diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index cd10985..a21d2a3 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -306,10 +306,10 @@ type App struct { // 全站主题("dark" | "pink"),从 DB 读 theme string // 显式指定的 spider91 上传目标 drive ID。 - // 空字符串表示本地保存不上传,不再自动挑选 pikpak/p115/onedrive drive。 + // 空字符串表示本地保存不上传,不再自动挑选 pikpak/p115/p123/onedrive drive。 spider91UploadDriveID string - // spider91Migrator 周期把 spider91 视频上传到目标 drive(PikPak、115 或 OneDrive)。 + // spider91Migrator 周期把 spider91 视频上传到目标 drive(PikPak、115、123 或 OneDrive)。 spider91Migrator *spider91migrate.Migrator // nightlyRunner 是凌晨流水线调度器:每天 cron_hour 串行跑扫盘 → 91 爬虫 → 迁移。 @@ -400,7 +400,7 @@ func (a *App) loadTheme(ctx context.Context) { } // Spider91UploadDriveID 返回当前配置的 spider91 上传目标 drive ID。 -// 空字符串表示本地保存不上传;只有管理员显式选择 pikpak/p115/onedrive drive 时才迁移上传。 +// 空字符串表示本地保存不上传;只有管理员显式选择 pikpak/p115/p123/onedrive drive 时才迁移上传。 func (a *App) Spider91UploadDriveID() string { a.mu.Lock() explicit := a.spider91UploadDriveID @@ -417,7 +417,7 @@ func (a *App) Spider91UploadDriveID() string { // SetSpider91UploadDriveID 设置 spider91 上传目标 drive ID 并持久化。 // 接受空字符串(本地保存不上传)。 -// 设置一个不存在或 kind 不是 pikpak / p115 / onedrive 的 drive 会返回错误。 +// 设置一个不存在或 kind 不是 pikpak / p115 / p123 / onedrive 的 drive 会返回错误。 func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) error { driveID = strings.TrimSpace(driveID) if driveID != "" { @@ -426,7 +426,7 @@ func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) erro return fmt.Errorf("drive %q not found", driveID) } if !isSpider91UploadKind(d.Kind()) { - return fmt.Errorf("drive %q kind=%s, only pikpak, p115 or onedrive can be spider91 upload target", driveID, d.Kind()) + return fmt.Errorf("drive %q kind=%s, only pikpak, p115, p123 or onedrive can be spider91 upload target", driveID, d.Kind()) } } a.mu.Lock() @@ -459,7 +459,7 @@ func formatOptionalRFC3339(t time.Time) string { // isSpider91UploadKind 是 spider91 迁移目标盘的 allowlist。 // 与 spider91migrate.adaptUploadTarget 的支持范围保持一致。 func isSpider91UploadKind(kind string) bool { - return kind == "pikpak" || kind == "p115" || kind == "onedrive" + return kind == "pikpak" || kind == "p115" || kind == "p123" || kind == "onedrive" } // loadSpider91UploadDriveID 从 DB 读上传目标 drive ID 设置;不存在时使用空串。 diff --git a/backend/cmd/server/main_spider91_test.go b/backend/cmd/server/main_spider91_test.go index a81bd3d..8aa2bd4 100644 --- a/backend/cmd/server/main_spider91_test.go +++ b/backend/cmd/server/main_spider91_test.go @@ -38,6 +38,7 @@ func TestSpider91IntCredFallbacks(t *testing.T) { func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) { reg := proxy.NewRegistry() reg.Set("p115-one", &spider91UploadTargetFakeDrive{id: "p115-one", kind: "p115"}) + reg.Set("p123-one", &spider91UploadTargetFakeDrive{id: "p123-one", kind: "p123"}) reg.Set("onedrive-one", &spider91UploadTargetFakeDrive{id: "onedrive-one", kind: "onedrive"}) app := &App{registry: reg} @@ -50,6 +51,11 @@ func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) { t.Fatalf("explicit upload target = %q, want p115-one", got) } + app.spider91UploadDriveID = "p123-one" + if got := app.Spider91UploadDriveID(); got != "p123-one" { + t.Fatalf("explicit p123 upload target = %q, want p123-one", got) + } + app.spider91UploadDriveID = "onedrive-one" if got := app.Spider91UploadDriveID(); got != "onedrive-one" { t.Fatalf("explicit onedrive upload target = %q, want onedrive-one", got) diff --git a/backend/internal/api/admin.go b/backend/internal/api/admin.go index 62e7393..453ef65 100644 --- a/backend/internal/api/admin.go +++ b/backend/internal/api/admin.go @@ -59,7 +59,7 @@ type AdminServer struct { // Theme 读写("dark" | "pink") GetTheme func() string SetTheme func(theme string) error - // Spider91 → 115/PikPak 上传目标 drive ID 读写 + // Spider91 → 115/123/PikPak/OneDrive 上传目标 drive ID 读写 GetSpider91UploadDriveID func() string SetSpider91UploadDriveID func(driveID string) error // OnRunNightlyJob 触发一次完整的凌晨流水线(Phase1 扫盘 + Phase2 91 爬虫 + diff --git a/backend/internal/drives/p123/driver.go b/backend/internal/drives/p123/driver.go index 73192b1..683b629 100644 --- a/backend/internal/drives/p123/driver.go +++ b/backend/internal/drives/p123/driver.go @@ -2,7 +2,9 @@ package p123 import ( "context" + "crypto/md5" "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -13,6 +15,7 @@ import ( "math/rand" "net/http" "net/url" + "os" "path" "strconv" "strings" @@ -38,9 +41,16 @@ const ( endpointFileList = "/file/list/new" endpointDownloadInfo = "/file/download_info" endpointMkdir = "/file/upload_request" + endpointRename = "/file/rename" + endpointUpload = "/file/upload_request" + endpointS3Auth = "/file/s3_upload_object/auth" + endpointS3Parts = "/file/s3_repare_upload_parts_batch" + endpointUploadDone = "/file/upload_complete/v2" listInterval = 700 * time.Millisecond listCooldown = 10 * time.Minute + + uploadChunkSize = int64(16 * 1024 * 1024) ) type Driver struct { @@ -237,8 +247,302 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi return d.resolveDownloadURL(ctx, downloadURL) } -func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) { - return "", drives.ErrNotSupported +// Upload 实现 drives.Drive 接口;只返回 fileID。 +// 完整上传元数据见 UploadAndReportHash。 +func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) { + res, err := d.UploadAndReportHash(ctx, parentID, name, r, size) + if err != nil { + return "", err + } + return res.FileID, nil +} + +// UploadResult 是 UploadAndReportHash 的返回值。 +// +// FileID 是 123 云盘分配的新文件 ID;Hash 是本次上传的 MD5 HEX(小写), +// 与 123 云盘列表返回的 Etag 一致;Size 是实际上传字节数。 +type UploadResult struct { + FileID string + Hash string + Size int64 +} + +// UploadAndReportHash 把 r 上传到 parentID 目录下的指定文件名,返回新文件元数据。 +// +// 123 云盘 Web 上传协议需要先计算文件 MD5 作为 etag 申请 upload_request。 +// 命中 Reuse 时服务端已经秒传;否则用返回的 S3 预签名 URL 分片 PUT,最后 +// 调 upload_complete/v2 完成。 +func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) { + if r == nil { + return UploadResult{}, errors.New("123pan upload: nil reader") + } + if size < 0 { + return UploadResult{}, fmt.Errorf("123pan upload: invalid size %d", size) + } + name = strings.TrimSpace(name) + if name == "" { + return UploadResult{}, errors.New("123pan upload: empty file name") + } + parentID = strings.TrimSpace(parentID) + if parentID == "" || parentID == "/" { + parentID = d.rootID + } + + tmp, md5Hex, actualSize, err := bufferAndHashMD5(r, size) + if err != nil { + return UploadResult{}, err + } + defer func() { + _ = tmp.Close() + _ = os.Remove(tmp.Name()) + }() + + body := map[string]any{ + "driveId": 0, + "duplicate": 2, + "etag": md5Hex, + "fileName": name, + "parentFileId": parentID, + "size": actualSize, + "type": 0, + } + var resp uploadResp + if _, err := d.request(ctx, endpointUpload, http.MethodPost, func(req *resty.Request) { + req.SetBody(body) + }, &resp); err != nil { + return UploadResult{}, fmt.Errorf("123pan upload: request session: %w", err) + } + + result := UploadResult{ + FileID: strconv.FormatInt(resp.Data.FileID, 10), + Hash: md5Hex, + Size: actualSize, + } + if resp.Data.FileID == 0 { + result.FileID = "" + } + + if resp.Data.Reuse || strings.TrimSpace(resp.Data.Key) == "" { + if result.FileID == "" { + fileID, err := d.findUploadedFileID(ctx, parentID, name, md5Hex) + if err != nil { + return UploadResult{}, err + } + result.FileID = fileID + } + d.cacheUploadedFile(result.FileID, parentID, name, md5Hex, actualSize) + return result, nil + } + + if err := d.uploadToPresignedURLs(ctx, &resp, tmp, actualSize); err != nil { + return UploadResult{}, err + } + if err := d.completeUpload(ctx, &resp, actualSize); err != nil { + return UploadResult{}, err + } + if result.FileID == "" { + fileID, err := d.findUploadedFileID(ctx, parentID, name, md5Hex) + if err != nil { + return UploadResult{}, err + } + result.FileID = fileID + } + d.cacheUploadedFile(result.FileID, parentID, name, md5Hex, actualSize) + return result, nil +} + +func (d *Driver) uploadToPresignedURLs(ctx context.Context, up *uploadResp, tmp *os.File, size int64) error { + if strings.TrimSpace(up.Data.Bucket) == "" || strings.TrimSpace(up.Data.Key) == "" || strings.TrimSpace(up.Data.UploadID) == "" { + return errors.New("123pan upload: incomplete upload session") + } + chunkCount := int64(1) + if size > uploadChunkSize { + chunkCount = (size + uploadChunkSize - 1) / uploadChunkSize + } + batchSize := int64(1) + endpoint := endpointS3Auth + if chunkCount > 1 { + batchSize = 10 + endpoint = endpointS3Parts + } + for start := int64(1); start <= chunkCount; start += batchSize { + end := minInt64(start+batchSize, chunkCount+1) + urls, err := d.getUploadURLs(ctx, endpoint, up, start, end) + if err != nil { + return err + } + for part := start; part < end; part++ { + offset := (part - 1) * uploadChunkSize + partSize := minInt64(uploadChunkSize, size-offset) + uploadURL := strings.TrimSpace(urls.Data.PreSignedURLs[strconv.FormatInt(part, 10)]) + if uploadURL == "" { + return fmt.Errorf("123pan upload: empty presigned url for part %d", part) + } + if err := d.putUploadPart(ctx, uploadURL, tmp, offset, partSize); err != nil { + if !isForbiddenUploadPart(err) { + return err + } + refreshed, refreshErr := d.getUploadURLs(ctx, endpoint, up, part, part+1) + if refreshErr != nil { + return refreshErr + } + uploadURL = strings.TrimSpace(refreshed.Data.PreSignedURLs[strconv.FormatInt(part, 10)]) + if uploadURL == "" { + return fmt.Errorf("123pan upload: empty refreshed presigned url for part %d", part) + } + if retryErr := d.putUploadPart(ctx, uploadURL, tmp, offset, partSize); retryErr != nil { + return retryErr + } + } + } + } + return nil +} + +func (d *Driver) getUploadURLs(ctx context.Context, endpoint string, up *uploadResp, start, end int64) (*s3PreSignedURLsResp, error) { + body := map[string]any{ + "StorageNode": up.Data.StorageNode, + "bucket": up.Data.Bucket, + "key": up.Data.Key, + "partNumberEnd": end, + "partNumberStart": start, + "uploadId": up.Data.UploadID, + } + var resp s3PreSignedURLsResp + if _, err := d.request(ctx, endpoint, http.MethodPost, func(req *resty.Request) { + req.SetBody(body) + }, &resp); err != nil { + return nil, fmt.Errorf("123pan upload: presigned urls: %w", err) + } + return &resp, nil +} + +type forbiddenUploadPartError struct { + status int +} + +func (e *forbiddenUploadPartError) Error() string { + return fmt.Sprintf("123pan upload: presigned put status=%d", e.status) +} + +func isForbiddenUploadPart(err error) bool { + var forbidden *forbiddenUploadPartError + return errors.As(err, &forbidden) +} + +func (d *Driver) putUploadPart(ctx context.Context, uploadURL string, tmp *os.File, offset, size int64) error { + reader := io.NewSectionReader(tmp, offset, size) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, reader) + if err != nil { + return err + } + req.ContentLength = size + req.Header.Set("User-Agent", d.userAgent) + res, err := d.httpClient.Do(req) + if err != nil { + return fmt.Errorf("123pan upload: presigned put: %w", err) + } + defer res.Body.Close() + if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusCreated || res.StatusCode == http.StatusNoContent { + return nil + } + body, _ := io.ReadAll(io.LimitReader(res.Body, 4096)) + if isP123RateLimitHTTPResponse(res.StatusCode, res.Header.Get("Retry-After"), string(body)) { + return p123RateLimitErrorFromHTTP("upload part", res.StatusCode, res.Header.Get("Retry-After"), string(body)) + } + if res.StatusCode == http.StatusForbidden { + return &forbiddenUploadPartError{status: res.StatusCode} + } + return fmt.Errorf("123pan upload: presigned put status=%d body=%s", res.StatusCode, strings.TrimSpace(string(body))) +} + +func (d *Driver) completeUpload(ctx context.Context, up *uploadResp, size int64) error { + if up.Data.FileID == 0 { + return errors.New("123pan upload: empty file id") + } + body := map[string]any{ + "StorageNode": up.Data.StorageNode, + "bucket": up.Data.Bucket, + "fileId": up.Data.FileID, + "fileSize": size, + "isMultipart": size > uploadChunkSize, + "key": up.Data.Key, + "uploadId": up.Data.UploadID, + } + if _, err := d.request(ctx, endpointUploadDone, http.MethodPost, func(req *resty.Request) { + req.SetBody(body) + }, nil); err != nil { + return fmt.Errorf("123pan upload: complete: %w", err) + } + return nil +} + +func (d *Driver) findUploadedFileID(ctx context.Context, parentID, name, md5Hex string) (string, error) { + entries, err := d.List(ctx, parentID) + if err != nil { + return "", fmt.Errorf("123pan upload verify: %w", err) + } + var hashHit string + for _, e := range entries { + if e.IsDir { + continue + } + if !strings.EqualFold(e.Hash, md5Hex) { + continue + } + if e.Name == name { + return e.ID, nil + } + if hashHit == "" { + hashHit = e.ID + } + } + if hashHit != "" { + return hashHit, nil + } + for _, e := range entries { + if !e.IsDir && e.Name == name { + return e.ID, nil + } + } + return "", fmt.Errorf("123pan upload: uploaded file %q not found in parent %q", name, parentID) +} + +func (d *Driver) cacheUploadedFile(fileID, parentID, name, md5Hex string, size int64) { + id, err := strconv.ParseInt(strings.TrimSpace(fileID), 10, 64) + if err != nil || id == 0 { + return + } + d.cacheFile(panFile{ + FileName: name, + Size: size, + FileID: id, + Type: 0, + Etag: md5Hex, + }, parentID) +} + +// Rename 调用 123 云盘 Web API 把指定 fileID 重命名为 newName。 +func (d *Driver) Rename(ctx context.Context, fileID, newName string) error { + fileID = strings.TrimSpace(fileID) + if fileID == "" { + return errors.New("123pan rename: empty file id") + } + newName = strings.TrimSpace(newName) + if newName == "" { + return errors.New("123pan rename: empty new name") + } + if _, err := d.request(ctx, endpointRename, http.MethodPost, func(req *resty.Request) { + req.SetBody(map[string]any{ + "driveId": 0, + "fileId": fileID, + "fileName": newName, + }) + }, nil); err != nil { + return fmt.Errorf("123pan rename: %w", err) + } + d.renameCachedFile(fileID, newName) + return nil } func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) { @@ -629,6 +933,15 @@ func (d *Driver) cacheFile(f panFile, parentID string) { d.fileMu.Unlock() } +func (d *Driver) renameCachedFile(fileID, newName string) { + d.fileMu.Lock() + defer d.fileMu.Unlock() + if c, ok := d.files[fileID]; ok { + c.file.FileName = newName + d.files[fileID] = c + } +} + func (d *Driver) cachedFile(fileID string) (panFile, string, bool) { d.fileMu.RLock() defer d.fileMu.RUnlock() @@ -738,6 +1051,33 @@ func splitPath(p string) []string { return strings.Split(p, "/") } +func bufferAndHashMD5(r io.Reader, declaredSize int64) (*os.File, string, int64, error) { + tmp, err := os.CreateTemp("", "p123-upload-*.bin") + if err != nil { + return nil, "", 0, fmt.Errorf("123pan upload: create tmp: %w", err) + } + h := md5.New() + written, err := io.Copy(io.MultiWriter(tmp, h), r) + if err != nil { + _ = tmp.Close() + _ = os.Remove(tmp.Name()) + return nil, "", 0, fmt.Errorf("123pan upload: buffer body: %w", err) + } + if declaredSize >= 0 && written != declaredSize { + _ = tmp.Close() + _ = os.Remove(tmp.Name()) + return nil, "", 0, fmt.Errorf("123pan upload: size mismatch: declared %d, copied %d", declaredSize, written) + } + return tmp, strings.ToLower(hex.EncodeToString(h.Sum(nil))), written, nil +} + +func minInt64(a, b int64) int64 { + if a < b { + return a + } + return b +} + func fileToEntry(f panFile, parentID string) drives.Entry { return drives.Entry{ ID: strconv.FormatInt(f.FileID, 10), diff --git a/backend/internal/drives/p123/driver_test.go b/backend/internal/drives/p123/driver_test.go index b5ea8ef..83dfe9b 100644 --- a/backend/internal/drives/p123/driver_test.go +++ b/backend/internal/drives/p123/driver_test.go @@ -1,10 +1,14 @@ package p123 import ( + "bytes" "context" + "crypto/md5" "encoding/base64" "encoding/json" "errors" + "fmt" + "io" "net/http" "net/http/httptest" "strings" @@ -254,3 +258,230 @@ func TestResolveDownloadURL429ReturnsRateLimitError(t *testing.T) { t.Fatalf("RetryAfter = %s, want 3s", rateLimit.RetryAfter) } } + +func TestUploadAndReportHashUsesPresignedPUTAndComplete(t *testing.T) { + ctx := context.Background() + body := []byte("video bytes for 123 upload") + wantMD5 := fmt.Sprintf("%x", md5.Sum(body)) + + var putBody []byte + upload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut { + t.Fatalf("upload method = %s, want PUT", r.Method) + } + if r.ContentLength != int64(len(body)) { + t.Fatalf("ContentLength = %d, want %d", r.ContentLength, len(body)) + } + got, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read upload body: %v", err) + } + putBody = got + w.WriteHeader(http.StatusOK) + })) + defer upload.Close() + + var uploadRequest map[string]any + var uploadURLRequest map[string]any + var completeRequest map[string]any + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/file/upload_request": + if err := json.NewDecoder(r.Body).Decode(&uploadRequest); err != nil { + t.Fatalf("decode upload_request: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "code": 0, + "data": map[string]any{ + "FileId": 9001, + "Bucket": "bucket-1", + "Key": "key-1", + "StorageNode": "node-1", + "UploadId": "upload-1", + }, + }) + case "/file/s3_upload_object/auth": + if err := json.NewDecoder(r.Body).Decode(&uploadURLRequest); err != nil { + t.Fatalf("decode s3 auth: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "code": 0, + "data": map[string]any{ + "presignedUrls": map[string]string{ + "1": upload.URL + "/part-1", + }, + }, + }) + case "/file/upload_complete/v2": + if err := json.NewDecoder(r.Body).Decode(&completeRequest); err != nil { + t.Fatalf("decode complete: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{}}) + default: + http.NotFound(w, r) + } + })) + defer api.Close() + + d := New(Config{ + ID: "123-main", + AccessToken: "token-1", + MainAPIBaseURL: api.URL, + }) + res, err := d.UploadAndReportHash(ctx, "parent-1", "video.mp4", bytes.NewReader(body), int64(len(body))) + if err != nil { + t.Fatalf("UploadAndReportHash() error = %v", err) + } + if res.FileID != "9001" { + t.Fatalf("FileID = %q, want 9001", res.FileID) + } + if res.Hash != wantMD5 { + t.Fatalf("Hash = %q, want %q", res.Hash, wantMD5) + } + if res.Size != int64(len(body)) { + t.Fatalf("Size = %d, want %d", res.Size, len(body)) + } + if !bytes.Equal(putBody, body) { + t.Fatalf("PUT body = %q, want %q", putBody, body) + } + if uploadRequest["etag"] != wantMD5 { + t.Fatalf("upload etag = %#v, want %q", uploadRequest["etag"], wantMD5) + } + if uploadRequest["fileName"] != "video.mp4" || uploadRequest["parentFileId"] != "parent-1" { + t.Fatalf("upload request = %#v, want fileName and parentFileId", uploadRequest) + } + if uploadURLRequest["partNumberStart"].(float64) != 1 || uploadURLRequest["partNumberEnd"].(float64) != 2 { + t.Fatalf("s3 auth request = %#v, want part range 1..2", uploadURLRequest) + } + if completeRequest["fileId"].(float64) != 9001 || completeRequest["fileSize"].(float64) != float64(len(body)) { + t.Fatalf("complete request = %#v, want file id and size", completeRequest) + } + if completeRequest["isMultipart"].(bool) { + t.Fatalf("complete isMultipart = true, want false") + } +} + +func TestUploadAndReportHashReuseSkipsPUTAndComplete(t *testing.T) { + body := []byte("reused body") + var presignedCalled bool + var completeCalled bool + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/file/upload_request": + _ = json.NewEncoder(w).Encode(map[string]any{ + "code": 0, + "data": map[string]any{ + "FileId": 7001, + "Reuse": true, + }, + }) + case "/file/s3_upload_object/auth", "/file/s3_repare_upload_parts_batch": + presignedCalled = true + _ = json.NewEncoder(w).Encode(map[string]any{"code": 0}) + case "/file/upload_complete/v2": + completeCalled = true + _ = json.NewEncoder(w).Encode(map[string]any{"code": 0}) + default: + http.NotFound(w, r) + } + })) + defer api.Close() + + d := New(Config{ + ID: "123-main", + AccessToken: "token-1", + MainAPIBaseURL: api.URL, + }) + res, err := d.UploadAndReportHash(context.Background(), "parent-1", "reused.mp4", bytes.NewReader(body), int64(len(body))) + if err != nil { + t.Fatalf("UploadAndReportHash() error = %v", err) + } + if res.FileID != "7001" { + t.Fatalf("FileID = %q, want 7001", res.FileID) + } + if presignedCalled { + t.Fatal("reuse upload should not request presigned URLs") + } + if completeCalled { + t.Fatal("reuse upload should not call upload_complete") + } +} + +func TestUploadPresignedPUT429ReturnsRateLimitError(t *testing.T) { + upload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "4") + http.Error(w, "too many requests", http.StatusTooManyRequests) + })) + defer upload.Close() + + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/file/upload_request": + _ = json.NewEncoder(w).Encode(map[string]any{ + "code": 0, + "data": map[string]any{ + "FileId": 9001, + "Bucket": "bucket-1", + "Key": "key-1", + "StorageNode": "node-1", + "UploadId": "upload-1", + }, + }) + case "/file/s3_upload_object/auth": + _ = json.NewEncoder(w).Encode(map[string]any{ + "code": 0, + "data": map[string]any{ + "presignedUrls": map[string]string{"1": upload.URL}, + }, + }) + default: + http.NotFound(w, r) + } + })) + defer api.Close() + + d := New(Config{ + ID: "123-main", + AccessToken: "token-1", + MainAPIBaseURL: api.URL, + }) + _, err := d.UploadAndReportHash(context.Background(), "parent-1", "limited.mp4", strings.NewReader("limited"), int64(len("limited"))) + var rateLimit *drives.RateLimitError + if !errors.As(err, &rateLimit) { + t.Fatalf("error = %T %[1]v, want RateLimitError", err) + } + if rateLimit.RetryAfter != 4*time.Second { + t.Fatalf("RetryAfter = %s, want 4s", rateLimit.RetryAfter) + } +} + +func TestRenameSendsExpectedBody(t *testing.T) { + var renameRequest map[string]any + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path != "/file/rename" { + http.NotFound(w, r) + return + } + if err := json.NewDecoder(r.Body).Decode(&renameRequest); err != nil { + t.Fatalf("decode rename: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{}}) + })) + defer api.Close() + + d := New(Config{ + ID: "123-main", + AccessToken: "token-1", + MainAPIBaseURL: api.URL, + }) + if err := d.Rename(context.Background(), "9001", "new name.mp4"); err != nil { + t.Fatalf("Rename() error = %v", err) + } + if renameRequest["driveId"].(float64) != 0 || renameRequest["fileId"] != "9001" || renameRequest["fileName"] != "new name.mp4" { + t.Fatalf("rename request = %#v, want driveId/fileId/fileName", renameRequest) + } +} diff --git a/backend/internal/drives/p123/types.go b/backend/internal/drives/p123/types.go index fd5c2e7..b378da6 100644 --- a/backend/internal/drives/p123/types.go +++ b/backend/internal/drives/p123/types.go @@ -129,6 +129,27 @@ type mkdirResp struct { } `json:"data"` } +type uploadResp struct { + Data struct { + AccessKeyID string `json:"AccessKeyId"` + Bucket string `json:"Bucket"` + Key string `json:"Key"` + SecretAccessKey string `json:"SecretAccessKey"` + SessionToken string `json:"SessionToken"` + FileID int64 `json:"FileId"` + Reuse bool `json:"Reuse"` + EndPoint string `json:"EndPoint"` + StorageNode string `json:"StorageNode"` + UploadID string `json:"UploadId"` + } `json:"data"` +} + +type s3PreSignedURLsResp struct { + Data struct { + PreSignedURLs map[string]string `json:"presignedUrls"` + } `json:"data"` +} + type flexibleTime struct { t time.Time } diff --git a/backend/internal/spider91migrate/migrator.go b/backend/internal/spider91migrate/migrator.go index d29ab7c..e79f0a5 100644 --- a/backend/internal/spider91migrate/migrator.go +++ b/backend/internal/spider91migrate/migrator.go @@ -1,5 +1,5 @@ // Package spider91migrate 周期性把 spider91 drive 下载到本地的视频 -// 上传到一个指定的目标 drive 目录(PikPak、115 或 OneDrive),上传成功后: +// 上传到一个指定的目标 drive 目录(PikPak、115、123 或 OneDrive),上传成功后: // // - 改写 catalog 行:drive_id / file_id / content_hash 改成目标盘的; // 视频自身的 id 不变(仍是 spider91--),video_tags、 @@ -31,17 +31,19 @@ import ( "github.com/video-site/backend/internal/drives" "github.com/video-site/backend/internal/drives/onedrive" "github.com/video-site/backend/internal/drives/p115" + "github.com/video-site/backend/internal/drives/p123" "github.com/video-site/backend/internal/drives/pikpak" "github.com/video-site/backend/internal/drives/spider91" "github.com/video-site/backend/internal/mediaasset" ) // uploadTarget 是 migrator 调用目标 drive 的最小接口。任何一种"接收 spider91 上传"的 -// 网盘都要实现它;当前 PikPak 和 115 各自通过适配器满足。 +// 网盘都要实现它;当前 PikPak、115、123 和 OneDrive 各自通过适配器满足。 // // 这一层抽象把"迁移调用方"和"具体盘的 SDK 协议"解耦: // - PikPak 走 GCID + OSS PutObject(pikpak.UploadResult) // - 115 走 SHA1 + 秒传 / OSS / 分片(p115.UploadResult) +// - 123 走 MD5 + 秒传 / S3 预签名分片(p123.UploadResult) // - OneDrive 走 SHA1 + 小文件 PUT / 大文件 upload session // // 各家返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。 @@ -57,7 +59,7 @@ type uploadTarget interface { // UploadResult 是 uploadTarget.UploadAndReportHash 的归一返回。 // // FileID 目标盘上的新文件 ID; -// Hash GCID(PikPak)或 SHA1 HEX(115 / OneDrive),写入 catalog.content_hash 用于跨盘去重; +// Hash GCID(PikPak)、MD5 HEX(123)或 SHA1 HEX(115 / OneDrive),写入 catalog.content_hash 用于跨盘去重; // Size 实际上传字节数。 type UploadResult struct { FileID string @@ -67,7 +69,7 @@ type UploadResult struct { const spider91UploadDirName = "91 Spider" -// pikpakAdapter / p115Adapter / onedriveAdapter 把具体 driver 包装成 uploadTarget。 +// pikpakAdapter / p115Adapter / p123Adapter / onedriveAdapter 把具体 driver 包装成 uploadTarget。 // // 之所以不让 driver 直接实现 uploadTarget: // @@ -116,6 +118,27 @@ func (a *p115Adapter) Rename(ctx context.Context, fileID, newName string) error return a.d.Rename(ctx, fileID, newName) } +type p123Adapter struct { + d *p123.Driver +} + +func (a *p123Adapter) ID() string { return a.d.ID() } +func (a *p123Adapter) Kind() string { return a.d.Kind() } +func (a *p123Adapter) RootID() string { return a.d.RootID() } +func (a *p123Adapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) { + return a.d.EnsureDir(ctx, pathFromRoot) +} +func (a *p123Adapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) { + res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size) + if err != nil { + return UploadResult{}, err + } + return UploadResult{FileID: res.FileID, Hash: res.Hash, Size: res.Size}, nil +} +func (a *p123Adapter) Rename(ctx context.Context, fileID, newName string) error { + return a.d.Rename(ctx, fileID, newName) +} + type onedriveAdapter struct { d *onedrive.Driver } @@ -145,6 +168,8 @@ func adaptUploadTarget(d drives.Drive) (uploadTarget, error) { return &pikpakAdapter{d: v}, nil case *p115.Driver: return &p115Adapter{d: v}, nil + case *p123.Driver: + return &p123Adapter{d: v}, nil case *onedrive.Driver: return &onedriveAdapter{d: v}, nil case uploadTarget: @@ -760,7 +785,7 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driv return deleted, nil } -// backfillFileNames 扫描目标 drive(PikPak、115 或 OneDrive)下所有 spider91-* 起始 ID 的视频, +// backfillFileNames 扫描目标 drive(PikPak、115、123 或 OneDrive)下所有 spider91-* 起始 ID 的视频, // 对文件名不是 desiredPikPakName(...) 期望格式的,调 target.Rename 修正, // 并把 catalog.file_name 同步到新名字。 // diff --git a/backend/internal/spider91migrate/migrator_test.go b/backend/internal/spider91migrate/migrator_test.go index 634c308..5b96700 100644 --- a/backend/internal/spider91migrate/migrator_test.go +++ b/backend/internal/spider91migrate/migrator_test.go @@ -14,6 +14,7 @@ import ( "github.com/video-site/backend/internal/catalog" "github.com/video-site/backend/internal/drives" + "github.com/video-site/backend/internal/drives/p123" "github.com/video-site/backend/internal/drives/pikpak" "github.com/video-site/backend/internal/drives/spider91" ) @@ -134,6 +135,19 @@ func (d *fakeP115) Kind() string { return "p115" } var _ drives.Drive = (*fakeP115)(nil) var _ uploadTarget = (*fakeP115)(nil) +type fakeP123 struct { + *fakePikPak +} + +func newFakeP123(id, rootID string) *fakeP123 { + return &fakeP123{fakePikPak: newFakePikPak(id, rootID)} +} + +func (d *fakeP123) Kind() string { return "p123" } + +var _ drives.Drive = (*fakeP123)(nil) +var _ uploadTarget = (*fakeP123)(nil) + type fakeOneDrive struct { *fakePikPak } @@ -946,6 +960,66 @@ func TestRunOnceMigratesToP115Target(t *testing.T) { } } +func TestRunOnceMigratesToP123Target(t *testing.T) { + cat := setupCatalog(t) + src, _ := setupSpider91(t) + target := newFakeP123("p123-target", "p123-root-id") + reg := newFakeRegistry() + reg.Add(src) + reg.Add(target) + + now := time.Now() + id := writeSpider91Video(t, cat, src, "vk-123-001", ".mp4", []byte("video bytes 123"), now) + + m := New(Config{ + Catalog: cat, + Registry: reg, + GetTargetDriveID: func() string { return target.ID() }, + KeepLatestN: -1, + }) + m.runOnce(context.Background()) + + if target.uploadCalls != 1 { + t.Fatalf("p123 upload calls = %d, want 1", target.uploadCalls) + } + + got, err := cat.GetVideo(context.Background(), id) + if err != nil { + t.Fatalf("get video: %v", err) + } + if got.DriveID != target.ID() { + t.Fatalf("drive_id = %q, want %q", got.DriveID, target.ID()) + } + wantName := "Sample vk-123-001-001.mp4" + if _, ok := target.gotBodies[wantName]; !ok { + t.Fatalf("p123 did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies)) + } + if gotParent := target.gotParents[wantName]; gotParent != "p123-root-id/"+spider91UploadDirName { + t.Fatalf("p123 upload parent = %q, want root/91 Spider", gotParent) + } + if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName { + t.Fatalf("p123 ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName) + } + if got.FileID != "remote-"+wantName { + t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName) + } + if got.FileName != wantName { + t.Fatalf("file_name = %q, want %q", got.FileName, wantName) + } + if got.ContentHash == "" { + t.Fatal("content_hash should be set after p123 migration") + } + + videoPath, _ := src.VideoPath("vk-123-001.mp4") + if _, err := os.Stat(videoPath); !os.IsNotExist(err) { + t.Fatalf("local mp4 still exists after p123 migration or stat error: %v", err) + } + thumbPath, _ := src.ThumbPath("vk-123-001.jpg") + if _, err := os.Stat(thumbPath); !os.IsNotExist(err) { + t.Fatalf("local thumb still exists after p123 migration or stat error: %v", err) + } +} + func TestRunOnceMigratesToOneDriveTarget(t *testing.T) { cat := setupCatalog(t) src, _ := setupSpider91(t) @@ -1006,7 +1080,22 @@ func TestRunOnceMigratesToOneDriveTarget(t *testing.T) { } } -// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115 也不是 OneDrive 时, +func TestAdaptUploadTargetSupportsP123Driver(t *testing.T) { + d := p123.New(p123.Config{ + ID: "p123-target", + RootID: "root-123", + AccessToken: "token-1", + }) + target, err := adaptUploadTarget(d) + if err != nil { + t.Fatalf("adaptUploadTarget() error = %v", err) + } + if target.ID() != "p123-target" || target.Kind() != "p123" || target.RootID() != "root-123" { + t.Fatalf("target id/kind/root = %q/%q/%q, want p123-target/p123/root-123", target.ID(), target.Kind(), target.RootID()) + } +} + +// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115、123 也不是 OneDrive 时, // resolveTarget 拒绝并返回 error,让 runOnce 静默跳过(不会做破坏性变更)。 func TestResolveTargetRejectsUnsupportedKind(t *testing.T) { cat := setupCatalog(t) diff --git a/src/admin/DrivesPage.tsx b/src/admin/DrivesPage.tsx index 942dd4b..997d8f6 100644 --- a/src/admin/DrivesPage.tsx +++ b/src/admin/DrivesPage.tsx @@ -68,7 +68,7 @@ export function DrivesPage() { const formDirty = !sameForm(form, initialForm); const uploadTargets = useMemo( - () => list.filter((d) => d.kind === "pikpak" || d.kind === "p115" || d.kind === "onedrive"), + () => list.filter((d) => d.kind === "pikpak" || d.kind === "p115" || d.kind === "p123" || d.kind === "onedrive"), [list] ); diff --git a/src/admin/api.ts b/src/admin/api.ts index 5effaf1..adfc09a 100644 --- a/src/admin/api.ts +++ b/src/admin/api.ts @@ -388,9 +388,9 @@ export type Theme = "dark" | "pink"; export type Settings = { theme: Theme; /** - * spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115 或 onedrive drive)。 + * spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115、p123 或 onedrive drive)。 * - 空字符串:本地保存,不上传到云盘。 - * - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, onedrive}。 + * - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, p123, onedrive}。 */ spider91UploadDriveId: string; }; diff --git a/src/admin/drive/Spider91UploadTargetField.tsx b/src/admin/drive/Spider91UploadTargetField.tsx index 14a9f1d..6eba5c1 100644 --- a/src/admin/drive/Spider91UploadTargetField.tsx +++ b/src/admin/drive/Spider91UploadTargetField.tsx @@ -25,7 +25,7 @@ export function Spider91UploadTargetField({ ))}
- 选择本地保存时,爬取视频只保存在服务器本地;选择 115 网盘、PikPak 或 OneDrive 后,较早的视频会上传到该云盘根目录下的 91 Spider 文件夹。该设置全局生效。 + 选择本地保存时,爬取视频只保存在服务器本地;选择 115 网盘、123 云盘、PikPak 或 OneDrive 后,较早的视频会上传到该云盘根目录下的 91 Spider 文件夹。该设置全局生效。
); diff --git a/tests/adminDriveForm.test.ts b/tests/adminDriveForm.test.ts index 4a19be3..a010e6e 100644 --- a/tests/adminDriveForm.test.ts +++ b/tests/adminDriveForm.test.ts @@ -52,7 +52,7 @@ test("spider91 upload target uses explicit local-save option instead of auto tar assert.match(combinedSource, /本地保存,不上传/); assert.match( combinedSource, - /d\.kind === "pikpak" \|\| d\.kind === "p115" \|\| d\.kind === "onedrive"/ + /d\.kind === "pikpak" \|\| d\.kind === "p115" \|\| d\.kind === "p123" \|\| d\.kind === "onedrive"/ ); assert.doesNotMatch(combinedSource, /自动:唯一/); assert.doesNotMatch(combinedSource, /自动模式/);