feat: support spider91 upload to 123pan

This commit is contained in:
nianzhibai
2026-06-03 21:49:27 +08:00
parent 8f0d52aec4
commit df6f0ebbbf
12 changed files with 732 additions and 20 deletions
+6 -6
View File
@@ -306,10 +306,10 @@ type App struct {
// 全站主题("dark" | "pink"),从 DB 读
theme string
// 显式指定的 spider91 上传目标 drive ID。
// 空字符串表示本地保存不上传,不再自动挑选 pikpak/p115/onedrive drive。
// 空字符串表示本地保存不上传,不再自动挑选 pikpak/p115/p123/onedrive drive。
spider91UploadDriveID string
// spider91Migrator 周期把 spider91 视频上传到目标 drivePikPak、115 或 OneDrive)。
// spider91Migrator 周期把 spider91 视频上传到目标 drivePikPak、115、123 或 OneDrive)。
spider91Migrator *spider91migrate.Migrator
// nightlyRunner 是凌晨流水线调度器:每天 cron_hour 串行跑扫盘 → 91 爬虫 → 迁移。
@@ -400,7 +400,7 @@ func (a *App) loadTheme(ctx context.Context) {
}
// Spider91UploadDriveID 返回当前配置的 spider91 上传目标 drive ID。
// 空字符串表示本地保存不上传;只有管理员显式选择 pikpak/p115/onedrive drive 时才迁移上传。
// 空字符串表示本地保存不上传;只有管理员显式选择 pikpak/p115/p123/onedrive drive 时才迁移上传。
func (a *App) Spider91UploadDriveID() string {
a.mu.Lock()
explicit := a.spider91UploadDriveID
@@ -417,7 +417,7 @@ func (a *App) Spider91UploadDriveID() string {
// SetSpider91UploadDriveID 设置 spider91 上传目标 drive ID 并持久化。
// 接受空字符串(本地保存不上传)。
// 设置一个不存在或 kind 不是 pikpak / p115 / onedrive 的 drive 会返回错误。
// 设置一个不存在或 kind 不是 pikpak / p115 / p123 / onedrive 的 drive 会返回错误。
func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) error {
driveID = strings.TrimSpace(driveID)
if driveID != "" {
@@ -426,7 +426,7 @@ func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) erro
return fmt.Errorf("drive %q not found", driveID)
}
if !isSpider91UploadKind(d.Kind()) {
return fmt.Errorf("drive %q kind=%s, only pikpak, p115 or onedrive can be spider91 upload target", driveID, d.Kind())
return fmt.Errorf("drive %q kind=%s, only pikpak, p115, p123 or onedrive can be spider91 upload target", driveID, d.Kind())
}
}
a.mu.Lock()
@@ -459,7 +459,7 @@ func formatOptionalRFC3339(t time.Time) string {
// isSpider91UploadKind 是 spider91 迁移目标盘的 allowlist。
// 与 spider91migrate.adaptUploadTarget 的支持范围保持一致。
func isSpider91UploadKind(kind string) bool {
return kind == "pikpak" || kind == "p115" || kind == "onedrive"
return kind == "pikpak" || kind == "p115" || kind == "p123" || kind == "onedrive"
}
// loadSpider91UploadDriveID 从 DB 读上传目标 drive ID 设置;不存在时使用空串。
+6
View File
@@ -38,6 +38,7 @@ func TestSpider91IntCredFallbacks(t *testing.T) {
func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
reg := proxy.NewRegistry()
reg.Set("p115-one", &spider91UploadTargetFakeDrive{id: "p115-one", kind: "p115"})
reg.Set("p123-one", &spider91UploadTargetFakeDrive{id: "p123-one", kind: "p123"})
reg.Set("onedrive-one", &spider91UploadTargetFakeDrive{id: "onedrive-one", kind: "onedrive"})
app := &App{registry: reg}
@@ -50,6 +51,11 @@ func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
t.Fatalf("explicit upload target = %q, want p115-one", got)
}
app.spider91UploadDriveID = "p123-one"
if got := app.Spider91UploadDriveID(); got != "p123-one" {
t.Fatalf("explicit p123 upload target = %q, want p123-one", got)
}
app.spider91UploadDriveID = "onedrive-one"
if got := app.Spider91UploadDriveID(); got != "onedrive-one" {
t.Fatalf("explicit onedrive upload target = %q, want onedrive-one", got)
+1 -1
View File
@@ -59,7 +59,7 @@ type AdminServer struct {
// Theme 读写("dark" | "pink"
GetTheme func() string
SetTheme func(theme string) error
// Spider91 → 115/PikPak 上传目标 drive ID 读写
// Spider91 → 115/123/PikPak/OneDrive 上传目标 drive ID 读写
GetSpider91UploadDriveID func() string
SetSpider91UploadDriveID func(driveID string) error
// OnRunNightlyJob 触发一次完整的凌晨流水线(Phase1 扫盘 + Phase2 91 爬虫 +
+342 -2
View File
@@ -2,7 +2,9 @@ package p123
import (
"context"
"crypto/md5"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
@@ -13,6 +15,7 @@ import (
"math/rand"
"net/http"
"net/url"
"os"
"path"
"strconv"
"strings"
@@ -38,9 +41,16 @@ const (
endpointFileList = "/file/list/new"
endpointDownloadInfo = "/file/download_info"
endpointMkdir = "/file/upload_request"
endpointRename = "/file/rename"
endpointUpload = "/file/upload_request"
endpointS3Auth = "/file/s3_upload_object/auth"
endpointS3Parts = "/file/s3_repare_upload_parts_batch"
endpointUploadDone = "/file/upload_complete/v2"
listInterval = 700 * time.Millisecond
listCooldown = 10 * time.Minute
uploadChunkSize = int64(16 * 1024 * 1024)
)
type Driver struct {
@@ -237,8 +247,302 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
return d.resolveDownloadURL(ctx, downloadURL)
}
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
return "", drives.ErrNotSupported
// Upload 实现 drives.Drive 接口;只返回 fileID。
// 完整上传元数据见 UploadAndReportHash。
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
res, err := d.UploadAndReportHash(ctx, parentID, name, r, size)
if err != nil {
return "", err
}
return res.FileID, nil
}
// UploadResult 是 UploadAndReportHash 的返回值。
//
// FileID 是 123 云盘分配的新文件 IDHash 是本次上传的 MD5 HEX(小写),
// 与 123 云盘列表返回的 Etag 一致;Size 是实际上传字节数。
type UploadResult struct {
FileID string
Hash string
Size int64
}
// UploadAndReportHash 把 r 上传到 parentID 目录下的指定文件名,返回新文件元数据。
//
// 123 云盘 Web 上传协议需要先计算文件 MD5 作为 etag 申请 upload_request。
// 命中 Reuse 时服务端已经秒传;否则用返回的 S3 预签名 URL 分片 PUT,最后
// 调 upload_complete/v2 完成。
func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
if r == nil {
return UploadResult{}, errors.New("123pan upload: nil reader")
}
if size < 0 {
return UploadResult{}, fmt.Errorf("123pan upload: invalid size %d", size)
}
name = strings.TrimSpace(name)
if name == "" {
return UploadResult{}, errors.New("123pan upload: empty file name")
}
parentID = strings.TrimSpace(parentID)
if parentID == "" || parentID == "/" {
parentID = d.rootID
}
tmp, md5Hex, actualSize, err := bufferAndHashMD5(r, size)
if err != nil {
return UploadResult{}, err
}
defer func() {
_ = tmp.Close()
_ = os.Remove(tmp.Name())
}()
body := map[string]any{
"driveId": 0,
"duplicate": 2,
"etag": md5Hex,
"fileName": name,
"parentFileId": parentID,
"size": actualSize,
"type": 0,
}
var resp uploadResp
if _, err := d.request(ctx, endpointUpload, http.MethodPost, func(req *resty.Request) {
req.SetBody(body)
}, &resp); err != nil {
return UploadResult{}, fmt.Errorf("123pan upload: request session: %w", err)
}
result := UploadResult{
FileID: strconv.FormatInt(resp.Data.FileID, 10),
Hash: md5Hex,
Size: actualSize,
}
if resp.Data.FileID == 0 {
result.FileID = ""
}
if resp.Data.Reuse || strings.TrimSpace(resp.Data.Key) == "" {
if result.FileID == "" {
fileID, err := d.findUploadedFileID(ctx, parentID, name, md5Hex)
if err != nil {
return UploadResult{}, err
}
result.FileID = fileID
}
d.cacheUploadedFile(result.FileID, parentID, name, md5Hex, actualSize)
return result, nil
}
if err := d.uploadToPresignedURLs(ctx, &resp, tmp, actualSize); err != nil {
return UploadResult{}, err
}
if err := d.completeUpload(ctx, &resp, actualSize); err != nil {
return UploadResult{}, err
}
if result.FileID == "" {
fileID, err := d.findUploadedFileID(ctx, parentID, name, md5Hex)
if err != nil {
return UploadResult{}, err
}
result.FileID = fileID
}
d.cacheUploadedFile(result.FileID, parentID, name, md5Hex, actualSize)
return result, nil
}
func (d *Driver) uploadToPresignedURLs(ctx context.Context, up *uploadResp, tmp *os.File, size int64) error {
if strings.TrimSpace(up.Data.Bucket) == "" || strings.TrimSpace(up.Data.Key) == "" || strings.TrimSpace(up.Data.UploadID) == "" {
return errors.New("123pan upload: incomplete upload session")
}
chunkCount := int64(1)
if size > uploadChunkSize {
chunkCount = (size + uploadChunkSize - 1) / uploadChunkSize
}
batchSize := int64(1)
endpoint := endpointS3Auth
if chunkCount > 1 {
batchSize = 10
endpoint = endpointS3Parts
}
for start := int64(1); start <= chunkCount; start += batchSize {
end := minInt64(start+batchSize, chunkCount+1)
urls, err := d.getUploadURLs(ctx, endpoint, up, start, end)
if err != nil {
return err
}
for part := start; part < end; part++ {
offset := (part - 1) * uploadChunkSize
partSize := minInt64(uploadChunkSize, size-offset)
uploadURL := strings.TrimSpace(urls.Data.PreSignedURLs[strconv.FormatInt(part, 10)])
if uploadURL == "" {
return fmt.Errorf("123pan upload: empty presigned url for part %d", part)
}
if err := d.putUploadPart(ctx, uploadURL, tmp, offset, partSize); err != nil {
if !isForbiddenUploadPart(err) {
return err
}
refreshed, refreshErr := d.getUploadURLs(ctx, endpoint, up, part, part+1)
if refreshErr != nil {
return refreshErr
}
uploadURL = strings.TrimSpace(refreshed.Data.PreSignedURLs[strconv.FormatInt(part, 10)])
if uploadURL == "" {
return fmt.Errorf("123pan upload: empty refreshed presigned url for part %d", part)
}
if retryErr := d.putUploadPart(ctx, uploadURL, tmp, offset, partSize); retryErr != nil {
return retryErr
}
}
}
}
return nil
}
func (d *Driver) getUploadURLs(ctx context.Context, endpoint string, up *uploadResp, start, end int64) (*s3PreSignedURLsResp, error) {
body := map[string]any{
"StorageNode": up.Data.StorageNode,
"bucket": up.Data.Bucket,
"key": up.Data.Key,
"partNumberEnd": end,
"partNumberStart": start,
"uploadId": up.Data.UploadID,
}
var resp s3PreSignedURLsResp
if _, err := d.request(ctx, endpoint, http.MethodPost, func(req *resty.Request) {
req.SetBody(body)
}, &resp); err != nil {
return nil, fmt.Errorf("123pan upload: presigned urls: %w", err)
}
return &resp, nil
}
type forbiddenUploadPartError struct {
status int
}
func (e *forbiddenUploadPartError) Error() string {
return fmt.Sprintf("123pan upload: presigned put status=%d", e.status)
}
func isForbiddenUploadPart(err error) bool {
var forbidden *forbiddenUploadPartError
return errors.As(err, &forbidden)
}
func (d *Driver) putUploadPart(ctx context.Context, uploadURL string, tmp *os.File, offset, size int64) error {
reader := io.NewSectionReader(tmp, offset, size)
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, reader)
if err != nil {
return err
}
req.ContentLength = size
req.Header.Set("User-Agent", d.userAgent)
res, err := d.httpClient.Do(req)
if err != nil {
return fmt.Errorf("123pan upload: presigned put: %w", err)
}
defer res.Body.Close()
if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusCreated || res.StatusCode == http.StatusNoContent {
return nil
}
body, _ := io.ReadAll(io.LimitReader(res.Body, 4096))
if isP123RateLimitHTTPResponse(res.StatusCode, res.Header.Get("Retry-After"), string(body)) {
return p123RateLimitErrorFromHTTP("upload part", res.StatusCode, res.Header.Get("Retry-After"), string(body))
}
if res.StatusCode == http.StatusForbidden {
return &forbiddenUploadPartError{status: res.StatusCode}
}
return fmt.Errorf("123pan upload: presigned put status=%d body=%s", res.StatusCode, strings.TrimSpace(string(body)))
}
func (d *Driver) completeUpload(ctx context.Context, up *uploadResp, size int64) error {
if up.Data.FileID == 0 {
return errors.New("123pan upload: empty file id")
}
body := map[string]any{
"StorageNode": up.Data.StorageNode,
"bucket": up.Data.Bucket,
"fileId": up.Data.FileID,
"fileSize": size,
"isMultipart": size > uploadChunkSize,
"key": up.Data.Key,
"uploadId": up.Data.UploadID,
}
if _, err := d.request(ctx, endpointUploadDone, http.MethodPost, func(req *resty.Request) {
req.SetBody(body)
}, nil); err != nil {
return fmt.Errorf("123pan upload: complete: %w", err)
}
return nil
}
func (d *Driver) findUploadedFileID(ctx context.Context, parentID, name, md5Hex string) (string, error) {
entries, err := d.List(ctx, parentID)
if err != nil {
return "", fmt.Errorf("123pan upload verify: %w", err)
}
var hashHit string
for _, e := range entries {
if e.IsDir {
continue
}
if !strings.EqualFold(e.Hash, md5Hex) {
continue
}
if e.Name == name {
return e.ID, nil
}
if hashHit == "" {
hashHit = e.ID
}
}
if hashHit != "" {
return hashHit, nil
}
for _, e := range entries {
if !e.IsDir && e.Name == name {
return e.ID, nil
}
}
return "", fmt.Errorf("123pan upload: uploaded file %q not found in parent %q", name, parentID)
}
func (d *Driver) cacheUploadedFile(fileID, parentID, name, md5Hex string, size int64) {
id, err := strconv.ParseInt(strings.TrimSpace(fileID), 10, 64)
if err != nil || id == 0 {
return
}
d.cacheFile(panFile{
FileName: name,
Size: size,
FileID: id,
Type: 0,
Etag: md5Hex,
}, parentID)
}
// Rename 调用 123 云盘 Web API 把指定 fileID 重命名为 newName。
func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
fileID = strings.TrimSpace(fileID)
if fileID == "" {
return errors.New("123pan rename: empty file id")
}
newName = strings.TrimSpace(newName)
if newName == "" {
return errors.New("123pan rename: empty new name")
}
if _, err := d.request(ctx, endpointRename, http.MethodPost, func(req *resty.Request) {
req.SetBody(map[string]any{
"driveId": 0,
"fileId": fileID,
"fileName": newName,
})
}, nil); err != nil {
return fmt.Errorf("123pan rename: %w", err)
}
d.renameCachedFile(fileID, newName)
return nil
}
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
@@ -629,6 +933,15 @@ func (d *Driver) cacheFile(f panFile, parentID string) {
d.fileMu.Unlock()
}
func (d *Driver) renameCachedFile(fileID, newName string) {
d.fileMu.Lock()
defer d.fileMu.Unlock()
if c, ok := d.files[fileID]; ok {
c.file.FileName = newName
d.files[fileID] = c
}
}
func (d *Driver) cachedFile(fileID string) (panFile, string, bool) {
d.fileMu.RLock()
defer d.fileMu.RUnlock()
@@ -738,6 +1051,33 @@ func splitPath(p string) []string {
return strings.Split(p, "/")
}
func bufferAndHashMD5(r io.Reader, declaredSize int64) (*os.File, string, int64, error) {
tmp, err := os.CreateTemp("", "p123-upload-*.bin")
if err != nil {
return nil, "", 0, fmt.Errorf("123pan upload: create tmp: %w", err)
}
h := md5.New()
written, err := io.Copy(io.MultiWriter(tmp, h), r)
if err != nil {
_ = tmp.Close()
_ = os.Remove(tmp.Name())
return nil, "", 0, fmt.Errorf("123pan upload: buffer body: %w", err)
}
if declaredSize >= 0 && written != declaredSize {
_ = tmp.Close()
_ = os.Remove(tmp.Name())
return nil, "", 0, fmt.Errorf("123pan upload: size mismatch: declared %d, copied %d", declaredSize, written)
}
return tmp, strings.ToLower(hex.EncodeToString(h.Sum(nil))), written, nil
}
func minInt64(a, b int64) int64 {
if a < b {
return a
}
return b
}
func fileToEntry(f panFile, parentID string) drives.Entry {
return drives.Entry{
ID: strconv.FormatInt(f.FileID, 10),
+231
View File
@@ -1,10 +1,14 @@
package p123
import (
"bytes"
"context"
"crypto/md5"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
@@ -254,3 +258,230 @@ func TestResolveDownloadURL429ReturnsRateLimitError(t *testing.T) {
t.Fatalf("RetryAfter = %s, want 3s", rateLimit.RetryAfter)
}
}
func TestUploadAndReportHashUsesPresignedPUTAndComplete(t *testing.T) {
ctx := context.Background()
body := []byte("video bytes for 123 upload")
wantMD5 := fmt.Sprintf("%x", md5.Sum(body))
var putBody []byte
upload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPut {
t.Fatalf("upload method = %s, want PUT", r.Method)
}
if r.ContentLength != int64(len(body)) {
t.Fatalf("ContentLength = %d, want %d", r.ContentLength, len(body))
}
got, err := io.ReadAll(r.Body)
if err != nil {
t.Fatalf("read upload body: %v", err)
}
putBody = got
w.WriteHeader(http.StatusOK)
}))
defer upload.Close()
var uploadRequest map[string]any
var uploadURLRequest map[string]any
var completeRequest map[string]any
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/file/upload_request":
if err := json.NewDecoder(r.Body).Decode(&uploadRequest); err != nil {
t.Fatalf("decode upload_request: %v", err)
}
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 0,
"data": map[string]any{
"FileId": 9001,
"Bucket": "bucket-1",
"Key": "key-1",
"StorageNode": "node-1",
"UploadId": "upload-1",
},
})
case "/file/s3_upload_object/auth":
if err := json.NewDecoder(r.Body).Decode(&uploadURLRequest); err != nil {
t.Fatalf("decode s3 auth: %v", err)
}
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 0,
"data": map[string]any{
"presignedUrls": map[string]string{
"1": upload.URL + "/part-1",
},
},
})
case "/file/upload_complete/v2":
if err := json.NewDecoder(r.Body).Decode(&completeRequest); err != nil {
t.Fatalf("decode complete: %v", err)
}
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{}})
default:
http.NotFound(w, r)
}
}))
defer api.Close()
d := New(Config{
ID: "123-main",
AccessToken: "token-1",
MainAPIBaseURL: api.URL,
})
res, err := d.UploadAndReportHash(ctx, "parent-1", "video.mp4", bytes.NewReader(body), int64(len(body)))
if err != nil {
t.Fatalf("UploadAndReportHash() error = %v", err)
}
if res.FileID != "9001" {
t.Fatalf("FileID = %q, want 9001", res.FileID)
}
if res.Hash != wantMD5 {
t.Fatalf("Hash = %q, want %q", res.Hash, wantMD5)
}
if res.Size != int64(len(body)) {
t.Fatalf("Size = %d, want %d", res.Size, len(body))
}
if !bytes.Equal(putBody, body) {
t.Fatalf("PUT body = %q, want %q", putBody, body)
}
if uploadRequest["etag"] != wantMD5 {
t.Fatalf("upload etag = %#v, want %q", uploadRequest["etag"], wantMD5)
}
if uploadRequest["fileName"] != "video.mp4" || uploadRequest["parentFileId"] != "parent-1" {
t.Fatalf("upload request = %#v, want fileName and parentFileId", uploadRequest)
}
if uploadURLRequest["partNumberStart"].(float64) != 1 || uploadURLRequest["partNumberEnd"].(float64) != 2 {
t.Fatalf("s3 auth request = %#v, want part range 1..2", uploadURLRequest)
}
if completeRequest["fileId"].(float64) != 9001 || completeRequest["fileSize"].(float64) != float64(len(body)) {
t.Fatalf("complete request = %#v, want file id and size", completeRequest)
}
if completeRequest["isMultipart"].(bool) {
t.Fatalf("complete isMultipart = true, want false")
}
}
func TestUploadAndReportHashReuseSkipsPUTAndComplete(t *testing.T) {
body := []byte("reused body")
var presignedCalled bool
var completeCalled bool
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/file/upload_request":
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 0,
"data": map[string]any{
"FileId": 7001,
"Reuse": true,
},
})
case "/file/s3_upload_object/auth", "/file/s3_repare_upload_parts_batch":
presignedCalled = true
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0})
case "/file/upload_complete/v2":
completeCalled = true
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0})
default:
http.NotFound(w, r)
}
}))
defer api.Close()
d := New(Config{
ID: "123-main",
AccessToken: "token-1",
MainAPIBaseURL: api.URL,
})
res, err := d.UploadAndReportHash(context.Background(), "parent-1", "reused.mp4", bytes.NewReader(body), int64(len(body)))
if err != nil {
t.Fatalf("UploadAndReportHash() error = %v", err)
}
if res.FileID != "7001" {
t.Fatalf("FileID = %q, want 7001", res.FileID)
}
if presignedCalled {
t.Fatal("reuse upload should not request presigned URLs")
}
if completeCalled {
t.Fatal("reuse upload should not call upload_complete")
}
}
func TestUploadPresignedPUT429ReturnsRateLimitError(t *testing.T) {
upload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Retry-After", "4")
http.Error(w, "too many requests", http.StatusTooManyRequests)
}))
defer upload.Close()
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/file/upload_request":
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 0,
"data": map[string]any{
"FileId": 9001,
"Bucket": "bucket-1",
"Key": "key-1",
"StorageNode": "node-1",
"UploadId": "upload-1",
},
})
case "/file/s3_upload_object/auth":
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 0,
"data": map[string]any{
"presignedUrls": map[string]string{"1": upload.URL},
},
})
default:
http.NotFound(w, r)
}
}))
defer api.Close()
d := New(Config{
ID: "123-main",
AccessToken: "token-1",
MainAPIBaseURL: api.URL,
})
_, err := d.UploadAndReportHash(context.Background(), "parent-1", "limited.mp4", strings.NewReader("limited"), int64(len("limited")))
var rateLimit *drives.RateLimitError
if !errors.As(err, &rateLimit) {
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
}
if rateLimit.RetryAfter != 4*time.Second {
t.Fatalf("RetryAfter = %s, want 4s", rateLimit.RetryAfter)
}
}
func TestRenameSendsExpectedBody(t *testing.T) {
var renameRequest map[string]any
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.URL.Path != "/file/rename" {
http.NotFound(w, r)
return
}
if err := json.NewDecoder(r.Body).Decode(&renameRequest); err != nil {
t.Fatalf("decode rename: %v", err)
}
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{}})
}))
defer api.Close()
d := New(Config{
ID: "123-main",
AccessToken: "token-1",
MainAPIBaseURL: api.URL,
})
if err := d.Rename(context.Background(), "9001", "new name.mp4"); err != nil {
t.Fatalf("Rename() error = %v", err)
}
if renameRequest["driveId"].(float64) != 0 || renameRequest["fileId"] != "9001" || renameRequest["fileName"] != "new name.mp4" {
t.Fatalf("rename request = %#v, want driveId/fileId/fileName", renameRequest)
}
}
+21
View File
@@ -129,6 +129,27 @@ type mkdirResp struct {
} `json:"data"`
}
type uploadResp struct {
Data struct {
AccessKeyID string `json:"AccessKeyId"`
Bucket string `json:"Bucket"`
Key string `json:"Key"`
SecretAccessKey string `json:"SecretAccessKey"`
SessionToken string `json:"SessionToken"`
FileID int64 `json:"FileId"`
Reuse bool `json:"Reuse"`
EndPoint string `json:"EndPoint"`
StorageNode string `json:"StorageNode"`
UploadID string `json:"UploadId"`
} `json:"data"`
}
type s3PreSignedURLsResp struct {
Data struct {
PreSignedURLs map[string]string `json:"presignedUrls"`
} `json:"data"`
}
type flexibleTime struct {
t time.Time
}
+30 -5
View File
@@ -1,5 +1,5 @@
// Package spider91migrate 周期性把 spider91 drive 下载到本地的视频
// 上传到一个指定的目标 drive 目录(PikPak、115 或 OneDrive),上传成功后:
// 上传到一个指定的目标 drive 目录(PikPak、115、123 或 OneDrive),上传成功后:
//
// - 改写 catalog 行:drive_id / file_id / content_hash 改成目标盘的;
// 视频自身的 id 不变(仍是 spider91-<driveID>-<viewkey>),video_tags、
@@ -31,17 +31,19 @@ import (
"github.com/video-site/backend/internal/drives"
"github.com/video-site/backend/internal/drives/onedrive"
"github.com/video-site/backend/internal/drives/p115"
"github.com/video-site/backend/internal/drives/p123"
"github.com/video-site/backend/internal/drives/pikpak"
"github.com/video-site/backend/internal/drives/spider91"
"github.com/video-site/backend/internal/mediaasset"
)
// uploadTarget 是 migrator 调用目标 drive 的最小接口。任何一种"接收 spider91 上传"的
// 网盘都要实现它;当前 PikPak 和 115 各自通过适配器满足。
// 网盘都要实现它;当前 PikPak、115、123 和 OneDrive 各自通过适配器满足。
//
// 这一层抽象把"迁移调用方"和"具体盘的 SDK 协议"解耦:
// - PikPak 走 GCID + OSS PutObjectpikpak.UploadResult
// - 115 走 SHA1 + 秒传 / OSS / 分片(p115.UploadResult
// - 123 走 MD5 + 秒传 / S3 预签名分片(p123.UploadResult
// - OneDrive 走 SHA1 + 小文件 PUT / 大文件 upload session
//
// 各家返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
@@ -57,7 +59,7 @@ type uploadTarget interface {
// UploadResult 是 uploadTarget.UploadAndReportHash 的归一返回。
//
// FileID 目标盘上的新文件 ID;
// Hash GCIDPikPak)或 SHA1 HEX115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;
// Hash GCIDPikPak、MD5 HEX123或 SHA1 HEX115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;
// Size 实际上传字节数。
type UploadResult struct {
FileID string
@@ -67,7 +69,7 @@ type UploadResult struct {
const spider91UploadDirName = "91 Spider"
// pikpakAdapter / p115Adapter / onedriveAdapter 把具体 driver 包装成 uploadTarget。
// pikpakAdapter / p115Adapter / p123Adapter / onedriveAdapter 把具体 driver 包装成 uploadTarget。
//
// 之所以不让 driver 直接实现 uploadTarget
//
@@ -116,6 +118,27 @@ func (a *p115Adapter) Rename(ctx context.Context, fileID, newName string) error
return a.d.Rename(ctx, fileID, newName)
}
type p123Adapter struct {
d *p123.Driver
}
func (a *p123Adapter) ID() string { return a.d.ID() }
func (a *p123Adapter) Kind() string { return a.d.Kind() }
func (a *p123Adapter) RootID() string { return a.d.RootID() }
func (a *p123Adapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
return a.d.EnsureDir(ctx, pathFromRoot)
}
func (a *p123Adapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
if err != nil {
return UploadResult{}, err
}
return UploadResult{FileID: res.FileID, Hash: res.Hash, Size: res.Size}, nil
}
func (a *p123Adapter) Rename(ctx context.Context, fileID, newName string) error {
return a.d.Rename(ctx, fileID, newName)
}
type onedriveAdapter struct {
d *onedrive.Driver
}
@@ -145,6 +168,8 @@ func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
return &pikpakAdapter{d: v}, nil
case *p115.Driver:
return &p115Adapter{d: v}, nil
case *p123.Driver:
return &p123Adapter{d: v}, nil
case *onedrive.Driver:
return &onedriveAdapter{d: v}, nil
case uploadTarget:
@@ -760,7 +785,7 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driv
return deleted, nil
}
// backfillFileNames 扫描目标 drivePikPak、115 或 OneDrive)下所有 spider91-* 起始 ID 的视频,
// backfillFileNames 扫描目标 drivePikPak、115、123 或 OneDrive)下所有 spider91-* 起始 ID 的视频,
// 对文件名不是 desiredPikPakName(...) 期望格式的,调 target.Rename 修正,
// 并把 catalog.file_name 同步到新名字。
//
@@ -14,6 +14,7 @@ import (
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/drives"
"github.com/video-site/backend/internal/drives/p123"
"github.com/video-site/backend/internal/drives/pikpak"
"github.com/video-site/backend/internal/drives/spider91"
)
@@ -134,6 +135,19 @@ func (d *fakeP115) Kind() string { return "p115" }
var _ drives.Drive = (*fakeP115)(nil)
var _ uploadTarget = (*fakeP115)(nil)
type fakeP123 struct {
*fakePikPak
}
func newFakeP123(id, rootID string) *fakeP123 {
return &fakeP123{fakePikPak: newFakePikPak(id, rootID)}
}
func (d *fakeP123) Kind() string { return "p123" }
var _ drives.Drive = (*fakeP123)(nil)
var _ uploadTarget = (*fakeP123)(nil)
type fakeOneDrive struct {
*fakePikPak
}
@@ -946,6 +960,66 @@ func TestRunOnceMigratesToP115Target(t *testing.T) {
}
}
func TestRunOnceMigratesToP123Target(t *testing.T) {
cat := setupCatalog(t)
src, _ := setupSpider91(t)
target := newFakeP123("p123-target", "p123-root-id")
reg := newFakeRegistry()
reg.Add(src)
reg.Add(target)
now := time.Now()
id := writeSpider91Video(t, cat, src, "vk-123-001", ".mp4", []byte("video bytes 123"), now)
m := New(Config{
Catalog: cat,
Registry: reg,
GetTargetDriveID: func() string { return target.ID() },
KeepLatestN: -1,
})
m.runOnce(context.Background())
if target.uploadCalls != 1 {
t.Fatalf("p123 upload calls = %d, want 1", target.uploadCalls)
}
got, err := cat.GetVideo(context.Background(), id)
if err != nil {
t.Fatalf("get video: %v", err)
}
if got.DriveID != target.ID() {
t.Fatalf("drive_id = %q, want %q", got.DriveID, target.ID())
}
wantName := "Sample vk-123-001-001.mp4"
if _, ok := target.gotBodies[wantName]; !ok {
t.Fatalf("p123 did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
}
if gotParent := target.gotParents[wantName]; gotParent != "p123-root-id/"+spider91UploadDirName {
t.Fatalf("p123 upload parent = %q, want root/91 Spider", gotParent)
}
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
t.Fatalf("p123 ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
}
if got.FileID != "remote-"+wantName {
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
}
if got.FileName != wantName {
t.Fatalf("file_name = %q, want %q", got.FileName, wantName)
}
if got.ContentHash == "" {
t.Fatal("content_hash should be set after p123 migration")
}
videoPath, _ := src.VideoPath("vk-123-001.mp4")
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
t.Fatalf("local mp4 still exists after p123 migration or stat error: %v", err)
}
thumbPath, _ := src.ThumbPath("vk-123-001.jpg")
if _, err := os.Stat(thumbPath); !os.IsNotExist(err) {
t.Fatalf("local thumb still exists after p123 migration or stat error: %v", err)
}
}
func TestRunOnceMigratesToOneDriveTarget(t *testing.T) {
cat := setupCatalog(t)
src, _ := setupSpider91(t)
@@ -1006,7 +1080,22 @@ func TestRunOnceMigratesToOneDriveTarget(t *testing.T) {
}
}
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115 也不是 OneDrive 时,
func TestAdaptUploadTargetSupportsP123Driver(t *testing.T) {
d := p123.New(p123.Config{
ID: "p123-target",
RootID: "root-123",
AccessToken: "token-1",
})
target, err := adaptUploadTarget(d)
if err != nil {
t.Fatalf("adaptUploadTarget() error = %v", err)
}
if target.ID() != "p123-target" || target.Kind() != "p123" || target.RootID() != "root-123" {
t.Fatalf("target id/kind/root = %q/%q/%q, want p123-target/p123/root-123", target.ID(), target.Kind(), target.RootID())
}
}
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115、123 也不是 OneDrive 时,
// resolveTarget 拒绝并返回 error,让 runOnce 静默跳过(不会做破坏性变更)。
func TestResolveTargetRejectsUnsupportedKind(t *testing.T) {
cat := setupCatalog(t)
+1 -1
View File
@@ -68,7 +68,7 @@ export function DrivesPage() {
const formDirty = !sameForm(form, initialForm);
const uploadTargets = useMemo(
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115" || d.kind === "onedrive"),
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115" || d.kind === "p123" || d.kind === "onedrive"),
[list]
);
+2 -2
View File
@@ -388,9 +388,9 @@ export type Theme = "dark" | "pink";
export type Settings = {
theme: Theme;
/**
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115 或 onedrive drive)。
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115、p123 或 onedrive drive)。
* - 空字符串:本地保存,不上传到云盘。
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, onedrive}。
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, p123, onedrive}。
*/
spider91UploadDriveId: string;
};
@@ -25,7 +25,7 @@ export function Spider91UploadTargetField({
))}
</select>
<div className="admin-form__help">
115 PikPak OneDrive 91 Spider
115 123 PikPak OneDrive 91 Spider
</div>
</div>
);
+1 -1
View File
@@ -52,7 +52,7 @@ test("spider91 upload target uses explicit local-save option instead of auto tar
assert.match(combinedSource, /本地保存,不上传/);
assert.match(
combinedSource,
/d\.kind === "pikpak" \|\| d\.kind === "p115" \|\| d\.kind === "onedrive"/
/d\.kind === "pikpak" \|\| d\.kind === "p115" \|\| d\.kind === "p123" \|\| d\.kind === "onedrive"/
);
assert.doesNotMatch(combinedSource, /自动:唯一/);
assert.doesNotMatch(combinedSource, /自动模式/);