From 940e5dd76d7fe6857cefea4b2431453fbf9e0f58 Mon Sep 17 00:00:00 2001 From: nianzhibai <177086871+nianzhibai@users.noreply.github.com> Date: Mon, 8 Jun 2026 23:50:19 +0800 Subject: [PATCH] feat: support spider91 uploads to google drive --- backend/README.md | 6 +- backend/cmd/server/main.go | 10 +- backend/internal/drives/googledrive/driver.go | 558 +++++++++++++++++- .../drives/googledrive/driver_test.go | 217 +++++++ backend/internal/drives/googledrive/types.go | 14 +- backend/internal/fingerprint/worker.go | 54 ++ backend/internal/fingerprint/worker_test.go | 28 + backend/internal/preview/ffmpeg.go | 58 +- backend/internal/preview/worker_test.go | 24 +- backend/internal/spider91migrate/migrator.go | 35 +- .../internal/spider91migrate/migrator_test.go | 18 +- src/admin/DrivesPage.tsx | 10 +- src/admin/api.ts | 4 +- tests/adminDriveForm.test.ts | 2 +- 14 files changed, 997 insertions(+), 41 deletions(-) diff --git a/backend/README.md b/backend/README.md index 8a04191..b566b6d 100644 --- a/backend/README.md +++ b/backend/README.md @@ -120,7 +120,7 @@ go run ./cmd/server 后端 9192 OneDrive 按 OpenList 默认应用方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token，不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。后台新建 OneDrive 时只需要填 OpenList 代刷得到的 `refresh_token`；服务端会默认挂载根目录并自动回写新 token。 -Google Drive 默认按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/renewapi` 刷新 token。后台新建 Google Drive 时只需要填 OpenList Google Drive 获取到的 `refresh_token`。如果不想依赖 OpenList 在线 API，可以关闭“使用 OpenList 在线续期 API”，并填写同一个 Google OAuth 客户端授权得到的 `refresh_token`、`client_id`、`client_secret`，服务端会直接请求 Google OAuth token 接口续期。Google Drive 下载地址必须携带 `Authorization` 头，浏览器不能直接 302 使用，所以本站会由后端代理 `/p/stream` 播放，不加入零带宽 302 白名单。 +Google Drive 默认按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/renewapi` 刷新 token。后台新建 Google Drive 时只需要填 OpenList Google Drive 获取到的 `refresh_token`。如果不想依赖 OpenList 在线 API，可以关闭“使用 OpenList 在线续期 API”，并填写同一个 Google OAuth 客户端授权得到的 `refresh_token`、`client_id`、`client_secret`，服务端会直接请求 Google OAuth token 接口续期。Google Drive 下载地址必须携带 `Authorization` 头，浏览器不能直接 302 使用，所以本站会由后端代理 `/p/stream` 播放，不加入零带宽 302 白名单。91 爬虫迁移到 Google Drive 时使用 Google Drive resumable upload session 上传，并把上传文件的 MD5 写入 catalog 用于去重。 ## 文件名约定 @@ -147,7 +147,7 @@ Google Drive 默认按 OpenList 在线 API 调用 `https://api.oplist.org/google 2. 扫描时优先按网盘侧 `content_hash` 去重；没有 hash 时退化为 `file_name + size_bytes`。 3. 扫描、爬虫、本地上传或服务启动挂载网盘后，后台指纹 worker 会异步读取视频的少量 Range 片段，生成 `sampled_sha256`。前台列表、首页、搜索、推荐会按 `size_bytes + sampled_sha256` 只展示最早入库的 canonical 视频。 -`sampled_sha256` 是文件级去重：适合识别同一个视频文件被复制到 115 / PikPak / OneDrive 等不同网盘的情况。它不会删除任何网盘文件，也不用于识别转码、裁剪、加水印后的同源视频。 +`sampled_sha256` 是文件级去重：适合识别同一个视频文件被复制到 115 / PikPak / OneDrive / Google Drive 等不同网盘的情况。它不会删除任何网盘文件，也不用于识别转码、裁剪、加水印后的同源视频。 封面和预览视频仍然优先生成，不等待指纹完成。夜间流水线最后会做一次重复资产清理：对 `size_bytes + sampled_sha256` 命中的非 canonical 视频，只删除本机生成的重复封面和预览视频，并把对应字段重置为 `pending`。网盘原文件和视频元数据记录不会被删除；如果 canonical 视频以后被移除，这些重复项会重新进入生成队列。 @@ -170,7 +170,7 @@ ffmpeg -ss <起点> -headers "UA/Cookie/Referer" -i <直链> \ 当前策略是每段固定 3 秒；30 秒以下最多 3 段，30 秒及以上固定 4 段；长视频在 20% 到 80% 区间均匀取段。生成的预览视频和封面都只保存在本地 `data/previews/`，不会回写到网盘；旧数据中的 `preview_file_id` 会被忽略。 -服务启动或网盘重新挂载时，如果预览视频开关已开启，后端会把历史 `pending` 任务重新入队，避免重启后长期停在“待生成”。OneDrive 扫盘和直链生成预览视频 / 封面时可能触发 Microsoft Graph 429、`TooManyRequests`、`activityLimitReached` 或 throttled 文本；后端会识别这类错误并让当前网盘进入冷却期，保留任务为 `pending`，避免连续请求触发更严重限流。扫盘阶段会按 `Retry-After` 或默认冷却时间等待后继续当前目录。 +服务启动或网盘重新挂载时，如果预览视频开关已开启，后端会把历史 `pending` 任务重新入队，避免重启后长期停在“待生成”。OneDrive 扫盘和直链生成预览视频 / 封面时可能触发 Microsoft Graph 429、`TooManyRequests`、`activityLimitReached` 或 throttled 文本；Google Drive 可能返回 429、`usageLimits`、`userRateLimitExceeded`、`downloadQuotaExceeded` 等限制标识。后端会识别这类错误并让当前网盘进入冷却期，保留任务为 `pending`，避免连续请求触发更严重限流。扫盘阶段会按 `Retry-After` 或默认冷却时间等待后继续当前目录。 前端卡片的 `previewSrc` 统一指向 `/p/preview/`，后端只从本地 `preview_local` 文件读取。 diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index eaf0311..59b221b 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -317,7 +317,7 @@ type App struct { // 空字符串表示本地保存不上传，不再自动挑选 pikpak/p115/p123/onedrive drive。 spider91UploadDriveID string - // spider91Migrator 把 spider91 视频上传到目标 drive（PikPak、115、123 或 OneDrive）。 + // spider91Migrator 把 spider91 视频上传到目标 drive（PikPak、115、123、OneDrive 或 Google Drive）。 spider91Migrator spider91MigrationRunner // nightlyRunner 是凌晨流水线调度器：每天 cron_hour 串行跑扫盘 → 91 爬虫 → 迁移。 @@ -415,7 +415,7 @@ func (a *App) loadTheme(ctx context.Context) { } // Spider91UploadDriveID 返回当前配置的 spider91 上传目标 drive ID。 -// 空字符串表示本地保存不上传；只有管理员显式选择 pikpak/p115/p123/onedrive drive 时才迁移上传。 +// 空字符串表示本地保存不上传；只有管理员显式选择 pikpak/p115/p123/onedrive/googledrive drive 时才迁移上传。 func (a *App) Spider91UploadDriveID() string { a.mu.Lock() explicit := a.spider91UploadDriveID @@ -432,7 +432,7 @@ func (a *App) Spider91UploadDriveID() string { // SetSpider91UploadDriveID 设置 spider91 上传目标 drive ID 并持久化。 // 接受空字符串（本地保存不上传）。 -// 设置一个不存在或 kind 不是 pikpak / p115 / p123 / onedrive 的 drive 会返回错误。 +// 设置一个不存在或 kind 不是 pikpak / p115 / p123 / onedrive / googledrive 的 drive 会返回错误。 func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) error { driveID = strings.TrimSpace(driveID) if driveID != "" { @@ -441,7 +441,7 @@ func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) erro return fmt.Errorf("drive %q not found", driveID) } if !isSpider91UploadKind(d.Kind()) { - return fmt.Errorf("drive %q kind=%s, only pikpak, p115, p123 or onedrive can be spider91 upload target", driveID, d.Kind()) + return fmt.Errorf("drive %q kind=%s, only pikpak, p115, p123, onedrive or googledrive can be spider91 upload target", driveID, d.Kind()) } } a.mu.Lock() @@ -474,7 +474,7 @@ func formatOptionalRFC3339(t time.Time) string { // isSpider91UploadKind 是 spider91 迁移目标盘的 allowlist。 // 与 spider91migrate.adaptUploadTarget 的支持范围保持一致。 func isSpider91UploadKind(kind string) bool { - return kind == "pikpak" || kind == "p115" || kind == "p123" || kind == "onedrive" + return kind == "pikpak" || kind == "p115" || kind == "p123" || kind == "onedrive" || kind == "googledrive" } // loadSpider91UploadDriveID 从 DB 读上传目标 drive ID 设置；不存在时使用空串。 diff --git a/backend/internal/drives/googledrive/driver.go b/backend/internal/drives/googledrive/driver.go index 562b326..456cd88 100644 --- a/backend/internal/drives/googledrive/driver.go +++ b/backend/internal/drives/googledrive/driver.go @@ -1,10 +1,17 @@ package googledrive import ( + "bytes" "context" + "crypto/md5" + "encoding/hex" + "encoding/json" "errors" "fmt" + "hash" "io" + "log" + "math" "net/http" "net/url" "path" @@ -21,10 +28,13 @@ import ( const ( Kind = "googledrive" defaultAPIBaseURL = "https://www.googleapis.com/drive/v3" + defaultUploadAPIURL = "https://www.googleapis.com/upload/drive/v3" defaultOAuthURL = "https://www.googleapis.com/oauth2/v4/token" defaultRenewAPIURL = "https://api.oplist.org/googleui/renewapi" defaultListInterval = 1 * time.Second defaultListCooldown = 5 * time.Minute + defaultLinkCooldown = 5 * time.Minute + uploadChunkSize = int64(8 * 1024 * 1024) filesListFields = "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken" fileInfoFields = "id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum" @@ -41,13 +51,19 @@ type Driver struct { renewAPIURL string oauthURL string apiBaseURL string + uploadBaseURL string client *resty.Client + httpClient *http.Client onTokenUpdate func(access, refresh string) listMu sync.Mutex lastListAt time.Time listInterval time.Duration listCooldown time.Duration + + linkCooldownMu sync.Mutex + linkCooldownUntil time.Time + linkCooldownDuration time.Duration } type Config struct { @@ -61,6 +77,7 @@ type Config struct { RenewAPIURL string OAuthURL string APIBaseURL string + UploadAPIURL string OnTokenUpdate func(access, refresh string) } @@ -82,6 +99,10 @@ func New(c Config) *Driver { if apiBaseURL == "" { apiBaseURL = defaultAPIBaseURL } + uploadBaseURL := strings.TrimRight(strings.TrimSpace(c.UploadAPIURL), "/") + if uploadBaseURL == "" { + uploadBaseURL = deriveUploadBaseURL(apiBaseURL) + } return &Driver{ id: c.ID, rootID: rootID, @@ -93,15 +114,34 @@ func New(c Config) *Driver { renewAPIURL: renewAPIURL, oauthURL: oauthURL, apiBaseURL: apiBaseURL, + uploadBaseURL: uploadBaseURL, onTokenUpdate: c.OnTokenUpdate, client: resty.New(). SetTimeout(30*time.Second). SetHeader("Accept", "application/json, text/plain, */*"), - listInterval: defaultListInterval, - listCooldown: defaultListCooldown, + httpClient: &http.Client{ + Timeout: 0, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + listInterval: defaultListInterval, + listCooldown: defaultListCooldown, + linkCooldownDuration: defaultLinkCooldown, } } +func deriveUploadBaseURL(apiBaseURL string) string { + apiBaseURL = strings.TrimRight(strings.TrimSpace(apiBaseURL), "/") + if apiBaseURL == "" || apiBaseURL == defaultAPIBaseURL { + return defaultUploadAPIURL + } + if strings.HasSuffix(apiBaseURL, "/drive/v3") { + return strings.TrimSuffix(apiBaseURL, "/drive/v3") + "/upload/drive/v3" + } + return apiBaseURL +} + func (d *Driver) Kind() string { return Kind } func (d *Driver) ID() string { return d.id } func (d *Driver) RootID() string { return d.rootID } @@ -209,8 +249,19 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi if fileID == "" { return nil, errors.New("googledrive stream: empty file id") } + if err := ctx.Err(); err != nil { + return nil, err + } + if err := d.linkCooldownError(time.Now()); err != nil { + return nil, err + } if _, err := d.Stat(ctx, fileID); err != nil { - return nil, fmt.Errorf("googledrive stream: %w", err) + err = fmt.Errorf("googledrive stream: %w", err) + if wait, ok := drives.RateLimitRetryAfter(err); ok { + until := d.pauseLinkCooldown(wait) + log.Printf("[googledrive] stream link cooling down drive=%s until=%s err=%v", d.id, until.Format(time.RFC3339), err) + } + return nil, err } u := d.fileURL(fileID) + "?alt=media&acknowledgeAbuse=true&supportsAllDrives=true" return &drives.StreamLink{ @@ -222,12 +273,383 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi }, nil } -func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) { - return "", drives.ErrNotSupported +func (d *Driver) linkCooldownError(now time.Time) error { + d.linkCooldownMu.Lock() + defer d.linkCooldownMu.Unlock() + if d.linkCooldownUntil.IsZero() { + return nil + } + if !now.Before(d.linkCooldownUntil) { + d.linkCooldownUntil = time.Time{} + return nil + } + wait := d.linkCooldownUntil.Sub(now) + if wait <= 0 { + return nil + } + return &drives.RateLimitError{ + Provider: Kind, + RetryAfter: wait, + Err: fmt.Errorf("googledrive stream link cooling down until %s", d.linkCooldownUntil.Format(time.RFC3339)), + } } -func (d *Driver) EnsureDir(context.Context, string) (string, error) { - return "", drives.ErrNotSupported +func (d *Driver) pauseLinkCooldown(wait time.Duration) time.Time { + if wait <= 0 { + wait = d.linkCooldownDuration + } + if wait <= 0 { + wait = defaultLinkCooldown + } + until := time.Now().Add(wait) + d.linkCooldownMu.Lock() + if until.After(d.linkCooldownUntil) { + d.linkCooldownUntil = until + } else { + until = d.linkCooldownUntil + } + d.linkCooldownMu.Unlock() + return until +} + +func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) { + res, err := d.UploadAndReportHash(ctx, parentID, name, r, size) + if err != nil { + return "", err + } + return res.FileID, nil +} + +func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) { + parentID, name, err := d.normalizeUploadArgs(parentID, name, r, size) + if err != nil { + return UploadResult{}, err + } + sessionURL, err := d.createUploadSession(ctx, parentID, name, size) + if err != nil { + return UploadResult{}, err + } + if strings.TrimSpace(sessionURL) == "" { + return UploadResult{}, errors.New("googledrive upload session: empty upload url") + } + + hasher := md5.New() + var item driveFile + var copied int64 + if size == 0 { + completed, err := d.putUploadSessionChunkWithRetry(ctx, sessionURL, 0, 0, nil, hasher) + if err != nil { + return UploadResult{}, err + } + if completed != nil { + item = *completed + } + } else { + chunkSize := uploadChunkSize + if chunkSize <= 0 { + chunkSize = 8 * 1024 * 1024 + } + if chunkSize > int64(math.MaxInt32) { + chunkSize = int64(math.MaxInt32) + } + buf := make([]byte, int(chunkSize)) + for copied < size { + partSize := minInt64(chunkSize, size-copied) + chunk := buf[:int(partSize)] + n, err := io.ReadFull(r, chunk) + if err != nil { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return UploadResult{}, fmt.Errorf("googledrive upload: size mismatch: declared %d, copied %d", size, copied+int64(n)) + } + return UploadResult{}, fmt.Errorf("googledrive upload: read body: %w", err) + } + chunk = chunk[:n] + completed, err := d.putUploadSessionChunkWithRetry(ctx, sessionURL, copied, size, chunk, hasher) + if err != nil { + return UploadResult{}, err + } + if completed != nil { + item = *completed + } + copied += int64(n) + } + } + + hashHex := hex.EncodeToString(hasher.Sum(nil)) + if item.ID == "" { + fileID, err := d.findUploadedFileID(ctx, parentID, name, hashHex) + if err != nil { + return UploadResult{}, err + } + item.ID = fileID + } + return UploadResult{FileID: item.ID, Hash: hashHex, Size: copied}, nil +} + +func (d *Driver) normalizeUploadArgs(parentID, name string, r io.Reader, size int64) (string, string, error) { + if r == nil { + return "", "", errors.New("googledrive upload: body is required") + } + if size < 0 { + return "", "", fmt.Errorf("googledrive upload: invalid size %d", size) + } + parentID = strings.TrimSpace(parentID) + if parentID == "" || parentID == "/" { + parentID = d.rootID + } + name = strings.TrimSpace(name) + if name == "" { + return "", "", errors.New("googledrive upload: empty file name") + } + return parentID, name, nil +} + +func (d *Driver) createUploadSession(ctx context.Context, parentID, name string, size int64) (string, error) { + return d.createUploadSessionOnce(ctx, parentID, name, size, true) +} + +func (d *Driver) createUploadSessionOnce(ctx context.Context, parentID, name string, size int64, retry bool) (string, error) { + var apiErr apiErrorResp + res, err := d.client.R(). + SetContext(ctx). + SetHeader("Authorization", "Bearer "+d.accessToken). + SetHeader("X-Upload-Content-Type", mimeType(driveFile{Name: name})). + SetHeader("X-Upload-Content-Length", strconv.FormatInt(size, 10)). + SetQueryParams(map[string]string{ + "uploadType": "resumable", + "supportsAllDrives": "true", + "fields": fileInfoFields, + }). + SetBody(map[string]any{ + "name": name, + "parents": []string{parentID}, + }). + SetError(&apiErr). + Post(d.uploadFilesURL()) + if err != nil { + return "", fmt.Errorf("googledrive upload session: %w", err) + } + if isGoogleRateLimit(res, apiErr.Error) { + return "", googleRateLimitError(res, apiErr.Error.Message) + } + if apiErr.Error.Code != 0 { + if apiErr.Error.Code == http.StatusUnauthorized && retry { + if err := d.refresh(ctx); err != nil { + return "", err + } + return d.createUploadSessionOnce(ctx, parentID, name, size, false) + } + return "", googleAPIError(apiErr.Error) + } + if res.IsError() { + return "", fmt.Errorf("googledrive upload session: status=%d body=%s", res.StatusCode(), strings.TrimSpace(res.String())) + } + return strings.TrimSpace(res.Header().Get("Location")), nil +} + +func (d *Driver) putUploadSessionChunkWithRetry(ctx context.Context, uploadURL string, start, total int64, data []byte, hasher hash.Hash) (*driveFile, error) { + var last error + for attempt := 0; attempt < 3; attempt++ { + if attempt > 0 { + if err := sleepContext(ctx, time.Duration(attempt)*time.Second); err != nil { + return nil, err + } + } + item, retryable, err := d.putUploadSessionChunk(ctx, uploadURL, start, total, data) + if err == nil { + if hasher != nil && len(data) > 0 { + _, _ = hasher.Write(data) + } + return item, nil + } + last = err + if !retryable { + return nil, err + } + } + if last == nil { + last = errors.New("googledrive upload session: retry attempts exhausted") + } + return nil, last +} + +func (d *Driver) putUploadSessionChunk(ctx context.Context, uploadURL string, start, total int64, data []byte) (*driveFile, bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(data)) + if err != nil { + return nil, false, err + } + req.ContentLength = int64(len(data)) + req.Header.Set("Authorization", "Bearer "+d.accessToken) + req.Header.Set("Content-Length", strconv.Itoa(len(data))) + if total == 0 { + req.Header.Set("Content-Range", "bytes */0") + } else { + end := start + int64(len(data)) - 1 + req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, total)) + } + hc := d.httpClient + if hc == nil { + hc = http.DefaultClient + } + res, err := hc.Do(req) + if err != nil { + return nil, true, fmt.Errorf("googledrive upload session: put chunk: %w", err) + } + defer res.Body.Close() + + switch res.StatusCode { + case http.StatusOK, http.StatusCreated: + var item driveFile + if err := json.NewDecoder(res.Body).Decode(&item); err != nil { + return nil, false, fmt.Errorf("googledrive upload session: decode completed file: %w", err) + } + return &item, false, nil + case http.StatusPermanentRedirect: + return nil, false, nil + case http.StatusUnauthorized: + if err := d.refresh(ctx); err != nil { + return nil, false, err + } + return nil, true, fmt.Errorf("googledrive upload session: unauthorized") + default: + body, _ := io.ReadAll(io.LimitReader(res.Body, 64*1024)) + var apiErr apiErrorResp + _ = json.Unmarshal(body, &apiErr) + if isGoogleUploadHTTPRateLimit(res.StatusCode, res.Header, body, apiErr.Error) { + return nil, false, googleUploadRateLimitError(res.StatusCode, res.Header, body, apiErr.Error.Message) + } + retryable := res.StatusCode == http.StatusTooManyRequests || (res.StatusCode >= 500 && res.StatusCode <= 504) + return nil, retryable, fmt.Errorf("googledrive upload session: status=%d body=%s", res.StatusCode, strings.TrimSpace(string(body))) + } +} + +func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) { + currentID := d.rootID + for _, name := range splitPath(pathFromRoot) { + childID, err := d.findChildDir(ctx, currentID, name) + if err != nil { + return "", err + } + if childID == "" { + childID, err = d.makeDir(ctx, currentID, name) + if err != nil { + return "", err + } + } + currentID = childID + } + return currentID, nil +} + +func (d *Driver) findChildDir(ctx context.Context, parentID, name string) (string, error) { + entries, err := d.List(ctx, parentID) + if err != nil { + return "", err + } + for _, e := range entries { + if e.IsDir && e.Name == name { + return e.ID, nil + } + } + return "", nil +} + +func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, error) { + var item driveFile + err := d.request(ctx, d.filesURL(), http.MethodPost, func(req *resty.Request) { + req.SetQueryParam("fields", fileInfoFields) + req.SetBody(map[string]any{ + "name": name, + "parents": []string{parentID}, + "mimeType": "application/vnd.google-apps.folder", + }) + }, &item) + if err != nil { + return "", fmt.Errorf("googledrive mkdir %s: %w", name, err) + } + if item.ID == "" { + return "", fmt.Errorf("googledrive mkdir %s: empty file id", name) + } + return item.ID, nil +} + +func (d *Driver) Rename(ctx context.Context, fileID, newName string) error { + fileID = strings.TrimSpace(fileID) + if fileID == "" { + return errors.New("googledrive rename: empty file id") + } + newName = strings.TrimSpace(newName) + if newName == "" { + return errors.New("googledrive rename: empty new name") + } + var item driveFile + err := d.request(ctx, d.fileURL(fileID), http.MethodPatch, func(req *resty.Request) { + req.SetQueryParam("fields", fileInfoFields) + req.SetBody(map[string]string{"name": newName}) + }, &item) + if err != nil { + return fmt.Errorf("googledrive rename: %w", err) + } + return nil +} + +func (d *Driver) findUploadedFileID(ctx context.Context, parentID, name, md5Hex string) (string, error) { + entries, err := d.List(ctx, parentID) + if err != nil { + return "", fmt.Errorf("googledrive upload verify: %w", err) + } + var hashHit string + for _, e := range entries { + if e.IsDir { + continue + } + if !strings.EqualFold(e.Hash, md5Hex) { + continue + } + if e.Name == name { + return e.ID, nil + } + if hashHit == "" { + hashHit = e.ID + } + } + if hashHit != "" { + return hashHit, nil + } + for _, e := range entries { + if !e.IsDir && e.Name == name { + return e.ID, nil + } + } + return "", fmt.Errorf("googledrive upload: uploaded file %q not found in parent %q", name, parentID) +} + +func isGoogleUploadHTTPRateLimit(status int, header http.Header, body []byte, apiErr apiErrorBody) bool { + if status == http.StatusTooManyRequests { + return true + } + if status == http.StatusForbidden && strings.TrimSpace(header.Get("Retry-After")) != "" { + return true + } + if isGoogleRateLimit(nil, apiErr) { + return true + } + return googleLimitText(string(body)) +} + +func googleUploadRateLimitError(status int, header http.Header, body []byte, message string) error { + if strings.TrimSpace(message) == "" { + message = "google drive upload rate limited" + } + bodyText := strings.TrimSpace(string(body)) + if bodyText != "" { + message = fmt.Sprintf("%s: status=%d body=%s", message, status, bodyText) + } + return &drives.RateLimitError{ + Provider: Kind, + RetryAfter: parseRetryAfterHeader(header.Get("Retry-After")), + Err: errors.New(message), + } } func (d *Driver) refresh(ctx context.Context) error { @@ -288,6 +710,26 @@ func (d *Driver) applyToken(out tokenResp) { } func tokenResponseError(prefix string, res *resty.Response, out tokenResp, requireRefresh bool) error { + if isGoogleTokenRateLimit(res, out) { + message := strings.TrimSpace(out.Text) + if message == "" { + message = strings.TrimSpace(out.ErrorDescription) + } + if message == "" { + message = strings.TrimSpace(out.Error) + } + if message == "" { + message = "google drive token refresh rate limited" + } + if res != nil && strings.TrimSpace(res.String()) != "" { + message = fmt.Sprintf("%s: status=%d body=%s", message, res.StatusCode(), strings.TrimSpace(res.String())) + } + return &drives.RateLimitError{ + Provider: Kind, + RetryAfter: parseRetryAfter(res), + Err: fmt.Errorf("%s: %s", prefix, message), + } + } if out.Text != "" { return fmt.Errorf("%s: %s", prefix, out.Text) } @@ -380,6 +822,10 @@ func (d *Driver) filesURL() string { return d.apiBaseURL + "/files" } +func (d *Driver) uploadFilesURL() string { + return d.uploadBaseURL + "/files" +} + func (d *Driver) fileURL(fileID string) string { return d.filesURL() + "/" + url.PathEscape(fileID) } @@ -444,18 +890,85 @@ func isGoogleRateLimit(res *resty.Response, body apiErrorBody) bool { if res != nil && res.StatusCode() == http.StatusTooManyRequests { return true } + if res != nil && res.StatusCode() == http.StatusForbidden && strings.TrimSpace(res.Header().Get("Retry-After")) != "" { + return true + } if body.Code == http.StatusTooManyRequests { return true } for _, e := range body.Errors { - reason := strings.ToLower(strings.TrimSpace(e.Reason)) - switch reason { - case "ratelimitexceeded", "userratelimitexceeded", "downloadquotaexceeded", "sharingratelimitexceeded": + if googleLimitReason(e.Reason) || googleLimitText(e.Message) { + return true + } + domain := compactGoogleLimitText(e.Domain) + if domain == "usagelimits" && (body.Code == http.StatusForbidden || body.Code == http.StatusTooManyRequests) { return true } } - msg := strings.ToLower(body.Message) - return strings.Contains(msg, "rate limit") || strings.Contains(msg, "too many requests") || strings.Contains(msg, "quota exceeded") + return googleLimitText(body.Message) +} + +func isGoogleTokenRateLimit(res *resty.Response, out tokenResp) bool { + if res != nil { + if res.StatusCode() == http.StatusTooManyRequests { + return true + } + if res.StatusCode() == http.StatusForbidden && strings.TrimSpace(res.Header().Get("Retry-After")) != "" { + return true + } + } + return googleLimitText(out.Text) || + googleLimitText(out.Error) || + googleLimitText(out.ErrorDescription) +} + +func googleLimitReason(reason string) bool { + switch compactGoogleLimitText(reason) { + case "ratelimitexceeded", + "userratelimitexceeded", + "dailylimitexceeded", + "dailylimitexceededunreg", + "downloadquotaexceeded", + "sharingratelimitexceeded", + "quotaexceeded", + "uploadlimitexceeded", + "storagelimitexceeded", + "storagequotaexceeded": + return true + default: + return false + } +} + +func googleLimitText(text string) bool { + text = strings.ToLower(strings.TrimSpace(text)) + if text == "" { + return false + } + compact := compactGoogleLimitText(text) + if strings.Contains(compact, "ratelimitexceeded") || + strings.Contains(compact, "userratelimitexceeded") || + strings.Contains(compact, "dailylimitexceeded") || + strings.Contains(compact, "downloadquotaexceeded") || + strings.Contains(compact, "sharingratelimitexceeded") || + strings.Contains(compact, "quotaexceeded") || + strings.Contains(compact, "toomanyrequests") { + return true + } + return strings.Contains(text, "rate limit") || + strings.Contains(text, "too many requests") || + strings.Contains(text, "quota exceeded") || + strings.Contains(text, "download quota") || + strings.Contains(text, "sharing rate") || + strings.Contains(text, "daily limit") || + strings.Contains(text, "user rate") || + strings.Contains(text, "usage limit") +} + +func compactGoogleLimitText(text string) string { + text = strings.ToLower(strings.TrimSpace(text)) + replacer := strings.NewReplacer("_", "", "-", "", " ", "", ".", "", ":", "") + return replacer.Replace(text) } func googleRateLimitError(res *resty.Response, message string) error { @@ -486,7 +999,11 @@ func parseRetryAfter(res *resty.Response) time.Duration { if res == nil { return 0 } - raw := strings.TrimSpace(res.Header().Get("Retry-After")) + return parseRetryAfterHeader(res.Header().Get("Retry-After")) +} + +func parseRetryAfterHeader(raw string) time.Duration { + raw = strings.TrimSpace(raw) if raw == "" { return 0 } @@ -502,4 +1019,19 @@ func parseRetryAfter(res *resty.Response) time.Duration { return 0 } +func splitPath(p string) []string { + p = strings.Trim(p, "/") + if p == "" { + return nil + } + return strings.Split(p, "/") +} + +func minInt64(a, b int64) int64 { + if a < b { + return a + } + return b +} + var _ drives.Drive = (*Driver)(nil) diff --git a/backend/internal/drives/googledrive/driver_test.go b/backend/internal/drives/googledrive/driver_test.go index 15269f8..5d0690e 100644 --- a/backend/internal/drives/googledrive/driver_test.go +++ b/backend/internal/drives/googledrive/driver_test.go @@ -2,11 +2,18 @@ package googledrive import ( "context" + "crypto/md5" + "encoding/hex" "encoding/json" + "errors" + "io" "net/http" "net/http/httptest" "strings" "testing" + "time" + + "github.com/video-site/backend/internal/drives" ) func TestInitUsesOnlineRenewAPI(t *testing.T) { @@ -131,6 +138,134 @@ func TestStreamURLReturnsAuthenticatedMediaLinkWithoutRedirectRequirement(t *tes } } +func TestUploadAndReportHashUsesResumableSession(t *testing.T) { + body := "hello google drive" + wantHash := md5.Sum([]byte(body)) + var sawSession bool + var sawUpload bool + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/upload/drive/v3/files": + sawSession = true + if got := r.Header.Get("Authorization"); got != "Bearer access" { + t.Fatalf("session Authorization = %q", got) + } + if got := r.URL.Query().Get("uploadType"); got != "resumable" { + t.Fatalf("uploadType = %q", got) + } + if got := r.Header.Get("X-Upload-Content-Length"); got != "18" { + t.Fatalf("X-Upload-Content-Length = %q", got) + } + var meta struct { + Name string `json:"name"` + Parents []string `json:"parents"` + } + if err := json.NewDecoder(r.Body).Decode(&meta); err != nil { + t.Fatalf("decode session metadata: %v", err) + } + if meta.Name != "clip.mp4" || len(meta.Parents) != 1 || meta.Parents[0] != "parent-1" { + t.Fatalf("metadata = %+v", meta) + } + w.Header().Set("Location", srv.URL+"/upload/session/1") + w.WriteHeader(http.StatusOK) + case "/upload/session/1": + sawUpload = true + if got := r.Header.Get("Authorization"); got != "Bearer access" { + t.Fatalf("upload Authorization = %q", got) + } + if got := r.Header.Get("Content-Range"); got != "bytes 0-17/18" { + t.Fatalf("Content-Range = %q", got) + } + gotBody, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read upload body: %v", err) + } + if string(gotBody) != body { + t.Fatalf("upload body = %q", string(gotBody)) + } + writeTestJSONStatus(w, http.StatusCreated, driveFile{ + ID: "file-uploaded", + Name: "clip.mp4", + Size: "18", + MD5Checksum: hex.EncodeToString(wantHash[:]), + }) + default: + t.Fatalf("unexpected path %s", r.URL.Path) + } + })) + defer srv.Close() + + d := New(Config{ID: "g", APIBaseURL: srv.URL + "/drive/v3"}) + d.accessToken = "access" + res, err := d.UploadAndReportHash(context.Background(), "parent-1", "clip.mp4", strings.NewReader(body), int64(len(body))) + if err != nil { + t.Fatalf("UploadAndReportHash() error = %v", err) + } + if !sawSession || !sawUpload { + t.Fatalf("saw session/upload = %v/%v, want both", sawSession, sawUpload) + } + if res.FileID != "file-uploaded" || res.Size != int64(len(body)) || res.Hash != hex.EncodeToString(wantHash[:]) { + t.Fatalf("upload result = %+v", res) + } +} + +func TestEnsureDirAndRenameUseGoogleDriveFileAPI(t *testing.T) { + var madeDir bool + var renamed bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/drive/v3/files": + writeTestJSON(w, filesResp{}) + case r.Method == http.MethodPost && r.URL.Path == "/drive/v3/files": + madeDir = true + var meta struct { + Name string `json:"name"` + Parents []string `json:"parents"` + MimeType string `json:"mimeType"` + } + if err := json.NewDecoder(r.Body).Decode(&meta); err != nil { + t.Fatalf("decode mkdir body: %v", err) + } + if meta.Name != "91 Spider" || len(meta.Parents) != 1 || meta.Parents[0] != "root" || meta.MimeType != "application/vnd.google-apps.folder" { + t.Fatalf("mkdir body = %+v", meta) + } + writeTestJSON(w, driveFile{ID: "folder-91", Name: "91 Spider", MimeType: "application/vnd.google-apps.folder"}) + case r.Method == http.MethodPatch && r.URL.Path == "/drive/v3/files/file-1": + renamed = true + var body map[string]string + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatalf("decode rename body: %v", err) + } + if body["name"] != "new-name.mp4" { + t.Fatalf("rename body = %+v", body) + } + writeTestJSON(w, driveFile{ID: "file-1", Name: "new-name.mp4"}) + default: + t.Fatalf("unexpected %s %s", r.Method, r.URL.Path) + } + })) + defer srv.Close() + + d := New(Config{ID: "g", RootID: "root", APIBaseURL: srv.URL + "/drive/v3"}) + d.accessToken = "access" + d.listInterval = -1 + + dirID, err := d.EnsureDir(context.Background(), "91 Spider") + if err != nil { + t.Fatalf("EnsureDir() error = %v", err) + } + if dirID != "folder-91" || !madeDir { + t.Fatalf("dirID/madeDir = %q/%v, want folder-91/true", dirID, madeDir) + } + if err := d.Rename(context.Background(), "file-1", "new-name.mp4"); err != nil { + t.Fatalf("Rename() error = %v", err) + } + if !renamed { + t.Fatal("rename endpoint was not called") + } +} + func TestRequestRefreshesOnUnauthorized(t *testing.T) { var fileCalls int srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -179,6 +314,88 @@ func TestRequestRefreshesOnUnauthorized(t *testing.T) { } } +func TestRateLimitReasonsFollowGoogleDriveErrorShape(t *testing.T) { + reasons := []string{ + "rateLimitExceeded", + "userRateLimitExceeded", + "dailyLimitExceeded", + "dailyLimitExceededUnreg", + "downloadQuotaExceeded", + "sharingRateLimitExceeded", + "quotaExceeded", + } + for _, reason := range reasons { + body := apiErrorBody{ + Code: http.StatusForbidden, + Message: "google drive quota or rate limited", + Errors: []struct { + Domain string `json:"domain"` + Reason string `json:"reason"` + Message string `json:"message"` + LocationType string `json:"location_type"` + Location string `json:"location"` + }{ + {Domain: "usageLimits", Reason: reason, Message: reason}, + }, + } + if !isGoogleRateLimit(nil, body) { + t.Fatalf("reason %q not treated as rate limit", reason) + } + } +} + +func TestStreamURLRateLimitStartsSharedLinkCooldown(t *testing.T) { + var calls int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.Header().Set("Retry-After", "120") + writeTestJSONStatus(w, http.StatusForbidden, apiErrorResp{Error: apiErrorBody{ + Code: http.StatusForbidden, + Message: "User rate limit exceeded.", + Errors: []struct { + Domain string `json:"domain"` + Reason string `json:"reason"` + Message string `json:"message"` + LocationType string `json:"location_type"` + Location string `json:"location"` + }{ + {Domain: "usageLimits", Reason: "userRateLimitExceeded", Message: "User rate limit exceeded."}, + }, + }}) + })) + defer srv.Close() + + d := New(Config{ID: "g", APIBaseURL: srv.URL}) + d.accessToken = "access" + d.linkCooldownDuration = time.Hour + + _, err := d.StreamURL(context.Background(), "file-1") + if err == nil { + t.Fatal("first StreamURL succeeded, want rate limit") + } + var rateLimit *drives.RateLimitError + if !errors.As(err, &rateLimit) { + t.Fatalf("first error = %T %[1]v, want RateLimitError", err) + } + if rateLimit.RetryAfter != 2*time.Minute { + t.Fatalf("retry after = %s, want 2m", rateLimit.RetryAfter) + } + + _, err = d.StreamURL(context.Background(), "file-1") + if err == nil { + t.Fatal("second StreamURL succeeded during cooldown") + } + if !errors.As(err, &rateLimit) { + t.Fatalf("second error = %T %[1]v, want RateLimitError", err) + } + if calls != 1 { + t.Fatalf("remote calls = %d, want 1; second call should use shared cooldown", calls) + } + if rateLimit.RetryAfter <= 0 || rateLimit.RetryAfter > 2*time.Minute { + t.Fatalf("second retry after = %s, want remaining cooldown", rateLimit.RetryAfter) + } +} + func writeTestJSON(w http.ResponseWriter, v any) { writeTestJSONStatus(w, http.StatusOK, v) } diff --git a/backend/internal/drives/googledrive/types.go b/backend/internal/drives/googledrive/types.go index bcc87fb..09da6ca 100644 --- a/backend/internal/drives/googledrive/types.go +++ b/backend/internal/drives/googledrive/types.go @@ -42,8 +42,16 @@ type apiErrorBody struct { Code int `json:"code"` Message string `json:"message"` Errors []struct { - Domain string `json:"domain"` - Reason string `json:"reason"` - Message string `json:"message"` + Domain string `json:"domain"` + Reason string `json:"reason"` + Message string `json:"message"` + LocationType string `json:"location_type"` + Location string `json:"location"` } `json:"errors"` } + +type UploadResult struct { + FileID string + Hash string + Size int64 +} diff --git a/backend/internal/fingerprint/worker.go b/backend/internal/fingerprint/worker.go index b3b4125..fa1e9dc 100644 --- a/backend/internal/fingerprint/worker.go +++ b/backend/internal/fingerprint/worker.go @@ -327,11 +327,65 @@ func readHTTPRange(ctx context.Context, hc *http.Client, link *drives.StreamLink return data, nil } } + body, _ := io.ReadAll(io.LimitReader(resp.Body, 64*1024)) + if remoteRangeResponseLooksRateLimited(link.URL, resp.StatusCode, body) { + return nil, &drives.RateLimitError{ + Provider: "fingerprint", + RetryAfter: parseRetryAfter(resp.Header.Get("Retry-After")), + Err: fmt.Errorf("remote sample rate limited: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(body))), + } + } return nil, fmt.Errorf("fingerprint: range request got status=%d for bytes=%d-%d", resp.StatusCode, r.start, end) } return io.ReadAll(io.LimitReader(resp.Body, r.length)) } +func remoteRangeResponseLooksRateLimited(rawURL string, status int, body []byte) bool { + if status == http.StatusTooManyRequests { + return true + } + text := strings.ToLower(strings.TrimSpace(string(body))) + compact := compactRemoteRangeErrorText(text) + if strings.Contains(text, "too many request") || + strings.Contains(text, "too many requests") || + strings.Contains(text, "rate limit") || + strings.Contains(text, "quota exceeded") || + strings.Contains(text, "download quota") || + strings.Contains(text, "sharing rate") || + strings.Contains(text, "daily limit") || + strings.Contains(text, "user rate") || + strings.Contains(text, "usage limit") || + strings.Contains(compact, "ratelimitexceeded") || + strings.Contains(compact, "userratelimitexceeded") || + strings.Contains(compact, "dailylimitexceeded") || + strings.Contains(compact, "downloadquotaexceeded") || + strings.Contains(compact, "sharingratelimitexceeded") || + strings.Contains(compact, "quotaexceeded") || + strings.Contains(compact, "toomanyrequests") || + strings.Contains(compact, "usagelimits") { + return true + } + if status == http.StatusForbidden && isGoogleDriveMediaURL(rawURL) { + return true + } + return false +} + +func isGoogleDriveMediaURL(rawURL string) bool { + u, err := url.Parse(rawURL) + if err != nil { + return false + } + host := strings.ToLower(u.Host) + path := strings.ToLower(u.Path) + return strings.Contains(host, "googleapis.com") && strings.Contains(path, "/drive/") +} + +func compactRemoteRangeErrorText(text string) string { + replacer := strings.NewReplacer("_", "", "-", "", " ", "", ".", "", ":", "") + return replacer.Replace(strings.ToLower(strings.TrimSpace(text))) +} + func parseRetryAfter(raw string) time.Duration { raw = strings.TrimSpace(raw) if raw == "" { diff --git a/backend/internal/fingerprint/worker_test.go b/backend/internal/fingerprint/worker_test.go index 9cf5e98..a4ade52 100644 --- a/backend/internal/fingerprint/worker_test.go +++ b/backend/internal/fingerprint/worker_test.go @@ -2,6 +2,7 @@ package fingerprint import ( "context" + "errors" "fmt" "io" "net/http" @@ -85,6 +86,33 @@ func TestComputeRemoteUsesRangeSamples(t *testing.T) { } } +func TestComputeRemoteGoogleQuotaExceededReturnsRateLimit(t *testing.T) { + ctx := context.Background() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "60") + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error":{"code":403,"message":"The download quota for this file has been exceeded.","errors":[{"domain":"usageLimits","reason":"downloadQuotaExceeded","message":"The download quota for this file has been exceeded."}]}}`)) + })) + defer srv.Close() + + drv := &fakeDrive{paths: map[string]string{"remote": srv.URL + "/drive/v3/files/file-1?alt=media"}} + _, err := Compute(ctx, drv, &catalog.Video{ID: "remote", FileID: "remote", Size: 1024 * 1024}, Config{ + SampleSizeBytes: 4, + FullHashMaxSize: 8, + HTTPClient: srv.Client(), + }, srv.Client()) + if err == nil { + t.Fatal("compute succeeded, want rate limit") + } + var rateLimit *drives.RateLimitError + if !errors.As(err, &rateLimit) { + t.Fatalf("error = %T %[1]v, want RateLimitError", err) + } + if rateLimit.RetryAfter != time.Minute { + t.Fatalf("retry after = %s, want 1m", rateLimit.RetryAfter) + } +} + type fakeDrive struct { paths map[string]string } diff --git a/backend/internal/preview/ffmpeg.go b/backend/internal/preview/ffmpeg.go index 7c597ff..f0453c4 100644 --- a/backend/internal/preview/ffmpeg.go +++ b/backend/internal/preview/ffmpeg.go @@ -1427,11 +1427,14 @@ func (w *Worker) skipIfRateLimited(v *catalog.Video) bool { } func (w *Worker) pauseForRateLimit(err error, step, title string) bool { - _, ok := drives.RateLimitRetryAfter(err) + wait, ok := drives.RateLimitRetryAfter(err) if !ok { return false } - until := w.rateLimit.pause(time.Now(), defaultGenerationRateLimitCooldown) + if wait <= 0 { + wait = defaultGenerationRateLimitCooldown + } + until := w.rateLimit.pause(time.Now(), wait) log.Printf("[preview] drive=%s rate-limited until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err) return true } @@ -1460,11 +1463,14 @@ func (w *ThumbWorker) skipIfRateLimited(v *catalog.Video) bool { } func (w *ThumbWorker) pauseForRateLimit(err error, step, title string) bool { - _, ok := drives.RateLimitRetryAfter(err) + wait, ok := drives.RateLimitRetryAfter(err) if !ok { return false } - until := w.rateLimit.pause(time.Now(), defaultGenerationRateLimitCooldown) + if wait <= 0 { + wait = defaultGenerationRateLimitCooldown + } + until := w.rateLimit.pause(time.Now(), wait) log.Printf("[thumb] drive=%s rate-limited until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err) return true } @@ -1560,10 +1566,54 @@ func driveErrorShouldCooldown(d drives.Drive, err error) bool { strings.Contains(text, "blocked") || strings.Contains(text, "访问被阻断") || strings.Contains(text, "service unavailable") + case "googledrive": + // Google Drive 下载/取样阶段常把频控和配额问题包装成 403， + // 具体标识在 error.errors[].reason/message 里（OpenList 也按该结构解析）。 + // ffmpeg/ffprobe 只能看到 stderr 文本时，按这些 reason/文本兜底冷却。 + text := strings.ToLower(err.Error()) + return googleDriveMediaErrorShouldCooldown(text) } return false } +func googleDriveMediaErrorShouldCooldown(text string) bool { + if text == "" { + return false + } + compact := compactGoogleDriveErrorText(text) + return strings.Contains(text, "server returned 403") || + strings.Contains(text, "403 forbidden") || + strings.Contains(text, "server returned 429") || + strings.Contains(text, "http 429") || + strings.Contains(text, "http 500") || + strings.Contains(text, "http 502") || + strings.Contains(text, "http 503") || + strings.Contains(text, "http 504") || + strings.Contains(text, "too many request") || + strings.Contains(text, "too many requests") || + strings.Contains(text, "rate limit") || + strings.Contains(text, "quota exceeded") || + strings.Contains(text, "download quota") || + strings.Contains(text, "sharing rate") || + strings.Contains(text, "daily limit") || + strings.Contains(text, "user rate") || + strings.Contains(text, "usage limit") || + strings.Contains(text, "service unavailable") || + strings.Contains(compact, "ratelimitexceeded") || + strings.Contains(compact, "userratelimitexceeded") || + strings.Contains(compact, "dailylimitexceeded") || + strings.Contains(compact, "downloadquotaexceeded") || + strings.Contains(compact, "sharingratelimitexceeded") || + strings.Contains(compact, "quotaexceeded") || + strings.Contains(compact, "toomanyrequests") || + strings.Contains(compact, "usagelimits") +} + +func compactGoogleDriveErrorText(text string) string { + replacer := strings.NewReplacer("_", "", "-", "", " ", "", ".", "", ":", "") + return replacer.Replace(strings.ToLower(strings.TrimSpace(text))) +} + func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) bool { if w.skipIfRateLimited(v) { return false diff --git a/backend/internal/preview/worker_test.go b/backend/internal/preview/worker_test.go index 85398c6..046a200 100644 --- a/backend/internal/preview/worker_test.go +++ b/backend/internal/preview/worker_test.go @@ -442,7 +442,7 @@ func TestPreviewWorkerRateLimitLeavesCurrentPendingAndSkipsNextVideo(t *testing. if gen.generateCalls != 1 { t.Fatalf("generate calls = %d, want 1", gen.generateCalls) } - assertCooldownAround(t, worker.Status().CooldownUntil, before, 5*time.Minute) + assertCooldownAround(t, worker.Status().CooldownUntil, before, 2*time.Hour) gen.generateErr = nil worker.process(ctx, &second) @@ -458,7 +458,7 @@ func TestPreviewWorkerRateLimitLeavesCurrentPendingAndSkipsNextVideo(t *testing. } } -func TestThumbWorkerRateLimitCoolsDownFiveMinutes(t *testing.T) { +func TestThumbWorkerRateLimitHonorsRetryAfter(t *testing.T) { ctx := context.Background() cat, video := seedPreviewTestVideo(t, "thumb-rate-limit") @@ -482,7 +482,7 @@ func TestThumbWorkerRateLimitCoolsDownFiveMinutes(t *testing.T) { if got.ThumbnailURL != "" { t.Fatalf("thumbnail = %q, want unchanged after rate limit", got.ThumbnailURL) } - assertCooldownAround(t, worker.Status().CooldownUntil, before, 5*time.Minute) + assertCooldownAround(t, worker.Status().CooldownUntil, before, 2*time.Hour) } func TestThumbWorkerP115TransientErrorFailsAfterRetryLimit(t *testing.T) { @@ -661,6 +661,24 @@ func TestP123TransientErrorsShouldCooldown(t *testing.T) { } } +func TestGoogleDriveMediaErrorsShouldCooldown(t *testing.T) { + drv := &previewFakeDrive{kind: "googledrive"} + for _, err := range []error{ + errors.New("google drive api error: usageLimits userRateLimitExceeded"), + errors.New("ffmpeg: Server returned 403 Forbidden"), + errors.New("downloadQuotaExceeded: The download quota for this file has been exceeded"), + errors.New("sharingRateLimitExceeded"), + errors.New("http 503 service unavailable"), + } { + if !driveErrorShouldCooldown(drv, err) { + t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err) + } + } + if driveErrorShouldCooldown(drv, errors.New("invalid credentials")) { + t.Fatal("invalid credentials should not trigger googledrive cooldown") + } +} + func assertCooldownAround(t *testing.T, until time.Time, before time.Time, want time.Duration) { t.Helper() if until.IsZero() { diff --git a/backend/internal/spider91migrate/migrator.go b/backend/internal/spider91migrate/migrator.go index e79f0a5..903dbe8 100644 --- a/backend/internal/spider91migrate/migrator.go +++ b/backend/internal/spider91migrate/migrator.go @@ -1,5 +1,5 @@ // Package spider91migrate 周期性把 spider91 drive 下载到本地的视频 -// 上传到一个指定的目标 drive 目录（PikPak、115、123 或 OneDrive），上传成功后： +// 上传到一个指定的目标 drive 目录（PikPak、115、123、OneDrive 或 Google Drive），上传成功后： // // - 改写 catalog 行：drive_id / file_id / content_hash 改成目标盘的； // 视频自身的 id 不变（仍是 spider91--），video_tags、 @@ -29,6 +29,7 @@ import ( "github.com/video-site/backend/internal/catalog" "github.com/video-site/backend/internal/drives" + "github.com/video-site/backend/internal/drives/googledrive" "github.com/video-site/backend/internal/drives/onedrive" "github.com/video-site/backend/internal/drives/p115" "github.com/video-site/backend/internal/drives/p123" @@ -38,13 +39,14 @@ import ( ) // uploadTarget 是 migrator 调用目标 drive 的最小接口。任何一种"接收 spider91 上传"的 -// 网盘都要实现它；当前 PikPak、115、123 和 OneDrive 各自通过适配器满足。 +// 网盘都要实现它；当前 PikPak、115、123、OneDrive 和 Google Drive 各自通过适配器满足。 // // 这一层抽象把"迁移调用方"和"具体盘的 SDK 协议"解耦： // - PikPak 走 GCID + OSS PutObject（pikpak.UploadResult） // - 115 走 SHA1 + 秒传 / OSS / 分片（p115.UploadResult） // - 123 走 MD5 + 秒传 / S3 预签名分片（p123.UploadResult） // - OneDrive 走 SHA1 + 小文件 PUT / 大文件 upload session +// - Google Drive 走 MD5 + resumable upload session // // 各家返回值都被归一成本地的 UploadResult，并在 catalog 改写阶段统一处理。 type uploadTarget interface { @@ -59,7 +61,7 @@ type uploadTarget interface { // UploadResult 是 uploadTarget.UploadAndReportHash 的归一返回。 // // FileID 目标盘上的新文件 ID； -// Hash GCID（PikPak）、MD5 HEX（123）或 SHA1 HEX（115 / OneDrive），写入 catalog.content_hash 用于跨盘去重； +// Hash GCID（PikPak）、MD5 HEX（123 / Google Drive）或 SHA1 HEX（115 / OneDrive），写入 catalog.content_hash 用于跨盘去重； // Size 实际上传字节数。 type UploadResult struct { FileID string @@ -69,7 +71,7 @@ type UploadResult struct { const spider91UploadDirName = "91 Spider" -// pikpakAdapter / p115Adapter / p123Adapter / onedriveAdapter 把具体 driver 包装成 uploadTarget。 +// pikpakAdapter / p115Adapter / p123Adapter / onedriveAdapter / googledriveAdapter 把具体 driver 包装成 uploadTarget。 // // 之所以不让 driver 直接实现 uploadTarget： // @@ -160,6 +162,27 @@ func (a *onedriveAdapter) Rename(ctx context.Context, fileID, newName string) er return a.d.Rename(ctx, fileID, newName) } +type googledriveAdapter struct { + d *googledrive.Driver +} + +func (a *googledriveAdapter) ID() string { return a.d.ID() } +func (a *googledriveAdapter) Kind() string { return a.d.Kind() } +func (a *googledriveAdapter) RootID() string { return a.d.RootID() } +func (a *googledriveAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) { + return a.d.EnsureDir(ctx, pathFromRoot) +} +func (a *googledriveAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) { + res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size) + if err != nil { + return UploadResult{}, err + } + return UploadResult{FileID: res.FileID, Hash: res.Hash, Size: res.Size}, nil +} +func (a *googledriveAdapter) Rename(ctx context.Context, fileID, newName string) error { + return a.d.Rename(ctx, fileID, newName) +} + // adaptUploadTarget 把通用 drive 包装成 uploadTarget。 // 不支持的盘 kind 返回 error；调用方静默跳过。 func adaptUploadTarget(d drives.Drive) (uploadTarget, error) { @@ -172,6 +195,8 @@ func adaptUploadTarget(d drives.Drive) (uploadTarget, error) { return &p123Adapter{d: v}, nil case *onedrive.Driver: return &onedriveAdapter{d: v}, nil + case *googledrive.Driver: + return &googledriveAdapter{d: v}, nil case uploadTarget: // 测试或自定义实现可以直接传入；优先使用具体类型分支以拿到适配器。 return v, nil @@ -785,7 +810,7 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driv return deleted, nil } -// backfillFileNames 扫描目标 drive（PikPak、115、123 或 OneDrive）下所有 spider91-* 起始 ID 的视频， +// backfillFileNames 扫描目标 drive（PikPak、115、123、OneDrive 或 Google Drive）下所有 spider91-* 起始 ID 的视频， // 对文件名不是 desiredPikPakName(...) 期望格式的，调 target.Rename 修正， // 并把 catalog.file_name 同步到新名字。 // diff --git a/backend/internal/spider91migrate/migrator_test.go b/backend/internal/spider91migrate/migrator_test.go index 5b96700..f0a20c3 100644 --- a/backend/internal/spider91migrate/migrator_test.go +++ b/backend/internal/spider91migrate/migrator_test.go @@ -14,6 +14,7 @@ import ( "github.com/video-site/backend/internal/catalog" "github.com/video-site/backend/internal/drives" + "github.com/video-site/backend/internal/drives/googledrive" "github.com/video-site/backend/internal/drives/p123" "github.com/video-site/backend/internal/drives/pikpak" "github.com/video-site/backend/internal/drives/spider91" @@ -1095,7 +1096,22 @@ func TestAdaptUploadTargetSupportsP123Driver(t *testing.T) { } } -// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115、123 也不是 OneDrive 时， +func TestAdaptUploadTargetSupportsGoogleDriveDriver(t *testing.T) { + d := googledrive.New(googledrive.Config{ + ID: "google-target", + RootID: "root-google", + RefreshToken: "refresh-token", + }) + target, err := adaptUploadTarget(d) + if err != nil { + t.Fatalf("adaptUploadTarget() error = %v", err) + } + if target.ID() != "google-target" || target.Kind() != "googledrive" || target.RootID() != "root-google" { + t.Fatalf("target id/kind/root = %q/%q/%q, want google-target/googledrive/root-google", target.ID(), target.Kind(), target.RootID()) + } +} + +// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115、123、OneDrive 也不是 Google Drive 时， // resolveTarget 拒绝并返回 error，让 runOnce 静默跳过（不会做破坏性变更）。 func TestResolveTargetRejectsUnsupportedKind(t *testing.T) { cat := setupCatalog(t) diff --git a/src/admin/DrivesPage.tsx b/src/admin/DrivesPage.tsx index c115d78..0f67652 100644 --- a/src/admin/DrivesPage.tsx +++ b/src/admin/DrivesPage.tsx @@ -92,7 +92,15 @@ export function DrivesPage() { : hasCreateFormChanges(form, initialForm); const uploadTargets = useMemo( - () => list.filter((d) => d.kind === "pikpak" || d.kind === "p115" || d.kind === "p123" || d.kind === "onedrive"), + () => + list.filter( + (d) => + d.kind === "pikpak" || + d.kind === "p115" || + d.kind === "p123" || + d.kind === "onedrive" || + d.kind === "googledrive" + ), [list] ); diff --git a/src/admin/api.ts b/src/admin/api.ts index b929b18..f403c3f 100644 --- a/src/admin/api.ts +++ b/src/admin/api.ts @@ -407,9 +407,9 @@ export type Theme = "dark" | "pink"; export type Settings = { theme: Theme; /** - * spider91 视频迁移到云盘时的目标 drive ID（必须是已挂载的 pikpak、p115、p123 或 onedrive drive）。 + * spider91 视频迁移到云盘时的目标 drive ID（必须是已挂载的 pikpak、p115、p123、onedrive 或 googledrive drive）。 * - 空字符串：本地保存，不上传到云盘。 - * - 非空：显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, p123, onedrive}。 + * - 非空：显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, p123, onedrive, googledrive}。 */ spider91UploadDriveId: string; }; diff --git a/tests/adminDriveForm.test.ts b/tests/adminDriveForm.test.ts index 79b3d06..08c39d2 100644 --- a/tests/adminDriveForm.test.ts +++ b/tests/adminDriveForm.test.ts @@ -65,7 +65,7 @@ test("spider91 upload target uses explicit local-save option instead of auto tar assert.match(combinedSource, /本地保存，不上传/); assert.match( combinedSource, - /d\.kind === "pikpak" \|\| d\.kind === "p115" \|\| d\.kind === "p123" \|\| d\.kind === "onedrive"/ + /d\.kind === "pikpak"[\s\S]*d\.kind === "p115"[\s\S]*d\.kind === "p123"[\s\S]*d\.kind === "onedrive"[\s\S]*d\.kind === "googledrive"/ ); assert.doesNotMatch(combinedSource, /自动：唯一/); assert.doesNotMatch(combinedSource, /自动模式/);