mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-15 16:55:42 +08:00
Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 71d4a16db1 | |||
| 940e5dd76d | |||
| e826c05d5c | |||
| 3465b9e837 | |||
| d33c1b1b20 | |||
| 5fc8e9ebb7 | |||
| dc7d2a5de3 | |||
| 2f2bfbfcdc | |||
| 9def08b0c5 |
@@ -35,3 +35,10 @@ tmp/
|
||||
91VideoSpider/__pycache__/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Local scratch images
|
||||
/image.jpg
|
||||
/image003.jpg
|
||||
/image004.jpg
|
||||
/image005.png
|
||||
/image02.png
|
||||
|
||||
+4
-4
@@ -109,7 +109,7 @@ go run ./cmd/server 后端 9192
|
||||
| pikpak | `username`、`password`(token、验证码和设备 ID 由服务端自动处理并保存) |
|
||||
| wopan | `access_token`、`refresh_token`,可选 `family_id` |
|
||||
| onedrive | `refresh_token` |
|
||||
| googledrive | `refresh_token` |
|
||||
| googledrive | 默认只需 `refresh_token`;自建 OAuth 客户端模式还需 `use_online_api=false`、`client_id`、`client_secret` |
|
||||
| localstorage | `path`(服务器上的已有视频目录,如 `/mnt/videos`) |
|
||||
|
||||
### PikPak 速度说明
|
||||
@@ -120,7 +120,7 @@ go run ./cmd/server 后端 9192
|
||||
|
||||
OneDrive 按 OpenList 默认应用方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。后台新建 OneDrive 时只需要填 OpenList 代刷得到的 `refresh_token`;服务端会默认挂载根目录并自动回写新 token。
|
||||
|
||||
Google Drive 按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/renewapi` 刷新 token。后台新建 Google Drive 时只需要填 OpenList Google Drive 获取到的 `refresh_token`。Google Drive 下载地址必须携带 `Authorization` 头,浏览器不能直接 302 使用,所以本站会由后端代理 `/p/stream` 播放,不加入零带宽 302 白名单。
|
||||
Google Drive 默认按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/renewapi` 刷新 token。后台新建 Google Drive 时只需要填 OpenList Google Drive 获取到的 `refresh_token`。如果不想依赖 OpenList 在线 API,可以关闭“使用 OpenList 在线续期 API”,并填写同一个 Google OAuth 客户端授权得到的 `refresh_token`、`client_id`、`client_secret`,服务端会直接请求 Google OAuth token 接口续期。Google Drive 下载地址必须携带 `Authorization` 头,浏览器不能直接 302 使用,所以本站会由后端代理 `/p/stream` 播放,不加入零带宽 302 白名单。91 爬虫迁移到 Google Drive 时使用 Google Drive resumable upload session 上传,并把上传文件的 MD5 写入 catalog 用于去重。
|
||||
|
||||
## 文件名约定
|
||||
|
||||
@@ -147,7 +147,7 @@ Google Drive 按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/ren
|
||||
2. 扫描时优先按网盘侧 `content_hash` 去重;没有 hash 时退化为 `file_name + size_bytes`。
|
||||
3. 扫描、爬虫、本地上传或服务启动挂载网盘后,后台指纹 worker 会异步读取视频的少量 Range 片段,生成 `sampled_sha256`。前台列表、首页、搜索、推荐会按 `size_bytes + sampled_sha256` 只展示最早入库的 canonical 视频。
|
||||
|
||||
`sampled_sha256` 是文件级去重:适合识别同一个视频文件被复制到 115 / PikPak / OneDrive 等不同网盘的情况。它不会删除任何网盘文件,也不用于识别转码、裁剪、加水印后的同源视频。
|
||||
`sampled_sha256` 是文件级去重:适合识别同一个视频文件被复制到 115 / PikPak / OneDrive / Google Drive 等不同网盘的情况。它不会删除任何网盘文件,也不用于识别转码、裁剪、加水印后的同源视频。
|
||||
|
||||
封面和预览视频仍然优先生成,不等待指纹完成。夜间流水线最后会做一次重复资产清理:对 `size_bytes + sampled_sha256` 命中的非 canonical 视频,只删除本机生成的重复封面和预览视频,并把对应字段重置为 `pending`。网盘原文件和视频元数据记录不会被删除;如果 canonical 视频以后被移除,这些重复项会重新进入生成队列。
|
||||
|
||||
@@ -170,7 +170,7 @@ ffmpeg -ss <起点> -headers "UA/Cookie/Referer" -i <直链> \
|
||||
|
||||
当前策略是每段固定 3 秒;30 秒以下最多 3 段,30 秒及以上固定 4 段;长视频在 20% 到 80% 区间均匀取段。生成的预览视频和封面都只保存在本地 `data/previews/`,不会回写到网盘;旧数据中的 `preview_file_id` 会被忽略。
|
||||
|
||||
服务启动或网盘重新挂载时,如果预览视频开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 扫盘和直链生成预览视频 / 封面时可能触发 Microsoft Graph 429、`TooManyRequests`、`activityLimitReached` 或 throttled 文本;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。扫盘阶段会按 `Retry-After` 或默认冷却时间等待后继续当前目录。
|
||||
服务启动或网盘重新挂载时,如果预览视频开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 扫盘和直链生成预览视频 / 封面时可能触发 Microsoft Graph 429、`TooManyRequests`、`activityLimitReached` 或 throttled 文本;Google Drive 可能返回 429、`usageLimits`、`userRateLimitExceeded`、`downloadQuotaExceeded` 等限制标识。后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。扫盘阶段会按 `Retry-After` 或默认冷却时间等待后继续当前目录。
|
||||
|
||||
前端卡片的 `previewSrc` 统一指向 `/p/preview/<videoID>`,后端只从本地 `preview_local` 文件读取。
|
||||
|
||||
|
||||
+251
-72
@@ -170,16 +170,15 @@ func main() {
|
||||
OnDriveRemoved: func(driveID string) {
|
||||
app.detachDrive(driveID)
|
||||
},
|
||||
OnScanRequested: func(driveID string) {
|
||||
OnScanRequested: func(driveID string) bool {
|
||||
// spider91 的"重扫"等同于手动触发一次爬取;其它 drive 走标准 scan
|
||||
app.mu.Lock()
|
||||
_, isSpider91 := app.spider91Crawlers[driveID]
|
||||
app.mu.Unlock()
|
||||
if isSpider91 {
|
||||
app.scheduleSpider91Crawl(ctx, driveID)
|
||||
return
|
||||
return app.scheduleSpider91Crawl(ctx, driveID)
|
||||
}
|
||||
app.scheduleScan(ctx, driveID)
|
||||
return app.scheduleScan(ctx, driveID)
|
||||
},
|
||||
OnStopDriveTasks: func(driveID string) bool {
|
||||
return app.stopDriveTasks(ctx, driveID)
|
||||
@@ -318,29 +317,20 @@ type App struct {
|
||||
// 空字符串表示本地保存不上传,不再自动挑选 pikpak/p115/p123/onedrive drive。
|
||||
spider91UploadDriveID string
|
||||
|
||||
// spider91Migrator 周期把 spider91 视频上传到目标 drive(PikPak、115、123 或 OneDrive)。
|
||||
spider91Migrator *spider91migrate.Migrator
|
||||
// spider91Migrator 把 spider91 视频上传到目标 drive(PikPak、115、123、OneDrive 或 Google Drive)。
|
||||
spider91Migrator spider91MigrationRunner
|
||||
|
||||
// nightlyRunner 是凌晨流水线调度器:每天 cron_hour 串行跑扫盘 → 91 爬虫 → 迁移。
|
||||
// 也响应 admin 「扫描所有网盘」按钮(TriggerNow)。
|
||||
nightlyRunner *nightly.Runner
|
||||
|
||||
// scanGlobalMu 串行化所有云盘扫盘任务,确保同一时刻全系统只有一个扫盘
|
||||
// 在跑(包括 admin 手动重扫和 nightly Phase 1)。即便用户同时点多个 drive
|
||||
// 的"重扫"按钮,goroutine 也会排队等这把锁,逐个执行。
|
||||
//
|
||||
// 设计取舍:
|
||||
// - 不同 drive 的扫盘技术上可以并行(互不干涉),但用户希望"线性来"以
|
||||
// 避免带宽 / CPU 抢占,所以做全局串行。
|
||||
// - nightly Phase 1 已经是 for 循环顺序调用 runScan,加了这把锁后行为
|
||||
// 不变,只是顺手把 admin 异步触发的请求也接入同一条队列。
|
||||
scanGlobalMu sync.Mutex
|
||||
// scanQueueMu 保护 scanQueued。
|
||||
// scanQueueMu 保护 scanQueued 和 scanProgress。
|
||||
scanQueueMu sync.Mutex
|
||||
// scanQueued 跟踪哪些 driveID 已经排队或正在跑扫盘/91 爬取,去重后续重复点击。
|
||||
// 一个 drive 在 scheduleScan/scheduleSpider91Crawl 入队时被加入,后台 goroutine
|
||||
// 结束时被移除。
|
||||
// 不同 drive 互不等待,可以并行扫;同一个 drive 只能有一个扫盘/抓取任务。
|
||||
scanQueued map[string]bool
|
||||
// scanProgress 跟踪每个正在扫盘/抓取的 drive 当前进度。
|
||||
scanProgress map[string]driveScanProgress
|
||||
|
||||
// taskCancelMu 保护 driveTaskCancels。这里登记的是可被"停止任务"按钮中断
|
||||
// 的 drive 级任务上下文:扫盘、91 爬取、指纹补队列、失败生成重试等。
|
||||
@@ -354,6 +344,15 @@ type App struct {
|
||||
fingerprintQueueing map[string]bool
|
||||
}
|
||||
|
||||
type driveScanProgress struct {
|
||||
Scanned int
|
||||
Added int
|
||||
}
|
||||
|
||||
type spider91MigrationRunner interface {
|
||||
RunOnce(ctx context.Context) error
|
||||
}
|
||||
|
||||
// teaserEnabledForDrive 查询某个 drive 当前的 per-drive 预览视频开关。
|
||||
//
|
||||
// 预览视频生成不再由全局 setting 控制,而是由 catalog.drives.teaser_enabled
|
||||
@@ -416,7 +415,7 @@ func (a *App) loadTheme(ctx context.Context) {
|
||||
}
|
||||
|
||||
// Spider91UploadDriveID 返回当前配置的 spider91 上传目标 drive ID。
|
||||
// 空字符串表示本地保存不上传;只有管理员显式选择 pikpak/p115/p123/onedrive drive 时才迁移上传。
|
||||
// 空字符串表示本地保存不上传;只有管理员显式选择 pikpak/p115/p123/onedrive/googledrive drive 时才迁移上传。
|
||||
func (a *App) Spider91UploadDriveID() string {
|
||||
a.mu.Lock()
|
||||
explicit := a.spider91UploadDriveID
|
||||
@@ -433,7 +432,7 @@ func (a *App) Spider91UploadDriveID() string {
|
||||
|
||||
// SetSpider91UploadDriveID 设置 spider91 上传目标 drive ID 并持久化。
|
||||
// 接受空字符串(本地保存不上传)。
|
||||
// 设置一个不存在或 kind 不是 pikpak / p115 / p123 / onedrive 的 drive 会返回错误。
|
||||
// 设置一个不存在或 kind 不是 pikpak / p115 / p123 / onedrive / googledrive 的 drive 会返回错误。
|
||||
func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) error {
|
||||
driveID = strings.TrimSpace(driveID)
|
||||
if driveID != "" {
|
||||
@@ -442,7 +441,7 @@ func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) erro
|
||||
return fmt.Errorf("drive %q not found", driveID)
|
||||
}
|
||||
if !isSpider91UploadKind(d.Kind()) {
|
||||
return fmt.Errorf("drive %q kind=%s, only pikpak, p115, p123 or onedrive can be spider91 upload target", driveID, d.Kind())
|
||||
return fmt.Errorf("drive %q kind=%s, only pikpak, p115, p123, onedrive or googledrive can be spider91 upload target", driveID, d.Kind())
|
||||
}
|
||||
}
|
||||
a.mu.Lock()
|
||||
@@ -475,7 +474,7 @@ func formatOptionalRFC3339(t time.Time) string {
|
||||
// isSpider91UploadKind 是 spider91 迁移目标盘的 allowlist。
|
||||
// 与 spider91migrate.adaptUploadTarget 的支持范围保持一致。
|
||||
func isSpider91UploadKind(kind string) bool {
|
||||
return kind == "pikpak" || kind == "p115" || kind == "p123" || kind == "onedrive"
|
||||
return kind == "pikpak" || kind == "p115" || kind == "p123" || kind == "onedrive" || kind == "googledrive"
|
||||
}
|
||||
|
||||
// loadSpider91UploadDriveID 从 DB 读上传目标 drive ID 设置;不存在时使用空串。
|
||||
@@ -491,6 +490,17 @@ func (a *App) loadSpider91UploadDriveID(ctx context.Context) {
|
||||
}
|
||||
|
||||
func (a *App) driveGenerationStatuses() map[string]api.DriveGenerationStatuses {
|
||||
a.scanQueueMu.Lock()
|
||||
scanningDrives := make(map[string]bool, len(a.scanQueued))
|
||||
for id, running := range a.scanQueued {
|
||||
scanningDrives[id] = running
|
||||
}
|
||||
scanProgresses := make(map[string]driveScanProgress, len(a.scanProgress))
|
||||
for id, progress := range a.scanProgress {
|
||||
scanProgresses[id] = progress
|
||||
}
|
||||
a.scanQueueMu.Unlock()
|
||||
|
||||
a.mu.Lock()
|
||||
previewWorkers := make(map[string]*preview.Worker, len(a.workers))
|
||||
for id, worker := range a.workers {
|
||||
@@ -506,7 +516,20 @@ func (a *App) driveGenerationStatuses() map[string]api.DriveGenerationStatuses {
|
||||
}
|
||||
a.mu.Unlock()
|
||||
|
||||
out := make(map[string]api.DriveGenerationStatuses, len(previewWorkers)+len(thumbWorkers)+len(fingerprintWorkers))
|
||||
out := make(map[string]api.DriveGenerationStatuses, len(scanningDrives)+len(previewWorkers)+len(thumbWorkers)+len(fingerprintWorkers))
|
||||
for id, running := range scanningDrives {
|
||||
if !running {
|
||||
continue
|
||||
}
|
||||
progress := scanProgresses[id]
|
||||
status := out[id]
|
||||
status.Scan = api.GenerationStatus{
|
||||
State: "scanning",
|
||||
ScannedCount: progress.Scanned,
|
||||
AddedCount: progress.Added,
|
||||
}
|
||||
out[id] = status
|
||||
}
|
||||
for id, worker := range previewWorkers {
|
||||
status := out[id]
|
||||
status.Preview = generationStatusFromPreview(worker.Status())
|
||||
@@ -856,6 +879,12 @@ func (a *App) attachSpider91Crawler(d *catalog.Drive, drv *spider91.Driver) {
|
||||
proxyURL := strings.TrimSpace(d.Credentials["proxy"])
|
||||
|
||||
driveID := d.ID
|
||||
var progressMu sync.Mutex
|
||||
checkedVideos := 0
|
||||
expectedNewVideos := 0
|
||||
updateProgress := func(scanned, added int) {
|
||||
a.updateDriveScanProgress(driveID, scanned, added)
|
||||
}
|
||||
c := spider91.NewCrawler(spider91.CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: a.cat,
|
||||
@@ -864,6 +893,35 @@ func (a *App) attachSpider91Crawler(d *catalog.Drive, drv *spider91.Driver) {
|
||||
WorkDir: filepath.Dir(scriptPath),
|
||||
CommonThumbDir: a.commonThumbsDir(),
|
||||
ProxyURL: proxyURL,
|
||||
OnProgress: func(progress spider91.CrawlProgress) {
|
||||
progressMu.Lock()
|
||||
if progress.TotalEntries == 0 && progress.NewVideos == 0 && progress.Skipped == 0 && progress.Failed == 0 {
|
||||
checkedVideos = 0
|
||||
expectedNewVideos = 0
|
||||
} else if progress.TotalEntries > expectedNewVideos {
|
||||
expectedNewVideos = progress.TotalEntries
|
||||
}
|
||||
scanned := checkedVideos
|
||||
added := expectedNewVideos
|
||||
progressMu.Unlock()
|
||||
updateProgress(scanned, added)
|
||||
},
|
||||
OnCheckedVideo: func() {
|
||||
progressMu.Lock()
|
||||
checkedVideos++
|
||||
scanned := checkedVideos
|
||||
added := expectedNewVideos
|
||||
progressMu.Unlock()
|
||||
updateProgress(scanned, added)
|
||||
},
|
||||
OnExtractedVideo: func() {
|
||||
progressMu.Lock()
|
||||
expectedNewVideos++
|
||||
scanned := checkedVideos
|
||||
added := expectedNewVideos
|
||||
progressMu.Unlock()
|
||||
updateProgress(scanned, added)
|
||||
},
|
||||
// 新流程:预览视频不在每条视频入库时立即入队,而是 RunOnce 全部下完后由
|
||||
// runSpider91Crawl 统一调 enqueueDriveGeneration 一次性入队。这样:
|
||||
// - 下载阶段不和 ffmpeg 抢 CPU/IO
|
||||
@@ -1008,6 +1066,7 @@ func (a *App) clearQueuedDriveTask(driveID string) bool {
|
||||
a.scanQueueMu.Lock()
|
||||
queued := a.scanQueued[driveID]
|
||||
delete(a.scanQueued, driveID)
|
||||
delete(a.scanProgress, driveID)
|
||||
a.scanQueueMu.Unlock()
|
||||
return queued
|
||||
}
|
||||
@@ -1019,6 +1078,7 @@ func (a *App) clearAllQueuedDriveTasks() []string {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
a.scanQueued = nil
|
||||
a.scanProgress = nil
|
||||
a.scanQueueMu.Unlock()
|
||||
return ids
|
||||
}
|
||||
@@ -1042,6 +1102,102 @@ func (a *App) clearAllFingerprintQueueing() []string {
|
||||
return ids
|
||||
}
|
||||
|
||||
func (a *App) beginDriveScanOrCrawl(driveID string) bool {
|
||||
driveID = strings.TrimSpace(driveID)
|
||||
if driveID == "" {
|
||||
return false
|
||||
}
|
||||
a.scanQueueMu.Lock()
|
||||
defer a.scanQueueMu.Unlock()
|
||||
if a.scanQueued == nil {
|
||||
a.scanQueued = make(map[string]bool)
|
||||
}
|
||||
if a.scanQueued[driveID] {
|
||||
return false
|
||||
}
|
||||
a.scanQueued[driveID] = true
|
||||
if a.scanProgress == nil {
|
||||
a.scanProgress = make(map[string]driveScanProgress)
|
||||
}
|
||||
a.scanProgress[driveID] = driveScanProgress{}
|
||||
return true
|
||||
}
|
||||
|
||||
func (a *App) endDriveScanOrCrawl(driveID string) {
|
||||
a.scanQueueMu.Lock()
|
||||
delete(a.scanQueued, driveID)
|
||||
delete(a.scanProgress, driveID)
|
||||
a.scanQueueMu.Unlock()
|
||||
}
|
||||
|
||||
func (a *App) updateDriveScanProgress(driveID string, scanned, added int) {
|
||||
driveID = strings.TrimSpace(driveID)
|
||||
if driveID == "" {
|
||||
return
|
||||
}
|
||||
a.scanQueueMu.Lock()
|
||||
if a.scanQueued[driveID] {
|
||||
if a.scanProgress == nil {
|
||||
a.scanProgress = make(map[string]driveScanProgress)
|
||||
}
|
||||
a.scanProgress[driveID] = driveScanProgress{Scanned: scanned, Added: added}
|
||||
}
|
||||
a.scanQueueMu.Unlock()
|
||||
}
|
||||
|
||||
func (a *App) driveHasActiveWork(driveID string) bool {
|
||||
driveID = strings.TrimSpace(driveID)
|
||||
if driveID == "" {
|
||||
return true
|
||||
}
|
||||
|
||||
a.scanQueueMu.Lock()
|
||||
scanning := a.scanQueued[driveID]
|
||||
a.scanQueueMu.Unlock()
|
||||
if scanning {
|
||||
return true
|
||||
}
|
||||
|
||||
a.taskCancelMu.Lock()
|
||||
taskContexts := len(a.driveTaskCancels[driveID])
|
||||
a.taskCancelMu.Unlock()
|
||||
if taskContexts > 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
a.fingerprintQueueMu.Lock()
|
||||
fingerprintQueueing := a.fingerprintQueueing[driveID]
|
||||
a.fingerprintQueueMu.Unlock()
|
||||
if fingerprintQueueing {
|
||||
return true
|
||||
}
|
||||
|
||||
a.mu.Lock()
|
||||
previewWorker := a.workers[driveID]
|
||||
thumbWorker := a.thumbWorkers[driveID]
|
||||
fingerprintWorker := a.fingerprintWorkers[driveID]
|
||||
a.mu.Unlock()
|
||||
|
||||
if previewTaskBusy(thumbWorker.Status()) {
|
||||
return true
|
||||
}
|
||||
if previewTaskBusy(previewWorker.Status()) {
|
||||
return true
|
||||
}
|
||||
if fingerprintTaskBusy(fingerprintWorker.Status()) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func previewTaskBusy(status preview.TaskStatus) bool {
|
||||
return status.State != "" && status.State != "idle"
|
||||
}
|
||||
|
||||
func fingerprintTaskBusy(status fingerprint.TaskStatus) bool {
|
||||
return status.State != "" && status.State != "idle"
|
||||
}
|
||||
|
||||
func (a *App) resetDriveGenerationWorkers(ctx context.Context, driveID string) bool {
|
||||
var drv drives.Drive
|
||||
var attached bool
|
||||
@@ -1355,52 +1511,41 @@ func (a *App) listDriveDirChildren(ctx context.Context, driveID, parentID string
|
||||
|
||||
// scheduleScan 异步触发某个 drive 的扫盘。
|
||||
//
|
||||
// 调用立即返回;扫盘任务在后台 goroutine 里排队执行 —— 系统中所有扫盘共享
|
||||
// 一把 scanGlobalMu,按提交顺序串行跑。
|
||||
//
|
||||
// 去重:如果该 drive 已经在排队或正在跑,重复请求会被丢弃并记日志。这样用户
|
||||
// 反复点同一个 drive 的"重扫"按钮,也只会有一次实际工作。
|
||||
//
|
||||
// 用于 admin UI「重扫」、「立即抓取」这类异步触发;nightly Phase 1 应继续直接
|
||||
// 调 runScan(同步、按 for 循环顺序),不需要走 scheduleScan。
|
||||
func (a *App) scheduleScan(ctx context.Context, driveID string) {
|
||||
taskCtx, done := a.registerDriveTaskContext(ctx, driveID)
|
||||
a.scanQueueMu.Lock()
|
||||
if a.scanQueued == nil {
|
||||
a.scanQueued = make(map[string]bool)
|
||||
// 调用立即返回。不同 drive 的扫盘可以并行;同一个 drive 如果已有扫盘、封面、
|
||||
// 预览视频或指纹任务在跑,本次请求会被拒绝。
|
||||
func (a *App) scheduleScan(ctx context.Context, driveID string) bool {
|
||||
if a.driveHasActiveWork(driveID) {
|
||||
log.Printf("[scan] drive=%s has active work, skip duplicate request", driveID)
|
||||
return false
|
||||
}
|
||||
if a.scanQueued[driveID] {
|
||||
a.scanQueueMu.Unlock()
|
||||
done()
|
||||
if !a.beginDriveScanOrCrawl(driveID) {
|
||||
log.Printf("[scan] drive=%s already queued or running, skip duplicate request", driveID)
|
||||
return
|
||||
return false
|
||||
}
|
||||
a.scanQueued[driveID] = true
|
||||
a.scanQueueMu.Unlock()
|
||||
taskCtx, done := a.registerDriveTaskContext(ctx, driveID)
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
a.scanQueueMu.Lock()
|
||||
delete(a.scanQueued, driveID)
|
||||
a.scanQueueMu.Unlock()
|
||||
a.endDriveScanOrCrawl(driveID)
|
||||
done()
|
||||
}()
|
||||
a.runScanWithTaskContext(taskCtx, driveID)
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
func (a *App) runScan(ctx context.Context, driveID string) {
|
||||
if !a.beginDriveScanOrCrawl(driveID) {
|
||||
log.Printf("[scan] drive=%s already queued or running, skip direct scan", driveID)
|
||||
return
|
||||
}
|
||||
defer a.endDriveScanOrCrawl(driveID)
|
||||
taskCtx, done := a.registerDriveTaskContext(ctx, driveID)
|
||||
defer done()
|
||||
a.runScanWithTaskContext(taskCtx, driveID)
|
||||
}
|
||||
|
||||
func (a *App) runScanWithTaskContext(ctx context.Context, driveID string) {
|
||||
// 全局串行:同一时刻只有一个扫盘任务在跑(admin 重扫 + nightly Phase 1 共用)。
|
||||
// 等待这把锁的 goroutine 在排队,按到达顺序逐个执行。
|
||||
a.scanGlobalMu.Lock()
|
||||
defer a.scanGlobalMu.Unlock()
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
log.Printf("[scan] drive=%s canceled before start: %v", driveID, err)
|
||||
return
|
||||
@@ -1438,6 +1583,9 @@ func (a *App) runScanWithTaskContext(ctx context.Context, driveID string) {
|
||||
return
|
||||
}
|
||||
sc := scanner.New(a.cat, drv, a.cfg.Scanner.VideoExtensions, d.SkipDirIDs, onNew)
|
||||
sc.OnProgress = func(stats scanner.Stats) {
|
||||
a.updateDriveScanProgress(driveID, stats.Scanned, stats.Added)
|
||||
}
|
||||
|
||||
startID := d.RootID
|
||||
|
||||
@@ -2310,30 +2458,27 @@ func shouldScanDrive(d drives.Drive) bool {
|
||||
|
||||
// ---------- spider91 crawl ----------
|
||||
|
||||
func (a *App) scheduleSpider91Crawl(ctx context.Context, driveID string) {
|
||||
taskCtx, done := a.registerDriveTaskContext(ctx, driveID)
|
||||
a.scanQueueMu.Lock()
|
||||
if a.scanQueued == nil {
|
||||
a.scanQueued = make(map[string]bool)
|
||||
func (a *App) scheduleSpider91Crawl(ctx context.Context, driveID string) bool {
|
||||
if a.driveHasActiveWork(driveID) {
|
||||
log.Printf("[spider91] drive=%s has active work, skip duplicate crawl request", driveID)
|
||||
return false
|
||||
}
|
||||
if a.scanQueued[driveID] {
|
||||
a.scanQueueMu.Unlock()
|
||||
done()
|
||||
if !a.beginDriveScanOrCrawl(driveID) {
|
||||
log.Printf("[spider91] drive=%s already queued or running, skip duplicate crawl request", driveID)
|
||||
return
|
||||
return false
|
||||
}
|
||||
a.scanQueued[driveID] = true
|
||||
a.scanQueueMu.Unlock()
|
||||
taskCtx, done := a.registerDriveTaskContext(ctx, driveID)
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
a.scanQueueMu.Lock()
|
||||
delete(a.scanQueued, driveID)
|
||||
a.scanQueueMu.Unlock()
|
||||
a.endDriveScanOrCrawl(driveID)
|
||||
done()
|
||||
}()
|
||||
a.runSpider91CrawlWithTaskContext(taskCtx, driveID)
|
||||
if a.runSpider91CrawlWithTaskContext(taskCtx, driveID) {
|
||||
a.runSpider91MigrationAfterManualCrawl(taskCtx, driveID)
|
||||
}
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
// runSpider91Crawl 运行一次完整爬取流程并把 last_crawl_at 写回 drive.credentials。
|
||||
@@ -2342,15 +2487,20 @@ func (a *App) scheduleSpider91Crawl(ctx context.Context, driveID string) {
|
||||
// 流水线重跑时仍会重试。该方法是阻塞的,被 nightly Phase 2 串行调用,以及被
|
||||
// admin "立即抓取" 单 drive 异步调用。
|
||||
func (a *App) runSpider91Crawl(ctx context.Context, driveID string) {
|
||||
if !a.beginDriveScanOrCrawl(driveID) {
|
||||
log.Printf("[spider91] drive=%s already queued or running, skip direct crawl", driveID)
|
||||
return
|
||||
}
|
||||
defer a.endDriveScanOrCrawl(driveID)
|
||||
taskCtx, done := a.registerDriveTaskContext(ctx, driveID)
|
||||
defer done()
|
||||
a.runSpider91CrawlWithTaskContext(taskCtx, driveID)
|
||||
}
|
||||
|
||||
func (a *App) runSpider91CrawlWithTaskContext(ctx context.Context, driveID string) {
|
||||
func (a *App) runSpider91CrawlWithTaskContext(ctx context.Context, driveID string) bool {
|
||||
if err := ctx.Err(); err != nil {
|
||||
log.Printf("[spider91] drive=%s crawl canceled before start: %v", driveID, err)
|
||||
return
|
||||
return false
|
||||
}
|
||||
a.mu.Lock()
|
||||
c := a.spider91Crawlers[driveID]
|
||||
@@ -2358,21 +2508,21 @@ func (a *App) runSpider91CrawlWithTaskContext(ctx context.Context, driveID strin
|
||||
if c == nil {
|
||||
if err := a.ensureDriveAttached(ctx, driveID); err != nil {
|
||||
log.Printf("[spider91] drive=%s attach failed: %v", driveID, err)
|
||||
return
|
||||
return false
|
||||
}
|
||||
a.mu.Lock()
|
||||
c = a.spider91Crawlers[driveID]
|
||||
a.mu.Unlock()
|
||||
if c == nil {
|
||||
log.Printf("[spider91] drive=%s crawler not attached", driveID)
|
||||
return
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
d, err := a.cat.GetDrive(ctx, driveID)
|
||||
if err != nil || d == nil {
|
||||
log.Printf("[spider91] drive=%s lookup failed: %v", driveID, err)
|
||||
return
|
||||
return false
|
||||
}
|
||||
targetNew := spider91IntCred(d, "target_new", spider91.DefaultTargetNew)
|
||||
if targetNew <= 0 {
|
||||
@@ -2406,7 +2556,7 @@ func (a *App) runSpider91CrawlWithTaskContext(ctx context.Context, driveID strin
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
log.Printf("[spider91] drive=%s crawl canceled after run: %v", driveID, err)
|
||||
return
|
||||
return false
|
||||
}
|
||||
|
||||
// 爬取全部完成后,统一把所有还 pending 的预览视频入队。
|
||||
@@ -2421,6 +2571,35 @@ func (a *App) runSpider91CrawlWithTaskContext(ctx context.Context, driveID strin
|
||||
a.mu.Unlock()
|
||||
a.scheduleFingerprintBackfill(ctx, driveID, fingerprintWorker)
|
||||
a.enqueueDriveGeneration(ctx, driveID, worker, thumbWorker)
|
||||
return runErr == nil
|
||||
}
|
||||
|
||||
func (a *App) runSpider91MigrationAfterManualCrawl(ctx context.Context, driveID string) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
log.Printf("[spider91] drive=%s skip post-crawl migration: %v", driveID, err)
|
||||
return
|
||||
}
|
||||
targetDriveID := a.Spider91UploadDriveID()
|
||||
if targetDriveID == "" {
|
||||
return
|
||||
}
|
||||
if a.spider91Migrator == nil {
|
||||
log.Printf("[spider91] drive=%s skip post-crawl migration: migrator not configured", driveID)
|
||||
return
|
||||
}
|
||||
log.Printf("[spider91] drive=%s waiting for generation queues before post-crawl migration target=%s", driveID, targetDriveID)
|
||||
if err := a.waitAllPreviewQueuesIdle(ctx); err != nil {
|
||||
log.Printf("[spider91] drive=%s post-crawl migration wait canceled: %v", driveID, err)
|
||||
return
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
log.Printf("[spider91] drive=%s skip post-crawl migration after wait: %v", driveID, err)
|
||||
return
|
||||
}
|
||||
log.Printf("[spider91] drive=%s running post-crawl migration target=%s", driveID, targetDriveID)
|
||||
if err := a.spider91Migrator.RunOnce(ctx); err != nil {
|
||||
log.Printf("[spider91] drive=%s post-crawl migration: %v", driveID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// spider91IntCred 解析 credentials 中的整数字段,缺省时返回 def。
|
||||
|
||||
@@ -260,6 +260,7 @@ func TestStopDriveTasksCancelsQueuedTasksAndReplacesWorkers(t *testing.T) {
|
||||
"drive-id": func() { close(oldCanceled) },
|
||||
},
|
||||
scanQueued: map[string]bool{"drive-id": true},
|
||||
scanProgress: map[string]driveScanProgress{"drive-id": {Scanned: 8, Added: 2}},
|
||||
fingerprintQueueing: map[string]bool{"drive-id": true},
|
||||
}
|
||||
taskCtx, done := app.registerDriveTaskContext(ctx, "drive-id")
|
||||
@@ -279,6 +280,9 @@ func TestStopDriveTasksCancelsQueuedTasksAndReplacesWorkers(t *testing.T) {
|
||||
if app.scanQueued["drive-id"] {
|
||||
t.Fatal("scan queue marker was not cleared")
|
||||
}
|
||||
if _, ok := app.scanProgress["drive-id"]; ok {
|
||||
t.Fatal("scan progress marker was not cleared")
|
||||
}
|
||||
if app.fingerprintQueueing["drive-id"] {
|
||||
t.Fatal("fingerprint queue marker was not cleared")
|
||||
}
|
||||
@@ -304,6 +308,117 @@ func TestStopDriveTasksCancelsQueuedTasksAndReplacesWorkers(t *testing.T) {
|
||||
newCancel()
|
||||
}
|
||||
|
||||
func TestScheduleScanRejectsDriveWithActiveGenerationWork(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
thumbWorker := preview.NewThumbWorker(&serverFakeTeaserGenerator{}, nil, &serverFakeDrive{})
|
||||
if !thumbWorker.Enqueue(&catalog.Video{ID: "busy-video", DriveID: "drive-id", Title: "Busy Video"}) {
|
||||
t.Fatal("failed to enqueue busy thumbnail task")
|
||||
}
|
||||
app := &App{
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{"drive-id": thumbWorker},
|
||||
}
|
||||
|
||||
if app.scheduleScan(ctx, "drive-id") {
|
||||
t.Fatal("scheduleScan accepted a drive with active generation work")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleScanRunsDifferentDrivesConcurrently(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
seedDriveWithTeaser(t, cat, "drive-a", true)
|
||||
seedDriveWithTeaser(t, cat, "drive-b", true)
|
||||
|
||||
started := make(chan string, 2)
|
||||
release := make(chan struct{})
|
||||
registry := proxy.NewRegistry()
|
||||
registry.Set("drive-a", &serverBlockingListDrive{id: "drive-a", started: started, release: release})
|
||||
registry.Set("drive-b", &serverBlockingListDrive{id: "drive-b", started: started, release: release})
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{
|
||||
Scanner: config.Scanner{VideoExtensions: []string{".mp4"}},
|
||||
},
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
}
|
||||
|
||||
if !app.scheduleScan(ctx, "drive-a") {
|
||||
t.Fatal("scheduleScan drive-a was rejected")
|
||||
}
|
||||
if !app.scheduleScan(ctx, "drive-b") {
|
||||
t.Fatal("scheduleScan drive-b was rejected")
|
||||
}
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
deadline := time.After(time.Second)
|
||||
for len(seen) < 2 {
|
||||
select {
|
||||
case id := <-started:
|
||||
seen[id] = struct{}{}
|
||||
case <-deadline:
|
||||
close(release)
|
||||
t.Fatalf("started drives = %#v, want both drives before releasing List", seen)
|
||||
}
|
||||
}
|
||||
close(release)
|
||||
}
|
||||
|
||||
func TestDriveGenerationStatusIncludesScanState(t *testing.T) {
|
||||
app := &App{
|
||||
scanQueued: map[string]bool{"drive-id": true},
|
||||
scanProgress: map[string]driveScanProgress{"drive-id": {Scanned: 12, Added: 3}},
|
||||
}
|
||||
|
||||
status := app.driveGenerationStatuses()["drive-id"].Scan
|
||||
if status.State != "scanning" {
|
||||
t.Fatalf("scan status = %#v, want scanning", status)
|
||||
}
|
||||
if status.ScannedCount != 12 || status.AddedCount != 3 {
|
||||
t.Fatalf("scan counts = scanned %d added %d, want 12 and 3", status.ScannedCount, status.AddedCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSpider91MigrationAfterManualCrawlRequiresConfiguredUploadTarget(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
registry := proxy.NewRegistry()
|
||||
migrator := &serverFakeSpider91MigrationRunner{}
|
||||
app := &App{
|
||||
registry: registry,
|
||||
spider91Migrator: migrator,
|
||||
workers: map[string]*preview.Worker{},
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{},
|
||||
fingerprintWorkers: map[string]*fingerprint.Worker{},
|
||||
}
|
||||
|
||||
app.runSpider91MigrationAfterManualCrawl(ctx, "91spider")
|
||||
if migrator.called != 0 {
|
||||
t.Fatalf("migration called without upload target")
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "pikpak"
|
||||
app.runSpider91MigrationAfterManualCrawl(ctx, "91spider")
|
||||
if migrator.called != 0 {
|
||||
t.Fatalf("migration called when upload target is not attached")
|
||||
}
|
||||
|
||||
registry.Set("pikpak", &serverFakeKindDrive{id: "pikpak", kind: "pikpak"})
|
||||
app.runSpider91MigrationAfterManualCrawl(ctx, "91spider")
|
||||
if migrator.called != 1 {
|
||||
t.Fatalf("migration calls = %d, want 1", migrator.called)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriveGenerationStatusUsesWorkerQueueNotPendingCatalogRows(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -1496,6 +1611,63 @@ func (d *serverFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
}
|
||||
func (d *serverFakeDrive) RootID() string { return "root" }
|
||||
|
||||
type serverFakeKindDrive struct {
|
||||
serverFakeDrive
|
||||
id string
|
||||
kind string
|
||||
}
|
||||
|
||||
func (d *serverFakeKindDrive) Kind() string { return d.kind }
|
||||
func (d *serverFakeKindDrive) ID() string { return d.id }
|
||||
|
||||
type serverFakeSpider91MigrationRunner struct {
|
||||
called int
|
||||
}
|
||||
|
||||
func (r *serverFakeSpider91MigrationRunner) RunOnce(context.Context) error {
|
||||
r.called++
|
||||
return nil
|
||||
}
|
||||
|
||||
type serverBlockingListDrive struct {
|
||||
id string
|
||||
started chan string
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (d *serverBlockingListDrive) Kind() string { return "fake" }
|
||||
func (d *serverBlockingListDrive) ID() string { return d.id }
|
||||
func (d *serverBlockingListDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *serverBlockingListDrive) List(ctx context.Context, _ string) ([]drives.Entry, error) {
|
||||
if d.started != nil {
|
||||
select {
|
||||
case d.started <- d.id:
|
||||
default:
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-d.release:
|
||||
return nil, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
func (d *serverBlockingListDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *serverBlockingListDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
|
||||
return &drives.StreamLink{URL: "https://video.example/clip.mp4"}, nil
|
||||
}
|
||||
func (d *serverBlockingListDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *serverBlockingListDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *serverBlockingListDrive) RootID() string { return "root" }
|
||||
|
||||
type serverFingerprintFakeDrive struct {
|
||||
serverFakeDrive
|
||||
path string
|
||||
|
||||
@@ -74,6 +74,11 @@ preview:
|
||||
# root_id: "root"
|
||||
# params:
|
||||
# refresh_token: "..."
|
||||
# # 默认 use_online_api=true,会使用 OpenList 在线续期 API。
|
||||
# # 如需使用自己创建的 Google OAuth 客户端,取消下面三行注释:
|
||||
# # use_online_api: "false"
|
||||
# # client_id: "..."
|
||||
# # client_secret: "..."
|
||||
# 本地存储示例:
|
||||
# - id: "local-media"
|
||||
# kind: "localstorage"
|
||||
|
||||
@@ -45,7 +45,7 @@ type AdminServer struct {
|
||||
OnDriveSaved func(driveID string) error
|
||||
OnDriveDeleteCleanup func(ctx context.Context, driveID string) (int, error)
|
||||
OnDriveRemoved func(driveID string)
|
||||
OnScanRequested func(driveID string)
|
||||
OnScanRequested func(driveID string) bool
|
||||
OnStopDriveTasks func(driveID string) bool
|
||||
OnStopAllTasks func() int
|
||||
OnRegenPreview func(videoID string)
|
||||
@@ -81,6 +81,11 @@ type AdminServer struct {
|
||||
P123HTTPClient *http.Client
|
||||
}
|
||||
|
||||
const (
|
||||
driveTaskBusyMessage = "当前存储有正在进行的任务,请稍后重试"
|
||||
fullScanBusyMessage = "当前有全量扫描任务正在进行,请稍后重试"
|
||||
)
|
||||
|
||||
// DriveDirEntry 是 dirtree 接口的一条返回项:网盘上的一个目录节点。
|
||||
type DriveDirEntry struct {
|
||||
ID string `json:"id"`
|
||||
@@ -92,9 +97,12 @@ type GenerationStatus struct {
|
||||
CurrentTitle string `json:"currentTitle,omitempty"`
|
||||
QueueLength int `json:"queueLength"`
|
||||
CooldownUntil string `json:"cooldownUntil,omitempty"`
|
||||
ScannedCount int `json:"scannedCount"`
|
||||
AddedCount int `json:"addedCount"`
|
||||
}
|
||||
|
||||
type DriveGenerationStatuses struct {
|
||||
Scan GenerationStatus `json:"scan"`
|
||||
Thumbnail GenerationStatus `json:"thumbnail"`
|
||||
Preview GenerationStatus `json:"preview"`
|
||||
Fingerprint GenerationStatus `json:"fingerprint"`
|
||||
@@ -417,6 +425,8 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
// 其它 kind 留 0;前端用它显示"上次抓取: N 小时前"。
|
||||
Spider91Proxy string `json:"spider91Proxy,omitempty"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
GoogleDriveUseOnlineAPI *bool `json:"googleDriveUseOnlineAPI,omitempty"`
|
||||
ScanGenerationStatus GenerationStatus `json:"scanGenerationStatus"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
|
||||
@@ -437,6 +447,9 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
thumbCounts := thumbnailCounts[d.ID]
|
||||
fingerprintCount := fingerprintCounts[d.ID]
|
||||
generation := generationStatuses[d.ID]
|
||||
if generation.Scan.State == "" {
|
||||
generation.Scan.State = "idle"
|
||||
}
|
||||
if generation.Thumbnail.State == "" {
|
||||
generation.Thumbnail.State = "idle"
|
||||
}
|
||||
@@ -476,6 +489,8 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
|
||||
Spider91Proxy: spider91ProxyForDrive(d),
|
||||
LastCrawlAt: lastCrawlAt,
|
||||
GoogleDriveUseOnlineAPI: googleDriveUseOnlineAPIForDrive(d),
|
||||
ScanGenerationStatus: generation.Scan,
|
||||
ThumbnailGenerationStatus: generation.Thumbnail,
|
||||
PreviewGenerationStatus: generation.Preview,
|
||||
FingerprintGenerationStatus: generation.Fingerprint,
|
||||
@@ -534,6 +549,8 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
body.Credentials = credentials
|
||||
} else if body.Kind == "googledrive" {
|
||||
body.Credentials = mergeGoogleDriveCredentials(existing, body.Credentials)
|
||||
} else if len(body.Credentials) == 0 && existing != nil && len(existing.Credentials) > 0 {
|
||||
body.Credentials = existing.Credentials
|
||||
}
|
||||
@@ -590,6 +607,47 @@ func spider91ProxyForDrive(d *catalog.Drive) string {
|
||||
return strings.TrimSpace(d.Credentials["proxy"])
|
||||
}
|
||||
|
||||
func googleDriveUseOnlineAPIForDrive(d *catalog.Drive) *bool {
|
||||
if d == nil || d.Kind != "googledrive" {
|
||||
return nil
|
||||
}
|
||||
result := true
|
||||
if d.Credentials == nil {
|
||||
return &result
|
||||
}
|
||||
raw := strings.TrimSpace(d.Credentials["use_online_api"])
|
||||
if raw == "" {
|
||||
return &result
|
||||
}
|
||||
v, err := strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return &result
|
||||
}
|
||||
result = v
|
||||
return &result
|
||||
}
|
||||
|
||||
func mergeGoogleDriveCredentials(existing *catalog.Drive, incoming map[string]string) map[string]string {
|
||||
merged := map[string]string{}
|
||||
if existing != nil {
|
||||
for k, v := range existing.Credentials {
|
||||
merged[k] = v
|
||||
}
|
||||
}
|
||||
for k, v := range incoming {
|
||||
key := strings.TrimSpace(k)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
value := strings.TrimSpace(v)
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
merged[key] = value
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func mergeSpider91Credentials(existing *catalog.Drive, incoming map[string]string) (map[string]string, error) {
|
||||
merged := map[string]string{}
|
||||
if existing != nil {
|
||||
@@ -675,10 +733,26 @@ type deleteDriveReq struct {
|
||||
|
||||
func (a *AdminServer) handleRescan(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
if a.OnScanRequested != nil {
|
||||
a.OnScanRequested(id)
|
||||
status := a.nightlyJobStatus()
|
||||
if status.Running || status.Queued {
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"ok": true,
|
||||
"accepted": false,
|
||||
"message": fullScanBusyMessage,
|
||||
"status": status,
|
||||
})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
|
||||
accepted := true
|
||||
if a.OnScanRequested != nil {
|
||||
accepted = a.OnScanRequested(id)
|
||||
}
|
||||
resp := map[string]any{"ok": true, "accepted": accepted}
|
||||
if !accepted {
|
||||
resp["message"] = driveTaskBusyMessage
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, resp)
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleStopDriveTasks(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -734,11 +808,15 @@ func (a *AdminServer) handleRunNightlyJob(w http.ResponseWriter, r *http.Request
|
||||
if a.OnRunNightlyJob != nil {
|
||||
accepted = a.OnRunNightlyJob()
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
resp := map[string]any{
|
||||
"ok": true,
|
||||
"accepted": accepted,
|
||||
"status": a.nightlyJobStatus(),
|
||||
})
|
||||
}
|
||||
if !accepted {
|
||||
resp["message"] = fullScanBusyMessage
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, resp)
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleNightlyJobStatus(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -278,6 +278,108 @@ func TestHandleRunNightlyJobReturnsAcceptedStatus(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRunNightlyJobReturnsBusyMessageWhenRejected(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/jobs/nightly/run", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
OnRunNightlyJob: func() bool {
|
||||
return false
|
||||
},
|
||||
GetNightlyJobStatus: func() NightlyJobStatus {
|
||||
return NightlyJobStatus{State: "running", Running: true}
|
||||
},
|
||||
}).handleRunNightlyJob(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, want 202; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
OK bool `json:"ok"`
|
||||
Accepted bool `json:"accepted"`
|
||||
Message string `json:"message"`
|
||||
Status NightlyJobStatus `json:"status"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if !got.OK || got.Accepted || got.Message != fullScanBusyMessage || !got.Status.Running {
|
||||
t.Fatalf("response = %#v, want rejected busy message", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRescanRejectsWhenNightlyBusy(t *testing.T) {
|
||||
called := false
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives/PikPak/rescan", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "PikPak")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
OnScanRequested: func(driveID string) bool {
|
||||
called = true
|
||||
return true
|
||||
},
|
||||
GetNightlyJobStatus: func() NightlyJobStatus {
|
||||
return NightlyJobStatus{State: "running", Running: true}
|
||||
},
|
||||
}).handleRescan(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, want 202; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if called {
|
||||
t.Fatal("OnScanRequested was called while nightly job was busy")
|
||||
}
|
||||
var got struct {
|
||||
OK bool `json:"ok"`
|
||||
Accepted bool `json:"accepted"`
|
||||
Message string `json:"message"`
|
||||
Status NightlyJobStatus `json:"status"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if !got.OK || got.Accepted || got.Message != fullScanBusyMessage || !got.Status.Running {
|
||||
t.Fatalf("response = %#v, want rejected full scan busy message", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRescanReturnsAcceptedFlagAndBusyMessage(t *testing.T) {
|
||||
calledWith := ""
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives/PikPak/rescan", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "PikPak")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
OnScanRequested: func(driveID string) bool {
|
||||
calledWith = driveID
|
||||
return false
|
||||
},
|
||||
}).handleRescan(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, want 202; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
OK bool `json:"ok"`
|
||||
Accepted bool `json:"accepted"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if calledWith != "PikPak" {
|
||||
t.Fatalf("hook called with %q, want PikPak", calledWith)
|
||||
}
|
||||
if !got.OK || got.Accepted || got.Message != driveTaskBusyMessage {
|
||||
t.Fatalf("response = %#v, want rejected busy message", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleNightlyJobStatusDefaultsToIdle(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/jobs/nightly/status", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
@@ -509,6 +611,67 @@ func TestHandleUpsertDriveReplacesExistingCredentialsWhenProvided(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertGoogleDriveMergesOAuthCredentials(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "google-main",
|
||||
Kind: "googledrive",
|
||||
Name: "Google Drive",
|
||||
RootID: "root",
|
||||
Credentials: map[string]string{
|
||||
"refresh_token": "existing-refresh",
|
||||
"access_token": "existing-access",
|
||||
"use_online_api": "true",
|
||||
"api_url_address": "https://api.oplist.org/googleui/renewapi",
|
||||
},
|
||||
Status: "ok",
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", bytes.NewBufferString(`{
|
||||
"id": "google-main",
|
||||
"kind": "googledrive",
|
||||
"name": "Google Drive",
|
||||
"rootId": "root",
|
||||
"credentials": {
|
||||
"use_online_api": "false",
|
||||
"client_id": "google-client-id",
|
||||
"client_secret": "google-client-secret"
|
||||
}
|
||||
}`))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, "google-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.Credentials["refresh_token"] != "existing-refresh" || got.Credentials["access_token"] != "existing-access" {
|
||||
t.Fatalf("tokens were not preserved: %#v", got.Credentials)
|
||||
}
|
||||
if got.Credentials["use_online_api"] != "false" {
|
||||
t.Fatalf("use_online_api = %q, want false", got.Credentials["use_online_api"])
|
||||
}
|
||||
if got.Credentials["client_id"] != "google-client-id" || got.Credentials["client_secret"] != "google-client-secret" {
|
||||
t.Fatalf("oauth client credentials = %#v, want saved", got.Credentials)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertSpider91ProxyPreservesRuntimeCredentials(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -803,6 +966,74 @@ func TestHandleListDrivesIncludesSpider91Proxy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListDrivesIncludesGoogleDriveOnlineAPIMode(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, d := range []*catalog.Drive{
|
||||
{
|
||||
ID: "google-legacy",
|
||||
Kind: "googledrive",
|
||||
Name: "Google Legacy",
|
||||
RootID: "root",
|
||||
Credentials: map[string]string{
|
||||
"refresh_token": "legacy-refresh",
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
{
|
||||
ID: "google-oauth",
|
||||
Kind: "googledrive",
|
||||
Name: "Google OAuth",
|
||||
RootID: "root",
|
||||
Credentials: map[string]string{
|
||||
"refresh_token": "oauth-refresh",
|
||||
"use_online_api": "false",
|
||||
"client_id": "client-id",
|
||||
"client_secret": "client-secret",
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
t.Fatalf("seed drive %s: %v", d.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/drives", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleListDrives(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
var got []struct {
|
||||
ID string `json:"id"`
|
||||
GoogleDriveUseOnlineAPI bool `json:"googleDriveUseOnlineAPI"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
byID := map[string]bool{}
|
||||
for _, d := range got {
|
||||
byID[d.ID] = d.GoogleDriveUseOnlineAPI
|
||||
}
|
||||
if !byID["google-legacy"] {
|
||||
t.Fatalf("legacy google drive use_online_api = false, want true")
|
||||
}
|
||||
if byID["google-oauth"] {
|
||||
t.Fatalf("oauth google drive use_online_api = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -854,6 +1085,7 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
GetDriveGenerationStatuses: func() map[string]DriveGenerationStatuses {
|
||||
return map[string]DriveGenerationStatuses{
|
||||
"OneDrive": {
|
||||
Scan: GenerationStatus{State: "scanning", ScannedCount: 12, AddedCount: 3},
|
||||
Thumbnail: GenerationStatus{State: "cooling", QueueLength: 3, CooldownUntil: "2026-05-16T21:00:00+08:00"},
|
||||
Preview: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
|
||||
Fingerprint: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
|
||||
@@ -867,6 +1099,7 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
}
|
||||
var got []struct {
|
||||
ID string `json:"id"`
|
||||
ScanGenerationStatus GenerationStatus `json:"scanGenerationStatus"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
|
||||
@@ -895,6 +1128,7 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
FingerprintReady int
|
||||
FingerprintPending int
|
||||
FingerprintFailed int
|
||||
Scan GenerationStatus
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
Fingerprint GenerationStatus
|
||||
@@ -911,6 +1145,7 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
FingerprintReady int
|
||||
FingerprintPending int
|
||||
FingerprintFailed int
|
||||
Scan GenerationStatus
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
Fingerprint GenerationStatus
|
||||
@@ -925,6 +1160,7 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
FingerprintReady: d.FingerprintReadyCount,
|
||||
FingerprintPending: d.FingerprintPendingCount,
|
||||
FingerprintFailed: d.FingerprintFailedCount,
|
||||
Scan: d.ScanGenerationStatus,
|
||||
Thumbnail: d.ThumbnailGenerationStatus,
|
||||
Preview: d.PreviewGenerationStatus,
|
||||
Fingerprint: d.FingerprintGenerationStatus,
|
||||
@@ -942,6 +1178,12 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
if byID["OneDrive"].Thumbnail.State != "cooling" || byID["OneDrive"].Preview.State != "generating" {
|
||||
t.Fatalf("OneDrive generation statuses = %#v, want thumbnail cooling and preview generating", byID["OneDrive"])
|
||||
}
|
||||
if byID["OneDrive"].Scan.State != "scanning" {
|
||||
t.Fatalf("OneDrive scan status = %#v, want scanning", byID["OneDrive"].Scan)
|
||||
}
|
||||
if byID["OneDrive"].Scan.ScannedCount != 12 || byID["OneDrive"].Scan.AddedCount != 3 {
|
||||
t.Fatalf("OneDrive scan counts = %#v, want scanned=12 added=3", byID["OneDrive"].Scan)
|
||||
}
|
||||
if byID["OneDrive"].FingerprintReady != 1 || byID["OneDrive"].FingerprintPending != 1 || byID["OneDrive"].FingerprintFailed != 1 {
|
||||
t.Fatalf("OneDrive fingerprint counts = %#v, want ready=1 pending=1 failed=1", byID["OneDrive"])
|
||||
}
|
||||
@@ -957,7 +1199,7 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
if byID["PikPak"].FingerprintPending != 2 {
|
||||
t.Fatalf("PikPak fingerprint counts = %#v, want pending=2", byID["PikPak"])
|
||||
}
|
||||
if byID["PikPak"].Thumbnail.State != "idle" || byID["PikPak"].Preview.State != "idle" || byID["PikPak"].Fingerprint.State != "idle" {
|
||||
if byID["PikPak"].Scan.State != "idle" || byID["PikPak"].Thumbnail.State != "idle" || byID["PikPak"].Preview.State != "idle" || byID["PikPak"].Fingerprint.State != "idle" {
|
||||
t.Fatalf("PikPak generation statuses = %#v, want idle defaults", byID["PikPak"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
package googledrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -21,10 +28,13 @@ import (
|
||||
const (
|
||||
Kind = "googledrive"
|
||||
defaultAPIBaseURL = "https://www.googleapis.com/drive/v3"
|
||||
defaultUploadAPIURL = "https://www.googleapis.com/upload/drive/v3"
|
||||
defaultOAuthURL = "https://www.googleapis.com/oauth2/v4/token"
|
||||
defaultRenewAPIURL = "https://api.oplist.org/googleui/renewapi"
|
||||
defaultListInterval = 1 * time.Second
|
||||
defaultListCooldown = 5 * time.Minute
|
||||
defaultLinkCooldown = 5 * time.Minute
|
||||
uploadChunkSize = int64(8 * 1024 * 1024)
|
||||
|
||||
filesListFields = "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken"
|
||||
fileInfoFields = "id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum"
|
||||
@@ -41,13 +51,19 @@ type Driver struct {
|
||||
renewAPIURL string
|
||||
oauthURL string
|
||||
apiBaseURL string
|
||||
uploadBaseURL string
|
||||
client *resty.Client
|
||||
httpClient *http.Client
|
||||
onTokenUpdate func(access, refresh string)
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
listInterval time.Duration
|
||||
listCooldown time.Duration
|
||||
|
||||
linkCooldownMu sync.Mutex
|
||||
linkCooldownUntil time.Time
|
||||
linkCooldownDuration time.Duration
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -61,6 +77,7 @@ type Config struct {
|
||||
RenewAPIURL string
|
||||
OAuthURL string
|
||||
APIBaseURL string
|
||||
UploadAPIURL string
|
||||
|
||||
OnTokenUpdate func(access, refresh string)
|
||||
}
|
||||
@@ -82,6 +99,10 @@ func New(c Config) *Driver {
|
||||
if apiBaseURL == "" {
|
||||
apiBaseURL = defaultAPIBaseURL
|
||||
}
|
||||
uploadBaseURL := strings.TrimRight(strings.TrimSpace(c.UploadAPIURL), "/")
|
||||
if uploadBaseURL == "" {
|
||||
uploadBaseURL = deriveUploadBaseURL(apiBaseURL)
|
||||
}
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
rootID: rootID,
|
||||
@@ -93,15 +114,34 @@ func New(c Config) *Driver {
|
||||
renewAPIURL: renewAPIURL,
|
||||
oauthURL: oauthURL,
|
||||
apiBaseURL: apiBaseURL,
|
||||
uploadBaseURL: uploadBaseURL,
|
||||
onTokenUpdate: c.OnTokenUpdate,
|
||||
client: resty.New().
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
listInterval: defaultListInterval,
|
||||
listCooldown: defaultListCooldown,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 0,
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
},
|
||||
listInterval: defaultListInterval,
|
||||
listCooldown: defaultListCooldown,
|
||||
linkCooldownDuration: defaultLinkCooldown,
|
||||
}
|
||||
}
|
||||
|
||||
func deriveUploadBaseURL(apiBaseURL string) string {
|
||||
apiBaseURL = strings.TrimRight(strings.TrimSpace(apiBaseURL), "/")
|
||||
if apiBaseURL == "" || apiBaseURL == defaultAPIBaseURL {
|
||||
return defaultUploadAPIURL
|
||||
}
|
||||
if strings.HasSuffix(apiBaseURL, "/drive/v3") {
|
||||
return strings.TrimSuffix(apiBaseURL, "/drive/v3") + "/upload/drive/v3"
|
||||
}
|
||||
return apiBaseURL
|
||||
}
|
||||
|
||||
func (d *Driver) Kind() string { return Kind }
|
||||
func (d *Driver) ID() string { return d.id }
|
||||
func (d *Driver) RootID() string { return d.rootID }
|
||||
@@ -209,8 +249,19 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
if fileID == "" {
|
||||
return nil, errors.New("googledrive stream: empty file id")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := d.linkCooldownError(time.Now()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := d.Stat(ctx, fileID); err != nil {
|
||||
return nil, fmt.Errorf("googledrive stream: %w", err)
|
||||
err = fmt.Errorf("googledrive stream: %w", err)
|
||||
if wait, ok := drives.RateLimitRetryAfter(err); ok {
|
||||
until := d.pauseLinkCooldown(wait)
|
||||
log.Printf("[googledrive] stream link cooling down drive=%s until=%s err=%v", d.id, until.Format(time.RFC3339), err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
u := d.fileURL(fileID) + "?alt=media&acknowledgeAbuse=true&supportsAllDrives=true"
|
||||
return &drives.StreamLink{
|
||||
@@ -222,12 +273,383 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
func (d *Driver) linkCooldownError(now time.Time) error {
|
||||
d.linkCooldownMu.Lock()
|
||||
defer d.linkCooldownMu.Unlock()
|
||||
if d.linkCooldownUntil.IsZero() {
|
||||
return nil
|
||||
}
|
||||
if !now.Before(d.linkCooldownUntil) {
|
||||
d.linkCooldownUntil = time.Time{}
|
||||
return nil
|
||||
}
|
||||
wait := d.linkCooldownUntil.Sub(now)
|
||||
if wait <= 0 {
|
||||
return nil
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: wait,
|
||||
Err: fmt.Errorf("googledrive stream link cooling down until %s", d.linkCooldownUntil.Format(time.RFC3339)),
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
func (d *Driver) pauseLinkCooldown(wait time.Duration) time.Time {
|
||||
if wait <= 0 {
|
||||
wait = d.linkCooldownDuration
|
||||
}
|
||||
if wait <= 0 {
|
||||
wait = defaultLinkCooldown
|
||||
}
|
||||
until := time.Now().Add(wait)
|
||||
d.linkCooldownMu.Lock()
|
||||
if until.After(d.linkCooldownUntil) {
|
||||
d.linkCooldownUntil = until
|
||||
} else {
|
||||
until = d.linkCooldownUntil
|
||||
}
|
||||
d.linkCooldownMu.Unlock()
|
||||
return until
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
res, err := d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return res.FileID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
parentID, name, err := d.normalizeUploadArgs(parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
sessionURL, err := d.createUploadSession(ctx, parentID, name, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if strings.TrimSpace(sessionURL) == "" {
|
||||
return UploadResult{}, errors.New("googledrive upload session: empty upload url")
|
||||
}
|
||||
|
||||
hasher := md5.New()
|
||||
var item driveFile
|
||||
var copied int64
|
||||
if size == 0 {
|
||||
completed, err := d.putUploadSessionChunkWithRetry(ctx, sessionURL, 0, 0, nil, hasher)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if completed != nil {
|
||||
item = *completed
|
||||
}
|
||||
} else {
|
||||
chunkSize := uploadChunkSize
|
||||
if chunkSize <= 0 {
|
||||
chunkSize = 8 * 1024 * 1024
|
||||
}
|
||||
if chunkSize > int64(math.MaxInt32) {
|
||||
chunkSize = int64(math.MaxInt32)
|
||||
}
|
||||
buf := make([]byte, int(chunkSize))
|
||||
for copied < size {
|
||||
partSize := minInt64(chunkSize, size-copied)
|
||||
chunk := buf[:int(partSize)]
|
||||
n, err := io.ReadFull(r, chunk)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return UploadResult{}, fmt.Errorf("googledrive upload: size mismatch: declared %d, copied %d", size, copied+int64(n))
|
||||
}
|
||||
return UploadResult{}, fmt.Errorf("googledrive upload: read body: %w", err)
|
||||
}
|
||||
chunk = chunk[:n]
|
||||
completed, err := d.putUploadSessionChunkWithRetry(ctx, sessionURL, copied, size, chunk, hasher)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if completed != nil {
|
||||
item = *completed
|
||||
}
|
||||
copied += int64(n)
|
||||
}
|
||||
}
|
||||
|
||||
hashHex := hex.EncodeToString(hasher.Sum(nil))
|
||||
if item.ID == "" {
|
||||
fileID, err := d.findUploadedFileID(ctx, parentID, name, hashHex)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
item.ID = fileID
|
||||
}
|
||||
return UploadResult{FileID: item.ID, Hash: hashHex, Size: copied}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) normalizeUploadArgs(parentID, name string, r io.Reader, size int64) (string, string, error) {
|
||||
if r == nil {
|
||||
return "", "", errors.New("googledrive upload: body is required")
|
||||
}
|
||||
if size < 0 {
|
||||
return "", "", fmt.Errorf("googledrive upload: invalid size %d", size)
|
||||
}
|
||||
parentID = strings.TrimSpace(parentID)
|
||||
if parentID == "" || parentID == "/" {
|
||||
parentID = d.rootID
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return "", "", errors.New("googledrive upload: empty file name")
|
||||
}
|
||||
return parentID, name, nil
|
||||
}
|
||||
|
||||
func (d *Driver) createUploadSession(ctx context.Context, parentID, name string, size int64) (string, error) {
|
||||
return d.createUploadSessionOnce(ctx, parentID, name, size, true)
|
||||
}
|
||||
|
||||
func (d *Driver) createUploadSessionOnce(ctx context.Context, parentID, name string, size int64, retry bool) (string, error) {
|
||||
var apiErr apiErrorResp
|
||||
res, err := d.client.R().
|
||||
SetContext(ctx).
|
||||
SetHeader("Authorization", "Bearer "+d.accessToken).
|
||||
SetHeader("X-Upload-Content-Type", mimeType(driveFile{Name: name})).
|
||||
SetHeader("X-Upload-Content-Length", strconv.FormatInt(size, 10)).
|
||||
SetQueryParams(map[string]string{
|
||||
"uploadType": "resumable",
|
||||
"supportsAllDrives": "true",
|
||||
"fields": fileInfoFields,
|
||||
}).
|
||||
SetBody(map[string]any{
|
||||
"name": name,
|
||||
"parents": []string{parentID},
|
||||
}).
|
||||
SetError(&apiErr).
|
||||
Post(d.uploadFilesURL())
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("googledrive upload session: %w", err)
|
||||
}
|
||||
if isGoogleRateLimit(res, apiErr.Error) {
|
||||
return "", googleRateLimitError(res, apiErr.Error.Message)
|
||||
}
|
||||
if apiErr.Error.Code != 0 {
|
||||
if apiErr.Error.Code == http.StatusUnauthorized && retry {
|
||||
if err := d.refresh(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return d.createUploadSessionOnce(ctx, parentID, name, size, false)
|
||||
}
|
||||
return "", googleAPIError(apiErr.Error)
|
||||
}
|
||||
if res.IsError() {
|
||||
return "", fmt.Errorf("googledrive upload session: status=%d body=%s", res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return strings.TrimSpace(res.Header().Get("Location")), nil
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunkWithRetry(ctx context.Context, uploadURL string, start, total int64, data []byte, hasher hash.Hash) (*driveFile, error) {
|
||||
var last error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if attempt > 0 {
|
||||
if err := sleepContext(ctx, time.Duration(attempt)*time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
item, retryable, err := d.putUploadSessionChunk(ctx, uploadURL, start, total, data)
|
||||
if err == nil {
|
||||
if hasher != nil && len(data) > 0 {
|
||||
_, _ = hasher.Write(data)
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
last = err
|
||||
if !retryable {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if last == nil {
|
||||
last = errors.New("googledrive upload session: retry attempts exhausted")
|
||||
}
|
||||
return nil, last
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunk(ctx context.Context, uploadURL string, start, total int64, data []byte) (*driveFile, bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
req.ContentLength = int64(len(data))
|
||||
req.Header.Set("Authorization", "Bearer "+d.accessToken)
|
||||
req.Header.Set("Content-Length", strconv.Itoa(len(data)))
|
||||
if total == 0 {
|
||||
req.Header.Set("Content-Range", "bytes */0")
|
||||
} else {
|
||||
end := start + int64(len(data)) - 1
|
||||
req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, total))
|
||||
}
|
||||
hc := d.httpClient
|
||||
if hc == nil {
|
||||
hc = http.DefaultClient
|
||||
}
|
||||
res, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, true, fmt.Errorf("googledrive upload session: put chunk: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK, http.StatusCreated:
|
||||
var item driveFile
|
||||
if err := json.NewDecoder(res.Body).Decode(&item); err != nil {
|
||||
return nil, false, fmt.Errorf("googledrive upload session: decode completed file: %w", err)
|
||||
}
|
||||
return &item, false, nil
|
||||
case http.StatusPermanentRedirect:
|
||||
return nil, false, nil
|
||||
case http.StatusUnauthorized:
|
||||
if err := d.refresh(ctx); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return nil, true, fmt.Errorf("googledrive upload session: unauthorized")
|
||||
default:
|
||||
body, _ := io.ReadAll(io.LimitReader(res.Body, 64*1024))
|
||||
var apiErr apiErrorResp
|
||||
_ = json.Unmarshal(body, &apiErr)
|
||||
if isGoogleUploadHTTPRateLimit(res.StatusCode, res.Header, body, apiErr.Error) {
|
||||
return nil, false, googleUploadRateLimitError(res.StatusCode, res.Header, body, apiErr.Error.Message)
|
||||
}
|
||||
retryable := res.StatusCode == http.StatusTooManyRequests || (res.StatusCode >= 500 && res.StatusCode <= 504)
|
||||
return nil, retryable, fmt.Errorf("googledrive upload session: status=%d body=%s", res.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
currentID := d.rootID
|
||||
for _, name := range splitPath(pathFromRoot) {
|
||||
childID, err := d.findChildDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if childID == "" {
|
||||
childID, err = d.makeDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
currentID = childID
|
||||
}
|
||||
return currentID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) findChildDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
entries, err := d.List(ctx, parentID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir && e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
var item driveFile
|
||||
err := d.request(ctx, d.filesURL(), http.MethodPost, func(req *resty.Request) {
|
||||
req.SetQueryParam("fields", fileInfoFields)
|
||||
req.SetBody(map[string]any{
|
||||
"name": name,
|
||||
"parents": []string{parentID},
|
||||
"mimeType": "application/vnd.google-apps.folder",
|
||||
})
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("googledrive mkdir %s: %w", name, err)
|
||||
}
|
||||
if item.ID == "" {
|
||||
return "", fmt.Errorf("googledrive mkdir %s: empty file id", name)
|
||||
}
|
||||
return item.ID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("googledrive rename: empty file id")
|
||||
}
|
||||
newName = strings.TrimSpace(newName)
|
||||
if newName == "" {
|
||||
return errors.New("googledrive rename: empty new name")
|
||||
}
|
||||
var item driveFile
|
||||
err := d.request(ctx, d.fileURL(fileID), http.MethodPatch, func(req *resty.Request) {
|
||||
req.SetQueryParam("fields", fileInfoFields)
|
||||
req.SetBody(map[string]string{"name": newName})
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return fmt.Errorf("googledrive rename: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) findUploadedFileID(ctx context.Context, parentID, name, md5Hex string) (string, error) {
|
||||
entries, err := d.List(ctx, parentID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("googledrive upload verify: %w", err)
|
||||
}
|
||||
var hashHit string
|
||||
for _, e := range entries {
|
||||
if e.IsDir {
|
||||
continue
|
||||
}
|
||||
if !strings.EqualFold(e.Hash, md5Hex) {
|
||||
continue
|
||||
}
|
||||
if e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
if hashHit == "" {
|
||||
hashHit = e.ID
|
||||
}
|
||||
}
|
||||
if hashHit != "" {
|
||||
return hashHit, nil
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir && e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("googledrive upload: uploaded file %q not found in parent %q", name, parentID)
|
||||
}
|
||||
|
||||
func isGoogleUploadHTTPRateLimit(status int, header http.Header, body []byte, apiErr apiErrorBody) bool {
|
||||
if status == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if status == http.StatusForbidden && strings.TrimSpace(header.Get("Retry-After")) != "" {
|
||||
return true
|
||||
}
|
||||
if isGoogleRateLimit(nil, apiErr) {
|
||||
return true
|
||||
}
|
||||
return googleLimitText(string(body))
|
||||
}
|
||||
|
||||
func googleUploadRateLimitError(status int, header http.Header, body []byte, message string) error {
|
||||
if strings.TrimSpace(message) == "" {
|
||||
message = "google drive upload rate limited"
|
||||
}
|
||||
bodyText := strings.TrimSpace(string(body))
|
||||
if bodyText != "" {
|
||||
message = fmt.Sprintf("%s: status=%d body=%s", message, status, bodyText)
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: parseRetryAfterHeader(header.Get("Retry-After")),
|
||||
Err: errors.New(message),
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) refresh(ctx context.Context) error {
|
||||
@@ -288,6 +710,26 @@ func (d *Driver) applyToken(out tokenResp) {
|
||||
}
|
||||
|
||||
func tokenResponseError(prefix string, res *resty.Response, out tokenResp, requireRefresh bool) error {
|
||||
if isGoogleTokenRateLimit(res, out) {
|
||||
message := strings.TrimSpace(out.Text)
|
||||
if message == "" {
|
||||
message = strings.TrimSpace(out.ErrorDescription)
|
||||
}
|
||||
if message == "" {
|
||||
message = strings.TrimSpace(out.Error)
|
||||
}
|
||||
if message == "" {
|
||||
message = "google drive token refresh rate limited"
|
||||
}
|
||||
if res != nil && strings.TrimSpace(res.String()) != "" {
|
||||
message = fmt.Sprintf("%s: status=%d body=%s", message, res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: parseRetryAfter(res),
|
||||
Err: fmt.Errorf("%s: %s", prefix, message),
|
||||
}
|
||||
}
|
||||
if out.Text != "" {
|
||||
return fmt.Errorf("%s: %s", prefix, out.Text)
|
||||
}
|
||||
@@ -380,6 +822,10 @@ func (d *Driver) filesURL() string {
|
||||
return d.apiBaseURL + "/files"
|
||||
}
|
||||
|
||||
func (d *Driver) uploadFilesURL() string {
|
||||
return d.uploadBaseURL + "/files"
|
||||
}
|
||||
|
||||
func (d *Driver) fileURL(fileID string) string {
|
||||
return d.filesURL() + "/" + url.PathEscape(fileID)
|
||||
}
|
||||
@@ -444,18 +890,85 @@ func isGoogleRateLimit(res *resty.Response, body apiErrorBody) bool {
|
||||
if res != nil && res.StatusCode() == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if res != nil && res.StatusCode() == http.StatusForbidden && strings.TrimSpace(res.Header().Get("Retry-After")) != "" {
|
||||
return true
|
||||
}
|
||||
if body.Code == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
for _, e := range body.Errors {
|
||||
reason := strings.ToLower(strings.TrimSpace(e.Reason))
|
||||
switch reason {
|
||||
case "ratelimitexceeded", "userratelimitexceeded", "downloadquotaexceeded", "sharingratelimitexceeded":
|
||||
if googleLimitReason(e.Reason) || googleLimitText(e.Message) {
|
||||
return true
|
||||
}
|
||||
domain := compactGoogleLimitText(e.Domain)
|
||||
if domain == "usagelimits" && (body.Code == http.StatusForbidden || body.Code == http.StatusTooManyRequests) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
msg := strings.ToLower(body.Message)
|
||||
return strings.Contains(msg, "rate limit") || strings.Contains(msg, "too many requests") || strings.Contains(msg, "quota exceeded")
|
||||
return googleLimitText(body.Message)
|
||||
}
|
||||
|
||||
func isGoogleTokenRateLimit(res *resty.Response, out tokenResp) bool {
|
||||
if res != nil {
|
||||
if res.StatusCode() == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if res.StatusCode() == http.StatusForbidden && strings.TrimSpace(res.Header().Get("Retry-After")) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return googleLimitText(out.Text) ||
|
||||
googleLimitText(out.Error) ||
|
||||
googleLimitText(out.ErrorDescription)
|
||||
}
|
||||
|
||||
func googleLimitReason(reason string) bool {
|
||||
switch compactGoogleLimitText(reason) {
|
||||
case "ratelimitexceeded",
|
||||
"userratelimitexceeded",
|
||||
"dailylimitexceeded",
|
||||
"dailylimitexceededunreg",
|
||||
"downloadquotaexceeded",
|
||||
"sharingratelimitexceeded",
|
||||
"quotaexceeded",
|
||||
"uploadlimitexceeded",
|
||||
"storagelimitexceeded",
|
||||
"storagequotaexceeded":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func googleLimitText(text string) bool {
|
||||
text = strings.ToLower(strings.TrimSpace(text))
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
compact := compactGoogleLimitText(text)
|
||||
if strings.Contains(compact, "ratelimitexceeded") ||
|
||||
strings.Contains(compact, "userratelimitexceeded") ||
|
||||
strings.Contains(compact, "dailylimitexceeded") ||
|
||||
strings.Contains(compact, "downloadquotaexceeded") ||
|
||||
strings.Contains(compact, "sharingratelimitexceeded") ||
|
||||
strings.Contains(compact, "quotaexceeded") ||
|
||||
strings.Contains(compact, "toomanyrequests") {
|
||||
return true
|
||||
}
|
||||
return strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "quota exceeded") ||
|
||||
strings.Contains(text, "download quota") ||
|
||||
strings.Contains(text, "sharing rate") ||
|
||||
strings.Contains(text, "daily limit") ||
|
||||
strings.Contains(text, "user rate") ||
|
||||
strings.Contains(text, "usage limit")
|
||||
}
|
||||
|
||||
func compactGoogleLimitText(text string) string {
|
||||
text = strings.ToLower(strings.TrimSpace(text))
|
||||
replacer := strings.NewReplacer("_", "", "-", "", " ", "", ".", "", ":", "")
|
||||
return replacer.Replace(text)
|
||||
}
|
||||
|
||||
func googleRateLimitError(res *resty.Response, message string) error {
|
||||
@@ -486,7 +999,11 @@ func parseRetryAfter(res *resty.Response) time.Duration {
|
||||
if res == nil {
|
||||
return 0
|
||||
}
|
||||
raw := strings.TrimSpace(res.Header().Get("Retry-After"))
|
||||
return parseRetryAfterHeader(res.Header().Get("Retry-After"))
|
||||
}
|
||||
|
||||
func parseRetryAfterHeader(raw string) time.Duration {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return 0
|
||||
}
|
||||
@@ -502,4 +1019,19 @@ func parseRetryAfter(res *resty.Response) time.Duration {
|
||||
return 0
|
||||
}
|
||||
|
||||
func splitPath(p string) []string {
|
||||
p = strings.Trim(p, "/")
|
||||
if p == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(p, "/")
|
||||
}
|
||||
|
||||
func minInt64(a, b int64) int64 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
|
||||
@@ -2,11 +2,18 @@ package googledrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestInitUsesOnlineRenewAPI(t *testing.T) {
|
||||
@@ -131,6 +138,134 @@ func TestStreamURLReturnsAuthenticatedMediaLinkWithoutRedirectRequirement(t *tes
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAndReportHashUsesResumableSession(t *testing.T) {
|
||||
body := "hello google drive"
|
||||
wantHash := md5.Sum([]byte(body))
|
||||
var sawSession bool
|
||||
var sawUpload bool
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/upload/drive/v3/files":
|
||||
sawSession = true
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("session Authorization = %q", got)
|
||||
}
|
||||
if got := r.URL.Query().Get("uploadType"); got != "resumable" {
|
||||
t.Fatalf("uploadType = %q", got)
|
||||
}
|
||||
if got := r.Header.Get("X-Upload-Content-Length"); got != "18" {
|
||||
t.Fatalf("X-Upload-Content-Length = %q", got)
|
||||
}
|
||||
var meta struct {
|
||||
Name string `json:"name"`
|
||||
Parents []string `json:"parents"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&meta); err != nil {
|
||||
t.Fatalf("decode session metadata: %v", err)
|
||||
}
|
||||
if meta.Name != "clip.mp4" || len(meta.Parents) != 1 || meta.Parents[0] != "parent-1" {
|
||||
t.Fatalf("metadata = %+v", meta)
|
||||
}
|
||||
w.Header().Set("Location", srv.URL+"/upload/session/1")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case "/upload/session/1":
|
||||
sawUpload = true
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("upload Authorization = %q", got)
|
||||
}
|
||||
if got := r.Header.Get("Content-Range"); got != "bytes 0-17/18" {
|
||||
t.Fatalf("Content-Range = %q", got)
|
||||
}
|
||||
gotBody, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read upload body: %v", err)
|
||||
}
|
||||
if string(gotBody) != body {
|
||||
t.Fatalf("upload body = %q", string(gotBody))
|
||||
}
|
||||
writeTestJSONStatus(w, http.StatusCreated, driveFile{
|
||||
ID: "file-uploaded",
|
||||
Name: "clip.mp4",
|
||||
Size: "18",
|
||||
MD5Checksum: hex.EncodeToString(wantHash[:]),
|
||||
})
|
||||
default:
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", APIBaseURL: srv.URL + "/drive/v3"})
|
||||
d.accessToken = "access"
|
||||
res, err := d.UploadAndReportHash(context.Background(), "parent-1", "clip.mp4", strings.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("UploadAndReportHash() error = %v", err)
|
||||
}
|
||||
if !sawSession || !sawUpload {
|
||||
t.Fatalf("saw session/upload = %v/%v, want both", sawSession, sawUpload)
|
||||
}
|
||||
if res.FileID != "file-uploaded" || res.Size != int64(len(body)) || res.Hash != hex.EncodeToString(wantHash[:]) {
|
||||
t.Fatalf("upload result = %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDirAndRenameUseGoogleDriveFileAPI(t *testing.T) {
|
||||
var madeDir bool
|
||||
var renamed bool
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/drive/v3/files":
|
||||
writeTestJSON(w, filesResp{})
|
||||
case r.Method == http.MethodPost && r.URL.Path == "/drive/v3/files":
|
||||
madeDir = true
|
||||
var meta struct {
|
||||
Name string `json:"name"`
|
||||
Parents []string `json:"parents"`
|
||||
MimeType string `json:"mimeType"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&meta); err != nil {
|
||||
t.Fatalf("decode mkdir body: %v", err)
|
||||
}
|
||||
if meta.Name != "91 Spider" || len(meta.Parents) != 1 || meta.Parents[0] != "root" || meta.MimeType != "application/vnd.google-apps.folder" {
|
||||
t.Fatalf("mkdir body = %+v", meta)
|
||||
}
|
||||
writeTestJSON(w, driveFile{ID: "folder-91", Name: "91 Spider", MimeType: "application/vnd.google-apps.folder"})
|
||||
case r.Method == http.MethodPatch && r.URL.Path == "/drive/v3/files/file-1":
|
||||
renamed = true
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode rename body: %v", err)
|
||||
}
|
||||
if body["name"] != "new-name.mp4" {
|
||||
t.Fatalf("rename body = %+v", body)
|
||||
}
|
||||
writeTestJSON(w, driveFile{ID: "file-1", Name: "new-name.mp4"})
|
||||
default:
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", RootID: "root", APIBaseURL: srv.URL + "/drive/v3"})
|
||||
d.accessToken = "access"
|
||||
d.listInterval = -1
|
||||
|
||||
dirID, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureDir() error = %v", err)
|
||||
}
|
||||
if dirID != "folder-91" || !madeDir {
|
||||
t.Fatalf("dirID/madeDir = %q/%v, want folder-91/true", dirID, madeDir)
|
||||
}
|
||||
if err := d.Rename(context.Background(), "file-1", "new-name.mp4"); err != nil {
|
||||
t.Fatalf("Rename() error = %v", err)
|
||||
}
|
||||
if !renamed {
|
||||
t.Fatal("rename endpoint was not called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestRefreshesOnUnauthorized(t *testing.T) {
|
||||
var fileCalls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -179,6 +314,88 @@ func TestRequestRefreshesOnUnauthorized(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateLimitReasonsFollowGoogleDriveErrorShape(t *testing.T) {
|
||||
reasons := []string{
|
||||
"rateLimitExceeded",
|
||||
"userRateLimitExceeded",
|
||||
"dailyLimitExceeded",
|
||||
"dailyLimitExceededUnreg",
|
||||
"downloadQuotaExceeded",
|
||||
"sharingRateLimitExceeded",
|
||||
"quotaExceeded",
|
||||
}
|
||||
for _, reason := range reasons {
|
||||
body := apiErrorBody{
|
||||
Code: http.StatusForbidden,
|
||||
Message: "google drive quota or rate limited",
|
||||
Errors: []struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
LocationType string `json:"location_type"`
|
||||
Location string `json:"location"`
|
||||
}{
|
||||
{Domain: "usageLimits", Reason: reason, Message: reason},
|
||||
},
|
||||
}
|
||||
if !isGoogleRateLimit(nil, body) {
|
||||
t.Fatalf("reason %q not treated as rate limit", reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRateLimitStartsSharedLinkCooldown(t *testing.T) {
|
||||
var calls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
w.Header().Set("Retry-After", "120")
|
||||
writeTestJSONStatus(w, http.StatusForbidden, apiErrorResp{Error: apiErrorBody{
|
||||
Code: http.StatusForbidden,
|
||||
Message: "User rate limit exceeded.",
|
||||
Errors: []struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
LocationType string `json:"location_type"`
|
||||
Location string `json:"location"`
|
||||
}{
|
||||
{Domain: "usageLimits", Reason: "userRateLimitExceeded", Message: "User rate limit exceeded."},
|
||||
},
|
||||
}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", APIBaseURL: srv.URL})
|
||||
d.accessToken = "access"
|
||||
d.linkCooldownDuration = time.Hour
|
||||
|
||||
_, err := d.StreamURL(context.Background(), "file-1")
|
||||
if err == nil {
|
||||
t.Fatal("first StreamURL succeeded, want rate limit")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("first error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != 2*time.Minute {
|
||||
t.Fatalf("retry after = %s, want 2m", rateLimit.RetryAfter)
|
||||
}
|
||||
|
||||
_, err = d.StreamURL(context.Background(), "file-1")
|
||||
if err == nil {
|
||||
t.Fatal("second StreamURL succeeded during cooldown")
|
||||
}
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("second error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("remote calls = %d, want 1; second call should use shared cooldown", calls)
|
||||
}
|
||||
if rateLimit.RetryAfter <= 0 || rateLimit.RetryAfter > 2*time.Minute {
|
||||
t.Fatalf("second retry after = %s, want remaining cooldown", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestJSON(w http.ResponseWriter, v any) {
|
||||
writeTestJSONStatus(w, http.StatusOK, v)
|
||||
}
|
||||
|
||||
@@ -42,8 +42,16 @@ type apiErrorBody struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Errors []struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
LocationType string `json:"location_type"`
|
||||
Location string `json:"location"`
|
||||
} `json:"errors"`
|
||||
}
|
||||
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
Hash string
|
||||
Size int64
|
||||
}
|
||||
|
||||
@@ -64,6 +64,12 @@ type CrawlerConfig struct {
|
||||
|
||||
// OnNewVideo 是新视频成功入库后的回调,用于触发预览视频 worker。
|
||||
OnNewVideo func(v *catalog.Video)
|
||||
// OnProgress 在抓取统计变化时触发,用于后台管理页展示实时进度。
|
||||
OnProgress func(progress CrawlProgress)
|
||||
// OnCheckedVideo 在 Python 爬虫开始检查一个列表页视频时触发。
|
||||
OnCheckedVideo func()
|
||||
// OnExtractedVideo 在 Python 爬虫提取到一个新视频直链时触发。
|
||||
OnExtractedVideo func()
|
||||
}
|
||||
|
||||
// Crawler 把 Python 爬虫产出包装成 catalog 入库流程。
|
||||
@@ -219,6 +225,16 @@ type CrawlResult struct {
|
||||
SeenFile string
|
||||
}
|
||||
|
||||
// CrawlProgress 是 RunOnce 过程中可安全对外发布的实时计数。
|
||||
type CrawlProgress struct {
|
||||
TargetNew int
|
||||
TotalEntries int
|
||||
NewVideos int
|
||||
Skipped int
|
||||
Failed int
|
||||
SeenSnapshot int
|
||||
}
|
||||
|
||||
// spiderVideoEntry 对应 spider_91porn.py 输出 JSON 中的单条视频。
|
||||
type spiderVideoEntry struct {
|
||||
Title string `json:"title"`
|
||||
@@ -266,6 +282,20 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
|
||||
result := &CrawlResult{TargetNew: targetNew, StartedAt: time.Now()}
|
||||
defer func() { result.FinishedAt = time.Now() }()
|
||||
emitProgress := func() {
|
||||
if c.cfg.OnProgress == nil {
|
||||
return
|
||||
}
|
||||
c.cfg.OnProgress(CrawlProgress{
|
||||
TargetNew: result.TargetNew,
|
||||
TotalEntries: result.TotalEntries,
|
||||
NewVideos: result.NewVideos,
|
||||
Skipped: result.Skipped,
|
||||
Failed: result.Failed,
|
||||
SeenSnapshot: result.SeenSnapshot,
|
||||
})
|
||||
}
|
||||
emitProgress()
|
||||
|
||||
// 1. 准备 .crawl/ 目录 + 已知源视频 ID 列表
|
||||
//
|
||||
@@ -291,6 +321,7 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
return result, fmt.Errorf("spider91 crawler: build seen list: %w", err)
|
||||
}
|
||||
result.SeenSnapshot = seenCount
|
||||
emitProgress()
|
||||
|
||||
// 2-3. 启动 Python 爬虫(流式 stdout 协议),并边读边处理。
|
||||
//
|
||||
@@ -321,9 +352,11 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
continue
|
||||
}
|
||||
result.TotalEntries++
|
||||
emitProgress()
|
||||
sourceID := sourceIDForItem(item)
|
||||
if sourceID == "" || strings.TrimSpace(item.VideoURL) == "" {
|
||||
result.Failed++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
if result.NewVideos >= targetNew {
|
||||
@@ -335,22 +368,27 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
if err != nil {
|
||||
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s check deleted: %v", c.cfg.Driver.ID(), item.Viewkey, sourceID, err)
|
||||
result.Failed++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
if deleted {
|
||||
result.Skipped++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
if existing, _ := c.cfg.Catalog.GetVideo(ctx, videoID); existing != nil {
|
||||
result.Skipped++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
if perr := c.processOne(ctx, videoID, item); perr != nil {
|
||||
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s failed: %v", c.cfg.Driver.ID(), item.Viewkey, sourceID, perr)
|
||||
result.Failed++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
result.NewVideos++
|
||||
emitProgress()
|
||||
}
|
||||
if scerr := scanner.Err(); scerr != nil {
|
||||
log.Printf("[spider91] drive=%s stdout scan: %v", c.cfg.Driver.ID(), scerr)
|
||||
@@ -458,12 +496,12 @@ func (c *Crawler) startSpiderTargetNew(ctx context.Context, targetNew int, seenP
|
||||
return nil, nil, fmt.Errorf("start: %w", err)
|
||||
}
|
||||
// stderr 转发到 backend log。子进程退出时 reader 自动 EOF,goroutine 自然结束。
|
||||
go forwardSpiderLog(c.cfg.Driver.ID(), stderr)
|
||||
go forwardSpiderLog(c.cfg.Driver.ID(), stderr, c.cfg.OnCheckedVideo, c.cfg.OnExtractedVideo)
|
||||
return cmd, stdout, nil
|
||||
}
|
||||
|
||||
// forwardSpiderLog 把 Python stderr 逐行转发到 backend log,便于调试。
|
||||
func forwardSpiderLog(driveID string, r io.Reader) {
|
||||
func forwardSpiderLog(driveID string, r io.Reader, onCheckedVideo func(), onExtractedVideo func()) {
|
||||
scanner := bufio.NewScanner(r)
|
||||
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
@@ -472,9 +510,23 @@ func forwardSpiderLog(driveID string, r io.Reader) {
|
||||
continue
|
||||
}
|
||||
log.Printf("[spider91:py] drive=%s %s", driveID, line)
|
||||
if onCheckedVideo != nil && isSpider91CheckedVideoLogLine(line) {
|
||||
onCheckedVideo()
|
||||
}
|
||||
if onExtractedVideo != nil && isSpider91ExtractedVideoLogLine(line) {
|
||||
onExtractedVideo()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isSpider91CheckedVideoLogLine(line string) bool {
|
||||
return checkedVideoLogRE.MatchString(line)
|
||||
}
|
||||
|
||||
func isSpider91ExtractedVideoLogLine(line string) bool {
|
||||
return strings.Contains(line, "[OK] 成功提取视频直链")
|
||||
}
|
||||
|
||||
// processOne 处理单个 91 源视频:下载视频 + 封面 + 复制封面 + 入库。
|
||||
// 任一步失败会清理已写入的临时文件,不留半成品。
|
||||
func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVideoEntry) error {
|
||||
@@ -847,9 +899,10 @@ func spider91CookieHeader(cookies []*http.Cookie) string {
|
||||
}
|
||||
|
||||
var (
|
||||
strencode2RE = regexp.MustCompile(`strencode2\(["']([^"']+)["']\)`)
|
||||
srcAttrRE = regexp.MustCompile(`src=['"]([^'"]+)['"]`)
|
||||
mp4URLRE = regexp.MustCompile(`https?://[^\s"'<>]+\.mp4[^\s"'<>]*`)
|
||||
checkedVideoLogRE = regexp.MustCompile(`处理视频\s+\d+/\d+:`)
|
||||
strencode2RE = regexp.MustCompile(`strencode2\(["']([^"']+)["']\)`)
|
||||
srcAttrRE = regexp.MustCompile(`src=['"]([^'"]+)['"]`)
|
||||
mp4URLRE = regexp.MustCompile(`https?://[^\s"'<>]+\.mp4[^\s"'<>]*`)
|
||||
)
|
||||
|
||||
func parseSpider91VideoURL(html string) string {
|
||||
|
||||
@@ -707,6 +707,18 @@ func TestSpider91CookieHeader(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpider91ProgressLogLineClassifiers(t *testing.T) {
|
||||
if !isSpider91CheckedVideoLogLine("[2026-06-08 16:49:17] 处理视频 3/24: 标题") {
|
||||
t.Fatal("checked video log line was not recognized")
|
||||
}
|
||||
if isSpider91CheckedVideoLogLine("[2026-06-08 16:49:17] [页 2] 发现 24 个视频") {
|
||||
t.Fatal("page summary log line should not count as checked video")
|
||||
}
|
||||
if !isSpider91ExtractedVideoLogLine("[2026-06-08 16:49:39] [OK] 成功提取视频直链") {
|
||||
t.Fatal("extracted video log line was not recognized")
|
||||
}
|
||||
}
|
||||
|
||||
func spider91DetailHTML(videoURL string) string {
|
||||
fragment := `<video><source src="` + videoURL + `" type="video/mp4"></video>`
|
||||
return `document.write(strencode2("` + url.PathEscape(fragment) + `"));`
|
||||
|
||||
@@ -327,11 +327,65 @@ func readHTTPRange(ctx context.Context, hc *http.Client, link *drives.StreamLink
|
||||
return data, nil
|
||||
}
|
||||
}
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
|
||||
if remoteRangeResponseLooksRateLimited(link.URL, resp.StatusCode, body) {
|
||||
return nil, &drives.RateLimitError{
|
||||
Provider: "fingerprint",
|
||||
RetryAfter: parseRetryAfter(resp.Header.Get("Retry-After")),
|
||||
Err: fmt.Errorf("remote sample rate limited: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(body))),
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("fingerprint: range request got status=%d for bytes=%d-%d", resp.StatusCode, r.start, end)
|
||||
}
|
||||
return io.ReadAll(io.LimitReader(resp.Body, r.length))
|
||||
}
|
||||
|
||||
func remoteRangeResponseLooksRateLimited(rawURL string, status int, body []byte) bool {
|
||||
if status == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
text := strings.ToLower(strings.TrimSpace(string(body)))
|
||||
compact := compactRemoteRangeErrorText(text)
|
||||
if strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "quota exceeded") ||
|
||||
strings.Contains(text, "download quota") ||
|
||||
strings.Contains(text, "sharing rate") ||
|
||||
strings.Contains(text, "daily limit") ||
|
||||
strings.Contains(text, "user rate") ||
|
||||
strings.Contains(text, "usage limit") ||
|
||||
strings.Contains(compact, "ratelimitexceeded") ||
|
||||
strings.Contains(compact, "userratelimitexceeded") ||
|
||||
strings.Contains(compact, "dailylimitexceeded") ||
|
||||
strings.Contains(compact, "downloadquotaexceeded") ||
|
||||
strings.Contains(compact, "sharingratelimitexceeded") ||
|
||||
strings.Contains(compact, "quotaexceeded") ||
|
||||
strings.Contains(compact, "toomanyrequests") ||
|
||||
strings.Contains(compact, "usagelimits") {
|
||||
return true
|
||||
}
|
||||
if status == http.StatusForbidden && isGoogleDriveMediaURL(rawURL) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isGoogleDriveMediaURL(rawURL string) bool {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Host)
|
||||
path := strings.ToLower(u.Path)
|
||||
return strings.Contains(host, "googleapis.com") && strings.Contains(path, "/drive/")
|
||||
}
|
||||
|
||||
func compactRemoteRangeErrorText(text string) string {
|
||||
replacer := strings.NewReplacer("_", "", "-", "", " ", "", ".", "", ":", "")
|
||||
return replacer.Replace(strings.ToLower(strings.TrimSpace(text)))
|
||||
}
|
||||
|
||||
func parseRetryAfter(raw string) time.Duration {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
|
||||
@@ -2,6 +2,7 @@ package fingerprint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -85,6 +86,33 @@ func TestComputeRemoteUsesRangeSamples(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeRemoteGoogleQuotaExceededReturnsRateLimit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Retry-After", "60")
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
_, _ = w.Write([]byte(`{"error":{"code":403,"message":"The download quota for this file has been exceeded.","errors":[{"domain":"usageLimits","reason":"downloadQuotaExceeded","message":"The download quota for this file has been exceeded."}]}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
drv := &fakeDrive{paths: map[string]string{"remote": srv.URL + "/drive/v3/files/file-1?alt=media"}}
|
||||
_, err := Compute(ctx, drv, &catalog.Video{ID: "remote", FileID: "remote", Size: 1024 * 1024}, Config{
|
||||
SampleSizeBytes: 4,
|
||||
FullHashMaxSize: 8,
|
||||
HTTPClient: srv.Client(),
|
||||
}, srv.Client())
|
||||
if err == nil {
|
||||
t.Fatal("compute succeeded, want rate limit")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != time.Minute {
|
||||
t.Fatalf("retry after = %s, want 1m", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeDrive struct {
|
||||
paths map[string]string
|
||||
}
|
||||
|
||||
@@ -1427,11 +1427,14 @@ func (w *Worker) skipIfRateLimited(v *catalog.Video) bool {
|
||||
}
|
||||
|
||||
func (w *Worker) pauseForRateLimit(err error, step, title string) bool {
|
||||
_, ok := drives.RateLimitRetryAfter(err)
|
||||
wait, ok := drives.RateLimitRetryAfter(err)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), defaultGenerationRateLimitCooldown)
|
||||
if wait <= 0 {
|
||||
wait = defaultGenerationRateLimitCooldown
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), wait)
|
||||
log.Printf("[preview] drive=%s rate-limited until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err)
|
||||
return true
|
||||
}
|
||||
@@ -1460,11 +1463,14 @@ func (w *ThumbWorker) skipIfRateLimited(v *catalog.Video) bool {
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) pauseForRateLimit(err error, step, title string) bool {
|
||||
_, ok := drives.RateLimitRetryAfter(err)
|
||||
wait, ok := drives.RateLimitRetryAfter(err)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), defaultGenerationRateLimitCooldown)
|
||||
if wait <= 0 {
|
||||
wait = defaultGenerationRateLimitCooldown
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), wait)
|
||||
log.Printf("[thumb] drive=%s rate-limited until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err)
|
||||
return true
|
||||
}
|
||||
@@ -1560,10 +1566,54 @@ func driveErrorShouldCooldown(d drives.Drive, err error) bool {
|
||||
strings.Contains(text, "blocked") ||
|
||||
strings.Contains(text, "访问被阻断") ||
|
||||
strings.Contains(text, "service unavailable")
|
||||
case "googledrive":
|
||||
// Google Drive 下载/取样阶段常把频控和配额问题包装成 403,
|
||||
// 具体标识在 error.errors[].reason/message 里(OpenList 也按该结构解析)。
|
||||
// ffmpeg/ffprobe 只能看到 stderr 文本时,按这些 reason/文本兜底冷却。
|
||||
text := strings.ToLower(err.Error())
|
||||
return googleDriveMediaErrorShouldCooldown(text)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func googleDriveMediaErrorShouldCooldown(text string) bool {
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
compact := compactGoogleDriveErrorText(text)
|
||||
return strings.Contains(text, "server returned 403") ||
|
||||
strings.Contains(text, "403 forbidden") ||
|
||||
strings.Contains(text, "server returned 429") ||
|
||||
strings.Contains(text, "http 429") ||
|
||||
strings.Contains(text, "http 500") ||
|
||||
strings.Contains(text, "http 502") ||
|
||||
strings.Contains(text, "http 503") ||
|
||||
strings.Contains(text, "http 504") ||
|
||||
strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "quota exceeded") ||
|
||||
strings.Contains(text, "download quota") ||
|
||||
strings.Contains(text, "sharing rate") ||
|
||||
strings.Contains(text, "daily limit") ||
|
||||
strings.Contains(text, "user rate") ||
|
||||
strings.Contains(text, "usage limit") ||
|
||||
strings.Contains(text, "service unavailable") ||
|
||||
strings.Contains(compact, "ratelimitexceeded") ||
|
||||
strings.Contains(compact, "userratelimitexceeded") ||
|
||||
strings.Contains(compact, "dailylimitexceeded") ||
|
||||
strings.Contains(compact, "downloadquotaexceeded") ||
|
||||
strings.Contains(compact, "sharingratelimitexceeded") ||
|
||||
strings.Contains(compact, "quotaexceeded") ||
|
||||
strings.Contains(compact, "toomanyrequests") ||
|
||||
strings.Contains(compact, "usagelimits")
|
||||
}
|
||||
|
||||
func compactGoogleDriveErrorText(text string) string {
|
||||
replacer := strings.NewReplacer("_", "", "-", "", " ", "", ".", "", ":", "")
|
||||
return replacer.Replace(strings.ToLower(strings.TrimSpace(text)))
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) bool {
|
||||
if w.skipIfRateLimited(v) {
|
||||
return false
|
||||
|
||||
@@ -442,7 +442,7 @@ func TestPreviewWorkerRateLimitLeavesCurrentPendingAndSkipsNextVideo(t *testing.
|
||||
if gen.generateCalls != 1 {
|
||||
t.Fatalf("generate calls = %d, want 1", gen.generateCalls)
|
||||
}
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 5*time.Minute)
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 2*time.Hour)
|
||||
|
||||
gen.generateErr = nil
|
||||
worker.process(ctx, &second)
|
||||
@@ -458,7 +458,7 @@ func TestPreviewWorkerRateLimitLeavesCurrentPendingAndSkipsNextVideo(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerRateLimitCoolsDownFiveMinutes(t *testing.T) {
|
||||
func TestThumbWorkerRateLimitHonorsRetryAfter(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-rate-limit")
|
||||
|
||||
@@ -482,7 +482,7 @@ func TestThumbWorkerRateLimitCoolsDownFiveMinutes(t *testing.T) {
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want unchanged after rate limit", got.ThumbnailURL)
|
||||
}
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 5*time.Minute)
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 2*time.Hour)
|
||||
}
|
||||
|
||||
func TestThumbWorkerP115TransientErrorFailsAfterRetryLimit(t *testing.T) {
|
||||
@@ -661,6 +661,24 @@ func TestP123TransientErrorsShouldCooldown(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoogleDriveMediaErrorsShouldCooldown(t *testing.T) {
|
||||
drv := &previewFakeDrive{kind: "googledrive"}
|
||||
for _, err := range []error{
|
||||
errors.New("google drive api error: usageLimits userRateLimitExceeded"),
|
||||
errors.New("ffmpeg: Server returned 403 Forbidden"),
|
||||
errors.New("downloadQuotaExceeded: The download quota for this file has been exceeded"),
|
||||
errors.New("sharingRateLimitExceeded"),
|
||||
errors.New("http 503 service unavailable"),
|
||||
} {
|
||||
if !driveErrorShouldCooldown(drv, err) {
|
||||
t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err)
|
||||
}
|
||||
}
|
||||
if driveErrorShouldCooldown(drv, errors.New("invalid credentials")) {
|
||||
t.Fatal("invalid credentials should not trigger googledrive cooldown")
|
||||
}
|
||||
}
|
||||
|
||||
func assertCooldownAround(t *testing.T, until time.Time, before time.Time, want time.Duration) {
|
||||
t.Helper()
|
||||
if until.IsZero() {
|
||||
|
||||
@@ -25,6 +25,8 @@ type Scanner struct {
|
||||
SkipDirIDs map[string]struct{}
|
||||
// 回调:新视频被加入后触发预览视频生成
|
||||
OnNewVideo func(v *catalog.Video)
|
||||
// OnProgress 在扫描进度变化时触发。回调只应读取 Stats 里的计数,不应修改 map 字段。
|
||||
OnProgress func(stats Stats)
|
||||
// ProgressInterval 控制扫描内部 heartbeat 的最小输出间隔。
|
||||
// 0 → 默认 30s;< 0 → 关闭 heartbeat(仅留外层 start / done 两行)。
|
||||
// heartbeat 单行格式:
|
||||
@@ -91,6 +93,9 @@ func (s *Scanner) Run(ctx context.Context, startDirID string) (Stats, error) {
|
||||
driveID = s.Drive.ID()
|
||||
}
|
||||
progress := func(currentDir string) {
|
||||
if s.OnProgress != nil {
|
||||
s.OnProgress(stats)
|
||||
}
|
||||
if interval < 0 {
|
||||
return
|
||||
}
|
||||
@@ -149,7 +154,6 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
continue
|
||||
}
|
||||
|
||||
stats.Scanned++
|
||||
ext := strings.ToLower(path.Ext(e.Name))
|
||||
if !s.Exts[ext] {
|
||||
continue
|
||||
@@ -157,6 +161,8 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
if e.Size <= 0 {
|
||||
continue
|
||||
}
|
||||
stats.Scanned++
|
||||
progress(dirName)
|
||||
stats.SeenFileIDs[e.ID] = struct{}{}
|
||||
|
||||
id := s.Drive.Kind() + "-" + s.Drive.ID() + "-" + e.ID
|
||||
@@ -266,6 +272,7 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
return err
|
||||
}
|
||||
stats.Added++
|
||||
progress(dirName)
|
||||
if s.OnNewVideo != nil {
|
||||
s.OnNewVideo(v)
|
||||
}
|
||||
|
||||
@@ -91,6 +91,39 @@ func TestRunIgnoresZeroSizeVideoFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunScannedCountsOnlyVideoCandidates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := &scannerFakeDrive{
|
||||
entries: []drives.Entry{
|
||||
{ID: "file-1", Name: "clip.mp4", Size: 123},
|
||||
{ID: "file-2", Name: "notes.txt", Size: 123},
|
||||
{ID: "file-3", Name: "empty.mp4", Size: 0},
|
||||
},
|
||||
}
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
|
||||
stats, err := sc.Run(ctx, "")
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Scanned != 1 {
|
||||
t.Fatalf("scanned = %d, want one non-empty video candidate", stats.Scanned)
|
||||
}
|
||||
if stats.Added != 1 {
|
||||
t.Fatalf("added = %d, want one added video", stats.Added)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStopsWhenContextCanceledDuringFileLoop(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Package spider91migrate 周期性把 spider91 drive 下载到本地的视频
|
||||
// 上传到一个指定的目标 drive 目录(PikPak、115、123 或 OneDrive),上传成功后:
|
||||
// 上传到一个指定的目标 drive 目录(PikPak、115、123、OneDrive 或 Google Drive),上传成功后:
|
||||
//
|
||||
// - 改写 catalog 行:drive_id / file_id / content_hash 改成目标盘的;
|
||||
// 视频自身的 id 不变(仍是 spider91-<driveID>-<viewkey>),video_tags、
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/googledrive"
|
||||
"github.com/video-site/backend/internal/drives/onedrive"
|
||||
"github.com/video-site/backend/internal/drives/p115"
|
||||
"github.com/video-site/backend/internal/drives/p123"
|
||||
@@ -38,13 +39,14 @@ import (
|
||||
)
|
||||
|
||||
// uploadTarget 是 migrator 调用目标 drive 的最小接口。任何一种"接收 spider91 上传"的
|
||||
// 网盘都要实现它;当前 PikPak、115、123 和 OneDrive 各自通过适配器满足。
|
||||
// 网盘都要实现它;当前 PikPak、115、123、OneDrive 和 Google Drive 各自通过适配器满足。
|
||||
//
|
||||
// 这一层抽象把"迁移调用方"和"具体盘的 SDK 协议"解耦:
|
||||
// - PikPak 走 GCID + OSS PutObject(pikpak.UploadResult)
|
||||
// - 115 走 SHA1 + 秒传 / OSS / 分片(p115.UploadResult)
|
||||
// - 123 走 MD5 + 秒传 / S3 预签名分片(p123.UploadResult)
|
||||
// - OneDrive 走 SHA1 + 小文件 PUT / 大文件 upload session
|
||||
// - Google Drive 走 MD5 + resumable upload session
|
||||
//
|
||||
// 各家返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
|
||||
type uploadTarget interface {
|
||||
@@ -59,7 +61,7 @@ type uploadTarget interface {
|
||||
// UploadResult 是 uploadTarget.UploadAndReportHash 的归一返回。
|
||||
//
|
||||
// FileID 目标盘上的新文件 ID;
|
||||
// Hash GCID(PikPak)、MD5 HEX(123)或 SHA1 HEX(115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;
|
||||
// Hash GCID(PikPak)、MD5 HEX(123 / Google Drive)或 SHA1 HEX(115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;
|
||||
// Size 实际上传字节数。
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
@@ -69,7 +71,7 @@ type UploadResult struct {
|
||||
|
||||
const spider91UploadDirName = "91 Spider"
|
||||
|
||||
// pikpakAdapter / p115Adapter / p123Adapter / onedriveAdapter 把具体 driver 包装成 uploadTarget。
|
||||
// pikpakAdapter / p115Adapter / p123Adapter / onedriveAdapter / googledriveAdapter 把具体 driver 包装成 uploadTarget。
|
||||
//
|
||||
// 之所以不让 driver 直接实现 uploadTarget:
|
||||
//
|
||||
@@ -160,6 +162,27 @@ func (a *onedriveAdapter) Rename(ctx context.Context, fileID, newName string) er
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
type googledriveAdapter struct {
|
||||
d *googledrive.Driver
|
||||
}
|
||||
|
||||
func (a *googledriveAdapter) ID() string { return a.d.ID() }
|
||||
func (a *googledriveAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *googledriveAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *googledriveAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *googledriveAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
return UploadResult{FileID: res.FileID, Hash: res.Hash, Size: res.Size}, nil
|
||||
}
|
||||
func (a *googledriveAdapter) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
// adaptUploadTarget 把通用 drive 包装成 uploadTarget。
|
||||
// 不支持的盘 kind 返回 error;调用方静默跳过。
|
||||
func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
@@ -172,6 +195,8 @@ func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
return &p123Adapter{d: v}, nil
|
||||
case *onedrive.Driver:
|
||||
return &onedriveAdapter{d: v}, nil
|
||||
case *googledrive.Driver:
|
||||
return &googledriveAdapter{d: v}, nil
|
||||
case uploadTarget:
|
||||
// 测试或自定义实现可以直接传入;优先使用具体类型分支以拿到适配器。
|
||||
return v, nil
|
||||
@@ -785,7 +810,7 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driv
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// backfillFileNames 扫描目标 drive(PikPak、115、123 或 OneDrive)下所有 spider91-* 起始 ID 的视频,
|
||||
// backfillFileNames 扫描目标 drive(PikPak、115、123、OneDrive 或 Google Drive)下所有 spider91-* 起始 ID 的视频,
|
||||
// 对文件名不是 desiredPikPakName(...) 期望格式的,调 target.Rename 修正,
|
||||
// 并把 catalog.file_name 同步到新名字。
|
||||
//
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/googledrive"
|
||||
"github.com/video-site/backend/internal/drives/p123"
|
||||
"github.com/video-site/backend/internal/drives/pikpak"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
@@ -1095,7 +1096,22 @@ func TestAdaptUploadTargetSupportsP123Driver(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115、123 也不是 OneDrive 时,
|
||||
func TestAdaptUploadTargetSupportsGoogleDriveDriver(t *testing.T) {
|
||||
d := googledrive.New(googledrive.Config{
|
||||
ID: "google-target",
|
||||
RootID: "root-google",
|
||||
RefreshToken: "refresh-token",
|
||||
})
|
||||
target, err := adaptUploadTarget(d)
|
||||
if err != nil {
|
||||
t.Fatalf("adaptUploadTarget() error = %v", err)
|
||||
}
|
||||
if target.ID() != "google-target" || target.Kind() != "googledrive" || target.RootID() != "root-google" {
|
||||
t.Fatalf("target id/kind/root = %q/%q/%q, want google-target/googledrive/root-google", target.ID(), target.Kind(), target.RootID())
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115、123、OneDrive 也不是 Google Drive 时,
|
||||
// resolveTarget 拒绝并返回 error,让 runOnce 静默跳过(不会做破坏性变更)。
|
||||
func TestResolveTargetRejectsUnsupportedKind(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
|
||||
Generated
+37
-2
@@ -1,14 +1,16 @@
|
||||
{
|
||||
"name": "video-site",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.5",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "video-site",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.5",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"artplayer": "^5.4.0",
|
||||
"hls.js": "^1.6.16",
|
||||
"lucide-react": "0.453.0",
|
||||
"react": "18.3.1",
|
||||
"react-dom": "18.3.1",
|
||||
@@ -475,6 +477,15 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/artplayer": {
|
||||
"version": "5.4.0",
|
||||
"resolved": "https://registry.npmjs.org/artplayer/-/artplayer-5.4.0.tgz",
|
||||
"integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"option-validator": "^2.0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/csstype": {
|
||||
"version": "3.2.3",
|
||||
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz",
|
||||
@@ -525,12 +536,27 @@
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/hls.js": {
|
||||
"version": "1.6.16",
|
||||
"resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.6.16.tgz",
|
||||
"integrity": "sha512-VSIRpLfRwlAAdGL4wiTucx2ScRipo0ed1FBatWkyt832jC4CReKstga6yIhYVwGu9LOBjuX9wzmRMeQdBJtzEA==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/js-tokens": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
|
||||
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/kind-of": {
|
||||
"version": "6.0.3",
|
||||
"resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
|
||||
"integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/lightningcss": {
|
||||
"version": "1.32.0",
|
||||
"resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz",
|
||||
@@ -832,6 +858,15 @@
|
||||
"node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/option-validator": {
|
||||
"version": "2.0.6",
|
||||
"resolved": "https://registry.npmjs.org/option-validator/-/option-validator-2.0.6.tgz",
|
||||
"integrity": "sha512-tmZDan2LRIRQyhUGvkff68/O0R8UmF+Btmiiz0SmSw2ng3CfPZB9wJlIjHpe/MKUZqyIZkVIXCrwr1tIN+0Dzg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"kind-of": "^6.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/picocolors": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
|
||||
|
||||
+3
-1
@@ -2,7 +2,7 @@
|
||||
"name": "video-site",
|
||||
"private": true,
|
||||
"license": "MIT",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.5",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
@@ -13,6 +13,8 @@
|
||||
"test": "node --import tsx --test tests/*.test.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"artplayer": "^5.4.0",
|
||||
"hls.js": "^1.6.16",
|
||||
"lucide-react": "0.453.0",
|
||||
"react": "18.3.1",
|
||||
"react-dom": "18.3.1",
|
||||
|
||||
+74
-15
@@ -20,6 +20,7 @@ import { formatBytes } from "./storageFormat";
|
||||
import { makeUniqueDriveId } from "./driveId";
|
||||
import {
|
||||
FormState,
|
||||
driveKindAbbr,
|
||||
kindLabel,
|
||||
emptyForm,
|
||||
idleNightlyStatus,
|
||||
@@ -38,6 +39,21 @@ import { DriveForm } from "./drive/DriveForm";
|
||||
import { DeleteDriveModal } from "./drive/DeleteDriveModal";
|
||||
import { SkipDirsPanel } from "./drive/SkipDirsPanel";
|
||||
|
||||
const DRIVE_BUSY_MESSAGE = "当前存储有正在进行的任务,请稍后重试";
|
||||
const NIGHTLY_BUSY_MESSAGE = "当前有全量扫描任务正在进行,请稍后重试";
|
||||
|
||||
function isDriveBusy(d: api.AdminDrive) {
|
||||
return [
|
||||
d.scanGenerationStatus,
|
||||
d.thumbnailGenerationStatus,
|
||||
d.previewGenerationStatus,
|
||||
d.fingerprintGenerationStatus,
|
||||
].some((status) => {
|
||||
const state = status?.state || "idle";
|
||||
return state !== "idle";
|
||||
});
|
||||
}
|
||||
|
||||
export function DrivesPage() {
|
||||
const [list, setList] = useState<api.AdminDrive[]>([]);
|
||||
const [storage, setStorage] = useState<api.AdminDriveStorage | null>(null);
|
||||
@@ -61,7 +77,8 @@ export function DrivesPage() {
|
||||
const [scanningAll, setScanningAll] = useState(false);
|
||||
const [stoppingAll, setStoppingAll] = useState(false);
|
||||
const [trackingNightly, setTrackingNightly] = useState(false);
|
||||
const [scanningDriveId, setScanningDriveId] = useState("");
|
||||
const [scanningDriveIds, setScanningDriveIds] = useState<Record<string, boolean>>({});
|
||||
const scanningDriveIdsRef = useRef(new Set<string>());
|
||||
const [stoppingDriveId, setStoppingDriveId] = useState("");
|
||||
const [searchParams, setSearchParams] = useSearchParams();
|
||||
const selectedDriveId = searchParams.get("drive") || null;
|
||||
@@ -75,7 +92,15 @@ export function DrivesPage() {
|
||||
: hasCreateFormChanges(form, initialForm);
|
||||
|
||||
const uploadTargets = useMemo(
|
||||
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115" || d.kind === "p123" || d.kind === "onedrive"),
|
||||
() =>
|
||||
list.filter(
|
||||
(d) =>
|
||||
d.kind === "pikpak" ||
|
||||
d.kind === "p115" ||
|
||||
d.kind === "p123" ||
|
||||
d.kind === "onedrive" ||
|
||||
d.kind === "googledrive"
|
||||
),
|
||||
[list]
|
||||
);
|
||||
|
||||
@@ -184,7 +209,12 @@ export function DrivesPage() {
|
||||
kind: d.kind,
|
||||
name: d.name,
|
||||
rootId: d.rootId,
|
||||
creds: d.kind === "spider91" ? { proxy: d.spider91Proxy ?? "" } : {},
|
||||
creds:
|
||||
d.kind === "spider91"
|
||||
? { proxy: d.spider91Proxy ?? "" }
|
||||
: d.kind === "googledrive"
|
||||
? { use_online_api: (d.googleDriveUseOnlineAPI ?? true) ? "true" : "false" }
|
||||
: {},
|
||||
spider91UploadDriveId: settings?.spider91UploadDriveId ?? "",
|
||||
};
|
||||
setForm(nextForm);
|
||||
@@ -295,25 +325,47 @@ export function DrivesPage() {
|
||||
}
|
||||
|
||||
async function handleRescan(d: api.AdminDrive) {
|
||||
if (scanningDriveId) return;
|
||||
setScanningDriveId(d.id);
|
||||
if (nightlyBusy) {
|
||||
show(nightlyBusyText(nightlyStatus) || NIGHTLY_BUSY_MESSAGE, "info");
|
||||
return;
|
||||
}
|
||||
if (isDriveBusy(d) || scanningDriveIdsRef.current.has(d.id)) {
|
||||
show(DRIVE_BUSY_MESSAGE, "info");
|
||||
return;
|
||||
}
|
||||
scanningDriveIdsRef.current.add(d.id);
|
||||
setScanningDriveIds((prev) => ({ ...prev, [d.id]: true }));
|
||||
try {
|
||||
await api.rescan(d.id);
|
||||
const resp = await api.rescan(d.id);
|
||||
if (!resp.accepted) {
|
||||
if (resp.status) {
|
||||
setNightlyStatus(resp.status);
|
||||
}
|
||||
show(resp.message || DRIVE_BUSY_MESSAGE, "info");
|
||||
refreshDriveList();
|
||||
return;
|
||||
}
|
||||
if (d.kind === "spider91") {
|
||||
show("已触发抓取任务,需要 2-4 分钟,可稍后刷新视频列表查看", "success");
|
||||
} else {
|
||||
show("已触发扫描,可稍后刷新视频列表查看", "success");
|
||||
}
|
||||
refreshDriveList();
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "触发失败", "error");
|
||||
} finally {
|
||||
setScanningDriveId("");
|
||||
scanningDriveIdsRef.current.delete(d.id);
|
||||
setScanningDriveIds((prev) => {
|
||||
const next = { ...prev };
|
||||
delete next[d.id];
|
||||
return next;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function handleRunNightly() {
|
||||
if (nightlyBusy) {
|
||||
show(nightlyBusyText(nightlyStatus) || "当前已有扫描所有网盘任务", "info");
|
||||
show(nightlyBusyText(nightlyStatus) || NIGHTLY_BUSY_MESSAGE, "info");
|
||||
return;
|
||||
}
|
||||
setScanningAll(true);
|
||||
@@ -324,7 +376,7 @@ export function DrivesPage() {
|
||||
setTrackingNightly(!resp.status.running);
|
||||
show("已触发扫描所有网盘,耗时较长,可在任务状态和 backend 日志观察进度", "success");
|
||||
} else {
|
||||
show("当前已有扫描所有网盘任务", "info");
|
||||
show(resp.message || NIGHTLY_BUSY_MESSAGE, "info");
|
||||
}
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "触发失败", "error");
|
||||
@@ -515,17 +567,24 @@ export function DrivesPage() {
|
||||
type="button"
|
||||
className="admin-btn is-primary"
|
||||
onClick={() => handleRescan(d)}
|
||||
disabled={!!scanningDriveId}
|
||||
aria-disabled={nightlyBusy || isDriveBusy(d) || !!scanningDriveIds[d.id]}
|
||||
title={
|
||||
nightlyBusy
|
||||
? nightlyBusyText(nightlyStatus) || NIGHTLY_BUSY_MESSAGE
|
||||
: isDriveBusy(d) || scanningDriveIds[d.id]
|
||||
? DRIVE_BUSY_MESSAGE
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
{d.kind === "spider91" ? (
|
||||
<>
|
||||
<Download size={13} className={scanningDriveId === d.id ? "admin-spin" : undefined} />
|
||||
{scanningDriveId === d.id ? "触发中..." : "立即抓取"}
|
||||
<Download size={13} className={scanningDriveIds[d.id] ? "admin-spin" : undefined} />
|
||||
{scanningDriveIds[d.id] ? "触发中..." : "立即抓取"}
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<RefreshCw size={13} className={scanningDriveId === d.id ? "admin-spin" : undefined} />
|
||||
{scanningDriveId === d.id ? "触发中..." : "立即重扫"}
|
||||
<RefreshCw size={13} className={scanningDriveIds[d.id] ? "admin-spin" : undefined} />
|
||||
{scanningDriveIds[d.id] ? "触发中..." : "立即重扫"}
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
@@ -720,7 +779,7 @@ export function DrivesPage() {
|
||||
<div className="admin-drive-card__header">
|
||||
<div className="admin-drive-card__title">
|
||||
<span className="admin-drive-card__brand-icon" data-kind={d.kind}>
|
||||
{d.kind.substring(0, 2)}
|
||||
{driveKindAbbr(d.kind)}
|
||||
</span>
|
||||
<span>{d.name || d.id}</span>
|
||||
</div>
|
||||
|
||||
+9
-4
@@ -95,6 +95,9 @@ export type AdminDrive = {
|
||||
lastCrawlAt?: number;
|
||||
// spider91 专用代理地址;仅后台管理接口返回,用于编辑表单回显。
|
||||
spider91Proxy?: string;
|
||||
// Google Drive 是否使用 OpenList 在线续期 API;未配置时后端按 true 返回。
|
||||
googleDriveUseOnlineAPI?: boolean;
|
||||
scanGenerationStatus?: DriveGenerationStatus;
|
||||
thumbnailGenerationStatus?: DriveGenerationStatus;
|
||||
previewGenerationStatus?: DriveGenerationStatus;
|
||||
fingerprintGenerationStatus?: DriveGenerationStatus;
|
||||
@@ -115,6 +118,8 @@ export type DriveGenerationStatus = {
|
||||
currentTitle?: string;
|
||||
queueLength: number;
|
||||
cooldownUntil?: string;
|
||||
scannedCount: number;
|
||||
addedCount: number;
|
||||
};
|
||||
|
||||
export function listDrives() {
|
||||
@@ -170,7 +175,7 @@ export function deleteDrive(id: string, body: DeleteDriveInput) {
|
||||
}
|
||||
|
||||
export function rescan(id: string) {
|
||||
return request<{ ok: boolean }>(
|
||||
return request<{ ok: boolean; accepted: boolean; message?: string; status?: NightlyJobStatus }>(
|
||||
`/drives/${encodeURIComponent(id)}/rescan`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
@@ -402,9 +407,9 @@ export type Theme = "dark" | "pink";
|
||||
export type Settings = {
|
||||
theme: Theme;
|
||||
/**
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115、p123 或 onedrive drive)。
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115、p123、onedrive 或 googledrive drive)。
|
||||
* - 空字符串:本地保存,不上传到云盘。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, p123, onedrive}。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, p123, onedrive, googledrive}。
|
||||
*/
|
||||
spider91UploadDriveId: string;
|
||||
};
|
||||
@@ -448,7 +453,7 @@ export function getNightlyJobStatus() {
|
||||
}
|
||||
|
||||
export function runNightlyJob() {
|
||||
return request<{ ok: boolean; accepted: boolean; status: NightlyJobStatus }>(
|
||||
return request<{ ok: boolean; accepted: boolean; status: NightlyJobStatus; message?: string }>(
|
||||
"/jobs/nightly/run",
|
||||
{ method: "POST" }
|
||||
);
|
||||
|
||||
@@ -204,6 +204,11 @@ export function DriveGenerationPanel({
|
||||
</header>
|
||||
|
||||
<div className="admin-gen-columns">
|
||||
<DriveGenCol
|
||||
label={d.kind === "spider91" ? "抓取" : "扫盘"}
|
||||
status={d.scanGenerationStatus}
|
||||
showCounts={false}
|
||||
/>
|
||||
<DriveGenCol
|
||||
label="封面"
|
||||
status={d.thumbnailGenerationStatus}
|
||||
@@ -265,6 +270,7 @@ function DriveGenCol({
|
||||
pending,
|
||||
failed,
|
||||
extra,
|
||||
showCounts = true,
|
||||
}: {
|
||||
label: string;
|
||||
status?: api.DriveGenerationStatus;
|
||||
@@ -272,10 +278,14 @@ function DriveGenCol({
|
||||
pending?: number;
|
||||
failed?: number;
|
||||
extra?: number;
|
||||
showCounts?: boolean;
|
||||
}) {
|
||||
const state = status?.state || "idle";
|
||||
const detail = generationDetail(status);
|
||||
const title = generationTitle(status, detail);
|
||||
const stateLabel = label === "抓取" && state === "scanning" ? "抓取中" : generationStateLabel(state);
|
||||
const showScanProgress = !showCounts && (state === "scanning" || (status?.scannedCount ?? 0) > 0 || (status?.addedCount ?? 0) > 0);
|
||||
const scannedLabel = label === "抓取" ? "已抓取" : "已扫描";
|
||||
return (
|
||||
<div className="admin-gen-col">
|
||||
<div className="admin-gen-col__head">
|
||||
@@ -284,18 +294,26 @@ function DriveGenCol({
|
||||
className={`admin-status admin-generation-state is-${generationStateClass(state)}`}
|
||||
title={title || undefined}
|
||||
>
|
||||
{generationStateLabel(state)}
|
||||
{stateLabel}
|
||||
</span>
|
||||
</div>
|
||||
{detail && <div className="admin-gen-col__detail">{detail}</div>}
|
||||
<div className="admin-gen-col__counts">
|
||||
<div className="admin-gen-col__count"><span>就绪</span><strong>{ready ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>待生成</span><strong>{pending ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>失败</span><strong>{failed ?? 0}</strong></div>
|
||||
{(extra ?? 0) > 0 && (
|
||||
<div className="admin-gen-col__count"><span>待补时长</span><strong>{extra}</strong></div>
|
||||
)}
|
||||
</div>
|
||||
{showScanProgress && (
|
||||
<div className="admin-gen-col__counts admin-gen-col__counts--scan">
|
||||
<div className="admin-gen-col__count"><span>{scannedLabel}</span><strong>{status?.scannedCount ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>预计新增</span><strong>{status?.addedCount ?? 0}</strong></div>
|
||||
</div>
|
||||
)}
|
||||
{showCounts && (
|
||||
<div className="admin-gen-col__counts">
|
||||
<div className="admin-gen-col__count"><span>就绪</span><strong>{ready ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>待生成</span><strong>{pending ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>失败</span><strong>{failed ?? 0}</strong></div>
|
||||
{(extra ?? 0) > 0 && (
|
||||
<div className="admin-gen-col__count"><span>待补时长</span><strong>{extra}</strong></div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useId, useMemo, useState } from "react";
|
||||
import { ArrowLeft } from "lucide-react";
|
||||
import { ArrowLeft, ChevronDown } from "lucide-react";
|
||||
import { P123QRCodeLogin } from "./P123QRCodeLogin";
|
||||
import { Spider91UploadTargetField } from "./Spider91UploadTargetField";
|
||||
import {
|
||||
@@ -49,7 +49,7 @@ export function DriveForm({
|
||||
onBack?: () => void;
|
||||
}) {
|
||||
const idPrefix = useId();
|
||||
const fields = useMemo(() => credentialFields(form.kind), [form.kind]);
|
||||
const fields = useMemo(() => credentialFields(form.kind, form.creds), [form.kind, form.creds]);
|
||||
const help = credentialHelp(form.kind, isEdit);
|
||||
const [step, setStep] = useState<"type" | "form">(isEdit ? "form" : "type");
|
||||
const nameId = `${idPrefix}-drive-name`;
|
||||
@@ -180,25 +180,53 @@ export function DriveForm({
|
||||
|
||||
{fields.map((f) => (
|
||||
<div key={f.key} className="admin-form__row">
|
||||
<label htmlFor={`${idPrefix}-credential-${f.key}`}>
|
||||
{f.label}
|
||||
{f.required && " *"}
|
||||
</label>
|
||||
{f.multiline ? (
|
||||
<textarea
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
value={form.creds[f.key] ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
placeholder={f.placeholder}
|
||||
/>
|
||||
{f.type === "select" ? (
|
||||
<>
|
||||
<label htmlFor={`${idPrefix}-credential-${f.key}`}>
|
||||
{f.label}
|
||||
{f.required && " *"}
|
||||
</label>
|
||||
<div className="admin-form-select-wrap">
|
||||
<select
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
className="admin-form-select"
|
||||
value={form.creds[f.key] ?? f.defaultValue ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
>
|
||||
{(f.options ?? []).map((option) => (
|
||||
<option key={option.value} value={option.value}>
|
||||
{option.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<ChevronDown size={15} className="admin-form-select__icon" aria-hidden="true" />
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<input
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
type={credentialInputType(f.key)}
|
||||
value={form.creds[f.key] ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
placeholder={f.placeholder}
|
||||
/>
|
||||
<>
|
||||
<label htmlFor={`${idPrefix}-credential-${f.key}`}>
|
||||
{f.label}
|
||||
{f.required && " *"}
|
||||
</label>
|
||||
{f.multiline ? (
|
||||
<textarea
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
value={form.creds[f.key] ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
placeholder={f.placeholder}
|
||||
required={f.required && !isEdit}
|
||||
/>
|
||||
) : (
|
||||
<input
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
type={credentialInputType(f.key)}
|
||||
value={form.creds[f.key] ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
placeholder={f.placeholder}
|
||||
required={f.required && !isEdit}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
{f.help && <div className="admin-form__help">{f.help}</div>}
|
||||
</div>
|
||||
|
||||
@@ -12,6 +12,16 @@ export const kindAbbr: Record<string, string> = {
|
||||
spider91: "91",
|
||||
};
|
||||
|
||||
export function driveKindAbbr(kind: string): string {
|
||||
const explicit = kindAbbr[kind];
|
||||
if (explicit) return explicit;
|
||||
|
||||
const trimmed = kind.trim();
|
||||
if (!trimmed) return "??";
|
||||
const compact = trimmed.replace(/[^a-zA-Z0-9]+/g, "");
|
||||
return (compact || trimmed).slice(0, 2).toUpperCase();
|
||||
}
|
||||
|
||||
export const kindLabel: Record<string, string> = {
|
||||
quark: "夸克网盘",
|
||||
p115: "115 网盘",
|
||||
@@ -56,12 +66,12 @@ export function nightlyButtonText(status: { running: boolean; queued: boolean },
|
||||
}
|
||||
|
||||
export function nightlyBusyText(status: { running: boolean; queued: boolean }) {
|
||||
if (status.running) return "扫描任务正在运行";
|
||||
if (status.queued) return "扫描任务已排队";
|
||||
if (status.running || status.queued) return "当前有全量扫描任务正在进行,请稍后重试";
|
||||
return "";
|
||||
}
|
||||
|
||||
export function generationStateLabel(state: string): string {
|
||||
if (state === "scanning") return "扫盘中";
|
||||
if (state === "generating") return "生成中";
|
||||
if (state === "cooling") return "冷却中";
|
||||
if (state === "queued") return "排队中";
|
||||
@@ -69,7 +79,8 @@ export function generationStateLabel(state: string): string {
|
||||
}
|
||||
|
||||
export function generationStateClass(state: string): string {
|
||||
if (state === "generating" || state === "cooling" || state === "queued") {
|
||||
if (state === "scanning" || state === "generating" || state === "cooling" || state === "queued") {
|
||||
if (state === "scanning") return "generating";
|
||||
return state;
|
||||
}
|
||||
return "idle";
|
||||
@@ -146,7 +157,9 @@ export function credentialHelp(kind: Kind, isEdit: boolean): string {
|
||||
case "onedrive":
|
||||
return `按 OpenList 默认应用在线挂载,只需要 refresh_token;保存时会自动刷新并保存 token。${note}`;
|
||||
case "googledrive":
|
||||
return `按 OpenList 在线 API 挂载,只需要 Google Drive refresh_token;保存时会自动刷新并保存 token。播放不走 302,会由后端带 Authorization 代理转发。${note}`;
|
||||
return isEdit
|
||||
? "请参考OpenList文档中关于谷歌云盘的配置方法;如不修改凭证,留空即可,保存时会沿用旧值"
|
||||
: "请参考OpenList文档中关于谷歌云盘的配置方法";
|
||||
case "localstorage":
|
||||
return `填写服务器可访问的本地目录绝对路径,例如 /mnt/videos。系统会扫描该目录及子目录中的视频文件和 .strm 文件;.strm 可指向 HTTP/HTTPS 直链,或指向本地存储根目录内的真实视频路径。Docker 部署时请填写容器内路径。${note}`;
|
||||
case "spider91":
|
||||
@@ -156,14 +169,31 @@ export function credentialHelp(kind: Kind, isEdit: boolean): string {
|
||||
}
|
||||
}
|
||||
|
||||
export function credentialFields(kind: Kind): Array<{
|
||||
export type CredentialField = {
|
||||
key: string;
|
||||
label: string;
|
||||
placeholder: string;
|
||||
type?: "text" | "select";
|
||||
options?: Array<{ value: string; label: string }>;
|
||||
multiline?: boolean;
|
||||
required?: boolean;
|
||||
defaultValue?: string;
|
||||
help?: string;
|
||||
}> {
|
||||
};
|
||||
|
||||
export function credentialBoolValue(value: string | undefined, defaultValue: boolean): boolean {
|
||||
const normalized = (value ?? "").trim().toLowerCase();
|
||||
if (normalized === "") return defaultValue;
|
||||
if (normalized === "true" || normalized === "1" || normalized === "yes" || normalized === "on") return true;
|
||||
if (normalized === "false" || normalized === "0" || normalized === "no" || normalized === "off") return false;
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
export function googleDriveUsesOnlineAPI(creds: Record<string, string> = {}): boolean {
|
||||
return credentialBoolValue(creds.use_online_api, true);
|
||||
}
|
||||
|
||||
export function credentialFields(kind: Kind, creds: Record<string, string> = {}): CredentialField[] {
|
||||
switch (kind) {
|
||||
case "quark":
|
||||
return [
|
||||
@@ -252,6 +282,17 @@ export function credentialFields(kind: Kind): Array<{
|
||||
];
|
||||
case "googledrive":
|
||||
return [
|
||||
{
|
||||
key: "use_online_api",
|
||||
label: "认证方式",
|
||||
placeholder: "",
|
||||
type: "select",
|
||||
defaultValue: "true",
|
||||
options: [
|
||||
{ value: "true", label: "OpenList 在线 API" },
|
||||
{ value: "false", label: "自建 Google OAuth 客户端" },
|
||||
],
|
||||
},
|
||||
{
|
||||
key: "refresh_token",
|
||||
label: "refresh_token",
|
||||
@@ -259,6 +300,24 @@ export function credentialFields(kind: Kind): Array<{
|
||||
multiline: true,
|
||||
required: true,
|
||||
},
|
||||
...(googleDriveUsesOnlineAPI(creds)
|
||||
? []
|
||||
: [
|
||||
{
|
||||
key: "client_id",
|
||||
label: "客户端 ID",
|
||||
placeholder: "xxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.apps.googleusercontent.com",
|
||||
required: true,
|
||||
help: "Google Cloud Console 中 OAuth 2.0 客户端的 Client ID",
|
||||
},
|
||||
{
|
||||
key: "client_secret",
|
||||
label: "客户端密钥",
|
||||
placeholder: "Google OAuth client secret",
|
||||
required: true,
|
||||
help: "Google Cloud Console 中同一个 OAuth 客户端的 Client Secret",
|
||||
},
|
||||
]),
|
||||
];
|
||||
case "localstorage":
|
||||
return [
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { ReactNode } from "react";
|
||||
import { ReactNode, useEffect, useState } from "react";
|
||||
import { TopBar } from "./TopBar";
|
||||
import { MainNav } from "./MainNav";
|
||||
import { SubNav } from "./SubNav";
|
||||
@@ -7,14 +7,84 @@ import { BackToTop } from "./BackToTop";
|
||||
|
||||
type Props = {
|
||||
children: ReactNode;
|
||||
mobileAutoHideNav?: boolean;
|
||||
};
|
||||
|
||||
export function AppShell({ children }: Props) {
|
||||
const MOBILE_NAV_QUERY = "(max-width: 768px)";
|
||||
const SCROLL_DELTA_THRESHOLD = 6;
|
||||
const HIDE_AFTER_SCROLL_Y = 56;
|
||||
|
||||
export function AppShell({ children, mobileAutoHideNav = false }: Props) {
|
||||
const [mobileNavHidden, setMobileNavHidden] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!mobileAutoHideNav) {
|
||||
setMobileNavHidden(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const mediaQuery = window.matchMedia(MOBILE_NAV_QUERY);
|
||||
let lastScrollY = Math.max(window.scrollY, 0);
|
||||
let ticking = false;
|
||||
|
||||
const showNav = () => setMobileNavHidden(false);
|
||||
|
||||
const updateNavVisibility = () => {
|
||||
ticking = false;
|
||||
const currentScrollY = Math.max(window.scrollY, 0);
|
||||
|
||||
if (!mediaQuery.matches || currentScrollY <= 0) {
|
||||
showNav();
|
||||
lastScrollY = currentScrollY;
|
||||
return;
|
||||
}
|
||||
|
||||
const delta = currentScrollY - lastScrollY;
|
||||
if (Math.abs(delta) < SCROLL_DELTA_THRESHOLD) return;
|
||||
|
||||
if (delta > 0 && currentScrollY > HIDE_AFTER_SCROLL_Y) {
|
||||
setMobileNavHidden(true);
|
||||
} else if (delta < 0) {
|
||||
showNav();
|
||||
}
|
||||
|
||||
lastScrollY = currentScrollY;
|
||||
};
|
||||
|
||||
const handleScroll = () => {
|
||||
if (ticking) return;
|
||||
ticking = true;
|
||||
window.requestAnimationFrame(updateNavVisibility);
|
||||
};
|
||||
|
||||
const handleMediaChange = () => {
|
||||
lastScrollY = Math.max(window.scrollY, 0);
|
||||
showNav();
|
||||
};
|
||||
|
||||
handleMediaChange();
|
||||
window.addEventListener("scroll", handleScroll, { passive: true });
|
||||
mediaQuery.addEventListener("change", handleMediaChange);
|
||||
|
||||
return () => {
|
||||
window.removeEventListener("scroll", handleScroll);
|
||||
mediaQuery.removeEventListener("change", handleMediaChange);
|
||||
};
|
||||
}, [mobileAutoHideNav]);
|
||||
|
||||
const className = [
|
||||
"app-shell",
|
||||
mobileAutoHideNav ? "app-shell--mobile-auto-hide-nav" : "",
|
||||
mobileNavHidden ? "is-mobile-nav-hidden" : "",
|
||||
].filter(Boolean).join(" ");
|
||||
|
||||
return (
|
||||
<div className="app-shell">
|
||||
<TopBar />
|
||||
<MainNav />
|
||||
<SubNav />
|
||||
<div className={className}>
|
||||
<div className="app-shell__nav-stack">
|
||||
<TopBar />
|
||||
<MainNav />
|
||||
<SubNav />
|
||||
</div>
|
||||
<main className="app-shell__main">{children}</main>
|
||||
<Footer />
|
||||
<BackToTop />
|
||||
|
||||
@@ -38,13 +38,11 @@ export function RecommendedRail({ videos }: Props) {
|
||||
return (
|
||||
<aside className="vd-rail" aria-label="推荐视频">
|
||||
<header className="vd-rail__head">
|
||||
<span className="vd-rail__head-bar" aria-hidden="true" />
|
||||
<div className="vd-rail__head-text">
|
||||
<h2 className="vd-rail__head-title">推荐视频</h2>
|
||||
<span className="vd-rail__head-sub">
|
||||
根据当前视频 · {videos.length} 条
|
||||
</span>
|
||||
</div>
|
||||
<span className="vd-rail__head-icon" aria-hidden="true">
|
||||
<span />
|
||||
<span />
|
||||
</span>
|
||||
<h2 className="vd-rail__head-title">推荐视频</h2>
|
||||
</header>
|
||||
<ul className="vd-rail__list">
|
||||
{videos.map((v) => (
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useState } from "react";
|
||||
import { useEffect, useState } from "react";
|
||||
import { EyeOff, ThumbsDown, ThumbsUp } from "lucide-react";
|
||||
import type { VideoDetail } from "@/types";
|
||||
import { formatCount } from "@/lib/format";
|
||||
@@ -12,11 +12,11 @@ type Props = {
|
||||
/**
|
||||
* 视频操作工具条。
|
||||
* - 整体是一张浮起的圆角玻璃卡,比上一版的横线分隔更"成体"。
|
||||
* - 点赞 + 点踩组成一个胶囊(中间一道竖线分隔),两侧分别带计数。
|
||||
* - 点赞 + 点踩是两个独立按钮。
|
||||
* - "不再显示" 单独成一个次要按钮,hover 时露出 danger 色。
|
||||
*
|
||||
* 功能没变:
|
||||
* - 后端只有点赞接口(POST /api/video/:id/like),点踩仅本地 state。
|
||||
* - 后端只有点赞计数接口,点踩仅本地 state。
|
||||
* - 失败回滚已经处理。
|
||||
*/
|
||||
export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
@@ -25,11 +25,20 @@ export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
const [bursting, setBursting] = useState(false);
|
||||
const [liked, setLiked] = useState(false);
|
||||
const [disliked, setDisliked] = useState(false);
|
||||
const [likeSubmitted, setLikeSubmitted] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
setLikes(video.likes ?? 0);
|
||||
setDislikes(video.dislikes ?? 0);
|
||||
setBursting(false);
|
||||
setLiked(false);
|
||||
setDisliked(false);
|
||||
setLikeSubmitted(false);
|
||||
}, [video.id, video.likes, video.dislikes]);
|
||||
|
||||
async function handleLike() {
|
||||
if (liked) return;
|
||||
setLiked(true);
|
||||
setLikes((n) => n + 1);
|
||||
setBursting(true);
|
||||
window.setTimeout(() => setBursting(false), 320);
|
||||
|
||||
@@ -38,6 +47,11 @@ export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
setDislikes((n) => Math.max(0, n - 1));
|
||||
}
|
||||
|
||||
if (likeSubmitted) return;
|
||||
|
||||
setLikeSubmitted(true);
|
||||
setLikes((n) => n + 1);
|
||||
|
||||
try {
|
||||
const res = await fetch(
|
||||
`/api/video/${encodeURIComponent(video.id)}/like`,
|
||||
@@ -51,6 +65,7 @@ export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
} catch {
|
||||
setLikes((n) => Math.max(0, n - 1));
|
||||
setLiked(false);
|
||||
setLikeSubmitted(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,7 +79,6 @@ export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
setDislikes((n) => n + 1);
|
||||
if (liked) {
|
||||
setLiked(false);
|
||||
setLikes((n) => Math.max(0, n - 1));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,7 +97,6 @@ export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
<ThumbsUp size={18} fill={liked ? "currentColor" : "none"} />
|
||||
<span className="vd-actions__count">{formatCount(likes)}</span>
|
||||
</button>
|
||||
<span className="vd-actions__divider" aria-hidden="true" />
|
||||
<button
|
||||
type="button"
|
||||
className={`vd-actions__pill vd-actions__dislike${
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useMemo, useState } from "react";
|
||||
import { Hash, Pencil, X } from "lucide-react";
|
||||
import { Pencil, Tag, X } from "lucide-react";
|
||||
import type { TagItem, VideoDetail } from "@/types";
|
||||
|
||||
type Props = {
|
||||
@@ -17,7 +17,7 @@ type Props = {
|
||||
* 视觉上和上一版的"两张分离卡"相比,整体感更强:
|
||||
* - 一张大卡内分两个小区块,区块之间用细分隔线
|
||||
* - 简介区块加 "简介" 标题前缀
|
||||
* - 标签区块加 # 图标暗示
|
||||
* - 标签区块加标签轮廓图标暗示
|
||||
*/
|
||||
export function VideoInfoPanel({
|
||||
video,
|
||||
@@ -99,7 +99,7 @@ export function VideoInfoPanel({
|
||||
<div className="vd-info__tags">
|
||||
<div className="vd-info__section-head">
|
||||
<span className="vd-info__section-title">
|
||||
<Hash size={14} aria-hidden="true" />
|
||||
<Tag size={15} strokeWidth={2} aria-hidden="true" />
|
||||
标签
|
||||
</span>
|
||||
{onTagsChange && (
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { CalendarDays, Clock3, Eye } from "lucide-react";
|
||||
import type { VideoDetail } from "@/types";
|
||||
import { formatCount } from "@/lib/format";
|
||||
|
||||
@@ -9,12 +10,11 @@ type Props = {
|
||||
* 详情页标题块。
|
||||
*
|
||||
* 视觉:
|
||||
* - 标题:大、粗、最高两行
|
||||
* - meta:作者首字头像 + 名字 + 一组小胶囊(来源、画质、时长、观看数、发布时间)
|
||||
* - meta:一组小胶囊(来源、画质、时长、观看数、发布时间)
|
||||
* 每个胶囊有自己的语义色彩,避免传统 "·" 分隔列表的列表感。
|
||||
* - 标题:大、粗、最高两行,位于 meta 下方
|
||||
*/
|
||||
export function VideoMetaHeader({ video }: Props) {
|
||||
const author = (video.author ?? "").trim();
|
||||
const source = (video.sourceLabel ?? "").trim();
|
||||
const quality = (video.quality ?? "").trim();
|
||||
const duration = (video.duration ?? "").trim();
|
||||
@@ -23,20 +23,7 @@ export function VideoMetaHeader({ video }: Props) {
|
||||
|
||||
return (
|
||||
<header className="vd-header">
|
||||
<h1 className="vd-header__title" title={video.title}>
|
||||
{video.title}
|
||||
</h1>
|
||||
|
||||
<div className="vd-header__row">
|
||||
{author && (
|
||||
<div className="vd-author" aria-label={`作者 ${author}`}>
|
||||
<span className="vd-author__avatar" aria-hidden="true">
|
||||
{author.slice(0, 1)}
|
||||
</span>
|
||||
<span className="vd-author__name">{author}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<ul className="vd-meta" aria-label="视频信息">
|
||||
{source && (
|
||||
<li className="vd-meta__chip" data-tone={sourceKind || "neutral"}>
|
||||
@@ -52,13 +39,28 @@ export function VideoMetaHeader({ video }: Props) {
|
||||
{quality}
|
||||
</li>
|
||||
)}
|
||||
{duration && <li className="vd-meta__chip">{duration}</li>}
|
||||
<li className="vd-meta__chip">
|
||||
{duration && (
|
||||
<li className="vd-meta__chip vd-meta__chip--plain">
|
||||
<Clock3 size={14} aria-hidden="true" />
|
||||
{duration}
|
||||
</li>
|
||||
)}
|
||||
<li className="vd-meta__chip vd-meta__chip--plain">
|
||||
<Eye size={14} aria-hidden="true" />
|
||||
<strong>{formatCount(video.views)}</strong> 次观看
|
||||
</li>
|
||||
{published && <li className="vd-meta__chip">{published}</li>}
|
||||
{published && (
|
||||
<li className="vd-meta__chip vd-meta__chip--plain">
|
||||
<CalendarDays size={14} aria-hidden="true" />
|
||||
{published}
|
||||
</li>
|
||||
)}
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<h1 className="vd-header__title" title={video.title}>
|
||||
{video.title}
|
||||
</h1>
|
||||
</header>
|
||||
);
|
||||
}
|
||||
|
||||
+1451
-133
File diff suppressed because it is too large
Load Diff
@@ -105,7 +105,7 @@ export default function HomePage() {
|
||||
const latest = latestVideos.slice(0, displayCount);
|
||||
|
||||
return (
|
||||
<AppShell>
|
||||
<AppShell mobileAutoHideNav>
|
||||
<div className="container page-section">
|
||||
<PromoStrip />
|
||||
<SearchPanel />
|
||||
|
||||
@@ -84,14 +84,66 @@ export default function VideoDetailPage() {
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<AppShell>
|
||||
<AppShell mobileAutoHideNav>
|
||||
<div className="vd-page">
|
||||
<div className="vd-ambient" aria-hidden="true" />
|
||||
<div className="container vd-page__inner">
|
||||
<div className="vd-skeleton">
|
||||
<div className="vd-skeleton__player" />
|
||||
<div className="vd-skeleton__title" />
|
||||
<div className="vd-skeleton__meta" />
|
||||
<div
|
||||
className="vd-layout vd-skeleton"
|
||||
aria-busy="true"
|
||||
aria-label="视频详情加载中"
|
||||
>
|
||||
<div className="vd-main">
|
||||
<div className="vd-skeleton__player" />
|
||||
|
||||
<div className="vd-skeleton__summary">
|
||||
<div className="vd-skeleton__chips">
|
||||
<span className="vd-skeleton__chip vd-skeleton__chip--source" />
|
||||
<span className="vd-skeleton__chip" />
|
||||
<span className="vd-skeleton__chip vd-skeleton__chip--plain" />
|
||||
<span className="vd-skeleton__chip vd-skeleton__chip--plain" />
|
||||
</div>
|
||||
<div className="vd-skeleton__title" />
|
||||
<div className="vd-skeleton__actions">
|
||||
<span />
|
||||
<span />
|
||||
<span />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="vd-skeleton__info">
|
||||
<span className="vd-skeleton__section-head" />
|
||||
<span className="vd-skeleton__line" />
|
||||
<span className="vd-skeleton__line vd-skeleton__line--short" />
|
||||
<div className="vd-skeleton__tag-row">
|
||||
<span />
|
||||
<span />
|
||||
<span />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<aside className="vd-rail vd-skeleton__rail">
|
||||
<div className="vd-rail__head">
|
||||
<span className="vd-rail__head-icon" aria-hidden="true">
|
||||
<span />
|
||||
<span />
|
||||
</span>
|
||||
<span className="vd-skeleton__rail-head" />
|
||||
</div>
|
||||
<ul className="vd-rail__list vd-skeleton__rail-list">
|
||||
{Array.from({ length: 6 }).map((_, index) => (
|
||||
<li key={index} className="vd-skeleton__rail-item">
|
||||
<span className="vd-skeleton__rail-thumb" />
|
||||
<span className="vd-skeleton__rail-body">
|
||||
<span className="vd-skeleton__rail-title" />
|
||||
<span className="vd-skeleton__rail-title vd-skeleton__rail-title--short" />
|
||||
<span className="vd-skeleton__rail-meta" />
|
||||
</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</aside>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -101,7 +153,7 @@ export default function VideoDetailPage() {
|
||||
|
||||
if (!detail) {
|
||||
return (
|
||||
<AppShell>
|
||||
<AppShell mobileAutoHideNav>
|
||||
<div className="vd-page">
|
||||
<div className="container vd-page__inner">
|
||||
<div className="vd-empty">视频不存在或已被移除</div>
|
||||
@@ -112,7 +164,7 @@ export default function VideoDetailPage() {
|
||||
}
|
||||
|
||||
return (
|
||||
<AppShell>
|
||||
<AppShell mobileAutoHideNav>
|
||||
<div className="vd-page">
|
||||
{/* Ambient 背景层:用海报作模糊底色,叠加渐变过渡到页面背景 */}
|
||||
<div
|
||||
@@ -131,21 +183,25 @@ export default function VideoDetailPage() {
|
||||
<div className="vd-player-wrap">
|
||||
<div className="vd-player">
|
||||
<VideoPlayer
|
||||
id={detail.id}
|
||||
src={detail.videoSrc}
|
||||
poster={detail.poster}
|
||||
previewSrc={detail.previewSrc}
|
||||
title={detail.title}
|
||||
onFirstPlay={handleFirstPlay}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<VideoMetaHeader video={detail} />
|
||||
<section className="vd-summary" aria-label="当前视频">
|
||||
<VideoMetaHeader video={detail} />
|
||||
|
||||
<VideoActions
|
||||
video={detail}
|
||||
onHideVideo={handleHideVideo}
|
||||
hideSaving={hideSaving}
|
||||
/>
|
||||
<VideoActions
|
||||
video={detail}
|
||||
onHideVideo={handleHideVideo}
|
||||
hideSaving={hideSaving}
|
||||
/>
|
||||
</section>
|
||||
|
||||
<VideoInfoPanel
|
||||
video={detail}
|
||||
|
||||
@@ -2971,6 +2971,7 @@
|
||||
display: grid;
|
||||
place-items: center;
|
||||
color: #fff;
|
||||
background: var(--accent);
|
||||
font-size: 13px;
|
||||
font-weight: var(--weight-bold);
|
||||
text-transform: capitalize;
|
||||
@@ -2983,6 +2984,7 @@
|
||||
.admin-drive-card__brand-icon[data-kind="pikpak"] { background: var(--drive-pikpak); }
|
||||
.admin-drive-card__brand-icon[data-kind="wopan"] { background: var(--drive-wopan); }
|
||||
.admin-drive-card__brand-icon[data-kind="onedrive"] { background: var(--drive-onedrive); }
|
||||
.admin-drive-card__brand-icon[data-kind="googledrive"] { background: #4285f4; }
|
||||
.admin-drive-card__brand-icon[data-kind="localstorage"] { background: var(--drive-localstorage); }
|
||||
.admin-drive-card__brand-icon[data-kind="spider91"] { background: var(--accent); }
|
||||
|
||||
@@ -3107,11 +3109,11 @@
|
||||
}
|
||||
|
||||
/* =========================================================
|
||||
* Drive Generation — 3-Column Layout
|
||||
* Drive Generation
|
||||
* ========================================================= */
|
||||
.admin-gen-columns {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, 1fr);
|
||||
grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
|
||||
gap: var(--space-3);
|
||||
margin-bottom: var(--space-4);
|
||||
}
|
||||
|
||||
@@ -10,6 +10,10 @@
|
||||
font-size: var(--font-base);
|
||||
}
|
||||
|
||||
.app-shell__nav-stack {
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
.app-shell__main {
|
||||
flex: 1;
|
||||
width: 100%;
|
||||
|
||||
@@ -179,6 +179,24 @@
|
||||
|
||||
/* ----- 响应式 ----- */
|
||||
@media (max-width: 768px) {
|
||||
.app-shell--mobile-auto-hide-nav .app-shell__nav-stack {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: var(--z-nav);
|
||||
transform: translateY(0);
|
||||
transition: transform 220ms var(--ease-out);
|
||||
will-change: transform;
|
||||
}
|
||||
|
||||
.app-shell--mobile-auto-hide-nav.is-mobile-nav-hidden .app-shell__nav-stack {
|
||||
transform: translateY(-100%);
|
||||
}
|
||||
|
||||
.app-shell--mobile-auto-hide-nav .main-nav {
|
||||
position: relative;
|
||||
z-index: auto;
|
||||
}
|
||||
|
||||
.main-nav__inner {
|
||||
height: 56px;
|
||||
gap: var(--space-3);
|
||||
|
||||
+789
-292
File diff suppressed because it is too large
Load Diff
@@ -65,7 +65,7 @@ test("spider91 upload target uses explicit local-save option instead of auto tar
|
||||
assert.match(combinedSource, /本地保存,不上传/);
|
||||
assert.match(
|
||||
combinedSource,
|
||||
/d\.kind === "pikpak" \|\| d\.kind === "p115" \|\| d\.kind === "p123" \|\| d\.kind === "onedrive"/
|
||||
/d\.kind === "pikpak"[\s\S]*d\.kind === "p115"[\s\S]*d\.kind === "p123"[\s\S]*d\.kind === "onedrive"[\s\S]*d\.kind === "googledrive"/
|
||||
);
|
||||
assert.doesNotMatch(combinedSource, /自动:唯一/);
|
||||
assert.doesNotMatch(combinedSource, /自动模式/);
|
||||
@@ -112,7 +112,7 @@ test("onedrive drive form only exposes required default-app fields", () => {
|
||||
assert.doesNotMatch(fields, /key: "site_id"/);
|
||||
});
|
||||
|
||||
test("googledrive drive form only exposes refresh token", () => {
|
||||
test("googledrive drive form supports online API and custom OAuth client modes", () => {
|
||||
assertDriveTypeOption("googledrive", "Google Drive");
|
||||
|
||||
const match =
|
||||
@@ -123,10 +123,25 @@ test("googledrive drive form only exposes refresh token", () => {
|
||||
const fields = match[1];
|
||||
|
||||
assert.match(fields, /key: "refresh_token"/);
|
||||
assert.doesNotMatch(fields, /key: "access_token"/);
|
||||
assert.match(fields, /key: "use_online_api"/);
|
||||
assert.match(fields, /type: "select"/);
|
||||
assert.match(fields, /defaultValue: "true"/);
|
||||
assert.match(fields, /OpenList 在线 API/);
|
||||
assert.match(fields, /自建 Google OAuth 客户端/);
|
||||
assert.match(fields, /key: "client_id"/);
|
||||
assert.match(fields, /key: "client_secret"/);
|
||||
assert.match(fields, /googleDriveUsesOnlineAPI\(creds\)/);
|
||||
assert.doesNotMatch(fields, /key: "api_url_address"/);
|
||||
assert.doesNotMatch(fields, /key: "client_id"/);
|
||||
assert.doesNotMatch(fields, /key: "client_secret"/);
|
||||
assert.doesNotMatch(fields, /在线 API 模式填写 OpenList 获取的 refresh_token/);
|
||||
assert.doesNotMatch(constantsSource, /请参考OpenList文档中关于谷歌云盘的配置方法。/);
|
||||
assert.doesNotMatch(constantsSource, /选择自建 Google OAuth 客户端后,服务端会直接请求 Google OAuth token 接口续期。/);
|
||||
assert.match(driveFormSource, /<select/);
|
||||
assert.match(driveFormSource, /value=\{form\.creds\[f\.key\] \?\? f\.defaultValue \?\? ""\}/);
|
||||
assert.match(driveFormSource, /className="admin-form-select"/);
|
||||
assert.match(driveFormSource, /ChevronDown/);
|
||||
assert.match(drivesPageSource, /googleDriveUseOnlineAPI/);
|
||||
assert.match(apiSource, /googleDriveUseOnlineAPI\?: boolean/);
|
||||
assert.doesNotMatch(fields, /key: "access_token"/);
|
||||
});
|
||||
|
||||
test("pikpak drive form only exposes account login fields", () => {
|
||||
@@ -176,6 +191,15 @@ test("drive type selector keeps primary source order", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
test("drive cards use configured abbreviations and visible fallback icon colors", () => {
|
||||
assert.match(constantsSource, /googledrive:\s*"GD"/);
|
||||
assert.match(constantsSource, /function driveKindAbbr\(kind: string\)/);
|
||||
assert.match(constantsSource, /\.slice\(0, 2\)\.toUpperCase\(\)/);
|
||||
assert.match(drivesPageSource, /driveKindAbbr\(d\.kind\)/);
|
||||
assert.match(adminCss, /\.admin-drive-card__brand-icon\s*\{[^}]*background:\s*var\(--accent\);/s);
|
||||
assert.match(adminCss, /\.admin-drive-card__brand-icon\[data-kind="googledrive"\]\s*\{\s*background:\s*#4285f4;\s*\}/);
|
||||
});
|
||||
|
||||
test("drive management exposes stop task controls", () => {
|
||||
assert.match(apiSource, /stopDriveTasks/);
|
||||
assert.match(apiSource, /\/drives\/\$\{encodeURIComponent\(id\)\}\/tasks\/stop/);
|
||||
@@ -186,6 +210,37 @@ test("drive management exposes stop task controls", () => {
|
||||
assert.match(drivesPageSource, /停止所有网盘任务/);
|
||||
});
|
||||
|
||||
test("drive rescan reports busy storage tasks instead of queueing duplicates", () => {
|
||||
assert.match(apiSource, /accepted:\s*boolean;\s*message\?:\s*string/);
|
||||
assert.match(apiSource, /scanGenerationStatus\?: DriveGenerationStatus/);
|
||||
assert.match(drivesPageSource, /当前存储有正在进行的任务,请稍后重试/);
|
||||
assert.match(drivesPageSource, /function isDriveBusy\(d: api\.AdminDrive\)/);
|
||||
assert.match(drivesPageSource, /d\.scanGenerationStatus/);
|
||||
assert.match(drivesPageSource, /status\?\.state \|\| "idle"/);
|
||||
assert.match(drivesPageSource, /scanningDriveIdsRef\.current\.has\(d\.id\)/);
|
||||
assert.match(drivesPageSource, /if \(!resp\.accepted\)/);
|
||||
assert.doesNotMatch(drivesPageSource, /disabled=\{!!scanningDriveId\}/);
|
||||
});
|
||||
|
||||
test("nightly scan duplicate trigger uses full-scan busy message", () => {
|
||||
assert.match(apiSource, /status:\s*NightlyJobStatus;\s*message\?:\s*string/);
|
||||
assert.match(drivesPageSource, /当前有全量扫描任务正在进行,请稍后重试/);
|
||||
assert.match(drivesPageSource, /resp\.message \|\| NIGHTLY_BUSY_MESSAGE/);
|
||||
assert.match(constantsSource, /当前有全量扫描任务正在进行,请稍后重试/);
|
||||
});
|
||||
|
||||
test("drive generation panel shows scan or crawler status first", () => {
|
||||
assert.match(driveComponentsSource, /label=\{d\.kind === "spider91" \? "抓取" : "扫盘"\}/);
|
||||
assert.match(driveComponentsSource, /status=\{d\.scanGenerationStatus\}/);
|
||||
assert.match(driveComponentsSource, /showCounts=\{false\}/);
|
||||
assert.match(driveComponentsSource, /label === "抓取" && state === "scanning" \? "抓取中"/);
|
||||
assert.match(driveComponentsSource, /status\?\.scannedCount/);
|
||||
assert.match(driveComponentsSource, /预计新增/);
|
||||
assert.match(apiSource, /scannedCount:\s*number/);
|
||||
assert.match(apiSource, /addedCount:\s*number/);
|
||||
assert.match(constantsSource, /if \(state === "scanning"\) return "扫盘中"/);
|
||||
});
|
||||
|
||||
test("drive detail selection is stored in the URL history", () => {
|
||||
assert.match(drivesPageSource, /useSearchParams/);
|
||||
assert.match(drivesPageSource, /searchParams\.get\("drive"\)/);
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import test from "node:test";
|
||||
|
||||
const actionsSource = readFileSync(
|
||||
new URL("../src/components/VideoActions.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
const detailCss = readFileSync(
|
||||
new URL("../src/styles/video-detail.css", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
|
||||
test("detail dislike does not locally decrement persisted likes", () => {
|
||||
const match = /function handleDislike\(\) \{([\s\S]*?)\n return \(/.exec(
|
||||
actionsSource
|
||||
);
|
||||
assert.ok(match, "handleDislike block should be present");
|
||||
assert.match(match[1], /setDisliked\(true\)/);
|
||||
assert.doesNotMatch(match[1], /setLikes/);
|
||||
});
|
||||
|
||||
test("detail like and dislike buttons are visually separated", () => {
|
||||
assert.doesNotMatch(actionsSource, /vd-actions__divider/);
|
||||
assert.match(
|
||||
detailCss,
|
||||
/\.vd-actions__group\s*\{[^}]*gap:\s*var\(--space-2\)/s
|
||||
);
|
||||
assert.match(
|
||||
detailCss,
|
||||
/\.vd-actions__pill\s*\{[^}]*border:\s*1px solid var\(--border-subtle\)[^}]*border-radius:\s*var\(--radius-sm\)/s
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,155 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import test from "node:test";
|
||||
|
||||
const detailCss = readFileSync(
|
||||
new URL("../src/styles/video-detail.css", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
const playerSource = readFileSync(
|
||||
new URL("../src/components/VideoPlayer.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
const detailPageSource = readFileSync(
|
||||
new URL("../src/pages/VideoDetailPage.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
|
||||
test("detail player poster uses full-frame contain scaling", () => {
|
||||
assert.match(
|
||||
detailCss,
|
||||
/\.video-player \.art-poster\s*\{[^}]*background-position:\s*center[^}]*background-repeat:\s*no-repeat[^}]*background-size:\s*contain/s
|
||||
);
|
||||
});
|
||||
|
||||
test("detail player does not keep playback resume state", () => {
|
||||
assert.doesNotMatch(playerSource, /ResumePrompt/);
|
||||
assert.doesNotMatch(playerSource, /PlaybackRecord/);
|
||||
assert.doesNotMatch(playerSource, /PLAYBACK_KEY_PREFIX/);
|
||||
assert.doesNotMatch(playerSource, /maybeOfferResume/);
|
||||
assert.doesNotMatch(playerSource, /savePlaybackRecord/);
|
||||
assert.doesNotMatch(playerSource, /clearPlaybackRecord/);
|
||||
assert.doesNotMatch(playerSource, /video-player__resume/);
|
||||
assert.doesNotMatch(detailCss, /video-player__resume/);
|
||||
});
|
||||
|
||||
test("detail loading skeleton matches current desktop video page layout", () => {
|
||||
assert.match(detailPageSource, /className="vd-layout vd-skeleton"/);
|
||||
assert.match(detailPageSource, /className="vd-skeleton__summary"/);
|
||||
assert.match(detailPageSource, /className="vd-skeleton__info"/);
|
||||
assert.match(detailPageSource, /className="vd-rail vd-skeleton__rail"/);
|
||||
assert.match(detailPageSource, /Array\.from\(\{ length: 6 \}\)/);
|
||||
assert.doesNotMatch(detailPageSource, /className="vd-skeleton__meta"/);
|
||||
assert.match(
|
||||
detailCss,
|
||||
/\.vd-skeleton__player\s*\{[^}]*aspect-ratio:\s*16 \/ 9[^}]*border-radius:\s*0/s
|
||||
);
|
||||
assert.match(
|
||||
detailCss,
|
||||
/\.vd-skeleton__summary,\s*\.vd-skeleton__info\s*\{[^}]*border:\s*1px solid var\(--border-default\)[^}]*border-radius:\s*var\(--radius-md\)/s
|
||||
);
|
||||
assert.match(
|
||||
detailCss,
|
||||
/\.vd-skeleton__rail-item\s*\{[^}]*grid-template-columns:\s*150px minmax\(0,\s*1fr\)/s
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
detailCss,
|
||||
/\.vd-skeleton__player\s*\{[^}]*box-shadow:\s*var\(--shadow-lg\)/s
|
||||
);
|
||||
});
|
||||
|
||||
test("detail video title uses a restrained size", () => {
|
||||
assert.match(
|
||||
detailCss,
|
||||
/\.vd-header__title\s*\{[^}]*font-size:\s*var\(--font-xl\)[^}]*line-height:\s*1\.34/s
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
detailCss,
|
||||
/\.vd-header__title\s*\{[^}]*font-size:\s*var\(--font-2xl\)/s
|
||||
);
|
||||
assert.match(
|
||||
detailCss,
|
||||
/@media \(max-width:\s*480px\)\s*\{[\s\S]*\.vd-header__title\s*\{[^}]*font-size:\s*var\(--font-base\)/s
|
||||
);
|
||||
});
|
||||
|
||||
test("detail player uses custom mobile gestures instead of ArtPlayer native gestures", () => {
|
||||
assert.match(playerSource, /gesture:\s*false/);
|
||||
assert.match(playerSource, /fastForward:\s*false/);
|
||||
assert.match(playerSource, /function bindMobilePlayerGestures/);
|
||||
assert.match(playerSource, /let suppressNextClick = false/);
|
||||
assert.match(playerSource, /endPress\(true\)/);
|
||||
assert.match(playerSource, /event\.stopImmediatePropagation\(\)/);
|
||||
assert.match(playerSource, /addEventListener\("click", handleClick, true\)/);
|
||||
assert.match(playerSource, /state\.mode = "seek"/);
|
||||
assert.match(playerSource, /state\.side === "right" \? "volume" : "brightness"/);
|
||||
assert.doesNotMatch(playerSource, /function isPlayerLandscapeExpanded/);
|
||||
assert.doesNotMatch(playerSource, /getEffectivePlayerOrientation\(art\) === "landscape"/);
|
||||
assert.match(playerSource, /if \(!isPlayerExpanded\(art\)\) \{\s*resetGesture\(\);/);
|
||||
assert.match(playerSource, /if \(!isPlayerExpanded\(art\)\) return;\s*onGestureHud\(seekGestureLabel/);
|
||||
assert.match(playerSource, /const FAST_RATE_CLASS = "art-fast-rate-active"/);
|
||||
assert.match(playerSource, /const FAST_RATE_HINT_CLASS = "video-player__art-rate-hint"/);
|
||||
assert.match(playerSource, /const PLAYER_GESTURE_HUD_CLASS = "video-player__art-gesture-hud"/);
|
||||
assert.match(playerSource, /setPlayerFastRateHint\(art, active\)/);
|
||||
assert.match(playerSource, /player\.appendChild\(hint\)/);
|
||||
assert.match(playerSource, /showPlayerGestureHud\(art, "volume", formatPercent\(normalized\)\)/);
|
||||
assert.match(playerSource, /showPlayerGestureHud\(art, "brightness", formatBrightnessPercent\(nextBrightness\)\)/);
|
||||
assert.match(playerSource, /stroke-width="1\.7"/);
|
||||
assert.match(playerSource, /M15\.4 9\.2a4\.2 4\.2 0 0 1 0 5\.6/);
|
||||
assert.match(playerSource, /M4\.8 9\.7h3l4\.3-3\.6v11\.8l-4\.3-3\.6h-3/);
|
||||
assert.doesNotMatch(playerSource, /stroke-width="2\.2"/);
|
||||
assert.doesNotMatch(playerSource, /onGestureHud\(`音量 /);
|
||||
assert.doesNotMatch(playerSource, /onGestureHud\(`亮度 /);
|
||||
assert.match(playerSource, /fullscreen:\s*true/);
|
||||
assert.match(playerSource, /fullscreenWeb:\s*!enableOrientationControl/);
|
||||
assert.doesNotMatch(playerSource, /addTextTrack\("captions", "Playback rate"/);
|
||||
assert.doesNotMatch(playerSource, /new VTTCue\(/);
|
||||
assert.doesNotMatch(playerSource, /onGestureHud\(`\$\{FAST_RATE\}x`/);
|
||||
assert.match(playerSource, /addEventListener\("touchmove", handleTouchMove, \{ passive: false \}\)/);
|
||||
});
|
||||
|
||||
test("detail player fullscreen long-press rate hint lives inside ArtPlayer", () => {
|
||||
assert.match(
|
||||
detailCss,
|
||||
/\.video-player__rate-hint,\s*\.video-player__art-rate-hint\s*\{[\s\S]*position:\s*absolute[\s\S]*top:\s*12px/s
|
||||
);
|
||||
assert.match(
|
||||
detailCss,
|
||||
/\.video-player__art-rate-hint\s*\{[^}]*z-index:\s*130/s
|
||||
);
|
||||
assert.match(
|
||||
detailCss,
|
||||
/\.art-video-player\.art-fullscreen \.video-player__art-rate-hint,[\s\S]*\.art-video-player\.art-fullscreen-web \.video-player__art-rate-hint,[\s\S]*position:\s*fixed/s
|
||||
);
|
||||
});
|
||||
|
||||
test("detail player mobile brightness gesture only filters the video surface", () => {
|
||||
assert.match(
|
||||
detailCss,
|
||||
/\.video-player \.art-video,\s*\.video-player \.art-poster\s*\{[^}]*filter:\s*brightness\(var\(--video-player-brightness, 1\)\)/s
|
||||
);
|
||||
assert.match(
|
||||
detailCss,
|
||||
/@media \(hover: none\) and \(pointer: coarse\)\s*\{[\s\S]*\.video-player \.art-video-player,[\s\S]*touch-action:\s*pan-y/s
|
||||
);
|
||||
assert.match(
|
||||
detailCss,
|
||||
/\.video-player \.art-video-player\.art-fullscreen,[\s\S]*\.video-player \.art-video-player\.art-fullscreen-web,[\s\S]*touch-action:\s*none/s
|
||||
);
|
||||
assert.match(
|
||||
detailCss,
|
||||
/\.video-player__art-gesture-hud\s*\{[^}]*top:\s*16%[^}]*background:\s*rgba\(18,\s*18,\s*20,\s*0\.8\)[^}]*font-size:\s*18px/s
|
||||
);
|
||||
assert.match(
|
||||
detailCss,
|
||||
/\.video-player__art-gesture-hud-icon\s*\{[^}]*width:\s*18px[^}]*height:\s*18px[^}]*transform:\s*translateY\(-1px\)/s
|
||||
);
|
||||
assert.match(
|
||||
detailCss,
|
||||
/\.video-player__art-gesture-hud-icon svg\s*\{[^}]*width:\s*18px[^}]*height:\s*18px/s
|
||||
);
|
||||
assert.match(
|
||||
detailCss,
|
||||
/\.art-video-player\.art-fullscreen \.video-player__art-gesture-hud,[\s\S]*\.art-video-player\.art-manual-orientation \.video-player__art-gesture-hud\s*\{[^}]*position:\s*fixed/s
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user