mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-15 16:55:42 +08:00
Compare commits
28 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 91352ac681 | |||
| d70cea7320 | |||
| 6654a1b730 | |||
| bcc887e088 | |||
| 2c0cfe1d15 | |||
| a276e6b32d | |||
| fa4ea469c3 | |||
| 92885748fd | |||
| 093724a59d | |||
| 9cd30c8059 | |||
| bec6d9496c | |||
| f187302b8e | |||
| 739baf1294 | |||
| af18bbbf4c | |||
| 309b621084 | |||
| 286329c446 | |||
| 1d5b5c2495 | |||
| fac60b0054 | |||
| 19a939e80f | |||
| 16a2a7e03c | |||
| b9b6c5e098 | |||
| 4200919774 | |||
| 33d970a322 | |||
| 59e9b435a0 | |||
| dcda0e2e36 | |||
| 87709792f1 | |||
| 655da05b94 | |||
| 674a92be16 |
@@ -68,6 +68,7 @@ import time
|
||||
import random
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import html
|
||||
from urllib.parse import urljoin, unquote, urlparse
|
||||
@@ -80,6 +81,28 @@ except ImportError:
|
||||
print("请运行: pip install beautifulsoup4 lxml")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def prefer_ipv4_for_plain_socks5_proxy():
|
||||
"""PySocks may pick IPv6 first for socks5://; some SOCKS5 servers only accept IPv4."""
|
||||
proxy_envs = (
|
||||
os.environ.get("HTTPS_PROXY", ""),
|
||||
os.environ.get("HTTP_PROXY", ""),
|
||||
os.environ.get("https_proxy", ""),
|
||||
os.environ.get("http_proxy", ""),
|
||||
)
|
||||
uses_plain_socks5 = any(v.strip().lower().startswith("socks5://") for v in proxy_envs)
|
||||
if not uses_plain_socks5 or getattr(socket, "_spider91_ipv4_first", False):
|
||||
return
|
||||
|
||||
original_getaddrinfo = socket.getaddrinfo
|
||||
|
||||
def getaddrinfo_ipv4_first(*args, **kwargs):
|
||||
infos = original_getaddrinfo(*args, **kwargs)
|
||||
return sorted(infos, key=lambda info: 0 if info[0] == socket.AF_INET else 1)
|
||||
|
||||
socket.getaddrinfo = getaddrinfo_ipv4_first
|
||||
socket._spider91_ipv4_first = True
|
||||
|
||||
# ===================== 配置区域 =====================
|
||||
BASE_URL = "https://www.91porn.com/v.php"
|
||||
LIST_PARAMS = {
|
||||
@@ -757,13 +780,15 @@ def main():
|
||||
"日志改走 stderr。配合 backend 边读边下载使用。")
|
||||
|
||||
args, _ = parser.parse_known_args()
|
||||
cli_out = sys.stderr if args.stream_output else sys.stdout
|
||||
prefer_ipv4_for_plain_socks5_proxy()
|
||||
|
||||
print("""
|
||||
================================================
|
||||
91porn 视频爬虫启动中...
|
||||
================================================
|
||||
按 Ctrl+C 可随时中断并保存进度
|
||||
""")
|
||||
""", file=cli_out)
|
||||
|
||||
# 加载已知 ID(来自 backend 的 catalog 已入库列表;兼容旧参数名)
|
||||
seen_viewkeys = []
|
||||
@@ -775,9 +800,9 @@ def main():
|
||||
if line:
|
||||
seen_viewkeys.append(line)
|
||||
except FileNotFoundError:
|
||||
print(f"警告: --seen-viewkeys-file 不存在: {args.seen_viewkeys_file}")
|
||||
print(f"警告: --seen-viewkeys-file 不存在: {args.seen_viewkeys_file}", file=cli_out)
|
||||
except Exception as e:
|
||||
print(f"警告: 读取 --seen-viewkeys-file 失败: {e}")
|
||||
print(f"警告: 读取 --seen-viewkeys-file 失败: {e}", file=cli_out)
|
||||
|
||||
# 决定运行模式
|
||||
if args.target_new is not None:
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Common Commands
|
||||
|
||||
### Full Local Development
|
||||
|
||||
```bash
|
||||
npm install
|
||||
./start.sh
|
||||
```
|
||||
|
||||
`./start.sh` starts the Go backend on `127.0.0.1:9192` and the frontend on `0.0.0.0:9191`. By default it builds the frontend and runs Vite preview mode. Use hot reload with:
|
||||
|
||||
```bash
|
||||
FRONTEND_MODE=dev ./start.sh --restart
|
||||
```
|
||||
|
||||
Useful variants:
|
||||
|
||||
```bash
|
||||
./start.sh --status
|
||||
./start.sh --restart
|
||||
./start.sh --stop
|
||||
```
|
||||
|
||||
### Frontend
|
||||
|
||||
Run from the repository root:
|
||||
|
||||
```bash
|
||||
npm run dev # Vite dev server; default port comes from Vite unless overridden
|
||||
npm run dev:raw # Vite dev server on 127.0.0.1:5173
|
||||
npm run build # tsc -b && vite build
|
||||
npm run preview # Vite preview; vite.config.ts uses port 9191
|
||||
npm run lint # TypeScript no-emit check
|
||||
npm test # node --import tsx --test tests/*.test.ts
|
||||
```
|
||||
|
||||
Run one frontend test:
|
||||
|
||||
```bash
|
||||
node --import tsx --test tests/previewIntent.test.ts
|
||||
```
|
||||
|
||||
### Backend
|
||||
|
||||
Run from `backend/` unless noted:
|
||||
|
||||
```bash
|
||||
go run ./cmd/server
|
||||
go test ./... -count=1
|
||||
go build -o video-server ./cmd/server
|
||||
```
|
||||
|
||||
Run one backend package or test:
|
||||
|
||||
```bash
|
||||
go test ./internal/scanner -count=1
|
||||
go test ./internal/scanner -run TestParse -count=1
|
||||
```
|
||||
|
||||
The backend requires Go 1.23+ and uses vendored dependencies in `backend/vendor/`, so keep `go mod vendor` in sync after dependency changes.
|
||||
|
||||
### Release and Deployment
|
||||
|
||||
```bash
|
||||
scripts/build-release.sh # builds Linux amd64/arm64 release tarballs into release/
|
||||
sudo bash install.sh # prebuilt installer flow used by README
|
||||
sudo bash deploy.sh # build from current checkout and install systemd services
|
||||
```
|
||||
|
||||
Docker uses the root `Dockerfile` and `docker-compose.yml`. The runtime image exposes port `9191` and stores persistent data under `/opt/video-site-91/data`.
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
This is a private video aggregation site with a React/Vite frontend and a Go backend.
|
||||
|
||||
### Frontend
|
||||
|
||||
The frontend is a React 18 SPA under `src/`. `src/main.tsx` mounts `BrowserRouter`, `ToastProvider`, and `AuthProvider`, then renders `src/App.tsx`. `App.tsx` defines the public app routes (`/`, `/list`, `/shorts`, `/upload`, `/video/:id`) and admin routes under `/admin`; both main-site and admin pages are wrapped in `RequireAuth`, while `/login` is public.
|
||||
|
||||
Frontend API calls are split by surface:
|
||||
|
||||
- `src/data/videos.ts` calls the main authenticated API under `/api` and upload/proxy-related endpoints.
|
||||
- `src/admin/api.ts` is the admin API client for `/admin/api`, always sending cookies and raising `UnauthorizedError` on `401`.
|
||||
|
||||
`vite.config.ts` proxies `/api`, `/p`, and `/admin/api` to `http://127.0.0.1:9192`, with frontend dev/preview served on port `9191` by default. The alias `@` maps to `src`.
|
||||
|
||||
Styling is plain CSS loaded from `src/main.tsx` in token/base/layout/navigation/search/video/admin layers. Shared UI lives in `src/components`, page-level screens in `src/pages`, and admin screens in `src/admin`.
|
||||
|
||||
### Backend
|
||||
|
||||
The backend entrypoint is `backend/cmd/server/main.go`. It loads `config.yaml` or `VIDEO_CONFIG`, creates the SQLite catalog and preview directories, builds the app state, registers API routes, starts the nightly runner, and then asynchronously attaches configured external drives so slow upstream login checks do not block port binding.
|
||||
|
||||
Important backend packages:
|
||||
|
||||
- `internal/config`: YAML config loading and first-run admin credential setup.
|
||||
- `internal/catalog`: SQLite catalog, schema migration, video metadata, settings, tags, drive records, generation status, and deduplication state. It opens SQLite with WAL and a busy timeout.
|
||||
- `internal/drives`: provider abstraction. Implementations include `quark`, `p115`, `pikpak`, `wopan`, `onedrive`, `localstorage`, `localupload`, and `spider91`.
|
||||
- `internal/scanner`: recursively lists drive directories, parses filenames/tags, upserts catalog videos, applies skip-directory rules, and enqueues newly discovered videos.
|
||||
- `internal/preview`: ffprobe/ffmpeg thumbnail and teaser generation workers. Generated assets are local files under the configured preview directory.
|
||||
- `internal/fingerprint`: asynchronous sampled SHA-256 worker used for cross-drive duplicate detection.
|
||||
- `internal/proxy`: `/p/*` media serving. Some providers redirect with `302` to signed CDN URLs, while providers requiring backend-held headers are reverse-proxied with Range support.
|
||||
- `internal/api`: main API and admin API route handlers.
|
||||
- `internal/nightly`: daily pipeline for drive scans, spider91 crawl, migration, queue drain, and duplicate asset cleanup.
|
||||
- `internal/spider91migrate`: migration from spider91 downloads to a configured cloud drive.
|
||||
|
||||
### Runtime Flow
|
||||
|
||||
1. Admin adds or edits drives through `/admin/drives`, which persists drive config in the catalog.
|
||||
2. The server attaches the drive implementation into the proxy registry and can trigger scans.
|
||||
3. Scans convert provider files into catalog video rows, parse titles/authors/tags from filenames, and queue preview/fingerprint work.
|
||||
4. The frontend lists videos through `/api/home`, `/api/list`, `/api/video/:id`, and streams media through `/p/*` endpoints.
|
||||
5. The nightly runner performs the scheduled end-to-end maintenance pipeline; admins can trigger it manually through `/admin/api/jobs/nightly/run`.
|
||||
|
||||
### Configuration and Data
|
||||
|
||||
Backend defaults come from `backend/config.example.yaml`. On first backend start, `config.yaml` is created automatically if missing. Default local development paths are:
|
||||
|
||||
- Backend listen address: `127.0.0.1:9192`
|
||||
- SQLite DB: `backend/data/video-site.db`
|
||||
- Generated previews/thumbs: `backend/data/previews`
|
||||
|
||||
Docker and installer deployments rewrite config paths so data lives under `/opt/video-site-91/data` or the mounted `./data` directory.
|
||||
|
||||
`VIDEO_FRONTEND_DIR` controls where the Go server looks for built frontend assets. If unset, it serves `./dist` when present. Backend routes (`/api`, `/admin/api`, `/p`) are excluded from the SPA fallback.
|
||||
|
||||
## Notes for Changes
|
||||
|
||||
- Main-site API routes and proxy routes require authentication; only login/setup and `/api/settings/theme` are intentionally public.
|
||||
- When adding a new drive provider, implement `internal/drives.Drive`, persist any needed config through catalog/admin APIs, attach it in `cmd/server`, and decide whether `/p/stream` should redirect or reverse-proxy in `internal/proxy`.
|
||||
- Generated thumbnails and teasers are local runtime assets; do not treat them as source files.
|
||||
- Frontend tests use Node's built-in test runner with `tsx`; TypeScript linting only checks `src` through the root `tsconfig.json`.
|
||||
@@ -36,6 +36,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
python3-bs4 \
|
||||
python3-lxml \
|
||||
python3-requests \
|
||||
python3-socks \
|
||||
tar \
|
||||
tzdata \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
@@ -19,8 +19,8 @@
|
||||
|
||||
## 功能特性
|
||||
|
||||
- **多后端支持** — 兼容 115 云盘、PikPak 云盘、OneDrive和本地存储
|
||||
- **零带宽消耗** — 云盘视频采用 302 重定向播放,不占用服务器带宽
|
||||
- **多后端支持** — 兼容 115 云盘、PikPak 云盘、OneDrive、Google Drive 和本地存储
|
||||
- **低带宽播放** — 115 云盘、PikPak 云盘、OneDrive 都支持302模式,在线播放视频时,不占用服务器带宽,播放体验不受服务器带宽影响;Google Drive 不支持302模式,走服务器中转,观看体验会受服务器带宽影响
|
||||
- **封面 & 预览片段** — 自动为每个视频生成封面图和预览片段,首页快速选片
|
||||
- **91 爬虫** — 内置爬虫,支持抓取 91 本月最热视频
|
||||
- **双主题** — 黑黄经典主题 / 粉白清新主题,随时切换
|
||||
@@ -121,10 +121,7 @@ services:
|
||||
- ./data:/opt/video-site-91/data
|
||||
restart: unless-stopped
|
||||
```
|
||||
|
||||
`stable` 只会在发布 `v*` 正式 Release 时更新,不会跟随 `main` 分支开发镜像变化。
|
||||
升级到最新正式版:
|
||||
|
||||
创建yml文件后运行下面指令
|
||||
```bash
|
||||
docker compose pull
|
||||
docker compose up -d
|
||||
@@ -133,7 +130,7 @@ docker compose up -d
|
||||
如果想固定某个 Release 版本,可以改成明确的 tag,例如:
|
||||
|
||||
```yaml
|
||||
image: ghcr.io/nianzhibai/91:v0.0.4
|
||||
image: ghcr.io/nianzhibai/91:v0.0.6
|
||||
```
|
||||
|
||||
或直接拉取仓库内置配置:
|
||||
|
||||
+16
-7
@@ -2,7 +2,7 @@
|
||||
|
||||
视频聚合站的 Go 后端。提供三件事:
|
||||
|
||||
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive / 本地存储)
|
||||
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive / Google Drive / 本地存储)
|
||||
2. 视频元数据目录(SQLite)+ 扫描 + teaser 预生成
|
||||
3. REST API(前台)+ 管理后台 + 直链代理
|
||||
4. 标签池、视频隐藏、按网盘统计和详情页来源网盘类型展示能力
|
||||
@@ -21,6 +21,7 @@ internal/
|
||||
pikpak/ PikPak(自己实现,参考 OpenList pikpak)
|
||||
wopan/ 联通沃盘(壳子 + OpenListTeam/wopan-sdk-go)
|
||||
onedrive/ OneDrive(OpenList 在线续期 + Microsoft Graph 文件接口)
|
||||
googledrive/ Google Drive(OpenList 在线续期 + Google Drive API;播放走后端代理)
|
||||
localstorage/ 本地目录扫描(服务器已有视频目录)
|
||||
scanner/ 扫目录 → 落库
|
||||
preview/ ffmpeg 抽封面和生成多段 teaser
|
||||
@@ -92,7 +93,6 @@ go run ./cmd/server 后端 9192
|
||||
"kind": "quark",
|
||||
"name": "我的夸克盘",
|
||||
"rootId": "0",
|
||||
"scanRootId": "0",
|
||||
"credentials": {
|
||||
"cookie": "粘贴浏览器 F12 复制的 pan.quark.cn Cookie"
|
||||
}
|
||||
@@ -109,6 +109,7 @@ go run ./cmd/server 后端 9192
|
||||
| pikpak | `username`、`password`(token、验证码和设备 ID 由服务端自动处理并保存) |
|
||||
| wopan | `access_token`、`refresh_token`,可选 `family_id` |
|
||||
| onedrive | `refresh_token` |
|
||||
| googledrive | `refresh_token` |
|
||||
| localstorage | `path`(服务器上的已有视频目录,如 `/mnt/videos`) |
|
||||
|
||||
### PikPak 速度说明
|
||||
@@ -119,16 +120,24 @@ go run ./cmd/server 后端 9192
|
||||
|
||||
OneDrive 按 OpenList 默认应用方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。后台新建 OneDrive 时只需要填 OpenList 代刷得到的 `refresh_token`;服务端会默认挂载根目录并自动回写新 token。
|
||||
|
||||
Google Drive 按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/renewapi` 刷新 token。后台新建 Google Drive 时只需要填 OpenList Google Drive 获取到的 `refresh_token`。Google Drive 下载地址必须携带 `Authorization` 头,浏览器不能直接 302 使用,所以本站会由后端代理 `/p/stream` 播放,不加入零带宽 302 白名单。
|
||||
|
||||
## 文件名约定
|
||||
|
||||
扫描器按以下顺序解析文件名:
|
||||
扫描器按以下顺序解析文件名,用于提取标题和作者:
|
||||
|
||||
1. `[tag1,tag2] 标题 - 作者.mp4`
|
||||
2. `[tag1,tag2] 标题.mp4`
|
||||
1. `[前缀] 标题 - 作者.mp4`
|
||||
2. `[前缀] 标题.mp4`
|
||||
3. `标题 - 作者.mp4`
|
||||
4. `标题.mp4`
|
||||
|
||||
标签分隔符支持 `, , 、` 和空格。解析结果会和系统标签池匹配,常见番号类噪声会归并到 `AV` 等系统标签,避免把每个番号都变成独立标签。解析结果可在管理后台覆盖。
|
||||
开头的 `[前缀]` 只会从标题里剥离,不会按分隔符作为任意标签入库。视频标签来自三类规则:
|
||||
|
||||
1. 文件名、作者和目录名命中系统标签或已有标签的标签名 / 别名。
|
||||
2. 符合条件的目录名会自动创建 `collection` 合集标签,并给同目录视频打上该标签。
|
||||
3. 常见番号类噪声会统一归并到 `AV`,避免把每个番号都变成独立标签。
|
||||
|
||||
当前内置系统标签为:`后入`、`奶子`、`口交`、`臀`、`人妻`、`女大`、`AV`。解析结果可在管理后台覆盖;手动保存后,该视频会标记为人工标签,后续扫描不会再自动覆盖。
|
||||
|
||||
## 视频去重
|
||||
|
||||
@@ -146,7 +155,7 @@ OneDrive 按 OpenList 默认应用方式调用 `https://api.oplist.org/onedrive/
|
||||
|
||||
- `/admin/drives`:新增、编辑、删除网盘,触发扫描。
|
||||
- `/admin/videos`:按网盘筛选视频,每页 100 条分页,查看各网盘 Teaser 统计,编辑标题/作者/分类/标签,单条或全量重生 teaser。
|
||||
- `/admin/tags`:新增标签并用内置规则自动匹配已有视频。
|
||||
- `/admin/tags`:新增标签并用内置规则自动匹配已有视频;删除非系统标签时会从所有视频上同步移除该标签。
|
||||
- 播放页视频信息会展示来源网盘类型;同时提供“不再展示”,点击后会把视频标记为全局隐藏。隐藏视频不会再出现在首页、列表、搜索、相关推荐和详情接口中。目前没有管理后台恢复入口,如需恢复可把数据库里对应视频的 `hidden` 字段改回 `0`。
|
||||
|
||||
## Teaser 生成
|
||||
|
||||
@@ -67,3 +67,40 @@ func TestFrontendHandlerDoesNotSwallowBackendRoutes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveFrontendDirFallsBackToParentDist(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
backendDir := filepath.Join(workspace, "backend")
|
||||
distDir := filepath.Join(workspace, "dist")
|
||||
if err := os.MkdirAll(backendDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir backend: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(distDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir dist: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(distDir, "index.html"), []byte("<html>app</html>"), 0o644); err != nil {
|
||||
t.Fatalf("write index: %v", err)
|
||||
}
|
||||
|
||||
oldWD, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("getwd: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := os.Chdir(oldWD); err != nil {
|
||||
t.Fatalf("restore wd: %v", err)
|
||||
}
|
||||
})
|
||||
t.Setenv("VIDEO_FRONTEND_DIR", "")
|
||||
if err := os.Chdir(backendDir); err != nil {
|
||||
t.Fatalf("chdir backend: %v", err)
|
||||
}
|
||||
|
||||
got, ok := resolveFrontendDir()
|
||||
if !ok {
|
||||
t.Fatal("resolveFrontendDir ok = false, want true")
|
||||
}
|
||||
if got != "../dist" {
|
||||
t.Fatalf("frontend dir = %q, want ../dist", got)
|
||||
}
|
||||
}
|
||||
|
||||
+122
-26
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/config"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/googledrive"
|
||||
"github.com/video-site/backend/internal/drives/localstorage"
|
||||
"github.com/video-site/backend/internal/drives/localupload"
|
||||
"github.com/video-site/backend/internal/drives/onedrive"
|
||||
@@ -181,6 +182,9 @@ func main() {
|
||||
OnRegenFailedThumbnails: func(driveID string) {
|
||||
go app.regenFailedThumbnails(ctx, driveID)
|
||||
},
|
||||
OnRegenFailedFingerprints: func(driveID string) {
|
||||
go app.regenFailedFingerprints(ctx, driveID)
|
||||
},
|
||||
GetDriveGenerationStatuses: func() map[string]api.DriveGenerationStatuses {
|
||||
return app.driveGenerationStatuses()
|
||||
},
|
||||
@@ -204,10 +208,14 @@ func main() {
|
||||
SetSpider91UploadDriveID: func(id string) error {
|
||||
return app.SetSpider91UploadDriveID(ctx, id)
|
||||
},
|
||||
OnRunNightlyJob: func() {
|
||||
OnRunNightlyJob: func() bool {
|
||||
if app.nightlyRunner != nil {
|
||||
app.nightlyRunner.TriggerNow()
|
||||
return app.nightlyRunner.TriggerNow()
|
||||
}
|
||||
return false
|
||||
},
|
||||
GetNightlyJobStatus: func() api.NightlyJobStatus {
|
||||
return app.nightlyJobStatus()
|
||||
},
|
||||
ListDriveDirChildren: func(reqCtx context.Context, driveID, parentID string) ([]api.DriveDirEntry, error) {
|
||||
return app.listDriveDirChildren(reqCtx, driveID, parentID)
|
||||
@@ -416,6 +424,27 @@ func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) erro
|
||||
return a.cat.SetSetting(ctx, "spider91.upload_drive_id", driveID)
|
||||
}
|
||||
|
||||
func (a *App) nightlyJobStatus() api.NightlyJobStatus {
|
||||
if a.nightlyRunner == nil {
|
||||
return api.NightlyJobStatus{State: "idle"}
|
||||
}
|
||||
status := a.nightlyRunner.Status()
|
||||
return api.NightlyJobStatus{
|
||||
State: status.State,
|
||||
Running: status.Running,
|
||||
Queued: status.Queued,
|
||||
StartedAt: formatOptionalRFC3339(status.StartedAt),
|
||||
LastFinishedAt: formatOptionalRFC3339(status.LastFinishedAt),
|
||||
}
|
||||
}
|
||||
|
||||
func formatOptionalRFC3339(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return ""
|
||||
}
|
||||
return t.Format(time.RFC3339)
|
||||
}
|
||||
|
||||
// isSpider91UploadKind 是 spider91 迁移目标盘的 allowlist。
|
||||
// 与 spider91migrate.adaptUploadTarget 的支持范围保持一致。
|
||||
func isSpider91UploadKind(kind string) bool {
|
||||
@@ -634,6 +663,27 @@ func (a *App) attachDriveUnlocked(ctx context.Context, d *catalog.Drive) error {
|
||||
_ = a.cat.UpsertDrive(ctx, d)
|
||||
},
|
||||
})
|
||||
case googledrive.Kind:
|
||||
drv = googledrive.New(googledrive.Config{
|
||||
ID: d.ID,
|
||||
RootID: d.RootID,
|
||||
AccessToken: d.Credentials["access_token"],
|
||||
RefreshToken: d.Credentials["refresh_token"],
|
||||
ClientID: d.Credentials["client_id"],
|
||||
ClientSecret: d.Credentials["client_secret"],
|
||||
UseOnlineAPI: parseBoolDefault(d.Credentials["use_online_api"], true),
|
||||
RenewAPIURL: d.Credentials["api_url_address"],
|
||||
OAuthURL: d.Credentials["oauth_url"],
|
||||
APIBaseURL: d.Credentials["api_base_url"],
|
||||
OnTokenUpdate: func(access, refresh string) {
|
||||
if d.Credentials == nil {
|
||||
d.Credentials = make(map[string]string)
|
||||
}
|
||||
d.Credentials["access_token"] = access
|
||||
d.Credentials["refresh_token"] = refresh
|
||||
_ = a.cat.UpsertDrive(ctx, d)
|
||||
},
|
||||
})
|
||||
case localstorage.Kind:
|
||||
drv = localstorage.New(localstorage.Config{
|
||||
ID: d.ID,
|
||||
@@ -807,13 +857,14 @@ func (a *App) attachSpider91Crawler(d *catalog.Drive, drv *spider91.Driver) {
|
||||
a.spider91Crawlers[driveID] = c
|
||||
a.mu.Unlock()
|
||||
|
||||
// 确保 "91porn" 系统标签存在,并把已入库的 spider91 视频按 author 字段
|
||||
// 匹配补打这个标签(CreateTagAndClassify 内部对所有视频走一遍 classify)。
|
||||
// 重复调用是幂等的:tags 用 INSERT OR IGNORE,video_tags 也是 INSERT OR IGNORE。
|
||||
// 确保 "91porn" 系统标签存在,并按 spider91 来源前缀给历史视频补打。
|
||||
// 不能只靠文本匹配:老版本入库的视频可能没有 author/tags 字段,但 id 前缀
|
||||
// "spider91-<driveID>-" 会一直保留,即使后续迁移到 PikPak/115 也不变。
|
||||
bgCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
go func() {
|
||||
defer cancel()
|
||||
if _, err := a.cat.CreateTagAndClassify(bgCtx, spider91.DefaultTag, nil, "system"); err != nil {
|
||||
prefix := "spider91-" + driveID + "-"
|
||||
if _, err := a.cat.EnsureTagForVideoIDPrefix(bgCtx, prefix, spider91.DefaultTag, nil, "system"); err != nil {
|
||||
log.Printf("[spider91] ensure %q tag: %v", spider91.DefaultTag, err)
|
||||
}
|
||||
}()
|
||||
@@ -1001,9 +1052,8 @@ func (a *App) detachDrive(id string) {
|
||||
// listDriveDirChildren 实现 AdminServer.ListDriveDirChildren:
|
||||
// 列指定 drive 在 parentID 下的直接子目录,仅返回目录条目(IsDir=true),文件忽略。
|
||||
//
|
||||
// parentID 为空时使用 drive 实例的 RootID(),与扫描起点保持一致 —— 但有意不
|
||||
// 用 ScanRootID:用户在"设置跳过目录"弹窗里浏览的是整个网盘逻辑根,方便从 0
|
||||
// 起逐层挑跳过点;ScanRootID 仅用于实际扫描起点。
|
||||
// parentID 为空时使用 drive 实例的 RootID()。用户在"设置跳过目录"弹窗里
|
||||
// 浏览的是整个网盘逻辑根,方便从根目录起逐层挑跳过点。
|
||||
//
|
||||
// 性能优化:p115 的 Driver.List 走 SDK 的 ListWithLimit,会把目录里全部文件 +
|
||||
// 目录分页拉完才返回;某些 115 根目录累积了几万个视频,单次列目录可能卡几十
|
||||
@@ -1113,7 +1163,7 @@ func (a *App) runScan(ctx context.Context, driveID string) {
|
||||
}
|
||||
}
|
||||
|
||||
// 使用 drive 的 scan_root_id,否则 root_id;同时把 admin 配置的 SkipDirIDs
|
||||
// 扫描入口固定使用 drive 的 root_id;同时把 admin 配置的 SkipDirIDs
|
||||
// 传给 scanner(命中即不递归)。
|
||||
d, err := a.cat.GetDrive(ctx, driveID)
|
||||
if err != nil {
|
||||
@@ -1122,10 +1172,7 @@ func (a *App) runScan(ctx context.Context, driveID string) {
|
||||
}
|
||||
sc := scanner.New(a.cat, drv, a.cfg.Scanner.VideoExtensions, d.SkipDirIDs, onNew)
|
||||
|
||||
startID := d.ScanRootID
|
||||
if startID == "" {
|
||||
startID = d.RootID
|
||||
}
|
||||
startID := d.RootID
|
||||
|
||||
log.Printf("[scan] drive=%s start=%s skip_dirs=%d", driveID, startID, len(d.SkipDirIDs))
|
||||
stats, err := sc.Run(ctx, startID)
|
||||
@@ -1491,8 +1538,9 @@ func (a *App) regenFailedThumbnails(ctx context.Context, driveID string) {
|
||||
// 来判断是否真的要再生)。但既然之前是 failed 说明 url 没写过,所以这里
|
||||
// 把 url 一并清空更稳。
|
||||
if err := a.cat.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
ThumbnailURL: "",
|
||||
ThumbnailStatus: "pending",
|
||||
ThumbnailURL: "",
|
||||
ThumbnailStatus: "pending",
|
||||
ResetThumbnailFailures: true,
|
||||
}); err != nil {
|
||||
log.Printf("[thumb] reset failed video %s drive=%s: %v", v.ID, driveID, err)
|
||||
continue
|
||||
@@ -1507,6 +1555,42 @@ func (a *App) regenFailedThumbnails(ctx context.Context, driveID string) {
|
||||
log.Printf("[thumb] enqueued failed thumbnails for regen drive=%s queued=%d", driveID, queued)
|
||||
}
|
||||
|
||||
func (a *App) regenFailedFingerprints(ctx context.Context, driveID string) {
|
||||
items, err := a.cat.ListVideosByFingerprintStatus(ctx, driveID, "failed", 0)
|
||||
if err != nil {
|
||||
log.Printf("[fingerprint] list failed videos for regen drive=%s: %v", driveID, err)
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
fingerprintWorker := a.fingerprintWorkers[driveID]
|
||||
a.mu.Unlock()
|
||||
if fingerprintWorker == nil {
|
||||
log.Printf("[fingerprint] regen failed drive=%s skipped: fingerprint worker not found", driveID)
|
||||
return
|
||||
}
|
||||
log.Printf("[fingerprint] enqueue failed videos for regen drive=%s count=%d", driveID, len(items))
|
||||
queued := 0
|
||||
for _, v := range items {
|
||||
if err := ctx.Err(); err != nil {
|
||||
log.Printf("[fingerprint] enqueue failed canceled drive=%s queued=%d: %v", driveID, queued, err)
|
||||
return
|
||||
}
|
||||
if err := a.cat.UpdateVideoFingerprint(ctx, v.ID, "", "pending", ""); err != nil {
|
||||
log.Printf("[fingerprint] reset failed video %s drive=%s: %v", v.ID, driveID, err)
|
||||
continue
|
||||
}
|
||||
v.SampledSHA256 = ""
|
||||
v.FingerprintStatus = "pending"
|
||||
v.FingerprintError = ""
|
||||
if !fingerprintWorker.EnqueueBlocking(ctx, v) {
|
||||
log.Printf("[fingerprint] enqueue failed canceled drive=%s queued=%d", driveID, queued)
|
||||
return
|
||||
}
|
||||
queued++
|
||||
}
|
||||
log.Printf("[fingerprint] enqueued failed videos for regen drive=%s queued=%d", driveID, queued)
|
||||
}
|
||||
|
||||
// listScanTargetIDs 返回 nightly Phase 1 应扫描的所有 drive ID
|
||||
// (非 spider91、非 localupload)。它直接读 catalog,而不是 registry,这样
|
||||
// 进程刚启动、云盘还在后台挂载时,nightly 也不会漏掉配置过的 drive。
|
||||
@@ -1743,22 +1827,34 @@ func corsMiddleware(allowedOrigins []string) func(http.Handler) http.Handler {
|
||||
}
|
||||
|
||||
func mountFrontend(r chi.Router) {
|
||||
dir := strings.TrimSpace(os.Getenv("VIDEO_FRONTEND_DIR"))
|
||||
if dir == "" {
|
||||
dir = "./dist"
|
||||
}
|
||||
info, err := os.Stat(dir)
|
||||
if err != nil || !info.IsDir() {
|
||||
return
|
||||
}
|
||||
indexPath := filepath.Join(dir, "index.html")
|
||||
if st, err := os.Stat(indexPath); err != nil || st.IsDir() {
|
||||
dir, ok := resolveFrontendDir()
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
log.Printf("serving frontend from %s", dir)
|
||||
r.NotFound(frontendHandler(dir))
|
||||
}
|
||||
|
||||
func resolveFrontendDir() (string, bool) {
|
||||
candidates := []string{}
|
||||
if dir := strings.TrimSpace(os.Getenv("VIDEO_FRONTEND_DIR")); dir != "" {
|
||||
candidates = append(candidates, dir)
|
||||
} else {
|
||||
candidates = append(candidates, "./dist", "../dist")
|
||||
}
|
||||
for _, dir := range candidates {
|
||||
info, err := os.Stat(dir)
|
||||
if err != nil || !info.IsDir() {
|
||||
continue
|
||||
}
|
||||
indexPath := filepath.Join(dir, "index.html")
|
||||
if st, err := os.Stat(indexPath); err == nil && !st.IsDir() {
|
||||
return dir, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func frontendHandler(dir string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet && r.Method != http.MethodHead {
|
||||
|
||||
@@ -59,7 +59,7 @@ preview:
|
||||
width: 480
|
||||
|
||||
# 盘列表。上线后请通过管理后台添加,本文件可留空。
|
||||
# kind 支持 quark / p115 / pikpak / wopan / onedrive / localstorage。
|
||||
# kind 支持 quark / p115 / pikpak / wopan / onedrive / googledrive / localstorage。
|
||||
# OneDrive 示例:
|
||||
# - id: "my-onedrive"
|
||||
# kind: "onedrive"
|
||||
@@ -67,12 +67,18 @@ preview:
|
||||
# root_id: "root"
|
||||
# params:
|
||||
# refresh_token: "..."
|
||||
# Google Drive 示例:
|
||||
# - id: "my-google"
|
||||
# kind: "googledrive"
|
||||
# name: "我的 Google Drive"
|
||||
# root_id: "root"
|
||||
# params:
|
||||
# refresh_token: "..."
|
||||
# 本地存储示例:
|
||||
# - id: "local-media"
|
||||
# kind: "localstorage"
|
||||
# name: "本地视频目录"
|
||||
# root_id: "/"
|
||||
# scan_root_id: "/"
|
||||
# params:
|
||||
# path: "/mnt/videos"
|
||||
drives: []
|
||||
|
||||
+3
-3
@@ -7,15 +7,17 @@ toolchain go1.23.4
|
||||
require (
|
||||
github.com/OpenListTeam/wopan-sdk-go v0.2.0
|
||||
github.com/SheltonZhu/115driver v1.3.2
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
|
||||
github.com/go-chi/chi/v5 v5.1.0
|
||||
github.com/go-resty/resty/v2 v2.14.0
|
||||
golang.org/x/net v0.27.0
|
||||
golang.org/x/sys v0.30.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
modernc.org/sqlite v1.33.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aead/ecdh v0.2.0 // indirect
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect
|
||||
github.com/andreburgaud/crypt2go v1.1.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
@@ -28,8 +30,6 @@ require (
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
|
||||
golang.org/x/crypto v0.25.0 // indirect
|
||||
golang.org/x/net v0.27.0 // indirect
|
||||
golang.org/x/sys v0.30.0 // indirect
|
||||
golang.org/x/time v0.8.0 // indirect
|
||||
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
|
||||
modernc.org/libc v1.55.3 // indirect
|
||||
|
||||
+170
-40
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -46,6 +47,7 @@ type AdminServer struct {
|
||||
OnRegenAllPreviews func()
|
||||
OnRegenFailedPreviews func(driveID string)
|
||||
OnRegenFailedThumbnails func(driveID string)
|
||||
OnRegenFailedFingerprints func(driveID string)
|
||||
GetDriveGenerationStatuses func() map[string]DriveGenerationStatuses
|
||||
// OnTeaserEnabledChanged 在 per-drive teaser 开关被切换后调用。
|
||||
// enabled=true 时上层应该重新把 pending teaser 入队(类似旧的全局开关从关到开);
|
||||
@@ -59,8 +61,10 @@ type AdminServer struct {
|
||||
SetSpider91UploadDriveID func(driveID string) error
|
||||
// OnRunNightlyJob 触发一次完整的凌晨流水线(Phase1 扫盘 + Phase2 91 爬虫 +
|
||||
// Phase3 迁移)。立即返回 —— 实际任务在后台跑,admin 在日志或下次状态查询里
|
||||
// 看进度。若流水线正在跑,Runner 最多保留一个待触发请求,当前轮结束后再跑一轮。
|
||||
OnRunNightlyJob func()
|
||||
// 看进度。若流水线正在跑或已排队,Runner 会拒绝重复触发。
|
||||
OnRunNightlyJob func() bool
|
||||
// GetNightlyJobStatus 返回凌晨流水线当前状态,用于前端禁用重复触发按钮。
|
||||
GetNightlyJobStatus func() NightlyJobStatus
|
||||
// ListDriveDirChildren 列出某个 drive 在 parentID 目录下的直接子目录。
|
||||
// parentID 为空时使用 drive 的 RootID。返回 (子目录列表, error)。
|
||||
// 用于"设置跳过目录"弹窗按需展开浏览网盘目录树;只返回目录条目,文件忽略。
|
||||
@@ -87,6 +91,14 @@ type DriveGenerationStatuses struct {
|
||||
Fingerprint GenerationStatus `json:"fingerprint"`
|
||||
}
|
||||
|
||||
type NightlyJobStatus struct {
|
||||
State string `json:"state"`
|
||||
Running bool `json:"running"`
|
||||
Queued bool `json:"queued"`
|
||||
StartedAt string `json:"startedAt,omitempty"`
|
||||
LastFinishedAt string `json:"lastFinishedAt,omitempty"`
|
||||
}
|
||||
|
||||
func (a *AdminServer) Register(r chi.Router) {
|
||||
r.Route("/admin/api", func(r chi.Router) {
|
||||
// 登录、登出和首次部署初始化不需要鉴权
|
||||
@@ -111,6 +123,7 @@ func (a *AdminServer) Register(r chi.Router) {
|
||||
r.Get("/drives/{id}/dirtree", a.handleListDriveDirTree)
|
||||
r.Post("/drives/{id}/previews/failed/regenerate", a.handleRegenFailedPreviews)
|
||||
r.Post("/drives/{id}/thumbnails/failed/regenerate", a.handleRegenFailedThumbnails)
|
||||
r.Post("/drives/{id}/fingerprints/failed/regenerate", a.handleRegenFailedFingerprints)
|
||||
|
||||
// 视频
|
||||
r.Get("/videos", a.handleAdminListVideos)
|
||||
@@ -121,6 +134,7 @@ func (a *AdminServer) Register(r chi.Router) {
|
||||
// 标签
|
||||
r.Get("/tags", a.handleListTags)
|
||||
r.Post("/tags", a.handleCreateTag)
|
||||
r.Delete("/tags/{id}", a.handleDeleteTag)
|
||||
|
||||
// 运行时设置
|
||||
r.Get("/settings", a.handleGetSettings)
|
||||
@@ -128,6 +142,7 @@ func (a *AdminServer) Register(r chi.Router) {
|
||||
|
||||
// 运维任务
|
||||
r.Get("/update/check", a.handleCheckUpdate)
|
||||
r.Get("/jobs/nightly/status", a.handleNightlyJobStatus)
|
||||
r.Post("/jobs/nightly/run", a.handleRunNightlyJob)
|
||||
})
|
||||
})
|
||||
@@ -381,19 +396,21 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
SkipDirIDs []string `json:"skipDirIds"`
|
||||
// LastCrawlAt 是 spider91 上次成功爬取的 unix 秒(来自 credentials.last_crawl_at)。
|
||||
// 其它 kind 留 0;前端用它显示"上次抓取: N 小时前"。
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
FingerprintReadyCount int `json:"fingerprintReadyCount"`
|
||||
FingerprintPendingCount int `json:"fingerprintPendingCount"`
|
||||
FingerprintFailedCount int `json:"fingerprintFailedCount"`
|
||||
Spider91Proxy string `json:"spider91Proxy,omitempty"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
ThumbnailDurationPendingCount int `json:"thumbnailDurationPendingCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
FingerprintReadyCount int `json:"fingerprintReadyCount"`
|
||||
FingerprintPendingCount int `json:"fingerprintPendingCount"`
|
||||
FingerprintFailedCount int `json:"fingerprintFailedCount"`
|
||||
}
|
||||
list := make([]out, 0, len(drives))
|
||||
for _, d := range drives {
|
||||
@@ -435,32 +452,35 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
ID: d.ID, Kind: d.Kind, Name: d.Name,
|
||||
RootID: d.RootID, ScanRootID: d.ScanRootID,
|
||||
Status: d.Status, LastError: d.LastError,
|
||||
HasCredential: hasCred,
|
||||
TeaserEnabled: d.TeaserEnabled,
|
||||
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
|
||||
LastCrawlAt: lastCrawlAt,
|
||||
ThumbnailGenerationStatus: generation.Thumbnail,
|
||||
PreviewGenerationStatus: generation.Preview,
|
||||
FingerprintGenerationStatus: generation.Fingerprint,
|
||||
ThumbnailReadyCount: thumbCounts.Ready,
|
||||
ThumbnailPendingCount: thumbCounts.Pending,
|
||||
ThumbnailFailedCount: thumbCounts.Failed,
|
||||
TeaserReadyCount: counts.Ready,
|
||||
TeaserPendingCount: counts.Pending,
|
||||
TeaserFailedCount: counts.Failed,
|
||||
FingerprintReadyCount: fingerprintCount.Ready,
|
||||
FingerprintPendingCount: fingerprintCount.Pending,
|
||||
FingerprintFailedCount: fingerprintCount.Failed,
|
||||
HasCredential: hasCred,
|
||||
TeaserEnabled: d.TeaserEnabled,
|
||||
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
|
||||
Spider91Proxy: spider91ProxyForDrive(d),
|
||||
LastCrawlAt: lastCrawlAt,
|
||||
ThumbnailGenerationStatus: generation.Thumbnail,
|
||||
PreviewGenerationStatus: generation.Preview,
|
||||
FingerprintGenerationStatus: generation.Fingerprint,
|
||||
ThumbnailReadyCount: thumbCounts.Ready,
|
||||
ThumbnailPendingCount: thumbCounts.Pending,
|
||||
ThumbnailFailedCount: thumbCounts.Failed,
|
||||
ThumbnailDurationPendingCount: thumbCounts.DurationPending,
|
||||
TeaserReadyCount: counts.Ready,
|
||||
TeaserPendingCount: counts.Pending,
|
||||
TeaserFailedCount: counts.Failed,
|
||||
FingerprintReadyCount: fingerprintCount.Ready,
|
||||
FingerprintPendingCount: fingerprintCount.Pending,
|
||||
FingerprintFailedCount: fingerprintCount.Failed,
|
||||
})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, list)
|
||||
}
|
||||
|
||||
type upsertDriveReq struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
RootID string `json:"rootId"`
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
RootID string `json:"rootId"`
|
||||
// Deprecated: 扫描起点已固定为 rootId;保留字段只为兼容旧客户端请求体。
|
||||
ScanRootID string `json:"scanRootId"`
|
||||
Credentials map[string]string `json:"credentials"`
|
||||
// TeaserEnabled 是 per-drive teaser/封面生成开关。
|
||||
@@ -488,7 +508,14 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
|
||||
if existingDrive, err := a.Catalog.GetDrive(r.Context(), body.ID); err == nil {
|
||||
existing = existingDrive
|
||||
}
|
||||
if len(body.Credentials) == 0 && existing != nil && len(existing.Credentials) > 0 {
|
||||
if body.Kind == "spider91" {
|
||||
credentials, err := mergeSpider91Credentials(existing, body.Credentials)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
body.Credentials = credentials
|
||||
} else if len(body.Credentials) == 0 && existing != nil && len(existing.Credentials) > 0 {
|
||||
body.Credentials = existing.Credentials
|
||||
}
|
||||
|
||||
@@ -518,7 +545,7 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
d := &catalog.Drive{
|
||||
ID: body.ID, Kind: body.Kind, Name: body.Name,
|
||||
RootID: body.RootID, ScanRootID: body.ScanRootID,
|
||||
RootID: body.RootID,
|
||||
Credentials: body.Credentials,
|
||||
Status: "disconnected",
|
||||
TeaserEnabled: teaserEnabled,
|
||||
@@ -537,6 +564,58 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
|
||||
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
func spider91ProxyForDrive(d *catalog.Drive) string {
|
||||
if d == nil || d.Kind != "spider91" || d.Credentials == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(d.Credentials["proxy"])
|
||||
}
|
||||
|
||||
func mergeSpider91Credentials(existing *catalog.Drive, incoming map[string]string) (map[string]string, error) {
|
||||
merged := map[string]string{}
|
||||
if existing != nil {
|
||||
for k, v := range existing.Credentials {
|
||||
merged[k] = v
|
||||
}
|
||||
}
|
||||
for k, v := range incoming {
|
||||
if strings.TrimSpace(k) == "" {
|
||||
continue
|
||||
}
|
||||
if k == "proxy" {
|
||||
proxy, err := normalizeSpider91ProxyURL(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if proxy == "" {
|
||||
delete(merged, "proxy")
|
||||
} else {
|
||||
merged["proxy"] = proxy
|
||||
}
|
||||
continue
|
||||
}
|
||||
merged[k] = v
|
||||
}
|
||||
return merged, nil
|
||||
}
|
||||
|
||||
func normalizeSpider91ProxyURL(raw string) (string, error) {
|
||||
proxy := strings.TrimSpace(raw)
|
||||
if proxy == "" {
|
||||
return "", nil
|
||||
}
|
||||
u, err := url.Parse(proxy)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return "", fmt.Errorf("91Spider 代理地址格式无效,请填写类似 http://127.0.0.1:7890 的地址")
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https", "socks5", "socks5h":
|
||||
return proxy, nil
|
||||
default:
|
||||
return "", fmt.Errorf("91Spider 代理地址仅支持 http://、https://、socks5:// 或 socks5h://")
|
||||
}
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleDeleteDrive(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
if err := a.Catalog.DeleteDrive(r.Context(), id); err != nil {
|
||||
@@ -559,12 +638,32 @@ func (a *AdminServer) handleRescan(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// handleRunNightlyJob 触发一次完整的凌晨流水线(不论当前时间,不论今日是否已跑)。
|
||||
// 立即返回 202;进度通过 backend 日志和下次 GET /admin/api/drives 的状态变化观察。
|
||||
// 流水线已在跑时 Runner 最多排队一个后续触发;如果已有待触发请求,新的点击会被忽略。
|
||||
// 流水线已在跑或已排队时,Runner 会拒绝重复触发。
|
||||
func (a *AdminServer) handleRunNightlyJob(w http.ResponseWriter, r *http.Request) {
|
||||
accepted := false
|
||||
if a.OnRunNightlyJob != nil {
|
||||
a.OnRunNightlyJob()
|
||||
accepted = a.OnRunNightlyJob()
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"ok": true,
|
||||
"accepted": accepted,
|
||||
"status": a.nightlyJobStatus(),
|
||||
})
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleNightlyJobStatus(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.nightlyJobStatus())
|
||||
}
|
||||
|
||||
func (a *AdminServer) nightlyJobStatus() NightlyJobStatus {
|
||||
if a.GetNightlyJobStatus == nil {
|
||||
return NightlyJobStatus{State: "idle"}
|
||||
}
|
||||
status := a.GetNightlyJobStatus()
|
||||
if status.State == "" {
|
||||
status.State = "idle"
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
// teaserEnabledReq 是 POST /admin/api/drives/{id}/teaser-enabled 的入参。
|
||||
@@ -745,6 +844,27 @@ func (a *AdminServer) handleCreateTag(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleDeleteTag(w http.ResponseWriter, r *http.Request) {
|
||||
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
||||
if err != nil || id <= 0 {
|
||||
writeErr(w, http.StatusBadRequest, errors.New("invalid tag id"))
|
||||
return
|
||||
}
|
||||
removedVideos, err := a.Catalog.DeleteTag(r.Context(), id)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
writeErr(w, http.StatusNotFound, err)
|
||||
case errors.Is(err, catalog.ErrSystemTag):
|
||||
writeErr(w, http.StatusBadRequest, err)
|
||||
default:
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "removedVideos": removedVideos})
|
||||
}
|
||||
|
||||
type updateVideoReq struct {
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
@@ -851,6 +971,16 @@ func (a *AdminServer) handleRegenFailedThumbnails(w http.ResponseWriter, r *http
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
// handleRegenFailedFingerprints triggers regeneration for all failed sampled
|
||||
// fingerprints on a drive. It mirrors the failed teaser/thumbnail retry endpoints.
|
||||
func (a *AdminServer) handleRegenFailedFingerprints(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
if a.OnRegenFailedFingerprints != nil {
|
||||
a.OnRegenFailedFingerprints(id)
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
// ---------- Settings ----------
|
||||
|
||||
// settingsDTO 是 GET/PUT /admin/api/settings 的入参/出参。
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -242,6 +243,58 @@ func TestInstalledVersionPrefersDockerImageVersionOverVersionFile(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRunNightlyJobReturnsAcceptedStatus(t *testing.T) {
|
||||
called := false
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/jobs/nightly/run", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
OnRunNightlyJob: func() bool {
|
||||
called = true
|
||||
return true
|
||||
},
|
||||
GetNightlyJobStatus: func() NightlyJobStatus {
|
||||
return NightlyJobStatus{State: "queued", Queued: true}
|
||||
},
|
||||
}).handleRunNightlyJob(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, want 202; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !called {
|
||||
t.Fatal("OnRunNightlyJob was not called")
|
||||
}
|
||||
var got struct {
|
||||
OK bool `json:"ok"`
|
||||
Accepted bool `json:"accepted"`
|
||||
Status NightlyJobStatus `json:"status"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if !got.OK || !got.Accepted || got.Status.State != "queued" || !got.Status.Queued {
|
||||
t.Fatalf("response = %#v, want accepted queued status", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleNightlyJobStatusDefaultsToIdle(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/jobs/nightly/status", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{}).handleNightlyJobStatus(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got NightlyJobStatus
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.State != "idle" || got.Running || got.Queued {
|
||||
t.Fatalf("status = %#v, want idle", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertDrivePreservesExistingCredentialsWhenRequestCredentialsEmpty(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -290,14 +343,52 @@ func TestHandleUpsertDrivePreservesExistingCredentialsWhenRequestCredentialsEmpt
|
||||
if got.Name != "New name" {
|
||||
t.Fatalf("name = %q, want New name", got.Name)
|
||||
}
|
||||
if got.ScanRootID != "scan-root" {
|
||||
t.Fatalf("scanRootId = %q, want scan-root", got.ScanRootID)
|
||||
if got.ScanRootID != "0" {
|
||||
t.Fatalf("scanRootId = %q, want rootId 0", got.ScanRootID)
|
||||
}
|
||||
if got.Credentials["cookie"] != "existing-cookie" {
|
||||
t.Fatalf("cookie credential = %q, want existing-cookie", got.Credentials["cookie"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertDriveDefaultsEmptyRootID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "onedrive-main",
|
||||
"kind": "onedrive",
|
||||
"name": "OneDrive",
|
||||
"rootId": "",
|
||||
"credentials": {"refresh_token": "token"}
|
||||
}`))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, "onedrive-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.RootID != "root" {
|
||||
t.Fatalf("rootId = %q, want root", got.RootID)
|
||||
}
|
||||
if got.ScanRootID != got.RootID {
|
||||
t.Fatalf("scanRootId = %q, want rootId %q", got.ScanRootID, got.RootID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertDriveReplacesExistingCredentialsWhenProvided(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -348,6 +439,190 @@ func TestHandleUpsertDriveReplacesExistingCredentialsWhenProvided(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertSpider91ProxyPreservesRuntimeCredentials(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "spider91-main",
|
||||
Kind: "spider91",
|
||||
Name: "91 Spider",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"last_crawl_at": "1800000000",
|
||||
"proxy": "http://old-proxy.local:7890",
|
||||
"script_path": "/opt/video-site-91/91VideoSpider/spider_91porn.py",
|
||||
},
|
||||
Status: "ok",
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"kind": "spider91",
|
||||
"name": "91 Spider",
|
||||
"rootId": "/",
|
||||
"credentials": {"proxy": " socks5h://proxy-user:proxy-pass@127.0.0.1:7891 "}
|
||||
}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
got, err := cat.GetDrive(ctx, "spider91-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.Credentials["proxy"] != "socks5h://proxy-user:proxy-pass@127.0.0.1:7891" {
|
||||
t.Fatalf("proxy = %q, want trimmed new proxy", got.Credentials["proxy"])
|
||||
}
|
||||
if got.Credentials["last_crawl_at"] != "1800000000" {
|
||||
t.Fatalf("last_crawl_at = %q, want preserved", got.Credentials["last_crawl_at"])
|
||||
}
|
||||
if got.Credentials["script_path"] == "" {
|
||||
t.Fatalf("script_path should be preserved")
|
||||
}
|
||||
|
||||
req = httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"kind": "spider91",
|
||||
"name": "91 Spider",
|
||||
"rootId": "/",
|
||||
"credentials": {"proxy": " "}
|
||||
}`))
|
||||
rr = httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("clear status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
got, err = cat.GetDrive(ctx, "spider91-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get cleared drive: %v", err)
|
||||
}
|
||||
if _, ok := got.Credentials["proxy"]; ok {
|
||||
t.Fatalf("proxy should be removed after empty save, got %q", got.Credentials["proxy"])
|
||||
}
|
||||
if got.Credentials["last_crawl_at"] != "1800000000" {
|
||||
t.Fatalf("last_crawl_at after clear = %q, want preserved", got.Credentials["last_crawl_at"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertSpider91RejectsUnsupportedProxyScheme(t *testing.T) {
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"kind": "spider91",
|
||||
"name": "91 Spider",
|
||||
"rootId": "/",
|
||||
"credentials": {"proxy": "ftp://127.0.0.1:21"}
|
||||
}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), "socks5:// 或 socks5h://") {
|
||||
t.Fatalf("body = %q, want supported schemes message", rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListDrivesIncludesSpider91Proxy(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, d := range []*catalog.Drive{
|
||||
{
|
||||
ID: "spider91-main",
|
||||
Kind: "spider91",
|
||||
Name: "91 Spider",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"last_crawl_at": "1800000000",
|
||||
"proxy": " http://127.0.0.1:7890 ",
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
{
|
||||
ID: "onedrive-main",
|
||||
Kind: "onedrive",
|
||||
Name: "OneDrive",
|
||||
RootID: "root",
|
||||
Credentials: map[string]string{
|
||||
"proxy": "http://should-not-leak.local:7890",
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
t.Fatalf("seed drive %s: %v", d.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/drives", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleListDrives(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
var got []struct {
|
||||
ID string `json:"id"`
|
||||
Spider91Proxy string `json:"spider91Proxy"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
byID := map[string]struct {
|
||||
Spider91Proxy string
|
||||
LastCrawlAt int64
|
||||
}{}
|
||||
for _, d := range got {
|
||||
byID[d.ID] = struct {
|
||||
Spider91Proxy string
|
||||
LastCrawlAt int64
|
||||
}{Spider91Proxy: d.Spider91Proxy, LastCrawlAt: d.LastCrawlAt}
|
||||
}
|
||||
if byID["spider91-main"].Spider91Proxy != "http://127.0.0.1:7890" {
|
||||
t.Fatalf("spider91 proxy = %q, want trimmed proxy", byID["spider91-main"].Spider91Proxy)
|
||||
}
|
||||
if byID["spider91-main"].LastCrawlAt != 1800000000 {
|
||||
t.Fatalf("lastCrawlAt = %d, want 1800000000", byID["spider91-main"].LastCrawlAt)
|
||||
}
|
||||
if byID["onedrive-main"].Spider91Proxy != "" {
|
||||
t.Fatalf("onedrive spider91Proxy = %q, want empty", byID["onedrive-main"].Spider91Proxy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -411,64 +686,68 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got []struct {
|
||||
ID string `json:"id"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
FingerprintReadyCount int `json:"fingerprintReadyCount"`
|
||||
FingerprintPendingCount int `json:"fingerprintPendingCount"`
|
||||
FingerprintFailedCount int `json:"fingerprintFailedCount"`
|
||||
ID string `json:"id"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
ThumbnailDurationPendingCount int `json:"thumbnailDurationPendingCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
FingerprintReadyCount int `json:"fingerprintReadyCount"`
|
||||
FingerprintPendingCount int `json:"fingerprintPendingCount"`
|
||||
FingerprintFailedCount int `json:"fingerprintFailedCount"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
byID := map[string]struct {
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
FingerprintReady int
|
||||
FingerprintPending int
|
||||
FingerprintFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
Fingerprint GenerationStatus
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
ThumbnailDurationPending int
|
||||
FingerprintReady int
|
||||
FingerprintPending int
|
||||
FingerprintFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
Fingerprint GenerationStatus
|
||||
}{}
|
||||
for _, d := range got {
|
||||
byID[d.ID] = struct {
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
FingerprintReady int
|
||||
FingerprintPending int
|
||||
FingerprintFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
Fingerprint GenerationStatus
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
ThumbnailDurationPending int
|
||||
FingerprintReady int
|
||||
FingerprintPending int
|
||||
FingerprintFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
Fingerprint GenerationStatus
|
||||
}{
|
||||
TeaserReady: d.TeaserReadyCount,
|
||||
TeaserPending: d.TeaserPendingCount,
|
||||
TeaserFailed: d.TeaserFailedCount,
|
||||
ThumbnailReady: d.ThumbnailReadyCount,
|
||||
ThumbnailPending: d.ThumbnailPendingCount,
|
||||
ThumbnailFailed: d.ThumbnailFailedCount,
|
||||
FingerprintReady: d.FingerprintReadyCount,
|
||||
FingerprintPending: d.FingerprintPendingCount,
|
||||
FingerprintFailed: d.FingerprintFailedCount,
|
||||
Thumbnail: d.ThumbnailGenerationStatus,
|
||||
Preview: d.PreviewGenerationStatus,
|
||||
Fingerprint: d.FingerprintGenerationStatus,
|
||||
TeaserReady: d.TeaserReadyCount,
|
||||
TeaserPending: d.TeaserPendingCount,
|
||||
TeaserFailed: d.TeaserFailedCount,
|
||||
ThumbnailReady: d.ThumbnailReadyCount,
|
||||
ThumbnailPending: d.ThumbnailPendingCount,
|
||||
ThumbnailFailed: d.ThumbnailFailedCount,
|
||||
ThumbnailDurationPending: d.ThumbnailDurationPendingCount,
|
||||
FingerprintReady: d.FingerprintReadyCount,
|
||||
FingerprintPending: d.FingerprintPendingCount,
|
||||
FingerprintFailed: d.FingerprintFailedCount,
|
||||
Thumbnail: d.ThumbnailGenerationStatus,
|
||||
Preview: d.PreviewGenerationStatus,
|
||||
Fingerprint: d.FingerprintGenerationStatus,
|
||||
}
|
||||
}
|
||||
if byID["OneDrive"].TeaserReady != 2 || byID["OneDrive"].TeaserPending != 1 || byID["OneDrive"].TeaserFailed != 0 {
|
||||
@@ -477,6 +756,9 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
if byID["OneDrive"].ThumbnailReady != 1 || byID["OneDrive"].ThumbnailPending != 1 || byID["OneDrive"].ThumbnailFailed != 1 {
|
||||
t.Fatalf("OneDrive thumbnail counts = %#v, want ready=1 pending=1 failed=1", byID["OneDrive"])
|
||||
}
|
||||
if byID["OneDrive"].ThumbnailDurationPending != 1 {
|
||||
t.Fatalf("OneDrive thumbnail duration pending = %#v, want 1", byID["OneDrive"])
|
||||
}
|
||||
if byID["OneDrive"].Thumbnail.State != "cooling" || byID["OneDrive"].Preview.State != "generating" {
|
||||
t.Fatalf("OneDrive generation statuses = %#v, want thumbnail cooling and preview generating", byID["OneDrive"])
|
||||
}
|
||||
@@ -500,6 +782,28 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRegenFailedFingerprintsInvokesHook(t *testing.T) {
|
||||
called := ""
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives/drive-one/fingerprints/failed/regenerate", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "drive-one")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
OnRegenFailedFingerprints: func(driveID string) {
|
||||
called = driveID
|
||||
},
|
||||
}).handleRegenFailedFingerprints(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if called != "drive-one" {
|
||||
t.Fatalf("called drive = %q, want drive-one", called)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleDriveStorageReportsLocalMediaUsage(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
@@ -663,6 +967,64 @@ func TestHandleCreateTagClassifiesExistingVideos(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleDeleteTagRemovesTagFromVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "video-1",
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
Title: "清纯短发",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
if _, err := cat.CreateTagAndClassify(ctx, "清纯", nil, "user"); err != nil {
|
||||
t.Fatalf("create tag: %v", err)
|
||||
}
|
||||
tags, err := cat.ListTags(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list tags: %v", err)
|
||||
}
|
||||
var tagID int64
|
||||
for _, tag := range tags {
|
||||
if tag.Label == "清纯" {
|
||||
tagID = tag.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
if tagID == 0 {
|
||||
t.Fatal("created tag not found")
|
||||
}
|
||||
|
||||
req := requestWithRouteParam(http.MethodDelete, "/admin/api/tags/1", "id", strconv.FormatInt(tagID, 10), strings.NewReader(``))
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleDeleteTag(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
video, err := cat.GetVideo(ctx, "video-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if len(video.Tags) != 0 {
|
||||
t.Fatalf("video tags = %#v, want none", video.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleAdminListVideosFiltersByDriveID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
+53
-26
@@ -40,7 +40,7 @@ var allowedUploadExtensions = map[string]struct{}{
|
||||
var allowedUploadTags = map[string]struct{}{
|
||||
"奶子": {},
|
||||
"臀": {},
|
||||
"口角": {},
|
||||
"口交": {},
|
||||
"女大": {},
|
||||
"人妻": {},
|
||||
"AV": {},
|
||||
@@ -156,41 +156,60 @@ func (s *Server) handleGetTheme(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) {
|
||||
// 首页优先展示封面已经生成好的视频,避免新盘扫盘时大量黑封面占满首页。
|
||||
// 候选仍按发布时间覆盖最近 200 个,随后随机洗牌;封面不足时再用普通可见视频补齐。
|
||||
const candidatePool = 200
|
||||
readyItems, _, err := s.Catalog.ListVideos(r.Context(), catalog.ListParams{
|
||||
Sort: "latest", Page: 1, PageSize: candidatePool, ThumbnailReadyOnly: true,
|
||||
})
|
||||
// 首页优先从全量已有封面的视频里随机抽取,避免只在最近一小段候选中反复出现。
|
||||
excludeIDs := parseVideoIDQuery(r, "exclude", 120)
|
||||
items, err := s.Catalog.RandomVideosWithReadyThumbnailsExcluding(r.Context(), excludeIDs, homePageSize)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
rand.Shuffle(len(readyItems), func(i, j int) {
|
||||
readyItems[i], readyItems[j] = readyItems[j], readyItems[i]
|
||||
})
|
||||
|
||||
items := appendUniqueVideos(nil, readyItems, homePageSize)
|
||||
if len(items) > homePageSize {
|
||||
items = items[:homePageSize]
|
||||
}
|
||||
if len(items) < homePageSize {
|
||||
fallback, _, err := s.Catalog.ListVideos(r.Context(), catalog.ListParams{
|
||||
Sort: "latest", Page: 1, PageSize: candidatePool,
|
||||
})
|
||||
fallbackExclude := append([]string{}, excludeIDs...)
|
||||
for _, item := range items {
|
||||
if item != nil {
|
||||
fallbackExclude = append(fallbackExclude, item.ID)
|
||||
}
|
||||
}
|
||||
fallback, err := s.Catalog.RandomVideosExcluding(r.Context(), fallbackExclude, homePageSize-len(items))
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
rand.Shuffle(len(fallback), func(i, j int) {
|
||||
fallback[i], fallback[j] = fallback[j], fallback[i]
|
||||
})
|
||||
items = appendUniqueVideos(items, fallback, homePageSize)
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, http.StatusOK, mapVideos(items))
|
||||
}
|
||||
|
||||
func parseVideoIDQuery(r *http.Request, key string, limit int) []string {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
values := r.URL.Query()[key]
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
out := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
for _, id := range strings.Split(value, ",") {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
out = append(out, id)
|
||||
if limit > 0 && len(out) >= limit {
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func appendUniqueVideos(dst []*catalog.Video, candidates []*catalog.Video, limit int) []*catalog.Video {
|
||||
if len(dst) >= limit {
|
||||
return dst[:limit]
|
||||
@@ -440,11 +459,12 @@ func (s *Server) handleTags(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// shortsNextReq 客户端把当前轮已看过的 video id 列表传上来,
|
||||
// 服务器从未在列表中的视频里随机抽 count 个返回。
|
||||
// shortsNextReq 客户端把当前轮已看过的 video id 列表传上来。
|
||||
// PreferredFromVideoID 来自短视频页最近一次点赞成功的视频,用于优先推荐相似标签。
|
||||
type shortsNextReq struct {
|
||||
SeenIDs []string `json:"seenIds"`
|
||||
Count int `json:"count"`
|
||||
SeenIDs []string `json:"seenIds"`
|
||||
Count int `json:"count"`
|
||||
PreferredFromVideoID string `json:"preferredFromVideoId"`
|
||||
}
|
||||
|
||||
// ShortsItemDTO 是短视频流单条的精简结构。比 VideoDTO 多 videoSrc / poster,
|
||||
@@ -490,7 +510,12 @@ func (s *Server) handleShortsNext(w http.ResponseWriter, r *http.Request) {
|
||||
exclude = nil
|
||||
}
|
||||
|
||||
items, err := s.Catalog.RandomVideosExcluding(r.Context(), exclude, count)
|
||||
var items []*catalog.Video
|
||||
if strings.TrimSpace(body.PreferredFromVideoID) != "" {
|
||||
items, err = s.Catalog.RandomVideosForPreferredVideoExcluding(r.Context(), body.PreferredFromVideoID, exclude, count)
|
||||
} else {
|
||||
items, err = s.Catalog.RandomVideosExcluding(r.Context(), exclude, count)
|
||||
}
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
@@ -900,6 +925,8 @@ func driveKindLabel(kind string) string {
|
||||
return "联通沃盘"
|
||||
case "onedrive":
|
||||
return "OneDrive"
|
||||
case "googledrive":
|
||||
return "Google Drive"
|
||||
case localstorage.Kind:
|
||||
return "本地存储"
|
||||
case spider91.Kind:
|
||||
|
||||
@@ -166,6 +166,59 @@ func TestHandleHomePrioritizesVideosWithReadyThumbnails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleHomeExcludesRecentlyShownVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < homePageSize+4; i++ {
|
||||
id := "ready-video-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
|
||||
PublishedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
CreatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
UpdatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/home?exclude=ready-video-0&exclude=ready-video-1", nil)
|
||||
(&Server{Catalog: cat}).handleHome(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got []VideoDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(got) != homePageSize {
|
||||
t.Fatalf("home items = %d, want %d", len(got), homePageSize)
|
||||
}
|
||||
for _, item := range got {
|
||||
if item.ID == "ready-video-0" || item.ID == "ready-video-1" {
|
||||
t.Fatalf("home returned excluded video %q; items=%#v", item.ID, got)
|
||||
}
|
||||
if !strings.HasPrefix(item.ID, "ready-video-") {
|
||||
t.Fatalf("home returned %q without a ready thumbnail; items=%#v", item.ID, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListLatestPrefersReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -261,7 +314,7 @@ func TestHandleUploadVideoSavesFileVideoTagsAndQueuesPreview(t *testing.T) {
|
||||
}
|
||||
req := multipartUploadRequest(t, map[string]string{
|
||||
"title": "用户上传标题",
|
||||
"tags": "奶子,AV,女大",
|
||||
"tags": "奶子,口交,AV,女大",
|
||||
}, "clip.mp4", "video-bytes")
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
@@ -287,7 +340,7 @@ func TestHandleUploadVideoSavesFileVideoTagsAndQueuesPreview(t *testing.T) {
|
||||
if got.Title != "用户上传标题" {
|
||||
t.Fatalf("title = %q, want submitted title", got.Title)
|
||||
}
|
||||
if !sameStringSet(got.Tags, []string{"奶子", "AV", "女大"}) {
|
||||
if !sameStringSet(got.Tags, []string{"奶子", "口交", "AV", "女大"}) {
|
||||
t.Fatalf("tags = %#v, want selected tags", got.Tags)
|
||||
}
|
||||
if got.PreviewStatus != "pending" {
|
||||
@@ -523,6 +576,66 @@ func TestHandleTagsReturnsUnifiedTagPool(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleShortsNextUsesPreferredVideoLeastPopulatedTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*catalog.Video{
|
||||
{ID: "current", DriveID: "drive", FileID: "f-current", Title: "current", Tags: []string{"common", "rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-1", DriveID: "drive", FileID: "f-common-1", Title: "common 1", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-2", DriveID: "drive", FileID: "f-common-2", Title: "common 2", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "rare-1", DriveID: "drive", FileID: "f-rare-1", Title: "rare 1", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/shorts/next", strings.NewReader(`{"seenIds":["current"],"count":3,"preferredFromVideoId":"current"}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&Server{Catalog: cat}).handleShortsNext(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
Items []ShortsItemDTO `json:"items"`
|
||||
Total int `json:"total"`
|
||||
RoundComplete bool `json:"roundComplete"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
ids := make([]string, 0, len(got.Items))
|
||||
for _, item := range got.Items {
|
||||
ids = append(ids, item.ID)
|
||||
}
|
||||
if got.Total != 4 {
|
||||
t.Fatalf("total = %d, want 4", got.Total)
|
||||
}
|
||||
if got.RoundComplete {
|
||||
t.Fatalf("roundComplete = true, want false with fallback-filled batch")
|
||||
}
|
||||
if !containsString(ids, "rare-1") {
|
||||
t.Fatalf("ids = %#v, want rare-1 from least populated tag", ids)
|
||||
}
|
||||
if containsString(ids, "current") {
|
||||
t.Fatalf("ids = %#v, should exclude current", ids)
|
||||
}
|
||||
if len(ids) != 3 {
|
||||
t.Fatalf("ids = %#v, want 3 items", ids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpdateVideoTagsRejectsUnknownTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
@@ -321,14 +321,15 @@ func (c *Catalog) IncrementView(ctx context.Context, id string) (int, error) {
|
||||
|
||||
// VideoMetaPatch 轻量更新视频元数据(仅非零值字段会被写入)
|
||||
type VideoMetaPatch struct {
|
||||
ThumbnailURL string
|
||||
ThumbnailStatus string
|
||||
DurationSeconds int
|
||||
Category string
|
||||
ContentHash string
|
||||
FileName string
|
||||
Tags []string
|
||||
TagsSet bool
|
||||
ThumbnailURL string
|
||||
ThumbnailStatus string
|
||||
ResetThumbnailFailures bool
|
||||
DurationSeconds int
|
||||
Category string
|
||||
ContentHash string
|
||||
FileName string
|
||||
Tags []string
|
||||
TagsSet bool
|
||||
}
|
||||
|
||||
func (c *Catalog) UpdateVideoMeta(ctx context.Context, id string, p VideoMetaPatch) error {
|
||||
@@ -342,8 +343,12 @@ func (c *Catalog) UpdateVideoMeta(ctx context.Context, id string, p VideoMetaPat
|
||||
case p.ThumbnailStatus != "":
|
||||
// 调用方显式指定 status —— 信任之;典型是 worker 把状态置 'failed' 或
|
||||
// 在重试时显式置 'pending'。
|
||||
status := nullableStatus(p.ThumbnailStatus)
|
||||
parts = append(parts, "thumbnail_status = ?")
|
||||
args = append(args, nullableStatus(p.ThumbnailStatus))
|
||||
args = append(args, status)
|
||||
if status == "ready" {
|
||||
p.ResetThumbnailFailures = true
|
||||
}
|
||||
case p.ThumbnailURL != "":
|
||||
// 调用方写了 url 但没显式给 status —— 视为"封面就绪"。url 非空意味着
|
||||
// 浏览器访问那个 URL 能拿到图(要么是本地 /p/thumb/<id>,要么是网盘 API
|
||||
@@ -351,6 +356,10 @@ func (c *Catalog) UpdateVideoMeta(ctx context.Context, id string, p VideoMetaPat
|
||||
// 仍是 'pending' 的脏状态(修过的历史 bug)。
|
||||
parts = append(parts, "thumbnail_status = ?")
|
||||
args = append(args, nullableStatus("ready"))
|
||||
p.ResetThumbnailFailures = true
|
||||
}
|
||||
if p.ResetThumbnailFailures {
|
||||
parts = append(parts, "thumbnail_failures = 0")
|
||||
}
|
||||
if p.DurationSeconds > 0 {
|
||||
parts = append(parts, "duration_seconds = ?")
|
||||
@@ -389,6 +398,38 @@ func (c *Catalog) UpdateVideoMeta(ctx context.Context, id string, p VideoMetaPat
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Catalog) IncrementThumbnailFailures(ctx context.Context, id string) (int, error) {
|
||||
tx, err := c.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
res, err := tx.ExecContext(ctx,
|
||||
`UPDATE videos
|
||||
SET thumbnail_failures = COALESCE(thumbnail_failures, 0) + 1,
|
||||
updated_at = ?
|
||||
WHERE id = ?`,
|
||||
time.Now().UnixMilli(), id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
|
||||
return 0, sql.ErrNoRows
|
||||
}
|
||||
|
||||
var failures int
|
||||
if err := tx.QueryRowContext(ctx,
|
||||
`SELECT COALESCE(thumbnail_failures, 0) FROM videos WHERE id = ?`,
|
||||
id).Scan(&failures); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return failures, nil
|
||||
}
|
||||
|
||||
// ListCategories 聚合所有 category,按视频数降序
|
||||
type CategoryStat struct {
|
||||
Category string
|
||||
@@ -724,6 +765,37 @@ func (c *Catalog) ListVideosNeedingFingerprint(ctx context.Context, driveID stri
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListVideosByFingerprintStatus lists visible videos on a drive by fingerprint status.
|
||||
// It is used by the admin "retry failed fingerprints" action to reset failed rows
|
||||
// back to pending and enqueue them again.
|
||||
func (c *Catalog) ListVideosByFingerprintStatus(ctx context.Context, driveID, status string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10000
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = ?
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT ?`,
|
||||
driveID, status, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) UpdateVideoFingerprint(ctx context.Context, id, sampledSHA256, status, errText string) error {
|
||||
sampledSHA256 = normalizeContentHash(sampledSHA256)
|
||||
if status == "" {
|
||||
@@ -781,12 +853,7 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
|
||||
args = append(args, p.DriveID)
|
||||
}
|
||||
if p.Tag != "" {
|
||||
where = append(where, `EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
JOIN tags t ON t.id = vt.tag_id
|
||||
WHERE vt.video_id = videos.id AND t.label = ? COLLATE NOCASE
|
||||
)`)
|
||||
where = append(where, videoMatchesTagLabelSQL("videos"))
|
||||
args = append(args, p.Tag)
|
||||
}
|
||||
if p.Category != "" && p.Category != "all" {
|
||||
@@ -865,28 +932,25 @@ func (c *Catalog) CountVisibleVideos(ctx context.Context) (int, error) {
|
||||
// 如果剩余可选数量 < limit,就返回所有可选项;调用方负责判断是否需要开新一轮。
|
||||
// limit <= 0 时返回 nil, nil。
|
||||
func (c *Catalog) RandomVideosExcluding(ctx context.Context, excludeIDs []string, limit int) ([]*Video, error) {
|
||||
return c.randomVideosExcluding(ctx, excludeIDs, limit, false)
|
||||
}
|
||||
|
||||
func (c *Catalog) RandomVideosWithReadyThumbnailsExcluding(ctx context.Context, excludeIDs []string, limit int) ([]*Video, error) {
|
||||
return c.randomVideosExcluding(ctx, excludeIDs, limit, true)
|
||||
}
|
||||
|
||||
func (c *Catalog) randomVideosExcluding(ctx context.Context, excludeIDs []string, limit int, thumbnailReadyOnly bool) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// 去重 excludeIDs,过滤空串
|
||||
seen := make(map[string]struct{}, len(excludeIDs))
|
||||
cleaned := make([]string, 0, len(excludeIDs))
|
||||
for _, id := range excludeIDs {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
cleaned = append(cleaned, id)
|
||||
}
|
||||
|
||||
cleaned := cleanVideoIDs(excludeIDs)
|
||||
args := make([]any, 0, len(cleaned)+1)
|
||||
whereSQL := `WHERE COALESCE(hidden, 0) = 0
|
||||
AND ` + uniqueVideoWhereSQL
|
||||
if thumbnailReadyOnly {
|
||||
whereSQL += " AND COALESCE(thumbnail_url, '') != ''"
|
||||
}
|
||||
if len(cleaned) > 0 {
|
||||
placeholders := strings.Repeat("?,", len(cleaned))
|
||||
placeholders = placeholders[:len(placeholders)-1]
|
||||
@@ -921,6 +985,175 @@ func (c *Catalog) RandomVideosExcluding(ctx context.Context, excludeIDs []string
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func cleanVideoIDs(ids []string) []string {
|
||||
seen := make(map[string]struct{}, len(ids))
|
||||
cleaned := make([]string, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
cleaned = append(cleaned, id)
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func cleanTagLabels(labels []string) []string {
|
||||
seen := make(map[string]struct{}, len(labels))
|
||||
cleaned := make([]string, 0, len(labels))
|
||||
for _, label := range labels {
|
||||
label = strings.TrimSpace(label)
|
||||
if label == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(label)
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
cleaned = append(cleaned, label)
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func (c *Catalog) LeastPopulatedVisibleUniqueTag(ctx context.Context, labels []string) (string, error) {
|
||||
cleaned := cleanTagLabels(labels)
|
||||
bestLabel := ""
|
||||
bestCount := 0
|
||||
for _, label := range cleaned {
|
||||
var count int
|
||||
if err := c.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*)
|
||||
FROM videos
|
||||
WHERE COALESCE(hidden, 0) = 0
|
||||
AND `+uniqueVideoWhereSQL+`
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
JOIN tags t ON t.id = vt.tag_id
|
||||
WHERE vt.video_id = videos.id
|
||||
AND t.label = ? COLLATE NOCASE
|
||||
)`,
|
||||
label,
|
||||
).Scan(&count); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if count == 0 {
|
||||
continue
|
||||
}
|
||||
if bestLabel == "" || count < bestCount {
|
||||
bestLabel = label
|
||||
bestCount = count
|
||||
}
|
||||
}
|
||||
return bestLabel, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) RandomVideosByTagExcluding(ctx context.Context, tag string, excludeIDs []string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
tag = strings.TrimSpace(tag)
|
||||
if tag == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
cleaned := cleanVideoIDs(excludeIDs)
|
||||
args := make([]any, 0, len(cleaned)+2)
|
||||
args = append(args, tag)
|
||||
whereSQL := `WHERE COALESCE(hidden, 0) = 0
|
||||
AND ` + uniqueVideoWhereSQL + `
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
JOIN tags t ON t.id = vt.tag_id
|
||||
WHERE vt.video_id = videos.id
|
||||
AND t.label = ? COLLATE NOCASE
|
||||
)`
|
||||
if len(cleaned) > 0 {
|
||||
placeholders := strings.Repeat("?,", len(cleaned))
|
||||
placeholders = placeholders[:len(placeholders)-1]
|
||||
whereSQL += " AND id NOT IN (" + placeholders + ")"
|
||||
for _, id := range cleaned {
|
||||
args = append(args, id)
|
||||
}
|
||||
}
|
||||
args = append(args, limit)
|
||||
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos `+whereSQL+`
|
||||
ORDER BY RANDOM() LIMIT ?`,
|
||||
args...,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) RandomVideosForPreferredVideoExcluding(ctx context.Context, preferredVideoID string, excludeIDs []string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
preferredVideoID = strings.TrimSpace(preferredVideoID)
|
||||
if preferredVideoID == "" {
|
||||
return c.RandomVideosExcluding(ctx, excludeIDs, limit)
|
||||
}
|
||||
|
||||
preferredExclude := append([]string{}, excludeIDs...)
|
||||
preferredExclude = append(preferredExclude, preferredVideoID)
|
||||
|
||||
preferred, err := c.GetVideo(ctx, preferredVideoID)
|
||||
if err != nil || preferred == nil || preferred.Hidden || len(preferred.Tags) == 0 {
|
||||
return c.RandomVideosExcluding(ctx, preferredExclude, limit)
|
||||
}
|
||||
tag, err := c.LeastPopulatedVisibleUniqueTag(ctx, preferred.Tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tag == "" {
|
||||
return c.RandomVideosExcluding(ctx, preferredExclude, limit)
|
||||
}
|
||||
|
||||
items, err := c.RandomVideosByTagExcluding(ctx, tag, preferredExclude, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(items) >= limit {
|
||||
return items, nil
|
||||
}
|
||||
|
||||
mergedExclude := make([]string, 0, len(preferredExclude)+len(items))
|
||||
mergedExclude = append(mergedExclude, preferredExclude...)
|
||||
for _, item := range items {
|
||||
if item != nil {
|
||||
mergedExclude = append(mergedExclude, item.ID)
|
||||
}
|
||||
}
|
||||
fallback, err := c.RandomVideosExcluding(ctx, mergedExclude, limit-len(items))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(items, fallback...), nil
|
||||
}
|
||||
|
||||
type DriveTeaserCounts struct {
|
||||
Ready int
|
||||
Pending int
|
||||
@@ -928,9 +1161,10 @@ type DriveTeaserCounts struct {
|
||||
}
|
||||
|
||||
type DriveThumbnailCounts struct {
|
||||
Ready int
|
||||
Pending int
|
||||
Failed int
|
||||
Ready int
|
||||
Pending int
|
||||
Failed int
|
||||
DurationPending int
|
||||
}
|
||||
|
||||
type DriveFingerprintCounts struct {
|
||||
@@ -974,9 +1208,12 @@ func (c *Catalog) CountThumbnailsByDrive(ctx context.Context) (map[string]DriveT
|
||||
`SELECT drive_id,
|
||||
COUNT(CASE WHEN COALESCE(thumbnail_url, '') != '' THEN 1 END) AS ready_count,
|
||||
COUNT(CASE WHEN COALESCE(thumbnail_url, '') = ''
|
||||
AND COALESCE(thumbnail_status, 'pending') != 'failed' THEN 1 END) AS pending_count,
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('failed', 'skipped') THEN 1 END) AS pending_count,
|
||||
COUNT(CASE WHEN COALESCE(thumbnail_url, '') = ''
|
||||
AND COALESCE(thumbnail_status, 'pending') = 'failed' THEN 1 END) AS failed_count
|
||||
AND COALESCE(thumbnail_status, 'pending') = 'failed' THEN 1 END) AS failed_count,
|
||||
COUNT(CASE WHEN COALESCE(thumbnail_url, '') != ''
|
||||
AND COALESCE(duration_seconds, 0) <= 0
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('failed', 'skipped') THEN 1 END) AS duration_pending_count
|
||||
FROM videos
|
||||
WHERE COALESCE(hidden, 0) = 0
|
||||
AND `+uniqueVideoWhereSQL+`
|
||||
@@ -990,7 +1227,7 @@ func (c *Catalog) CountThumbnailsByDrive(ctx context.Context) (map[string]DriveT
|
||||
for rows.Next() {
|
||||
var driveID string
|
||||
var counts DriveThumbnailCounts
|
||||
if err := rows.Scan(&driveID, &counts.Ready, &counts.Pending, &counts.Failed); err != nil {
|
||||
if err := rows.Scan(&driveID, &counts.Ready, &counts.Pending, &counts.Failed, &counts.DurationPending); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[driveID] = counts
|
||||
@@ -1197,10 +1434,11 @@ func (c *Catalog) ClearGeneratedAssets(ctx context.Context, videoID string, clea
|
||||
// ---------- Drive ----------
|
||||
|
||||
type Drive struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
RootID string `json:"rootId"`
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
RootID string `json:"rootId"`
|
||||
// Deprecated: 扫描入口固定等于 RootID;字段保留用于兼容旧数据/API。
|
||||
ScanRootID string `json:"scanRootId"`
|
||||
Credentials map[string]string `json:"credentials,omitempty"`
|
||||
Status string `json:"status"`
|
||||
@@ -1218,6 +1456,7 @@ type Drive struct {
|
||||
}
|
||||
|
||||
func (c *Catalog) UpsertDrive(ctx context.Context, d *Drive) error {
|
||||
normalizeDriveRootFields(d)
|
||||
cred, _ := json.Marshal(d.Credentials)
|
||||
skipDirs := d.SkipDirIDs
|
||||
if skipDirs == nil {
|
||||
@@ -1248,6 +1487,46 @@ ON CONFLICT(id) DO UPDATE SET
|
||||
return err
|
||||
}
|
||||
|
||||
func normalizeDriveRootFields(d *Drive) {
|
||||
if d == nil {
|
||||
return
|
||||
}
|
||||
d.RootID = normalizeDriveRootID(d.Kind, d.RootID)
|
||||
d.ScanRootID = d.RootID
|
||||
}
|
||||
|
||||
func normalizeDriveRootID(kind, rootID string) string {
|
||||
rootID = strings.TrimSpace(rootID)
|
||||
switch kind {
|
||||
case "pikpak":
|
||||
if rootID == "0" {
|
||||
return ""
|
||||
}
|
||||
return rootID
|
||||
case "onedrive", "googledrive":
|
||||
if rootID == "" {
|
||||
return "root"
|
||||
}
|
||||
return rootID
|
||||
case "localstorage", "spider91":
|
||||
return "/"
|
||||
default:
|
||||
if rootID == "" {
|
||||
return "0"
|
||||
}
|
||||
return rootID
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Catalog) syncDriveScanRootIDToRootID(ctx context.Context) error {
|
||||
_, err := c.db.ExecContext(ctx, `
|
||||
UPDATE drives
|
||||
SET scan_root_id = root_id,
|
||||
updated_at = ?
|
||||
WHERE COALESCE(scan_root_id, '') != COALESCE(root_id, '')`, time.Now().UnixMilli())
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) ListDrives(ctx context.Context) ([]*Drive, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT id, kind, name, root_id, COALESCE(scan_root_id, ''), COALESCE(credentials, '{}'), status, COALESCE(last_error, ''), COALESCE(teaser_enabled, 1), COALESCE(skip_dir_ids, '[]'), created_at, updated_at FROM drives ORDER BY created_at ASC`)
|
||||
if err != nil {
|
||||
@@ -1265,6 +1544,7 @@ func (c *Catalog) ListDrives(ctx context.Context) ([]*Drive, error) {
|
||||
}
|
||||
_ = json.Unmarshal([]byte(credsStr), &d.Credentials)
|
||||
_ = json.Unmarshal([]byte(skipDirsStr), &d.SkipDirIDs)
|
||||
normalizeDriveRootFields(d)
|
||||
d.TeaserEnabled = teaserEnabled != 0
|
||||
d.CreatedAt = time.UnixMilli(createdAt)
|
||||
d.UpdatedAt = time.UnixMilli(updatedAt)
|
||||
@@ -1284,6 +1564,7 @@ func (c *Catalog) GetDrive(ctx context.Context, id string) (*Drive, error) {
|
||||
}
|
||||
_ = json.Unmarshal([]byte(credsStr), &d.Credentials)
|
||||
_ = json.Unmarshal([]byte(skipDirsStr), &d.SkipDirIDs)
|
||||
normalizeDriveRootFields(d)
|
||||
d.TeaserEnabled = teaserEnabled != 0
|
||||
d.CreatedAt = time.UnixMilli(createdAt)
|
||||
d.UpdatedAt = time.UnixMilli(updatedAt)
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUpsertDriveUsesRootIDAsScanRootID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: "drive",
|
||||
Kind: "p115",
|
||||
Name: "115",
|
||||
RootID: "root-folder",
|
||||
ScanRootID: "ignored-scan-root",
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert drive: %v", err)
|
||||
}
|
||||
|
||||
got, err := cat.GetDrive(ctx, "drive")
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.RootID != "root-folder" {
|
||||
t.Fatalf("rootId = %q, want root-folder", got.RootID)
|
||||
}
|
||||
if got.ScanRootID != "root-folder" {
|
||||
t.Fatalf("scanRootId = %q, want root-folder", got.ScanRootID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertDriveDefaultsRootIDByKind(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
cases := []struct {
|
||||
id string
|
||||
kind string
|
||||
want string
|
||||
}{
|
||||
{id: "p115", kind: "p115", want: "0"},
|
||||
{id: "pikpak", kind: "pikpak", want: ""},
|
||||
{id: "onedrive", kind: "onedrive", want: "root"},
|
||||
{id: "googledrive", kind: "googledrive", want: "root"},
|
||||
{id: "localstorage", kind: "localstorage", want: "/"},
|
||||
{id: "spider91", kind: "spider91", want: "/"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: tc.id,
|
||||
Kind: tc.kind,
|
||||
Name: tc.kind,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert %s: %v", tc.kind, err)
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, tc.id)
|
||||
if err != nil {
|
||||
t.Fatalf("get %s: %v", tc.kind, err)
|
||||
}
|
||||
if got.RootID != tc.want {
|
||||
t.Fatalf("%s rootId = %q, want %q", tc.kind, got.RootID, tc.want)
|
||||
}
|
||||
if got.ScanRootID != tc.want {
|
||||
t.Fatalf("%s scanRootId = %q, want %q", tc.kind, got.ScanRootID, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertDriveIgnoresRootIDForLocalStorageAndSpider91(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, tc := range []struct {
|
||||
id string
|
||||
kind string
|
||||
}{
|
||||
{id: "localstorage", kind: "localstorage"},
|
||||
{id: "spider91", kind: "spider91"},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: tc.id,
|
||||
Kind: tc.kind,
|
||||
Name: tc.kind,
|
||||
RootID: "manual-root",
|
||||
ScanRootID: "manual-scan-root",
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert %s: %v", tc.kind, err)
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, tc.id)
|
||||
if err != nil {
|
||||
t.Fatalf("get %s: %v", tc.kind, err)
|
||||
}
|
||||
if got.RootID != "/" {
|
||||
t.Fatalf("%s rootId = %q, want /", tc.kind, got.RootID)
|
||||
}
|
||||
if got.ScanRootID != "/" {
|
||||
t.Fatalf("%s scanRootId = %q, want /", tc.kind, got.ScanRootID)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,7 +17,8 @@ CREATE TABLE IF NOT EXISTS videos (
|
||||
ext TEXT,
|
||||
quality TEXT, -- HD / SD
|
||||
thumbnail_url TEXT,
|
||||
thumbnail_status TEXT DEFAULT 'pending', -- pending / ready / failed
|
||||
thumbnail_status TEXT DEFAULT 'pending', -- pending / ready / failed / skipped
|
||||
thumbnail_failures INTEGER DEFAULT 0, -- consecutive transient thumbnail generation failures
|
||||
preview_file_id TEXT, -- deprecated: 旧版回写网盘后的 teaser file id
|
||||
preview_local TEXT, -- 本地 teaser 路径(兜底)
|
||||
preview_status TEXT DEFAULT 'pending', -- pending / ready / failed
|
||||
@@ -61,13 +62,21 @@ CREATE TABLE IF NOT EXISTS video_tags (
|
||||
CREATE INDEX IF NOT EXISTS idx_video_tags_tag ON video_tags(tag_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_video_tags_video ON video_tags(video_id);
|
||||
|
||||
-- 用户手动删除过的非系统标签。自动扫描/迁移不再重新创建同名标签;
|
||||
-- 管理员手动新建同名标签时会移除这里的记录。
|
||||
CREATE TABLE IF NOT EXISTS deleted_tags (
|
||||
label TEXT PRIMARY KEY COLLATE NOCASE,
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
deleted_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- 网盘账户
|
||||
CREATE TABLE IF NOT EXISTS drives (
|
||||
id TEXT PRIMARY KEY,
|
||||
kind TEXT NOT NULL, -- quark / p115 / pikpak / wopan / onedrive / localstorage / spider91
|
||||
kind TEXT NOT NULL, -- quark / p115 / pikpak / wopan / onedrive / googledrive / localstorage / spider91
|
||||
name TEXT NOT NULL,
|
||||
root_id TEXT NOT NULL DEFAULT '0',
|
||||
scan_root_id TEXT, -- 扫描起点(默认 root_id)
|
||||
scan_root_id TEXT, -- deprecated: 扫描起点固定等于 root_id
|
||||
credentials TEXT, -- JSON: cookie / refresh_token 等
|
||||
status TEXT DEFAULT 'disconnected', -- disconnected / ok / error
|
||||
last_error TEXT,
|
||||
|
||||
@@ -109,3 +109,227 @@ func TestRandomVideosExcluding(t *testing.T) {
|
||||
t.Fatalf("limit 0 should return nil, got %v", got4)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosWithReadyThumbnailsExcluding(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < 4; i++ {
|
||||
id := "ready-" + string(rune('a'+i))
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: "f-" + id,
|
||||
Title: id,
|
||||
ThumbnailURL: "/p/thumb/" + id,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
id := "pending-" + string(rune('a'+i))
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: "f-" + id,
|
||||
Title: id,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosWithReadyThumbnailsExcluding(ctx, []string{"ready-a"}, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("random ready excluding: %v", err)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("ready random count = %d, want 3", len(got))
|
||||
}
|
||||
for _, v := range got {
|
||||
if v.ID == "ready-a" {
|
||||
t.Fatal("excluded ready video was returned")
|
||||
}
|
||||
if v.ThumbnailURL == "" {
|
||||
t.Fatalf("pending video %q was returned", v.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoChoosesLeastPopulatedTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "current", DriveID: "drive", FileID: "f-current", Title: "current", Tags: []string{"common", "rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-1", DriveID: "drive", FileID: "f-common-1", Title: "common 1", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-2", DriveID: "drive", FileID: "f-common-2", Title: "common 2", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "rare-1", DriveID: "drive", FileID: "f-rare-1", Title: "rare 1", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
tag, err := cat.LeastPopulatedVisibleUniqueTag(ctx, []string{"common", "rare"})
|
||||
if err != nil {
|
||||
t.Fatalf("least populated tag: %v", err)
|
||||
}
|
||||
if tag != "rare" {
|
||||
t.Fatalf("least populated tag = %q, want rare", tag)
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosForPreferredVideoExcluding(ctx, "current", []string{"current"}, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("random preferred: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID != "rare-1" {
|
||||
t.Fatalf("preferred result = %#v, want rare-1", videoIDs(got))
|
||||
}
|
||||
|
||||
got, err = cat.RandomVideosForPreferredVideoExcluding(ctx, "current", nil, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("random preferred without explicit exclude: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID == "current" {
|
||||
t.Fatalf("preferred result without explicit exclude = %#v, should not return current", videoIDs(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoFallsBackToFillBatch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "current", DriveID: "drive", FileID: "f-current", Title: "current", Tags: []string{"common", "rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-1", DriveID: "drive", FileID: "f-common-1", Title: "common 1", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-2", DriveID: "drive", FileID: "f-common-2", Title: "common 2", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "rare-1", DriveID: "drive", FileID: "f-rare-1", Title: "rare 1", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "hidden-rare", DriveID: "drive", FileID: "f-hidden-rare", Title: "hidden rare", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
if err := cat.HideVideo(ctx, "hidden-rare"); err != nil {
|
||||
t.Fatalf("hide hidden-rare: %v", err)
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosForPreferredVideoExcluding(ctx, "current", []string{"current"}, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("random preferred: %v", err)
|
||||
}
|
||||
ids := videoIDs(got)
|
||||
if len(ids) != 3 {
|
||||
t.Fatalf("result ids = %#v, want 3 items", ids)
|
||||
}
|
||||
for _, excluded := range []string{"current", "hidden-rare"} {
|
||||
if hasVideoID(ids, excluded) {
|
||||
t.Fatalf("result ids = %#v, should not include %s", ids, excluded)
|
||||
}
|
||||
}
|
||||
if !hasVideoID(ids, "rare-1") {
|
||||
t.Fatalf("result ids = %#v, want rare-1 from least populated tag", ids)
|
||||
}
|
||||
if len(uniqueVideoIDs(ids)) != len(ids) {
|
||||
t.Fatalf("result ids = %#v, want no duplicates", ids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoFallbacksWhenPreferenceUnavailable(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "untagged", DriveID: "drive", FileID: "f-untagged", Title: "untagged", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "visible-1", DriveID: "drive", FileID: "f-visible-1", Title: "visible 1", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "visible-2", DriveID: "drive", FileID: "f-visible-2", Title: "visible 2", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosForPreferredVideoExcluding(ctx, "missing", []string{"untagged"}, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("random missing preferred: %v", err)
|
||||
}
|
||||
if !sameVideoIDSet(videoIDs(got), []string{"visible-1", "visible-2"}) {
|
||||
t.Fatalf("missing preferred ids = %#v, want visible fallback videos", videoIDs(got))
|
||||
}
|
||||
|
||||
got, err = cat.RandomVideosForPreferredVideoExcluding(ctx, "untagged", []string{"untagged"}, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("random untagged preferred: %v", err)
|
||||
}
|
||||
if !sameVideoIDSet(videoIDs(got), []string{"visible-1", "visible-2"}) {
|
||||
t.Fatalf("untagged preferred ids = %#v, want visible fallback videos", videoIDs(got))
|
||||
}
|
||||
}
|
||||
|
||||
func videoIDs(videos []*Video) []string {
|
||||
ids := make([]string, 0, len(videos))
|
||||
for _, v := range videos {
|
||||
ids = append(ids, v.ID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func hasVideoID(ids []string, want string) bool {
|
||||
for _, id := range ids {
|
||||
if id == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func uniqueVideoIDs(ids []string) map[string]struct{} {
|
||||
seen := make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
seen[id] = struct{}{}
|
||||
}
|
||||
return seen
|
||||
}
|
||||
|
||||
func sameVideoIDSet(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
seen := make(map[string]int, len(a))
|
||||
for _, value := range a {
|
||||
seen[value]++
|
||||
}
|
||||
for _, value := range b {
|
||||
if seen[value] == 0 {
|
||||
return false
|
||||
}
|
||||
seen[value]--
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -17,6 +17,8 @@ import (
|
||||
)
|
||||
|
||||
var ErrUnknownTag = errors.New("unknown tag")
|
||||
var ErrSystemTag = errors.New("system tag cannot be deleted")
|
||||
var ErrDeletedTag = errors.New("tag was previously deleted")
|
||||
|
||||
const avTagLabel = "AV"
|
||||
|
||||
@@ -61,6 +63,9 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "thumbnail_status", "TEXT DEFAULT 'pending'"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "thumbnail_failures", "INTEGER DEFAULT 0"); err != nil {
|
||||
return err
|
||||
}
|
||||
// drives.teaser_enabled:每盘 teaser 开关,替代旧的全局 preview.enabled。
|
||||
// 升级路径:直接让 ALTER TABLE 的 DEFAULT 1 兜底 —— 每个现存 drive 都默认开启,
|
||||
// 不读旧的 settings.preview.enabled 字段。这样老用户即便之前关过全局开关,
|
||||
@@ -74,6 +79,9 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "drives", "skip_dir_ids", "TEXT NOT NULL DEFAULT '[]'"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.syncDriveScanRootIDToRootID(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
// 一次性修正:早期版本(短暂存在过)会把现存 drive 的 teaser_enabled 同步成
|
||||
// 旧的全局 preview.enabled 值,导致升级后所有 drive 都是关。"默认开启"约定下,
|
||||
// 这里一次性把所有 drive 强制重置为 1,并用 marker setting 记号,避免之后
|
||||
@@ -201,8 +209,9 @@ func (c *Catalog) resetDriveTeaserEnabledToDefaultOnce(ctx context.Context) erro
|
||||
// - 管理员凭直觉认知字段名时会被误导
|
||||
//
|
||||
// 修正策略:
|
||||
// - thumbnail_url 非空 + status 非 'ready' + status 非 'failed' → 改成 'ready'
|
||||
// - thumbnail_url 非空 + status 非 'ready' + status 非 'failed' + status 非 'skipped' → 改成 'ready'
|
||||
// - status='failed' 不动(这是 worker 显式标的失败,要保留以便管理员手动重生)
|
||||
// - status='skipped' 不动(已有封面但时长探测不可用,避免重启后重复排队)
|
||||
//
|
||||
// 幂等保证:marker setting 写过就不再跑,避免每次重启都 update 一遍。
|
||||
func (c *Catalog) reconcileThumbnailStatusOnce(ctx context.Context) error {
|
||||
@@ -219,7 +228,7 @@ UPDATE videos
|
||||
SET thumbnail_status = 'ready',
|
||||
updated_at = ?
|
||||
WHERE COALESCE(thumbnail_url, '') != ''
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('ready', 'failed')
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('ready', 'failed', 'skipped')
|
||||
`, time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
return fmt.Errorf("reconcile thumbnail_status: %w", err)
|
||||
@@ -293,7 +302,15 @@ func (c *Catalog) classifySystemTags(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (c *Catalog) backfillVideoTags(ctx context.Context) error {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT id, COALESCE(tags, '[]') FROM videos`)
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT id, COALESCE(tags, '[]')
|
||||
FROM videos
|
||||
WHERE COALESCE(tags, '') NOT IN ('', '[]', 'null')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
WHERE vt.video_id = videos.id
|
||||
)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -310,11 +327,14 @@ func (c *Catalog) backfillVideoTags(ctx context.Context) error {
|
||||
if len(labels) == 0 {
|
||||
continue
|
||||
}
|
||||
if err := c.addVideoTags(ctx, videoID, labels, "legacy", true); err != nil {
|
||||
added, err := c.addVideoTags(ctx, videoID, labels, "legacy", true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return err
|
||||
if added {
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -362,6 +382,9 @@ GROUP BY category`)
|
||||
if !LooksLikeCollectionTag(stat.category) {
|
||||
continue
|
||||
}
|
||||
if c.tagDeleted(ctx, stat.category) {
|
||||
continue
|
||||
}
|
||||
if _, err := c.ensureTag(ctx, stat.category, nil, "collection"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -380,12 +403,178 @@ func (c *Catalog) CreateTagAndClassify(ctx context.Context, label string, aliase
|
||||
return c.classifyTag(ctx, tag)
|
||||
}
|
||||
|
||||
func (c *Catalog) EnsureTagForVideoIDPrefix(ctx context.Context, prefix, label string, aliases []string, source string) (int, error) {
|
||||
prefix = strings.TrimSpace(prefix)
|
||||
if prefix == "" {
|
||||
return 0, errors.New("video id prefix is required")
|
||||
}
|
||||
tag, err := c.ensureTag(ctx, label, aliases, source)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT v.id
|
||||
FROM videos v
|
||||
WHERE v.id LIKE ? || '%'
|
||||
AND COALESCE(v.tags_manual, 0) = 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
WHERE vt.video_id = v.id
|
||||
AND vt.tag_id = ?
|
||||
)
|
||||
ORDER BY v.id ASC`, prefix, tag.ID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var videoIDs []string
|
||||
for rows.Next() {
|
||||
var videoID string
|
||||
if err := rows.Scan(&videoID); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
videoIDs = append(videoIDs, videoID)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, videoID := range videoIDs {
|
||||
if err := c.insertVideoTag(ctx, videoID, tag.ID, "auto"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return len(videoIDs), nil
|
||||
}
|
||||
|
||||
func (c *Catalog) DeleteTag(ctx context.Context, tagID int64) (int, error) {
|
||||
tx, err := c.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
tag, err := c.getTagByIDTx(ctx, tx, tagID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if tag.Source == "system" {
|
||||
return 0, ErrSystemTag
|
||||
}
|
||||
|
||||
rows, err := tx.QueryContext(ctx, `SELECT video_id FROM video_tags WHERE tag_id = ?`, tagID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var videoIDs []string
|
||||
for rows.Next() {
|
||||
var videoID string
|
||||
if err := rows.Scan(&videoID); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
videoIDs = append(videoIDs, videoID)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM video_tags WHERE tag_id = ?`, tagID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM tags WHERE id = ?`, tagID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := markDeletedTagTx(ctx, tx, tag); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
for _, videoID := range videoIDs {
|
||||
manual := hasManualTagsTx(ctx, tx, videoID)
|
||||
if err := syncVideoTagsJSONTx(ctx, tx, videoID, manual); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(videoIDs), nil
|
||||
}
|
||||
|
||||
func (c *Catalog) ListTags(ctx context.Context) ([]Tag, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT t.id, t.label, t.aliases, t.source, COUNT(v.id) AS cnt
|
||||
WITH tagged_tags AS (
|
||||
SELECT vt.tag_id,
|
||||
tagged.id,
|
||||
COALESCE(tagged.content_hash, '') AS content_hash,
|
||||
COALESCE(tagged.sampled_sha256, '') AS sampled_sha256,
|
||||
tagged.size_bytes,
|
||||
COALESCE(tagged.file_name, '') AS file_name
|
||||
FROM video_tags vt
|
||||
JOIN videos tagged ON tagged.id = vt.video_id
|
||||
WHERE COALESCE(tagged.hidden, 0) = 0
|
||||
),
|
||||
tag_candidates AS (
|
||||
SELECT tag_id, id AS video_id
|
||||
FROM tagged_tags
|
||||
UNION ALL
|
||||
SELECT tag_id,
|
||||
(SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_tags.content_hash != ''
|
||||
AND canonical.content_hash = tagged_tags.content_hash
|
||||
AND COALESCE(canonical.content_hash, '') != ''
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_tags
|
||||
WHERE content_hash != ''
|
||||
UNION ALL
|
||||
SELECT tag_id,
|
||||
(SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_tags.sampled_sha256 != ''
|
||||
AND tagged_tags.size_bytes > 0
|
||||
AND canonical.sampled_sha256 = tagged_tags.sampled_sha256
|
||||
AND canonical.size_bytes = tagged_tags.size_bytes
|
||||
AND COALESCE(canonical.sampled_sha256, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_tags
|
||||
WHERE sampled_sha256 != '' AND size_bytes > 0
|
||||
UNION ALL
|
||||
SELECT tag_id,
|
||||
(SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_tags.file_name != ''
|
||||
AND tagged_tags.size_bytes > 0
|
||||
AND canonical.file_name = tagged_tags.file_name
|
||||
AND canonical.size_bytes = tagged_tags.size_bytes
|
||||
AND COALESCE(canonical.file_name, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_tags
|
||||
WHERE file_name != '' AND size_bytes > 0
|
||||
)
|
||||
SELECT t.id, t.label, t.aliases, t.source, COUNT(DISTINCT videos.id) AS cnt
|
||||
FROM tags t
|
||||
LEFT JOIN video_tags vt ON vt.tag_id = t.id
|
||||
LEFT JOIN videos v ON v.id = vt.video_id AND COALESCE(v.hidden, 0) = 0
|
||||
LEFT JOIN tag_candidates tc ON tc.tag_id = t.id AND tc.video_id IS NOT NULL
|
||||
LEFT JOIN videos ON videos.id = tc.video_id
|
||||
AND COALESCE(videos.hidden, 0) = 0
|
||||
AND `+uniqueVideoWhereSQL+`
|
||||
GROUP BY t.id, t.label, t.aliases, t.source
|
||||
ORDER BY cnt DESC, t.label ASC`)
|
||||
if err != nil {
|
||||
@@ -403,6 +592,66 @@ ORDER BY cnt DESC, t.label ASC`)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func videoMatchesTagLabelSQL(videoAlias string) string {
|
||||
return fmt.Sprintf(`%s.id IN (
|
||||
WITH tagged_videos AS (
|
||||
SELECT tagged.id,
|
||||
COALESCE(tagged.content_hash, '') AS content_hash,
|
||||
COALESCE(tagged.sampled_sha256, '') AS sampled_sha256,
|
||||
tagged.size_bytes,
|
||||
COALESCE(tagged.file_name, '') AS file_name
|
||||
FROM video_tags vt
|
||||
JOIN tags tag_filter ON tag_filter.id = vt.tag_id
|
||||
JOIN videos tagged ON tagged.id = vt.video_id
|
||||
WHERE tag_filter.label = ? COLLATE NOCASE
|
||||
AND COALESCE(tagged.hidden, 0) = 0
|
||||
),
|
||||
tag_candidates AS (
|
||||
SELECT id AS video_id
|
||||
FROM tagged_videos
|
||||
UNION ALL
|
||||
SELECT (SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_videos.content_hash != ''
|
||||
AND canonical.content_hash = tagged_videos.content_hash
|
||||
AND COALESCE(canonical.content_hash, '') != ''
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_videos
|
||||
WHERE content_hash != ''
|
||||
UNION ALL
|
||||
SELECT (SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_videos.sampled_sha256 != ''
|
||||
AND tagged_videos.size_bytes > 0
|
||||
AND canonical.sampled_sha256 = tagged_videos.sampled_sha256
|
||||
AND canonical.size_bytes = tagged_videos.size_bytes
|
||||
AND COALESCE(canonical.sampled_sha256, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_videos
|
||||
WHERE sampled_sha256 != '' AND size_bytes > 0
|
||||
UNION ALL
|
||||
SELECT (SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_videos.file_name != ''
|
||||
AND tagged_videos.size_bytes > 0
|
||||
AND canonical.file_name = tagged_videos.file_name
|
||||
AND canonical.size_bytes = tagged_videos.size_bytes
|
||||
AND COALESCE(canonical.file_name, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_videos
|
||||
WHERE file_name != '' AND size_bytes > 0
|
||||
)
|
||||
SELECT video_id
|
||||
FROM tag_candidates
|
||||
WHERE video_id IS NOT NULL
|
||||
)`, videoAlias)
|
||||
}
|
||||
|
||||
func (c *Catalog) SetManualVideoTags(ctx context.Context, videoID string, labels []string) error {
|
||||
if _, err := c.GetVideo(ctx, videoID); err != nil {
|
||||
return err
|
||||
@@ -453,6 +702,9 @@ func (c *Catalog) EnsureCollectionTag(ctx context.Context, label string) (string
|
||||
if !LooksLikeCollectionTag(label) {
|
||||
return "", false, nil
|
||||
}
|
||||
if c.tagDeleted(ctx, label) {
|
||||
return "", false, nil
|
||||
}
|
||||
if !c.tagExists(ctx, label) {
|
||||
count, err := c.categoryVideoCount(ctx, label)
|
||||
if err != nil {
|
||||
@@ -484,6 +736,14 @@ func (c *Catalog) ensureTag(ctx context.Context, label string, aliases []string,
|
||||
if source == "" {
|
||||
source = "user"
|
||||
}
|
||||
if source != "system" && source != "user" && c.tagDeleted(ctx, label) {
|
||||
return Tag{}, ErrDeletedTag
|
||||
}
|
||||
if source == "system" || source == "user" {
|
||||
if err := c.restoreDeletedTag(ctx, label); err != nil {
|
||||
return Tag{}, err
|
||||
}
|
||||
}
|
||||
aliases = cleanAliases(aliases, label)
|
||||
aliasesJSON, _ := json.Marshal(aliases)
|
||||
now := time.Now().UnixMilli()
|
||||
@@ -510,6 +770,10 @@ func (c *Catalog) getTagByLabel(ctx context.Context, label string) (Tag, error)
|
||||
}
|
||||
|
||||
func (c *Catalog) classifyTag(ctx context.Context, tag Tag) (int, error) {
|
||||
existingIDs, err := c.videoIDSetForTagID(ctx, tag.ID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT id, title, COALESCE(author, ''), COALESCE(category, ''), COALESCE(tags_manual, 0)
|
||||
FROM videos`)
|
||||
@@ -541,13 +805,14 @@ FROM videos`)
|
||||
continue
|
||||
}
|
||||
}
|
||||
added, err := c.addVideoTag(ctx, videoID, tag.Label, "auto", false)
|
||||
if err != nil {
|
||||
if existingIDs[videoID] {
|
||||
continue
|
||||
}
|
||||
if err := c.insertVideoTag(ctx, videoID, tag.ID, "auto"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if added {
|
||||
classified++
|
||||
}
|
||||
existingIDs[videoID] = true
|
||||
classified++
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -557,9 +822,15 @@ FROM videos`)
|
||||
|
||||
func (c *Catalog) replaceVideoTags(ctx context.Context, videoID string, labels []string, source string, manual bool, createMissing bool) error {
|
||||
labels = uniqueStrings(cleanLabels(labels))
|
||||
if source != "manual" {
|
||||
labels = c.filterDeletedTagLabels(ctx, labels)
|
||||
}
|
||||
if createMissing {
|
||||
for _, label := range labels {
|
||||
if _, err := c.ensureTag(ctx, label, nil, "legacy"); err != nil {
|
||||
if errors.Is(err, ErrDeletedTag) {
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -601,18 +872,33 @@ func (c *Catalog) replaceVideoTags(ctx context.Context, videoID string, labels [
|
||||
return c.syncVideoTagsJSON(ctx, videoID, manual)
|
||||
}
|
||||
|
||||
func (c *Catalog) addVideoTags(ctx context.Context, videoID string, labels []string, source string, createMissing bool) error {
|
||||
for _, label := range uniqueStrings(cleanLabels(labels)) {
|
||||
if _, err := c.addVideoTag(ctx, videoID, label, source, createMissing); err != nil {
|
||||
return err
|
||||
func (c *Catalog) addVideoTags(ctx context.Context, videoID string, labels []string, source string, createMissing bool) (bool, error) {
|
||||
labels = uniqueStrings(cleanLabels(labels))
|
||||
if source != "manual" {
|
||||
labels = c.filterDeletedTagLabels(ctx, labels)
|
||||
}
|
||||
changed := false
|
||||
for _, label := range labels {
|
||||
added, err := c.addVideoTag(ctx, videoID, label, source, createMissing)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if added {
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) addVideoTag(ctx context.Context, videoID, label, source string, createMissing bool) (bool, error) {
|
||||
if source != "manual" && c.tagDeleted(ctx, label) {
|
||||
return false, nil
|
||||
}
|
||||
if createMissing {
|
||||
if _, err := c.ensureTag(ctx, label, nil, "legacy"); err != nil {
|
||||
if errors.Is(err, ErrDeletedTag) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
@@ -631,12 +917,33 @@ func (c *Catalog) addVideoTag(ctx context.Context, videoID, label, source string
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) insertVideoTag(ctx context.Context, videoID string, tagID int64, source string) error {
|
||||
_, err := c.db.ExecContext(ctx,
|
||||
`INSERT OR IGNORE INTO video_tags (video_id, tag_id, source, created_at) VALUES (?, ?, ?, ?)`,
|
||||
videoID, tagID, source, time.Now().UnixMilli())
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) addCollectionTagToVideos(ctx context.Context, category string) error {
|
||||
return c.addTagToVideosByCategory(ctx, category, category, "auto")
|
||||
}
|
||||
|
||||
func (c *Catalog) addTagToVideosByCategory(ctx context.Context, category, label, source string) error {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT id FROM videos WHERE category = ? AND COALESCE(tags_manual, 0) = 0`, category)
|
||||
tag, err := c.getTagByLabel(ctx, label)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT v.id
|
||||
FROM videos v
|
||||
WHERE v.category = ?
|
||||
AND COALESCE(v.tags_manual, 0) = 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
WHERE vt.video_id = v.id
|
||||
AND vt.tag_id = ?
|
||||
)`, category, tag.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -655,7 +962,7 @@ func (c *Catalog) addTagToVideosByCategory(ctx context.Context, category, label,
|
||||
return err
|
||||
}
|
||||
for _, videoID := range videoIDs {
|
||||
if _, err := c.addVideoTag(ctx, videoID, label, source, false); err != nil {
|
||||
if err := c.insertVideoTag(ctx, videoID, tag.ID, source); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
@@ -739,6 +1046,23 @@ func (c *Catalog) videoIDsForTagID(ctx context.Context, tagID int64) ([]string,
|
||||
return videoIDs, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) videoIDSetForTagID(ctx context.Context, tagID int64) (map[string]bool, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT video_id FROM video_tags WHERE tag_id = ?`, tagID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var videoID string
|
||||
if err := rows.Scan(&videoID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[videoID] = true
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) validateTagsExist(ctx context.Context, labels []string) error {
|
||||
for _, label := range labels {
|
||||
if _, err := c.getTagByLabel(ctx, label); err != nil {
|
||||
@@ -804,6 +1128,39 @@ func (c *Catalog) tagExists(ctx context.Context, label string) bool {
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (c *Catalog) tagDeleted(ctx context.Context, label string) bool {
|
||||
label = cleanTagLabel(label)
|
||||
if label == "" {
|
||||
return false
|
||||
}
|
||||
var exists int
|
||||
err := c.db.QueryRowContext(ctx, `SELECT 1 FROM deleted_tags WHERE label = ? COLLATE NOCASE`, label).Scan(&exists)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (c *Catalog) filterDeletedTagLabels(ctx context.Context, labels []string) []string {
|
||||
if len(labels) == 0 {
|
||||
return labels
|
||||
}
|
||||
out := labels[:0]
|
||||
for _, label := range labels {
|
||||
if c.tagDeleted(ctx, label) {
|
||||
continue
|
||||
}
|
||||
out = append(out, label)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (c *Catalog) restoreDeletedTag(ctx context.Context, label string) error {
|
||||
label = cleanTagLabel(label)
|
||||
if label == "" {
|
||||
return nil
|
||||
}
|
||||
_, err := c.db.ExecContext(ctx, `DELETE FROM deleted_tags WHERE label = ? COLLATE NOCASE`, label)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) categoryVideoCount(ctx context.Context, category string) (int, error) {
|
||||
var count int
|
||||
err := c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM videos WHERE category = ?`, category).Scan(&count)
|
||||
@@ -817,6 +1174,71 @@ func (c *Catalog) getTagByLabelTx(ctx context.Context, tx *sql.Tx, label string)
|
||||
return scanTag(row)
|
||||
}
|
||||
|
||||
func (c *Catalog) getTagByIDTx(ctx context.Context, tx *sql.Tx, id int64) (Tag, error) {
|
||||
row := tx.QueryRowContext(ctx,
|
||||
`SELECT id, label, aliases, source, 0 FROM tags WHERE id = ?`,
|
||||
id)
|
||||
return scanTag(row)
|
||||
}
|
||||
|
||||
func hasManualTagsTx(ctx context.Context, tx *sql.Tx, videoID string) bool {
|
||||
var manual int
|
||||
err := tx.QueryRowContext(ctx, `SELECT COALESCE(tags_manual, 0) FROM videos WHERE id = ?`, videoID).Scan(&manual)
|
||||
return err == nil && manual == 1
|
||||
}
|
||||
|
||||
func markDeletedTagTx(ctx context.Context, tx *sql.Tx, tag Tag) error {
|
||||
label := cleanTagLabel(tag.Label)
|
||||
if label == "" {
|
||||
return nil
|
||||
}
|
||||
now := time.Now().UnixMilli()
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO deleted_tags (label, source, deleted_at)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(label) DO UPDATE SET
|
||||
source = excluded.source,
|
||||
deleted_at = excluded.deleted_at`, label, tag.Source, now)
|
||||
return err
|
||||
}
|
||||
|
||||
func syncVideoTagsJSONTx(ctx context.Context, tx *sql.Tx, videoID string, manual bool) error {
|
||||
rows, err := tx.QueryContext(ctx, `
|
||||
SELECT t.label
|
||||
FROM video_tags vt
|
||||
JOIN tags t ON t.id = vt.tag_id
|
||||
WHERE vt.video_id = ?
|
||||
ORDER BY t.id ASC`, videoID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var labels []string
|
||||
for rows.Next() {
|
||||
var label string
|
||||
if err := rows.Scan(&label); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
labels = append(labels, label)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
labelsJSON, _ := json.Marshal(labels)
|
||||
manualValue := 0
|
||||
if manual {
|
||||
manualValue = 1
|
||||
}
|
||||
_, err = tx.ExecContext(ctx,
|
||||
`UPDATE videos SET tags = ?, tags_manual = ?, updated_at = ? WHERE id = ?`,
|
||||
string(labelsJSON), manualValue, time.Now().UnixMilli(), videoID)
|
||||
return err
|
||||
}
|
||||
|
||||
type tagRowScanner interface {
|
||||
Scan(dest ...any) error
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package catalog
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -89,6 +90,32 @@ func TestListVideosNeedingThumbnailIncludesExistingThumbnailMissingDuration(t *t
|
||||
if count != 2 {
|
||||
t.Fatalf("count = %d, want 2", count)
|
||||
}
|
||||
|
||||
counts, err := cat.CountThumbnailsByDrive(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("count thumbnails by drive: %v", err)
|
||||
}
|
||||
if got := counts["drive"]; got.Ready != 2 || got.Pending != 1 || got.Failed != 1 || got.DurationPending != 1 {
|
||||
t.Fatalf("thumbnail counts = %#v, want ready=2 pending=1 failed=1 durationPending=1", got)
|
||||
}
|
||||
|
||||
if err := cat.UpdateVideoMeta(ctx, "duration-only", VideoMetaPatch{ThumbnailStatus: "skipped"}); err != nil {
|
||||
t.Fatalf("mark duration-only skipped: %v", err)
|
||||
}
|
||||
count, err = cat.CountVideosNeedingThumbnail(ctx, "drive")
|
||||
if err != nil {
|
||||
t.Fatalf("count videos needing thumbnail after skip: %v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("count after skip = %d, want 1", count)
|
||||
}
|
||||
counts, err = cat.CountThumbnailsByDrive(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("count thumbnails by drive after skip: %v", err)
|
||||
}
|
||||
if got := counts["drive"]; got.Ready != 2 || got.Pending != 1 || got.Failed != 1 || got.DurationPending != 0 {
|
||||
t.Fatalf("thumbnail counts after skip = %#v, want ready=2 pending=1 failed=1 durationPending=0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateTagAndClassifyAddsTagToMatchingExistingVideos(t *testing.T) {
|
||||
@@ -154,6 +181,242 @@ func TestCreateTagAndClassifyAddsTagToMatchingExistingVideos(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteTagRemovesTagFromVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: "video-1",
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
Title: "清纯短发",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
if _, err := cat.CreateTagAndClassify(ctx, "清纯", nil, "user"); err != nil {
|
||||
t.Fatalf("create tag: %v", err)
|
||||
}
|
||||
|
||||
tag := mustTagByLabel(t, ctx, cat, "清纯")
|
||||
removed, err := cat.DeleteTag(ctx, tag.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("delete tag: %v", err)
|
||||
}
|
||||
if removed != 1 {
|
||||
t.Fatalf("removed = %d, want 1", removed)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(ctx, "video-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if len(got.Tags) != 0 {
|
||||
t.Fatalf("video tags = %#v, want none", got.Tags)
|
||||
}
|
||||
for _, tag := range mustListTags(t, ctx, cat) {
|
||||
if tag.Label == "清纯" {
|
||||
t.Fatal("deleted tag still appears in ListTags")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteTagSuppressesAutomaticCollectionRecreation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, id := range []string{"video-1", "video-2"} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: "合集视频",
|
||||
Category: "sunny",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
if label, ok, err := cat.EnsureCollectionTag(ctx, "sunny"); err != nil || !ok || label != "sunny" {
|
||||
t.Fatalf("ensure collection = %q, %v, %v; want sunny true nil", label, ok, err)
|
||||
}
|
||||
tag := mustTagByLabel(t, ctx, cat, "sunny")
|
||||
if _, err := cat.DeleteTag(ctx, tag.ID); err != nil {
|
||||
t.Fatalf("delete tag: %v", err)
|
||||
}
|
||||
|
||||
if label, ok, err := cat.EnsureCollectionTag(ctx, "sunny"); err != nil || ok || label != "" {
|
||||
t.Fatalf("ensure deleted collection = %q, %v, %v; want empty false nil", label, ok, err)
|
||||
}
|
||||
for _, tag := range mustListTags(t, ctx, cat) {
|
||||
if tag.Label == "sunny" {
|
||||
t.Fatal("deleted collection tag was recreated automatically")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateTagAndClassifyRestoresDeletedTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: "video-1",
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
Title: "清纯短发",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
if _, err := cat.CreateTagAndClassify(ctx, "清纯", nil, "user"); err != nil {
|
||||
t.Fatalf("create tag: %v", err)
|
||||
}
|
||||
tag := mustTagByLabel(t, ctx, cat, "清纯")
|
||||
if _, err := cat.DeleteTag(ctx, tag.ID); err != nil {
|
||||
t.Fatalf("delete tag: %v", err)
|
||||
}
|
||||
|
||||
classified, err := cat.CreateTagAndClassify(ctx, "清纯", nil, "user")
|
||||
if err != nil {
|
||||
t.Fatalf("recreate tag: %v", err)
|
||||
}
|
||||
if classified != 1 {
|
||||
t.Fatalf("classified = %d, want 1", classified)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, "video-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if !sameStrings(got.Tags, []string{"清纯"}) {
|
||||
t.Fatalf("video tags = %#v, want 清纯", got.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureTagForVideoIDPrefixBackfillsSourceTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, seed := range []struct {
|
||||
id string
|
||||
manual bool
|
||||
}{
|
||||
{id: "spider91-91-spider-1200001"},
|
||||
{id: "spider91-91-spider-1200002", manual: true},
|
||||
{id: "spider91-other-1200003"},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: seed.id,
|
||||
DriveID: "91-spider",
|
||||
FileID: seed.id + ".mp4",
|
||||
Title: "legacy title without source text",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", seed.id, err)
|
||||
}
|
||||
if seed.manual {
|
||||
if err := cat.SetManualVideoTags(ctx, seed.id, nil); err != nil {
|
||||
t.Fatalf("mark %s manual: %v", seed.id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
added, err := cat.EnsureTagForVideoIDPrefix(ctx, "spider91-91-spider-", "91porn", nil, "system")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure prefix tag: %v", err)
|
||||
}
|
||||
if added != 1 {
|
||||
t.Fatalf("added = %d, want 1", added)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, "spider91-91-spider-1200001")
|
||||
if err != nil {
|
||||
t.Fatalf("get tagged video: %v", err)
|
||||
}
|
||||
if !sameStrings(got.Tags, []string{"91porn"}) {
|
||||
t.Fatalf("tagged video tags = %#v, want 91porn", got.Tags)
|
||||
}
|
||||
manual, err := cat.GetVideo(ctx, "spider91-91-spider-1200002")
|
||||
if err != nil {
|
||||
t.Fatalf("get manual video: %v", err)
|
||||
}
|
||||
if len(manual.Tags) != 0 {
|
||||
t.Fatalf("manual video tags = %#v, want unchanged", manual.Tags)
|
||||
}
|
||||
other, err := cat.GetVideo(ctx, "spider91-other-1200003")
|
||||
if err != nil {
|
||||
t.Fatalf("get other prefix video: %v", err)
|
||||
}
|
||||
if len(other.Tags) != 0 {
|
||||
t.Fatalf("other prefix video tags = %#v, want unchanged", other.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteTagRejectsSystemTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
tag := mustTagByLabel(t, ctx, cat, "AV")
|
||||
if _, err := cat.DeleteTag(ctx, tag.ID); !errors.Is(err, ErrSystemTag) {
|
||||
t.Fatalf("delete system tag err = %v, want ErrSystemTag", err)
|
||||
}
|
||||
|
||||
if tag := mustTagByLabel(t, ctx, cat, "AV"); tag.Source != "system" {
|
||||
t.Fatalf("AV source = %q, want system", tag.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenClassifiesSystemTagsForExistingVideos(t *testing.T) {
|
||||
path := t.TempDir() + "/catalog.db"
|
||||
db, err := sql.Open("sqlite", path)
|
||||
@@ -204,6 +467,84 @@ VALUES
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateDoesNotRewriteAlreadySyncedVideoTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, id := range []string{"video-1", "video-2", "video-3"} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: "巨乳后入合集",
|
||||
Category: "Better Call Saul S03",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
if err := cat.migrate(ctx); err != nil {
|
||||
t.Fatalf("first migrate: %v", err)
|
||||
}
|
||||
|
||||
before := videoUpdatedAtByID(t, ctx, cat, "video-1", "video-2", "video-3")
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
if err := cat.migrate(ctx); err != nil {
|
||||
t.Fatalf("second migrate: %v", err)
|
||||
}
|
||||
after := videoUpdatedAtByID(t, ctx, cat, "video-1", "video-2", "video-3")
|
||||
for id, want := range before {
|
||||
if got := after[id]; got != want {
|
||||
t.Fatalf("%s updated_at changed on no-op migrate: got %d, want %d", id, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateBackfillsLegacyTagsWithoutRelations(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
if _, err := cat.db.ExecContext(ctx, `
|
||||
INSERT INTO videos (id, drive_id, file_id, title, tags, tags_manual, published_at, created_at, updated_at)
|
||||
VALUES ('legacy-video', 'drive', 'file-legacy', 'legacy title', '["legacy-tag"]', 0, ?, ?, ?)`,
|
||||
now, now, now); err != nil {
|
||||
t.Fatalf("seed legacy video: %v", err)
|
||||
}
|
||||
if err := cat.migrate(ctx); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
|
||||
tag := mustTagByLabel(t, ctx, cat, "legacy-tag")
|
||||
var count int
|
||||
if err := cat.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM video_tags WHERE video_id = 'legacy-video' AND tag_id = ?`, tag.ID).Scan(&count); err != nil {
|
||||
t.Fatalf("count video tag: %v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("legacy video tag relation count = %d, want 1", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenMigratesLegacyVideosWithoutFileName(t *testing.T) {
|
||||
path := t.TempDir() + "/catalog.db"
|
||||
db, err := sql.Open("sqlite", path)
|
||||
@@ -665,6 +1006,98 @@ func TestListVideosHidesDuplicateContentHashes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTagFilterMatchesCanonicalDuplicateVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{
|
||||
ID: "pikpak-canonical",
|
||||
DriveID: "pikpak",
|
||||
FileID: "canonical.mp4",
|
||||
Title: "Canonical",
|
||||
Size: 1024,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "spider91-dup-1",
|
||||
DriveID: "91-spider",
|
||||
FileID: "dup-1.mp4",
|
||||
Title: "Spider duplicate 1",
|
||||
Tags: []string{"91porn"},
|
||||
Size: 1024,
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
{
|
||||
ID: "spider91-dup-2",
|
||||
DriveID: "91-spider",
|
||||
FileID: "dup-2.mp4",
|
||||
Title: "Spider duplicate 2",
|
||||
Tags: []string{"91porn"},
|
||||
Size: 1024,
|
||||
PublishedAt: now.Add(2 * time.Second),
|
||||
CreatedAt: now.Add(2 * time.Second),
|
||||
UpdatedAt: now.Add(2 * time.Second),
|
||||
},
|
||||
{
|
||||
ID: "spider91-visible",
|
||||
DriveID: "91-spider",
|
||||
FileID: "visible.mp4",
|
||||
Title: "Spider visible",
|
||||
Tags: []string{"91porn"},
|
||||
Size: 2048,
|
||||
PublishedAt: now.Add(3 * time.Second),
|
||||
CreatedAt: now.Add(3 * time.Second),
|
||||
UpdatedAt: now.Add(3 * time.Second),
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"pikpak-canonical", "spider91-dup-1", "spider91-dup-2"} {
|
||||
if err := cat.UpdateVideoFingerprint(ctx, id, "same-sampled-sha256", "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "spider91-visible", "unique-sampled-sha256", "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint visible: %v", err)
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{Tag: "91porn", Page: 1, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list videos by tag: %v", err)
|
||||
}
|
||||
if total != 2 || len(items) != 2 {
|
||||
t.Fatalf("tagged videos total=%d len=%d, want 2", total, len(items))
|
||||
}
|
||||
gotIDs := map[string]bool{}
|
||||
for _, item := range items {
|
||||
gotIDs[item.ID] = true
|
||||
}
|
||||
for _, want := range []string{"pikpak-canonical", "spider91-visible"} {
|
||||
if !gotIDs[want] {
|
||||
t.Fatalf("tagged video ids = %#v, want %s", gotIDs, want)
|
||||
}
|
||||
}
|
||||
if got := mustTagByLabel(t, ctx, cat, "91porn").Count; got != 2 {
|
||||
t.Fatalf("91porn count = %d, want 2 visible canonical videos", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListVideosCanFilterReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
@@ -730,6 +1163,39 @@ func sameStrings(a, b []string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func mustListTags(t *testing.T, ctx context.Context, cat *Catalog) []Tag {
|
||||
t.Helper()
|
||||
tags, err := cat.ListTags(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list tags: %v", err)
|
||||
}
|
||||
return tags
|
||||
}
|
||||
|
||||
func mustTagByLabel(t *testing.T, ctx context.Context, cat *Catalog, label string) Tag {
|
||||
t.Helper()
|
||||
for _, tag := range mustListTags(t, ctx, cat) {
|
||||
if tag.Label == label {
|
||||
return tag
|
||||
}
|
||||
}
|
||||
t.Fatalf("tag %q not found", label)
|
||||
return Tag{}
|
||||
}
|
||||
|
||||
func videoUpdatedAtByID(t *testing.T, ctx context.Context, cat *Catalog, ids ...string) map[string]int64 {
|
||||
t.Helper()
|
||||
out := make(map[string]int64, len(ids))
|
||||
for _, id := range ids {
|
||||
var updatedAt int64
|
||||
if err := cat.db.QueryRowContext(ctx, `SELECT updated_at FROM videos WHERE id = ?`, id).Scan(&updatedAt); err != nil {
|
||||
t.Fatalf("read updated_at for %s: %v", id, err)
|
||||
}
|
||||
out[id] = updatedAt
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// 删除 collection 标签的最后一个引用视频后,标签应当自动从 tags 表里消失。
|
||||
// user/system 标签不受影响:用户/系统标签的语义由人维护,孤儿状态保留。
|
||||
func TestDeleteVideoPrunesOrphanCollectionTag(t *testing.T) {
|
||||
@@ -923,11 +1389,12 @@ func TestReconcileThumbnailStatusOnce(t *testing.T) {
|
||||
id, url, status string
|
||||
wantStatus string
|
||||
}{
|
||||
{"v-pending-url", "/p/thumb/v-pending-url", "pending", "ready"}, // 主要修复目标
|
||||
{"v-empty-url-pending", "", "pending", "pending"}, // 没 url 不动
|
||||
{"v-failed-with-url", "/p/thumb/v-failed-with-url", "failed", "failed"}, // 显式失败保留
|
||||
{"v-empty-url-failed", "", "failed", "failed"}, // 失败 + 没 url 也保留
|
||||
{"v-already-ready", "/p/thumb/v-already-ready", "ready", "ready"}, // 幂等
|
||||
{"v-pending-url", "/p/thumb/v-pending-url", "pending", "ready"}, // 主要修复目标
|
||||
{"v-empty-url-pending", "", "pending", "pending"}, // 没 url 不动
|
||||
{"v-failed-with-url", "/p/thumb/v-failed-with-url", "failed", "failed"}, // 显式失败保留
|
||||
{"v-empty-url-failed", "", "failed", "failed"}, // 失败 + 没 url 也保留
|
||||
{"v-skipped-with-url", "/p/thumb/v-skipped-with-url", "skipped", "skipped"}, // 已跳过的时长补全保留
|
||||
{"v-already-ready", "/p/thumb/v-already-ready", "ready", "ready"}, // 幂等
|
||||
}
|
||||
for _, c := range cases {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
|
||||
@@ -202,7 +202,7 @@ type Nightly struct {
|
||||
// 这里保留 yaml 中的静态定义,用于启动时预置盘。生产建议只在 DB 里维护。
|
||||
type Drive struct {
|
||||
ID string `yaml:"id"`
|
||||
Kind string `yaml:"kind"` // quark / p115 / pikpak / wopan / onedrive / localstorage
|
||||
Kind string `yaml:"kind"` // quark / p115 / pikpak / wopan / onedrive / googledrive / localstorage
|
||||
Name string `yaml:"name"`
|
||||
RootID string `yaml:"root_id"`
|
||||
Params map[string]string `yaml:"params,omitempty"`
|
||||
|
||||
@@ -0,0 +1,505 @@
|
||||
package googledrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-resty/resty/v2"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
const (
|
||||
Kind = "googledrive"
|
||||
defaultAPIBaseURL = "https://www.googleapis.com/drive/v3"
|
||||
defaultOAuthURL = "https://www.googleapis.com/oauth2/v4/token"
|
||||
defaultRenewAPIURL = "https://api.oplist.org/googleui/renewapi"
|
||||
defaultListInterval = 1 * time.Second
|
||||
defaultListCooldown = 5 * time.Minute
|
||||
|
||||
filesListFields = "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken"
|
||||
fileInfoFields = "id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum"
|
||||
)
|
||||
|
||||
type Driver struct {
|
||||
id string
|
||||
rootID string
|
||||
refreshToken string
|
||||
accessToken string
|
||||
clientID string
|
||||
clientSecret string
|
||||
useOnlineAPI bool
|
||||
renewAPIURL string
|
||||
oauthURL string
|
||||
apiBaseURL string
|
||||
client *resty.Client
|
||||
onTokenUpdate func(access, refresh string)
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
listInterval time.Duration
|
||||
listCooldown time.Duration
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ID string
|
||||
RootID string
|
||||
RefreshToken string
|
||||
AccessToken string
|
||||
ClientID string
|
||||
ClientSecret string
|
||||
UseOnlineAPI bool
|
||||
RenewAPIURL string
|
||||
OAuthURL string
|
||||
APIBaseURL string
|
||||
|
||||
OnTokenUpdate func(access, refresh string)
|
||||
}
|
||||
|
||||
func New(c Config) *Driver {
|
||||
rootID := strings.TrimSpace(c.RootID)
|
||||
if rootID == "" {
|
||||
rootID = "root"
|
||||
}
|
||||
renewAPIURL := strings.TrimSpace(c.RenewAPIURL)
|
||||
if renewAPIURL == "" {
|
||||
renewAPIURL = defaultRenewAPIURL
|
||||
}
|
||||
oauthURL := strings.TrimSpace(c.OAuthURL)
|
||||
if oauthURL == "" {
|
||||
oauthURL = defaultOAuthURL
|
||||
}
|
||||
apiBaseURL := strings.TrimRight(strings.TrimSpace(c.APIBaseURL), "/")
|
||||
if apiBaseURL == "" {
|
||||
apiBaseURL = defaultAPIBaseURL
|
||||
}
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
rootID: rootID,
|
||||
refreshToken: strings.TrimSpace(c.RefreshToken),
|
||||
accessToken: strings.TrimSpace(c.AccessToken),
|
||||
clientID: strings.TrimSpace(c.ClientID),
|
||||
clientSecret: strings.TrimSpace(c.ClientSecret),
|
||||
useOnlineAPI: c.UseOnlineAPI,
|
||||
renewAPIURL: renewAPIURL,
|
||||
oauthURL: oauthURL,
|
||||
apiBaseURL: apiBaseURL,
|
||||
onTokenUpdate: c.OnTokenUpdate,
|
||||
client: resty.New().
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
listInterval: defaultListInterval,
|
||||
listCooldown: defaultListCooldown,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Kind() string { return Kind }
|
||||
func (d *Driver) ID() string { return d.id }
|
||||
func (d *Driver) RootID() string { return d.rootID }
|
||||
|
||||
func (d *Driver) Init(ctx context.Context) error {
|
||||
if d.refreshToken == "" {
|
||||
return errors.New("googledrive init: refresh_token is required")
|
||||
}
|
||||
return d.refresh(ctx)
|
||||
}
|
||||
|
||||
func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error) {
|
||||
if dirID == "" {
|
||||
dirID = d.rootID
|
||||
}
|
||||
d.listMu.Lock()
|
||||
defer d.listMu.Unlock()
|
||||
|
||||
pageToken := ""
|
||||
out := make([]drives.Entry, 0)
|
||||
for {
|
||||
if err := d.waitForListSlotLocked(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var resp filesResp
|
||||
err := d.request(ctx, d.filesURL(), http.MethodGet, func(req *resty.Request) {
|
||||
params := map[string]string{
|
||||
"fields": filesListFields,
|
||||
"pageSize": "1000",
|
||||
"q": fmt.Sprintf("'%s' in parents and trashed = false", strings.ReplaceAll(dirID, "'", "\\'")),
|
||||
"orderBy": "folder,name,modifiedTime desc",
|
||||
}
|
||||
if pageToken != "" {
|
||||
params["pageToken"] = pageToken
|
||||
}
|
||||
req.SetQueryParams(params)
|
||||
}, &resp)
|
||||
if err != nil {
|
||||
if wait, ok := drives.RateLimitRetryAfter(err); ok {
|
||||
if wait <= 0 {
|
||||
wait = d.listCooldown
|
||||
}
|
||||
if sleepErr := sleepContext(ctx, wait); sleepErr != nil {
|
||||
return nil, sleepErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("googledrive list: %w", err)
|
||||
}
|
||||
if err := d.fillShortcutFileMetadata(ctx, resp.Files); err != nil {
|
||||
return nil, fmt.Errorf("googledrive shortcut metadata: %w", err)
|
||||
}
|
||||
for _, f := range resp.Files {
|
||||
out = append(out, fileToEntry(f, dirID))
|
||||
}
|
||||
pageToken = resp.NextPageToken
|
||||
if pageToken == "" {
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) waitForListSlotLocked(ctx context.Context) error {
|
||||
if d.listInterval <= 0 || d.lastListAt.IsZero() {
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
next := d.lastListAt.Add(d.listInterval)
|
||||
now := time.Now()
|
||||
if now.Before(next) {
|
||||
if err := sleepContext(ctx, next.Sub(now)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, d time.Duration) error {
|
||||
if d <= 0 {
|
||||
return ctx.Err()
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
|
||||
var f driveFile
|
||||
if err := d.request(ctx, d.fileURL(fileID), http.MethodGet, func(req *resty.Request) {
|
||||
req.SetQueryParam("fields", fileInfoFields)
|
||||
}, &f); err != nil {
|
||||
return nil, fmt.Errorf("googledrive stat: %w", err)
|
||||
}
|
||||
e := fileToEntry(f, "")
|
||||
return &e, nil
|
||||
}
|
||||
|
||||
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
if fileID == "" {
|
||||
return nil, errors.New("googledrive stream: empty file id")
|
||||
}
|
||||
if _, err := d.Stat(ctx, fileID); err != nil {
|
||||
return nil, fmt.Errorf("googledrive stream: %w", err)
|
||||
}
|
||||
u := d.fileURL(fileID) + "?alt=media&acknowledgeAbuse=true&supportsAllDrives=true"
|
||||
return &drives.StreamLink{
|
||||
URL: u,
|
||||
Headers: http.Header{
|
||||
"Authorization": []string{"Bearer " + d.accessToken},
|
||||
},
|
||||
Expires: time.Now().Add(30 * time.Minute),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) refresh(ctx context.Context) error {
|
||||
if d.useOnlineAPI && d.renewAPIURL != "" {
|
||||
var out tokenResp
|
||||
res, err := d.client.R().
|
||||
SetContext(ctx).
|
||||
SetQueryParams(map[string]string{
|
||||
"refresh_ui": d.refreshToken,
|
||||
"server_use": "true",
|
||||
"driver_txt": "googleui_go",
|
||||
}).
|
||||
SetResult(&out).
|
||||
SetError(&out).
|
||||
Get(d.renewAPIURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("googledrive refresh token: %w", err)
|
||||
}
|
||||
if err := tokenResponseError("googledrive refresh token", res, out, true); err != nil {
|
||||
return err
|
||||
}
|
||||
d.applyToken(out)
|
||||
return nil
|
||||
}
|
||||
if d.clientID == "" || d.clientSecret == "" {
|
||||
return errors.New("googledrive refresh token: client_id and client_secret are required when online API is disabled")
|
||||
}
|
||||
var out tokenResp
|
||||
res, err := d.client.R().
|
||||
SetContext(ctx).
|
||||
SetFormData(map[string]string{
|
||||
"client_id": d.clientID,
|
||||
"client_secret": d.clientSecret,
|
||||
"refresh_token": d.refreshToken,
|
||||
"grant_type": "refresh_token",
|
||||
}).
|
||||
SetResult(&out).
|
||||
SetError(&out).
|
||||
Post(d.oauthURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("googledrive refresh token: %w", err)
|
||||
}
|
||||
if err := tokenResponseError("googledrive refresh token", res, out, false); err != nil {
|
||||
return err
|
||||
}
|
||||
d.applyToken(out)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) applyToken(out tokenResp) {
|
||||
d.accessToken = out.AccessToken
|
||||
if strings.TrimSpace(out.RefreshToken) != "" {
|
||||
d.refreshToken = out.RefreshToken
|
||||
}
|
||||
if d.onTokenUpdate != nil {
|
||||
d.onTokenUpdate(d.accessToken, d.refreshToken)
|
||||
}
|
||||
}
|
||||
|
||||
func tokenResponseError(prefix string, res *resty.Response, out tokenResp, requireRefresh bool) error {
|
||||
if out.Text != "" {
|
||||
return fmt.Errorf("%s: %s", prefix, out.Text)
|
||||
}
|
||||
if out.Error != "" {
|
||||
if out.ErrorDescription != "" {
|
||||
return fmt.Errorf("%s: %s", prefix, out.ErrorDescription)
|
||||
}
|
||||
return fmt.Errorf("%s: %s", prefix, out.Error)
|
||||
}
|
||||
if res != nil && res.IsError() {
|
||||
return fmt.Errorf("%s: status=%d body=%s", prefix, res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
if out.AccessToken == "" || (requireRefresh && out.RefreshToken == "") {
|
||||
return fmt.Errorf("%s: empty token", prefix)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) request(ctx context.Context, rawURL, method string, configure func(*resty.Request), out any) error {
|
||||
return d.requestOnce(ctx, rawURL, method, configure, out, true)
|
||||
}
|
||||
|
||||
func (d *Driver) requestOnce(ctx context.Context, rawURL, method string, configure func(*resty.Request), out any, retry bool) error {
|
||||
req := d.client.R().
|
||||
SetContext(ctx).
|
||||
SetHeader("Authorization", "Bearer "+d.accessToken).
|
||||
SetQueryParam("includeItemsFromAllDrives", "true").
|
||||
SetQueryParam("supportsAllDrives", "true")
|
||||
if configure != nil {
|
||||
configure(req)
|
||||
}
|
||||
if out != nil {
|
||||
req.SetResult(out)
|
||||
}
|
||||
var apiErr apiErrorResp
|
||||
req.SetError(&apiErr)
|
||||
res, err := req.Execute(method, rawURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isGoogleRateLimit(res, apiErr.Error) {
|
||||
return googleRateLimitError(res, apiErr.Error.Message)
|
||||
}
|
||||
if apiErr.Error.Code != 0 {
|
||||
if apiErr.Error.Code == http.StatusUnauthorized && retry {
|
||||
if err := d.refresh(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return d.requestOnce(ctx, rawURL, method, configure, out, false)
|
||||
}
|
||||
return googleAPIError(apiErr.Error)
|
||||
}
|
||||
if res.IsError() {
|
||||
return fmt.Errorf("google drive api error: status=%d body=%s", res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) fillShortcutFileMetadata(ctx context.Context, files []driveFile) error {
|
||||
for i := range files {
|
||||
f := &files[i]
|
||||
if f.MimeType != "application/vnd.google-apps.shortcut" ||
|
||||
f.Shortcut.TargetID == "" ||
|
||||
f.Shortcut.TargetMimeType == "application/vnd.google-apps.folder" {
|
||||
continue
|
||||
}
|
||||
var target driveFile
|
||||
if err := d.request(ctx, d.fileURL(f.Shortcut.TargetID), http.MethodGet, func(req *resty.Request) {
|
||||
req.SetQueryParam("fields", fileInfoFields)
|
||||
}, &target); err != nil {
|
||||
return err
|
||||
}
|
||||
if target.Size != "" {
|
||||
f.Size = target.Size
|
||||
}
|
||||
if target.MD5Checksum != "" {
|
||||
f.MD5Checksum = target.MD5Checksum
|
||||
}
|
||||
if target.SHA1Checksum != "" {
|
||||
f.SHA1Checksum = target.SHA1Checksum
|
||||
}
|
||||
if target.SHA256Checksum != "" {
|
||||
f.SHA256Checksum = target.SHA256Checksum
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) filesURL() string {
|
||||
return d.apiBaseURL + "/files"
|
||||
}
|
||||
|
||||
func (d *Driver) fileURL(fileID string) string {
|
||||
return d.filesURL() + "/" + url.PathEscape(fileID)
|
||||
}
|
||||
|
||||
func fileToEntry(f driveFile, fallbackParentID string) drives.Entry {
|
||||
id := f.ID
|
||||
isDir := f.MimeType == "application/vnd.google-apps.folder"
|
||||
if f.MimeType == "application/vnd.google-apps.shortcut" && f.Shortcut.TargetID != "" {
|
||||
id = f.Shortcut.TargetID
|
||||
isDir = f.Shortcut.TargetMimeType == "application/vnd.google-apps.folder"
|
||||
}
|
||||
size, _ := strconv.ParseInt(f.Size, 10, 64)
|
||||
hash := f.MD5Checksum
|
||||
if hash == "" {
|
||||
hash = f.SHA1Checksum
|
||||
}
|
||||
if hash == "" {
|
||||
hash = f.SHA256Checksum
|
||||
}
|
||||
return drives.Entry{
|
||||
ID: id,
|
||||
Name: f.Name,
|
||||
Size: size,
|
||||
Hash: hash,
|
||||
IsDir: isDir,
|
||||
ParentID: fallbackParentID,
|
||||
MimeType: mimeType(f),
|
||||
ModTime: f.ModifiedTime,
|
||||
ThumbnailURL: f.ThumbnailLink,
|
||||
}
|
||||
}
|
||||
|
||||
func mimeType(f driveFile) string {
|
||||
if f.MimeType != "" && f.MimeType != "application/vnd.google-apps.shortcut" {
|
||||
return f.MimeType
|
||||
}
|
||||
if f.Shortcut.TargetMimeType != "" {
|
||||
return f.Shortcut.TargetMimeType
|
||||
}
|
||||
ext := strings.ToLower(path.Ext(f.Name))
|
||||
switch ext {
|
||||
case ".mp4":
|
||||
return "video/mp4"
|
||||
case ".mkv":
|
||||
return "video/x-matroska"
|
||||
case ".mov":
|
||||
return "video/quicktime"
|
||||
case ".webm":
|
||||
return "video/webm"
|
||||
case ".avi":
|
||||
return "video/x-msvideo"
|
||||
case ".jpg", ".jpeg":
|
||||
return "image/jpeg"
|
||||
case ".png":
|
||||
return "image/png"
|
||||
default:
|
||||
return "application/octet-stream"
|
||||
}
|
||||
}
|
||||
|
||||
func isGoogleRateLimit(res *resty.Response, body apiErrorBody) bool {
|
||||
if res != nil && res.StatusCode() == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if body.Code == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
for _, e := range body.Errors {
|
||||
reason := strings.ToLower(strings.TrimSpace(e.Reason))
|
||||
switch reason {
|
||||
case "ratelimitexceeded", "userratelimitexceeded", "downloadquotaexceeded", "sharingratelimitexceeded":
|
||||
return true
|
||||
}
|
||||
}
|
||||
msg := strings.ToLower(body.Message)
|
||||
return strings.Contains(msg, "rate limit") || strings.Contains(msg, "too many requests") || strings.Contains(msg, "quota exceeded")
|
||||
}
|
||||
|
||||
func googleRateLimitError(res *resty.Response, message string) error {
|
||||
if strings.TrimSpace(message) == "" {
|
||||
message = "google drive rate limited"
|
||||
}
|
||||
if res != nil && strings.TrimSpace(res.String()) != "" {
|
||||
message = fmt.Sprintf("%s: status=%d body=%s", message, res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: parseRetryAfter(res),
|
||||
Err: errors.New(message),
|
||||
}
|
||||
}
|
||||
|
||||
func googleAPIError(body apiErrorBody) error {
|
||||
if body.Message != "" {
|
||||
return errors.New(body.Message)
|
||||
}
|
||||
if body.Code != 0 {
|
||||
return fmt.Errorf("google drive api error: code=%d", body.Code)
|
||||
}
|
||||
return errors.New("google drive api error")
|
||||
}
|
||||
|
||||
func parseRetryAfter(res *resty.Response) time.Duration {
|
||||
if res == nil {
|
||||
return 0
|
||||
}
|
||||
raw := strings.TrimSpace(res.Header().Get("Retry-After"))
|
||||
if raw == "" {
|
||||
return 0
|
||||
}
|
||||
if seconds, err := strconv.Atoi(raw); err == nil && seconds > 0 {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
if when, err := http.ParseTime(raw); err == nil {
|
||||
d := time.Until(when)
|
||||
if d > 0 {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
@@ -0,0 +1,190 @@
|
||||
package googledrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestInitUsesOnlineRenewAPI(t *testing.T) {
|
||||
var savedAccess, savedRefresh string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/renew" {
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
if got := r.URL.Query().Get("refresh_ui"); got != "old-refresh" {
|
||||
t.Fatalf("refresh_ui = %q", got)
|
||||
}
|
||||
if got := r.URL.Query().Get("server_use"); got != "true" {
|
||||
t.Fatalf("server_use = %q", got)
|
||||
}
|
||||
if got := r.URL.Query().Get("driver_txt"); got != "googleui_go" {
|
||||
t.Fatalf("driver_txt = %q", got)
|
||||
}
|
||||
writeTestJSON(w, tokenResp{
|
||||
AccessToken: "new-access",
|
||||
RefreshToken: "new-refresh",
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "g",
|
||||
RefreshToken: "old-refresh",
|
||||
UseOnlineAPI: true,
|
||||
RenewAPIURL: srv.URL + "/renew",
|
||||
OnTokenUpdate: func(access, refresh string) {
|
||||
savedAccess = access
|
||||
savedRefresh = refresh
|
||||
},
|
||||
})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("Init() error = %v", err)
|
||||
}
|
||||
if d.accessToken != "new-access" || d.refreshToken != "new-refresh" {
|
||||
t.Fatalf("tokens not applied: access=%q refresh=%q", d.accessToken, d.refreshToken)
|
||||
}
|
||||
if savedAccess != "new-access" || savedRefresh != "new-refresh" {
|
||||
t.Fatalf("tokens not persisted: access=%q refresh=%q", savedAccess, savedRefresh)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListMapsGoogleDriveFiles(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("Authorization = %q", got)
|
||||
}
|
||||
if r.URL.Path != "/drive/v3/files" {
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
if !strings.Contains(r.URL.Query().Get("q"), "'root' in parents") {
|
||||
t.Fatalf("unexpected q = %q", r.URL.Query().Get("q"))
|
||||
}
|
||||
writeTestJSON(w, filesResp{Files: []driveFile{
|
||||
{ID: "folder-1", Name: "Movies", MimeType: "application/vnd.google-apps.folder"},
|
||||
{
|
||||
ID: "file-1",
|
||||
Name: "clip.mp4",
|
||||
MimeType: "video/mp4",
|
||||
Size: "1234",
|
||||
MD5Checksum: "abc",
|
||||
ThumbnailLink: "https://thumb.example/1",
|
||||
},
|
||||
}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", RootID: "root", APIBaseURL: srv.URL + "/drive/v3"})
|
||||
d.accessToken = "access"
|
||||
d.listInterval = -1
|
||||
|
||||
entries, err := d.List(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("List() error = %v", err)
|
||||
}
|
||||
if len(entries) != 2 {
|
||||
t.Fatalf("len(entries) = %d", len(entries))
|
||||
}
|
||||
if !entries[0].IsDir || entries[0].ID != "folder-1" {
|
||||
t.Fatalf("folder entry = %+v", entries[0])
|
||||
}
|
||||
if entries[1].ID != "file-1" || entries[1].Size != 1234 || entries[1].Hash != "abc" || entries[1].ThumbnailURL == "" {
|
||||
t.Fatalf("file entry = %+v", entries[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLReturnsAuthenticatedMediaLinkWithoutRedirectRequirement(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("Authorization = %q", got)
|
||||
}
|
||||
if r.URL.Path != "/drive/v3/files/file-1" {
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
writeTestJSON(w, driveFile{
|
||||
ID: "file-1",
|
||||
Name: "clip.mp4",
|
||||
MimeType: "video/mp4",
|
||||
Size: "1234",
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", APIBaseURL: srv.URL + "/drive/v3"})
|
||||
d.accessToken = "access"
|
||||
|
||||
link, err := d.StreamURL(context.Background(), "file-1")
|
||||
if err != nil {
|
||||
t.Fatalf("StreamURL() error = %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(link.URL, srv.URL+"/drive/v3/files/file-1?") {
|
||||
t.Fatalf("link URL = %q", link.URL)
|
||||
}
|
||||
if !strings.Contains(link.URL, "alt=media") {
|
||||
t.Fatalf("link URL missing alt=media: %q", link.URL)
|
||||
}
|
||||
if got := link.Headers.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("link Authorization = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestRefreshesOnUnauthorized(t *testing.T) {
|
||||
var fileCalls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/renew":
|
||||
writeTestJSON(w, tokenResp{
|
||||
AccessToken: "new-access",
|
||||
RefreshToken: "new-refresh",
|
||||
})
|
||||
case "/drive/v3/files/file-1":
|
||||
fileCalls++
|
||||
if fileCalls == 1 {
|
||||
writeTestJSONStatus(w, http.StatusUnauthorized, apiErrorResp{Error: apiErrorBody{
|
||||
Code: http.StatusUnauthorized,
|
||||
Message: "Invalid Credentials",
|
||||
}})
|
||||
return
|
||||
}
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer new-access" {
|
||||
t.Fatalf("Authorization after refresh = %q", got)
|
||||
}
|
||||
writeTestJSON(w, driveFile{ID: "file-1", Name: "clip.mp4", Size: "1"})
|
||||
default:
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "g",
|
||||
RefreshToken: "old-refresh",
|
||||
UseOnlineAPI: true,
|
||||
RenewAPIURL: srv.URL + "/renew",
|
||||
APIBaseURL: srv.URL + "/drive/v3",
|
||||
})
|
||||
d.accessToken = "old-access"
|
||||
|
||||
if _, err := d.Stat(context.Background(), "file-1"); err != nil {
|
||||
t.Fatalf("Stat() error = %v", err)
|
||||
}
|
||||
if fileCalls != 2 {
|
||||
t.Fatalf("fileCalls = %d", fileCalls)
|
||||
}
|
||||
if d.accessToken != "new-access" || d.refreshToken != "new-refresh" {
|
||||
t.Fatalf("tokens not refreshed: access=%q refresh=%q", d.accessToken, d.refreshToken)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestJSON(w http.ResponseWriter, v any) {
|
||||
writeTestJSONStatus(w, http.StatusOK, v)
|
||||
}
|
||||
|
||||
func writeTestJSONStatus(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package googledrive
|
||||
|
||||
import "time"
|
||||
|
||||
type tokenResp struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
RefreshToken string `json:"refresh_token"`
|
||||
ExpiresIn int64 `json:"expires_in"`
|
||||
Error string `json:"error"`
|
||||
ErrorDescription string `json:"error_description"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
type filesResp struct {
|
||||
NextPageToken string `json:"nextPageToken"`
|
||||
Files []driveFile `json:"files"`
|
||||
Error apiErrorBody `json:"error"`
|
||||
}
|
||||
|
||||
type driveFile struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
MimeType string `json:"mimeType"`
|
||||
ModifiedTime time.Time `json:"modifiedTime"`
|
||||
CreatedTime time.Time `json:"createdTime"`
|
||||
Size string `json:"size"`
|
||||
ThumbnailLink string `json:"thumbnailLink"`
|
||||
MD5Checksum string `json:"md5Checksum"`
|
||||
SHA1Checksum string `json:"sha1Checksum"`
|
||||
SHA256Checksum string `json:"sha256Checksum"`
|
||||
Shortcut struct {
|
||||
TargetID string `json:"targetId"`
|
||||
TargetMimeType string `json:"targetMimeType"`
|
||||
} `json:"shortcutDetails"`
|
||||
}
|
||||
|
||||
type apiErrorResp struct {
|
||||
Error apiErrorBody `json:"error"`
|
||||
}
|
||||
|
||||
type apiErrorBody struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Errors []struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
} `json:"errors"`
|
||||
}
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
// Drive 是多家网盘统一抽象。上层不区分盘,只区分 Kind。
|
||||
type Drive interface {
|
||||
// Kind 返回驱动代号:"quark" / "p115" / "pikpak" / "wopan" / "onedrive" / "localstorage"
|
||||
// Kind 返回驱动代号:"quark" / "p115" / "pikpak" / "wopan" / "onedrive" / "googledrive" / "localstorage"
|
||||
Kind() string
|
||||
|
||||
// ID 返回该盘在 catalog 中的唯一标识
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -20,6 +21,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"golang.org/x/net/proxy"
|
||||
)
|
||||
|
||||
// 默认 author/tag 标签,便于在前端筛选 spider91 来源的视频。
|
||||
@@ -79,29 +81,123 @@ func NewCrawler(cfg CrawlerConfig) *Crawler {
|
||||
cfg.DownloadTimeout = 30 * time.Minute
|
||||
}
|
||||
if cfg.HTTPClient == nil {
|
||||
// 选 proxy 函数:显式 ProxyURL > 环境变量 > 直连
|
||||
proxyFn := http.ProxyFromEnvironment
|
||||
if strings.TrimSpace(cfg.ProxyURL) != "" {
|
||||
if u, err := url.Parse(cfg.ProxyURL); err == nil {
|
||||
proxyFn = http.ProxyURL(u)
|
||||
} else {
|
||||
log.Printf("[spider91] invalid proxy URL %q, falling back to env: %v", cfg.ProxyURL, err)
|
||||
}
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
ResponseHeaderTimeout: 60 * time.Second,
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
}
|
||||
if err := configureExplicitProxy(transport, cfg.ProxyURL); err != nil {
|
||||
log.Printf("[spider91] invalid configured proxy URL, falling back to env: %v", err)
|
||||
}
|
||||
cfg.HTTPClient = &http.Client{
|
||||
// 不限制总下载时长,靠 ctx 控制;只挡 dial / handshake / header
|
||||
Timeout: 0,
|
||||
Transport: &http.Transport{
|
||||
Proxy: proxyFn,
|
||||
ResponseHeaderTimeout: 60 * time.Second,
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
},
|
||||
Timeout: 0,
|
||||
Transport: transport,
|
||||
}
|
||||
}
|
||||
return &Crawler{cfg: cfg}
|
||||
}
|
||||
|
||||
func configureExplicitProxy(transport *http.Transport, raw string) error {
|
||||
proxyURL := strings.TrimSpace(raw)
|
||||
if proxyURL == "" {
|
||||
return nil
|
||||
}
|
||||
u, err := url.Parse(proxyURL)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return fmt.Errorf("invalid proxy URL")
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https":
|
||||
transport.Proxy = http.ProxyURL(u)
|
||||
transport.DialContext = nil
|
||||
return nil
|
||||
case "socks5", "socks5h":
|
||||
dialContext, err := socksProxyDialContext(u)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
transport.Proxy = nil
|
||||
transport.DialContext = dialContext
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("unsupported proxy scheme %q", u.Scheme)
|
||||
}
|
||||
}
|
||||
|
||||
func socksProxyDialContext(proxyURL *url.URL) (func(context.Context, string, string) (net.Conn, error), error) {
|
||||
var auth *proxy.Auth
|
||||
if proxyURL.User != nil {
|
||||
username := proxyURL.User.Username()
|
||||
password, _ := proxyURL.User.Password()
|
||||
auth = &proxy.Auth{User: username, Password: password}
|
||||
}
|
||||
dialer, err := proxy.SOCKS5("tcp", proxyURL.Host, auth, &net.Dialer{Timeout: 60 * time.Second})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remoteDNS := strings.EqualFold(proxyURL.Scheme, "socks5h")
|
||||
return func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
target := addr
|
||||
if !remoteDNS {
|
||||
resolved, err := resolveSocksTarget(ctx, addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target = resolved
|
||||
}
|
||||
if ctxDialer, ok := dialer.(proxy.ContextDialer); ok {
|
||||
return ctxDialer.DialContext(ctx, network, target)
|
||||
}
|
||||
type result struct {
|
||||
conn net.Conn
|
||||
err error
|
||||
}
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
conn, err := dialer.Dial(network, target)
|
||||
ch <- result{conn: conn, err: err}
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case res := <-ch:
|
||||
return res.conn, res.err
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveSocksTarget(ctx context.Context, addr string) (string, error) {
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil || net.ParseIP(host) != nil {
|
||||
return addr, nil
|
||||
}
|
||||
ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
ip := selectSocksTargetIP(ips)
|
||||
if ip == nil {
|
||||
return "", fmt.Errorf("resolve %s: no address", host)
|
||||
}
|
||||
return net.JoinHostPort(ip.String(), port), nil
|
||||
}
|
||||
|
||||
func selectSocksTargetIP(ips []net.IPAddr) net.IP {
|
||||
for _, addr := range ips {
|
||||
if ip4 := addr.IP.To4(); ip4 != nil {
|
||||
return ip4
|
||||
}
|
||||
}
|
||||
for _, addr := range ips {
|
||||
if addr.IP != nil {
|
||||
return addr.IP
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CrawlResult 汇总一次 RunOnce 的结果。
|
||||
type CrawlResult struct {
|
||||
// TargetNew 是本次 RunOnce 的目标新增数(来自 drive.Credentials.target_new)。
|
||||
@@ -324,6 +420,16 @@ func (c *Crawler) startSpiderTargetNew(ctx context.Context, targetNew int, seenP
|
||||
if c.cfg.WorkDir != "" {
|
||||
cmd.Dir = c.cfg.WorkDir
|
||||
}
|
||||
if proxyURL := strings.TrimSpace(c.cfg.ProxyURL); proxyURL != "" {
|
||||
cmd.Env = append(os.Environ(),
|
||||
"HTTP_PROXY="+proxyURL,
|
||||
"HTTPS_PROXY="+proxyURL,
|
||||
"http_proxy="+proxyURL,
|
||||
"https_proxy="+proxyURL,
|
||||
"NO_PROXY=",
|
||||
"no_proxy=",
|
||||
)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("stdout pipe: %w", err)
|
||||
@@ -427,6 +533,17 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
}
|
||||
}
|
||||
|
||||
title := strings.TrimSpace(item.Title)
|
||||
if title == "" {
|
||||
title = sourceID
|
||||
}
|
||||
tags := []string{DefaultTag}
|
||||
if matched, err := c.cfg.Catalog.MatchTags(ctx, title+" "+DefaultAuthor); err == nil {
|
||||
tags = mergeCatalogTags(tags, matched)
|
||||
} else {
|
||||
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s match tags: %v", c.cfg.Driver.ID(), viewkey, sourceID, err)
|
||||
}
|
||||
|
||||
// 入库
|
||||
now := time.Now()
|
||||
v := &catalog.Video{
|
||||
@@ -434,9 +551,9 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
DriveID: c.cfg.Driver.ID(),
|
||||
FileID: videoFile,
|
||||
FileName: videoFile,
|
||||
Title: strings.TrimSpace(item.Title),
|
||||
Title: title,
|
||||
Author: DefaultAuthor,
|
||||
Tags: []string{DefaultTag},
|
||||
Tags: tags,
|
||||
Ext: strings.TrimPrefix(videoExt, "."),
|
||||
Quality: "HD",
|
||||
Size: videoSize,
|
||||
@@ -445,9 +562,6 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if v.Title == "" {
|
||||
v.Title = sourceID
|
||||
}
|
||||
if thumbReady {
|
||||
// 设了 ThumbnailURL 后 thumb worker 会跳过这条视频,
|
||||
// 不再尝试用 ffmpeg 抽帧(封面已经是网站原图)。
|
||||
@@ -888,6 +1002,26 @@ func copyFileAtomic(src, dst string) error {
|
||||
return os.Rename(tmp, dst)
|
||||
}
|
||||
|
||||
func mergeCatalogTags(lists ...[]string) []string {
|
||||
out := []string{}
|
||||
seen := map[string]bool{}
|
||||
for _, list := range lists {
|
||||
for _, tag := range list {
|
||||
tag = strings.TrimSpace(tag)
|
||||
if tag == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(tag)
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, tag)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BuildVideoID 给定 driveID + 91 源视频 ID,按统一规则生成 catalog 中 videos.id。
|
||||
// 与 scanner 用法一致:<kind>-<driveID>-<fileID>。
|
||||
func BuildVideoID(driveID, sourceID string) string {
|
||||
|
||||
@@ -3,6 +3,8 @@ package spider91
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
@@ -53,7 +55,7 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
|
||||
// 同时仍写 --output 文件作归档。
|
||||
videoEntries := []map[string]string{
|
||||
{
|
||||
"title": "Video One",
|
||||
"title": "Video One 口交",
|
||||
"thumb_url": srv.URL + "/thumb/not-120001.jpg",
|
||||
"video_url": srv.URL + "/videos/120001.mp4",
|
||||
"viewkey": "vk-001",
|
||||
@@ -94,6 +96,9 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert drive: %v", err)
|
||||
}
|
||||
if _, err := cat.CreateTagAndClassify(context.Background(), "Video One", nil, "user"); err != nil {
|
||||
t.Fatalf("create user tag: %v", err)
|
||||
}
|
||||
|
||||
var newVideos []*catalog.Video
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
@@ -188,6 +193,17 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
|
||||
if !hasDefaultTag {
|
||||
t.Fatalf("video %s tags = %v, want contain %q", videoID, v.Tags, DefaultTag)
|
||||
}
|
||||
if sourceID == "120001" {
|
||||
if !containsString(v.Tags, "口交") {
|
||||
t.Fatalf("video %s tags = %v, want contain built-in tag 口交", videoID, v.Tags)
|
||||
}
|
||||
if !containsString(v.Tags, "Video One") {
|
||||
t.Fatalf("video %s tags = %v, want contain user tag Video One", videoID, v.Tags)
|
||||
}
|
||||
}
|
||||
if sourceID == "120002" && (containsString(v.Tags, "口交") || containsString(v.Tags, "Video One")) {
|
||||
t.Fatalf("video %s tags = %v, should not inherit tags from other spider91 videos", videoID, v.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
// 7. 第二次 RunOnce:源视频 ID 已存在 → 全部 skipped,无新文件下载
|
||||
@@ -233,6 +249,108 @@ func TestCrawlerRunOnceMissingScript(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerPassesProxyToSpiderProcess(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell-based fake script only on unix")
|
||||
}
|
||||
|
||||
tmp := t.TempDir()
|
||||
scriptPath := filepath.Join(tmp, "print_proxy_env.sh")
|
||||
script := `#!/bin/sh
|
||||
printf 'HTTP_PROXY=%s\n' "$HTTP_PROXY"
|
||||
printf 'HTTPS_PROXY=%s\n' "$HTTPS_PROXY"
|
||||
printf 'http_proxy=%s\n' "$http_proxy"
|
||||
printf 'https_proxy=%s\n' "$https_proxy"
|
||||
printf 'NO_PROXY=%s\n' "$NO_PROXY"
|
||||
printf 'no_proxy=%s\n' "$no_proxy"
|
||||
`
|
||||
if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
|
||||
proxyURL := "socks5h://proxy.local:1080"
|
||||
drv := New(Config{ID: "proxy-drive", RootDir: filepath.Join(tmp, "proxy-drive")})
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
PythonPath: "sh",
|
||||
ScriptPath: scriptPath,
|
||||
ProxyURL: proxyURL,
|
||||
})
|
||||
cmd, stdout, err := c.startSpiderTargetNew(
|
||||
context.Background(),
|
||||
1,
|
||||
filepath.Join(tmp, "seen.txt"),
|
||||
filepath.Join(tmp, "out.json"),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("startSpiderTargetNew: %v", err)
|
||||
}
|
||||
raw, err := io.ReadAll(stdout)
|
||||
if err != nil {
|
||||
t.Fatalf("read stdout: %v", err)
|
||||
}
|
||||
if err := cmd.Wait(); err != nil {
|
||||
t.Fatalf("wait: %v", err)
|
||||
}
|
||||
|
||||
want := strings.Join([]string{
|
||||
"HTTP_PROXY=" + proxyURL,
|
||||
"HTTPS_PROXY=" + proxyURL,
|
||||
"http_proxy=" + proxyURL,
|
||||
"https_proxy=" + proxyURL,
|
||||
"NO_PROXY=",
|
||||
"no_proxy=",
|
||||
}, "\n") + "\n"
|
||||
if string(raw) != want {
|
||||
t.Fatalf("proxy env = %q, want %q", string(raw), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigureExplicitProxySupportsSocksSchemes(t *testing.T) {
|
||||
for _, raw := range []string{
|
||||
"socks5://127.0.0.1:1080",
|
||||
"socks5h://proxy-user:proxy-pass@127.0.0.1:1080",
|
||||
} {
|
||||
t.Run(raw, func(t *testing.T) {
|
||||
transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
|
||||
if err := configureExplicitProxy(transport, raw); err != nil {
|
||||
t.Fatalf("configureExplicitProxy: %v", err)
|
||||
}
|
||||
if transport.Proxy != nil {
|
||||
t.Fatalf("Transport.Proxy should be nil for SOCKS proxy")
|
||||
}
|
||||
if transport.DialContext == nil {
|
||||
t.Fatalf("Transport.DialContext should be set for SOCKS proxy")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
|
||||
if err := configureExplicitProxy(transport, "http://127.0.0.1:7890"); err != nil {
|
||||
t.Fatalf("configureExplicitProxy http: %v", err)
|
||||
}
|
||||
if transport.Proxy == nil {
|
||||
t.Fatalf("Transport.Proxy should be set for HTTP proxy")
|
||||
}
|
||||
if transport.DialContext != nil {
|
||||
t.Fatalf("Transport.DialContext should not be set for HTTP proxy")
|
||||
}
|
||||
|
||||
if err := configureExplicitProxy(&http.Transport{}, "ftp://127.0.0.1:21"); err == nil {
|
||||
t.Fatalf("expected unsupported proxy scheme error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSocksTargetIPPrefersIPv4(t *testing.T) {
|
||||
got := selectSocksTargetIP([]net.IPAddr{
|
||||
{IP: net.ParseIP("2606:4700:20::681a:229")},
|
||||
{IP: net.ParseIP("104.26.3.41")},
|
||||
})
|
||||
if got == nil || got.String() != "104.26.3.41" {
|
||||
t.Fatalf("selectSocksTargetIP = %v, want IPv4 104.26.3.41", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCrawlerThumbDownloadFailureMarksStatusFailed 验证:网站封面下载失败时
|
||||
// crawler 把 thumbnail_status 显式标 'failed',避免后续封面补队列一直重复
|
||||
// 捞到这条 spider91 视频。
|
||||
@@ -658,3 +776,12 @@ func buildFakeSpiderScript(entries []map[string]string) string {
|
||||
sb.WriteString("fi\n")
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func containsString(values []string, want string) bool {
|
||||
for _, value := range values {
|
||||
if value == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -96,11 +96,25 @@ type Config struct {
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type Status struct {
|
||||
State string
|
||||
Running bool
|
||||
Queued bool
|
||||
StartedAt time.Time
|
||||
LastFinishedAt time.Time
|
||||
}
|
||||
|
||||
// Runner drives the nightly pipeline.
|
||||
type Runner struct {
|
||||
cfg Config
|
||||
trigger chan struct{} // buffered(1); manual "run now"
|
||||
runMu sync.Mutex // prevents overlapping pipeline runs
|
||||
|
||||
stateMu sync.Mutex
|
||||
running bool
|
||||
queued bool
|
||||
startedAt time.Time
|
||||
lastFinishedAt time.Time
|
||||
}
|
||||
|
||||
// New constructs a Runner. cfg is shallow-copied; defaults are applied.
|
||||
@@ -138,14 +152,53 @@ func (r *Runner) Run(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// TriggerNow asks the running loop to fire a pipeline ASAP. The trigger channel
|
||||
// is buffered(1): if a pipeline is already in progress, one follow-up run may
|
||||
// remain pending and will start after the current run finishes. Additional
|
||||
// clicks while that follow-up is pending are ignored.
|
||||
func (r *Runner) TriggerNow() {
|
||||
// TriggerNow asks the running loop to fire a pipeline ASAP. Only one manual
|
||||
// trigger can be active at a time: if a pipeline is already running or waiting
|
||||
// in the trigger channel, the request is ignored and returns false.
|
||||
func (r *Runner) TriggerNow() bool {
|
||||
r.stateMu.Lock()
|
||||
if r.running || r.queued {
|
||||
r.stateMu.Unlock()
|
||||
return false
|
||||
}
|
||||
r.queued = true
|
||||
r.stateMu.Unlock()
|
||||
|
||||
select {
|
||||
case r.trigger <- struct{}{}:
|
||||
return true
|
||||
default:
|
||||
r.stateMu.Lock()
|
||||
r.queued = false
|
||||
r.stateMu.Unlock()
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) Status() Status {
|
||||
r.stateMu.Lock()
|
||||
running := r.running
|
||||
queued := r.queued
|
||||
startedAt := r.startedAt
|
||||
lastFinishedAt := r.lastFinishedAt
|
||||
r.stateMu.Unlock()
|
||||
|
||||
state := "idle"
|
||||
switch {
|
||||
case running && queued:
|
||||
state = "running_queued"
|
||||
case running:
|
||||
state = "running"
|
||||
case queued:
|
||||
state = "queued"
|
||||
}
|
||||
|
||||
return Status{
|
||||
State: state,
|
||||
Running: running,
|
||||
Queued: queued,
|
||||
StartedAt: startedAt,
|
||||
LastFinishedAt: lastFinishedAt,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -183,9 +236,13 @@ func (r *Runner) runPipelineLocked(ctx context.Context, manual bool) {
|
||||
log.Printf("[nightly] another pipeline is already running, skipping this trigger")
|
||||
return
|
||||
}
|
||||
defer r.runMu.Unlock()
|
||||
|
||||
started := r.cfg.Now()
|
||||
r.markStarted(started)
|
||||
defer func() {
|
||||
r.markFinished(r.cfg.Now())
|
||||
r.runMu.Unlock()
|
||||
}()
|
||||
|
||||
mode := "scheduled"
|
||||
if manual {
|
||||
@@ -207,6 +264,22 @@ func (r *Runner) runPipelineLocked(ctx context.Context, manual bool) {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) markStarted(started time.Time) {
|
||||
r.stateMu.Lock()
|
||||
defer r.stateMu.Unlock()
|
||||
r.running = true
|
||||
r.queued = false
|
||||
r.startedAt = started
|
||||
}
|
||||
|
||||
func (r *Runner) markFinished(finished time.Time) {
|
||||
r.stateMu.Lock()
|
||||
defer r.stateMu.Unlock()
|
||||
r.running = false
|
||||
r.startedAt = time.Time{}
|
||||
r.lastFinishedAt = finished
|
||||
}
|
||||
|
||||
// runPipeline executes the three phases. It returns when the pipeline finishes
|
||||
// OR ctx is done (deadline / cancel). Errors are logged but not propagated —
|
||||
// each phase is best-effort; downstream phases still attempt to run unless ctx
|
||||
|
||||
@@ -312,11 +312,14 @@ func TestCtxCancelPreventsLaterPhases(t *testing.T) {
|
||||
func TestTriggerNowIsNonBlocking(t *testing.T) {
|
||||
r := New(Config{Settings: newStubSettings()})
|
||||
// fill the trigger channel
|
||||
r.TriggerNow()
|
||||
if !r.TriggerNow() {
|
||||
t.Fatal("first TriggerNow should be accepted")
|
||||
}
|
||||
// Second call must not block
|
||||
done := make(chan struct{})
|
||||
var accepted bool
|
||||
go func() {
|
||||
r.TriggerNow()
|
||||
accepted = r.TriggerNow()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
@@ -324,4 +327,98 @@ func TestTriggerNowIsNonBlocking(t *testing.T) {
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
t.Fatal("TriggerNow blocked when channel is full")
|
||||
}
|
||||
if accepted {
|
||||
t.Fatal("second TriggerNow should be ignored when trigger channel is full")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusTracksQueuedRunningAndFinished(t *testing.T) {
|
||||
blockScan := make(chan struct{})
|
||||
scanStarted := make(chan struct{})
|
||||
var startedOnce sync.Once
|
||||
r := New(Config{
|
||||
Settings: newStubSettings(),
|
||||
ListScanTargets: func(context.Context) []string {
|
||||
return []string{"drive"}
|
||||
},
|
||||
RunScan: func(context.Context, string) {
|
||||
startedOnce.Do(func() { close(scanStarted) })
|
||||
<-blockScan
|
||||
},
|
||||
})
|
||||
|
||||
if got := r.Status(); got.State != "idle" || got.Running || got.Queued {
|
||||
t.Fatalf("initial status = %#v, want idle", got)
|
||||
}
|
||||
|
||||
if !r.TriggerNow() {
|
||||
t.Fatal("TriggerNow should queue a manual run")
|
||||
}
|
||||
if got := r.Status(); got.State != "queued" || got.Running || !got.Queued {
|
||||
t.Fatalf("queued status = %#v, want queued", got)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go r.Run(ctx)
|
||||
|
||||
select {
|
||||
case <-scanStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("pipeline did not start")
|
||||
}
|
||||
|
||||
if got := r.Status(); got.State != "running" || !got.Running || got.Queued || got.StartedAt.IsZero() {
|
||||
t.Fatalf("running status = %#v, want running with startedAt", got)
|
||||
}
|
||||
|
||||
if r.TriggerNow() {
|
||||
t.Fatal("TriggerNow during a run should be ignored")
|
||||
}
|
||||
if got := r.Status(); got.State != "running" || !got.Running || got.Queued {
|
||||
t.Fatalf("status after ignored trigger = %#v, want running", got)
|
||||
}
|
||||
|
||||
close(blockScan)
|
||||
deadline := time.After(time.Second)
|
||||
for {
|
||||
got := r.Status()
|
||||
if !got.Running && !got.Queued && !got.LastFinishedAt.IsZero() {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatalf("status did not finish; got=%#v", got)
|
||||
default:
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTriggerNowAcceptsOnlyOneConcurrentRequest(t *testing.T) {
|
||||
r := New(Config{Settings: newStubSettings()})
|
||||
|
||||
const callers = 16
|
||||
start := make(chan struct{})
|
||||
results := make(chan bool, callers)
|
||||
for i := 0; i < callers; i++ {
|
||||
go func() {
|
||||
<-start
|
||||
results <- r.TriggerNow()
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
|
||||
accepted := 0
|
||||
for i := 0; i < callers; i++ {
|
||||
if <-results {
|
||||
accepted++
|
||||
}
|
||||
}
|
||||
if accepted != 1 {
|
||||
t.Fatalf("accepted triggers = %d, want 1", accepted)
|
||||
}
|
||||
if got := r.Status(); got.State != "queued" || got.Running || !got.Queued {
|
||||
t.Fatalf("status = %#v, want one queued trigger", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package preview
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -345,9 +346,15 @@ func (g *Generator) Probe(ctx context.Context, link *drives.StreamLink) (float64
|
||||
args = append(args, ffmpegLink.URL)
|
||||
|
||||
cmd := exec.CommandContext(ctx2, g.cfg.FFprobePath, args...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return 0, ffmpegCommandError("ffprobe", err, out)
|
||||
errOut := stderr.Bytes()
|
||||
if len(errOut) == 0 {
|
||||
errOut = out
|
||||
}
|
||||
return 0, ffmpegCommandError("ffprobe", err, errOut)
|
||||
}
|
||||
raw := strings.TrimSpace(string(out))
|
||||
if raw == "" || raw == "N/A" {
|
||||
@@ -1033,11 +1040,12 @@ type ThumbWorker struct {
|
||||
}
|
||||
|
||||
const (
|
||||
defaultTransientMediaCooldown = 5 * time.Minute
|
||||
defaultGenerationRateLimitCooldown = 5 * time.Minute
|
||||
defaultWorkerQueueSize = 10000
|
||||
maxPreviewTeaserSizeBytes int64 = 5 * 1024 * 1024 * 1024
|
||||
previewStatusSkipped = "skipped"
|
||||
defaultTransientMediaCooldown = 5 * time.Minute
|
||||
defaultGenerationRateLimitCooldown = 5 * time.Minute
|
||||
defaultThumbTransientMediaMaxFailures = 3
|
||||
defaultWorkerQueueSize = 10000
|
||||
maxPreviewTeaserSizeBytes int64 = 5 * 1024 * 1024 * 1024
|
||||
previewStatusSkipped = "skipped"
|
||||
)
|
||||
|
||||
type rateLimitState struct {
|
||||
@@ -1339,13 +1347,16 @@ func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
defer w.queue.release(v)
|
||||
w.activity.start(v)
|
||||
defer w.activity.done()
|
||||
if !waitForRateLimitCooldown(ctx, &w.rateLimit, "thumb", w.Drive) {
|
||||
return
|
||||
retry := false
|
||||
if waitForRateLimitCooldown(ctx, &w.rateLimit, "thumb", w.Drive) {
|
||||
retry = w.process(ctx, v)
|
||||
}
|
||||
w.activity.done()
|
||||
w.queue.release(v)
|
||||
if retry && ctx.Err() == nil {
|
||||
w.EnqueueBlocking(ctx, v)
|
||||
}
|
||||
w.process(ctx, v)
|
||||
}
|
||||
|
||||
func waitForRateLimitCooldown(ctx context.Context, state *rateLimitState, label string, drive drives.Drive) bool {
|
||||
@@ -1427,15 +1438,34 @@ func (w *ThumbWorker) pauseForRateLimit(err error, step, title string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) pauseForRecoverableError(err error, step, title string) bool {
|
||||
func (w *ThumbWorker) pauseForRecoverableError(ctx context.Context, v *catalog.Video, err error, step string) bool {
|
||||
title := ""
|
||||
videoID := ""
|
||||
if v != nil {
|
||||
title = v.Title
|
||||
videoID = v.ID
|
||||
}
|
||||
if w.pauseForRateLimit(err, step, title) {
|
||||
return true
|
||||
}
|
||||
if !driveErrorShouldCooldown(w.Drive, err) {
|
||||
return false
|
||||
}
|
||||
failures := 1
|
||||
if w.Catalog != nil && videoID != "" {
|
||||
count, countErr := w.Catalog.IncrementThumbnailFailures(ctx, videoID)
|
||||
if countErr != nil {
|
||||
log.Printf("[thumb] drive=%s transient media source error count failed step=%s video=%s: %v", w.Drive.ID(), step, title, countErr)
|
||||
} else {
|
||||
failures = count
|
||||
}
|
||||
}
|
||||
if failures >= defaultThumbTransientMediaMaxFailures {
|
||||
log.Printf("[thumb] drive=%s transient media source error reached retry limit failures=%d/%d step=%s video=%s: %v", w.Drive.ID(), failures, defaultThumbTransientMediaMaxFailures, step, title, err)
|
||||
return false
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), w.RateLimitCooldown)
|
||||
log.Printf("[thumb] drive=%s transient media source error until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err)
|
||||
log.Printf("[thumb] drive=%s transient media source error until=%s failures=%d/%d step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), failures, defaultThumbTransientMediaMaxFailures, step, title, err)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -1481,9 +1511,9 @@ func driveErrorShouldCooldown(d drives.Drive, err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
|
||||
func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) bool {
|
||||
if w.skipIfRateLimited(v) {
|
||||
return
|
||||
return false
|
||||
}
|
||||
queued := v
|
||||
current := v
|
||||
@@ -1495,54 +1525,64 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
|
||||
v = loaded
|
||||
if loaded.ThumbnailURL != "" && loaded.DurationSeconds > 0 {
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
|
||||
return
|
||||
return false
|
||||
}
|
||||
}
|
||||
if current.ThumbnailURL != "" {
|
||||
durationBackfillFailed := false
|
||||
if current.DurationSeconds <= 0 {
|
||||
link, err := w.streamLink(ctx, current)
|
||||
if err != nil {
|
||||
if w.pauseForRecoverableError(err, "streamURL", current.Title) {
|
||||
return
|
||||
if w.pauseForRecoverableError(ctx, current, err, "streamURL") {
|
||||
return true
|
||||
}
|
||||
log.Printf("[thumb] probe streamURL %s: %v", current.Title, err)
|
||||
durationBackfillFailed = true
|
||||
} else if w.probeDuration(ctx, current, link) {
|
||||
return
|
||||
return true
|
||||
} else if current.DurationSeconds <= 0 {
|
||||
durationBackfillFailed = true
|
||||
}
|
||||
}
|
||||
if durationBackfillFailed {
|
||||
log.Printf("[thumb] skip duration backfill %s: thumbnail already exists but duration could not be probed", current.Title)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, current.ID, catalog.VideoMetaPatch{ThumbnailStatus: "skipped"})
|
||||
return false
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, current.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
|
||||
return
|
||||
return false
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "pending"})
|
||||
link, err := w.streamLink(ctx, v)
|
||||
if err != nil {
|
||||
if w.pauseForRecoverableError(err, "streamURL", v.Title) {
|
||||
return
|
||||
if w.pauseForRecoverableError(ctx, v, err, "streamURL") {
|
||||
return true
|
||||
}
|
||||
log.Printf("[thumb] streamURL %s: %v", v.Title, err)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
|
||||
return
|
||||
return false
|
||||
}
|
||||
if w.probeDuration(ctx, v, link) {
|
||||
return
|
||||
return true
|
||||
}
|
||||
|
||||
if err := w.generateThumbnailFromLink(ctx, v, link); err != nil {
|
||||
if localLink, ok := localPreviewLink(v); ok && link.URL != localLink.URL {
|
||||
if w.probeDuration(ctx, v, localLink) {
|
||||
return
|
||||
return true
|
||||
}
|
||||
if localErr := w.generateThumbnailFromLink(ctx, v, localLink); localErr == nil {
|
||||
return
|
||||
return false
|
||||
}
|
||||
}
|
||||
if w.pauseForRecoverableError(err, "generate", v.Title) {
|
||||
return
|
||||
if w.pauseForRecoverableError(ctx, v, err, "generate") {
|
||||
return true
|
||||
}
|
||||
log.Printf("[thumb] generate %s: %v", v.Title, err)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
|
||||
return
|
||||
return false
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) streamLink(ctx context.Context, v *catalog.Video) (*drives.StreamLink, error) {
|
||||
@@ -1570,7 +1610,7 @@ func (w *ThumbWorker) probeDuration(ctx context.Context, v *catalog.Video, link
|
||||
}
|
||||
return false
|
||||
}
|
||||
if w.pauseForRecoverableError(err, "probe", v.Title) {
|
||||
if w.pauseForRecoverableError(ctx, v, err, "probe") {
|
||||
return true
|
||||
}
|
||||
log.Printf("[thumb] probe %s: %v", v.Title, err)
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"errors"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -95,6 +97,24 @@ func TestTinyVideoPreviewPlanUsesWholeVideoAsSingleSegment(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeIgnoresStderrWarnings(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ffprobePath := filepath.Join(dir, "ffprobe")
|
||||
script := "#!/bin/sh\nprintf '%s\\n' 'h264 warning' >&2\nprintf '%s\\n' '364.800000'\n"
|
||||
if err := os.WriteFile(ffprobePath, []byte(script), 0o755); err != nil {
|
||||
t.Fatalf("write ffprobe stub: %v", err)
|
||||
}
|
||||
|
||||
gen := New(Config{FFprobePath: ffprobePath})
|
||||
got, err := gen.Probe(context.Background(), &drives.StreamLink{URL: filepath.Join(dir, "video.mp4")})
|
||||
if err != nil {
|
||||
t.Fatalf("probe: %v", err)
|
||||
}
|
||||
if got != 364.8 {
|
||||
t.Fatalf("duration = %v, want 364.8", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTeaserCandidateStartsKeepPrimaryAndAddFallbacks(t *testing.T) {
|
||||
primary := []float64{10.2, 64.65, 119.1, 173.55}
|
||||
got := teaserCandidateStarts(204, primary, 3)
|
||||
|
||||
@@ -89,6 +89,46 @@ func TestThumbWorkerBackfillsDurationWhenThumbnailAlreadyExists(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerSkipsDurationBackfillWhenExistingThumbnailCannotBeProbed(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-existing-thumbnail-probe-fails")
|
||||
video.ThumbnailURL = "/p/thumb/" + video.ID
|
||||
if err := cat.UpsertVideo(ctx, video); err != nil {
|
||||
t.Fatalf("update video: %v", err)
|
||||
}
|
||||
|
||||
gen := &fakeThumbGenerator{probeErr: errors.New("invalid media")}
|
||||
drv := &previewFakeDrive{}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
worker.process(ctx, video)
|
||||
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "/p/thumb/"+video.ID {
|
||||
t.Fatalf("thumbnail = %q, want unchanged existing thumbnail", got.ThumbnailURL)
|
||||
}
|
||||
if got.DurationSeconds != 0 {
|
||||
t.Fatalf("duration = %d, want still unknown", got.DurationSeconds)
|
||||
}
|
||||
skipped, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "skipped", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list skipped thumbnails: %v", err)
|
||||
}
|
||||
if len(skipped) != 1 || skipped[0].ID != video.ID {
|
||||
t.Fatalf("skipped thumbnails = %#v, want only %s", skipped, video.ID)
|
||||
}
|
||||
missing, err := cat.CountVideosNeedingThumbnail(ctx, video.DriveID)
|
||||
if err != nil {
|
||||
t.Fatalf("count videos needing thumbnail: %v", err)
|
||||
}
|
||||
if missing != 0 {
|
||||
t.Fatalf("missing thumbnails = %d, want 0 after duration backfill is skipped", missing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerFallsBackToLocalPreviewWhenDriveStreamFails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-local-preview")
|
||||
@@ -416,6 +456,113 @@ func TestThumbWorkerRateLimitCoolsDownFiveMinutes(t *testing.T) {
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 5*time.Minute)
|
||||
}
|
||||
|
||||
func TestThumbWorkerP115TransientErrorFailsAfterRetryLimit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-p115-transient")
|
||||
|
||||
gen := &fakeThumbGenerator{
|
||||
generateErr: errors.New("ffmpeg thumb: exit status 183, stderr: partial file Cannot determine format of input 0:0 after EOF"),
|
||||
}
|
||||
drv := &previewFakeDrive{kind: "p115"}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
for attempt := 1; attempt <= defaultThumbTransientMediaMaxFailures; attempt++ {
|
||||
worker.rateLimit = rateLimitState{}
|
||||
worker.process(ctx, video)
|
||||
|
||||
if attempt < defaultThumbTransientMediaMaxFailures {
|
||||
pending, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "pending", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list pending thumbnails: %v", err)
|
||||
}
|
||||
if len(pending) != 1 || pending[0].ID != video.ID {
|
||||
t.Fatalf("attempt %d pending thumbnails = %#v, want only %s", attempt, pending, video.ID)
|
||||
}
|
||||
missing, err := cat.CountVideosNeedingThumbnail(ctx, video.DriveID)
|
||||
if err != nil {
|
||||
t.Fatalf("count missing thumbnails: %v", err)
|
||||
}
|
||||
if missing != 1 {
|
||||
t.Fatalf("attempt %d missing thumbnails = %d, want 1 before retry limit", attempt, missing)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list failed thumbnails: %v", err)
|
||||
}
|
||||
if len(failed) != 1 || failed[0].ID != video.ID {
|
||||
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
|
||||
}
|
||||
missing, err := cat.CountVideosNeedingThumbnail(ctx, video.DriveID)
|
||||
if err != nil {
|
||||
t.Fatalf("count missing thumbnails: %v", err)
|
||||
}
|
||||
if missing != 0 {
|
||||
t.Fatalf("missing thumbnails = %d, want 0 after retry limit marks failed", missing)
|
||||
}
|
||||
}
|
||||
|
||||
if gen.generateCalls != defaultThumbTransientMediaMaxFailures {
|
||||
t.Fatalf("generate calls = %d, want %d", gen.generateCalls, defaultThumbTransientMediaMaxFailures)
|
||||
}
|
||||
|
||||
if err := cat.UpdateVideoMeta(ctx, video.ID, catalog.VideoMetaPatch{
|
||||
ThumbnailStatus: "pending",
|
||||
ResetThumbnailFailures: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("reset thumbnail status: %v", err)
|
||||
}
|
||||
worker.rateLimit = rateLimitState{}
|
||||
worker.process(ctx, video)
|
||||
|
||||
pending, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "pending", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list pending thumbnails after reset: %v", err)
|
||||
}
|
||||
if len(pending) != 1 || pending[0].ID != video.ID {
|
||||
t.Fatalf("pending thumbnails after reset = %#v, want only %s", pending, video.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerRequeuesP115TransientErrorBeforeRetryLimit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-p115-requeue")
|
||||
|
||||
gen := &fakeThumbGenerator{
|
||||
generateErr: errors.New("ffmpeg thumb: partial file Cannot determine format of input 0:0 after EOF"),
|
||||
}
|
||||
drv := &previewFakeDrive{kind: "p115"}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
worker.processQueued(ctx, video)
|
||||
|
||||
select {
|
||||
case queued := <-worker.ch:
|
||||
if queued.ID != video.ID {
|
||||
t.Fatalf("requeued video id = %q, want %q", queued.ID, video.ID)
|
||||
}
|
||||
default:
|
||||
t.Fatal("expected transient thumbnail failure to requeue the same video")
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want empty after transient failure", got.ThumbnailURL)
|
||||
}
|
||||
pending, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "pending", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list pending thumbnails: %v", err)
|
||||
}
|
||||
if len(pending) != 1 || pending[0].ID != video.ID {
|
||||
t.Fatalf("pending thumbnails = %#v, want only %s", pending, video.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewWorkerP115TransientErrorKeepsVideoPending(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "preview-p115-transient")
|
||||
@@ -508,6 +655,7 @@ type fakeThumbGenerator struct {
|
||||
thumbnailDuration float64
|
||||
thumbnailURL string
|
||||
probeCalls int
|
||||
generateCalls int
|
||||
probeDuration float64
|
||||
probeErr error
|
||||
generateErr error
|
||||
@@ -522,6 +670,7 @@ func (g *fakeThumbGenerator) Probe(context.Context, *drives.StreamLink) (float64
|
||||
}
|
||||
|
||||
func (g *fakeThumbGenerator) GenerateThumbnail(_ context.Context, link *drives.StreamLink, videoID string, duration float64) (string, error) {
|
||||
g.generateCalls++
|
||||
g.thumbnailVideoID = videoID
|
||||
g.thumbnailDuration = duration
|
||||
if link != nil {
|
||||
|
||||
@@ -16,11 +16,11 @@ type ParsedName struct {
|
||||
}
|
||||
|
||||
var (
|
||||
reTags = regexp.MustCompile(`^\s*\[([^\]]+)\]\s*`) // [tag1,tag2]
|
||||
reTags = regexp.MustCompile(`^\s*\[([^\]]+)\]\s*`) // [前缀]
|
||||
reAuthor = regexp.MustCompile(`\s*-\s*([^-]+?)\s*$`) // - author
|
||||
)
|
||||
|
||||
// Parse 按约定解析:[tag1,tag2] 标题 - 作者.ext
|
||||
// Parse 按约定解析:[前缀] 标题 - 作者.ext
|
||||
// 任何字段缺失都能降级
|
||||
func Parse(filename string) ParsedName {
|
||||
name := strings.TrimSuffix(filename, path.Ext(filename))
|
||||
|
||||
@@ -254,6 +254,93 @@ func TestRunAddsShortCollectionDirectoryAsTag(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDoesNotRecreateDeletedCollectionDirectoryTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
now := time.Now()
|
||||
for _, id := range []string{"existing-1", "existing-2"} {
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: "Existing",
|
||||
Category: "sunny",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed existing sunny video: %v", err)
|
||||
}
|
||||
}
|
||||
if label, ok, err := cat.EnsureCollectionTag(ctx, "sunny"); err != nil || !ok || label != "sunny" {
|
||||
t.Fatalf("ensure collection = %q, %v, %v; want sunny true nil", label, ok, err)
|
||||
}
|
||||
tags, err := cat.ListTags(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list tags: %v", err)
|
||||
}
|
||||
var tagID int64
|
||||
for _, tag := range tags {
|
||||
if tag.Label == "sunny" {
|
||||
tagID = tag.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
if tagID == 0 {
|
||||
t.Fatal("sunny tag not found before delete")
|
||||
}
|
||||
if _, err := cat.DeleteTag(ctx, tagID); err != nil {
|
||||
t.Fatalf("delete tag: %v", err)
|
||||
}
|
||||
|
||||
drv := &scannerTreeFakeDrive{
|
||||
entries: map[string][]drives.Entry{
|
||||
"root": {{
|
||||
ID: "dir-1",
|
||||
Name: "sunny",
|
||||
IsDir: true,
|
||||
}},
|
||||
"dir-1": {{
|
||||
ID: "file-1",
|
||||
ParentID: "dir-1",
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
ModTime: now,
|
||||
}},
|
||||
},
|
||||
}
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
|
||||
if _, err := sc.Run(ctx, ""); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(ctx, "fake-drive-file-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if len(got.Tags) != 0 {
|
||||
t.Fatalf("tags = %#v, want none", got.Tags)
|
||||
}
|
||||
tags, err = cat.ListTags(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list tags after scan: %v", err)
|
||||
}
|
||||
for _, tag := range tags {
|
||||
if tag.Label == "sunny" {
|
||||
t.Fatal("deleted collection tag was recreated during scan")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMapsAVCodeDirectoryToAVTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
// Copyright 2018 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package socks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
noDeadline = time.Time{}
|
||||
aLongTimeAgo = time.Unix(1, 0)
|
||||
)
|
||||
|
||||
func (d *Dialer) connect(ctx context.Context, c net.Conn, address string) (_ net.Addr, ctxErr error) {
|
||||
host, port, err := splitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if deadline, ok := ctx.Deadline(); ok && !deadline.IsZero() {
|
||||
c.SetDeadline(deadline)
|
||||
defer c.SetDeadline(noDeadline)
|
||||
}
|
||||
if ctx != context.Background() {
|
||||
errCh := make(chan error, 1)
|
||||
done := make(chan struct{})
|
||||
defer func() {
|
||||
close(done)
|
||||
if ctxErr == nil {
|
||||
ctxErr = <-errCh
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
c.SetDeadline(aLongTimeAgo)
|
||||
errCh <- ctx.Err()
|
||||
case <-done:
|
||||
errCh <- nil
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
b := make([]byte, 0, 6+len(host)) // the size here is just an estimate
|
||||
b = append(b, Version5)
|
||||
if len(d.AuthMethods) == 0 || d.Authenticate == nil {
|
||||
b = append(b, 1, byte(AuthMethodNotRequired))
|
||||
} else {
|
||||
ams := d.AuthMethods
|
||||
if len(ams) > 255 {
|
||||
return nil, errors.New("too many authentication methods")
|
||||
}
|
||||
b = append(b, byte(len(ams)))
|
||||
for _, am := range ams {
|
||||
b = append(b, byte(am))
|
||||
}
|
||||
}
|
||||
if _, ctxErr = c.Write(b); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if _, ctxErr = io.ReadFull(c, b[:2]); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
if b[0] != Version5 {
|
||||
return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0])))
|
||||
}
|
||||
am := AuthMethod(b[1])
|
||||
if am == AuthMethodNoAcceptableMethods {
|
||||
return nil, errors.New("no acceptable authentication methods")
|
||||
}
|
||||
if d.Authenticate != nil {
|
||||
if ctxErr = d.Authenticate(ctx, c, am); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
b = b[:0]
|
||||
b = append(b, Version5, byte(d.cmd), 0)
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
b = append(b, AddrTypeIPv4)
|
||||
b = append(b, ip4...)
|
||||
} else if ip6 := ip.To16(); ip6 != nil {
|
||||
b = append(b, AddrTypeIPv6)
|
||||
b = append(b, ip6...)
|
||||
} else {
|
||||
return nil, errors.New("unknown address type")
|
||||
}
|
||||
} else {
|
||||
if len(host) > 255 {
|
||||
return nil, errors.New("FQDN too long")
|
||||
}
|
||||
b = append(b, AddrTypeFQDN)
|
||||
b = append(b, byte(len(host)))
|
||||
b = append(b, host...)
|
||||
}
|
||||
b = append(b, byte(port>>8), byte(port))
|
||||
if _, ctxErr = c.Write(b); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if _, ctxErr = io.ReadFull(c, b[:4]); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
if b[0] != Version5 {
|
||||
return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0])))
|
||||
}
|
||||
if cmdErr := Reply(b[1]); cmdErr != StatusSucceeded {
|
||||
return nil, errors.New("unknown error " + cmdErr.String())
|
||||
}
|
||||
if b[2] != 0 {
|
||||
return nil, errors.New("non-zero reserved field")
|
||||
}
|
||||
l := 2
|
||||
var a Addr
|
||||
switch b[3] {
|
||||
case AddrTypeIPv4:
|
||||
l += net.IPv4len
|
||||
a.IP = make(net.IP, net.IPv4len)
|
||||
case AddrTypeIPv6:
|
||||
l += net.IPv6len
|
||||
a.IP = make(net.IP, net.IPv6len)
|
||||
case AddrTypeFQDN:
|
||||
if _, err := io.ReadFull(c, b[:1]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l += int(b[0])
|
||||
default:
|
||||
return nil, errors.New("unknown address type " + strconv.Itoa(int(b[3])))
|
||||
}
|
||||
if cap(b) < l {
|
||||
b = make([]byte, l)
|
||||
} else {
|
||||
b = b[:l]
|
||||
}
|
||||
if _, ctxErr = io.ReadFull(c, b); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
if a.IP != nil {
|
||||
copy(a.IP, b)
|
||||
} else {
|
||||
a.Name = string(b[:len(b)-2])
|
||||
}
|
||||
a.Port = int(b[len(b)-2])<<8 | int(b[len(b)-1])
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
func splitHostPort(address string) (string, int, error) {
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
portnum, err := strconv.Atoi(port)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if 1 > portnum || portnum > 0xffff {
|
||||
return "", 0, errors.New("port number out of range " + port)
|
||||
}
|
||||
return host, portnum, nil
|
||||
}
|
||||
+317
@@ -0,0 +1,317 @@
|
||||
// Copyright 2018 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package socks provides a SOCKS version 5 client implementation.
|
||||
//
|
||||
// SOCKS protocol version 5 is defined in RFC 1928.
|
||||
// Username/Password authentication for SOCKS version 5 is defined in
|
||||
// RFC 1929.
|
||||
package socks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// A Command represents a SOCKS command.
|
||||
type Command int
|
||||
|
||||
func (cmd Command) String() string {
|
||||
switch cmd {
|
||||
case CmdConnect:
|
||||
return "socks connect"
|
||||
case cmdBind:
|
||||
return "socks bind"
|
||||
default:
|
||||
return "socks " + strconv.Itoa(int(cmd))
|
||||
}
|
||||
}
|
||||
|
||||
// An AuthMethod represents a SOCKS authentication method.
|
||||
type AuthMethod int
|
||||
|
||||
// A Reply represents a SOCKS command reply code.
|
||||
type Reply int
|
||||
|
||||
func (code Reply) String() string {
|
||||
switch code {
|
||||
case StatusSucceeded:
|
||||
return "succeeded"
|
||||
case 0x01:
|
||||
return "general SOCKS server failure"
|
||||
case 0x02:
|
||||
return "connection not allowed by ruleset"
|
||||
case 0x03:
|
||||
return "network unreachable"
|
||||
case 0x04:
|
||||
return "host unreachable"
|
||||
case 0x05:
|
||||
return "connection refused"
|
||||
case 0x06:
|
||||
return "TTL expired"
|
||||
case 0x07:
|
||||
return "command not supported"
|
||||
case 0x08:
|
||||
return "address type not supported"
|
||||
default:
|
||||
return "unknown code: " + strconv.Itoa(int(code))
|
||||
}
|
||||
}
|
||||
|
||||
// Wire protocol constants.
|
||||
const (
|
||||
Version5 = 0x05
|
||||
|
||||
AddrTypeIPv4 = 0x01
|
||||
AddrTypeFQDN = 0x03
|
||||
AddrTypeIPv6 = 0x04
|
||||
|
||||
CmdConnect Command = 0x01 // establishes an active-open forward proxy connection
|
||||
cmdBind Command = 0x02 // establishes a passive-open forward proxy connection
|
||||
|
||||
AuthMethodNotRequired AuthMethod = 0x00 // no authentication required
|
||||
AuthMethodUsernamePassword AuthMethod = 0x02 // use username/password
|
||||
AuthMethodNoAcceptableMethods AuthMethod = 0xff // no acceptable authentication methods
|
||||
|
||||
StatusSucceeded Reply = 0x00
|
||||
)
|
||||
|
||||
// An Addr represents a SOCKS-specific address.
|
||||
// Either Name or IP is used exclusively.
|
||||
type Addr struct {
|
||||
Name string // fully-qualified domain name
|
||||
IP net.IP
|
||||
Port int
|
||||
}
|
||||
|
||||
func (a *Addr) Network() string { return "socks" }
|
||||
|
||||
func (a *Addr) String() string {
|
||||
if a == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
port := strconv.Itoa(a.Port)
|
||||
if a.IP == nil {
|
||||
return net.JoinHostPort(a.Name, port)
|
||||
}
|
||||
return net.JoinHostPort(a.IP.String(), port)
|
||||
}
|
||||
|
||||
// A Conn represents a forward proxy connection.
|
||||
type Conn struct {
|
||||
net.Conn
|
||||
|
||||
boundAddr net.Addr
|
||||
}
|
||||
|
||||
// BoundAddr returns the address assigned by the proxy server for
|
||||
// connecting to the command target address from the proxy server.
|
||||
func (c *Conn) BoundAddr() net.Addr {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
return c.boundAddr
|
||||
}
|
||||
|
||||
// A Dialer holds SOCKS-specific options.
|
||||
type Dialer struct {
|
||||
cmd Command // either CmdConnect or cmdBind
|
||||
proxyNetwork string // network between a proxy server and a client
|
||||
proxyAddress string // proxy server address
|
||||
|
||||
// ProxyDial specifies the optional dial function for
|
||||
// establishing the transport connection.
|
||||
ProxyDial func(context.Context, string, string) (net.Conn, error)
|
||||
|
||||
// AuthMethods specifies the list of request authentication
|
||||
// methods.
|
||||
// If empty, SOCKS client requests only AuthMethodNotRequired.
|
||||
AuthMethods []AuthMethod
|
||||
|
||||
// Authenticate specifies the optional authentication
|
||||
// function. It must be non-nil when AuthMethods is not empty.
|
||||
// It must return an error when the authentication is failed.
|
||||
Authenticate func(context.Context, io.ReadWriter, AuthMethod) error
|
||||
}
|
||||
|
||||
// DialContext connects to the provided address on the provided
|
||||
// network.
|
||||
//
|
||||
// The returned error value may be a net.OpError. When the Op field of
|
||||
// net.OpError contains "socks", the Source field contains a proxy
|
||||
// server address and the Addr field contains a command target
|
||||
// address.
|
||||
//
|
||||
// See func Dial of the net package of standard library for a
|
||||
// description of the network and address parameters.
|
||||
func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
if err := d.validateTarget(network, address); err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
if ctx == nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")}
|
||||
}
|
||||
var err error
|
||||
var c net.Conn
|
||||
if d.ProxyDial != nil {
|
||||
c, err = d.ProxyDial(ctx, d.proxyNetwork, d.proxyAddress)
|
||||
} else {
|
||||
var dd net.Dialer
|
||||
c, err = dd.DialContext(ctx, d.proxyNetwork, d.proxyAddress)
|
||||
}
|
||||
if err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
a, err := d.connect(ctx, c, address)
|
||||
if err != nil {
|
||||
c.Close()
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
return &Conn{Conn: c, boundAddr: a}, nil
|
||||
}
|
||||
|
||||
// DialWithConn initiates a connection from SOCKS server to the target
|
||||
// network and address using the connection c that is already
|
||||
// connected to the SOCKS server.
|
||||
//
|
||||
// It returns the connection's local address assigned by the SOCKS
|
||||
// server.
|
||||
func (d *Dialer) DialWithConn(ctx context.Context, c net.Conn, network, address string) (net.Addr, error) {
|
||||
if err := d.validateTarget(network, address); err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
if ctx == nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")}
|
||||
}
|
||||
a, err := d.connect(ctx, c, address)
|
||||
if err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// Dial connects to the provided address on the provided network.
|
||||
//
|
||||
// Unlike DialContext, it returns a raw transport connection instead
|
||||
// of a forward proxy connection.
|
||||
//
|
||||
// Deprecated: Use DialContext or DialWithConn instead.
|
||||
func (d *Dialer) Dial(network, address string) (net.Conn, error) {
|
||||
if err := d.validateTarget(network, address); err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
var err error
|
||||
var c net.Conn
|
||||
if d.ProxyDial != nil {
|
||||
c, err = d.ProxyDial(context.Background(), d.proxyNetwork, d.proxyAddress)
|
||||
} else {
|
||||
c, err = net.Dial(d.proxyNetwork, d.proxyAddress)
|
||||
}
|
||||
if err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
if _, err := d.DialWithConn(context.Background(), c, network, address); err != nil {
|
||||
c.Close()
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (d *Dialer) validateTarget(network, address string) error {
|
||||
switch network {
|
||||
case "tcp", "tcp6", "tcp4":
|
||||
default:
|
||||
return errors.New("network not implemented")
|
||||
}
|
||||
switch d.cmd {
|
||||
case CmdConnect, cmdBind:
|
||||
default:
|
||||
return errors.New("command not implemented")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Dialer) pathAddrs(address string) (proxy, dst net.Addr, err error) {
|
||||
for i, s := range []string{d.proxyAddress, address} {
|
||||
host, port, err := splitHostPort(s)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
a := &Addr{Port: port}
|
||||
a.IP = net.ParseIP(host)
|
||||
if a.IP == nil {
|
||||
a.Name = host
|
||||
}
|
||||
if i == 0 {
|
||||
proxy = a
|
||||
} else {
|
||||
dst = a
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// NewDialer returns a new Dialer that dials through the provided
|
||||
// proxy server's network and address.
|
||||
func NewDialer(network, address string) *Dialer {
|
||||
return &Dialer{proxyNetwork: network, proxyAddress: address, cmd: CmdConnect}
|
||||
}
|
||||
|
||||
const (
|
||||
authUsernamePasswordVersion = 0x01
|
||||
authStatusSucceeded = 0x00
|
||||
)
|
||||
|
||||
// UsernamePassword are the credentials for the username/password
|
||||
// authentication method.
|
||||
type UsernamePassword struct {
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
// Authenticate authenticates a pair of username and password with the
|
||||
// proxy server.
|
||||
func (up *UsernamePassword) Authenticate(ctx context.Context, rw io.ReadWriter, auth AuthMethod) error {
|
||||
switch auth {
|
||||
case AuthMethodNotRequired:
|
||||
return nil
|
||||
case AuthMethodUsernamePassword:
|
||||
if len(up.Username) == 0 || len(up.Username) > 255 || len(up.Password) > 255 {
|
||||
return errors.New("invalid username/password")
|
||||
}
|
||||
b := []byte{authUsernamePasswordVersion}
|
||||
b = append(b, byte(len(up.Username)))
|
||||
b = append(b, up.Username...)
|
||||
b = append(b, byte(len(up.Password)))
|
||||
b = append(b, up.Password...)
|
||||
// TODO(mikio): handle IO deadlines and cancelation if
|
||||
// necessary
|
||||
if _, err := rw.Write(b); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.ReadFull(rw, b[:2]); err != nil {
|
||||
return err
|
||||
}
|
||||
if b[0] != authUsernamePasswordVersion {
|
||||
return errors.New("invalid username/password version")
|
||||
}
|
||||
if b[1] != authStatusSucceeded {
|
||||
return errors.New("username/password authentication failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errors.New("unsupported authentication method " + strconv.Itoa(int(auth)))
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
// Copyright 2019 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
)
|
||||
|
||||
// A ContextDialer dials using a context.
|
||||
type ContextDialer interface {
|
||||
DialContext(ctx context.Context, network, address string) (net.Conn, error)
|
||||
}
|
||||
|
||||
// Dial works like DialContext on net.Dialer but using a dialer returned by FromEnvironment.
|
||||
//
|
||||
// The passed ctx is only used for returning the Conn, not the lifetime of the Conn.
|
||||
//
|
||||
// Custom dialers (registered via RegisterDialerType) that do not implement ContextDialer
|
||||
// can leak a goroutine for as long as it takes the underlying Dialer implementation to timeout.
|
||||
//
|
||||
// A Conn returned from a successful Dial after the context has been cancelled will be immediately closed.
|
||||
func Dial(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
d := FromEnvironment()
|
||||
if xd, ok := d.(ContextDialer); ok {
|
||||
return xd.DialContext(ctx, network, address)
|
||||
}
|
||||
return dialContext(ctx, d, network, address)
|
||||
}
|
||||
|
||||
// WARNING: this can leak a goroutine for as long as the underlying Dialer implementation takes to timeout
|
||||
// A Conn returned from a successful Dial after the context has been cancelled will be immediately closed.
|
||||
func dialContext(ctx context.Context, d Dialer, network, address string) (net.Conn, error) {
|
||||
var (
|
||||
conn net.Conn
|
||||
done = make(chan struct{}, 1)
|
||||
err error
|
||||
)
|
||||
go func() {
|
||||
conn, err = d.Dial(network, address)
|
||||
close(done)
|
||||
if conn != nil && ctx.Err() != nil {
|
||||
conn.Close()
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
case <-done:
|
||||
}
|
||||
return conn, err
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
)
|
||||
|
||||
type direct struct{}
|
||||
|
||||
// Direct implements Dialer by making network connections directly using net.Dial or net.DialContext.
|
||||
var Direct = direct{}
|
||||
|
||||
var (
|
||||
_ Dialer = Direct
|
||||
_ ContextDialer = Direct
|
||||
)
|
||||
|
||||
// Dial directly invokes net.Dial with the supplied parameters.
|
||||
func (direct) Dial(network, addr string) (net.Conn, error) {
|
||||
return net.Dial(network, addr)
|
||||
}
|
||||
|
||||
// DialContext instantiates a net.Dialer and invokes its DialContext receiver with the supplied parameters.
|
||||
func (direct) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, network, addr)
|
||||
}
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A PerHost directs connections to a default Dialer unless the host name
|
||||
// requested matches one of a number of exceptions.
|
||||
type PerHost struct {
|
||||
def, bypass Dialer
|
||||
|
||||
bypassNetworks []*net.IPNet
|
||||
bypassIPs []net.IP
|
||||
bypassZones []string
|
||||
bypassHosts []string
|
||||
}
|
||||
|
||||
// NewPerHost returns a PerHost Dialer that directs connections to either
|
||||
// defaultDialer or bypass, depending on whether the connection matches one of
|
||||
// the configured rules.
|
||||
func NewPerHost(defaultDialer, bypass Dialer) *PerHost {
|
||||
return &PerHost{
|
||||
def: defaultDialer,
|
||||
bypass: bypass,
|
||||
}
|
||||
}
|
||||
|
||||
// Dial connects to the address addr on the given network through either
|
||||
// defaultDialer or bypass.
|
||||
func (p *PerHost) Dial(network, addr string) (c net.Conn, err error) {
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return p.dialerForRequest(host).Dial(network, addr)
|
||||
}
|
||||
|
||||
// DialContext connects to the address addr on the given network through either
|
||||
// defaultDialer or bypass.
|
||||
func (p *PerHost) DialContext(ctx context.Context, network, addr string) (c net.Conn, err error) {
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := p.dialerForRequest(host)
|
||||
if x, ok := d.(ContextDialer); ok {
|
||||
return x.DialContext(ctx, network, addr)
|
||||
}
|
||||
return dialContext(ctx, d, network, addr)
|
||||
}
|
||||
|
||||
func (p *PerHost) dialerForRequest(host string) Dialer {
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
for _, net := range p.bypassNetworks {
|
||||
if net.Contains(ip) {
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
for _, bypassIP := range p.bypassIPs {
|
||||
if bypassIP.Equal(ip) {
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
return p.def
|
||||
}
|
||||
|
||||
for _, zone := range p.bypassZones {
|
||||
if strings.HasSuffix(host, zone) {
|
||||
return p.bypass
|
||||
}
|
||||
if host == zone[1:] {
|
||||
// For a zone ".example.com", we match "example.com"
|
||||
// too.
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
for _, bypassHost := range p.bypassHosts {
|
||||
if bypassHost == host {
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
return p.def
|
||||
}
|
||||
|
||||
// AddFromString parses a string that contains comma-separated values
|
||||
// specifying hosts that should use the bypass proxy. Each value is either an
|
||||
// IP address, a CIDR range, a zone (*.example.com) or a host name
|
||||
// (localhost). A best effort is made to parse the string and errors are
|
||||
// ignored.
|
||||
func (p *PerHost) AddFromString(s string) {
|
||||
hosts := strings.Split(s, ",")
|
||||
for _, host := range hosts {
|
||||
host = strings.TrimSpace(host)
|
||||
if len(host) == 0 {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(host, "/") {
|
||||
// We assume that it's a CIDR address like 127.0.0.0/8
|
||||
if _, net, err := net.ParseCIDR(host); err == nil {
|
||||
p.AddNetwork(net)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
p.AddIP(ip)
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(host, "*.") {
|
||||
p.AddZone(host[1:])
|
||||
continue
|
||||
}
|
||||
p.AddHost(host)
|
||||
}
|
||||
}
|
||||
|
||||
// AddIP specifies an IP address that will use the bypass proxy. Note that
|
||||
// this will only take effect if a literal IP address is dialed. A connection
|
||||
// to a named host will never match an IP.
|
||||
func (p *PerHost) AddIP(ip net.IP) {
|
||||
p.bypassIPs = append(p.bypassIPs, ip)
|
||||
}
|
||||
|
||||
// AddNetwork specifies an IP range that will use the bypass proxy. Note that
|
||||
// this will only take effect if a literal IP address is dialed. A connection
|
||||
// to a named host will never match.
|
||||
func (p *PerHost) AddNetwork(net *net.IPNet) {
|
||||
p.bypassNetworks = append(p.bypassNetworks, net)
|
||||
}
|
||||
|
||||
// AddZone specifies a DNS suffix that will use the bypass proxy. A zone of
|
||||
// "example.com" matches "example.com" and all of its subdomains.
|
||||
func (p *PerHost) AddZone(zone string) {
|
||||
zone = strings.TrimSuffix(zone, ".")
|
||||
if !strings.HasPrefix(zone, ".") {
|
||||
zone = "." + zone
|
||||
}
|
||||
p.bypassZones = append(p.bypassZones, zone)
|
||||
}
|
||||
|
||||
// AddHost specifies a host name that will use the bypass proxy.
|
||||
func (p *PerHost) AddHost(host string) {
|
||||
host = strings.TrimSuffix(host, ".")
|
||||
p.bypassHosts = append(p.bypassHosts, host)
|
||||
}
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package proxy provides support for a variety of protocols to proxy network
|
||||
// data.
|
||||
package proxy // import "golang.org/x/net/proxy"
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// A Dialer is a means to establish a connection.
|
||||
// Custom dialers should also implement ContextDialer.
|
||||
type Dialer interface {
|
||||
// Dial connects to the given address via the proxy.
|
||||
Dial(network, addr string) (c net.Conn, err error)
|
||||
}
|
||||
|
||||
// Auth contains authentication parameters that specific Dialers may require.
|
||||
type Auth struct {
|
||||
User, Password string
|
||||
}
|
||||
|
||||
// FromEnvironment returns the dialer specified by the proxy-related
|
||||
// variables in the environment and makes underlying connections
|
||||
// directly.
|
||||
func FromEnvironment() Dialer {
|
||||
return FromEnvironmentUsing(Direct)
|
||||
}
|
||||
|
||||
// FromEnvironmentUsing returns the dialer specify by the proxy-related
|
||||
// variables in the environment and makes underlying connections
|
||||
// using the provided forwarding Dialer (for instance, a *net.Dialer
|
||||
// with desired configuration).
|
||||
func FromEnvironmentUsing(forward Dialer) Dialer {
|
||||
allProxy := allProxyEnv.Get()
|
||||
if len(allProxy) == 0 {
|
||||
return forward
|
||||
}
|
||||
|
||||
proxyURL, err := url.Parse(allProxy)
|
||||
if err != nil {
|
||||
return forward
|
||||
}
|
||||
proxy, err := FromURL(proxyURL, forward)
|
||||
if err != nil {
|
||||
return forward
|
||||
}
|
||||
|
||||
noProxy := noProxyEnv.Get()
|
||||
if len(noProxy) == 0 {
|
||||
return proxy
|
||||
}
|
||||
|
||||
perHost := NewPerHost(proxy, forward)
|
||||
perHost.AddFromString(noProxy)
|
||||
return perHost
|
||||
}
|
||||
|
||||
// proxySchemes is a map from URL schemes to a function that creates a Dialer
|
||||
// from a URL with such a scheme.
|
||||
var proxySchemes map[string]func(*url.URL, Dialer) (Dialer, error)
|
||||
|
||||
// RegisterDialerType takes a URL scheme and a function to generate Dialers from
|
||||
// a URL with that scheme and a forwarding Dialer. Registered schemes are used
|
||||
// by FromURL.
|
||||
func RegisterDialerType(scheme string, f func(*url.URL, Dialer) (Dialer, error)) {
|
||||
if proxySchemes == nil {
|
||||
proxySchemes = make(map[string]func(*url.URL, Dialer) (Dialer, error))
|
||||
}
|
||||
proxySchemes[scheme] = f
|
||||
}
|
||||
|
||||
// FromURL returns a Dialer given a URL specification and an underlying
|
||||
// Dialer for it to make network requests.
|
||||
func FromURL(u *url.URL, forward Dialer) (Dialer, error) {
|
||||
var auth *Auth
|
||||
if u.User != nil {
|
||||
auth = new(Auth)
|
||||
auth.User = u.User.Username()
|
||||
if p, ok := u.User.Password(); ok {
|
||||
auth.Password = p
|
||||
}
|
||||
}
|
||||
|
||||
switch u.Scheme {
|
||||
case "socks5", "socks5h":
|
||||
addr := u.Hostname()
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
port = "1080"
|
||||
}
|
||||
return SOCKS5("tcp", net.JoinHostPort(addr, port), auth, forward)
|
||||
}
|
||||
|
||||
// If the scheme doesn't match any of the built-in schemes, see if it
|
||||
// was registered by another package.
|
||||
if proxySchemes != nil {
|
||||
if f, ok := proxySchemes[u.Scheme]; ok {
|
||||
return f(u, forward)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.New("proxy: unknown scheme: " + u.Scheme)
|
||||
}
|
||||
|
||||
var (
|
||||
allProxyEnv = &envOnce{
|
||||
names: []string{"ALL_PROXY", "all_proxy"},
|
||||
}
|
||||
noProxyEnv = &envOnce{
|
||||
names: []string{"NO_PROXY", "no_proxy"},
|
||||
}
|
||||
)
|
||||
|
||||
// envOnce looks up an environment variable (optionally by multiple
|
||||
// names) once. It mitigates expensive lookups on some platforms
|
||||
// (e.g. Windows).
|
||||
// (Borrowed from net/http/transport.go)
|
||||
type envOnce struct {
|
||||
names []string
|
||||
once sync.Once
|
||||
val string
|
||||
}
|
||||
|
||||
func (e *envOnce) Get() string {
|
||||
e.once.Do(e.init)
|
||||
return e.val
|
||||
}
|
||||
|
||||
func (e *envOnce) init() {
|
||||
for _, n := range e.names {
|
||||
e.val = os.Getenv(n)
|
||||
if e.val != "" {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reset is used by tests
|
||||
func (e *envOnce) reset() {
|
||||
e.once = sync.Once{}
|
||||
e.val = ""
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
|
||||
"golang.org/x/net/internal/socks"
|
||||
)
|
||||
|
||||
// SOCKS5 returns a Dialer that makes SOCKSv5 connections to the given
|
||||
// address with an optional username and password.
|
||||
// See RFC 1928 and RFC 1929.
|
||||
func SOCKS5(network, address string, auth *Auth, forward Dialer) (Dialer, error) {
|
||||
d := socks.NewDialer(network, address)
|
||||
if forward != nil {
|
||||
if f, ok := forward.(ContextDialer); ok {
|
||||
d.ProxyDial = func(ctx context.Context, network string, address string) (net.Conn, error) {
|
||||
return f.DialContext(ctx, network, address)
|
||||
}
|
||||
} else {
|
||||
d.ProxyDial = func(ctx context.Context, network string, address string) (net.Conn, error) {
|
||||
return dialContext(ctx, forward, network, address)
|
||||
}
|
||||
}
|
||||
}
|
||||
if auth != nil {
|
||||
up := socks.UsernamePassword{
|
||||
Username: auth.User,
|
||||
Password: auth.Password,
|
||||
}
|
||||
d.AuthMethods = []socks.AuthMethod{
|
||||
socks.AuthMethodNotRequired,
|
||||
socks.AuthMethodUsernamePassword,
|
||||
}
|
||||
d.Authenticate = up.Authenticate
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
Vendored
+2
@@ -67,6 +67,8 @@ github.com/skip2/go-qrcode/reedsolomon
|
||||
golang.org/x/crypto/curve25519
|
||||
# golang.org/x/net v0.27.0
|
||||
## explicit; go 1.18
|
||||
golang.org/x/net/internal/socks
|
||||
golang.org/x/net/proxy
|
||||
golang.org/x/net/publicsuffix
|
||||
# golang.org/x/sys v0.30.0
|
||||
## explicit; go 1.18
|
||||
|
||||
+193
-13
@@ -70,6 +70,7 @@ Options via environment:
|
||||
CONFIGURE_UFW=$CONFIGURE_UFW
|
||||
SELF_UPDATE=$SELF_UPDATE
|
||||
FORCE_UPDATE=$FORCE_UPDATE
|
||||
UNINSTALL_DELETE_FILES=0 Set to 1 for non-interactive uninstall to delete $INSTALL_PATH
|
||||
INSTALL_SCRIPT_REF=$INSTALL_SCRIPT_REF
|
||||
INSTALL_SCRIPT_URL=$INSTALL_SCRIPT_URL
|
||||
SERVICE_READY_TIMEOUT=$SERVICE_READY_TIMEOUT
|
||||
@@ -323,8 +324,8 @@ exec_latest_manager_update() {
|
||||
}
|
||||
|
||||
open_firewall_port() {
|
||||
[[ "$CONFIGURE_UFW" == "1" ]] || return
|
||||
command -v ufw >/dev/null 2>&1 || return
|
||||
[[ "$CONFIGURE_UFW" == "1" ]] || return 0
|
||||
command -v ufw >/dev/null 2>&1 || return 0
|
||||
if ufw status 2>/dev/null | grep -qi "Status: active"; then
|
||||
log "allowing ${FRONTEND_PORT}/tcp in UFW"
|
||||
ufw allow "${FRONTEND_PORT}/tcp"
|
||||
@@ -345,6 +346,171 @@ listen_port_from_config() {
|
||||
printf '%s' "$FRONTEND_PORT"
|
||||
}
|
||||
|
||||
append_unique() {
|
||||
local value="$1"
|
||||
shift
|
||||
for existing in "$@"; do
|
||||
[[ "$existing" == "$value" ]] && return 1
|
||||
done
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
app_service_names() {
|
||||
local names=()
|
||||
local name
|
||||
for name in "$SERVICE_NAME" "$APP_NAME" video-site-91 video-site-backend video-site-frontend; do
|
||||
[[ -n "$name" ]] || continue
|
||||
if append_unique "$name" "${names[@]}" >/dev/null; then
|
||||
names+=("$name")
|
||||
fi
|
||||
done
|
||||
printf '%s\n' "${names[@]}"
|
||||
}
|
||||
|
||||
stop_app_services() {
|
||||
local name unit
|
||||
while IFS= read -r name; do
|
||||
[[ -n "$name" ]] || continue
|
||||
unit="${name}.service"
|
||||
systemctl disable --now "$unit" 2>/dev/null || systemctl stop "$unit" 2>/dev/null || true
|
||||
rm -f "/etc/systemd/system/$unit"
|
||||
done < <(app_service_names)
|
||||
systemctl daemon-reload
|
||||
}
|
||||
|
||||
remove_app_containers() {
|
||||
command -v docker >/dev/null 2>&1 || return 0
|
||||
|
||||
local names=()
|
||||
local name
|
||||
for name in "$SERVICE_NAME" "$APP_NAME" video-site-91; do
|
||||
[[ -n "$name" ]] || continue
|
||||
if append_unique "$name" "${names[@]}" >/dev/null; then
|
||||
names+=("$name")
|
||||
fi
|
||||
done
|
||||
|
||||
for name in "${names[@]}"; do
|
||||
if docker ps -a --format '{{.Names}}' 2>/dev/null | grep -Fxq "$name"; then
|
||||
log "removing docker container $name"
|
||||
docker rm -f "$name" >/dev/null 2>&1 || true
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
pids_listening_on_port() {
|
||||
local port="$1"
|
||||
[[ "$port" =~ ^[0-9]+$ ]] || return 0
|
||||
command -v ss >/dev/null 2>&1 || return 0
|
||||
|
||||
ss -ltnp 2>/dev/null \
|
||||
| awk -v port="$port" '$4 ~ ":" port "$" {print}' \
|
||||
| grep -oE 'pid=[0-9]+' \
|
||||
| cut -d= -f2 \
|
||||
| sort -u || true
|
||||
}
|
||||
|
||||
process_looks_like_app() {
|
||||
local pid="$1"
|
||||
local exe="" cmd=""
|
||||
exe="$(readlink "/proc/$pid/exe" 2>/dev/null || true)"
|
||||
cmd="$(tr '\0' ' ' <"/proc/$pid/cmdline" 2>/dev/null || true)"
|
||||
|
||||
[[ "$exe" == "$INSTALL_PATH/server" ]] && return 0
|
||||
[[ "$cmd" == *"$INSTALL_PATH"* ]] && return 0
|
||||
[[ "$cmd" == *"VIDEO_FRONTEND_DIR=$INSTALL_PATH/dist"* ]] && return 0
|
||||
[[ "$cmd" == *"VIDEO_CONFIG=$INSTALL_PATH/config.yaml"* ]] && return 0
|
||||
[[ "$cmd" == *"video-site-91"* ]] && return 0
|
||||
[[ "$cmd" == *"91VideoSpider"* ]] && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
stop_lingering_app_processes() {
|
||||
local ports=("$@")
|
||||
local port pid pids=()
|
||||
|
||||
for port in "${ports[@]}"; do
|
||||
[[ "$port" =~ ^[0-9]+$ ]] || continue
|
||||
while IFS= read -r pid; do
|
||||
[[ -n "$pid" ]] || continue
|
||||
process_looks_like_app "$pid" || continue
|
||||
if append_unique "$pid" "${pids[@]}" >/dev/null; then
|
||||
pids+=("$pid")
|
||||
fi
|
||||
done < <(pids_listening_on_port "$port")
|
||||
done
|
||||
|
||||
if (( ${#pids[@]} == 0 )); then
|
||||
return
|
||||
fi
|
||||
|
||||
warn "stopping lingering app process(es): ${pids[*]}"
|
||||
kill "${pids[@]}" 2>/dev/null || true
|
||||
sleep 1
|
||||
|
||||
local alive=()
|
||||
for pid in "${pids[@]}"; do
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
alive+=("$pid")
|
||||
fi
|
||||
done
|
||||
if (( ${#alive[@]} > 0 )); then
|
||||
warn "force killing lingering app process(es): ${alive[*]}"
|
||||
kill -9 "${alive[@]}" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
warn_remaining_listeners() {
|
||||
local ports=("$@")
|
||||
local port pid cmd
|
||||
for port in "${ports[@]}"; do
|
||||
[[ "$port" =~ ^[0-9]+$ ]] || continue
|
||||
while IFS= read -r pid; do
|
||||
[[ -n "$pid" ]] || continue
|
||||
cmd="$(tr '\0' ' ' <"/proc/$pid/cmdline" 2>/dev/null || true)"
|
||||
warn "port $port is still listening after uninstall: pid=$pid ${cmd:-unknown}"
|
||||
done < <(pids_listening_on_port "$port")
|
||||
done
|
||||
}
|
||||
|
||||
has_interactive_tty() {
|
||||
[[ -t 0 ]]
|
||||
}
|
||||
|
||||
confirm_uninstall_app() {
|
||||
if ! has_interactive_tty; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local confirm=""
|
||||
printf '确认卸载 91 吗?这会停止服务、移除管理命令,并可选择是否删除项目文件。[y/N]: ' >/dev/tty
|
||||
IFS= read -r confirm </dev/tty || confirm=""
|
||||
case "$confirm" in
|
||||
[yY]) return 0 ;;
|
||||
*)
|
||||
log "uninstall cancelled"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
delete_install_path_requested() {
|
||||
if [[ "${UNINSTALL_DELETE_FILES:-0}" == "1" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if ! has_interactive_tty; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
local confirm=""
|
||||
printf '删除 %s 里的程序、配置和数据吗?[y/N]: ' "$INSTALL_PATH" >/dev/tty
|
||||
IFS= read -r confirm </dev/tty || confirm=""
|
||||
case "$confirm" in
|
||||
[yY]) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
service_health_url() {
|
||||
printf 'http://127.0.0.1:%s/admin/api/setup' "$(listen_port_from_config)"
|
||||
}
|
||||
@@ -557,20 +723,30 @@ update_app() {
|
||||
}
|
||||
|
||||
uninstall_app() {
|
||||
systemctl disable --now "${SERVICE_NAME}.service" 2>/dev/null || true
|
||||
rm -f "/etc/systemd/system/${SERVICE_NAME}.service"
|
||||
systemctl daemon-reload
|
||||
local listen_port port ports=()
|
||||
confirm_uninstall_app || return 1
|
||||
|
||||
listen_port="$(listen_port_from_config)"
|
||||
for port in "$listen_port" "$FRONTEND_PORT" 9191 9192; do
|
||||
[[ "$port" =~ ^[0-9]+$ ]] || continue
|
||||
if append_unique "$port" "${ports[@]}" >/dev/null; then
|
||||
ports+=("$port")
|
||||
fi
|
||||
done
|
||||
|
||||
stop_app_services
|
||||
remove_app_containers
|
||||
stop_lingering_app_processes "${ports[@]}"
|
||||
rm -f "$COMMAND_LINK" "$APP_COMMAND_LINK" "$MANAGER_PATH"
|
||||
|
||||
if [[ -t 0 ]]; then
|
||||
read -r -p "删除 $INSTALL_PATH 里的程序、配置和数据吗?[y/N]: " confirm
|
||||
case "$confirm" in
|
||||
[yY]) rm -rf "$INSTALL_PATH" ;;
|
||||
*) log "kept $INSTALL_PATH" ;;
|
||||
esac
|
||||
if delete_install_path_requested; then
|
||||
rm -rf "$INSTALL_PATH"
|
||||
log "removed $INSTALL_PATH"
|
||||
else
|
||||
log "removed service; kept $INSTALL_PATH"
|
||||
log "kept $INSTALL_PATH"
|
||||
fi
|
||||
|
||||
warn_remaining_listeners "${ports[@]}"
|
||||
}
|
||||
|
||||
show_menu() {
|
||||
@@ -600,7 +776,11 @@ show_menu() {
|
||||
3) main update ;;
|
||||
4) main restart ;;
|
||||
5) main stop ;;
|
||||
6) main uninstall ;;
|
||||
6)
|
||||
if main uninstall; then
|
||||
exit 0
|
||||
fi
|
||||
;;
|
||||
0) exit 0 ;;
|
||||
*) echo "无效的选项" ;;
|
||||
esac
|
||||
|
||||
@@ -110,7 +110,7 @@ export function AdminLayout() {
|
||||
{checkingUpdate ? "检查中" : "检查更新"}
|
||||
</button>
|
||||
<button className="admin-sidebar__logout" onClick={handleLogout}>
|
||||
<LogOut size={14} style={{ verticalAlign: -2, marginRight: 4 }} />
|
||||
<LogOut size={14} />
|
||||
退出登录
|
||||
</button>
|
||||
</div>
|
||||
|
||||
+144
-60
@@ -26,6 +26,7 @@ const kindLabel: Record<string, string> = {
|
||||
pikpak: "PikPak",
|
||||
wopan: "联通沃盘",
|
||||
onedrive: "OneDrive",
|
||||
googledrive: "Google Drive",
|
||||
localstorage: "本地存储",
|
||||
spider91: "91 爬虫",
|
||||
};
|
||||
@@ -41,7 +42,6 @@ type FormState = {
|
||||
kind: Kind;
|
||||
name: string;
|
||||
rootId: string;
|
||||
scanRootId: string;
|
||||
creds: Record<string, string>;
|
||||
/**
|
||||
* spider91 专用字段:把视频迁移到云盘的目标 drive ID。
|
||||
@@ -58,28 +58,50 @@ const emptyForm: FormState = {
|
||||
id: "",
|
||||
kind: "p115",
|
||||
name: "",
|
||||
rootId: "0",
|
||||
scanRootId: "0",
|
||||
rootId: "",
|
||||
creds: {},
|
||||
spider91UploadDriveId: "",
|
||||
};
|
||||
|
||||
const idleNightlyStatus: api.NightlyJobStatus = {
|
||||
state: "idle",
|
||||
running: false,
|
||||
queued: false,
|
||||
};
|
||||
|
||||
function nightlyButtonText(status: api.NightlyJobStatus, triggering: boolean) {
|
||||
if (triggering) return "触发中...";
|
||||
if (status.running) return "扫描运行中";
|
||||
if (status.queued) return "扫描已排队";
|
||||
return "扫描所有网盘";
|
||||
}
|
||||
|
||||
function nightlyBusyText(status: api.NightlyJobStatus) {
|
||||
if (status.running) return "扫描任务正在运行";
|
||||
if (status.queued) return "扫描任务已排队";
|
||||
return "";
|
||||
}
|
||||
|
||||
export function DrivesPage() {
|
||||
const [list, setList] = useState<api.AdminDrive[]>([]);
|
||||
const [storage, setStorage] = useState<api.AdminDriveStorage | null>(null);
|
||||
const [settings, setSettings] = useState<api.Settings | null>(null);
|
||||
const [nightlyStatus, setNightlyStatus] =
|
||||
useState<api.NightlyJobStatus>(idleNightlyStatus);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [modalOpen, setModalOpen] = useState(false);
|
||||
const [form, setForm] = useState<FormState>(emptyForm);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [regenFailedId, setRegenFailedId] = useState("");
|
||||
// 与 regenFailedId 并列:失败封面重新入队按钮的 disable 状态。两套独立按钮 →
|
||||
// 两个 state 互不阻塞,避免操作 teaser 时锁住封面那条按钮(反之亦然)。
|
||||
// 失败重试按钮各自维护 pending 状态,避免操作 teaser / 封面 / 指纹时互相锁住。
|
||||
const [regenFailedThumbId, setRegenFailedThumbId] = useState("");
|
||||
const [regenFailedFingerprintId, setRegenFailedFingerprintId] = useState("");
|
||||
// togglingTeaserId 在请求未返回前禁用按钮,避免连点导致两次切换互相覆盖。
|
||||
const [togglingTeaserId, setTogglingTeaserId] = useState("");
|
||||
const [scanningAll, setScanningAll] = useState(false);
|
||||
const [selectedDriveId, setSelectedDriveId] = useState<string | null>(null);
|
||||
const { show } = useToast();
|
||||
const nightlyBusy = scanningAll || nightlyStatus.running || nightlyStatus.queued;
|
||||
|
||||
// 当前系统中可作为 spider91 上传目标的 drive 列表(pikpak ∪ p115 ∪ onedrive)。
|
||||
// 用户保存 spider91 drive 时从这里挑一个;空表示本地保存不上传。
|
||||
@@ -91,14 +113,16 @@ export function DrivesPage() {
|
||||
async function refresh() {
|
||||
setLoading(true);
|
||||
try {
|
||||
const [data, storageData, settingsData] = await Promise.all([
|
||||
const [data, storageData, settingsData, jobStatus] = await Promise.all([
|
||||
api.listDrives(),
|
||||
api.getDriveStorage(),
|
||||
api.getSettings().catch(() => null),
|
||||
api.getNightlyJobStatus().catch(() => null),
|
||||
]);
|
||||
setList(data ?? []);
|
||||
setStorage(storageData);
|
||||
if (settingsData) setSettings(settingsData);
|
||||
if (jobStatus) setNightlyStatus(jobStatus);
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "加载失败", "error");
|
||||
} finally {
|
||||
@@ -108,8 +132,12 @@ export function DrivesPage() {
|
||||
|
||||
async function refreshDriveList() {
|
||||
try {
|
||||
const data = await api.listDrives();
|
||||
const [data, jobStatus] = await Promise.all([
|
||||
api.listDrives(),
|
||||
api.getNightlyJobStatus().catch(() => null),
|
||||
]);
|
||||
setList(data ?? []);
|
||||
if (jobStatus) setNightlyStatus(jobStatus);
|
||||
} catch {
|
||||
// 保持当前页面状态,下一次轮询或手动操作再刷新。
|
||||
}
|
||||
@@ -141,8 +169,7 @@ export function DrivesPage() {
|
||||
kind: d.kind,
|
||||
name: d.name,
|
||||
rootId: d.rootId,
|
||||
scanRootId: d.scanRootId || d.rootId,
|
||||
creds: {},
|
||||
creds: d.kind === "spider91" ? { proxy: d.spider91Proxy ?? "" } : {},
|
||||
spider91UploadDriveId: settings?.spider91UploadDriveId ?? "",
|
||||
});
|
||||
setModalOpen(true);
|
||||
@@ -158,6 +185,9 @@ export function DrivesPage() {
|
||||
const driveID = existing
|
||||
? form.id
|
||||
: makeUniqueDriveId(form.kind, name, list);
|
||||
const rootId = usesRootDirectoryID(form.kind)
|
||||
? form.rootId.trim() || defaultRootId(form.kind)
|
||||
: defaultRootId(form.kind);
|
||||
// 若编辑且没有提供凭证,提示一下但仍允许保存(不改凭证)
|
||||
setSaving(true);
|
||||
try {
|
||||
@@ -165,8 +195,7 @@ export function DrivesPage() {
|
||||
id: driveID,
|
||||
kind: form.kind,
|
||||
name,
|
||||
rootId: form.rootId || defaultRootId(form.kind),
|
||||
scanRootId: form.scanRootId || form.rootId || defaultRootId(form.kind),
|
||||
rootId,
|
||||
credentials: form.creds,
|
||||
});
|
||||
|
||||
@@ -233,14 +262,26 @@ export function DrivesPage() {
|
||||
/**
|
||||
* 立即触发完整凌晨流水线(Phase1 扫所有云盘 → Phase2 spider91 爬虫 →
|
||||
* Phase3 spider91 → 云盘迁移)。后端立即返回 202;进度看 backend 日志。
|
||||
* 如果当前已有流水线在跑,后端最多保留一个待触发请求,当前轮结束后再跑一轮。
|
||||
* 如果当前已有流水线在跑或已排队,前端只提示,不再提交新任务。
|
||||
*/
|
||||
async function handleRunNightly() {
|
||||
if (nightlyBusy) {
|
||||
show(nightlyBusyText(nightlyStatus) || "当前已有扫描所有网盘任务", "info");
|
||||
return;
|
||||
}
|
||||
setScanningAll(true);
|
||||
try {
|
||||
await api.runNightlyJob();
|
||||
show("已触发扫描所有网盘,耗时较长,可在 backend 日志观察进度", "success");
|
||||
const resp = await api.runNightlyJob();
|
||||
setNightlyStatus(resp.status);
|
||||
if (resp.accepted) {
|
||||
show("已触发扫描所有网盘,耗时较长,可在任务状态和 backend 日志观察进度", "success");
|
||||
} else {
|
||||
show("当前已有扫描所有网盘任务", "info");
|
||||
}
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "触发失败", "error");
|
||||
} finally {
|
||||
setScanningAll(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -271,6 +312,19 @@ export function DrivesPage() {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleRegenFailedFingerprints(d: api.AdminDrive) {
|
||||
setRegenFailedFingerprintId(d.id);
|
||||
try {
|
||||
await api.regenFailedFingerprints(d.id);
|
||||
show("已触发失败指纹重新生成", "success");
|
||||
refresh();
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "触发失败", "error");
|
||||
} finally {
|
||||
setRegenFailedFingerprintId("");
|
||||
}
|
||||
}
|
||||
|
||||
async function handleToggleTeaser(d: api.AdminDrive) {
|
||||
const next = !d.teaserEnabled;
|
||||
setTogglingTeaserId(d.id);
|
||||
@@ -356,16 +410,12 @@ export function DrivesPage() {
|
||||
<span className="admin-detail-label">网盘 ID</span>
|
||||
<span className="admin-detail-value admin-mono-cell">{d.id}</span>
|
||||
</div>
|
||||
{d.kind !== "spider91" && (
|
||||
{usesRootDirectoryID(d.kind) && (
|
||||
<>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">根目录 ID</span>
|
||||
<span className="admin-detail-value admin-mono-cell">{d.rootId}</span>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">扫描起点 ID</span>
|
||||
<span className="admin-detail-value admin-mono-cell">{d.scanRootId || d.rootId}</span>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
{d.kind === "spider91" && (
|
||||
@@ -444,14 +494,14 @@ export function DrivesPage() {
|
||||
style={{ padding: "4px 10px", fontSize: "11px" }}
|
||||
>
|
||||
{d.teaserEnabled ? <Power size={11} /> : <PowerOff size={11} />}
|
||||
<span>{d.teaserEnabled ? "Teaser: 开" : "Teaser: 关"}</span>
|
||||
<span>{d.teaserEnabled ? "预览视频生成:开" : "预览视频生成:关"}</span>
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div className="admin-detail-grid">
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">封面状态</span>
|
||||
<span className="admin-detail-label">封面生成状态</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationStatusLine label="封面" status={d.thumbnailGenerationStatus} />
|
||||
</div>
|
||||
@@ -463,17 +513,18 @@ export function DrivesPage() {
|
||||
ready={d.thumbnailReadyCount}
|
||||
pending={d.thumbnailPendingCount}
|
||||
failed={d.thumbnailFailedCount}
|
||||
durationPending={d.thumbnailDurationPendingCount}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">Teaser 状态</span>
|
||||
<span className="admin-detail-label">预览视频生成状态</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationStatusLine label="预览" status={d.previewGenerationStatus} />
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">Teaser 数量</span>
|
||||
<span className="admin-detail-label">预览视频数量</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationCounts
|
||||
ready={d.teaserReadyCount}
|
||||
@@ -483,13 +534,13 @@ export function DrivesPage() {
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">指纹状态</span>
|
||||
<span className="admin-detail-label">视频指纹生成状态</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationStatusLine label="指纹" status={d.fingerprintGenerationStatus} />
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">指纹数量</span>
|
||||
<span className="admin-detail-label">视频指纹数量</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationCounts
|
||||
ready={d.fingerprintReadyCount}
|
||||
@@ -517,6 +568,17 @@ export function DrivesPage() {
|
||||
<RotateCcw size={13} />
|
||||
<span>重试失败封面</span>
|
||||
</button>
|
||||
<button
|
||||
className="admin-btn"
|
||||
disabled={
|
||||
(d.fingerprintFailedCount ?? 0) <= 0 ||
|
||||
regenFailedFingerprintId === d.id
|
||||
}
|
||||
onClick={() => handleRegenFailedFingerprints(d)}
|
||||
>
|
||||
<RotateCcw size={13} />
|
||||
<span>重试失败指纹</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -534,7 +596,7 @@ export function DrivesPage() {
|
||||
<span className="admin-detail-value">{formatBytes(driveStorage?.thumbnailBytes ?? 0)}</span>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">Teaser 占用</span>
|
||||
<span className="admin-detail-label">预览视频占用</span>
|
||||
<span className="admin-detail-value">{formatBytes(driveStorage?.teaserBytes ?? 0)}</span>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
@@ -587,9 +649,10 @@ export function DrivesPage() {
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={handleRunNightly}
|
||||
title="立即扫描所有网盘。耗时较长,期间不要重复触发。"
|
||||
disabled={scanningAll}
|
||||
title={nightlyBusyText(nightlyStatus) || "立即扫描所有网盘。耗时较长,期间不要重复触发。"}
|
||||
>
|
||||
<PlayCircle size={14} /> 扫描所有网盘
|
||||
<PlayCircle size={14} /> {nightlyButtonText(nightlyStatus, scanningAll)}
|
||||
</button>
|
||||
<button className="admin-btn is-primary" onClick={openCreate}>
|
||||
<Plus size={14} /> 新建网盘
|
||||
@@ -702,7 +765,7 @@ function StorageSummary({ storage }: { storage: api.AdminDriveStorage }) {
|
||||
<strong>{formatBytes(storage.thumbnailBytes)}</strong>
|
||||
</div>
|
||||
<div className="admin-storage-summary__metric">
|
||||
<span>Teaser 占用</span>
|
||||
<span>预览视频占用</span>
|
||||
<strong>{formatBytes(storage.teaserBytes)}</strong>
|
||||
</div>
|
||||
<div className="admin-storage-summary__metric">
|
||||
@@ -721,10 +784,12 @@ function GenerationCounts({
|
||||
ready,
|
||||
pending,
|
||||
failed,
|
||||
durationPending,
|
||||
}: {
|
||||
ready?: number;
|
||||
pending?: number;
|
||||
failed?: number;
|
||||
durationPending?: number;
|
||||
}) {
|
||||
return (
|
||||
<div className="admin-generation-counts">
|
||||
@@ -737,6 +802,11 @@ function GenerationCounts({
|
||||
<span className="admin-drive-teaser__metric is-failed">
|
||||
失败 {failed ?? 0}
|
||||
</span>
|
||||
{(durationPending ?? 0) > 0 && (
|
||||
<span className="admin-drive-teaser__metric">
|
||||
待补时长 {durationPending}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -752,7 +822,7 @@ function GenerationStatusLine({
|
||||
const queueLength = status?.queueLength ?? 0;
|
||||
const detail = generationDetail(status);
|
||||
const title = generationTitle(status, detail);
|
||||
const countText = queueLength > 0 ? `${label === "封面" ? "剩余" : "队列"} ${queueLength}` : "";
|
||||
const countText = queueLength > 0 ? `${label === "封面" ? "待处理" : "队列"} ${queueLength}` : "";
|
||||
|
||||
return (
|
||||
<div className="admin-generation-row" title={title}>
|
||||
@@ -863,11 +933,6 @@ function DriveForm({
|
||||
}) {
|
||||
const fields = useMemo(() => credentialFields(form.kind), [form.kind]);
|
||||
const help = credentialHelp(form.kind, isEdit);
|
||||
const showDirectoryFields =
|
||||
form.kind !== "spider91" &&
|
||||
form.kind !== "onedrive" &&
|
||||
form.kind !== "localstorage" &&
|
||||
form.kind !== "pikpak";
|
||||
|
||||
function set<K extends keyof FormState>(k: K, v: FormState[K]) {
|
||||
onChange({ ...form, [k]: v });
|
||||
@@ -879,8 +944,7 @@ function DriveForm({
|
||||
onChange({
|
||||
...form,
|
||||
kind: v,
|
||||
rootId: defaultRootId(v),
|
||||
scanRootId: defaultRootId(v),
|
||||
rootId: "",
|
||||
creds: {},
|
||||
});
|
||||
}
|
||||
@@ -905,34 +969,25 @@ function DriveForm({
|
||||
<option value="p115">115 网盘</option>
|
||||
<option value="pikpak">PikPak</option>
|
||||
<option value="onedrive">OneDrive</option>
|
||||
<option value="googledrive">Google Drive</option>
|
||||
<option value="localstorage">本地存储</option>
|
||||
<option value="spider91">91 Spider</option>
|
||||
<option value="quark">夸克网盘</option>
|
||||
<option value="wopan">联通沃盘</option>
|
||||
</select>
|
||||
</div>
|
||||
{showDirectoryFields && (
|
||||
<>
|
||||
<div className="admin-form__row">
|
||||
<label>根目录 ID</label>
|
||||
<input
|
||||
value={form.rootId}
|
||||
onChange={(e) => set("rootId", e.target.value)}
|
||||
placeholder={form.kind === "pikpak" ? "留空表示根目录" : form.kind === "onedrive" ? "root" : "0"}
|
||||
/>
|
||||
{usesRootDirectoryID(form.kind) && (
|
||||
<div className="admin-form__row">
|
||||
<label>根目录 ID</label>
|
||||
<input
|
||||
value={form.rootId}
|
||||
onChange={(e) => set("rootId", e.target.value)}
|
||||
placeholder={rootIdPlaceholder(form.kind)}
|
||||
/>
|
||||
<div className="admin-form__help">
|
||||
留空时使用该网盘类型的默认根目录,具体目录ID获取方式请参考OpenList文档
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>扫描起点目录 ID</label>
|
||||
<input
|
||||
value={form.scanRootId}
|
||||
onChange={(e) => set("scanRootId", e.target.value)}
|
||||
placeholder="留空则使用根目录"
|
||||
/>
|
||||
<div className="admin-form__help">
|
||||
可以指定一个子目录作为视频库入口,避免扫描整个网盘
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{(help || fields.length > 0) && (
|
||||
@@ -1031,10 +1086,12 @@ function credentialHelp(kind: Kind, isEdit: boolean): string {
|
||||
return `需要 access_token 和 refresh_token。后续会加扫码/短信登录入口,第一版只能手工粘贴。${note}`;
|
||||
case "onedrive":
|
||||
return `按 OpenList 默认应用在线挂载,只需要 refresh_token;保存时会自动刷新并保存 token。${note}`;
|
||||
case "googledrive":
|
||||
return `按 OpenList 在线 API 挂载,只需要 Google Drive refresh_token;保存时会自动刷新并保存 token。播放不走 302,会由后端带 Authorization 代理转发。${note}`;
|
||||
case "localstorage":
|
||||
return `把服务器上的一个已有目录作为视频来源扫描。填写绝对路径,例如 /mnt/videos;系统会读取该目录及子目录中的视频,并生成封面、Teaser 和指纹。${note}`;
|
||||
case "spider91":
|
||||
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;它不是外部网盘,不需要填写 Cookie 或目录 ID。后续流水线会把较早的视频上传到你选择的 115 / PikPak / OneDrive 目标盘。";
|
||||
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;可按服务器网络情况单独配置代理。后续流水线会把较早的视频上传到你选择的 115 / PikPak / OneDrive 目标盘。";
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
@@ -1114,6 +1171,16 @@ function credentialFields(kind: Kind): Array<{
|
||||
required: true,
|
||||
},
|
||||
];
|
||||
case "googledrive":
|
||||
return [
|
||||
{
|
||||
key: "refresh_token",
|
||||
label: "refresh_token",
|
||||
placeholder: "OpenList Google Drive refresh_token",
|
||||
multiline: true,
|
||||
required: true,
|
||||
},
|
||||
];
|
||||
case "localstorage":
|
||||
return [
|
||||
{
|
||||
@@ -1125,18 +1192,35 @@ function credentialFields(kind: Kind): Array<{
|
||||
},
|
||||
];
|
||||
case "spider91":
|
||||
return [];
|
||||
return [
|
||||
{
|
||||
key: "proxy",
|
||||
label: "代理地址(可选)",
|
||||
placeholder: "http://127.0.0.1:7890",
|
||||
help: "仅用于 91Spider 的列表/详情请求和视频、封面下载;留空则使用服务器环境变量 HTTP_PROXY / HTTPS_PROXY 或直连。支持 http://、https://、socks5:// 或 socks5h://。",
|
||||
},
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
function defaultRootId(kind: Kind): string {
|
||||
if (kind === "pikpak") return "";
|
||||
if (kind === "onedrive") return "root";
|
||||
if (kind === "googledrive") return "root";
|
||||
if (kind === "localstorage") return "/";
|
||||
if (kind === "spider91") return "/";
|
||||
return "0";
|
||||
}
|
||||
|
||||
function usesRootDirectoryID(kind: Kind): boolean {
|
||||
return kind !== "localstorage" && kind !== "spider91";
|
||||
}
|
||||
|
||||
function rootIdPlaceholder(kind: Kind): string {
|
||||
const rootId = defaultRootId(kind);
|
||||
return rootId ? `默认:${rootId}` : "留空表示根目录";
|
||||
}
|
||||
|
||||
|
||||
// ---------- SkipDirsModal ----------
|
||||
//
|
||||
|
||||
@@ -82,11 +82,6 @@ export function LoginPage() {
|
||||
<Play size={18} fill="currentColor" /> {setupRequired ? "首次设置管理员" : "登录"}
|
||||
</h1>
|
||||
<div className="admin-form">
|
||||
{setupRequired && (
|
||||
<div className="admin-form__help admin-form__help--lead">
|
||||
请先设置后台管理员账号。保存后会写入本机配置文件,之后使用该账号登录。
|
||||
</div>
|
||||
)}
|
||||
<div className="admin-form__row">
|
||||
<label>用户名</label>
|
||||
<input
|
||||
|
||||
+268
-37
@@ -1,16 +1,26 @@
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { Film, Plus, RefreshCw, Search, Tags } from "lucide-react";
|
||||
import { CheckSquare, Film, Plus, RefreshCw, Search, Tags, Trash2 } from "lucide-react";
|
||||
import * as api from "./api";
|
||||
import { useToast } from "./ToastContext";
|
||||
|
||||
const DESKTOP_TAGS_PAGE_SIZE = 25;
|
||||
const MOBILE_TAGS_PAGE_SIZE = 8;
|
||||
const TAGS_MOBILE_QUERY = "(max-width: 640px)";
|
||||
|
||||
export function TagsPage() {
|
||||
const [tags, setTags] = useState<api.AdminTag[]>([]);
|
||||
const [label, setLabel] = useState("");
|
||||
const [aliases, setAliases] = useState("");
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [deletingId, setDeletingId] = useState<number | null>(null);
|
||||
const [searchQuery, setSearchQuery] = useState("");
|
||||
const [filterSource, setFilterSource] = useState<string>("all");
|
||||
const [selectMode, setSelectMode] = useState(false);
|
||||
const [selected, setSelected] = useState<Set<number>>(new Set());
|
||||
const [bulkDeleting, setBulkDeleting] = useState(false);
|
||||
const pageSize = useTagsPageSize();
|
||||
const [page, setPage] = useState(1);
|
||||
const { show } = useToast();
|
||||
|
||||
async function refresh() {
|
||||
@@ -45,6 +55,63 @@ export function TagsPage() {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleDelete(tag: api.AdminTag) {
|
||||
if (tag.source === "system") return;
|
||||
if (!window.confirm(`确定删除标签「${tag.label}」吗?此操作会从所有视频上移除该标签。`)) {
|
||||
return;
|
||||
}
|
||||
setDeletingId(tag.id);
|
||||
try {
|
||||
const r = await api.deleteTag(tag.id);
|
||||
show(`已删除标签,并从 ${r.removedVideos} 个视频移除`, "success");
|
||||
await refresh();
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "删除标签失败", "error");
|
||||
} finally {
|
||||
setDeletingId(null);
|
||||
}
|
||||
}
|
||||
|
||||
function toggleSelectMode() {
|
||||
setSelectMode((m) => !m);
|
||||
setSelected(new Set());
|
||||
}
|
||||
|
||||
function toggleSelect(id: number) {
|
||||
setSelected((prev) => {
|
||||
const next = new Set(prev);
|
||||
next.has(id) ? next.delete(id) : next.add(id);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
async function handleBulkDelete() {
|
||||
const ids = [...selected];
|
||||
if (ids.length === 0) return;
|
||||
if (!window.confirm(`确定删除选中的 ${ids.length} 个标签吗?此操作会从所有视频上移除这些标签。`)) {
|
||||
return;
|
||||
}
|
||||
setBulkDeleting(true);
|
||||
try {
|
||||
let ok = 0;
|
||||
for (const id of ids) {
|
||||
try {
|
||||
await api.deleteTag(id);
|
||||
ok += 1;
|
||||
} catch {
|
||||
/* 统计失败数,继续删除其余标签 */
|
||||
}
|
||||
}
|
||||
const failed = ids.length - ok;
|
||||
show(failed ? `已删除 ${ok} 个,${failed} 个失败` : `已删除 ${ok} 个标签`, failed ? "error" : "success");
|
||||
setSelected(new Set());
|
||||
setSelectMode(false);
|
||||
await refresh();
|
||||
} finally {
|
||||
setBulkDeleting(false);
|
||||
}
|
||||
}
|
||||
|
||||
const stats = useMemo(() => {
|
||||
let totalVideos = 0;
|
||||
let systemCount = 0;
|
||||
@@ -82,6 +149,41 @@ export function TagsPage() {
|
||||
});
|
||||
}, [tags, searchQuery, filterSource]);
|
||||
|
||||
const totalPages = Math.max(1, Math.ceil(filteredTags.length / pageSize));
|
||||
const currentPage = Math.min(page, totalPages);
|
||||
const pageStartIndex = (currentPage - 1) * pageSize;
|
||||
const pageEndIndex = pageStartIndex + pageSize;
|
||||
const pagedTags = useMemo(
|
||||
() => filteredTags.slice(pageStartIndex, pageEndIndex),
|
||||
[filteredTags, pageStartIndex, pageEndIndex]
|
||||
);
|
||||
const pageStart = filteredTags.length === 0 ? 0 : pageStartIndex + 1;
|
||||
const pageEnd = Math.min(filteredTags.length, pageEndIndex);
|
||||
|
||||
useEffect(() => {
|
||||
setPage(1);
|
||||
}, [searchQuery, filterSource, pageSize]);
|
||||
|
||||
useEffect(() => {
|
||||
setPage((p) => Math.min(Math.max(1, p), totalPages));
|
||||
}, [totalPages]);
|
||||
|
||||
const deletablePageTags = useMemo(
|
||||
() => pagedTags.filter((t) => t.source !== "system"),
|
||||
[pagedTags]
|
||||
);
|
||||
const allSelected =
|
||||
deletablePageTags.length > 0 && deletablePageTags.every((t) => selected.has(t.id));
|
||||
|
||||
function toggleSelectAll() {
|
||||
setSelected((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (allSelected) deletablePageTags.forEach((t) => next.delete(t.id));
|
||||
else deletablePageTags.forEach((t) => next.add(t.id));
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<section>
|
||||
<header className="admin-page__header">
|
||||
@@ -130,7 +232,7 @@ export function TagsPage() {
|
||||
|
||||
<div className="admin-card">
|
||||
<div className="admin-card__title">
|
||||
<Tags size={15} /> 系统标签统计
|
||||
<Tags size={15} /> 标签总览
|
||||
</div>
|
||||
<div className="admin-tag-stats-list">
|
||||
<div className="admin-tag-stat-item">
|
||||
@@ -141,14 +243,6 @@ export function TagsPage() {
|
||||
<span>关联视频次</span>
|
||||
<strong>{stats.totalVideos}</strong>
|
||||
</div>
|
||||
<div className="admin-tag-stat-item">
|
||||
<span>系统提取</span>
|
||||
<strong>{stats.systemCount}</strong>
|
||||
</div>
|
||||
<div className="admin-tag-stat-item">
|
||||
<span>用户创建</span>
|
||||
<strong>{stats.userCount}</strong>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -195,9 +289,52 @@ export function TagsPage() {
|
||||
>
|
||||
合集 ({stats.collectionCount})
|
||||
</button>
|
||||
{stats.legacyCount > 0 && (
|
||||
<button
|
||||
type="button"
|
||||
className={`admin-tags-filter-tab ${filterSource === "legacy" ? "is-active" : ""}`}
|
||||
onClick={() => setFilterSource("legacy")}
|
||||
>
|
||||
旧数据 ({stats.legacyCount})
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<button
|
||||
type="button"
|
||||
className={`admin-btn ${selectMode ? "is-primary" : ""}`}
|
||||
onClick={toggleSelectMode}
|
||||
>
|
||||
<CheckSquare size={13} /> {selectMode ? "退出批量" : "批量删除"}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{selectMode && (
|
||||
<div className="admin-tags-bulkbar">
|
||||
<label className="admin-check">
|
||||
<input type="checkbox" checked={allSelected} onChange={toggleSelectAll} />
|
||||
<span>全选本页 ({deletablePageTags.length})</span>
|
||||
</label>
|
||||
<span className="admin-tags-bulkbar__count">已选 {selected.size} 个</span>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setSelected(new Set())}
|
||||
disabled={selected.size === 0}
|
||||
>
|
||||
清空
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn is-danger"
|
||||
onClick={handleBulkDelete}
|
||||
disabled={selected.size === 0 || bulkDeleting}
|
||||
>
|
||||
<Trash2 size={13} /> {bulkDeleting ? "删除中..." : `删除选中 (${selected.size})`}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{loading ? (
|
||||
<div className="admin-empty">加载中...</div>
|
||||
) : filteredTags.length === 0 ? (
|
||||
@@ -205,36 +342,110 @@ export function TagsPage() {
|
||||
没有找到匹配的标签。
|
||||
</div>
|
||||
) : (
|
||||
<div className="admin-tags-grid">
|
||||
{filteredTags.map((tag) => (
|
||||
<div key={tag.id} className="admin-tag-card">
|
||||
<div className="admin-tag-card__head">
|
||||
<span className="admin-tag-card__title">{tag.label}</span>
|
||||
<span className="admin-tag-card__source-badge" data-source={tag.source}>
|
||||
{sourceLabel(tag.source)}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{tag.aliases && tag.aliases.length > 0 && (
|
||||
<div className="admin-tag-card__aliases">
|
||||
{tag.aliases.map((alias) => (
|
||||
<span key={alias} className="admin-tag-card__alias-pill">
|
||||
{alias}
|
||||
<>
|
||||
<div className="admin-tags-grid">
|
||||
{pagedTags.map((tag) => {
|
||||
const selectable = selectMode && tag.source !== "system";
|
||||
const isSelected = selected.has(tag.id);
|
||||
return (
|
||||
<div
|
||||
key={tag.id}
|
||||
className={`admin-tag-card${selectable ? " is-selectable" : ""}${
|
||||
selectable && isSelected ? " is-selected" : ""
|
||||
}`}
|
||||
onClick={selectable ? () => toggleSelect(tag.id) : undefined}
|
||||
>
|
||||
<div className="admin-tag-card__head">
|
||||
{selectable && (
|
||||
<input
|
||||
type="checkbox"
|
||||
className="admin-tag-card__check"
|
||||
checked={isSelected}
|
||||
readOnly
|
||||
/>
|
||||
)}
|
||||
<span className="admin-tag-card__title">{tag.label}</span>
|
||||
<span className="admin-tag-card__source-badge" data-source={tag.source}>
|
||||
{sourceLabel(tag.source)}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="admin-tag-card__footer">
|
||||
<span>ID: {tag.id}</span>
|
||||
<span className="admin-tag-card__count">
|
||||
<Film size={11} />
|
||||
<strong>{tag.count} 视频</strong>
|
||||
</span>
|
||||
</div>
|
||||
{tag.aliases && tag.aliases.length > 0 && (
|
||||
<div className="admin-tag-card__aliases">
|
||||
{tag.aliases.map((alias) => (
|
||||
<span key={alias} className="admin-tag-card__alias-pill">
|
||||
{alias}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="admin-tag-card__footer">
|
||||
<span className="admin-tag-card__count">
|
||||
<Film size={13} />
|
||||
<strong>{tag.count}</strong> 视频
|
||||
</span>
|
||||
<div className="admin-tag-card__footer-actions">
|
||||
<span className="admin-tag-card__id">#{tag.id}</span>
|
||||
{!selectMode && tag.source !== "system" && (
|
||||
<button
|
||||
type="button"
|
||||
className="admin-tag-card__delete"
|
||||
onClick={() => handleDelete(tag)}
|
||||
disabled={deletingId === tag.id}
|
||||
aria-label={`删除标签 ${tag.label}`}
|
||||
>
|
||||
<Trash2 size={11} />
|
||||
<span>{deletingId === tag.id ? "删除中" : "删除"}</span>
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
{totalPages > 1 && (
|
||||
<div className="admin-table-pagination admin-tags-pagination">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage(1)}
|
||||
disabled={currentPage <= 1}
|
||||
>
|
||||
首页
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage((p) => Math.max(1, p - 1))}
|
||||
disabled={currentPage <= 1}
|
||||
>
|
||||
上一页
|
||||
</button>
|
||||
<span className="admin-table-pagination__info">
|
||||
第 {currentPage} / {totalPages} 页,显示 {pageStart}-{pageEnd} / {filteredTags.length},每页 {pageSize} 个
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage((p) => Math.min(totalPages, p + 1))}
|
||||
disabled={currentPage >= totalPages}
|
||||
>
|
||||
下一页
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage(totalPages)}
|
||||
disabled={currentPage >= totalPages}
|
||||
>
|
||||
末页
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
@@ -242,6 +453,26 @@ export function TagsPage() {
|
||||
);
|
||||
}
|
||||
|
||||
function useTagsPageSize() {
|
||||
const [pageSize, setPageSize] = useState(() =>
|
||||
window.matchMedia(TAGS_MOBILE_QUERY).matches
|
||||
? MOBILE_TAGS_PAGE_SIZE
|
||||
: DESKTOP_TAGS_PAGE_SIZE
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const media = window.matchMedia(TAGS_MOBILE_QUERY);
|
||||
const update = () => {
|
||||
setPageSize(media.matches ? MOBILE_TAGS_PAGE_SIZE : DESKTOP_TAGS_PAGE_SIZE);
|
||||
};
|
||||
update();
|
||||
media.addEventListener("change", update);
|
||||
return () => media.removeEventListener("change", update);
|
||||
}, []);
|
||||
|
||||
return pageSize;
|
||||
}
|
||||
|
||||
function splitList(s: string): string[] {
|
||||
return s
|
||||
.split(/[,,、\s]+/)
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
useCallback,
|
||||
useContext,
|
||||
useEffect,
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
|
||||
@@ -18,13 +19,25 @@ const ToastCtx = createContext<Ctx | null>(null);
|
||||
|
||||
export function ToastProvider({ children }: { children: ReactNode }) {
|
||||
const [items, setItems] = useState<Toast[]>([]);
|
||||
const timers = useRef<Record<string, ReturnType<typeof window.setTimeout>>>({});
|
||||
|
||||
// Deduplicate: same text won't stack, just resets the dismiss timer
|
||||
const show = useCallback((text: string, kind: ToastKind = "info") => {
|
||||
// Reset timer if duplicate
|
||||
if (timers.current[text]) {
|
||||
window.clearTimeout(timers.current[text]);
|
||||
timers.current[text] = window.setTimeout(() => {
|
||||
setItems((list) => list.filter((t) => t.text !== text));
|
||||
delete timers.current[text];
|
||||
}, 2600);
|
||||
return;
|
||||
}
|
||||
const id = Date.now() + Math.random();
|
||||
setItems((list) => [...list, { id, kind, text }]);
|
||||
window.setTimeout(() => {
|
||||
timers.current[text] = window.setTimeout(() => {
|
||||
setItems((list) => list.filter((t) => t.id !== id));
|
||||
delete timers.current[text];
|
||||
}, 2600);
|
||||
setItems((list) => [...list, { id, kind, text }]);
|
||||
}, []);
|
||||
|
||||
return (
|
||||
|
||||
+38
-8
@@ -77,10 +77,9 @@ export function checkUpdate() {
|
||||
|
||||
export type AdminDrive = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "localstorage" | "spider91";
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
name: string;
|
||||
rootId: string;
|
||||
scanRootId: string;
|
||||
status: string;
|
||||
lastError?: string;
|
||||
hasCredential: boolean;
|
||||
@@ -94,12 +93,15 @@ export type AdminDrive = {
|
||||
skipDirIds: string[];
|
||||
// spider91 上次成功爬取时间(unix 秒);其它 kind 留空。
|
||||
lastCrawlAt?: number;
|
||||
// spider91 专用代理地址;仅后台管理接口返回,用于编辑表单回显。
|
||||
spider91Proxy?: string;
|
||||
thumbnailGenerationStatus?: DriveGenerationStatus;
|
||||
previewGenerationStatus?: DriveGenerationStatus;
|
||||
fingerprintGenerationStatus?: DriveGenerationStatus;
|
||||
thumbnailReadyCount: number;
|
||||
thumbnailPendingCount: number;
|
||||
thumbnailFailedCount: number;
|
||||
thumbnailDurationPendingCount: number;
|
||||
teaserReadyCount: number;
|
||||
teaserPendingCount: number;
|
||||
teaserFailedCount: number;
|
||||
@@ -137,10 +139,9 @@ export function getDriveStorage() {
|
||||
|
||||
export type UpsertDriveInput = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "localstorage" | "spider91";
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
name: string;
|
||||
rootId: string;
|
||||
scanRootId: string;
|
||||
credentials: Record<string, string>;
|
||||
/**
|
||||
* 可选:写入"扫描跳过目录"集合。`undefined` 表示不变(沿用服务端旧值),
|
||||
@@ -244,6 +245,13 @@ export function regenFailedThumbnails(id: string) {
|
||||
);
|
||||
}
|
||||
|
||||
export function regenFailedFingerprints(id: string) {
|
||||
return request<{ ok: boolean }>(
|
||||
`/drives/${encodeURIComponent(id)}/fingerprints/failed/regenerate`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- Videos ----------
|
||||
|
||||
export type AdminVideo = {
|
||||
@@ -333,6 +341,13 @@ export function createTag(label: string, aliases: string[]) {
|
||||
});
|
||||
}
|
||||
|
||||
export function deleteTag(id: number) {
|
||||
return request<{ ok: boolean; removedVideos: number }>(
|
||||
`/tags/${encodeURIComponent(String(id))}`,
|
||||
{ method: "DELETE" }
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- Settings ----------
|
||||
|
||||
export type Theme = "dark" | "pink";
|
||||
@@ -369,10 +384,25 @@ export function updateSettings(body: Partial<Settings>) {
|
||||
|
||||
/**
|
||||
* 立即触发一次完整的凌晨流水线(Phase1 扫盘 + Phase2 91 爬虫 + Phase3 迁移),
|
||||
* 不论当前时间或今日是否已跑。立即返回 202;进度通过 backend 日志观察。
|
||||
* 不论当前时间或今日是否已跑。立即返回 202;进度通过任务状态和 backend 日志观察。
|
||||
*
|
||||
* 流水线已在跑时后端最多保留一个待触发请求;已有待触发请求时,新的点击会被忽略。
|
||||
* 流水线已在跑或已排队时,后端会拒绝重复触发。
|
||||
*/
|
||||
export function runNightlyJob() {
|
||||
return request<{ ok: boolean }>("/jobs/nightly/run", { method: "POST" });
|
||||
export type NightlyJobStatus = {
|
||||
state: "idle" | "queued" | "running" | "running_queued";
|
||||
running: boolean;
|
||||
queued: boolean;
|
||||
startedAt?: string;
|
||||
lastFinishedAt?: string;
|
||||
};
|
||||
|
||||
export function getNightlyJobStatus() {
|
||||
return request<NightlyJobStatus>("/jobs/nightly/status");
|
||||
}
|
||||
|
||||
export function runNightlyJob() {
|
||||
return request<{ ok: boolean; accepted: boolean; status: NightlyJobStatus }>(
|
||||
"/jobs/nightly/run",
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
+12
-5
@@ -1,8 +1,13 @@
|
||||
import type { VideoDetail, VideoItem } from "@/types";
|
||||
|
||||
// 真实后端接口调用。未配置网盘时,各接口返回空数据。
|
||||
export function fetchHomeVideos(): Promise<VideoItem[]> {
|
||||
return apiGet<VideoItem[]>("/api/home").catch(() => []);
|
||||
export function fetchHomeVideos(excludeIds?: string[]): Promise<VideoItem[]> {
|
||||
const qs = new URLSearchParams();
|
||||
for (const id of excludeIds ?? []) {
|
||||
if (id.trim()) qs.append("exclude", id.trim());
|
||||
}
|
||||
const suffix = qs.toString() ? `?${qs.toString()}` : "";
|
||||
return apiGet<VideoItem[]>(`/api/home${suffix}`).catch(() => []);
|
||||
}
|
||||
|
||||
export function fetchListing(
|
||||
@@ -93,17 +98,19 @@ export type ShortsNextResponse = {
|
||||
|
||||
/**
|
||||
* 拉取短视频流的下一批候选。把当前轮已看过的 video id 列表传给后端,
|
||||
* 服务器从未在列表中的视频里随机抽 count 条返回。
|
||||
* 服务器从未在列表中的视频里随机抽 count 条返回。preferredFromVideoId
|
||||
* 来自用户最近一次点赞成功的视频,用于按相似标签优先推荐。
|
||||
*
|
||||
* 失败时返回空批 + roundComplete=false,由调用方决定是否重试。
|
||||
*/
|
||||
export function fetchShortsNext(
|
||||
seenIds: string[],
|
||||
count: number
|
||||
count: number,
|
||||
preferredFromVideoId?: string
|
||||
): Promise<ShortsNextResponse> {
|
||||
return apiJSON<ShortsNextResponse>("/api/shorts/next", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ seenIds, count }),
|
||||
body: JSON.stringify({ seenIds, count, preferredFromVideoId }),
|
||||
}).catch(() => ({ items: [], total: 0, roundComplete: false }));
|
||||
}
|
||||
|
||||
|
||||
+34
-1
@@ -10,6 +10,8 @@ import type { VideoItem } from "@/types";
|
||||
|
||||
const DESKTOP_COUNT = 12;
|
||||
const MOBILE_COUNT = 8;
|
||||
const HOME_RECENT_KEY = "home.random.recentVideoIds";
|
||||
const HOME_RECENT_LIMIT = 72;
|
||||
|
||||
function useIsMobile() {
|
||||
const [mobile, setMobile] = useState(window.innerWidth <= 640);
|
||||
@@ -26,6 +28,35 @@ function useIsMobile() {
|
||||
let cachedRanking: VideoItem[] | null = null;
|
||||
let cachedLatest: VideoItem[] | null = null;
|
||||
|
||||
function loadRecentHomeVideoIds(): string[] {
|
||||
try {
|
||||
const raw = window.localStorage.getItem(HOME_RECENT_KEY);
|
||||
const parsed = raw ? JSON.parse(raw) : [];
|
||||
return Array.isArray(parsed)
|
||||
? parsed.filter((id): id is string => typeof id === "string" && id.length > 0)
|
||||
: [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function rememberHomeVideos(items: VideoItem[]) {
|
||||
const merged = [...items.map((item) => item.id), ...loadRecentHomeVideoIds()];
|
||||
const seen = new Set<string>();
|
||||
const recent: string[] = [];
|
||||
for (const id of merged) {
|
||||
if (!id || seen.has(id)) continue;
|
||||
seen.add(id);
|
||||
recent.push(id);
|
||||
if (recent.length >= HOME_RECENT_LIMIT) break;
|
||||
}
|
||||
try {
|
||||
window.localStorage.setItem(HOME_RECENT_KEY, JSON.stringify(recent));
|
||||
} catch {
|
||||
// localStorage 不可用时只影响连续刷新去重,不影响首页展示。
|
||||
}
|
||||
}
|
||||
|
||||
export default function HomePage() {
|
||||
const [rankingVideos, setRankingVideos] = useState<VideoItem[]>(cachedRanking ?? []);
|
||||
const [latestVideos, setLatestVideos] = useState<VideoItem[]>(cachedLatest ?? []);
|
||||
@@ -40,11 +71,13 @@ export default function HomePage() {
|
||||
|
||||
let active = true;
|
||||
setLoading(true);
|
||||
const excludeIds = loadRecentHomeVideoIds();
|
||||
Promise.all([
|
||||
fetchHomeVideos(),
|
||||
fetchHomeVideos(excludeIds),
|
||||
fetchListing(1, DESKTOP_COUNT, { sort: "latest" }),
|
||||
]).then(([rankingItems, latestResult]) => {
|
||||
if (!active) return;
|
||||
rememberHomeVideos(rankingItems);
|
||||
cachedRanking = rankingItems;
|
||||
cachedLatest = latestResult.items;
|
||||
setRankingVideos(rankingItems);
|
||||
|
||||
+82
-12
@@ -120,12 +120,16 @@ export default function ShortsPage() {
|
||||
|
||||
// seenIds 用 ref 维护,方便在异步 callback 里读到最新值
|
||||
const seenIdsRef = useRef<string[]>(loadSeenIds());
|
||||
const preferredFromVideoIdRef = useRef<string | null>(null);
|
||||
|
||||
const containerRef = useRef<HTMLDivElement | null>(null);
|
||||
// 整个页面根元素,用于 requestFullscreen
|
||||
const pageRef = useRef<HTMLDivElement | null>(null);
|
||||
// index → video element,用来精确控制播放/暂停
|
||||
const videoRefs = useRef<Map<number, HTMLVideoElement>>(new Map());
|
||||
const activeIndexRef = useRef(0);
|
||||
const ignoreIntersectionUntilRef = useRef(0);
|
||||
const fullscreenRestoreTimersRef = useRef<number[]>([]);
|
||||
|
||||
// 当前是否处在浏览器全屏(Fullscreen API)状态。
|
||||
// iOS Safari 不支持元素级 Fullscreen API,这里会一直保持 false,
|
||||
@@ -139,6 +143,10 @@ export default function ShortsPage() {
|
||||
// 用户在操作栏点取消时会从这里移除,允许之后再次点赞。
|
||||
const likedIdsRef = useRef<Set<string>>(new Set());
|
||||
|
||||
useEffect(() => {
|
||||
activeIndexRef.current = activeIndex;
|
||||
}, [activeIndex]);
|
||||
|
||||
/**
|
||||
* 切换点赞状态。
|
||||
* - liked=true:发 POST /api/video/:id/like
|
||||
@@ -163,6 +171,11 @@ export default function ShortsPage() {
|
||||
);
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const data = (await res.json()) as { likes?: number };
|
||||
if (liked) {
|
||||
preferredFromVideoIdRef.current = videoId;
|
||||
} else if (preferredFromVideoIdRef.current === videoId) {
|
||||
preferredFromVideoIdRef.current = null;
|
||||
}
|
||||
return typeof data.likes === "number" ? data.likes : null;
|
||||
} catch {
|
||||
// 请求失败:回滚集合,让 Slide 自己回滚 UI
|
||||
@@ -191,7 +204,11 @@ export default function ShortsPage() {
|
||||
setLoading(true);
|
||||
try {
|
||||
const seen = seenIdsRef.current;
|
||||
const resp = await fetchShortsNext(seen, BATCH_SIZE);
|
||||
const resp = await fetchShortsNext(
|
||||
seen,
|
||||
BATCH_SIZE,
|
||||
preferredFromVideoIdRef.current ?? undefined
|
||||
);
|
||||
if (resp.items.length === 0) {
|
||||
setEmpty((prev) => prev || true /* 维持 true 即可 */);
|
||||
setRoundComplete(true);
|
||||
@@ -250,6 +267,8 @@ export default function ShortsPage() {
|
||||
|
||||
const observer = new IntersectionObserver(
|
||||
(entries) => {
|
||||
if (Date.now() < ignoreIntersectionUntilRef.current) return;
|
||||
|
||||
let bestIndex = -1;
|
||||
let bestRatio = 0.6;
|
||||
for (const entry of entries) {
|
||||
@@ -437,11 +456,37 @@ export default function ShortsPage() {
|
||||
};
|
||||
}, []);
|
||||
|
||||
function clearFullscreenRestoreTimers() {
|
||||
for (const timer of fullscreenRestoreTimersRef.current) {
|
||||
window.clearTimeout(timer);
|
||||
}
|
||||
fullscreenRestoreTimersRef.current = [];
|
||||
}
|
||||
|
||||
function restoreActiveSlideIntoView() {
|
||||
const idx = activeIndexRef.current;
|
||||
const slide = containerRef.current?.querySelector<HTMLElement>(
|
||||
`[data-index="${idx}"]`
|
||||
);
|
||||
if (!slide) return;
|
||||
slide.scrollIntoView({ block: "start", inline: "nearest", behavior: "auto" });
|
||||
}
|
||||
|
||||
function scheduleFullscreenActiveRestore() {
|
||||
ignoreIntersectionUntilRef.current = Date.now() + 700;
|
||||
clearFullscreenRestoreTimers();
|
||||
restoreActiveSlideIntoView();
|
||||
fullscreenRestoreTimersRef.current = [80, 220, 520].map((delay) =>
|
||||
window.setTimeout(restoreActiveSlideIntoView, delay)
|
||||
);
|
||||
}
|
||||
|
||||
// ---- 浏览器全屏(Fullscreen API) ----
|
||||
// 监听全屏状态变化,保持 React state 同步。
|
||||
// 用户按 ESC / 系统返回 / 浏览器退出全屏按钮 时也会走这里。
|
||||
useEffect(() => {
|
||||
function handleChange() {
|
||||
scheduleFullscreenActiveRestore();
|
||||
setIsFullscreen(
|
||||
document.fullscreenElement !== null ||
|
||||
// Safari (desktop) 旧前缀
|
||||
@@ -454,6 +499,7 @@ export default function ShortsPage() {
|
||||
return () => {
|
||||
document.removeEventListener("fullscreenchange", handleChange);
|
||||
document.removeEventListener("webkitfullscreenchange", handleChange);
|
||||
clearFullscreenRestoreTimers();
|
||||
};
|
||||
}, []);
|
||||
|
||||
@@ -530,6 +576,7 @@ export default function ShortsPage() {
|
||||
}
|
||||
|
||||
function toggleFullscreen() {
|
||||
scheduleFullscreenActiveRestore();
|
||||
if (isFullscreen) exitPageFullscreen();
|
||||
else requestPageFullscreen();
|
||||
}
|
||||
@@ -695,6 +742,7 @@ function ShortsSlide({
|
||||
const [duration, setDuration] = useState(0);
|
||||
const [currentTime, setCurrentTime] = useState(0);
|
||||
const [scrubbing, setScrubbing] = useState(false);
|
||||
const scrubbingRef = useRef(false);
|
||||
// 拖动开始时是否在播:用于拖完后判断要不要 resume
|
||||
const wasPlayingRef = useRef(true);
|
||||
|
||||
@@ -735,6 +783,7 @@ function ShortsSlide({
|
||||
if (!isActive) {
|
||||
setPaused(false);
|
||||
setScrubbing(false);
|
||||
scrubbingRef.current = false;
|
||||
setIsBuffering(false);
|
||||
setPlayPauseHud(null);
|
||||
}
|
||||
@@ -769,7 +818,7 @@ function ShortsSlide({
|
||||
};
|
||||
const handleTime = () => {
|
||||
// 拖动期间不要被 timeupdate 覆盖 UI
|
||||
if (!scrubbing) setCurrentTime(video.currentTime);
|
||||
if (!scrubbingRef.current) setCurrentTime(video.currentTime);
|
||||
};
|
||||
const handleWaiting = () => {
|
||||
setIsBuffering(true);
|
||||
@@ -811,7 +860,7 @@ function ShortsSlide({
|
||||
video.removeEventListener("canplay", handlePlayingOrCanPlay);
|
||||
video.removeEventListener("volumechange", handleVolumeChange);
|
||||
};
|
||||
}, [shouldMount, scrubbing, muted, volume, setMuted, setVolume]);
|
||||
}, [muted, volume, setMuted, setVolume]);
|
||||
|
||||
// 长按 2 倍速:直接绑原生事件
|
||||
useEffect(() => {
|
||||
@@ -1021,11 +1070,16 @@ function ShortsSlide({
|
||||
// ---- 进度条拖动 ----
|
||||
// 触摸进度条时:暂停 → 跟随手指更新 currentTime → 松手 resume
|
||||
function handleProgressPointerDown(e: React.PointerEvent<HTMLDivElement>) {
|
||||
const video = localRef.current;
|
||||
if (!video || !duration) return;
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
(e.currentTarget as HTMLElement).setPointerCapture(e.pointerId);
|
||||
const video = localRef.current;
|
||||
const seekDuration = getSeekDuration(video);
|
||||
if (!video || !seekDuration) return;
|
||||
try {
|
||||
(e.currentTarget as HTMLElement).setPointerCapture(e.pointerId);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
wasPlayingRef.current = !video.paused;
|
||||
if (!video.paused) {
|
||||
try {
|
||||
@@ -1034,17 +1088,19 @@ function ShortsSlide({
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
scrubbingRef.current = true;
|
||||
setScrubbing(true);
|
||||
applyProgressFromEvent(e);
|
||||
applyProgressFromEvent(e, seekDuration);
|
||||
}
|
||||
function handleProgressPointerMove(e: React.PointerEvent<HTMLDivElement>) {
|
||||
if (!scrubbing) return;
|
||||
if (!scrubbingRef.current) return;
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
applyProgressFromEvent(e);
|
||||
}
|
||||
function handleProgressPointerEnd(e: React.PointerEvent<HTMLDivElement>) {
|
||||
if (!scrubbing) return;
|
||||
if (!scrubbingRef.current) return;
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
try {
|
||||
(e.currentTarget as HTMLElement).releasePointerCapture(e.pointerId);
|
||||
@@ -1052,17 +1108,30 @@ function ShortsSlide({
|
||||
// ignore
|
||||
}
|
||||
const video = localRef.current;
|
||||
scrubbingRef.current = false;
|
||||
setScrubbing(false);
|
||||
if (video && wasPlayingRef.current) {
|
||||
video.play().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
function applyProgressFromEvent(e: React.PointerEvent<HTMLDivElement>) {
|
||||
function getSeekDuration(video: HTMLVideoElement | null) {
|
||||
if (duration > 0) return duration;
|
||||
if (video && Number.isFinite(video.duration) && video.duration > 0) {
|
||||
setDuration(video.duration);
|
||||
return video.duration;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
function applyProgressFromEvent(
|
||||
e: React.PointerEvent<HTMLDivElement>,
|
||||
knownDuration?: number
|
||||
) {
|
||||
const video = localRef.current;
|
||||
if (!video || !duration) return;
|
||||
const seekDuration = knownDuration ?? getSeekDuration(video);
|
||||
if (!video || !seekDuration) return;
|
||||
const rect = e.currentTarget.getBoundingClientRect();
|
||||
const ratio = clamp((e.clientX - rect.left) / rect.width, 0, 1);
|
||||
const next = ratio * duration;
|
||||
const next = ratio * seekDuration;
|
||||
setCurrentTime(next);
|
||||
try {
|
||||
video.currentTime = next;
|
||||
@@ -1235,6 +1304,7 @@ function ShortsSlide({
|
||||
onPointerMove={handleProgressPointerMove}
|
||||
onPointerUp={handleProgressPointerEnd}
|
||||
onPointerCancel={handleProgressPointerEnd}
|
||||
onLostPointerCapture={handleProgressPointerEnd}
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
>
|
||||
<div
|
||||
|
||||
@@ -7,7 +7,7 @@ import { uploadVideo } from "@/data/videos";
|
||||
import { defaultUploadTitleFromFileName } from "@/lib/uploadTitle";
|
||||
import type { VideoItem } from "@/types";
|
||||
|
||||
const UPLOAD_TAGS = ["奶子", "臀", "口角", "女大", "人妻", "AV"];
|
||||
const UPLOAD_TAGS = ["奶子", "臀", "口交", "女大", "人妻", "AV"];
|
||||
|
||||
export default function UploadPage() {
|
||||
const [file, setFile] = useState<File | null>(null);
|
||||
|
||||
+105
-18
@@ -2128,28 +2128,51 @@
|
||||
}
|
||||
|
||||
.admin-tags-grid {
|
||||
column-count: 3;
|
||||
column-gap: var(--space-3);
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(240px, 1fr));
|
||||
gap: var(--space-3);
|
||||
align-items: stretch;
|
||||
}
|
||||
|
||||
@media (max-width: 1200px) {
|
||||
.admin-tags-grid {
|
||||
column-count: 2;
|
||||
}
|
||||
.admin-tags-bulkbar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--space-3);
|
||||
flex-wrap: wrap;
|
||||
margin-bottom: var(--space-4);
|
||||
padding: var(--space-2) var(--space-3);
|
||||
background: var(--bg-sunken);
|
||||
border: 1px solid var(--border-subtle);
|
||||
border-radius: var(--radius-sm);
|
||||
}
|
||||
|
||||
@media (max-width: 600px) {
|
||||
.admin-tags-grid {
|
||||
column-count: 1;
|
||||
}
|
||||
.admin-tags-bulkbar__count {
|
||||
font-size: var(--font-xs);
|
||||
color: var(--text-muted);
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
.admin-tag-card.is-selectable {
|
||||
cursor: pointer;
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
.admin-tag-card.is-selected {
|
||||
border-color: var(--border-accent);
|
||||
box-shadow: 0 0 0 1px var(--border-accent), var(--shadow-md);
|
||||
}
|
||||
|
||||
.admin-tag-card__check {
|
||||
width: 15px;
|
||||
height: 15px;
|
||||
accent-color: var(--accent);
|
||||
cursor: pointer;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.admin-tag-card {
|
||||
break-inside: avoid;
|
||||
display: inline-flex;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
width: 100%;
|
||||
margin-bottom: var(--space-3);
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border-subtle);
|
||||
border-radius: var(--radius-sm);
|
||||
@@ -2176,6 +2199,7 @@
|
||||
font-size: var(--font-md);
|
||||
font-weight: var(--weight-bold);
|
||||
color: var(--text-strong);
|
||||
margin-right: auto;
|
||||
}
|
||||
|
||||
.admin-tag-card__source-badge {
|
||||
@@ -2228,7 +2252,7 @@
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-top: var(--space-2);
|
||||
margin-top: auto;
|
||||
padding-top: var(--space-2);
|
||||
border-top: 1px solid var(--border-subtle);
|
||||
font-size: var(--font-xs);
|
||||
@@ -2237,8 +2261,71 @@
|
||||
|
||||
.admin-tag-card__count {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
align-items: baseline;
|
||||
gap: 4px;
|
||||
font-weight: var(--weight-semibold);
|
||||
color: var(--text-default);
|
||||
font-size: var(--font-xs);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
.admin-tag-card__count svg {
|
||||
align-self: center;
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.admin-tag-card__count strong {
|
||||
font-size: var(--font-md);
|
||||
font-weight: var(--weight-bold);
|
||||
color: var(--text-strong);
|
||||
}
|
||||
|
||||
.admin-tag-card__id {
|
||||
font-size: 10px;
|
||||
color: var(--text-faint);
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.admin-tag-card__footer-actions {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
}
|
||||
|
||||
.admin-tag-card__delete {
|
||||
border: 1px solid var(--danger);
|
||||
background: var(--danger-soft);
|
||||
color: var(--danger);
|
||||
border-radius: var(--radius-xs);
|
||||
padding: 3px 6px;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 3px;
|
||||
font-size: 10px;
|
||||
font-weight: var(--weight-semibold);
|
||||
cursor: pointer;
|
||||
transition: all var(--transition-fast);
|
||||
}
|
||||
|
||||
.admin-tag-card__delete:hover:not(:disabled) {
|
||||
background: var(--danger);
|
||||
color: white;
|
||||
}
|
||||
|
||||
/* 仅在支持悬停的设备上隐藏删除按钮,悬停或键盘聚焦卡片时显示,避免满屏删除按钮 */
|
||||
@media (hover: hover) {
|
||||
.admin-tag-card__delete {
|
||||
opacity: 0;
|
||||
pointer-events: none;
|
||||
transition: opacity var(--transition-fast);
|
||||
}
|
||||
|
||||
.admin-tag-card:hover .admin-tag-card__delete,
|
||||
.admin-tag-card:focus-within .admin-tag-card__delete {
|
||||
opacity: 1;
|
||||
pointer-events: auto;
|
||||
}
|
||||
}
|
||||
|
||||
.admin-tag-card__delete:disabled {
|
||||
opacity: 0.55;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
@@ -8,6 +8,9 @@ const drivesPageSource = readFileSync(
|
||||
);
|
||||
|
||||
test("spider91 drive form does not expose advanced crawler credentials", () => {
|
||||
assert.match(drivesPageSource, /key: "proxy"/);
|
||||
assert.match(drivesPageSource, /label: "代理地址(可选)"/);
|
||||
assert.match(drivesPageSource, /支持 http:\/\/、https:\/\/、socks5:\/\/ 或 socks5h:\/\//);
|
||||
assert.doesNotMatch(drivesPageSource, /target_new/);
|
||||
assert.doesNotMatch(drivesPageSource, /crawl_hour/);
|
||||
assert.doesNotMatch(drivesPageSource, /python_path/);
|
||||
@@ -24,14 +27,22 @@ test("spider91 upload target uses explicit local-save option instead of auto tar
|
||||
assert.doesNotMatch(drivesPageSource, /自动模式/);
|
||||
});
|
||||
|
||||
test("onedrive drive form only exposes required default-app fields", () => {
|
||||
test("drive form hides root directory id for localstorage and spider91", () => {
|
||||
assert.match(drivesPageSource, /<label>根目录 ID<\/label>/);
|
||||
assert.match(
|
||||
drivesPageSource,
|
||||
/form\.kind !== "spider91" &&\s*form\.kind !== "onedrive" &&\s*form\.kind !== "localstorage" &&\s*form\.kind !== "pikpak"/
|
||||
/function usesRootDirectoryID\(kind: Kind\): boolean \{\s*return kind !== "localstorage" && kind !== "spider91";\s*\}/
|
||||
);
|
||||
assert.match(drivesPageSource, /\{usesRootDirectoryID\(form\.kind\) && \(/);
|
||||
assert.match(drivesPageSource, /\{usesRootDirectoryID\(d\.kind\) && \(/);
|
||||
assert.match(drivesPageSource, /placeholder=\{rootIdPlaceholder\(form\.kind\)\}/);
|
||||
assert.doesNotMatch(drivesPageSource, /扫描起点目录 ID/);
|
||||
assert.doesNotMatch(drivesPageSource, /set\("scanRootId"/);
|
||||
});
|
||||
|
||||
test("onedrive drive form only exposes required default-app fields", () => {
|
||||
const match =
|
||||
/function credentialFields[\s\S]*?case "onedrive":\s*return \[([\s\S]*?)\];\s*case "spider91":/.exec(
|
||||
/function credentialFields[\s\S]*?case "onedrive":\s*return \[([\s\S]*?)\];\s*case "googledrive":/.exec(
|
||||
drivesPageSource
|
||||
);
|
||||
assert.ok(match, "onedrive credential field block should be present");
|
||||
@@ -45,6 +56,23 @@ test("onedrive drive form only exposes required default-app fields", () => {
|
||||
assert.doesNotMatch(fields, /key: "site_id"/);
|
||||
});
|
||||
|
||||
test("googledrive drive form only exposes refresh token", () => {
|
||||
assert.match(drivesPageSource, /<option value="googledrive">Google Drive<\/option>/);
|
||||
|
||||
const match =
|
||||
/case "googledrive":\s*return \[([\s\S]*?)\];\s*case "localstorage":/.exec(
|
||||
drivesPageSource
|
||||
);
|
||||
assert.ok(match, "googledrive credential field block should be present");
|
||||
const fields = match[1];
|
||||
|
||||
assert.match(fields, /key: "refresh_token"/);
|
||||
assert.doesNotMatch(fields, /key: "access_token"/);
|
||||
assert.doesNotMatch(fields, /key: "api_url_address"/);
|
||||
assert.doesNotMatch(fields, /key: "client_id"/);
|
||||
assert.doesNotMatch(fields, /key: "client_secret"/);
|
||||
});
|
||||
|
||||
test("pikpak drive form only exposes account login fields", () => {
|
||||
const match =
|
||||
/case "pikpak":\s*return \[([\s\S]*?)\];\s*case "wopan":/.exec(
|
||||
@@ -75,6 +103,7 @@ test("localstorage drive form asks for a server directory path", () => {
|
||||
assert.match(fields, /key: "path"/);
|
||||
assert.match(fields, /label: "本地目录路径"/);
|
||||
assert.match(drivesPageSource, /if \(kind === "localstorage"\) return "\/"/);
|
||||
assert.match(drivesPageSource, /kind !== "localstorage" && kind !== "spider91"/);
|
||||
});
|
||||
|
||||
test("drive type selector keeps primary source order", () => {
|
||||
@@ -82,12 +111,13 @@ test("drive type selector keeps primary source order", () => {
|
||||
drivesPageSource.matchAll(/<option value="([^"]+)">([^<]+)<\/option>/g),
|
||||
(match) => ({ value: match[1], label: match[2] })
|
||||
);
|
||||
const driveOptions = options.slice(0, 7);
|
||||
const driveOptions = options.slice(0, 8);
|
||||
|
||||
assert.deepEqual(driveOptions, [
|
||||
{ value: "p115", label: "115 网盘" },
|
||||
{ value: "pikpak", label: "PikPak" },
|
||||
{ value: "onedrive", label: "OneDrive" },
|
||||
{ value: "googledrive", label: "Google Drive" },
|
||||
{ value: "localstorage", label: "本地存储" },
|
||||
{ value: "spider91", label: "91 Spider" },
|
||||
{ value: "quark", label: "夸克网盘" },
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import test from "node:test";
|
||||
|
||||
const tagsPageSource = readFileSync(
|
||||
new URL("../src/admin/TagsPage.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
|
||||
test("admin tags page limits visible tags by viewport", () => {
|
||||
assert.match(tagsPageSource, /const DESKTOP_TAGS_PAGE_SIZE = 25;/);
|
||||
assert.match(tagsPageSource, /const MOBILE_TAGS_PAGE_SIZE = 8;/);
|
||||
assert.match(tagsPageSource, /const TAGS_MOBILE_QUERY = "\(max-width: 640px\)";/);
|
||||
assert.match(tagsPageSource, /window\.matchMedia\(TAGS_MOBILE_QUERY\)/);
|
||||
});
|
||||
|
||||
test("admin tags page renders only the current page", () => {
|
||||
assert.match(tagsPageSource, /filteredTags\.slice\(pageStartIndex, pageEndIndex\)/);
|
||||
assert.match(tagsPageSource, /pagedTags\.map\(\(tag\) =>/);
|
||||
assert.doesNotMatch(tagsPageSource, /filteredTags\.map\(\(tag\) =>/);
|
||||
assert.match(tagsPageSource, /全选本页/);
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import test from "node:test";
|
||||
|
||||
const shortsPageSource = readFileSync(
|
||||
new URL("../src/pages/ShortsPage.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
|
||||
test("shorts recommendation preference follows successful likes instead of watch time", () => {
|
||||
assert.doesNotMatch(shortsPageSource, /currentTime\s*>=\s*3/);
|
||||
assert.doesNotMatch(shortsPageSource, /onPreferenceReady/);
|
||||
|
||||
const match = /const handleLikeToggle[\s\S]*?const hasLiked/.exec(
|
||||
shortsPageSource
|
||||
);
|
||||
assert.ok(match, "handleLikeToggle block should be present");
|
||||
|
||||
assert.match(
|
||||
match[0],
|
||||
/if \(liked\) \{\s*preferredFromVideoIdRef\.current = videoId;\s*\} else if \(preferredFromVideoIdRef\.current === videoId\) \{\s*preferredFromVideoIdRef\.current = null;/
|
||||
);
|
||||
});
|
||||
|
||||
test("shorts progress dragging uses immediate pointer state", () => {
|
||||
assert.match(shortsPageSource, /const scrubbingRef = useRef\(false\)/);
|
||||
assert.match(shortsPageSource, /scrubbingRef\.current = true;/);
|
||||
assert.match(shortsPageSource, /if \(!scrubbingRef\.current\) return;/);
|
||||
assert.doesNotMatch(shortsPageSource, /if \(!scrubbing\) return;/);
|
||||
assert.match(shortsPageSource, /function getSeekDuration/);
|
||||
assert.match(shortsPageSource, /onLostPointerCapture=\{handleProgressPointerEnd\}/);
|
||||
});
|
||||
|
||||
test("shorts fullscreen changes preserve the active slide", () => {
|
||||
assert.match(shortsPageSource, /const activeIndexRef = useRef\(0\)/);
|
||||
assert.match(shortsPageSource, /const ignoreIntersectionUntilRef = useRef\(0\)/);
|
||||
assert.match(
|
||||
shortsPageSource,
|
||||
/if \(Date\.now\(\) < ignoreIntersectionUntilRef\.current\) return;/
|
||||
);
|
||||
assert.match(shortsPageSource, /function scheduleFullscreenActiveRestore\(\)/);
|
||||
assert.match(shortsPageSource, /scheduleFullscreenActiveRestore\(\);\s*setIsFullscreen/);
|
||||
assert.match(
|
||||
shortsPageSource,
|
||||
/function toggleFullscreen\(\) \{\s*scheduleFullscreenActiveRestore\(\);/
|
||||
);
|
||||
assert.match(shortsPageSource, /scrollIntoView\(\{ block: "start", inline: "nearest", behavior: "auto" \}\)/);
|
||||
});
|
||||
@@ -1441,7 +1441,7 @@ VideoProject/
|
||||
| 项 | 决定 |
|
||||
|---|---|
|
||||
| 登录方式 | **B**:管理后台做完整登录流程。115 扫码、夸克扫码或 Cookie 导入、沃盘手机号 + 短信验证。Token 持久化到 SQLite 并自动刷新。 |
|
||||
| 元数据来源 | **默认文件名解析**:`标题.mp4` 或 `[tag1,tag2] 标题 - 作者.mp4`;同时提供后台录入 API 覆盖字段 |
|
||||
| 元数据来源 | **默认文件名解析**:`标题.mp4`、`标题 - 作者.mp4`,或带前缀的 `[前缀] 标题 - 作者.mp4`;前缀只用于标题清理,不作为任意标签列表入库。标签来自系统 / 用户标签匹配和目录合集规则;同时提供后台录入 API 覆盖字段 |
|
||||
| Hover teaser | **C 预生成**:scanner 发现新视频时异步生成 10s teaser 并存回网盘的 `previews/` 目录,详情页和列表页 hover 都秒开 |
|
||||
| 部署目标 | Linux 服务器;本地 Windows 开发 |
|
||||
| 扫描策略 | 启动时全量 + 每 6 小时增量 + 支持手动触发 |
|
||||
@@ -1485,14 +1485,14 @@ type StreamLink struct {
|
||||
|
||||
### 15.5 文件名解析规则
|
||||
|
||||
默认解析顺序(取第一个匹配):
|
||||
默认解析顺序(取第一个匹配),用于提取 `title` / `author`:
|
||||
|
||||
1. 完整格式:`[tag1,tag2] 标题 - 作者.ext`
|
||||
2. 去作者:`[tag1,tag2] 标题.ext`
|
||||
3. 去标签:`标题 - 作者.ext`
|
||||
1. 带前缀和作者:`[前缀] 标题 - 作者.ext`
|
||||
2. 带前缀:`[前缀] 标题.ext`
|
||||
3. 带作者:`标题 - 作者.ext`
|
||||
4. 最简单:`标题.ext`
|
||||
|
||||
解析出的字段:`title` / `author` / `tags[]`。其余字段(`duration` / `views` / `favorites` 等)由 scanner 读取文件元数据或置默认值。
|
||||
开头的 `[前缀]` 只会从标题里剥离,不会按 `,` / `,` / `、` / 空格拆成任意标签入库。`tags[]` 由 scanner 另行生成:文件名、作者和目录名命中系统标签或已有标签的标签名 / 别名时自动打标;符合条件的目录名会创建 `collection` 合集标签;常见番号类文本会归并为 `AV`。当前内置系统标签是 `后入`、`奶子`、`口交`、`臀`、`人妻`、`女大`、`AV`。其余字段(`duration` / `views` / `favorites` 等)由 scanner 读取文件元数据或置默认值。
|
||||
|
||||
后台录入接口可用来覆盖解析结果:
|
||||
|
||||
@@ -1570,6 +1570,10 @@ POST /admin/api/videos # 手动新建
|
||||
PUT /admin/api/videos/:id # 修改元数据
|
||||
DELETE /admin/api/videos/:id
|
||||
POST /admin/api/videos/:id/regen-preview
|
||||
|
||||
GET /admin/api/tags # 标签列表
|
||||
POST /admin/api/tags # 新增标签并自动归类历史视频
|
||||
DELETE /admin/api/tags/:id # 删除非系统标签,并从所有视频上移除
|
||||
```
|
||||
|
||||
登录流程三家各不相同:
|
||||
@@ -1684,9 +1688,10 @@ Teaser 不再是"固定从第 10 秒抽 10 秒",改为按视频时长分段挑
|
||||
- `backend/internal/catalog/tags.go` `migrate` / `pruneOrphanCollectionTags` / `pruneOrphanCollectionTagsByID` / `collectVideoTagIDs`
|
||||
- 测试:`backend/internal/catalog/tags_test.go` `TestDeleteVideoPrunesOrphanCollectionTag` / `TestMigratePrunesPreexistingOrphanCollectionTags`
|
||||
|
||||
**已知不在本次范围**:
|
||||
- `/admin/api/tags` 仍只有 `GET` / `POST`,没有 `DELETE`。如果将来要让管理员手动删 `user` 标签,再加 endpoint。
|
||||
- 数据迁移:上线时对运行中数据库一次性执行同样的 `DELETE` 即可(已对当前实例执行:清掉 10 条 `Season N` / `Better Call Saul SXX` / `东京爱情故事(1991)`,`tags` 总数 153 → 143)。
|
||||
**手动删除标签**:
|
||||
- `/admin/api/tags/{id}` 支持 `DELETE`。管理员可手动删除非系统标签,删除时同步清理 `video_tags` 并刷新相关视频的 `videos.tags` JSON。
|
||||
- `system` 标签由固定标签池维护,不开放删除;`user` / `collection` / `legacy` 标签可由管理员按需删除。
|
||||
- 历史孤儿 `collection` 标签仍由迁移自愈逻辑自动清理。
|
||||
|
||||
### 14.7 取消浏览器内本机转码,全部走 302 直链 + VLC 外部播放器按钮(2026-05-21)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user