mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-15 16:55:42 +08:00
Compare commits
100 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 71d4a16db1 | |||
| 940e5dd76d | |||
| e826c05d5c | |||
| 3465b9e837 | |||
| d33c1b1b20 | |||
| 5fc8e9ebb7 | |||
| dc7d2a5de3 | |||
| 2f2bfbfcdc | |||
| 9def08b0c5 | |||
| c87208117e | |||
| a770b3af6b | |||
| e1b8f0eae7 | |||
| 2d907da07d | |||
| 78cfb0a9e5 | |||
| fa7823ef3e | |||
| 5b0afcfc6c | |||
| 76ae3cea7d | |||
| abe335cea0 | |||
| 8dff0f07b9 | |||
| 5080203b7c | |||
| df6f0ebbbf | |||
| 8f0d52aec4 | |||
| 53327c9b8e | |||
| 57ed546b83 | |||
| 869c0d5f78 | |||
| 397823bb8d | |||
| 9e1acd4e56 | |||
| 2cd365acd4 | |||
| 48808ec568 | |||
| 5dc00e486d | |||
| 4ec1097496 | |||
| 95e46d8fbb | |||
| fdfc4771df | |||
| c8c6812ae6 | |||
| b938ff1221 | |||
| 7d63a6d265 | |||
| a8de7d2f6b | |||
| d4fcff896e | |||
| cada336e96 | |||
| 5bb93bd95b | |||
| b6be7d021c | |||
| e36a17f99d | |||
| e01b7cc3b7 | |||
| c78f22aedb | |||
| cf9de5b40a | |||
| be19f81e82 | |||
| 4d679ef64f | |||
| 4ba964b7e2 | |||
| cd3b3c6976 | |||
| 91c03947d1 | |||
| 7f1c1a51a3 | |||
| 077c2e2c38 | |||
| 30a62f265a | |||
| 38e62c6a2f | |||
| 6345cf74e0 | |||
| f004b14d20 | |||
| a407312dfa | |||
| a165605b0f | |||
| 0ac1a5b13e | |||
| a83449b129 | |||
| c68891e6f0 | |||
| 9892599412 | |||
| 0cb2a7a1c2 | |||
| 87d197496b | |||
| 0e3a5bd5cd | |||
| d72bfee10f | |||
| 389dd981a8 | |||
| 44d622d49c | |||
| d7ff0c98af | |||
| 66adf444ba | |||
| 8f8037b838 | |||
| 215d9596fd | |||
| e57058db79 | |||
| 6ec61833f2 | |||
| 6e87f88d53 | |||
| e78fa9d978 | |||
| afbff9eb55 | |||
| 039ec2a988 | |||
| da0683344e | |||
| 1a1282382e | |||
| 34b6fa8ea9 | |||
| 08e38bc4ca | |||
| c93d193efe | |||
| 08568c3951 | |||
| 7e394e2971 | |||
| d16e3168f9 | |||
| 81f348b246 | |||
| 1e71c1fb72 | |||
| d5122d289e | |||
| c146ad50ed | |||
| f5c20f9594 | |||
| 62e69d4c06 | |||
| 51725ba82f | |||
| c06db836dd | |||
| b8717da4fd | |||
| 2d57545e87 | |||
| 6518d772c0 | |||
| f2c0e7f854 | |||
| 3c7219ecd6 | |||
| 94669fd35e |
@@ -23,8 +23,10 @@ tools/
|
||||
|
||||
# 编译产物
|
||||
backend/server
|
||||
backend/server.*
|
||||
release/
|
||||
tsconfig.tsbuildinfo
|
||||
tmp/
|
||||
|
||||
# 91 爬虫脚本独立运行时的默认输出文件(backend 跑时会显式 --output 到 backend/data/spider91/,所以不会落在这里)
|
||||
91porn_videos.json
|
||||
@@ -33,3 +35,10 @@ tsconfig.tsbuildinfo
|
||||
91VideoSpider/__pycache__/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Local scratch images
|
||||
/image.jpg
|
||||
/image003.jpg
|
||||
/image004.jpg
|
||||
/image005.png
|
||||
/image02.png
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
- 视频直链 (MP4)
|
||||
|
||||
依赖安装:
|
||||
pip install requests beautifulsoup4 lxml
|
||||
pip install requests beautifulsoup4 lxml PySocks
|
||||
|
||||
使用方法:
|
||||
# 全量爬取(默认行为,从 page=1 一直爬到末尾,写到 OUTPUT_FILE)
|
||||
@@ -68,6 +68,7 @@ import time
|
||||
import random
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import html
|
||||
from urllib.parse import urljoin, unquote, urlparse
|
||||
@@ -80,6 +81,28 @@ except ImportError:
|
||||
print("请运行: pip install beautifulsoup4 lxml")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def prefer_ipv4_for_plain_socks5_proxy():
|
||||
"""PySocks may pick IPv6 first for socks5://; some SOCKS5 servers only accept IPv4."""
|
||||
proxy_envs = (
|
||||
os.environ.get("HTTPS_PROXY", ""),
|
||||
os.environ.get("HTTP_PROXY", ""),
|
||||
os.environ.get("https_proxy", ""),
|
||||
os.environ.get("http_proxy", ""),
|
||||
)
|
||||
uses_plain_socks5 = any(v.strip().lower().startswith("socks5://") for v in proxy_envs)
|
||||
if not uses_plain_socks5 or getattr(socket, "_spider91_ipv4_first", False):
|
||||
return
|
||||
|
||||
original_getaddrinfo = socket.getaddrinfo
|
||||
|
||||
def getaddrinfo_ipv4_first(*args, **kwargs):
|
||||
infos = original_getaddrinfo(*args, **kwargs)
|
||||
return sorted(infos, key=lambda info: 0 if info[0] == socket.AF_INET else 1)
|
||||
|
||||
socket.getaddrinfo = getaddrinfo_ipv4_first
|
||||
socket._spider91_ipv4_first = True
|
||||
|
||||
# ===================== 配置区域 =====================
|
||||
BASE_URL = "https://www.91porn.com/v.php"
|
||||
LIST_PARAMS = {
|
||||
@@ -706,7 +729,7 @@ def print_help():
|
||||
- 视频直链 (MP4)
|
||||
|
||||
依赖安装:
|
||||
pip install requests beautifulsoup4 lxml
|
||||
pip install requests beautifulsoup4 lxml PySocks
|
||||
|
||||
使用方法:
|
||||
python spider_91porn.py
|
||||
@@ -757,13 +780,15 @@ def main():
|
||||
"日志改走 stderr。配合 backend 边读边下载使用。")
|
||||
|
||||
args, _ = parser.parse_known_args()
|
||||
cli_out = sys.stderr if args.stream_output else sys.stdout
|
||||
prefer_ipv4_for_plain_socks5_proxy()
|
||||
|
||||
print("""
|
||||
================================================
|
||||
91porn 视频爬虫启动中...
|
||||
================================================
|
||||
按 Ctrl+C 可随时中断并保存进度
|
||||
""")
|
||||
""", file=cli_out)
|
||||
|
||||
# 加载已知 ID(来自 backend 的 catalog 已入库列表;兼容旧参数名)
|
||||
seen_viewkeys = []
|
||||
@@ -775,9 +800,9 @@ def main():
|
||||
if line:
|
||||
seen_viewkeys.append(line)
|
||||
except FileNotFoundError:
|
||||
print(f"警告: --seen-viewkeys-file 不存在: {args.seen_viewkeys_file}")
|
||||
print(f"警告: --seen-viewkeys-file 不存在: {args.seen_viewkeys_file}", file=cli_out)
|
||||
except Exception as e:
|
||||
print(f"警告: 读取 --seen-viewkeys-file 失败: {e}")
|
||||
print(f"警告: 读取 --seen-viewkeys-file 失败: {e}", file=cli_out)
|
||||
|
||||
# 决定运行模式
|
||||
if args.target_new is not None:
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Common Commands
|
||||
|
||||
### Full Local Development
|
||||
|
||||
```bash
|
||||
npm install
|
||||
./start.sh
|
||||
```
|
||||
|
||||
`./start.sh` starts the Go backend on `127.0.0.1:9192` and the frontend on `0.0.0.0:9191`. By default it builds the frontend and runs Vite preview mode. Use hot reload with:
|
||||
|
||||
```bash
|
||||
FRONTEND_MODE=dev ./start.sh --restart
|
||||
```
|
||||
|
||||
Useful variants:
|
||||
|
||||
```bash
|
||||
./start.sh --status
|
||||
./start.sh --restart
|
||||
./start.sh --stop
|
||||
```
|
||||
|
||||
### Frontend
|
||||
|
||||
Run from the repository root:
|
||||
|
||||
```bash
|
||||
npm run dev # Vite dev server; default port comes from Vite unless overridden
|
||||
npm run dev:raw # Vite dev server on 127.0.0.1:5173
|
||||
npm run build # tsc -b && vite build
|
||||
npm run preview # Vite preview; vite.config.ts uses port 9191
|
||||
npm run lint # TypeScript no-emit check
|
||||
npm test # node --import tsx --test tests/*.test.ts
|
||||
```
|
||||
|
||||
Run one frontend test:
|
||||
|
||||
```bash
|
||||
node --import tsx --test tests/previewIntent.test.ts
|
||||
```
|
||||
|
||||
### Backend
|
||||
|
||||
Run from `backend/` unless noted:
|
||||
|
||||
```bash
|
||||
go run ./cmd/server
|
||||
go test ./... -count=1
|
||||
go build -o video-server ./cmd/server
|
||||
```
|
||||
|
||||
Run one backend package or test:
|
||||
|
||||
```bash
|
||||
go test ./internal/scanner -count=1
|
||||
go test ./internal/scanner -run TestParse -count=1
|
||||
```
|
||||
|
||||
The backend requires Go 1.23+ and uses vendored dependencies in `backend/vendor/`, so keep `go mod vendor` in sync after dependency changes.
|
||||
|
||||
### Release and Deployment
|
||||
|
||||
```bash
|
||||
scripts/build-release.sh # builds Linux amd64/arm64 release tarballs into release/
|
||||
sudo bash install.sh # prebuilt installer flow used by README
|
||||
sudo bash deploy.sh # build from current checkout and install systemd services
|
||||
```
|
||||
|
||||
Docker uses the root `Dockerfile` and `docker-compose.yml`. The runtime image exposes port `9191` and stores persistent data under `/opt/video-site-91/data`.
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
This is a private video aggregation site with a React/Vite frontend and a Go backend.
|
||||
|
||||
### Frontend
|
||||
|
||||
The frontend is a React 18 SPA under `src/`. `src/main.tsx` mounts `BrowserRouter`, `ToastProvider`, and `AuthProvider`, then renders `src/App.tsx`. `App.tsx` defines the public app routes (`/`, `/list`, `/shorts`, `/upload`, `/video/:id`) and admin routes under `/admin`; both main-site and admin pages are wrapped in `RequireAuth`, while `/login` is public.
|
||||
|
||||
Frontend API calls are split by surface:
|
||||
|
||||
- `src/data/videos.ts` calls the main authenticated API under `/api` and upload/proxy-related endpoints.
|
||||
- `src/admin/api.ts` is the admin API client for `/admin/api`, always sending cookies and raising `UnauthorizedError` on `401`.
|
||||
|
||||
`vite.config.ts` proxies `/api`, `/p`, and `/admin/api` to `http://127.0.0.1:9192`, with frontend dev/preview served on port `9191` by default. The alias `@` maps to `src`.
|
||||
|
||||
Styling is plain CSS loaded from `src/main.tsx` in token/base/layout/navigation/search/video/admin layers. Shared UI lives in `src/components`, page-level screens in `src/pages`, and admin screens in `src/admin`.
|
||||
|
||||
### Backend
|
||||
|
||||
The backend entrypoint is `backend/cmd/server/main.go`. It loads `config.yaml` or `VIDEO_CONFIG`, creates the SQLite catalog and preview directories, builds the app state, registers API routes, starts the nightly runner, and then asynchronously attaches configured external drives so slow upstream login checks do not block port binding.
|
||||
|
||||
Important backend packages:
|
||||
|
||||
- `internal/config`: YAML config loading and first-run admin credential setup.
|
||||
- `internal/catalog`: SQLite catalog, schema migration, video metadata, settings, tags, drive records, generation status, and deduplication state. It opens SQLite with WAL and a busy timeout.
|
||||
- `internal/drives`: provider abstraction. Implementations include `quark`, `p115`, `pikpak`, `wopan`, `onedrive`, `localstorage`, `localupload`, and `spider91`.
|
||||
- `internal/scanner`: recursively lists drive directories, parses filenames/tags, upserts catalog videos, applies skip-directory rules, and enqueues newly discovered videos.
|
||||
- `internal/preview`: ffprobe/ffmpeg thumbnail and teaser generation workers. Generated assets are local files under the configured preview directory.
|
||||
- `internal/fingerprint`: asynchronous sampled SHA-256 worker used for cross-drive duplicate detection.
|
||||
- `internal/proxy`: `/p/*` media serving. Some providers redirect with `302` to signed CDN URLs, while providers requiring backend-held headers are reverse-proxied with Range support.
|
||||
- `internal/api`: main API and admin API route handlers.
|
||||
- `internal/nightly`: daily pipeline for drive scans, spider91 crawl, migration, queue drain, and duplicate asset cleanup.
|
||||
- `internal/spider91migrate`: migration from spider91 downloads to a configured cloud drive.
|
||||
|
||||
### Runtime Flow
|
||||
|
||||
1. Admin adds or edits drives through `/admin/drives`, which persists drive config in the catalog.
|
||||
2. The server attaches the drive implementation into the proxy registry and can trigger scans.
|
||||
3. Scans convert provider files into catalog video rows, parse titles/authors/tags from filenames, and queue preview/fingerprint work.
|
||||
4. The frontend lists videos through `/api/home`, `/api/list`, `/api/video/:id`, and streams media through `/p/*` endpoints.
|
||||
5. The nightly runner performs the scheduled end-to-end maintenance pipeline; admins can trigger it manually through `/admin/api/jobs/nightly/run`.
|
||||
|
||||
### Configuration and Data
|
||||
|
||||
Backend defaults come from `backend/config.example.yaml`. On first backend start, `config.yaml` is created automatically if missing. Default local development paths are:
|
||||
|
||||
- Backend listen address: `127.0.0.1:9192`
|
||||
- SQLite DB: `backend/data/video-site.db`
|
||||
- Generated previews/thumbs: `backend/data/previews`
|
||||
|
||||
Docker and installer deployments rewrite config paths so data lives under `/opt/video-site-91/data` or the mounted `./data` directory.
|
||||
|
||||
`VIDEO_FRONTEND_DIR` controls where the Go server looks for built frontend assets. If unset, it serves `./dist` when present. Backend routes (`/api`, `/admin/api`, `/p`) are excluded from the SPA fallback.
|
||||
|
||||
## Notes for Changes
|
||||
|
||||
- Main-site API routes and proxy routes require authentication; only login/setup and `/api/settings/theme` are intentionally public.
|
||||
- When adding a new drive provider, implement `internal/drives.Drive`, persist any needed config through catalog/admin APIs, attach it in `cmd/server`, and decide whether `/p/stream` should redirect or reverse-proxy in `internal/proxy`.
|
||||
- Generated thumbnails and teasers are local runtime assets; do not treat them as source files.
|
||||
- Frontend tests use Node's built-in test runner with `tsx`; TypeScript linting only checks `src` through the root `tsconfig.json`.
|
||||
@@ -36,10 +36,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
python3-bs4 \
|
||||
python3-lxml \
|
||||
python3-requests \
|
||||
python3-socks \
|
||||
tar \
|
||||
tzdata \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN python3 -c "import requests, bs4, lxml, socks"
|
||||
|
||||
WORKDIR /opt/video-site-91
|
||||
|
||||
COPY --from=backend /out/server ./server
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 nianzhibai
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -12,15 +12,16 @@
|
||||
<a href="#快速开始">快速开始</a> ·
|
||||
<a href="#功能特性">功能特性</a> ·
|
||||
<a href="#预览图">预览图</a> ·
|
||||
<a href="#数据存放位置">数据目录</a>
|
||||
<a href="#数据存放位置">数据目录</a> ·
|
||||
<a href="#许可证">许可证</a>
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
## 功能特性
|
||||
|
||||
- **多后端支持** — 兼容 115 云盘、PikPak 云盘、OneDrive、Google Drive 和本地存储
|
||||
- **低带宽播放** — 支持 302 的云盘可直连播放;Google Drive 等需鉴权直链的来源走后端代理
|
||||
- **多后端支持** — 兼容 115 云盘、PikPak 云盘、123云盘、OneDrive、Google Drive 和本地存储
|
||||
- **低带宽播放** — 115 云盘、PikPak 云盘、123云盘、OneDrive 都支持302模式,在线播放视频时,不占用服务器带宽,播放体验不受服务器带宽影响;Google Drive 不支持302模式,走服务器中转,观看体验会受服务器带宽影响
|
||||
- **封面 & 预览片段** — 自动为每个视频生成封面图和预览片段,首页快速选片
|
||||
- **91 爬虫** — 内置爬虫,支持抓取 91 本月最热视频
|
||||
- **双主题** — 黑黄经典主题 / 粉白清新主题,随时切换
|
||||
@@ -184,7 +185,6 @@ docker compose up -d # 更新并重启
|
||||
| 文档 | 内容 |
|
||||
|------|------|
|
||||
| [backend/README.md](backend/README.md) | 后端实现、接口说明、网盘字段 |
|
||||
| [video-site-implementation-plan.md](video-site-implementation-plan.md) | 完整实现方案 |
|
||||
|
||||
---
|
||||
|
||||
@@ -196,6 +196,12 @@ docker compose up -d # 更新并重启
|
||||
|
||||
---
|
||||
|
||||
## 许可证
|
||||
|
||||
本项目基于 [MIT License](LICENSE) 开源。
|
||||
|
||||
---
|
||||
|
||||
## 致谢
|
||||
|
||||
- [OpenList](https://github.com/OpenListTeam/OpenList) — 优秀的开源项目
|
||||
|
||||
+12
-12
@@ -3,7 +3,7 @@
|
||||
视频聚合站的 Go 后端。提供三件事:
|
||||
|
||||
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive / Google Drive / 本地存储)
|
||||
2. 视频元数据目录(SQLite)+ 扫描 + teaser 预生成
|
||||
2. 视频元数据目录(SQLite)+ 扫描 + 预览视频预生成
|
||||
3. REST API(前台)+ 管理后台 + 直链代理
|
||||
4. 标签池、视频隐藏、按网盘统计和详情页来源网盘类型展示能力
|
||||
|
||||
@@ -24,7 +24,7 @@ internal/
|
||||
googledrive/ Google Drive(OpenList 在线续期 + Google Drive API;播放走后端代理)
|
||||
localstorage/ 本地目录扫描(服务器已有视频目录)
|
||||
scanner/ 扫目录 → 落库
|
||||
preview/ ffmpeg 抽封面和生成多段 teaser
|
||||
preview/ ffmpeg 抽封面和生成多段预览视频
|
||||
proxy/ /p/stream/*、/p/preview/* 代理
|
||||
auth/ 管理员 session
|
||||
api/ REST 路由
|
||||
@@ -81,7 +81,7 @@ go run ./cmd/server 后端 9192
|
||||
|
||||
## 添加一个盘
|
||||
|
||||
推荐在前端管理后台 `/admin/drives` 新增网盘。保存后会立即挂载并触发扫描;视频结果可在 `/admin/videos` 按网盘查看,每页 100 条,页面会同时显示各网盘 Teaser 已生成、待生成、失败数量。
|
||||
推荐在前端管理后台 `/admin/drives` 新增网盘。保存后会立即挂载并触发扫描;视频结果可在 `/admin/videos` 按网盘查看,每页 100 条,页面会同时显示各网盘预览视频已生成、待生成、失败数量。
|
||||
|
||||
也可以直接调用后端接口:
|
||||
|
||||
@@ -109,7 +109,7 @@ go run ./cmd/server 后端 9192
|
||||
| pikpak | `username`、`password`(token、验证码和设备 ID 由服务端自动处理并保存) |
|
||||
| wopan | `access_token`、`refresh_token`,可选 `family_id` |
|
||||
| onedrive | `refresh_token` |
|
||||
| googledrive | `refresh_token` |
|
||||
| googledrive | 默认只需 `refresh_token`;自建 OAuth 客户端模式还需 `use_online_api=false`、`client_id`、`client_secret` |
|
||||
| localstorage | `path`(服务器上的已有视频目录,如 `/mnt/videos`) |
|
||||
|
||||
### PikPak 速度说明
|
||||
@@ -120,7 +120,7 @@ go run ./cmd/server 后端 9192
|
||||
|
||||
OneDrive 按 OpenList 默认应用方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。后台新建 OneDrive 时只需要填 OpenList 代刷得到的 `refresh_token`;服务端会默认挂载根目录并自动回写新 token。
|
||||
|
||||
Google Drive 按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/renewapi` 刷新 token。后台新建 Google Drive 时只需要填 OpenList Google Drive 获取到的 `refresh_token`。Google Drive 下载地址必须携带 `Authorization` 头,浏览器不能直接 302 使用,所以本站会由后端代理 `/p/stream` 播放,不加入零带宽 302 白名单。
|
||||
Google Drive 默认按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/renewapi` 刷新 token。后台新建 Google Drive 时只需要填 OpenList Google Drive 获取到的 `refresh_token`。如果不想依赖 OpenList 在线 API,可以关闭“使用 OpenList 在线续期 API”,并填写同一个 Google OAuth 客户端授权得到的 `refresh_token`、`client_id`、`client_secret`,服务端会直接请求 Google OAuth token 接口续期。Google Drive 下载地址必须携带 `Authorization` 头,浏览器不能直接 302 使用,所以本站会由后端代理 `/p/stream` 播放,不加入零带宽 302 白名单。91 爬虫迁移到 Google Drive 时使用 Google Drive resumable upload session 上传,并把上传文件的 MD5 写入 catalog 用于去重。
|
||||
|
||||
## 文件名约定
|
||||
|
||||
@@ -147,20 +147,20 @@ Google Drive 按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/ren
|
||||
2. 扫描时优先按网盘侧 `content_hash` 去重;没有 hash 时退化为 `file_name + size_bytes`。
|
||||
3. 扫描、爬虫、本地上传或服务启动挂载网盘后,后台指纹 worker 会异步读取视频的少量 Range 片段,生成 `sampled_sha256`。前台列表、首页、搜索、推荐会按 `size_bytes + sampled_sha256` 只展示最早入库的 canonical 视频。
|
||||
|
||||
`sampled_sha256` 是文件级去重:适合识别同一个视频文件被复制到 115 / PikPak / OneDrive 等不同网盘的情况。它不会删除任何网盘文件,也不用于识别转码、裁剪、加水印后的同源视频。
|
||||
`sampled_sha256` 是文件级去重:适合识别同一个视频文件被复制到 115 / PikPak / OneDrive / Google Drive 等不同网盘的情况。它不会删除任何网盘文件,也不用于识别转码、裁剪、加水印后的同源视频。
|
||||
|
||||
封面和 teaser 仍然优先生成,不等待指纹完成。夜间流水线最后会做一次重复资产清理:对 `size_bytes + sampled_sha256` 命中的非 canonical 视频,只删除本机生成的重复封面和 teaser,并把对应字段重置为 `pending`。网盘原文件和视频元数据记录不会被删除;如果 canonical 视频以后被移除,这些重复项会重新进入生成队列。
|
||||
封面和预览视频仍然优先生成,不等待指纹完成。夜间流水线最后会做一次重复资产清理:对 `size_bytes + sampled_sha256` 命中的非 canonical 视频,只删除本机生成的重复封面和预览视频,并把对应字段重置为 `pending`。网盘原文件和视频元数据记录不会被删除;如果 canonical 视频以后被移除,这些重复项会重新进入生成队列。
|
||||
|
||||
## 管理能力
|
||||
|
||||
- `/admin/drives`:新增、编辑、删除网盘,触发扫描。
|
||||
- `/admin/videos`:按网盘筛选视频,每页 100 条分页,查看各网盘 Teaser 统计,编辑标题/作者/分类/标签,单条或全量重生 teaser。
|
||||
- `/admin/videos`:按网盘筛选视频,每页 100 条分页,查看各网盘预览视频统计,编辑标题/作者/分类/标签,单条或全量重生预览视频。
|
||||
- `/admin/tags`:新增标签并用内置规则自动匹配已有视频;删除非系统标签时会从所有视频上同步移除该标签。
|
||||
- 播放页视频信息会展示来源网盘类型;同时提供“不再展示”,点击后会把视频标记为全局隐藏。隐藏视频不会再出现在首页、列表、搜索、相关推荐和详情接口中。目前没有管理后台恢复入口,如需恢复可把数据库里对应视频的 `hidden` 字段改回 `0`。
|
||||
|
||||
## Teaser 生成
|
||||
## 预览视频生成
|
||||
|
||||
scanner 扫到新视频会把 `(driveID, videoID)` 丢进 worker 队列。worker 会先用 `ffprobe` 探测时长,再用 `ffmpeg` 抽封面和生成无声 teaser:
|
||||
scanner 扫到新视频会把 `(driveID, videoID)` 丢进 worker 队列。worker 会先用 `ffprobe` 探测时长,再用 `ffmpeg` 抽封面和生成无声预览视频:
|
||||
|
||||
```
|
||||
ffmpeg -ss <起点> -headers "UA/Cookie/Referer" -i <直链> \
|
||||
@@ -168,9 +168,9 @@ ffmpeg -ss <起点> -headers "UA/Cookie/Referer" -i <直链> \
|
||||
-movflags +faststart -y <local>.mp4
|
||||
```
|
||||
|
||||
当前策略是每段固定 3 秒;30 秒以下最多 3 段,30 秒及以上固定 4 段;长视频在 20% 到 80% 区间均匀取段。生成的 teaser 和封面都只保存在本地 `data/previews/`,不会回写到网盘;旧数据中的 `preview_file_id` 会被忽略。
|
||||
当前策略是每段固定 3 秒;30 秒以下最多 3 段,30 秒及以上固定 4 段;长视频在 20% 到 80% 区间均匀取段。生成的预览视频和封面都只保存在本地 `data/previews/`,不会回写到网盘;旧数据中的 `preview_file_id` 会被忽略。
|
||||
|
||||
服务启动或网盘重新挂载时,如果 Teaser 开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 扫盘和直链生成 teaser / 封面时可能触发 Microsoft Graph 429、`TooManyRequests`、`activityLimitReached` 或 throttled 文本;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。扫盘阶段会按 `Retry-After` 或默认冷却时间等待后继续当前目录。
|
||||
服务启动或网盘重新挂载时,如果预览视频开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 扫盘和直链生成预览视频 / 封面时可能触发 Microsoft Graph 429、`TooManyRequests`、`activityLimitReached` 或 throttled 文本;Google Drive 可能返回 429、`usageLimits`、`userRateLimitExceeded`、`downloadQuotaExceeded` 等限制标识。后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。扫盘阶段会按 `Retry-After` 或默认冷却时间等待后继续当前目录。
|
||||
|
||||
前端卡片的 `previewSrc` 统一指向 `/p/preview/<videoID>`,后端只从本地 `preview_local` 文件读取。
|
||||
|
||||
|
||||
+1057
-158
File diff suppressed because it is too large
Load Diff
@@ -38,6 +38,7 @@ func TestSpider91IntCredFallbacks(t *testing.T) {
|
||||
func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
|
||||
reg := proxy.NewRegistry()
|
||||
reg.Set("p115-one", &spider91UploadTargetFakeDrive{id: "p115-one", kind: "p115"})
|
||||
reg.Set("p123-one", &spider91UploadTargetFakeDrive{id: "p123-one", kind: "p123"})
|
||||
reg.Set("onedrive-one", &spider91UploadTargetFakeDrive{id: "onedrive-one", kind: "onedrive"})
|
||||
|
||||
app := &App{registry: reg}
|
||||
@@ -50,6 +51,11 @@ func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
|
||||
t.Fatalf("explicit upload target = %q, want p115-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "p123-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "p123-one" {
|
||||
t.Fatalf("explicit p123 upload target = %q, want p123-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "onedrive-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "onedrive-one" {
|
||||
t.Fatalf("explicit onedrive upload target = %q, want onedrive-one", got)
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/config"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/fingerprint"
|
||||
"github.com/video-site/backend/internal/preview"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
@@ -224,6 +225,285 @@ func TestRegisterPreviewWorkersBackfillsHistoricalFingerprints(t *testing.T) {
|
||||
t.Fatalf("fingerprint status=%q sampled=%q, want ready with hash", got.FingerprintStatus, got.SampledSHA256)
|
||||
}
|
||||
|
||||
func TestStopDriveTasksCancelsQueuedTasksAndReplacesWorkers(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := &serverFakeDrive{}
|
||||
registry := proxy.NewRegistry()
|
||||
registry.Set("drive-id", drv)
|
||||
|
||||
gen := &serverFakeTeaserGenerator{}
|
||||
oldWorker := preview.NewWorker(gen, cat, drv)
|
||||
oldThumbWorker := preview.NewThumbWorker(gen, cat, drv)
|
||||
oldFingerprintWorker := fingerprint.NewWorker(cat, drv, fingerprint.Config{})
|
||||
oldCanceled := make(chan struct{})
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{},
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
workers: map[string]*preview.Worker{"drive-id": oldWorker},
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{"drive-id": oldThumbWorker},
|
||||
fingerprintWorkers: map[string]*fingerprint.Worker{"drive-id": oldFingerprintWorker},
|
||||
cancels: map[string]context.CancelFunc{
|
||||
"drive-id": func() { close(oldCanceled) },
|
||||
},
|
||||
scanQueued: map[string]bool{"drive-id": true},
|
||||
scanProgress: map[string]driveScanProgress{"drive-id": {Scanned: 8, Added: 2}},
|
||||
fingerprintQueueing: map[string]bool{"drive-id": true},
|
||||
}
|
||||
taskCtx, done := app.registerDriveTaskContext(ctx, "drive-id")
|
||||
defer done()
|
||||
|
||||
if !app.stopDriveTasks(ctx, "drive-id") {
|
||||
t.Fatal("stopDriveTasks returned false, want true")
|
||||
}
|
||||
select {
|
||||
case <-oldCanceled:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("old worker cancel was not called")
|
||||
}
|
||||
if err := taskCtx.Err(); err == nil {
|
||||
t.Fatal("registered drive task context was not canceled")
|
||||
}
|
||||
if app.scanQueued["drive-id"] {
|
||||
t.Fatal("scan queue marker was not cleared")
|
||||
}
|
||||
if _, ok := app.scanProgress["drive-id"]; ok {
|
||||
t.Fatal("scan progress marker was not cleared")
|
||||
}
|
||||
if app.fingerprintQueueing["drive-id"] {
|
||||
t.Fatal("fingerprint queue marker was not cleared")
|
||||
}
|
||||
|
||||
app.mu.Lock()
|
||||
newWorker := app.workers["drive-id"]
|
||||
newThumbWorker := app.thumbWorkers["drive-id"]
|
||||
newFingerprintWorker := app.fingerprintWorkers["drive-id"]
|
||||
newCancel := app.cancels["drive-id"]
|
||||
app.mu.Unlock()
|
||||
if newWorker == nil || newWorker == oldWorker {
|
||||
t.Fatalf("preview worker was not replaced")
|
||||
}
|
||||
if newThumbWorker == nil || newThumbWorker == oldThumbWorker {
|
||||
t.Fatalf("thumb worker was not replaced")
|
||||
}
|
||||
if newFingerprintWorker == nil || newFingerprintWorker == oldFingerprintWorker {
|
||||
t.Fatalf("fingerprint worker was not replaced")
|
||||
}
|
||||
if newCancel == nil {
|
||||
t.Fatalf("replacement worker cancel was not registered")
|
||||
}
|
||||
newCancel()
|
||||
}
|
||||
|
||||
func TestScheduleScanRejectsDriveWithActiveGenerationWork(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
thumbWorker := preview.NewThumbWorker(&serverFakeTeaserGenerator{}, nil, &serverFakeDrive{})
|
||||
if !thumbWorker.Enqueue(&catalog.Video{ID: "busy-video", DriveID: "drive-id", Title: "Busy Video"}) {
|
||||
t.Fatal("failed to enqueue busy thumbnail task")
|
||||
}
|
||||
app := &App{
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{"drive-id": thumbWorker},
|
||||
}
|
||||
|
||||
if app.scheduleScan(ctx, "drive-id") {
|
||||
t.Fatal("scheduleScan accepted a drive with active generation work")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleScanRunsDifferentDrivesConcurrently(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
seedDriveWithTeaser(t, cat, "drive-a", true)
|
||||
seedDriveWithTeaser(t, cat, "drive-b", true)
|
||||
|
||||
started := make(chan string, 2)
|
||||
release := make(chan struct{})
|
||||
registry := proxy.NewRegistry()
|
||||
registry.Set("drive-a", &serverBlockingListDrive{id: "drive-a", started: started, release: release})
|
||||
registry.Set("drive-b", &serverBlockingListDrive{id: "drive-b", started: started, release: release})
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{
|
||||
Scanner: config.Scanner{VideoExtensions: []string{".mp4"}},
|
||||
},
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
}
|
||||
|
||||
if !app.scheduleScan(ctx, "drive-a") {
|
||||
t.Fatal("scheduleScan drive-a was rejected")
|
||||
}
|
||||
if !app.scheduleScan(ctx, "drive-b") {
|
||||
t.Fatal("scheduleScan drive-b was rejected")
|
||||
}
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
deadline := time.After(time.Second)
|
||||
for len(seen) < 2 {
|
||||
select {
|
||||
case id := <-started:
|
||||
seen[id] = struct{}{}
|
||||
case <-deadline:
|
||||
close(release)
|
||||
t.Fatalf("started drives = %#v, want both drives before releasing List", seen)
|
||||
}
|
||||
}
|
||||
close(release)
|
||||
}
|
||||
|
||||
func TestDriveGenerationStatusIncludesScanState(t *testing.T) {
|
||||
app := &App{
|
||||
scanQueued: map[string]bool{"drive-id": true},
|
||||
scanProgress: map[string]driveScanProgress{"drive-id": {Scanned: 12, Added: 3}},
|
||||
}
|
||||
|
||||
status := app.driveGenerationStatuses()["drive-id"].Scan
|
||||
if status.State != "scanning" {
|
||||
t.Fatalf("scan status = %#v, want scanning", status)
|
||||
}
|
||||
if status.ScannedCount != 12 || status.AddedCount != 3 {
|
||||
t.Fatalf("scan counts = scanned %d added %d, want 12 and 3", status.ScannedCount, status.AddedCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSpider91MigrationAfterManualCrawlRequiresConfiguredUploadTarget(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
registry := proxy.NewRegistry()
|
||||
migrator := &serverFakeSpider91MigrationRunner{}
|
||||
app := &App{
|
||||
registry: registry,
|
||||
spider91Migrator: migrator,
|
||||
workers: map[string]*preview.Worker{},
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{},
|
||||
fingerprintWorkers: map[string]*fingerprint.Worker{},
|
||||
}
|
||||
|
||||
app.runSpider91MigrationAfterManualCrawl(ctx, "91spider")
|
||||
if migrator.called != 0 {
|
||||
t.Fatalf("migration called without upload target")
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "pikpak"
|
||||
app.runSpider91MigrationAfterManualCrawl(ctx, "91spider")
|
||||
if migrator.called != 0 {
|
||||
t.Fatalf("migration called when upload target is not attached")
|
||||
}
|
||||
|
||||
registry.Set("pikpak", &serverFakeKindDrive{id: "pikpak", kind: "pikpak"})
|
||||
app.runSpider91MigrationAfterManualCrawl(ctx, "91spider")
|
||||
if migrator.called != 1 {
|
||||
t.Fatalf("migration calls = %d, want 1", migrator.called)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriveGenerationStatusUsesWorkerQueueNotPendingCatalogRows(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "pending-thumb",
|
||||
DriveID: "drive-id",
|
||||
FileID: "file-id",
|
||||
Title: "Pending Thumb",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoMeta(ctx, "pending-thumb", catalog.VideoMetaPatch{ThumbnailStatus: "pending"}); err != nil {
|
||||
t.Fatalf("mark thumbnail pending: %v", err)
|
||||
}
|
||||
|
||||
thumbWorker := preview.NewThumbWorker(&serverFakeTeaserGenerator{}, cat, &serverFakeDrive{})
|
||||
app := &App{
|
||||
cat: cat,
|
||||
workers: map[string]*preview.Worker{},
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{"drive-id": thumbWorker},
|
||||
fingerprintWorkers: map[string]*fingerprint.Worker{},
|
||||
}
|
||||
|
||||
status := app.driveGenerationStatuses()["drive-id"].Thumbnail
|
||||
if status.State != "idle" || status.QueueLength != 0 {
|
||||
t.Fatalf("thumbnail status = %#v, want idle with empty worker queue", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegenFailedThumbnailsQueuesPendingRowsAfterStop(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "pending-thumb",
|
||||
DriveID: "drive-id",
|
||||
FileID: "file-id",
|
||||
Title: "Pending Thumb",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoMeta(ctx, "pending-thumb", catalog.VideoMetaPatch{ThumbnailStatus: "pending"}); err != nil {
|
||||
t.Fatalf("mark thumbnail pending: %v", err)
|
||||
}
|
||||
|
||||
thumbWorker := preview.NewThumbWorker(&serverFakeTeaserGenerator{}, cat, &serverFakeDrive{})
|
||||
app := &App{
|
||||
cat: cat,
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{"drive-id": thumbWorker},
|
||||
}
|
||||
|
||||
app.regenFailedThumbnails(ctx, "drive-id")
|
||||
|
||||
if got := thumbWorker.Status().QueueLength; got != 1 {
|
||||
t.Fatalf("thumb queue length = %d, want pending row re-enqueued", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunScanStartsFingerprintBeforeThumbnailAndPreviewDrain(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
@@ -671,6 +951,480 @@ func TestCleanupMissingPikPakVideosRemovesDatabaseRowsAndLocalAssets(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupDriveVideosForDeleteRemovesRowsAndGeneratedAssetsOnly(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
localDir := filepath.Join(root, "previews")
|
||||
originalDir := filepath.Join(root, "local-videos")
|
||||
originalVideo := filepath.Join(originalDir, "clip.mp4")
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, path := range []string{originalVideo} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("original"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "local-main",
|
||||
Kind: "localstorage",
|
||||
Name: "Local",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{"path": originalDir},
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
previewPath := filepath.Join(localDir, "localstorage-local-main-file.mp4")
|
||||
thumbPath := filepath.Join(localDir, "thumbs", "localstorage-local-main-file.jpg")
|
||||
for _, path := range []string{previewPath, thumbPath} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("generated"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "localstorage-local-main-file",
|
||||
DriveID: "local-main",
|
||||
FileID: "encoded-local-file",
|
||||
Title: "Local File",
|
||||
PreviewLocal: previewPath,
|
||||
PreviewStatus: "ready",
|
||||
ThumbnailURL: "/p/thumb/localstorage-local-main-file",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed local video: %v", err)
|
||||
}
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "pikpak-other",
|
||||
DriveID: "PikPak",
|
||||
FileID: "other",
|
||||
Title: "Other",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed other video: %v", err)
|
||||
}
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
registry: proxy.NewRegistry(),
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
fingerprintWorkers: make(map[string]*fingerprint.Worker),
|
||||
spider91Crawlers: make(map[string]*spider91.Crawler),
|
||||
}
|
||||
removed, err := app.cleanupDriveVideosForDelete(ctx, "local-main")
|
||||
if err != nil {
|
||||
t.Fatalf("cleanup drive videos: %v", err)
|
||||
}
|
||||
if removed != 1 {
|
||||
t.Fatalf("removed = %d, want 1", removed)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "localstorage-local-main-file"); err != sql.ErrNoRows {
|
||||
t.Fatalf("deleted video lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "pikpak-other"); err != nil {
|
||||
t.Fatalf("other drive video missing: %v", err)
|
||||
}
|
||||
for _, path := range []string{previewPath, thumbPath} {
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("generated asset %s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(originalVideo); err != nil {
|
||||
t.Fatalf("original local video should remain, stat err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteVideoRemovesGeneratedAssetsKeepsLocalOriginalAndTombstones(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
localDir := filepath.Join(root, "previews")
|
||||
originalDir := filepath.Join(root, "local-videos")
|
||||
originalVideo := filepath.Join(originalDir, "clip.mp4")
|
||||
if err := os.MkdirAll(originalDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir original dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(originalVideo, []byte("original"), 0o644); err != nil {
|
||||
t.Fatalf("write original: %v", err)
|
||||
}
|
||||
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "local-main",
|
||||
Kind: "localstorage",
|
||||
Name: "Local",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{"path": originalDir},
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
previewPath := filepath.Join(localDir, "localstorage-local-main-file.mp4")
|
||||
thumbPath := filepath.Join(localDir, "thumbs", "localstorage-local-main-file.jpg")
|
||||
for _, path := range []string{previewPath, thumbPath} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("generated"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "localstorage-local-main-file",
|
||||
DriveID: "local-main",
|
||||
FileID: "file",
|
||||
FileName: "clip.mp4",
|
||||
SampledSHA256: "sampled",
|
||||
FingerprintStatus: "ready",
|
||||
Title: "Local File",
|
||||
PreviewLocal: previewPath,
|
||||
PreviewStatus: "ready",
|
||||
ThumbnailURL: "/p/thumb/localstorage-local-main-file",
|
||||
Size: 123,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
}
|
||||
result, err := app.deleteVideo(ctx, "localstorage-local-main-file")
|
||||
if err != nil {
|
||||
t.Fatalf("delete video: %v", err)
|
||||
}
|
||||
if !result.OK || result.DeletedSource {
|
||||
t.Fatalf("delete result = %#v, want ok without source deletion", result)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "localstorage-local-main-file"); err != sql.ErrNoRows {
|
||||
t.Fatalf("deleted video lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
deleted, err := cat.IsDeletedVideoCandidate(ctx, "localstorage-local-main-file", "local-main", "file", "", "clip.mp4", 123)
|
||||
if err != nil {
|
||||
t.Fatalf("check tombstone: %v", err)
|
||||
}
|
||||
if !deleted {
|
||||
t.Fatal("deleted video tombstone missing")
|
||||
}
|
||||
for _, path := range []string{previewPath, thumbPath} {
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("generated asset %s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(originalVideo); err != nil {
|
||||
t.Fatalf("original local video was removed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteVideoRemovesSpider91SourceFile(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
localDir := filepath.Join(root, "previews")
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "spider-main",
|
||||
Kind: spider91.Kind,
|
||||
Name: "Spider",
|
||||
RootID: "/",
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
}
|
||||
sourceDir := app.spider91DriveDir("spider-main")
|
||||
sourceVideo := filepath.Join(sourceDir, "videos", "source.mp4")
|
||||
sourceThumb := filepath.Join(sourceDir, "thumbs", "source.jpg")
|
||||
previewPath := filepath.Join(localDir, "spider91-spider-main-source.mp4")
|
||||
commonThumb := filepath.Join(localDir, "thumbs", "spider91-spider-main-source.jpg")
|
||||
for _, path := range []string{sourceVideo, sourceThumb, previewPath, commonThumb} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("file"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "spider91-spider-main-source",
|
||||
DriveID: "spider-main",
|
||||
FileID: "source.mp4",
|
||||
FileName: "source.mp4",
|
||||
Ext: "mp4",
|
||||
Title: "Spider Source",
|
||||
PreviewLocal: previewPath,
|
||||
PreviewStatus: "ready",
|
||||
ThumbnailURL: "/p/thumb/spider91-spider-main-source",
|
||||
Size: 456,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
result, err := app.deleteVideo(ctx, "spider91-spider-main-source")
|
||||
if err != nil {
|
||||
t.Fatalf("delete spider video: %v", err)
|
||||
}
|
||||
if !result.OK || !result.DeletedSource {
|
||||
t.Fatalf("delete result = %#v, want source deleted", result)
|
||||
}
|
||||
for _, path := range []string{sourceVideo, sourceThumb, previewPath, commonThumb} {
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("deleted file %s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "spider91-spider-main-source"); err != sql.ErrNoRows {
|
||||
t.Fatalf("deleted video lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
deleted, err := cat.IsVideoDeleted(ctx, "spider91-spider-main-source")
|
||||
if err != nil {
|
||||
t.Fatalf("check tombstone: %v", err)
|
||||
}
|
||||
if !deleted {
|
||||
t.Fatal("deleted spider91 video tombstone missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupDriveVideosForDeleteSpider91RemovesCrawledDirAndOriginRecords(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
localDir := filepath.Join(root, "previews")
|
||||
driveID := "spider-main"
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: driveID,
|
||||
Kind: "spider91",
|
||||
Name: "91 Spider",
|
||||
RootID: "/",
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed spider91 drive: %v", err)
|
||||
}
|
||||
|
||||
spiderDriveDir := filepath.Join(root, "spider91", driveID)
|
||||
sourceVideo := filepath.Join(spiderDriveDir, "videos", "source.mp4")
|
||||
sourceThumb := filepath.Join(spiderDriveDir, "thumbs", "source.jpg")
|
||||
localPreview := filepath.Join(localDir, "spider91-spider-main-source.mp4")
|
||||
localThumb := filepath.Join(localDir, "thumbs", "spider91-spider-main-source.jpg")
|
||||
migratedPreview := filepath.Join(localDir, "spider91-spider-main-migrated.mp4")
|
||||
migratedThumb := filepath.Join(localDir, "thumbs", "spider91-spider-main-migrated.jpg")
|
||||
for _, path := range []string{sourceVideo, sourceThumb, localPreview, localThumb, migratedPreview, migratedThumb} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("asset"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*catalog.Video{
|
||||
{
|
||||
ID: "spider91-spider-main-source",
|
||||
DriveID: driveID,
|
||||
FileID: "source.mp4",
|
||||
Title: "Source",
|
||||
PreviewLocal: localPreview,
|
||||
PreviewStatus: "ready",
|
||||
ThumbnailURL: "/p/thumb/spider91-spider-main-source",
|
||||
},
|
||||
{
|
||||
ID: "spider91-spider-main-migrated",
|
||||
DriveID: "PikPak",
|
||||
FileID: "pikpak-file-id",
|
||||
Title: "Migrated",
|
||||
PreviewLocal: migratedPreview,
|
||||
PreviewStatus: "ready",
|
||||
ThumbnailURL: "/p/thumb/spider91-spider-main-migrated",
|
||||
},
|
||||
{
|
||||
ID: "pikpak-PikPak-other",
|
||||
DriveID: "PikPak",
|
||||
FileID: "other",
|
||||
Title: "Other",
|
||||
PreviewStatus: "ready",
|
||||
},
|
||||
} {
|
||||
v.PublishedAt = now
|
||||
v.CreatedAt = now
|
||||
v.UpdatedAt = now
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed video %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
registry: proxy.NewRegistry(),
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
fingerprintWorkers: make(map[string]*fingerprint.Worker),
|
||||
spider91Crawlers: make(map[string]*spider91.Crawler),
|
||||
}
|
||||
removed, err := app.cleanupDriveVideosForDelete(ctx, driveID)
|
||||
if err != nil {
|
||||
t.Fatalf("cleanup spider91 videos: %v", err)
|
||||
}
|
||||
if removed != 2 {
|
||||
t.Fatalf("removed = %d, want 2", removed)
|
||||
}
|
||||
for _, id := range []string{"spider91-spider-main-source", "spider91-spider-main-migrated"} {
|
||||
if _, err := cat.GetVideo(ctx, id); err != sql.ErrNoRows {
|
||||
t.Fatalf("%s lookup error = %v, want sql.ErrNoRows", id, err)
|
||||
}
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "pikpak-PikPak-other"); err != nil {
|
||||
t.Fatalf("unrelated pikpak video missing: %v", err)
|
||||
}
|
||||
for _, path := range []string{spiderDriveDir, localPreview, localThumb, migratedPreview, migratedThumb} {
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("%s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupOrphanDriveVideosRemovesRowsAndGeneratedAssets(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
localDir := filepath.Join(root, "previews")
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "active-drive",
|
||||
Kind: "pikpak",
|
||||
Name: "Active",
|
||||
RootID: "root",
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed active drive: %v", err)
|
||||
}
|
||||
|
||||
previewPath := filepath.Join(localDir, "p123-123-orphan.mp4")
|
||||
thumbPath := filepath.Join(localDir, "thumbs", "p123-123-orphan.jpg")
|
||||
for _, path := range []string{previewPath, thumbPath} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("generated"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*catalog.Video{
|
||||
{
|
||||
ID: "p123-123-orphan",
|
||||
DriveID: "123",
|
||||
FileID: "orphan-file",
|
||||
Title: "Orphan",
|
||||
PreviewLocal: previewPath,
|
||||
PreviewStatus: "ready",
|
||||
ThumbnailURL: "/p/thumb/p123-123-orphan",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "pikpak-active",
|
||||
DriveID: "active-drive",
|
||||
FileID: "active-file",
|
||||
Title: "Active",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed video %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
}
|
||||
removed, err := app.cleanupOrphanDriveVideos(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("cleanup orphan videos: %v", err)
|
||||
}
|
||||
if removed != 1 {
|
||||
t.Fatalf("removed = %d, want 1", removed)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "p123-123-orphan"); err != sql.ErrNoRows {
|
||||
t.Fatalf("orphan video lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "pikpak-active"); err != nil {
|
||||
t.Fatalf("active video missing: %v", err)
|
||||
}
|
||||
for _, path := range []string{previewPath, thumbPath} {
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("orphan asset %s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupDuplicateVideoAssetsRemovesOnlyDuplicateLocalAssets(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
localDir := t.TempDir()
|
||||
@@ -857,6 +1611,63 @@ func (d *serverFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
}
|
||||
func (d *serverFakeDrive) RootID() string { return "root" }
|
||||
|
||||
type serverFakeKindDrive struct {
|
||||
serverFakeDrive
|
||||
id string
|
||||
kind string
|
||||
}
|
||||
|
||||
func (d *serverFakeKindDrive) Kind() string { return d.kind }
|
||||
func (d *serverFakeKindDrive) ID() string { return d.id }
|
||||
|
||||
type serverFakeSpider91MigrationRunner struct {
|
||||
called int
|
||||
}
|
||||
|
||||
func (r *serverFakeSpider91MigrationRunner) RunOnce(context.Context) error {
|
||||
r.called++
|
||||
return nil
|
||||
}
|
||||
|
||||
type serverBlockingListDrive struct {
|
||||
id string
|
||||
started chan string
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (d *serverBlockingListDrive) Kind() string { return "fake" }
|
||||
func (d *serverBlockingListDrive) ID() string { return d.id }
|
||||
func (d *serverBlockingListDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *serverBlockingListDrive) List(ctx context.Context, _ string) ([]drives.Entry, error) {
|
||||
if d.started != nil {
|
||||
select {
|
||||
case d.started <- d.id:
|
||||
default:
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-d.release:
|
||||
return nil, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
func (d *serverBlockingListDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *serverBlockingListDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
|
||||
return &drives.StreamLink{URL: "https://video.example/clip.mp4"}, nil
|
||||
}
|
||||
func (d *serverBlockingListDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *serverBlockingListDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *serverBlockingListDrive) RootID() string { return "root" }
|
||||
|
||||
type serverFingerprintFakeDrive struct {
|
||||
serverFakeDrive
|
||||
path string
|
||||
|
||||
@@ -22,7 +22,7 @@ server:
|
||||
storage:
|
||||
# SQLite 数据库文件路径
|
||||
db_path: "./data/video-site.db"
|
||||
# 本地 teaser 和封面目录
|
||||
# 本地预览视频和封面目录
|
||||
local_preview_dir: "./data/previews"
|
||||
|
||||
scanner:
|
||||
@@ -33,33 +33,33 @@ scanner:
|
||||
# 单次扫描每家网盘目录递归层数上限
|
||||
max_depth: 5
|
||||
# 被扫描的扩展名
|
||||
video_extensions: [".mp4", ".mkv", ".mov", ".webm", ".avi"]
|
||||
video_extensions: [".mp4", ".mkv", ".mov", ".webm", ".avi", ".strm"]
|
||||
|
||||
nightly:
|
||||
# 凌晨流水线触发整点(0-23),默认 1 即每天 01:00。流程:
|
||||
# Phase 1 扫所有非 spider91 / 非 localupload 网盘 → 检测新增 / 删除
|
||||
# → 入队封面和 teaser → 等所有队列 idle
|
||||
# Phase 2 spider91 爬虫(如配置)→ 入队 teaser → 等队列 idle
|
||||
# → 入队封面和预览视频 → 等所有队列 idle
|
||||
# Phase 2 spider91 爬虫(如配置)→ 入队预览视频 → 等队列 idle
|
||||
# Phase 3 spider91 → 云盘迁移(一次性 sweep)
|
||||
cron_hour: 1
|
||||
# 单次流水线总耗时上限(软超时);超过后当前 phase 跑完不启动后续 phase。
|
||||
max_duration: 6h
|
||||
|
||||
preview:
|
||||
# 是否启用 ffmpeg 抽帧生成 teaser
|
||||
# 是否启用 ffmpeg 抽帧生成预览视频
|
||||
enabled: true
|
||||
# ffmpeg / ffprobe 可执行文件名或绝对路径
|
||||
ffmpeg_path: "ffmpeg"
|
||||
ffprobe_path: "ffprobe"
|
||||
# teaser 每段时长(秒),实际生成时每段最多 3 秒
|
||||
# 预览视频每段时长(秒),实际生成时每段最多 3 秒
|
||||
duration_seconds: 3
|
||||
# 兼容旧配置;当前 30 秒以下最多 3 段,30 秒及以上固定 4 段
|
||||
segments: 3
|
||||
# teaser 视频宽度
|
||||
# 预览视频宽度
|
||||
width: 480
|
||||
|
||||
# 盘列表。上线后请通过管理后台添加,本文件可留空。
|
||||
# kind 支持 quark / p115 / pikpak / wopan / onedrive / googledrive / localstorage。
|
||||
# kind 支持 quark / p115 / p123 / pikpak / wopan / onedrive / googledrive / localstorage。
|
||||
# OneDrive 示例:
|
||||
# - id: "my-onedrive"
|
||||
# kind: "onedrive"
|
||||
@@ -74,11 +74,18 @@ preview:
|
||||
# root_id: "root"
|
||||
# params:
|
||||
# refresh_token: "..."
|
||||
# # 默认 use_online_api=true,会使用 OpenList 在线续期 API。
|
||||
# # 如需使用自己创建的 Google OAuth 客户端,取消下面三行注释:
|
||||
# # use_online_api: "false"
|
||||
# # client_id: "..."
|
||||
# # client_secret: "..."
|
||||
# 本地存储示例:
|
||||
# - id: "local-media"
|
||||
# kind: "localstorage"
|
||||
# name: "本地视频目录"
|
||||
# root_id: "/"
|
||||
# params:
|
||||
# # Docker 部署时这里和 .strm 里的绝对路径都必须使用容器内路径。
|
||||
# # 例如宿主机 /mnt/videos 挂载为 /media,就填写 /media。
|
||||
# path: "/mnt/videos"
|
||||
drives: []
|
||||
|
||||
+4
-4
@@ -7,15 +7,18 @@ toolchain go1.23.4
|
||||
require (
|
||||
github.com/OpenListTeam/wopan-sdk-go v0.2.0
|
||||
github.com/SheltonZhu/115driver v1.3.2
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
|
||||
github.com/go-chi/chi/v5 v5.1.0
|
||||
github.com/go-resty/resty/v2 v2.14.0
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
|
||||
golang.org/x/net v0.27.0
|
||||
golang.org/x/sys v0.30.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
modernc.org/sqlite v1.33.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aead/ecdh v0.2.0 // indirect
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect
|
||||
github.com/andreburgaud/crypt2go v1.1.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
@@ -26,10 +29,7 @@ require (
|
||||
github.com/pierrec/lz4/v4 v4.1.17 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
|
||||
golang.org/x/crypto v0.25.0 // indirect
|
||||
golang.org/x/net v0.27.0 // indirect
|
||||
golang.org/x/sys v0.30.0 // indirect
|
||||
golang.org/x/time v0.8.0 // indirect
|
||||
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
|
||||
modernc.org/libc v1.55.3 // indirect
|
||||
|
||||
+303
-18
@@ -6,7 +6,9 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -16,6 +18,7 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/auth"
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives/p123"
|
||||
)
|
||||
|
||||
type AdminServer struct {
|
||||
@@ -36,25 +39,30 @@ type AdminServer struct {
|
||||
SetupRequired func() bool
|
||||
// OnSetup 持久化首次部署时设置的管理员账号密码,并更新运行中认证器。
|
||||
OnSetup func(username, password string) error
|
||||
// LocalPreviewDir is the local directory that stores generated teasers and thumbs.
|
||||
// LocalPreviewDir is the local directory that stores generated preview videos and thumbs.
|
||||
LocalPreviewDir string
|
||||
// Hooks:外层注入实际执行者
|
||||
OnDriveSaved func(driveID string) error
|
||||
OnDriveDeleteCleanup func(ctx context.Context, driveID string) (int, error)
|
||||
OnDriveRemoved func(driveID string)
|
||||
OnScanRequested func(driveID string)
|
||||
OnScanRequested func(driveID string) bool
|
||||
OnStopDriveTasks func(driveID string) bool
|
||||
OnStopAllTasks func() int
|
||||
OnRegenPreview func(videoID string)
|
||||
OnRegenAllPreviews func()
|
||||
OnRegenFailedPreviews func(driveID string)
|
||||
OnRegenFailedThumbnails func(driveID string)
|
||||
OnRegenFailedFingerprints func(driveID string)
|
||||
OnDeleteVideo func(ctx context.Context, videoID string) (DeleteVideoResult, error)
|
||||
GetDriveGenerationStatuses func() map[string]DriveGenerationStatuses
|
||||
// OnTeaserEnabledChanged 在 per-drive teaser 开关被切换后调用。
|
||||
// enabled=true 时上层应该重新把 pending teaser 入队(类似旧的全局开关从关到开);
|
||||
// OnTeaserEnabledChanged 在 per-drive 预览视频开关被切换后调用。
|
||||
// enabled=true 时上层应该重新把 pending 预览视频入队(类似旧的全局开关从关到开);
|
||||
// enabled=false 时通常不用做事 —— worker 入队前会再次查 catalog,自然停止。
|
||||
OnTeaserEnabledChanged func(driveID string, enabled bool)
|
||||
// Theme 读写("dark" | "pink")
|
||||
GetTheme func() string
|
||||
SetTheme func(theme string) error
|
||||
// Spider91 → 115/PikPak 上传目标 drive ID 读写
|
||||
// Spider91 → 115/123/PikPak/OneDrive 上传目标 drive ID 读写
|
||||
GetSpider91UploadDriveID func() string
|
||||
SetSpider91UploadDriveID func(driveID string) error
|
||||
// OnRunNightlyJob 触发一次完整的凌晨流水线(Phase1 扫盘 + Phase2 91 爬虫 +
|
||||
@@ -68,8 +76,16 @@ type AdminServer struct {
|
||||
// 用于"设置跳过目录"弹窗按需展开浏览网盘目录树;只返回目录条目,文件忽略。
|
||||
// 调用方应当处理 error 并以 5xx 返回前端。
|
||||
ListDriveDirChildren func(ctx context.Context, driveID, parentID string) ([]DriveDirEntry, error)
|
||||
// 123 云盘扫码登录接口测试注入;生产留空走官方 user.123pan.cn。
|
||||
P123UserAPIBaseURL string
|
||||
P123HTTPClient *http.Client
|
||||
}
|
||||
|
||||
const (
|
||||
driveTaskBusyMessage = "当前存储有正在进行的任务,请稍后重试"
|
||||
fullScanBusyMessage = "当前有全量扫描任务正在进行,请稍后重试"
|
||||
)
|
||||
|
||||
// DriveDirEntry 是 dirtree 接口的一条返回项:网盘上的一个目录节点。
|
||||
type DriveDirEntry struct {
|
||||
ID string `json:"id"`
|
||||
@@ -81,9 +97,12 @@ type GenerationStatus struct {
|
||||
CurrentTitle string `json:"currentTitle,omitempty"`
|
||||
QueueLength int `json:"queueLength"`
|
||||
CooldownUntil string `json:"cooldownUntil,omitempty"`
|
||||
ScannedCount int `json:"scannedCount"`
|
||||
AddedCount int `json:"addedCount"`
|
||||
}
|
||||
|
||||
type DriveGenerationStatuses struct {
|
||||
Scan GenerationStatus `json:"scan"`
|
||||
Thumbnail GenerationStatus `json:"thumbnail"`
|
||||
Preview GenerationStatus `json:"preview"`
|
||||
Fingerprint GenerationStatus `json:"fingerprint"`
|
||||
@@ -97,6 +116,11 @@ type NightlyJobStatus struct {
|
||||
LastFinishedAt string `json:"lastFinishedAt,omitempty"`
|
||||
}
|
||||
|
||||
type DeleteVideoResult struct {
|
||||
OK bool `json:"ok"`
|
||||
DeletedSource bool `json:"deletedSource"`
|
||||
}
|
||||
|
||||
func (a *AdminServer) Register(r chi.Router) {
|
||||
r.Route("/admin/api", func(r chi.Router) {
|
||||
// 登录、登出和首次部署初始化不需要鉴权
|
||||
@@ -114,17 +138,22 @@ func (a *AdminServer) Register(r chi.Router) {
|
||||
r.Get("/drives", a.handleListDrives)
|
||||
r.Get("/drives/storage", a.handleDriveStorage)
|
||||
r.Post("/drives", a.handleUpsertDrive)
|
||||
r.Post("/drives/p123/qr", a.handleP123QRStart)
|
||||
r.Get("/drives/p123/qr/{uniID}", a.handleP123QRStatus)
|
||||
r.Delete("/drives/{id}", a.handleDeleteDrive)
|
||||
r.Post("/drives/{id}/rescan", a.handleRescan)
|
||||
r.Post("/drives/{id}/tasks/stop", a.handleStopDriveTasks)
|
||||
r.Post("/drives/{id}/teaser-enabled", a.handleSetDriveTeaserEnabled)
|
||||
r.Post("/drives/{id}/skip-dirs", a.handleSetDriveSkipDirs)
|
||||
r.Get("/drives/{id}/dirtree", a.handleListDriveDirTree)
|
||||
r.Post("/drives/{id}/previews/failed/regenerate", a.handleRegenFailedPreviews)
|
||||
r.Post("/drives/{id}/thumbnails/failed/regenerate", a.handleRegenFailedThumbnails)
|
||||
r.Post("/drives/{id}/fingerprints/failed/regenerate", a.handleRegenFailedFingerprints)
|
||||
|
||||
// 视频
|
||||
r.Get("/videos", a.handleAdminListVideos)
|
||||
r.Put("/videos/{id}", a.handleUpdateVideo)
|
||||
r.Delete("/videos/{id}", a.handleDeleteVideo)
|
||||
r.Post("/videos/regen-preview", a.handleRegenAllPreviews)
|
||||
r.Post("/videos/{id}/regen-preview", a.handleRegenPreview)
|
||||
|
||||
@@ -141,6 +170,7 @@ func (a *AdminServer) Register(r chi.Router) {
|
||||
r.Get("/update/check", a.handleCheckUpdate)
|
||||
r.Get("/jobs/nightly/status", a.handleNightlyJobStatus)
|
||||
r.Post("/jobs/nightly/run", a.handleRunNightlyJob)
|
||||
r.Post("/tasks/stop", a.handleStopAllTasks)
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -385,7 +415,7 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
Status string `json:"status"`
|
||||
LastError string `json:"lastError,omitempty"`
|
||||
HasCredential bool `json:"hasCredential"`
|
||||
// TeaserEnabled 控制是否给本盘生成 teaser/封面。前端用它在网盘列表/编辑表单展示开关状态。
|
||||
// TeaserEnabled 控制是否给本盘生成预览视频/封面。前端用它在网盘列表/编辑表单展示开关状态。
|
||||
TeaserEnabled bool `json:"teaserEnabled"`
|
||||
// SkipDirIDs 是用户在 admin 配置的"扫描跳过目录"集合(drive 侧目录 fileID)。
|
||||
// 前端用它在"设置跳过目录"弹窗里回显已选项;JSON 字段名 camelCase 与
|
||||
@@ -393,7 +423,10 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
SkipDirIDs []string `json:"skipDirIds"`
|
||||
// LastCrawlAt 是 spider91 上次成功爬取的 unix 秒(来自 credentials.last_crawl_at)。
|
||||
// 其它 kind 留 0;前端用它显示"上次抓取: N 小时前"。
|
||||
Spider91Proxy string `json:"spider91Proxy,omitempty"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
GoogleDriveUseOnlineAPI *bool `json:"googleDriveUseOnlineAPI,omitempty"`
|
||||
ScanGenerationStatus GenerationStatus `json:"scanGenerationStatus"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
|
||||
@@ -414,6 +447,9 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
thumbCounts := thumbnailCounts[d.ID]
|
||||
fingerprintCount := fingerprintCounts[d.ID]
|
||||
generation := generationStatuses[d.ID]
|
||||
if generation.Scan.State == "" {
|
||||
generation.Scan.State = "idle"
|
||||
}
|
||||
if generation.Thumbnail.State == "" {
|
||||
generation.Thumbnail.State = "idle"
|
||||
}
|
||||
@@ -451,7 +487,10 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
HasCredential: hasCred,
|
||||
TeaserEnabled: d.TeaserEnabled,
|
||||
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
|
||||
Spider91Proxy: spider91ProxyForDrive(d),
|
||||
LastCrawlAt: lastCrawlAt,
|
||||
GoogleDriveUseOnlineAPI: googleDriveUseOnlineAPIForDrive(d),
|
||||
ScanGenerationStatus: generation.Scan,
|
||||
ThumbnailGenerationStatus: generation.Thumbnail,
|
||||
PreviewGenerationStatus: generation.Preview,
|
||||
FingerprintGenerationStatus: generation.Fingerprint,
|
||||
@@ -478,7 +517,7 @@ type upsertDriveReq struct {
|
||||
// Deprecated: 扫描起点已固定为 rootId;保留字段只为兼容旧客户端请求体。
|
||||
ScanRootID string `json:"scanRootId"`
|
||||
Credentials map[string]string `json:"credentials"`
|
||||
// TeaserEnabled 是 per-drive teaser/封面生成开关。
|
||||
// TeaserEnabled 是 per-drive 预览视频/封面生成开关。
|
||||
// 用 *bool 区分 "未传" / "传了 false":未传时表示客户端不打算改这个字段,
|
||||
// 沿用 catalog 现有值;新建时未传一律默认开启(true)。
|
||||
TeaserEnabled *bool `json:"teaserEnabled,omitempty"`
|
||||
@@ -503,7 +542,16 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
|
||||
if existingDrive, err := a.Catalog.GetDrive(r.Context(), body.ID); err == nil {
|
||||
existing = existingDrive
|
||||
}
|
||||
if len(body.Credentials) == 0 && existing != nil && len(existing.Credentials) > 0 {
|
||||
if body.Kind == "spider91" {
|
||||
credentials, err := mergeSpider91Credentials(existing, body.Credentials)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
body.Credentials = credentials
|
||||
} else if body.Kind == "googledrive" {
|
||||
body.Credentials = mergeGoogleDriveCredentials(existing, body.Credentials)
|
||||
} else if len(body.Credentials) == 0 && existing != nil && len(existing.Credentials) > 0 {
|
||||
body.Credentials = existing.Credentials
|
||||
}
|
||||
|
||||
@@ -552,8 +600,123 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
|
||||
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
func spider91ProxyForDrive(d *catalog.Drive) string {
|
||||
if d == nil || d.Kind != "spider91" || d.Credentials == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(d.Credentials["proxy"])
|
||||
}
|
||||
|
||||
func googleDriveUseOnlineAPIForDrive(d *catalog.Drive) *bool {
|
||||
if d == nil || d.Kind != "googledrive" {
|
||||
return nil
|
||||
}
|
||||
result := true
|
||||
if d.Credentials == nil {
|
||||
return &result
|
||||
}
|
||||
raw := strings.TrimSpace(d.Credentials["use_online_api"])
|
||||
if raw == "" {
|
||||
return &result
|
||||
}
|
||||
v, err := strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return &result
|
||||
}
|
||||
result = v
|
||||
return &result
|
||||
}
|
||||
|
||||
func mergeGoogleDriveCredentials(existing *catalog.Drive, incoming map[string]string) map[string]string {
|
||||
merged := map[string]string{}
|
||||
if existing != nil {
|
||||
for k, v := range existing.Credentials {
|
||||
merged[k] = v
|
||||
}
|
||||
}
|
||||
for k, v := range incoming {
|
||||
key := strings.TrimSpace(k)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
value := strings.TrimSpace(v)
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
merged[key] = value
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func mergeSpider91Credentials(existing *catalog.Drive, incoming map[string]string) (map[string]string, error) {
|
||||
merged := map[string]string{}
|
||||
if existing != nil {
|
||||
for k, v := range existing.Credentials {
|
||||
merged[k] = v
|
||||
}
|
||||
}
|
||||
for k, v := range incoming {
|
||||
if strings.TrimSpace(k) == "" {
|
||||
continue
|
||||
}
|
||||
if k == "proxy" {
|
||||
proxy, err := normalizeSpider91ProxyURL(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if proxy == "" {
|
||||
delete(merged, "proxy")
|
||||
} else {
|
||||
merged["proxy"] = proxy
|
||||
}
|
||||
continue
|
||||
}
|
||||
merged[k] = v
|
||||
}
|
||||
return merged, nil
|
||||
}
|
||||
|
||||
func normalizeSpider91ProxyURL(raw string) (string, error) {
|
||||
proxy := strings.TrimSpace(raw)
|
||||
if proxy == "" {
|
||||
return "", nil
|
||||
}
|
||||
u, err := url.Parse(proxy)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return "", fmt.Errorf("91Spider 代理地址格式无效,请填写类似 http://127.0.0.1:7890 的地址")
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https", "socks5", "socks5h":
|
||||
return proxy, nil
|
||||
default:
|
||||
return "", fmt.Errorf("91Spider 代理地址仅支持 http://、https://、socks5:// 或 socks5h://")
|
||||
}
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleDeleteDrive(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
var body deleteDriveReq
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
|
||||
writeErr(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
if !body.DeleteVideos {
|
||||
http.Error(w, "deleteVideos=true is required when deleting a drive", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
deletedVideos := 0
|
||||
if a.OnDriveDeleteCleanup == nil {
|
||||
http.Error(w, "drive video cleanup is not available", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
removed, err := a.OnDriveDeleteCleanup(r.Context(), id)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
deletedVideos = removed
|
||||
|
||||
if err := a.Catalog.DeleteDrive(r.Context(), id); err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
@@ -561,15 +724,80 @@ func (a *AdminServer) handleDeleteDrive(w http.ResponseWriter, r *http.Request)
|
||||
if a.OnDriveRemoved != nil {
|
||||
a.OnDriveRemoved(id)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
|
||||
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "deletedVideos": deletedVideos})
|
||||
}
|
||||
|
||||
type deleteDriveReq struct {
|
||||
DeleteVideos bool `json:"deleteVideos"`
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleRescan(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
if a.OnScanRequested != nil {
|
||||
a.OnScanRequested(id)
|
||||
status := a.nightlyJobStatus()
|
||||
if status.Running || status.Queued {
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"ok": true,
|
||||
"accepted": false,
|
||||
"message": fullScanBusyMessage,
|
||||
"status": status,
|
||||
})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
|
||||
accepted := true
|
||||
if a.OnScanRequested != nil {
|
||||
accepted = a.OnScanRequested(id)
|
||||
}
|
||||
resp := map[string]any{"ok": true, "accepted": accepted}
|
||||
if !accepted {
|
||||
resp["message"] = driveTaskBusyMessage
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, resp)
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleStopDriveTasks(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
stopped := false
|
||||
if a.OnStopDriveTasks != nil {
|
||||
stopped = a.OnStopDriveTasks(id)
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"ok": true,
|
||||
"stopped": stopped,
|
||||
})
|
||||
}
|
||||
|
||||
func (a *AdminServer) p123QRClient() *p123.QRClient {
|
||||
return p123.NewQRClient(p123.QRConfig{
|
||||
UserAPIBaseURL: a.P123UserAPIBaseURL,
|
||||
HTTPClient: a.P123HTTPClient,
|
||||
})
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleP123QRStart(w http.ResponseWriter, r *http.Request) {
|
||||
session, err := a.p123QRClient().Generate(r.Context())
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusBadGateway, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, http.StatusOK, session)
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleP123QRStatus(w http.ResponseWriter, r *http.Request) {
|
||||
uniID := chi.URLParam(r, "uniID")
|
||||
loginUUID := r.URL.Query().Get("loginUuid")
|
||||
if strings.TrimSpace(uniID) == "" || strings.TrimSpace(loginUUID) == "" {
|
||||
http.Error(w, "uniID and loginUuid are required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
status, err := a.p123QRClient().Poll(r.Context(), loginUUID, uniID)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusBadGateway, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, http.StatusOK, status)
|
||||
}
|
||||
|
||||
// handleRunNightlyJob 触发一次完整的凌晨流水线(不论当前时间,不论今日是否已跑)。
|
||||
@@ -580,17 +808,33 @@ func (a *AdminServer) handleRunNightlyJob(w http.ResponseWriter, r *http.Request
|
||||
if a.OnRunNightlyJob != nil {
|
||||
accepted = a.OnRunNightlyJob()
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
resp := map[string]any{
|
||||
"ok": true,
|
||||
"accepted": accepted,
|
||||
"status": a.nightlyJobStatus(),
|
||||
})
|
||||
}
|
||||
if !accepted {
|
||||
resp["message"] = fullScanBusyMessage
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, resp)
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleNightlyJobStatus(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.nightlyJobStatus())
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleStopAllTasks(w http.ResponseWriter, r *http.Request) {
|
||||
stoppedDrives := 0
|
||||
if a.OnStopAllTasks != nil {
|
||||
stoppedDrives = a.OnStopAllTasks()
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"ok": true,
|
||||
"stoppedDrives": stoppedDrives,
|
||||
"status": a.nightlyJobStatus(),
|
||||
})
|
||||
}
|
||||
|
||||
func (a *AdminServer) nightlyJobStatus() NightlyJobStatus {
|
||||
if a.GetNightlyJobStatus == nil {
|
||||
return NightlyJobStatus{State: "idle"}
|
||||
@@ -607,11 +851,11 @@ type teaserEnabledReq struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
// handleSetDriveTeaserEnabled 切换某盘的 teaser 生成开关。
|
||||
// handleSetDriveTeaserEnabled 切换某盘的预览视频生成开关。
|
||||
//
|
||||
// 行为:
|
||||
// - 写 catalog.drives.teaser_enabled
|
||||
// - 调 OnTeaserEnabledChanged(main 注入;从关到开时会重新入队 pending teaser)
|
||||
// - 调 OnTeaserEnabledChanged(main 注入;从关到开时会重新入队 pending 预览视频)
|
||||
// - 返回切换后的新值,方便前端乐观更新但又能以服务端为准
|
||||
//
|
||||
// 与 upsertDrive 的区别:那条接口要重传 kind / name / rootId 等,开关切换不该
|
||||
@@ -733,6 +977,7 @@ func (a *AdminServer) handleAdminListVideos(w http.ResponseWriter, r *http.Reque
|
||||
size = 100
|
||||
}
|
||||
items, total, err := a.Catalog.ListVideos(r.Context(), catalog.ListParams{
|
||||
Keyword: q.Get("keyword"),
|
||||
DriveID: q.Get("driveId"),
|
||||
Page: page,
|
||||
PageSize: size,
|
||||
@@ -871,6 +1116,36 @@ func (a *AdminServer) handleUpdateVideo(w http.ResponseWriter, r *http.Request)
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleDeleteVideo(w http.ResponseWriter, r *http.Request) {
|
||||
id := strings.TrimSpace(chi.URLParam(r, "id"))
|
||||
if id == "" {
|
||||
writeErr(w, http.StatusBadRequest, errors.New("invalid video id"))
|
||||
return
|
||||
}
|
||||
var (
|
||||
result DeleteVideoResult
|
||||
err error
|
||||
)
|
||||
if a.OnDeleteVideo != nil {
|
||||
result, err = a.OnDeleteVideo(r.Context(), id)
|
||||
} else {
|
||||
err = a.Catalog.DeleteVideoWithTombstone(r.Context(), id)
|
||||
result = DeleteVideoResult{OK: err == nil}
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
writeErr(w, http.StatusNotFound, err)
|
||||
return
|
||||
}
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
if !result.OK {
|
||||
result.OK = true
|
||||
}
|
||||
writeJSON(w, http.StatusOK, result)
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleRegenPreview(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
if a.OnRegenPreview != nil {
|
||||
@@ -895,7 +1170,7 @@ func (a *AdminServer) handleRegenFailedPreviews(w http.ResponseWriter, r *http.R
|
||||
}
|
||||
|
||||
// handleRegenFailedThumbnails 触发某 drive 下所有 thumbnail_status=failed 的封面
|
||||
// 重新入队生成。和 handleRegenFailedPreviews 行为对称(一个管 teaser,一个管封面)。
|
||||
// 重新入队生成。和 handleRegenFailedPreviews 行为对称(一个管预览视频,一个管封面)。
|
||||
//
|
||||
// 立即返回 202;实际执行在后台 goroutine 跑,状态可在下次 GET /admin/api/drives
|
||||
// 的 thumbnailFailedCount / thumbnailGenerationStatus 看变化。
|
||||
@@ -907,12 +1182,22 @@ func (a *AdminServer) handleRegenFailedThumbnails(w http.ResponseWriter, r *http
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
// handleRegenFailedFingerprints triggers regeneration for all failed sampled
|
||||
// fingerprints on a drive. It mirrors the failed preview-video/thumbnail retry endpoints.
|
||||
func (a *AdminServer) handleRegenFailedFingerprints(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
if a.OnRegenFailedFingerprints != nil {
|
||||
a.OnRegenFailedFingerprints(id)
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
// ---------- Settings ----------
|
||||
|
||||
// settingsDTO 是 GET/PUT /admin/api/settings 的入参/出参。
|
||||
//
|
||||
// 注意:早期的全局 previewEnabled 字段已经下沉为每盘 teaser_enabled,
|
||||
// 不再出现在这里;前端要切换某个盘的 teaser 生成请用 POST /admin/api/drives 上传
|
||||
// 不再出现在这里;前端要切换某个盘的预览视频生成请用 POST /admin/api/drives 上传
|
||||
// teaserEnabled 字段。保留 settings 用作主题、spider91 上传目标这类全局配置。
|
||||
type settingsDTO struct {
|
||||
Theme string `json:"theme"`
|
||||
|
||||
@@ -3,6 +3,7 @@ package api
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -277,6 +278,108 @@ func TestHandleRunNightlyJobReturnsAcceptedStatus(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRunNightlyJobReturnsBusyMessageWhenRejected(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/jobs/nightly/run", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
OnRunNightlyJob: func() bool {
|
||||
return false
|
||||
},
|
||||
GetNightlyJobStatus: func() NightlyJobStatus {
|
||||
return NightlyJobStatus{State: "running", Running: true}
|
||||
},
|
||||
}).handleRunNightlyJob(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, want 202; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
OK bool `json:"ok"`
|
||||
Accepted bool `json:"accepted"`
|
||||
Message string `json:"message"`
|
||||
Status NightlyJobStatus `json:"status"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if !got.OK || got.Accepted || got.Message != fullScanBusyMessage || !got.Status.Running {
|
||||
t.Fatalf("response = %#v, want rejected busy message", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRescanRejectsWhenNightlyBusy(t *testing.T) {
|
||||
called := false
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives/PikPak/rescan", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "PikPak")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
OnScanRequested: func(driveID string) bool {
|
||||
called = true
|
||||
return true
|
||||
},
|
||||
GetNightlyJobStatus: func() NightlyJobStatus {
|
||||
return NightlyJobStatus{State: "running", Running: true}
|
||||
},
|
||||
}).handleRescan(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, want 202; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if called {
|
||||
t.Fatal("OnScanRequested was called while nightly job was busy")
|
||||
}
|
||||
var got struct {
|
||||
OK bool `json:"ok"`
|
||||
Accepted bool `json:"accepted"`
|
||||
Message string `json:"message"`
|
||||
Status NightlyJobStatus `json:"status"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if !got.OK || got.Accepted || got.Message != fullScanBusyMessage || !got.Status.Running {
|
||||
t.Fatalf("response = %#v, want rejected full scan busy message", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRescanReturnsAcceptedFlagAndBusyMessage(t *testing.T) {
|
||||
calledWith := ""
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives/PikPak/rescan", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "PikPak")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
OnScanRequested: func(driveID string) bool {
|
||||
calledWith = driveID
|
||||
return false
|
||||
},
|
||||
}).handleRescan(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, want 202; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
OK bool `json:"ok"`
|
||||
Accepted bool `json:"accepted"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if calledWith != "PikPak" {
|
||||
t.Fatalf("hook called with %q, want PikPak", calledWith)
|
||||
}
|
||||
if !got.OK || got.Accepted || got.Message != driveTaskBusyMessage {
|
||||
t.Fatalf("response = %#v, want rejected busy message", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleNightlyJobStatusDefaultsToIdle(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/jobs/nightly/status", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
@@ -295,6 +398,75 @@ func TestHandleNightlyJobStatusDefaultsToIdle(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStopDriveTasksInvokesHookWithDriveID(t *testing.T) {
|
||||
calledWith := ""
|
||||
server := &AdminServer{
|
||||
OnStopDriveTasks: func(driveID string) bool {
|
||||
calledWith = driveID
|
||||
return true
|
||||
},
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives/PikPak/tasks/stop", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "PikPak")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
server.handleStopDriveTasks(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if calledWith != "PikPak" {
|
||||
t.Fatalf("hook called with %q, want PikPak", calledWith)
|
||||
}
|
||||
var got struct {
|
||||
OK bool `json:"ok"`
|
||||
Stopped bool `json:"stopped"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if !got.OK || !got.Stopped {
|
||||
t.Fatalf("response = %#v, want stopped", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStopAllTasksInvokesHookAndReturnsStatus(t *testing.T) {
|
||||
called := false
|
||||
server := &AdminServer{
|
||||
OnStopAllTasks: func() int {
|
||||
called = true
|
||||
return 2
|
||||
},
|
||||
GetNightlyJobStatus: func() NightlyJobStatus {
|
||||
return NightlyJobStatus{State: "running", Running: true}
|
||||
},
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/tasks/stop", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
server.handleStopAllTasks(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !called {
|
||||
t.Fatal("OnStopAllTasks was not called")
|
||||
}
|
||||
var got struct {
|
||||
OK bool `json:"ok"`
|
||||
StoppedDrives int `json:"stoppedDrives"`
|
||||
Status NightlyJobStatus `json:"status"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if !got.OK || got.StoppedDrives != 2 || got.Status.State != "running" || !got.Status.Running {
|
||||
t.Fatalf("response = %#v, want stopped drives and status", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertDrivePreservesExistingCredentialsWhenRequestCredentialsEmpty(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -439,6 +611,429 @@ func TestHandleUpsertDriveReplacesExistingCredentialsWhenProvided(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertGoogleDriveMergesOAuthCredentials(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "google-main",
|
||||
Kind: "googledrive",
|
||||
Name: "Google Drive",
|
||||
RootID: "root",
|
||||
Credentials: map[string]string{
|
||||
"refresh_token": "existing-refresh",
|
||||
"access_token": "existing-access",
|
||||
"use_online_api": "true",
|
||||
"api_url_address": "https://api.oplist.org/googleui/renewapi",
|
||||
},
|
||||
Status: "ok",
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", bytes.NewBufferString(`{
|
||||
"id": "google-main",
|
||||
"kind": "googledrive",
|
||||
"name": "Google Drive",
|
||||
"rootId": "root",
|
||||
"credentials": {
|
||||
"use_online_api": "false",
|
||||
"client_id": "google-client-id",
|
||||
"client_secret": "google-client-secret"
|
||||
}
|
||||
}`))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, "google-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.Credentials["refresh_token"] != "existing-refresh" || got.Credentials["access_token"] != "existing-access" {
|
||||
t.Fatalf("tokens were not preserved: %#v", got.Credentials)
|
||||
}
|
||||
if got.Credentials["use_online_api"] != "false" {
|
||||
t.Fatalf("use_online_api = %q, want false", got.Credentials["use_online_api"])
|
||||
}
|
||||
if got.Credentials["client_id"] != "google-client-id" || got.Credentials["client_secret"] != "google-client-secret" {
|
||||
t.Fatalf("oauth client credentials = %#v, want saved", got.Credentials)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertSpider91ProxyPreservesRuntimeCredentials(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "spider91-main",
|
||||
Kind: "spider91",
|
||||
Name: "91 Spider",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"last_crawl_at": "1800000000",
|
||||
"proxy": "http://old-proxy.local:7890",
|
||||
"script_path": "/opt/video-site-91/91VideoSpider/spider_91porn.py",
|
||||
},
|
||||
Status: "ok",
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"kind": "spider91",
|
||||
"name": "91 Spider",
|
||||
"rootId": "/",
|
||||
"credentials": {"proxy": " socks5h://proxy-user:proxy-pass@127.0.0.1:7891 "}
|
||||
}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
got, err := cat.GetDrive(ctx, "spider91-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.Credentials["proxy"] != "socks5h://proxy-user:proxy-pass@127.0.0.1:7891" {
|
||||
t.Fatalf("proxy = %q, want trimmed new proxy", got.Credentials["proxy"])
|
||||
}
|
||||
if got.Credentials["last_crawl_at"] != "1800000000" {
|
||||
t.Fatalf("last_crawl_at = %q, want preserved", got.Credentials["last_crawl_at"])
|
||||
}
|
||||
if got.Credentials["script_path"] == "" {
|
||||
t.Fatalf("script_path should be preserved")
|
||||
}
|
||||
|
||||
req = httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"kind": "spider91",
|
||||
"name": "91 Spider",
|
||||
"rootId": "/",
|
||||
"credentials": {"proxy": " "}
|
||||
}`))
|
||||
rr = httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("clear status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
got, err = cat.GetDrive(ctx, "spider91-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get cleared drive: %v", err)
|
||||
}
|
||||
if _, ok := got.Credentials["proxy"]; ok {
|
||||
t.Fatalf("proxy should be removed after empty save, got %q", got.Credentials["proxy"])
|
||||
}
|
||||
if got.Credentials["last_crawl_at"] != "1800000000" {
|
||||
t.Fatalf("last_crawl_at after clear = %q, want preserved", got.Credentials["last_crawl_at"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertSpider91RejectsUnsupportedProxyScheme(t *testing.T) {
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"kind": "spider91",
|
||||
"name": "91 Spider",
|
||||
"rootId": "/",
|
||||
"credentials": {"proxy": "ftp://127.0.0.1:21"}
|
||||
}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), "socks5:// 或 socks5h://") {
|
||||
t.Fatalf("body = %q, want supported schemes message", rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleDeleteDriveRunsRequestedCleanupBeforeDeletingDrive(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "drive-one",
|
||||
Kind: "pikpak",
|
||||
Name: "Drive One",
|
||||
RootID: "root",
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
cleanupCalled := ""
|
||||
removedCalled := ""
|
||||
req := httptest.NewRequest(http.MethodDelete, "/admin/api/drives/drive-one", strings.NewReader(`{"deleteVideos":true}`))
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "drive-one")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
Catalog: cat,
|
||||
OnDriveDeleteCleanup: func(cleanupCtx context.Context, driveID string) (int, error) {
|
||||
cleanupCalled = driveID
|
||||
if _, err := cat.GetDrive(cleanupCtx, driveID); err != nil {
|
||||
t.Fatalf("drive should still exist during cleanup: %v", err)
|
||||
}
|
||||
return 3, nil
|
||||
},
|
||||
OnDriveRemoved: func(driveID string) {
|
||||
removedCalled = driveID
|
||||
},
|
||||
}).handleDeleteDrive(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if cleanupCalled != "drive-one" {
|
||||
t.Fatalf("cleanup called with %q, want drive-one", cleanupCalled)
|
||||
}
|
||||
if removedCalled != "drive-one" {
|
||||
t.Fatalf("removed hook called with %q, want drive-one", removedCalled)
|
||||
}
|
||||
if _, err := cat.GetDrive(ctx, "drive-one"); err != sql.ErrNoRows {
|
||||
t.Fatalf("drive lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
var got struct {
|
||||
OK bool `json:"ok"`
|
||||
DeletedVideos int `json:"deletedVideos"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if !got.OK || got.DeletedVideos != 3 {
|
||||
t.Fatalf("response = %#v, want ok with deletedVideos=3", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleDeleteDriveRequiresCleanupConfirmation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "drive-one",
|
||||
Kind: "pikpak",
|
||||
Name: "Drive One",
|
||||
RootID: "root",
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodDelete, "/admin/api/drives/drive-one", strings.NewReader(`{"deleteVideos":false}`))
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "drive-one")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
Catalog: cat,
|
||||
OnDriveDeleteCleanup: func(context.Context, string) (int, error) {
|
||||
t.Fatal("cleanup hook should not be called without confirmation")
|
||||
return 0, nil
|
||||
},
|
||||
}).handleDeleteDrive(rr, req)
|
||||
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if _, err := cat.GetDrive(ctx, "drive-one"); err != nil {
|
||||
t.Fatalf("drive should remain after rejected delete: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListDrivesIncludesSpider91Proxy(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, d := range []*catalog.Drive{
|
||||
{
|
||||
ID: "spider91-main",
|
||||
Kind: "spider91",
|
||||
Name: "91 Spider",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"last_crawl_at": "1800000000",
|
||||
"proxy": " http://127.0.0.1:7890 ",
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
{
|
||||
ID: "onedrive-main",
|
||||
Kind: "onedrive",
|
||||
Name: "OneDrive",
|
||||
RootID: "root",
|
||||
Credentials: map[string]string{
|
||||
"proxy": "http://should-not-leak.local:7890",
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
t.Fatalf("seed drive %s: %v", d.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/drives", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleListDrives(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
var got []struct {
|
||||
ID string `json:"id"`
|
||||
Spider91Proxy string `json:"spider91Proxy"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
byID := map[string]struct {
|
||||
Spider91Proxy string
|
||||
LastCrawlAt int64
|
||||
}{}
|
||||
for _, d := range got {
|
||||
byID[d.ID] = struct {
|
||||
Spider91Proxy string
|
||||
LastCrawlAt int64
|
||||
}{Spider91Proxy: d.Spider91Proxy, LastCrawlAt: d.LastCrawlAt}
|
||||
}
|
||||
if byID["spider91-main"].Spider91Proxy != "http://127.0.0.1:7890" {
|
||||
t.Fatalf("spider91 proxy = %q, want trimmed proxy", byID["spider91-main"].Spider91Proxy)
|
||||
}
|
||||
if byID["spider91-main"].LastCrawlAt != 1800000000 {
|
||||
t.Fatalf("lastCrawlAt = %d, want 1800000000", byID["spider91-main"].LastCrawlAt)
|
||||
}
|
||||
if byID["onedrive-main"].Spider91Proxy != "" {
|
||||
t.Fatalf("onedrive spider91Proxy = %q, want empty", byID["onedrive-main"].Spider91Proxy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListDrivesIncludesGoogleDriveOnlineAPIMode(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, d := range []*catalog.Drive{
|
||||
{
|
||||
ID: "google-legacy",
|
||||
Kind: "googledrive",
|
||||
Name: "Google Legacy",
|
||||
RootID: "root",
|
||||
Credentials: map[string]string{
|
||||
"refresh_token": "legacy-refresh",
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
{
|
||||
ID: "google-oauth",
|
||||
Kind: "googledrive",
|
||||
Name: "Google OAuth",
|
||||
RootID: "root",
|
||||
Credentials: map[string]string{
|
||||
"refresh_token": "oauth-refresh",
|
||||
"use_online_api": "false",
|
||||
"client_id": "client-id",
|
||||
"client_secret": "client-secret",
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
t.Fatalf("seed drive %s: %v", d.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/drives", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleListDrives(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
var got []struct {
|
||||
ID string `json:"id"`
|
||||
GoogleDriveUseOnlineAPI bool `json:"googleDriveUseOnlineAPI"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
byID := map[string]bool{}
|
||||
for _, d := range got {
|
||||
byID[d.ID] = d.GoogleDriveUseOnlineAPI
|
||||
}
|
||||
if !byID["google-legacy"] {
|
||||
t.Fatalf("legacy google drive use_online_api = false, want true")
|
||||
}
|
||||
if byID["google-oauth"] {
|
||||
t.Fatalf("oauth google drive use_online_api = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -490,6 +1085,7 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
GetDriveGenerationStatuses: func() map[string]DriveGenerationStatuses {
|
||||
return map[string]DriveGenerationStatuses{
|
||||
"OneDrive": {
|
||||
Scan: GenerationStatus{State: "scanning", ScannedCount: 12, AddedCount: 3},
|
||||
Thumbnail: GenerationStatus{State: "cooling", QueueLength: 3, CooldownUntil: "2026-05-16T21:00:00+08:00"},
|
||||
Preview: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
|
||||
Fingerprint: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
|
||||
@@ -503,6 +1099,7 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
}
|
||||
var got []struct {
|
||||
ID string `json:"id"`
|
||||
ScanGenerationStatus GenerationStatus `json:"scanGenerationStatus"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
|
||||
@@ -531,6 +1128,7 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
FingerprintReady int
|
||||
FingerprintPending int
|
||||
FingerprintFailed int
|
||||
Scan GenerationStatus
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
Fingerprint GenerationStatus
|
||||
@@ -547,6 +1145,7 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
FingerprintReady int
|
||||
FingerprintPending int
|
||||
FingerprintFailed int
|
||||
Scan GenerationStatus
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
Fingerprint GenerationStatus
|
||||
@@ -561,6 +1160,7 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
FingerprintReady: d.FingerprintReadyCount,
|
||||
FingerprintPending: d.FingerprintPendingCount,
|
||||
FingerprintFailed: d.FingerprintFailedCount,
|
||||
Scan: d.ScanGenerationStatus,
|
||||
Thumbnail: d.ThumbnailGenerationStatus,
|
||||
Preview: d.PreviewGenerationStatus,
|
||||
Fingerprint: d.FingerprintGenerationStatus,
|
||||
@@ -578,6 +1178,12 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
if byID["OneDrive"].Thumbnail.State != "cooling" || byID["OneDrive"].Preview.State != "generating" {
|
||||
t.Fatalf("OneDrive generation statuses = %#v, want thumbnail cooling and preview generating", byID["OneDrive"])
|
||||
}
|
||||
if byID["OneDrive"].Scan.State != "scanning" {
|
||||
t.Fatalf("OneDrive scan status = %#v, want scanning", byID["OneDrive"].Scan)
|
||||
}
|
||||
if byID["OneDrive"].Scan.ScannedCount != 12 || byID["OneDrive"].Scan.AddedCount != 3 {
|
||||
t.Fatalf("OneDrive scan counts = %#v, want scanned=12 added=3", byID["OneDrive"].Scan)
|
||||
}
|
||||
if byID["OneDrive"].FingerprintReady != 1 || byID["OneDrive"].FingerprintPending != 1 || byID["OneDrive"].FingerprintFailed != 1 {
|
||||
t.Fatalf("OneDrive fingerprint counts = %#v, want ready=1 pending=1 failed=1", byID["OneDrive"])
|
||||
}
|
||||
@@ -593,11 +1199,33 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
if byID["PikPak"].FingerprintPending != 2 {
|
||||
t.Fatalf("PikPak fingerprint counts = %#v, want pending=2", byID["PikPak"])
|
||||
}
|
||||
if byID["PikPak"].Thumbnail.State != "idle" || byID["PikPak"].Preview.State != "idle" || byID["PikPak"].Fingerprint.State != "idle" {
|
||||
if byID["PikPak"].Scan.State != "idle" || byID["PikPak"].Thumbnail.State != "idle" || byID["PikPak"].Preview.State != "idle" || byID["PikPak"].Fingerprint.State != "idle" {
|
||||
t.Fatalf("PikPak generation statuses = %#v, want idle defaults", byID["PikPak"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRegenFailedFingerprintsInvokesHook(t *testing.T) {
|
||||
called := ""
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives/drive-one/fingerprints/failed/regenerate", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "drive-one")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
OnRegenFailedFingerprints: func(driveID string) {
|
||||
called = driveID
|
||||
},
|
||||
}).handleRegenFailedFingerprints(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if called != "drive-one" {
|
||||
t.Fatalf("called drive = %q, want drive-one", called)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleDriveStorageReportsLocalMediaUsage(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
|
||||
+137
-42
@@ -15,6 +15,7 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
@@ -24,6 +25,7 @@ import (
|
||||
"github.com/video-site/backend/internal/drives/localstorage"
|
||||
"github.com/video-site/backend/internal/drives/localupload"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
)
|
||||
|
||||
@@ -53,6 +55,10 @@ type Server struct {
|
||||
UploadDir string
|
||||
OnVideoUploaded func(*catalog.Video)
|
||||
|
||||
tagCacheMu sync.Mutex
|
||||
tagCacheUntil time.Time
|
||||
tagCache []TagDTO
|
||||
|
||||
// GetTheme 返回当前生效的主题("dark" | "pink")。前台 /api/settings/theme 用,
|
||||
// 不需要登录。无注入时返回 "dark"。
|
||||
GetTheme func() string
|
||||
@@ -86,6 +92,12 @@ type VideoDTO struct {
|
||||
Category string `json:"category,omitempty"`
|
||||
}
|
||||
|
||||
type TagDTO struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
type VideoDetailDTO struct {
|
||||
VideoDTO
|
||||
VideoSrc string `json:"videoSrc"`
|
||||
@@ -156,41 +168,81 @@ func (s *Server) handleGetTheme(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) {
|
||||
// 首页优先展示封面已经生成好的视频,避免新盘扫盘时大量黑封面占满首页。
|
||||
// 候选仍按发布时间覆盖最近 200 个,随后随机洗牌;封面不足时再用普通可见视频补齐。
|
||||
const candidatePool = 200
|
||||
readyItems, _, err := s.Catalog.ListVideos(r.Context(), catalog.ListParams{
|
||||
Sort: "latest", Page: 1, PageSize: candidatePool, ThumbnailReadyOnly: true,
|
||||
})
|
||||
// 首页优先从全量已有封面的视频里随机抽取,避免只在最近一小段候选中反复出现。
|
||||
excludeIDs := parseVideoIDQuery(r, "exclude", 120)
|
||||
items, err := s.Catalog.RandomVideosWithReadyThumbnailsExcluding(r.Context(), excludeIDs, homePageSize)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
rand.Shuffle(len(readyItems), func(i, j int) {
|
||||
readyItems[i], readyItems[j] = readyItems[j], readyItems[i]
|
||||
})
|
||||
|
||||
items := appendUniqueVideos(nil, readyItems, homePageSize)
|
||||
if len(items) > homePageSize {
|
||||
items = items[:homePageSize]
|
||||
}
|
||||
if len(items) < homePageSize {
|
||||
fallback, _, err := s.Catalog.ListVideos(r.Context(), catalog.ListParams{
|
||||
Sort: "latest", Page: 1, PageSize: candidatePool,
|
||||
})
|
||||
fallbackExclude := append([]string{}, excludeIDs...)
|
||||
for _, item := range items {
|
||||
if item != nil {
|
||||
fallbackExclude = append(fallbackExclude, item.ID)
|
||||
}
|
||||
}
|
||||
fallback, err := s.Catalog.RandomVideosExcluding(r.Context(), fallbackExclude, homePageSize-len(items))
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
items = appendUniqueVideos(items, fallback, homePageSize)
|
||||
}
|
||||
if len(items) < homePageSize && len(excludeIDs) > 0 {
|
||||
// The browser keeps a recent-video exclude list so normal refreshes do not
|
||||
// repeat too quickly. On small libraries that list can cover every visible
|
||||
// video; when that happens, start a new random round instead of returning
|
||||
// an empty home section.
|
||||
roundExclude := videoIDs(items)
|
||||
fallback, err := s.Catalog.RandomVideosWithReadyThumbnailsExcluding(r.Context(), roundExclude, homePageSize-len(items))
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
items = appendUniqueVideos(items, fallback, homePageSize)
|
||||
}
|
||||
if len(items) < homePageSize && len(excludeIDs) > 0 {
|
||||
fallback, err := s.Catalog.RandomVideosExcluding(r.Context(), videoIDs(items), homePageSize-len(items))
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
rand.Shuffle(len(fallback), func(i, j int) {
|
||||
fallback[i], fallback[j] = fallback[j], fallback[i]
|
||||
})
|
||||
items = appendUniqueVideos(items, fallback, homePageSize)
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, http.StatusOK, mapVideos(items))
|
||||
}
|
||||
|
||||
func parseVideoIDQuery(r *http.Request, key string, limit int) []string {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
values := r.URL.Query()[key]
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
out := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
for _, id := range strings.Split(value, ",") {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
out = append(out, id)
|
||||
if limit > 0 && len(out) >= limit {
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func appendUniqueVideos(dst []*catalog.Video, candidates []*catalog.Video, limit int) []*catalog.Video {
|
||||
if len(dst) >= limit {
|
||||
return dst[:limit]
|
||||
@@ -217,6 +269,16 @@ func appendUniqueVideos(dst []*catalog.Video, candidates []*catalog.Video, limit
|
||||
return dst
|
||||
}
|
||||
|
||||
func videoIDs(items []*catalog.Video) []string {
|
||||
out := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
if item != nil && item.ID != "" {
|
||||
out = append(out, item.ID)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Server) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
page, _ := strconv.Atoi(q.Get("page"))
|
||||
@@ -226,12 +288,13 @@ func (s *Server) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
sort := q.Get("sort")
|
||||
params := catalog.ListParams{
|
||||
Keyword: q.Get("q"),
|
||||
Tag: q.Get("tag"),
|
||||
Category: q.Get("cat"),
|
||||
Sort: sort,
|
||||
Page: page,
|
||||
PageSize: size,
|
||||
Keyword: q.Get("q"),
|
||||
Tag: q.Get("tag"),
|
||||
Category: q.Get("cat"),
|
||||
Sort: sort,
|
||||
Page: page,
|
||||
PageSize: size,
|
||||
SkipTotal: strings.EqualFold(q.Get("count"), "false"),
|
||||
}
|
||||
if sort == "" || sort == "latest" {
|
||||
params.PreferReadyThumbnails = true
|
||||
@@ -260,6 +323,15 @@ func (s *Server) handleVideoDetail(w http.ResponseWriter, r *http.Request) {
|
||||
writeErr(w, http.StatusNotFound, sql.ErrNoRows)
|
||||
return
|
||||
}
|
||||
if v.DriveID != localUploadDriveID {
|
||||
if _, err := s.Catalog.GetDrive(r.Context(), v.DriveID); err != nil {
|
||||
drives, listErr := s.Catalog.ListDrives(r.Context())
|
||||
if listErr != nil || len(drives) > 0 {
|
||||
writeErr(w, http.StatusNotFound, sql.ErrNoRows)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
related := s.pickRelatedVideos(r.Context(), v, 6)
|
||||
dto := mapVideo(v)
|
||||
if d, err := s.Catalog.GetDrive(r.Context(), v.DriveID); err == nil {
|
||||
@@ -423,20 +495,32 @@ func appendRandomRelated(picked []*catalog.Video, pool []*catalog.Video, targetL
|
||||
}
|
||||
|
||||
func (s *Server) handleTags(w http.ResponseWriter, r *http.Request) {
|
||||
now := time.Now()
|
||||
s.tagCacheMu.Lock()
|
||||
if s.tagCache != nil && now.Before(s.tagCacheUntil) {
|
||||
out := append([]TagDTO(nil), s.tagCache...)
|
||||
s.tagCacheMu.Unlock()
|
||||
w.Header().Set("Cache-Control", "private, max-age=15")
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
return
|
||||
}
|
||||
s.tagCacheMu.Unlock()
|
||||
|
||||
stats, err := s.Catalog.ListTags(r.Context())
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
type tag struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
out := make([]tag, 0, len(stats))
|
||||
out := make([]TagDTO, 0, len(stats))
|
||||
for _, stat := range stats {
|
||||
out = append(out, tag{ID: stat.Label, Label: stat.Label, Count: stat.Count})
|
||||
out = append(out, TagDTO{ID: stat.Label, Label: stat.Label, Count: stat.Count})
|
||||
}
|
||||
s.tagCacheMu.Lock()
|
||||
s.tagCache = append([]TagDTO(nil), out...)
|
||||
s.tagCacheUntil = now.Add(30 * time.Second)
|
||||
s.tagCacheMu.Unlock()
|
||||
|
||||
w.Header().Set("Cache-Control", "private, max-age=15")
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
@@ -808,14 +892,19 @@ func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
func (s *Server) handleThumb(w http.ResponseWriter, r *http.Request) {
|
||||
videoID := chi.URLParam(r, "videoID")
|
||||
// 直接读本地 thumbs 目录中 <videoID>.jpg
|
||||
path := filepath.Join(s.LocalDir, "thumbs", videoID+".jpg")
|
||||
clean := filepath.Clean(path)
|
||||
if !strings.HasPrefix(clean, filepath.Clean(s.LocalDir)) {
|
||||
http.Error(w, "invalid path", http.StatusForbidden)
|
||||
return
|
||||
var clean string
|
||||
for _, path := range mediaasset.ThumbnailPathCandidates(s.LocalDir, videoID) {
|
||||
candidate := filepath.Clean(path)
|
||||
if !strings.HasPrefix(candidate, filepath.Clean(s.LocalDir)) {
|
||||
http.Error(w, "invalid path", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if _, err := os.Stat(candidate); err == nil {
|
||||
clean = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(clean); err != nil {
|
||||
if clean == "" {
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
@@ -867,10 +956,14 @@ func previewURL(v *catalog.Video) string {
|
||||
}
|
||||
|
||||
func thumbnailURL(v *catalog.Video) string {
|
||||
base := "/p/thumb/" + v.ID
|
||||
if v.ThumbnailURL != "" {
|
||||
return v.ThumbnailURL
|
||||
base = v.ThumbnailURL
|
||||
}
|
||||
return "/p/thumb/" + v.ID
|
||||
if !strings.HasPrefix(base, "/p/thumb/") || v.UpdatedAt.IsZero() {
|
||||
return base
|
||||
}
|
||||
return base + "?v=" + strconv.FormatInt(v.UpdatedAt.UnixMilli(), 10)
|
||||
}
|
||||
|
||||
func (s *Server) videoSource(v *catalog.Video) string {
|
||||
@@ -900,6 +993,8 @@ func driveKindLabel(kind string) string {
|
||||
return "夸克网盘"
|
||||
case "p115":
|
||||
return "115 网盘"
|
||||
case "p123":
|
||||
return "123 云盘"
|
||||
case "pikpak":
|
||||
return "PikPak"
|
||||
case "wopan":
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
)
|
||||
|
||||
@@ -99,6 +100,27 @@ func TestPreviewURLFallsBackWithoutUpdatedAt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailURLVersionsLocalGeneratedThumbnails(t *testing.T) {
|
||||
got := thumbnailURL(&catalog.Video{
|
||||
ID: "video-1",
|
||||
ThumbnailURL: "/p/thumb/video-1",
|
||||
UpdatedAt: time.UnixMilli(1778863000123),
|
||||
})
|
||||
if got != "/p/thumb/video-1?v=1778863000123" {
|
||||
t.Fatalf("thumbnail URL = %q, want versioned local URL", got)
|
||||
}
|
||||
|
||||
remote := "https://thumb.example/video-1.jpg"
|
||||
got = thumbnailURL(&catalog.Video{
|
||||
ID: "video-1",
|
||||
ThumbnailURL: remote,
|
||||
UpdatedAt: time.UnixMilli(1778863000123),
|
||||
})
|
||||
if got != remote {
|
||||
t.Fatalf("remote thumbnail URL = %q, want unchanged %q", got, remote)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleHomePrioritizesVideosWithReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -166,6 +188,116 @@ func TestHandleHomePrioritizesVideosWithReadyThumbnails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleHomeExcludesRecentlyShownVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < homePageSize+4; i++ {
|
||||
id := "ready-video-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
|
||||
PublishedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
CreatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
UpdatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/home?exclude=ready-video-0&exclude=ready-video-1", nil)
|
||||
(&Server{Catalog: cat}).handleHome(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got []VideoDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(got) != homePageSize {
|
||||
t.Fatalf("home items = %d, want %d", len(got), homePageSize)
|
||||
}
|
||||
for _, item := range got {
|
||||
if item.ID == "ready-video-0" || item.ID == "ready-video-1" {
|
||||
t.Fatalf("home returned excluded video %q; items=%#v", item.ID, got)
|
||||
}
|
||||
if !strings.HasPrefix(item.ID, "ready-video-") {
|
||||
t.Fatalf("home returned %q without a ready thumbnail; items=%#v", item.ID, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleHomeStartsNewRoundWhenRecentExcludesAllVisibleVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
excludes := make([]string, 0, homePageSize+2)
|
||||
for i := 0; i < homePageSize+2; i++ {
|
||||
id := "ready-video-" + strconv.Itoa(i)
|
||||
excludes = append(excludes, "exclude="+id)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
|
||||
PublishedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
CreatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
UpdatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/home?"+strings.Join(excludes, "&"), nil)
|
||||
(&Server{Catalog: cat}).handleHome(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got []VideoDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(got) != homePageSize {
|
||||
t.Fatalf("home items = %d, want %d; body=%s", len(got), homePageSize, rr.Body.String())
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, item := range got {
|
||||
if seen[item.ID] {
|
||||
t.Fatalf("home returned duplicate video %q; items=%#v", item.ID, got)
|
||||
}
|
||||
seen[item.ID] = true
|
||||
if !strings.HasPrefix(item.ID, "ready-video-") {
|
||||
t.Fatalf("home returned unexpected video %q; items=%#v", item.ID, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListLatestPrefersReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -237,6 +369,26 @@ func TestHandleListLatestPrefersReadyThumbnails(t *testing.T) {
|
||||
t.Fatalf("thumbnail for %q = %q, want ready thumbnail URL", item.ID, item.Thumbnail)
|
||||
}
|
||||
}
|
||||
|
||||
rr = httptest.NewRecorder()
|
||||
req = httptest.NewRequest(http.MethodGet, "/api/list?page=1&size=12&sort=latest&count=false", nil)
|
||||
(&Server{Catalog: cat}).handleList(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("count=false status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
got = struct {
|
||||
Items []VideoDTO `json:"items"`
|
||||
Total int `json:"total"`
|
||||
}{}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode count=false response: %v", err)
|
||||
}
|
||||
if got.Total != 0 {
|
||||
t.Fatalf("count=false total = %d, want 0", got.Total)
|
||||
}
|
||||
if len(got.Items) != 12 {
|
||||
t.Fatalf("count=false items = %d, want 12", len(got.Items))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUploadVideoSavesFileVideoTagsAndQueuesPreview(t *testing.T) {
|
||||
@@ -458,6 +610,34 @@ func TestHandlePreviewIgnoresRemotePreviewFileIDAndServesLocalFile(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleThumbServesHashedPathForLongVideoID(t *testing.T) {
|
||||
localDir := t.TempDir()
|
||||
longID := "localstorage-" + strings.Repeat("x", 240)
|
||||
thumbPath := mediaasset.ThumbnailPath(localDir, longID)
|
||||
if err := os.MkdirAll(filepath.Dir(thumbPath), 0o755); err != nil {
|
||||
t.Fatalf("mkdir thumb dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(thumbPath, []byte("thumb-bytes"), 0o644); err != nil {
|
||||
t.Fatalf("write thumb: %v", err)
|
||||
}
|
||||
|
||||
server := &Server{
|
||||
LocalDir: localDir,
|
||||
Proxy: proxy.New(proxy.NewRegistry()),
|
||||
}
|
||||
req := requestWithRouteParam(http.MethodGet, "/p/thumb/"+longID, "videoID", longID, strings.NewReader(``))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
server.handleThumb(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if rr.Body.String() != "thumb-bytes" {
|
||||
t.Fatalf("body = %q, want thumb bytes", rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleTagsReturnsUnifiedTagPool(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"database/sql"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -546,7 +547,7 @@ func (c *Catalog) ListVideosByThumbnailStatus(ctx context.Context, driveID, stat
|
||||
// Besides missing thumbnails, this includes videos with an existing thumbnail but
|
||||
// missing duration metadata, because the thumbnail worker probes duration while
|
||||
// it already has a stream link.
|
||||
// Failed thumbnails are reported separately and should not block teaser generation.
|
||||
// Failed thumbnails are reported separately and should not block preview-video generation.
|
||||
// Videos whose local assets were cleared because they are fingerprint duplicates
|
||||
// stay pending in the DB, but uniqueVideoWhereSQL keeps them out of this queue
|
||||
// while their canonical sibling still exists.
|
||||
@@ -622,6 +623,56 @@ func (c *Catalog) ListVideosByDrive(ctx context.Context, driveID string) ([]*Vid
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) ListVideosByIDPrefix(ctx context.Context, prefix string) ([]*Video, error) {
|
||||
prefix = strings.TrimSpace(prefix)
|
||||
if prefix == "" {
|
||||
return nil, fmt.Errorf("catalog: list videos by id prefix: empty prefix")
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE SUBSTR(id, 1, LENGTH(?)) = ?
|
||||
ORDER BY created_at ASC, id ASC`,
|
||||
prefix, prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) ListVideosWithMissingDrive(ctx context.Context) ([]*Video, error) {
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE drive_id != 'local-upload'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM drives
|
||||
WHERE drives.id = videos.drive_id
|
||||
)
|
||||
ORDER BY drive_id ASC, id ASC`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListVideoFileIDsByDrive 只返回某 drive 下所有视频的 file_id 集合,
|
||||
// 比 ListVideosByDrive 轻量。
|
||||
func (c *Catalog) ListVideoFileIDsByDrive(ctx context.Context, driveID string) ([]string, error) {
|
||||
@@ -657,8 +708,10 @@ func (c *Catalog) ListVideoFileIDsByDrive(ctx context.Context, driveID string) (
|
||||
func (c *Catalog) ListSpider91Viewkeys(ctx context.Context, driveID string) ([]string, error) {
|
||||
prefix := "spider91-" + driveID + "-"
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT SUBSTR(id, ?) FROM videos WHERE id LIKE ? || '%'`,
|
||||
len(prefix)+1, prefix)
|
||||
`SELECT SUBSTR(id, ?) FROM videos WHERE id LIKE ? || '%'
|
||||
UNION
|
||||
SELECT SUBSTR(id, ?) FROM deleted_videos WHERE id LIKE ? || '%'`,
|
||||
len(prefix)+1, prefix, len(prefix)+1, prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -676,6 +729,69 @@ func (c *Catalog) ListSpider91Viewkeys(ctx context.Context, driveID string) ([]s
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// DeleteVideoWithTombstone records that an administrator explicitly deleted a
|
||||
// video, then removes the visible catalog row. The tombstone is used by
|
||||
// scanners/crawlers to avoid importing the same source file again.
|
||||
func (c *Catalog) DeleteVideoWithTombstone(ctx context.Context, id string) error {
|
||||
tx, err := c.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
var v struct {
|
||||
ID string
|
||||
DriveID string
|
||||
FileID string
|
||||
ContentHash string
|
||||
FileName string
|
||||
Size int64
|
||||
}
|
||||
row := tx.QueryRowContext(ctx, `
|
||||
SELECT id, drive_id, file_id, COALESCE(content_hash, ''), COALESCE(file_name, ''), size_bytes
|
||||
FROM videos
|
||||
WHERE id = ?`, id)
|
||||
if err := row.Scan(&v.ID, &v.DriveID, &v.FileID, &v.ContentHash, &v.FileName, &v.Size); err != nil {
|
||||
return err
|
||||
}
|
||||
v.ContentHash = normalizeContentHash(v.ContentHash)
|
||||
|
||||
// 先记录这次视频关联的 tag_id,便于事务末尾清理孤儿 collection 标签。
|
||||
tagIDs, err := collectVideoTagIDs(ctx, tx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO deleted_videos (id, drive_id, file_id, content_hash, file_name, size_bytes, deleted_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
drive_id = excluded.drive_id,
|
||||
file_id = excluded.file_id,
|
||||
content_hash = excluded.content_hash,
|
||||
file_name = excluded.file_name,
|
||||
size_bytes = excluded.size_bytes,
|
||||
deleted_at = excluded.deleted_at`,
|
||||
v.ID, v.DriveID, v.FileID, v.ContentHash, v.FileName, v.Size, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM video_tags WHERE video_id = ?`, id); err != nil {
|
||||
return err
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `DELETE FROM videos WHERE id = ?`, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rows, err := res.RowsAffected(); err == nil && rows == 0 {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
if err := pruneOrphanCollectionTagsByID(ctx, tx, tagIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (c *Catalog) DeleteVideo(ctx context.Context, id string) error {
|
||||
tx, err := c.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
@@ -709,6 +825,55 @@ func (c *Catalog) DeleteVideo(ctx context.Context, id string) error {
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (c *Catalog) IsVideoDeleted(ctx context.Context, id string) (bool, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
return false, nil
|
||||
}
|
||||
var found int
|
||||
err := c.db.QueryRowContext(ctx, `SELECT 1 FROM deleted_videos WHERE id = ? LIMIT 1`, id).Scan(&found)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) IsDeletedVideoCandidate(ctx context.Context, id, driveID, fileID, contentHash, fileName string, size int64) (bool, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
driveID = strings.TrimSpace(driveID)
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
contentHash = normalizeContentHash(contentHash)
|
||||
fileName = strings.TrimSpace(fileName)
|
||||
if id == "" && driveID == "" {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
var found int
|
||||
err := c.db.QueryRowContext(ctx, `
|
||||
SELECT 1
|
||||
FROM deleted_videos
|
||||
WHERE id = ?
|
||||
OR (drive_id = ? AND ? != '' AND file_id = ?)
|
||||
OR (drive_id = ? AND ? != '' AND content_hash = ?)
|
||||
OR (drive_id = ? AND ? != '' AND ? > 0 AND file_name = ? AND size_bytes = ?)
|
||||
LIMIT 1`,
|
||||
id,
|
||||
driveID, fileID, fileID,
|
||||
driveID, contentHash, contentHash,
|
||||
driveID, fileName, size, fileName, size,
|
||||
).Scan(&found)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) FindVideoByContentHash(ctx context.Context, hash string) (*Video, error) {
|
||||
hash = normalizeContentHash(hash)
|
||||
if hash == "" {
|
||||
@@ -765,6 +930,37 @@ func (c *Catalog) ListVideosNeedingFingerprint(ctx context.Context, driveID stri
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListVideosByFingerprintStatus lists visible videos on a drive by fingerprint status.
|
||||
// It is used by the admin "retry failed fingerprints" action to reset failed rows
|
||||
// back to pending and enqueue them again.
|
||||
func (c *Catalog) ListVideosByFingerprintStatus(ctx context.Context, driveID, status string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10000
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = ?
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT ?`,
|
||||
driveID, status, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) UpdateVideoFingerprint(ctx context.Context, id, sampledSHA256, status, errText string) error {
|
||||
sampledSHA256 = normalizeContentHash(sampledSHA256)
|
||||
if status == "" {
|
||||
@@ -798,6 +994,7 @@ type ListParams struct {
|
||||
Sort string // latest | hot | week | long
|
||||
ThumbnailReadyOnly bool
|
||||
PreferReadyThumbnails bool
|
||||
SkipTotal bool
|
||||
Page int
|
||||
PageSize int
|
||||
}
|
||||
@@ -822,12 +1019,7 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
|
||||
args = append(args, p.DriveID)
|
||||
}
|
||||
if p.Tag != "" {
|
||||
where = append(where, `EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
JOIN tags t ON t.id = vt.tag_id
|
||||
WHERE vt.video_id = videos.id AND t.label = ? COLLATE NOCASE
|
||||
)`)
|
||||
where = append(where, videoMatchesTagLabelSQL("videos"))
|
||||
args = append(args, p.Tag)
|
||||
}
|
||||
if p.Category != "" && p.Category != "all" {
|
||||
@@ -838,6 +1030,7 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
|
||||
where = append(where, "COALESCE(thumbnail_url, '') != ''")
|
||||
}
|
||||
where = append(where, "COALESCE(hidden, 0) = 0")
|
||||
where = append(where, activeDriveWhereSQL)
|
||||
where = append(where, uniqueVideoWhereSQL)
|
||||
|
||||
whereSQL := ""
|
||||
@@ -859,10 +1052,11 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
|
||||
orderBy = " ORDER BY " + readyOrderPrefix + "duration_seconds DESC"
|
||||
}
|
||||
|
||||
// count
|
||||
var total int
|
||||
if err := c.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM videos"+whereSQL, args...).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
if !p.SkipTotal {
|
||||
if err := c.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM videos"+whereSQL, args...).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
}
|
||||
|
||||
// list
|
||||
@@ -893,6 +1087,7 @@ func (c *Catalog) CountVisibleVideos(ctx context.Context) (int, error) {
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM videos
|
||||
WHERE COALESCE(hidden, 0) = 0
|
||||
AND `+activeDriveWhereSQL+`
|
||||
AND `+uniqueVideoWhereSQL,
|
||||
).Scan(&total)
|
||||
if err != nil {
|
||||
@@ -906,6 +1101,14 @@ func (c *Catalog) CountVisibleVideos(ctx context.Context) (int, error) {
|
||||
// 如果剩余可选数量 < limit,就返回所有可选项;调用方负责判断是否需要开新一轮。
|
||||
// limit <= 0 时返回 nil, nil。
|
||||
func (c *Catalog) RandomVideosExcluding(ctx context.Context, excludeIDs []string, limit int) ([]*Video, error) {
|
||||
return c.randomVideosExcluding(ctx, excludeIDs, limit, false)
|
||||
}
|
||||
|
||||
func (c *Catalog) RandomVideosWithReadyThumbnailsExcluding(ctx context.Context, excludeIDs []string, limit int) ([]*Video, error) {
|
||||
return c.randomVideosExcluding(ctx, excludeIDs, limit, true)
|
||||
}
|
||||
|
||||
func (c *Catalog) randomVideosExcluding(ctx context.Context, excludeIDs []string, limit int, thumbnailReadyOnly bool) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -913,7 +1116,11 @@ func (c *Catalog) RandomVideosExcluding(ctx context.Context, excludeIDs []string
|
||||
cleaned := cleanVideoIDs(excludeIDs)
|
||||
args := make([]any, 0, len(cleaned)+1)
|
||||
whereSQL := `WHERE COALESCE(hidden, 0) = 0
|
||||
AND ` + activeDriveWhereSQL + `
|
||||
AND ` + uniqueVideoWhereSQL
|
||||
if thumbnailReadyOnly {
|
||||
whereSQL += " AND COALESCE(thumbnail_url, '') != ''"
|
||||
}
|
||||
if len(cleaned) > 0 {
|
||||
placeholders := strings.Repeat("?,", len(cleaned))
|
||||
placeholders = placeholders[:len(placeholders)-1]
|
||||
@@ -993,6 +1200,7 @@ func (c *Catalog) LeastPopulatedVisibleUniqueTag(ctx context.Context, labels []s
|
||||
`SELECT COUNT(*)
|
||||
FROM videos
|
||||
WHERE COALESCE(hidden, 0) = 0
|
||||
AND `+activeDriveWhereSQL+`
|
||||
AND `+uniqueVideoWhereSQL+`
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
@@ -1029,6 +1237,7 @@ func (c *Catalog) RandomVideosByTagExcluding(ctx context.Context, tag string, ex
|
||||
args := make([]any, 0, len(cleaned)+2)
|
||||
args = append(args, tag)
|
||||
whereSQL := `WHERE COALESCE(hidden, 0) = 0
|
||||
AND ` + activeDriveWhereSQL + `
|
||||
AND ` + uniqueVideoWhereSQL + `
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
@@ -1278,7 +1487,7 @@ func (c *Catalog) ListLocalMediaRefs(ctx context.Context) ([]LocalMediaRef, erro
|
||||
|
||||
// DuplicateAssetCleanupCandidate points at a non-canonical video in a
|
||||
// size+sampled_sha256 duplicate group that still owns generated local assets.
|
||||
// The cleanup job uses this to remove duplicate thumbnails/teasers without
|
||||
// The cleanup job uses this to remove duplicate thumbnails/preview videos without
|
||||
// touching the original cloud file or deleting the catalog row.
|
||||
type DuplicateAssetCleanupCandidate struct {
|
||||
VideoID string
|
||||
@@ -1406,7 +1615,7 @@ type Drive struct {
|
||||
Credentials map[string]string `json:"credentials,omitempty"`
|
||||
Status string `json:"status"`
|
||||
LastError string `json:"lastError,omitempty"`
|
||||
// TeaserEnabled 控制是否给本盘生成 teaser/封面。
|
||||
// TeaserEnabled 控制是否给本盘生成预览视频/封面。
|
||||
// 替代早期的全局 preview.enabled 开关;新建 drive 时 UpsertDrive 默认置 true。
|
||||
TeaserEnabled bool `json:"teaserEnabled"`
|
||||
// SkipDirIDs 是用户在管理后台为该盘选定的"扫描跳过目录"集合(网盘侧的目录 fileID)。
|
||||
@@ -1472,10 +1681,7 @@ func normalizeDriveRootID(kind, rootID string) string {
|
||||
}
|
||||
return rootID
|
||||
case "localstorage", "spider91":
|
||||
if rootID == "" {
|
||||
return "/"
|
||||
}
|
||||
return rootID
|
||||
return "/"
|
||||
default:
|
||||
if rootID == "" {
|
||||
return "0"
|
||||
@@ -1542,7 +1748,7 @@ func (c *Catalog) DeleteDrive(ctx context.Context, id string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// SetDriveTeaserEnabled 切换某盘的 teaser/封面生成开关。
|
||||
// SetDriveTeaserEnabled 切换某盘的预览视频/封面生成开关。
|
||||
//
|
||||
// 与 UpsertDrive 的区别:只动 teaser_enabled + updated_at 一列,不要求调用方
|
||||
// 重传 kind / name / credentials 等容易踩坑的字段。
|
||||
@@ -1679,6 +1885,17 @@ COALESCE(category, ''), COALESCE(hidden, 0), COALESCE(badges, '[]'), COALESCE(de
|
||||
published_at, created_at, updated_at
|
||||
`
|
||||
|
||||
const activeDriveWhereSQL = `(videos.drive_id = 'local-upload'
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM drives
|
||||
WHERE drives.id = videos.drive_id
|
||||
)
|
||||
OR NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM drives
|
||||
))`
|
||||
|
||||
const uniqueVideoWhereSQL = `((COALESCE(videos.content_hash, '') = ''
|
||||
OR NOT EXISTS (
|
||||
SELECT 1
|
||||
|
||||
@@ -61,6 +61,7 @@ func TestUpsertDriveDefaultsRootIDByKind(t *testing.T) {
|
||||
{id: "onedrive", kind: "onedrive", want: "root"},
|
||||
{id: "googledrive", kind: "googledrive", want: "root"},
|
||||
{id: "localstorage", kind: "localstorage", want: "/"},
|
||||
{id: "spider91", kind: "spider91", want: "/"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
@@ -82,3 +83,44 @@ func TestUpsertDriveDefaultsRootIDByKind(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertDriveIgnoresRootIDForLocalStorageAndSpider91(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, tc := range []struct {
|
||||
id string
|
||||
kind string
|
||||
}{
|
||||
{id: "localstorage", kind: "localstorage"},
|
||||
{id: "spider91", kind: "spider91"},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: tc.id,
|
||||
Kind: tc.kind,
|
||||
Name: tc.kind,
|
||||
RootID: "manual-root",
|
||||
ScanRootID: "manual-scan-root",
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert %s: %v", tc.kind, err)
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, tc.id)
|
||||
if err != nil {
|
||||
t.Fatalf("get %s: %v", tc.kind, err)
|
||||
}
|
||||
if got.RootID != "/" {
|
||||
t.Fatalf("%s rootId = %q, want /", tc.kind, got.RootID)
|
||||
}
|
||||
if got.ScanRootID != "/" {
|
||||
t.Fatalf("%s scanRootId = %q, want /", tc.kind, got.ScanRootID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -126,3 +127,50 @@ func TestListSpider91ViewkeysFindsMigratedVideos(t *testing.T) {
|
||||
t.Fatalf("non-existent drive: got %v, want empty", other)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteVideoWithTombstonePreventsReimport(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: "spider91-91Spider-vk004",
|
||||
DriveID: "91Spider",
|
||||
FileID: "vk004.mp4",
|
||||
FileName: "vk004.mp4",
|
||||
ContentHash: "ABCDEF",
|
||||
Title: "Deleted Spider",
|
||||
Size: 2048,
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert: %v", err)
|
||||
}
|
||||
|
||||
if err := cat.DeleteVideoWithTombstone(ctx, "spider91-91Spider-vk004"); err != nil {
|
||||
t.Fatalf("delete with tombstone: %v", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "spider91-91Spider-vk004"); err != sql.ErrNoRows {
|
||||
t.Fatalf("get deleted video error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
deleted, err := cat.IsDeletedVideoCandidate(ctx, "spider91-91Spider-vk004", "91Spider", "vk004.mp4", "abcdef", "vk004.mp4", 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("check deleted candidate: %v", err)
|
||||
}
|
||||
if !deleted {
|
||||
t.Fatal("deleted candidate was not recognized")
|
||||
}
|
||||
viewkeys, err := cat.ListSpider91Viewkeys(ctx, "91Spider")
|
||||
if err != nil {
|
||||
t.Fatalf("ListSpider91Viewkeys: %v", err)
|
||||
}
|
||||
if len(viewkeys) != 1 || viewkeys[0] != "vk004" {
|
||||
t.Fatalf("viewkeys = %#v, want [vk004]", viewkeys)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestListVideosHidesMissingDriveVideosWhenDrivesExist(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: "active-drive",
|
||||
Kind: "pikpak",
|
||||
Name: "Active",
|
||||
RootID: "root",
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{
|
||||
ID: "visible-video",
|
||||
DriveID: "active-drive",
|
||||
FileID: "visible-file",
|
||||
Title: "Visible",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "orphan-video",
|
||||
DriveID: "deleted-drive",
|
||||
FileID: "orphan-file",
|
||||
Title: "Orphan",
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed video %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 10, Sort: "latest"})
|
||||
if err != nil {
|
||||
t.Fatalf("list videos: %v", err)
|
||||
}
|
||||
if total != 1 || len(items) != 1 || items[0].ID != "visible-video" {
|
||||
t.Fatalf("items total=%d items=%v, want only visible-video", total, items)
|
||||
}
|
||||
}
|
||||
@@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS videos (
|
||||
thumbnail_url TEXT,
|
||||
thumbnail_status TEXT DEFAULT 'pending', -- pending / ready / failed / skipped
|
||||
thumbnail_failures INTEGER DEFAULT 0, -- consecutive transient thumbnail generation failures
|
||||
preview_file_id TEXT, -- deprecated: 旧版回写网盘后的 teaser file id
|
||||
preview_local TEXT, -- 本地 teaser 路径(兜底)
|
||||
preview_file_id TEXT, -- deprecated: 旧版回写网盘后的预览视频 file id
|
||||
preview_local TEXT, -- 本地预览视频路径(兜底)
|
||||
preview_status TEXT DEFAULT 'pending', -- pending / ready / failed
|
||||
views INTEGER DEFAULT 0,
|
||||
favorites INTEGER DEFAULT 0,
|
||||
@@ -70,17 +70,36 @@ CREATE TABLE IF NOT EXISTS deleted_tags (
|
||||
deleted_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- 管理员显式删除过的视频。用于防止后续扫描 / spider91 爬虫把同一个源文件
|
||||
-- 再次入库;不代表原始云盘文件已被删除。
|
||||
CREATE TABLE IF NOT EXISTS deleted_videos (
|
||||
id TEXT PRIMARY KEY,
|
||||
drive_id TEXT NOT NULL DEFAULT '',
|
||||
file_id TEXT NOT NULL DEFAULT '',
|
||||
content_hash TEXT NOT NULL DEFAULT '',
|
||||
file_name TEXT NOT NULL DEFAULT '',
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
deleted_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_deleted_videos_drive_file
|
||||
ON deleted_videos(drive_id, file_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_deleted_videos_drive_hash
|
||||
ON deleted_videos(drive_id, content_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_deleted_videos_drive_signature
|
||||
ON deleted_videos(drive_id, file_name, size_bytes);
|
||||
|
||||
-- 网盘账户
|
||||
CREATE TABLE IF NOT EXISTS drives (
|
||||
id TEXT PRIMARY KEY,
|
||||
kind TEXT NOT NULL, -- quark / p115 / pikpak / wopan / onedrive / googledrive / localstorage / spider91
|
||||
kind TEXT NOT NULL, -- quark / p115 / p123 / pikpak / wopan / onedrive / googledrive / localstorage / spider91
|
||||
name TEXT NOT NULL,
|
||||
root_id TEXT NOT NULL DEFAULT '0',
|
||||
scan_root_id TEXT, -- deprecated: 扫描起点固定等于 root_id
|
||||
credentials TEXT, -- JSON: cookie / refresh_token 等
|
||||
status TEXT DEFAULT 'disconnected', -- disconnected / ok / error
|
||||
last_error TEXT,
|
||||
-- 是否给该盘生成 teaser/封面:1 开 / 0 关。
|
||||
-- 是否给该盘生成预览视频/封面:1 开 / 0 关。
|
||||
-- 替代了早期的全局 preview.enabled 设置(保留旧 setting 行不再读)。
|
||||
teaser_enabled INTEGER NOT NULL DEFAULT 1,
|
||||
-- 扫描时要跳过的目录 ID 集合(JSON array of string)。命中其中任意一个的目录及其
|
||||
|
||||
@@ -110,6 +110,62 @@ func TestRandomVideosExcluding(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosWithReadyThumbnailsExcluding(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < 4; i++ {
|
||||
id := "ready-" + string(rune('a'+i))
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: "f-" + id,
|
||||
Title: id,
|
||||
ThumbnailURL: "/p/thumb/" + id,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
id := "pending-" + string(rune('a'+i))
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: "f-" + id,
|
||||
Title: id,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosWithReadyThumbnailsExcluding(ctx, []string{"ready-a"}, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("random ready excluding: %v", err)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("ready random count = %d, want 3", len(got))
|
||||
}
|
||||
for _, v := range got {
|
||||
if v.ID == "ready-a" {
|
||||
t.Fatal("excluded ready video was returned")
|
||||
}
|
||||
if v.ThumbnailURL == "" {
|
||||
t.Fatalf("pending video %q was returned", v.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoChoosesLeastPopulatedTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
@@ -66,10 +66,10 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "thumbnail_failures", "INTEGER DEFAULT 0"); err != nil {
|
||||
return err
|
||||
}
|
||||
// drives.teaser_enabled:每盘 teaser 开关,替代旧的全局 preview.enabled。
|
||||
// drives.teaser_enabled:每盘预览视频开关,替代旧的全局 preview.enabled。
|
||||
// 升级路径:直接让 ALTER TABLE 的 DEFAULT 1 兜底 —— 每个现存 drive 都默认开启,
|
||||
// 不读旧的 settings.preview.enabled 字段。这样老用户即便之前关过全局开关,
|
||||
// 升级后所有盘也都恢复"默认生成 teaser",跟新建保持一致。
|
||||
// 升级后所有盘也都恢复"默认生成预览视频",跟新建保持一致。
|
||||
if _, err := c.addColumnIfMissingReportNew(ctx, "drives", "teaser_enabled", "INTEGER NOT NULL DEFAULT 1"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -79,6 +79,18 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "drives", "skip_dir_ids", "TEXT NOT NULL DEFAULT '[]'"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS deleted_videos (
|
||||
id TEXT PRIMARY KEY,
|
||||
drive_id TEXT NOT NULL DEFAULT '',
|
||||
file_id TEXT NOT NULL DEFAULT '',
|
||||
content_hash TEXT NOT NULL DEFAULT '',
|
||||
file_name TEXT NOT NULL DEFAULT '',
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
deleted_at INTEGER NOT NULL
|
||||
)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.syncDriveScanRootIDToRootID(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -100,15 +112,36 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_content_hash ON videos(content_hash)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_content_hash_created ON videos(content_hash, created_at, id)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_sampled_sha256 ON videos(size_bytes, sampled_sha256)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_sampled_sha256_created ON videos(size_bytes, sampled_sha256, created_at, id)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_hidden ON videos(hidden)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_visible_pub ON videos(COALESCE(hidden, 0), published_at DESC)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_file_name_size ON videos(file_name, size_bytes)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_file_name_size_created ON videos(file_name, size_bytes, created_at, id)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_deleted_videos_drive_file ON deleted_videos(drive_id, file_id)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_deleted_videos_drive_hash ON deleted_videos(drive_id, content_hash)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_deleted_videos_drive_signature ON deleted_videos(drive_id, file_name, size_bytes)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.seedSystemTags(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -127,6 +160,12 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.clearVolatileOneDriveThumbnails(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.clearRemoteP123ThumbnailsOnce(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.clearRemoteNonSpider91Thumbnails(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.hideZeroSizeVideosFromKnownDrives(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -175,7 +214,7 @@ func (c *Catalog) addColumnIfMissingReportNew(ctx context.Context, table, column
|
||||
// 设为 1(开启),但仅在历史上没跑过这条迁移时执行(用 marker setting 记号)。
|
||||
//
|
||||
// 为什么需要:早期短暂存在过的版本会从旧的全局 preview.enabled = "0" 同步到
|
||||
// 所有 drive 的 teaser_enabled = 0;用户报告升级后页面全显示"Teaser 关"。新版
|
||||
// 所有 drive 的 teaser_enabled = 0;用户报告升级后页面全显示"预览视频关"。新版
|
||||
// 约定 per-drive 默认开启,所以这里跑一次性修正。
|
||||
//
|
||||
// 幂等保证:marker setting 设过了就不再跑,确保用户在 UI 里把某盘关了不会被
|
||||
@@ -257,6 +296,85 @@ UPDATE videos
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) clearRemoteP123ThumbnailsOnce(ctx context.Context) error {
|
||||
// 123 云盘列表返回的缩略图尺寸和稳定性都不适合作为站内封面;清空历史写入的
|
||||
// 远程 URL,让封面 worker 统一从视频直链抽帧生成本地 /p/thumb/<id>。
|
||||
const markerKey = "videos.p123.remote_thumbnails_cleared"
|
||||
marker, err := c.GetSetting(ctx, markerKey, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read %s marker: %w", markerKey, err)
|
||||
}
|
||||
if strings.TrimSpace(marker) == "1" {
|
||||
return nil
|
||||
}
|
||||
|
||||
var p123Drives int
|
||||
if err := c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM drives WHERE kind = 'p123'`).Scan(&p123Drives); err != nil {
|
||||
return fmt.Errorf("count p123 drives: %w", err)
|
||||
}
|
||||
if p123Drives == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
res, err := c.db.ExecContext(ctx, `
|
||||
UPDATE videos
|
||||
SET thumbnail_url = '',
|
||||
thumbnail_status = 'pending',
|
||||
thumbnail_failures = 0,
|
||||
updated_at = ?
|
||||
WHERE EXISTS (
|
||||
SELECT 1
|
||||
FROM drives
|
||||
WHERE drives.id = videos.drive_id
|
||||
AND drives.kind = 'p123'
|
||||
)
|
||||
AND (
|
||||
lower(COALESCE(thumbnail_url, '')) LIKE 'http://%'
|
||||
OR lower(COALESCE(thumbnail_url, '')) LIKE 'https://%'
|
||||
)
|
||||
`, time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected > 0 {
|
||||
log.Printf("[catalog] cleared %d remote 123pan thumbnail(s) for local regeneration", affected)
|
||||
}
|
||||
if err := c.SetSetting(ctx, markerKey, "1"); err != nil {
|
||||
return fmt.Errorf("write %s marker: %w", markerKey, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Catalog) clearRemoteNonSpider91Thumbnails(ctx context.Context) error {
|
||||
// 非 91Spider 视频不再使用网盘侧返回的远程缩略图。清空历史 http/https
|
||||
// thumbnail_url 后,封面 worker 会重新从视频中间帧生成本地 /p/thumb/<id>。
|
||||
// 91Spider 的封面是爬虫下载后保存到本地 /p/thumb/<id>,不受这条规则影响。
|
||||
res, err := c.db.ExecContext(ctx, `
|
||||
UPDATE videos
|
||||
SET thumbnail_url = '',
|
||||
thumbnail_status = 'pending',
|
||||
thumbnail_failures = 0,
|
||||
updated_at = ?
|
||||
WHERE (
|
||||
lower(COALESCE(thumbnail_url, '')) LIKE 'http://%'
|
||||
OR lower(COALESCE(thumbnail_url, '')) LIKE 'https://%'
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM drives
|
||||
WHERE drives.id = videos.drive_id
|
||||
AND drives.kind = 'spider91'
|
||||
)
|
||||
`, time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected > 0 {
|
||||
log.Printf("[catalog] cleared %d remote non-91Spider thumbnail(s) for local regeneration", affected)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Catalog) hideZeroSizeVideosFromKnownDrives(ctx context.Context) error {
|
||||
_, err := c.db.ExecContext(ctx, `
|
||||
UPDATE videos
|
||||
@@ -302,7 +420,15 @@ func (c *Catalog) classifySystemTags(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (c *Catalog) backfillVideoTags(ctx context.Context) error {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT id, COALESCE(tags, '[]') FROM videos`)
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT id, COALESCE(tags, '[]')
|
||||
FROM videos
|
||||
WHERE COALESCE(tags, '') NOT IN ('', '[]', 'null')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
WHERE vt.video_id = videos.id
|
||||
)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -319,11 +445,14 @@ func (c *Catalog) backfillVideoTags(ctx context.Context) error {
|
||||
if len(labels) == 0 {
|
||||
continue
|
||||
}
|
||||
if err := c.addVideoTags(ctx, videoID, labels, "legacy", true); err != nil {
|
||||
added, err := c.addVideoTags(ctx, videoID, labels, "legacy", true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return err
|
||||
if added {
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -392,6 +521,57 @@ func (c *Catalog) CreateTagAndClassify(ctx context.Context, label string, aliase
|
||||
return c.classifyTag(ctx, tag)
|
||||
}
|
||||
|
||||
func (c *Catalog) EnsureTagForVideoIDPrefix(ctx context.Context, prefix, label string, aliases []string, source string) (int, error) {
|
||||
prefix = strings.TrimSpace(prefix)
|
||||
if prefix == "" {
|
||||
return 0, errors.New("video id prefix is required")
|
||||
}
|
||||
tag, err := c.ensureTag(ctx, label, aliases, source)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT v.id
|
||||
FROM videos v
|
||||
WHERE v.id LIKE ? || '%'
|
||||
AND COALESCE(v.tags_manual, 0) = 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
WHERE vt.video_id = v.id
|
||||
AND vt.tag_id = ?
|
||||
)
|
||||
ORDER BY v.id ASC`, prefix, tag.ID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var videoIDs []string
|
||||
for rows.Next() {
|
||||
var videoID string
|
||||
if err := rows.Scan(&videoID); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
videoIDs = append(videoIDs, videoID)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, videoID := range videoIDs {
|
||||
if err := c.insertVideoTag(ctx, videoID, tag.ID, "auto"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return len(videoIDs), nil
|
||||
}
|
||||
|
||||
func (c *Catalog) DeleteTag(ctx context.Context, tagID int64) (int, error) {
|
||||
tx, err := c.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
@@ -453,10 +633,66 @@ func (c *Catalog) DeleteTag(ctx context.Context, tagID int64) (int, error) {
|
||||
|
||||
func (c *Catalog) ListTags(ctx context.Context) ([]Tag, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT t.id, t.label, t.aliases, t.source, COUNT(v.id) AS cnt
|
||||
WITH tagged_tags AS (
|
||||
SELECT vt.tag_id,
|
||||
tagged.id,
|
||||
COALESCE(tagged.content_hash, '') AS content_hash,
|
||||
COALESCE(tagged.sampled_sha256, '') AS sampled_sha256,
|
||||
tagged.size_bytes,
|
||||
COALESCE(tagged.file_name, '') AS file_name
|
||||
FROM video_tags vt
|
||||
JOIN videos tagged ON tagged.id = vt.video_id
|
||||
WHERE COALESCE(tagged.hidden, 0) = 0
|
||||
),
|
||||
tag_candidates AS (
|
||||
SELECT tag_id, id AS video_id
|
||||
FROM tagged_tags
|
||||
UNION ALL
|
||||
SELECT tag_id,
|
||||
(SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_tags.content_hash != ''
|
||||
AND canonical.content_hash = tagged_tags.content_hash
|
||||
AND COALESCE(canonical.content_hash, '') != ''
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_tags
|
||||
WHERE content_hash != ''
|
||||
UNION ALL
|
||||
SELECT tag_id,
|
||||
(SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_tags.sampled_sha256 != ''
|
||||
AND tagged_tags.size_bytes > 0
|
||||
AND canonical.sampled_sha256 = tagged_tags.sampled_sha256
|
||||
AND canonical.size_bytes = tagged_tags.size_bytes
|
||||
AND COALESCE(canonical.sampled_sha256, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_tags
|
||||
WHERE sampled_sha256 != '' AND size_bytes > 0
|
||||
UNION ALL
|
||||
SELECT tag_id,
|
||||
(SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_tags.file_name != ''
|
||||
AND tagged_tags.size_bytes > 0
|
||||
AND canonical.file_name = tagged_tags.file_name
|
||||
AND canonical.size_bytes = tagged_tags.size_bytes
|
||||
AND COALESCE(canonical.file_name, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_tags
|
||||
WHERE file_name != '' AND size_bytes > 0
|
||||
)
|
||||
SELECT t.id, t.label, t.aliases, t.source, COUNT(DISTINCT videos.id) AS cnt
|
||||
FROM tags t
|
||||
LEFT JOIN video_tags vt ON vt.tag_id = t.id
|
||||
LEFT JOIN videos v ON v.id = vt.video_id AND COALESCE(v.hidden, 0) = 0
|
||||
LEFT JOIN tag_candidates tc ON tc.tag_id = t.id AND tc.video_id IS NOT NULL
|
||||
LEFT JOIN videos ON videos.id = tc.video_id
|
||||
AND COALESCE(videos.hidden, 0) = 0
|
||||
AND `+uniqueVideoWhereSQL+`
|
||||
GROUP BY t.id, t.label, t.aliases, t.source
|
||||
ORDER BY cnt DESC, t.label ASC`)
|
||||
if err != nil {
|
||||
@@ -474,6 +710,66 @@ ORDER BY cnt DESC, t.label ASC`)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func videoMatchesTagLabelSQL(videoAlias string) string {
|
||||
return fmt.Sprintf(`%s.id IN (
|
||||
WITH tagged_videos AS (
|
||||
SELECT tagged.id,
|
||||
COALESCE(tagged.content_hash, '') AS content_hash,
|
||||
COALESCE(tagged.sampled_sha256, '') AS sampled_sha256,
|
||||
tagged.size_bytes,
|
||||
COALESCE(tagged.file_name, '') AS file_name
|
||||
FROM video_tags vt
|
||||
JOIN tags tag_filter ON tag_filter.id = vt.tag_id
|
||||
JOIN videos tagged ON tagged.id = vt.video_id
|
||||
WHERE tag_filter.label = ? COLLATE NOCASE
|
||||
AND COALESCE(tagged.hidden, 0) = 0
|
||||
),
|
||||
tag_candidates AS (
|
||||
SELECT id AS video_id
|
||||
FROM tagged_videos
|
||||
UNION ALL
|
||||
SELECT (SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_videos.content_hash != ''
|
||||
AND canonical.content_hash = tagged_videos.content_hash
|
||||
AND COALESCE(canonical.content_hash, '') != ''
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_videos
|
||||
WHERE content_hash != ''
|
||||
UNION ALL
|
||||
SELECT (SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_videos.sampled_sha256 != ''
|
||||
AND tagged_videos.size_bytes > 0
|
||||
AND canonical.sampled_sha256 = tagged_videos.sampled_sha256
|
||||
AND canonical.size_bytes = tagged_videos.size_bytes
|
||||
AND COALESCE(canonical.sampled_sha256, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_videos
|
||||
WHERE sampled_sha256 != '' AND size_bytes > 0
|
||||
UNION ALL
|
||||
SELECT (SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_videos.file_name != ''
|
||||
AND tagged_videos.size_bytes > 0
|
||||
AND canonical.file_name = tagged_videos.file_name
|
||||
AND canonical.size_bytes = tagged_videos.size_bytes
|
||||
AND COALESCE(canonical.file_name, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_videos
|
||||
WHERE file_name != '' AND size_bytes > 0
|
||||
)
|
||||
SELECT video_id
|
||||
FROM tag_candidates
|
||||
WHERE video_id IS NOT NULL
|
||||
)`, videoAlias)
|
||||
}
|
||||
|
||||
func (c *Catalog) SetManualVideoTags(ctx context.Context, videoID string, labels []string) error {
|
||||
if _, err := c.GetVideo(ctx, videoID); err != nil {
|
||||
return err
|
||||
@@ -592,6 +888,10 @@ func (c *Catalog) getTagByLabel(ctx context.Context, label string) (Tag, error)
|
||||
}
|
||||
|
||||
func (c *Catalog) classifyTag(ctx context.Context, tag Tag) (int, error) {
|
||||
existingIDs, err := c.videoIDSetForTagID(ctx, tag.ID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT id, title, COALESCE(author, ''), COALESCE(category, ''), COALESCE(tags_manual, 0)
|
||||
FROM videos`)
|
||||
@@ -623,13 +923,14 @@ FROM videos`)
|
||||
continue
|
||||
}
|
||||
}
|
||||
added, err := c.addVideoTag(ctx, videoID, tag.Label, "auto", false)
|
||||
if err != nil {
|
||||
if existingIDs[videoID] {
|
||||
continue
|
||||
}
|
||||
if err := c.insertVideoTag(ctx, videoID, tag.ID, "auto"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if added {
|
||||
classified++
|
||||
}
|
||||
existingIDs[videoID] = true
|
||||
classified++
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -689,17 +990,22 @@ func (c *Catalog) replaceVideoTags(ctx context.Context, videoID string, labels [
|
||||
return c.syncVideoTagsJSON(ctx, videoID, manual)
|
||||
}
|
||||
|
||||
func (c *Catalog) addVideoTags(ctx context.Context, videoID string, labels []string, source string, createMissing bool) error {
|
||||
func (c *Catalog) addVideoTags(ctx context.Context, videoID string, labels []string, source string, createMissing bool) (bool, error) {
|
||||
labels = uniqueStrings(cleanLabels(labels))
|
||||
if source != "manual" {
|
||||
labels = c.filterDeletedTagLabels(ctx, labels)
|
||||
}
|
||||
changed := false
|
||||
for _, label := range labels {
|
||||
if _, err := c.addVideoTag(ctx, videoID, label, source, createMissing); err != nil {
|
||||
return err
|
||||
added, err := c.addVideoTag(ctx, videoID, label, source, createMissing)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if added {
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) addVideoTag(ctx context.Context, videoID, label, source string, createMissing bool) (bool, error) {
|
||||
@@ -729,12 +1035,33 @@ func (c *Catalog) addVideoTag(ctx context.Context, videoID, label, source string
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) insertVideoTag(ctx context.Context, videoID string, tagID int64, source string) error {
|
||||
_, err := c.db.ExecContext(ctx,
|
||||
`INSERT OR IGNORE INTO video_tags (video_id, tag_id, source, created_at) VALUES (?, ?, ?, ?)`,
|
||||
videoID, tagID, source, time.Now().UnixMilli())
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) addCollectionTagToVideos(ctx context.Context, category string) error {
|
||||
return c.addTagToVideosByCategory(ctx, category, category, "auto")
|
||||
}
|
||||
|
||||
func (c *Catalog) addTagToVideosByCategory(ctx context.Context, category, label, source string) error {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT id FROM videos WHERE category = ? AND COALESCE(tags_manual, 0) = 0`, category)
|
||||
tag, err := c.getTagByLabel(ctx, label)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT v.id
|
||||
FROM videos v
|
||||
WHERE v.category = ?
|
||||
AND COALESCE(v.tags_manual, 0) = 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
WHERE vt.video_id = v.id
|
||||
AND vt.tag_id = ?
|
||||
)`, category, tag.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -753,7 +1080,7 @@ func (c *Catalog) addTagToVideosByCategory(ctx context.Context, category, label,
|
||||
return err
|
||||
}
|
||||
for _, videoID := range videoIDs {
|
||||
if _, err := c.addVideoTag(ctx, videoID, label, source, false); err != nil {
|
||||
if err := c.insertVideoTag(ctx, videoID, tag.ID, source); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
@@ -837,6 +1164,23 @@ func (c *Catalog) videoIDsForTagID(ctx context.Context, tagID int64) ([]string,
|
||||
return videoIDs, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) videoIDSetForTagID(ctx context.Context, tagID int64) (map[string]bool, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT video_id FROM video_tags WHERE tag_id = ?`, tagID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var videoID string
|
||||
if err := rows.Scan(&videoID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[videoID] = true
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) validateTagsExist(ctx context.Context, labels []string) error {
|
||||
for _, label := range labels {
|
||||
if _, err := c.getTagByLabel(ctx, label); err != nil {
|
||||
|
||||
@@ -326,6 +326,75 @@ func TestCreateTagAndClassifyRestoresDeletedTag(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureTagForVideoIDPrefixBackfillsSourceTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, seed := range []struct {
|
||||
id string
|
||||
manual bool
|
||||
}{
|
||||
{id: "spider91-91-spider-1200001"},
|
||||
{id: "spider91-91-spider-1200002", manual: true},
|
||||
{id: "spider91-other-1200003"},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: seed.id,
|
||||
DriveID: "91-spider",
|
||||
FileID: seed.id + ".mp4",
|
||||
Title: "legacy title without source text",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", seed.id, err)
|
||||
}
|
||||
if seed.manual {
|
||||
if err := cat.SetManualVideoTags(ctx, seed.id, nil); err != nil {
|
||||
t.Fatalf("mark %s manual: %v", seed.id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
added, err := cat.EnsureTagForVideoIDPrefix(ctx, "spider91-91-spider-", "91porn", nil, "system")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure prefix tag: %v", err)
|
||||
}
|
||||
if added != 1 {
|
||||
t.Fatalf("added = %d, want 1", added)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, "spider91-91-spider-1200001")
|
||||
if err != nil {
|
||||
t.Fatalf("get tagged video: %v", err)
|
||||
}
|
||||
if !sameStrings(got.Tags, []string{"91porn"}) {
|
||||
t.Fatalf("tagged video tags = %#v, want 91porn", got.Tags)
|
||||
}
|
||||
manual, err := cat.GetVideo(ctx, "spider91-91-spider-1200002")
|
||||
if err != nil {
|
||||
t.Fatalf("get manual video: %v", err)
|
||||
}
|
||||
if len(manual.Tags) != 0 {
|
||||
t.Fatalf("manual video tags = %#v, want unchanged", manual.Tags)
|
||||
}
|
||||
other, err := cat.GetVideo(ctx, "spider91-other-1200003")
|
||||
if err != nil {
|
||||
t.Fatalf("get other prefix video: %v", err)
|
||||
}
|
||||
if len(other.Tags) != 0 {
|
||||
t.Fatalf("other prefix video tags = %#v, want unchanged", other.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteTagRejectsSystemTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
@@ -398,6 +467,84 @@ VALUES
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateDoesNotRewriteAlreadySyncedVideoTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, id := range []string{"video-1", "video-2", "video-3"} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: "巨乳后入合集",
|
||||
Category: "Better Call Saul S03",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
if err := cat.migrate(ctx); err != nil {
|
||||
t.Fatalf("first migrate: %v", err)
|
||||
}
|
||||
|
||||
before := videoUpdatedAtByID(t, ctx, cat, "video-1", "video-2", "video-3")
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
if err := cat.migrate(ctx); err != nil {
|
||||
t.Fatalf("second migrate: %v", err)
|
||||
}
|
||||
after := videoUpdatedAtByID(t, ctx, cat, "video-1", "video-2", "video-3")
|
||||
for id, want := range before {
|
||||
if got := after[id]; got != want {
|
||||
t.Fatalf("%s updated_at changed on no-op migrate: got %d, want %d", id, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateBackfillsLegacyTagsWithoutRelations(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
if _, err := cat.db.ExecContext(ctx, `
|
||||
INSERT INTO videos (id, drive_id, file_id, title, tags, tags_manual, published_at, created_at, updated_at)
|
||||
VALUES ('legacy-video', 'drive', 'file-legacy', 'legacy title', '["legacy-tag"]', 0, ?, ?, ?)`,
|
||||
now, now, now); err != nil {
|
||||
t.Fatalf("seed legacy video: %v", err)
|
||||
}
|
||||
if err := cat.migrate(ctx); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
|
||||
tag := mustTagByLabel(t, ctx, cat, "legacy-tag")
|
||||
var count int
|
||||
if err := cat.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM video_tags WHERE video_id = 'legacy-video' AND tag_id = ?`, tag.ID).Scan(&count); err != nil {
|
||||
t.Fatalf("count video tag: %v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("legacy video tag relation count = %d, want 1", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenMigratesLegacyVideosWithoutFileName(t *testing.T) {
|
||||
path := t.TempDir() + "/catalog.db"
|
||||
db, err := sql.Open("sqlite", path)
|
||||
@@ -657,7 +804,7 @@ func TestMigrateCollapsesAVCodeTagsIntoAV(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateClearsVolatileOneDriveThumbnailURLs(t *testing.T) {
|
||||
func TestMigrateClearsRemoteNonSpiderThumbnailURLs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -680,6 +827,36 @@ func TestMigrateClearsVolatileOneDriveThumbnailURLs(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatalf("seed onedrive: %v", err)
|
||||
}
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: "p123-main",
|
||||
Kind: "p123",
|
||||
Name: "123Pan",
|
||||
RootID: "root",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed p123: %v", err)
|
||||
}
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: "pikpak-main",
|
||||
Kind: "pikpak",
|
||||
Name: "PikPak",
|
||||
RootID: "root",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed pikpak: %v", err)
|
||||
}
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: "spider91-main",
|
||||
Kind: "spider91",
|
||||
Name: "91Spider",
|
||||
RootID: "root",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed spider91: %v", err)
|
||||
}
|
||||
|
||||
videos := []*Video{
|
||||
{
|
||||
@@ -703,6 +880,27 @@ func TestMigrateClearsVolatileOneDriveThumbnailURLs(t *testing.T) {
|
||||
Title: "PikPak",
|
||||
ThumbnailURL: "https://sg-thumbnail-drive.mypikpak.net/v0/screenshot-thumbnails/demo",
|
||||
},
|
||||
{
|
||||
ID: "p123-remote-thumb-video",
|
||||
DriveID: "p123-main",
|
||||
FileID: "file-4",
|
||||
Title: "123Pan remote thumb",
|
||||
ThumbnailURL: "https://download.123pan.com/thumb/file_70_70?w=70&h=70",
|
||||
},
|
||||
{
|
||||
ID: "p123-local-thumb-video",
|
||||
DriveID: "p123-main",
|
||||
FileID: "file-5",
|
||||
Title: "123Pan local thumb",
|
||||
ThumbnailURL: "/p/thumb/p123-local-thumb-video",
|
||||
},
|
||||
{
|
||||
ID: "spider91-local-thumb-video",
|
||||
DriveID: "spider91-main",
|
||||
FileID: "file-6",
|
||||
Title: "91Spider local thumb",
|
||||
ThumbnailURL: "/p/thumb/spider91-local-thumb-video",
|
||||
},
|
||||
}
|
||||
for _, v := range videos {
|
||||
v.PublishedAt = now
|
||||
@@ -737,8 +935,39 @@ func TestMigrateClearsVolatileOneDriveThumbnailURLs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("get pikpak video: %v", err)
|
||||
}
|
||||
if pikpak.ThumbnailURL == "" {
|
||||
t.Fatal("pikpak thumbnail was cleared")
|
||||
if pikpak.ThumbnailURL != "" {
|
||||
t.Fatalf("pikpak thumbnail = %q, want cleared", pikpak.ThumbnailURL)
|
||||
}
|
||||
|
||||
p123Remote, err := cat.GetVideo(ctx, "p123-remote-thumb-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get p123 remote thumb video: %v", err)
|
||||
}
|
||||
if p123Remote.ThumbnailURL != "" {
|
||||
t.Fatalf("p123 remote thumbnail = %q, want cleared", p123Remote.ThumbnailURL)
|
||||
}
|
||||
var p123Status string
|
||||
if err := cat.db.QueryRowContext(ctx, `SELECT thumbnail_status FROM videos WHERE id = ?`, "p123-remote-thumb-video").Scan(&p123Status); err != nil {
|
||||
t.Fatalf("read p123 thumbnail status: %v", err)
|
||||
}
|
||||
if p123Status != "pending" {
|
||||
t.Fatalf("p123 remote thumbnail_status = %q, want pending", p123Status)
|
||||
}
|
||||
|
||||
p123Local, err := cat.GetVideo(ctx, "p123-local-thumb-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get p123 local thumb video: %v", err)
|
||||
}
|
||||
if p123Local.ThumbnailURL != "/p/thumb/p123-local-thumb-video" {
|
||||
t.Fatalf("p123 local thumbnail = %q, want preserved", p123Local.ThumbnailURL)
|
||||
}
|
||||
|
||||
spider91Local, err := cat.GetVideo(ctx, "spider91-local-thumb-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get spider91 local thumb video: %v", err)
|
||||
}
|
||||
if spider91Local.ThumbnailURL != "/p/thumb/spider91-local-thumb-video" {
|
||||
t.Fatalf("spider91 local thumbnail = %q, want preserved", spider91Local.ThumbnailURL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -859,6 +1088,98 @@ func TestListVideosHidesDuplicateContentHashes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTagFilterMatchesCanonicalDuplicateVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{
|
||||
ID: "pikpak-canonical",
|
||||
DriveID: "pikpak",
|
||||
FileID: "canonical.mp4",
|
||||
Title: "Canonical",
|
||||
Size: 1024,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "spider91-dup-1",
|
||||
DriveID: "91-spider",
|
||||
FileID: "dup-1.mp4",
|
||||
Title: "Spider duplicate 1",
|
||||
Tags: []string{"91porn"},
|
||||
Size: 1024,
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
{
|
||||
ID: "spider91-dup-2",
|
||||
DriveID: "91-spider",
|
||||
FileID: "dup-2.mp4",
|
||||
Title: "Spider duplicate 2",
|
||||
Tags: []string{"91porn"},
|
||||
Size: 1024,
|
||||
PublishedAt: now.Add(2 * time.Second),
|
||||
CreatedAt: now.Add(2 * time.Second),
|
||||
UpdatedAt: now.Add(2 * time.Second),
|
||||
},
|
||||
{
|
||||
ID: "spider91-visible",
|
||||
DriveID: "91-spider",
|
||||
FileID: "visible.mp4",
|
||||
Title: "Spider visible",
|
||||
Tags: []string{"91porn"},
|
||||
Size: 2048,
|
||||
PublishedAt: now.Add(3 * time.Second),
|
||||
CreatedAt: now.Add(3 * time.Second),
|
||||
UpdatedAt: now.Add(3 * time.Second),
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"pikpak-canonical", "spider91-dup-1", "spider91-dup-2"} {
|
||||
if err := cat.UpdateVideoFingerprint(ctx, id, "same-sampled-sha256", "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "spider91-visible", "unique-sampled-sha256", "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint visible: %v", err)
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{Tag: "91porn", Page: 1, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list videos by tag: %v", err)
|
||||
}
|
||||
if total != 2 || len(items) != 2 {
|
||||
t.Fatalf("tagged videos total=%d len=%d, want 2", total, len(items))
|
||||
}
|
||||
gotIDs := map[string]bool{}
|
||||
for _, item := range items {
|
||||
gotIDs[item.ID] = true
|
||||
}
|
||||
for _, want := range []string{"pikpak-canonical", "spider91-visible"} {
|
||||
if !gotIDs[want] {
|
||||
t.Fatalf("tagged video ids = %#v, want %s", gotIDs, want)
|
||||
}
|
||||
}
|
||||
if got := mustTagByLabel(t, ctx, cat, "91porn").Count; got != 2 {
|
||||
t.Fatalf("91porn count = %d, want 2 visible canonical videos", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListVideosCanFilterReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
@@ -944,6 +1265,19 @@ func mustTagByLabel(t *testing.T, ctx context.Context, cat *Catalog, label strin
|
||||
return Tag{}
|
||||
}
|
||||
|
||||
func videoUpdatedAtByID(t *testing.T, ctx context.Context, cat *Catalog, ids ...string) map[string]int64 {
|
||||
t.Helper()
|
||||
out := make(map[string]int64, len(ids))
|
||||
for _, id := range ids {
|
||||
var updatedAt int64
|
||||
if err := cat.db.QueryRowContext(ctx, `SELECT updated_at FROM videos WHERE id = ?`, id).Scan(&updatedAt); err != nil {
|
||||
t.Fatalf("read updated_at for %s: %v", id, err)
|
||||
}
|
||||
out[id] = updatedAt
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// 删除 collection 标签的最后一个引用视频后,标签应当自动从 tags 表里消失。
|
||||
// user/system 标签不受影响:用户/系统标签的语义由人维护,孤儿状态保留。
|
||||
func TestDeleteVideoPrunesOrphanCollectionTag(t *testing.T) {
|
||||
|
||||
@@ -16,6 +16,11 @@ const (
|
||||
DefaultAdminPassword = "admin123"
|
||||
)
|
||||
|
||||
var (
|
||||
legacyDefaultVideoExtensions = []string{".mp4", ".mkv", ".mov", ".webm", ".avi"}
|
||||
defaultVideoExtensions = []string{".mp4", ".mkv", ".mov", ".webm", ".avi", ".strm"}
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Server Server `yaml:"server"`
|
||||
Storage Storage `yaml:"storage"`
|
||||
@@ -202,7 +207,7 @@ type Nightly struct {
|
||||
// 这里保留 yaml 中的静态定义,用于启动时预置盘。生产建议只在 DB 里维护。
|
||||
type Drive struct {
|
||||
ID string `yaml:"id"`
|
||||
Kind string `yaml:"kind"` // quark / p115 / pikpak / wopan / onedrive / googledrive / localstorage
|
||||
Kind string `yaml:"kind"` // quark / p115 / p123 / pikpak / wopan / onedrive / googledrive / localstorage
|
||||
Name string `yaml:"name"`
|
||||
RootID string `yaml:"root_id"`
|
||||
Params map[string]string `yaml:"params,omitempty"`
|
||||
@@ -247,7 +252,9 @@ func (c *Config) applyDefaults() {
|
||||
c.Scanner.MaxDepth = 5
|
||||
}
|
||||
if len(c.Scanner.VideoExtensions) == 0 {
|
||||
c.Scanner.VideoExtensions = []string{".mp4", ".mkv", ".mov", ".webm", ".avi"}
|
||||
c.Scanner.VideoExtensions = append([]string{}, defaultVideoExtensions...)
|
||||
} else if isLegacyDefaultVideoExtensions(c.Scanner.VideoExtensions) {
|
||||
c.Scanner.VideoExtensions = append(c.Scanner.VideoExtensions, ".strm")
|
||||
}
|
||||
if c.Preview.FFmpegPath == "" {
|
||||
c.Preview.FFmpegPath = "ffmpeg"
|
||||
@@ -276,3 +283,19 @@ func (c *Config) applyDefaults() {
|
||||
c.Nightly.CronHour = 1
|
||||
}
|
||||
}
|
||||
|
||||
func isLegacyDefaultVideoExtensions(exts []string) bool {
|
||||
if len(exts) != len(legacyDefaultVideoExtensions) {
|
||||
return false
|
||||
}
|
||||
seen := make(map[string]struct{}, len(exts))
|
||||
for _, ext := range exts {
|
||||
seen[strings.ToLower(strings.TrimSpace(ext))] = struct{}{}
|
||||
}
|
||||
for _, ext := range legacyDefaultVideoExtensions {
|
||||
if _, ok := seen[ext]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package config
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -50,3 +51,64 @@ storage:
|
||||
t.Fatalf("db path = %q, want preserved value", cfg.Storage.DBPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadDefaultScannerVideoExtensionsIncludeSTRM(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(`{}`), 0o644); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
if !hasVideoExtension(cfg.Scanner.VideoExtensions, ".strm") {
|
||||
t.Fatalf("video extensions = %#v, want .strm", cfg.Scanner.VideoExtensions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadLegacyDefaultScannerVideoExtensionsIncludeSTRM(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(`
|
||||
scanner:
|
||||
video_extensions: [".mp4", ".mkv", ".mov", ".webm", ".avi"]
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
if !hasVideoExtension(cfg.Scanner.VideoExtensions, ".strm") {
|
||||
t.Fatalf("video extensions = %#v, want .strm appended for legacy default list", cfg.Scanner.VideoExtensions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCustomScannerVideoExtensionsArePreserved(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(`
|
||||
scanner:
|
||||
video_extensions: [".mp4"]
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
if len(cfg.Scanner.VideoExtensions) != 1 || cfg.Scanner.VideoExtensions[0] != ".mp4" {
|
||||
t.Fatalf("video extensions = %#v, want custom list preserved", cfg.Scanner.VideoExtensions)
|
||||
}
|
||||
}
|
||||
|
||||
func hasVideoExtension(exts []string, want string) bool {
|
||||
want = strings.ToLower(strings.TrimSpace(want))
|
||||
for _, ext := range exts {
|
||||
if strings.ToLower(strings.TrimSpace(ext)) == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
package googledrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -21,10 +28,13 @@ import (
|
||||
const (
|
||||
Kind = "googledrive"
|
||||
defaultAPIBaseURL = "https://www.googleapis.com/drive/v3"
|
||||
defaultUploadAPIURL = "https://www.googleapis.com/upload/drive/v3"
|
||||
defaultOAuthURL = "https://www.googleapis.com/oauth2/v4/token"
|
||||
defaultRenewAPIURL = "https://api.oplist.org/googleui/renewapi"
|
||||
defaultListInterval = 1 * time.Second
|
||||
defaultListCooldown = 5 * time.Minute
|
||||
defaultLinkCooldown = 5 * time.Minute
|
||||
uploadChunkSize = int64(8 * 1024 * 1024)
|
||||
|
||||
filesListFields = "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken"
|
||||
fileInfoFields = "id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum"
|
||||
@@ -41,13 +51,19 @@ type Driver struct {
|
||||
renewAPIURL string
|
||||
oauthURL string
|
||||
apiBaseURL string
|
||||
uploadBaseURL string
|
||||
client *resty.Client
|
||||
httpClient *http.Client
|
||||
onTokenUpdate func(access, refresh string)
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
listInterval time.Duration
|
||||
listCooldown time.Duration
|
||||
|
||||
linkCooldownMu sync.Mutex
|
||||
linkCooldownUntil time.Time
|
||||
linkCooldownDuration time.Duration
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -61,6 +77,7 @@ type Config struct {
|
||||
RenewAPIURL string
|
||||
OAuthURL string
|
||||
APIBaseURL string
|
||||
UploadAPIURL string
|
||||
|
||||
OnTokenUpdate func(access, refresh string)
|
||||
}
|
||||
@@ -82,6 +99,10 @@ func New(c Config) *Driver {
|
||||
if apiBaseURL == "" {
|
||||
apiBaseURL = defaultAPIBaseURL
|
||||
}
|
||||
uploadBaseURL := strings.TrimRight(strings.TrimSpace(c.UploadAPIURL), "/")
|
||||
if uploadBaseURL == "" {
|
||||
uploadBaseURL = deriveUploadBaseURL(apiBaseURL)
|
||||
}
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
rootID: rootID,
|
||||
@@ -93,15 +114,34 @@ func New(c Config) *Driver {
|
||||
renewAPIURL: renewAPIURL,
|
||||
oauthURL: oauthURL,
|
||||
apiBaseURL: apiBaseURL,
|
||||
uploadBaseURL: uploadBaseURL,
|
||||
onTokenUpdate: c.OnTokenUpdate,
|
||||
client: resty.New().
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
listInterval: defaultListInterval,
|
||||
listCooldown: defaultListCooldown,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 0,
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
},
|
||||
listInterval: defaultListInterval,
|
||||
listCooldown: defaultListCooldown,
|
||||
linkCooldownDuration: defaultLinkCooldown,
|
||||
}
|
||||
}
|
||||
|
||||
func deriveUploadBaseURL(apiBaseURL string) string {
|
||||
apiBaseURL = strings.TrimRight(strings.TrimSpace(apiBaseURL), "/")
|
||||
if apiBaseURL == "" || apiBaseURL == defaultAPIBaseURL {
|
||||
return defaultUploadAPIURL
|
||||
}
|
||||
if strings.HasSuffix(apiBaseURL, "/drive/v3") {
|
||||
return strings.TrimSuffix(apiBaseURL, "/drive/v3") + "/upload/drive/v3"
|
||||
}
|
||||
return apiBaseURL
|
||||
}
|
||||
|
||||
func (d *Driver) Kind() string { return Kind }
|
||||
func (d *Driver) ID() string { return d.id }
|
||||
func (d *Driver) RootID() string { return d.rootID }
|
||||
@@ -209,8 +249,19 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
if fileID == "" {
|
||||
return nil, errors.New("googledrive stream: empty file id")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := d.linkCooldownError(time.Now()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := d.Stat(ctx, fileID); err != nil {
|
||||
return nil, fmt.Errorf("googledrive stream: %w", err)
|
||||
err = fmt.Errorf("googledrive stream: %w", err)
|
||||
if wait, ok := drives.RateLimitRetryAfter(err); ok {
|
||||
until := d.pauseLinkCooldown(wait)
|
||||
log.Printf("[googledrive] stream link cooling down drive=%s until=%s err=%v", d.id, until.Format(time.RFC3339), err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
u := d.fileURL(fileID) + "?alt=media&acknowledgeAbuse=true&supportsAllDrives=true"
|
||||
return &drives.StreamLink{
|
||||
@@ -222,12 +273,383 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
func (d *Driver) linkCooldownError(now time.Time) error {
|
||||
d.linkCooldownMu.Lock()
|
||||
defer d.linkCooldownMu.Unlock()
|
||||
if d.linkCooldownUntil.IsZero() {
|
||||
return nil
|
||||
}
|
||||
if !now.Before(d.linkCooldownUntil) {
|
||||
d.linkCooldownUntil = time.Time{}
|
||||
return nil
|
||||
}
|
||||
wait := d.linkCooldownUntil.Sub(now)
|
||||
if wait <= 0 {
|
||||
return nil
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: wait,
|
||||
Err: fmt.Errorf("googledrive stream link cooling down until %s", d.linkCooldownUntil.Format(time.RFC3339)),
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
func (d *Driver) pauseLinkCooldown(wait time.Duration) time.Time {
|
||||
if wait <= 0 {
|
||||
wait = d.linkCooldownDuration
|
||||
}
|
||||
if wait <= 0 {
|
||||
wait = defaultLinkCooldown
|
||||
}
|
||||
until := time.Now().Add(wait)
|
||||
d.linkCooldownMu.Lock()
|
||||
if until.After(d.linkCooldownUntil) {
|
||||
d.linkCooldownUntil = until
|
||||
} else {
|
||||
until = d.linkCooldownUntil
|
||||
}
|
||||
d.linkCooldownMu.Unlock()
|
||||
return until
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
res, err := d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return res.FileID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
parentID, name, err := d.normalizeUploadArgs(parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
sessionURL, err := d.createUploadSession(ctx, parentID, name, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if strings.TrimSpace(sessionURL) == "" {
|
||||
return UploadResult{}, errors.New("googledrive upload session: empty upload url")
|
||||
}
|
||||
|
||||
hasher := md5.New()
|
||||
var item driveFile
|
||||
var copied int64
|
||||
if size == 0 {
|
||||
completed, err := d.putUploadSessionChunkWithRetry(ctx, sessionURL, 0, 0, nil, hasher)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if completed != nil {
|
||||
item = *completed
|
||||
}
|
||||
} else {
|
||||
chunkSize := uploadChunkSize
|
||||
if chunkSize <= 0 {
|
||||
chunkSize = 8 * 1024 * 1024
|
||||
}
|
||||
if chunkSize > int64(math.MaxInt32) {
|
||||
chunkSize = int64(math.MaxInt32)
|
||||
}
|
||||
buf := make([]byte, int(chunkSize))
|
||||
for copied < size {
|
||||
partSize := minInt64(chunkSize, size-copied)
|
||||
chunk := buf[:int(partSize)]
|
||||
n, err := io.ReadFull(r, chunk)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return UploadResult{}, fmt.Errorf("googledrive upload: size mismatch: declared %d, copied %d", size, copied+int64(n))
|
||||
}
|
||||
return UploadResult{}, fmt.Errorf("googledrive upload: read body: %w", err)
|
||||
}
|
||||
chunk = chunk[:n]
|
||||
completed, err := d.putUploadSessionChunkWithRetry(ctx, sessionURL, copied, size, chunk, hasher)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if completed != nil {
|
||||
item = *completed
|
||||
}
|
||||
copied += int64(n)
|
||||
}
|
||||
}
|
||||
|
||||
hashHex := hex.EncodeToString(hasher.Sum(nil))
|
||||
if item.ID == "" {
|
||||
fileID, err := d.findUploadedFileID(ctx, parentID, name, hashHex)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
item.ID = fileID
|
||||
}
|
||||
return UploadResult{FileID: item.ID, Hash: hashHex, Size: copied}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) normalizeUploadArgs(parentID, name string, r io.Reader, size int64) (string, string, error) {
|
||||
if r == nil {
|
||||
return "", "", errors.New("googledrive upload: body is required")
|
||||
}
|
||||
if size < 0 {
|
||||
return "", "", fmt.Errorf("googledrive upload: invalid size %d", size)
|
||||
}
|
||||
parentID = strings.TrimSpace(parentID)
|
||||
if parentID == "" || parentID == "/" {
|
||||
parentID = d.rootID
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return "", "", errors.New("googledrive upload: empty file name")
|
||||
}
|
||||
return parentID, name, nil
|
||||
}
|
||||
|
||||
func (d *Driver) createUploadSession(ctx context.Context, parentID, name string, size int64) (string, error) {
|
||||
return d.createUploadSessionOnce(ctx, parentID, name, size, true)
|
||||
}
|
||||
|
||||
func (d *Driver) createUploadSessionOnce(ctx context.Context, parentID, name string, size int64, retry bool) (string, error) {
|
||||
var apiErr apiErrorResp
|
||||
res, err := d.client.R().
|
||||
SetContext(ctx).
|
||||
SetHeader("Authorization", "Bearer "+d.accessToken).
|
||||
SetHeader("X-Upload-Content-Type", mimeType(driveFile{Name: name})).
|
||||
SetHeader("X-Upload-Content-Length", strconv.FormatInt(size, 10)).
|
||||
SetQueryParams(map[string]string{
|
||||
"uploadType": "resumable",
|
||||
"supportsAllDrives": "true",
|
||||
"fields": fileInfoFields,
|
||||
}).
|
||||
SetBody(map[string]any{
|
||||
"name": name,
|
||||
"parents": []string{parentID},
|
||||
}).
|
||||
SetError(&apiErr).
|
||||
Post(d.uploadFilesURL())
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("googledrive upload session: %w", err)
|
||||
}
|
||||
if isGoogleRateLimit(res, apiErr.Error) {
|
||||
return "", googleRateLimitError(res, apiErr.Error.Message)
|
||||
}
|
||||
if apiErr.Error.Code != 0 {
|
||||
if apiErr.Error.Code == http.StatusUnauthorized && retry {
|
||||
if err := d.refresh(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return d.createUploadSessionOnce(ctx, parentID, name, size, false)
|
||||
}
|
||||
return "", googleAPIError(apiErr.Error)
|
||||
}
|
||||
if res.IsError() {
|
||||
return "", fmt.Errorf("googledrive upload session: status=%d body=%s", res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return strings.TrimSpace(res.Header().Get("Location")), nil
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunkWithRetry(ctx context.Context, uploadURL string, start, total int64, data []byte, hasher hash.Hash) (*driveFile, error) {
|
||||
var last error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if attempt > 0 {
|
||||
if err := sleepContext(ctx, time.Duration(attempt)*time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
item, retryable, err := d.putUploadSessionChunk(ctx, uploadURL, start, total, data)
|
||||
if err == nil {
|
||||
if hasher != nil && len(data) > 0 {
|
||||
_, _ = hasher.Write(data)
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
last = err
|
||||
if !retryable {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if last == nil {
|
||||
last = errors.New("googledrive upload session: retry attempts exhausted")
|
||||
}
|
||||
return nil, last
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunk(ctx context.Context, uploadURL string, start, total int64, data []byte) (*driveFile, bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
req.ContentLength = int64(len(data))
|
||||
req.Header.Set("Authorization", "Bearer "+d.accessToken)
|
||||
req.Header.Set("Content-Length", strconv.Itoa(len(data)))
|
||||
if total == 0 {
|
||||
req.Header.Set("Content-Range", "bytes */0")
|
||||
} else {
|
||||
end := start + int64(len(data)) - 1
|
||||
req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, total))
|
||||
}
|
||||
hc := d.httpClient
|
||||
if hc == nil {
|
||||
hc = http.DefaultClient
|
||||
}
|
||||
res, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, true, fmt.Errorf("googledrive upload session: put chunk: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK, http.StatusCreated:
|
||||
var item driveFile
|
||||
if err := json.NewDecoder(res.Body).Decode(&item); err != nil {
|
||||
return nil, false, fmt.Errorf("googledrive upload session: decode completed file: %w", err)
|
||||
}
|
||||
return &item, false, nil
|
||||
case http.StatusPermanentRedirect:
|
||||
return nil, false, nil
|
||||
case http.StatusUnauthorized:
|
||||
if err := d.refresh(ctx); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return nil, true, fmt.Errorf("googledrive upload session: unauthorized")
|
||||
default:
|
||||
body, _ := io.ReadAll(io.LimitReader(res.Body, 64*1024))
|
||||
var apiErr apiErrorResp
|
||||
_ = json.Unmarshal(body, &apiErr)
|
||||
if isGoogleUploadHTTPRateLimit(res.StatusCode, res.Header, body, apiErr.Error) {
|
||||
return nil, false, googleUploadRateLimitError(res.StatusCode, res.Header, body, apiErr.Error.Message)
|
||||
}
|
||||
retryable := res.StatusCode == http.StatusTooManyRequests || (res.StatusCode >= 500 && res.StatusCode <= 504)
|
||||
return nil, retryable, fmt.Errorf("googledrive upload session: status=%d body=%s", res.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
currentID := d.rootID
|
||||
for _, name := range splitPath(pathFromRoot) {
|
||||
childID, err := d.findChildDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if childID == "" {
|
||||
childID, err = d.makeDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
currentID = childID
|
||||
}
|
||||
return currentID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) findChildDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
entries, err := d.List(ctx, parentID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir && e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
var item driveFile
|
||||
err := d.request(ctx, d.filesURL(), http.MethodPost, func(req *resty.Request) {
|
||||
req.SetQueryParam("fields", fileInfoFields)
|
||||
req.SetBody(map[string]any{
|
||||
"name": name,
|
||||
"parents": []string{parentID},
|
||||
"mimeType": "application/vnd.google-apps.folder",
|
||||
})
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("googledrive mkdir %s: %w", name, err)
|
||||
}
|
||||
if item.ID == "" {
|
||||
return "", fmt.Errorf("googledrive mkdir %s: empty file id", name)
|
||||
}
|
||||
return item.ID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("googledrive rename: empty file id")
|
||||
}
|
||||
newName = strings.TrimSpace(newName)
|
||||
if newName == "" {
|
||||
return errors.New("googledrive rename: empty new name")
|
||||
}
|
||||
var item driveFile
|
||||
err := d.request(ctx, d.fileURL(fileID), http.MethodPatch, func(req *resty.Request) {
|
||||
req.SetQueryParam("fields", fileInfoFields)
|
||||
req.SetBody(map[string]string{"name": newName})
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return fmt.Errorf("googledrive rename: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) findUploadedFileID(ctx context.Context, parentID, name, md5Hex string) (string, error) {
|
||||
entries, err := d.List(ctx, parentID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("googledrive upload verify: %w", err)
|
||||
}
|
||||
var hashHit string
|
||||
for _, e := range entries {
|
||||
if e.IsDir {
|
||||
continue
|
||||
}
|
||||
if !strings.EqualFold(e.Hash, md5Hex) {
|
||||
continue
|
||||
}
|
||||
if e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
if hashHit == "" {
|
||||
hashHit = e.ID
|
||||
}
|
||||
}
|
||||
if hashHit != "" {
|
||||
return hashHit, nil
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir && e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("googledrive upload: uploaded file %q not found in parent %q", name, parentID)
|
||||
}
|
||||
|
||||
func isGoogleUploadHTTPRateLimit(status int, header http.Header, body []byte, apiErr apiErrorBody) bool {
|
||||
if status == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if status == http.StatusForbidden && strings.TrimSpace(header.Get("Retry-After")) != "" {
|
||||
return true
|
||||
}
|
||||
if isGoogleRateLimit(nil, apiErr) {
|
||||
return true
|
||||
}
|
||||
return googleLimitText(string(body))
|
||||
}
|
||||
|
||||
func googleUploadRateLimitError(status int, header http.Header, body []byte, message string) error {
|
||||
if strings.TrimSpace(message) == "" {
|
||||
message = "google drive upload rate limited"
|
||||
}
|
||||
bodyText := strings.TrimSpace(string(body))
|
||||
if bodyText != "" {
|
||||
message = fmt.Sprintf("%s: status=%d body=%s", message, status, bodyText)
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: parseRetryAfterHeader(header.Get("Retry-After")),
|
||||
Err: errors.New(message),
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) refresh(ctx context.Context) error {
|
||||
@@ -288,6 +710,26 @@ func (d *Driver) applyToken(out tokenResp) {
|
||||
}
|
||||
|
||||
func tokenResponseError(prefix string, res *resty.Response, out tokenResp, requireRefresh bool) error {
|
||||
if isGoogleTokenRateLimit(res, out) {
|
||||
message := strings.TrimSpace(out.Text)
|
||||
if message == "" {
|
||||
message = strings.TrimSpace(out.ErrorDescription)
|
||||
}
|
||||
if message == "" {
|
||||
message = strings.TrimSpace(out.Error)
|
||||
}
|
||||
if message == "" {
|
||||
message = "google drive token refresh rate limited"
|
||||
}
|
||||
if res != nil && strings.TrimSpace(res.String()) != "" {
|
||||
message = fmt.Sprintf("%s: status=%d body=%s", message, res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: parseRetryAfter(res),
|
||||
Err: fmt.Errorf("%s: %s", prefix, message),
|
||||
}
|
||||
}
|
||||
if out.Text != "" {
|
||||
return fmt.Errorf("%s: %s", prefix, out.Text)
|
||||
}
|
||||
@@ -380,6 +822,10 @@ func (d *Driver) filesURL() string {
|
||||
return d.apiBaseURL + "/files"
|
||||
}
|
||||
|
||||
func (d *Driver) uploadFilesURL() string {
|
||||
return d.uploadBaseURL + "/files"
|
||||
}
|
||||
|
||||
func (d *Driver) fileURL(fileID string) string {
|
||||
return d.filesURL() + "/" + url.PathEscape(fileID)
|
||||
}
|
||||
@@ -444,18 +890,85 @@ func isGoogleRateLimit(res *resty.Response, body apiErrorBody) bool {
|
||||
if res != nil && res.StatusCode() == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if res != nil && res.StatusCode() == http.StatusForbidden && strings.TrimSpace(res.Header().Get("Retry-After")) != "" {
|
||||
return true
|
||||
}
|
||||
if body.Code == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
for _, e := range body.Errors {
|
||||
reason := strings.ToLower(strings.TrimSpace(e.Reason))
|
||||
switch reason {
|
||||
case "ratelimitexceeded", "userratelimitexceeded", "downloadquotaexceeded", "sharingratelimitexceeded":
|
||||
if googleLimitReason(e.Reason) || googleLimitText(e.Message) {
|
||||
return true
|
||||
}
|
||||
domain := compactGoogleLimitText(e.Domain)
|
||||
if domain == "usagelimits" && (body.Code == http.StatusForbidden || body.Code == http.StatusTooManyRequests) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
msg := strings.ToLower(body.Message)
|
||||
return strings.Contains(msg, "rate limit") || strings.Contains(msg, "too many requests") || strings.Contains(msg, "quota exceeded")
|
||||
return googleLimitText(body.Message)
|
||||
}
|
||||
|
||||
func isGoogleTokenRateLimit(res *resty.Response, out tokenResp) bool {
|
||||
if res != nil {
|
||||
if res.StatusCode() == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if res.StatusCode() == http.StatusForbidden && strings.TrimSpace(res.Header().Get("Retry-After")) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return googleLimitText(out.Text) ||
|
||||
googleLimitText(out.Error) ||
|
||||
googleLimitText(out.ErrorDescription)
|
||||
}
|
||||
|
||||
func googleLimitReason(reason string) bool {
|
||||
switch compactGoogleLimitText(reason) {
|
||||
case "ratelimitexceeded",
|
||||
"userratelimitexceeded",
|
||||
"dailylimitexceeded",
|
||||
"dailylimitexceededunreg",
|
||||
"downloadquotaexceeded",
|
||||
"sharingratelimitexceeded",
|
||||
"quotaexceeded",
|
||||
"uploadlimitexceeded",
|
||||
"storagelimitexceeded",
|
||||
"storagequotaexceeded":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func googleLimitText(text string) bool {
|
||||
text = strings.ToLower(strings.TrimSpace(text))
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
compact := compactGoogleLimitText(text)
|
||||
if strings.Contains(compact, "ratelimitexceeded") ||
|
||||
strings.Contains(compact, "userratelimitexceeded") ||
|
||||
strings.Contains(compact, "dailylimitexceeded") ||
|
||||
strings.Contains(compact, "downloadquotaexceeded") ||
|
||||
strings.Contains(compact, "sharingratelimitexceeded") ||
|
||||
strings.Contains(compact, "quotaexceeded") ||
|
||||
strings.Contains(compact, "toomanyrequests") {
|
||||
return true
|
||||
}
|
||||
return strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "quota exceeded") ||
|
||||
strings.Contains(text, "download quota") ||
|
||||
strings.Contains(text, "sharing rate") ||
|
||||
strings.Contains(text, "daily limit") ||
|
||||
strings.Contains(text, "user rate") ||
|
||||
strings.Contains(text, "usage limit")
|
||||
}
|
||||
|
||||
func compactGoogleLimitText(text string) string {
|
||||
text = strings.ToLower(strings.TrimSpace(text))
|
||||
replacer := strings.NewReplacer("_", "", "-", "", " ", "", ".", "", ":", "")
|
||||
return replacer.Replace(text)
|
||||
}
|
||||
|
||||
func googleRateLimitError(res *resty.Response, message string) error {
|
||||
@@ -486,7 +999,11 @@ func parseRetryAfter(res *resty.Response) time.Duration {
|
||||
if res == nil {
|
||||
return 0
|
||||
}
|
||||
raw := strings.TrimSpace(res.Header().Get("Retry-After"))
|
||||
return parseRetryAfterHeader(res.Header().Get("Retry-After"))
|
||||
}
|
||||
|
||||
func parseRetryAfterHeader(raw string) time.Duration {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return 0
|
||||
}
|
||||
@@ -502,4 +1019,19 @@ func parseRetryAfter(res *resty.Response) time.Duration {
|
||||
return 0
|
||||
}
|
||||
|
||||
func splitPath(p string) []string {
|
||||
p = strings.Trim(p, "/")
|
||||
if p == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(p, "/")
|
||||
}
|
||||
|
||||
func minInt64(a, b int64) int64 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
|
||||
@@ -2,11 +2,18 @@ package googledrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestInitUsesOnlineRenewAPI(t *testing.T) {
|
||||
@@ -131,6 +138,134 @@ func TestStreamURLReturnsAuthenticatedMediaLinkWithoutRedirectRequirement(t *tes
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAndReportHashUsesResumableSession(t *testing.T) {
|
||||
body := "hello google drive"
|
||||
wantHash := md5.Sum([]byte(body))
|
||||
var sawSession bool
|
||||
var sawUpload bool
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/upload/drive/v3/files":
|
||||
sawSession = true
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("session Authorization = %q", got)
|
||||
}
|
||||
if got := r.URL.Query().Get("uploadType"); got != "resumable" {
|
||||
t.Fatalf("uploadType = %q", got)
|
||||
}
|
||||
if got := r.Header.Get("X-Upload-Content-Length"); got != "18" {
|
||||
t.Fatalf("X-Upload-Content-Length = %q", got)
|
||||
}
|
||||
var meta struct {
|
||||
Name string `json:"name"`
|
||||
Parents []string `json:"parents"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&meta); err != nil {
|
||||
t.Fatalf("decode session metadata: %v", err)
|
||||
}
|
||||
if meta.Name != "clip.mp4" || len(meta.Parents) != 1 || meta.Parents[0] != "parent-1" {
|
||||
t.Fatalf("metadata = %+v", meta)
|
||||
}
|
||||
w.Header().Set("Location", srv.URL+"/upload/session/1")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case "/upload/session/1":
|
||||
sawUpload = true
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("upload Authorization = %q", got)
|
||||
}
|
||||
if got := r.Header.Get("Content-Range"); got != "bytes 0-17/18" {
|
||||
t.Fatalf("Content-Range = %q", got)
|
||||
}
|
||||
gotBody, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read upload body: %v", err)
|
||||
}
|
||||
if string(gotBody) != body {
|
||||
t.Fatalf("upload body = %q", string(gotBody))
|
||||
}
|
||||
writeTestJSONStatus(w, http.StatusCreated, driveFile{
|
||||
ID: "file-uploaded",
|
||||
Name: "clip.mp4",
|
||||
Size: "18",
|
||||
MD5Checksum: hex.EncodeToString(wantHash[:]),
|
||||
})
|
||||
default:
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", APIBaseURL: srv.URL + "/drive/v3"})
|
||||
d.accessToken = "access"
|
||||
res, err := d.UploadAndReportHash(context.Background(), "parent-1", "clip.mp4", strings.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("UploadAndReportHash() error = %v", err)
|
||||
}
|
||||
if !sawSession || !sawUpload {
|
||||
t.Fatalf("saw session/upload = %v/%v, want both", sawSession, sawUpload)
|
||||
}
|
||||
if res.FileID != "file-uploaded" || res.Size != int64(len(body)) || res.Hash != hex.EncodeToString(wantHash[:]) {
|
||||
t.Fatalf("upload result = %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDirAndRenameUseGoogleDriveFileAPI(t *testing.T) {
|
||||
var madeDir bool
|
||||
var renamed bool
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/drive/v3/files":
|
||||
writeTestJSON(w, filesResp{})
|
||||
case r.Method == http.MethodPost && r.URL.Path == "/drive/v3/files":
|
||||
madeDir = true
|
||||
var meta struct {
|
||||
Name string `json:"name"`
|
||||
Parents []string `json:"parents"`
|
||||
MimeType string `json:"mimeType"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&meta); err != nil {
|
||||
t.Fatalf("decode mkdir body: %v", err)
|
||||
}
|
||||
if meta.Name != "91 Spider" || len(meta.Parents) != 1 || meta.Parents[0] != "root" || meta.MimeType != "application/vnd.google-apps.folder" {
|
||||
t.Fatalf("mkdir body = %+v", meta)
|
||||
}
|
||||
writeTestJSON(w, driveFile{ID: "folder-91", Name: "91 Spider", MimeType: "application/vnd.google-apps.folder"})
|
||||
case r.Method == http.MethodPatch && r.URL.Path == "/drive/v3/files/file-1":
|
||||
renamed = true
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode rename body: %v", err)
|
||||
}
|
||||
if body["name"] != "new-name.mp4" {
|
||||
t.Fatalf("rename body = %+v", body)
|
||||
}
|
||||
writeTestJSON(w, driveFile{ID: "file-1", Name: "new-name.mp4"})
|
||||
default:
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", RootID: "root", APIBaseURL: srv.URL + "/drive/v3"})
|
||||
d.accessToken = "access"
|
||||
d.listInterval = -1
|
||||
|
||||
dirID, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureDir() error = %v", err)
|
||||
}
|
||||
if dirID != "folder-91" || !madeDir {
|
||||
t.Fatalf("dirID/madeDir = %q/%v, want folder-91/true", dirID, madeDir)
|
||||
}
|
||||
if err := d.Rename(context.Background(), "file-1", "new-name.mp4"); err != nil {
|
||||
t.Fatalf("Rename() error = %v", err)
|
||||
}
|
||||
if !renamed {
|
||||
t.Fatal("rename endpoint was not called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestRefreshesOnUnauthorized(t *testing.T) {
|
||||
var fileCalls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -179,6 +314,88 @@ func TestRequestRefreshesOnUnauthorized(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateLimitReasonsFollowGoogleDriveErrorShape(t *testing.T) {
|
||||
reasons := []string{
|
||||
"rateLimitExceeded",
|
||||
"userRateLimitExceeded",
|
||||
"dailyLimitExceeded",
|
||||
"dailyLimitExceededUnreg",
|
||||
"downloadQuotaExceeded",
|
||||
"sharingRateLimitExceeded",
|
||||
"quotaExceeded",
|
||||
}
|
||||
for _, reason := range reasons {
|
||||
body := apiErrorBody{
|
||||
Code: http.StatusForbidden,
|
||||
Message: "google drive quota or rate limited",
|
||||
Errors: []struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
LocationType string `json:"location_type"`
|
||||
Location string `json:"location"`
|
||||
}{
|
||||
{Domain: "usageLimits", Reason: reason, Message: reason},
|
||||
},
|
||||
}
|
||||
if !isGoogleRateLimit(nil, body) {
|
||||
t.Fatalf("reason %q not treated as rate limit", reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRateLimitStartsSharedLinkCooldown(t *testing.T) {
|
||||
var calls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
w.Header().Set("Retry-After", "120")
|
||||
writeTestJSONStatus(w, http.StatusForbidden, apiErrorResp{Error: apiErrorBody{
|
||||
Code: http.StatusForbidden,
|
||||
Message: "User rate limit exceeded.",
|
||||
Errors: []struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
LocationType string `json:"location_type"`
|
||||
Location string `json:"location"`
|
||||
}{
|
||||
{Domain: "usageLimits", Reason: "userRateLimitExceeded", Message: "User rate limit exceeded."},
|
||||
},
|
||||
}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", APIBaseURL: srv.URL})
|
||||
d.accessToken = "access"
|
||||
d.linkCooldownDuration = time.Hour
|
||||
|
||||
_, err := d.StreamURL(context.Background(), "file-1")
|
||||
if err == nil {
|
||||
t.Fatal("first StreamURL succeeded, want rate limit")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("first error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != 2*time.Minute {
|
||||
t.Fatalf("retry after = %s, want 2m", rateLimit.RetryAfter)
|
||||
}
|
||||
|
||||
_, err = d.StreamURL(context.Background(), "file-1")
|
||||
if err == nil {
|
||||
t.Fatal("second StreamURL succeeded during cooldown")
|
||||
}
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("second error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("remote calls = %d, want 1; second call should use shared cooldown", calls)
|
||||
}
|
||||
if rateLimit.RetryAfter <= 0 || rateLimit.RetryAfter > 2*time.Minute {
|
||||
t.Fatalf("second retry after = %s, want remaining cooldown", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestJSON(w http.ResponseWriter, v any) {
|
||||
writeTestJSONStatus(w, http.StatusOK, v)
|
||||
}
|
||||
|
||||
@@ -42,8 +42,16 @@ type apiErrorBody struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Errors []struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
LocationType string `json:"location_type"`
|
||||
Location string `json:"location"`
|
||||
} `json:"errors"`
|
||||
}
|
||||
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
Hash string
|
||||
Size int64
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
// Drive 是多家网盘统一抽象。上层不区分盘,只区分 Kind。
|
||||
type Drive interface {
|
||||
// Kind 返回驱动代号:"quark" / "p115" / "pikpak" / "wopan" / "onedrive" / "googledrive" / "localstorage"
|
||||
// Kind 返回驱动代号:"quark" / "p115" / "p123" / "pikpak" / "wopan" / "onedrive" / "googledrive" / "localstorage"
|
||||
Kind() string
|
||||
|
||||
// ID 返回该盘在 catalog 中的唯一标识
|
||||
@@ -30,7 +30,7 @@ type Drive interface {
|
||||
StreamURL(ctx context.Context, fileID string) (*StreamLink, error)
|
||||
|
||||
// Upload 把本地流写入指定目录,返回新文件 fileID。
|
||||
// 当前 teaser 和封面只保存在本地,不再通过该方法写回网盘。
|
||||
// 当前预览视频和封面只保存在本地,不再通过该方法写回网盘。
|
||||
Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error)
|
||||
|
||||
// EnsureDir 保证指定路径存在(相对根目录),返回最终目录 fileID。
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -17,6 +18,8 @@ import (
|
||||
|
||||
const Kind = "localstorage"
|
||||
|
||||
const maxSTRMBytes = 64 * 1024
|
||||
|
||||
type Config struct {
|
||||
ID string
|
||||
RootPath string
|
||||
@@ -47,7 +50,7 @@ func (d *Driver) Init(context.Context) error {
|
||||
}
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("localstorage: stat root: %w", err)
|
||||
return fmt.Errorf("localstorage: stat root %q: %w%s", root, err, localStoragePathHint(d.rootPath))
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("localstorage: root is not a directory: %s", root)
|
||||
@@ -122,7 +125,13 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.IsDir() || !info.Mode().IsRegular() || info.Size() <= 0 {
|
||||
if info.IsDir() || !info.Mode().IsRegular() {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
if strings.EqualFold(filepath.Ext(p), ".strm") {
|
||||
return d.streamURLFromSTRM(ctx, p)
|
||||
}
|
||||
if info.Size() <= 0 {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return &drives.StreamLink{
|
||||
@@ -131,6 +140,115 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) streamURLFromSTRM(ctx context.Context, strmPath string) (*drives.StreamLink, error) {
|
||||
target, err := readSTRMTarget(strmPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if filepath.IsAbs(target) {
|
||||
return d.localSTRMLink(strmPath, target)
|
||||
}
|
||||
u, err := url.Parse(target)
|
||||
if err == nil {
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https":
|
||||
if u.Host == "" {
|
||||
return nil, fmt.Errorf("localstorage: invalid strm url %q", target)
|
||||
}
|
||||
return &drives.StreamLink{
|
||||
URL: target,
|
||||
Expires: time.Now().Add(24 * time.Hour),
|
||||
}, nil
|
||||
case "file":
|
||||
if u.Host != "" && !strings.EqualFold(u.Host, "localhost") {
|
||||
return nil, fmt.Errorf("localstorage: unsupported strm file url host %q", u.Host)
|
||||
}
|
||||
return d.localSTRMLink(strmPath, u.Path)
|
||||
case "":
|
||||
// Local path below.
|
||||
default:
|
||||
return nil, fmt.Errorf("localstorage: unsupported strm target scheme %q", u.Scheme)
|
||||
}
|
||||
} else if strings.Contains(target, "://") {
|
||||
return nil, fmt.Errorf("localstorage: invalid strm url %q: %w", target, err)
|
||||
}
|
||||
return d.localSTRMLink(strmPath, target)
|
||||
}
|
||||
|
||||
func readSTRMTarget(path string) (string, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(f, maxSTRMBytes+1))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(data) > maxSTRMBytes {
|
||||
return "", errors.New("localstorage: strm file is too large")
|
||||
}
|
||||
lines := strings.Split(string(data), "\n")
|
||||
for i, line := range lines {
|
||||
if i == 0 {
|
||||
line = strings.TrimPrefix(line, "\ufeff")
|
||||
}
|
||||
line = strings.TrimSpace(line)
|
||||
if line != "" {
|
||||
return line, nil
|
||||
}
|
||||
}
|
||||
return "", errors.New("localstorage: empty strm target")
|
||||
}
|
||||
|
||||
func (d *Driver) localSTRMLink(strmPath, target string) (*drives.StreamLink, error) {
|
||||
target = strings.TrimSpace(target)
|
||||
if target == "" {
|
||||
return nil, errors.New("localstorage: empty strm target")
|
||||
}
|
||||
|
||||
var p string
|
||||
if filepath.IsAbs(target) {
|
||||
p = filepath.Clean(target)
|
||||
} else {
|
||||
p = filepath.Join(filepath.Dir(strmPath), filepath.FromSlash(target))
|
||||
}
|
||||
p, err := filepath.Abs(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root, err := d.root()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
realPath, within, err := realPathWithinRoot(root, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !within {
|
||||
return nil, errors.New("localstorage: strm target escapes root")
|
||||
}
|
||||
if strings.EqualFold(filepath.Ext(p), ".strm") || strings.EqualFold(filepath.Ext(realPath), ".strm") {
|
||||
return nil, errors.New("localstorage: nested strm target is not supported")
|
||||
}
|
||||
info, err := os.Stat(realPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.IsDir() || !info.Mode().IsRegular() || info.Size() <= 0 {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return &drives.StreamLink{
|
||||
URL: realPath,
|
||||
Expires: time.Now().Add(24 * time.Hour),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
@@ -174,12 +292,63 @@ func (d *Driver) pathForID(id string) (string, string, error) {
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if p != root && !strings.HasPrefix(p, root+string(os.PathSeparator)) {
|
||||
if !pathWithinRoot(root, p) {
|
||||
return "", "", errors.New("localstorage: path escapes root")
|
||||
}
|
||||
if _, within, err := realPathWithinRoot(root, p); err != nil {
|
||||
return "", "", err
|
||||
} else if !within {
|
||||
return "", "", errors.New("localstorage: path escapes root")
|
||||
}
|
||||
return p, rel, nil
|
||||
}
|
||||
|
||||
func pathWithinRoot(root, path string) bool {
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator)))
|
||||
}
|
||||
|
||||
func realPathWithinRoot(root, path string) (string, bool, error) {
|
||||
realRoot, err := filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
realRoot, err = filepath.Abs(realRoot)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
realPath, err := filepath.EvalSymlinks(path)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
realPath, err = filepath.Abs(realPath)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return realPath, pathWithinRoot(realRoot, realPath), nil
|
||||
}
|
||||
|
||||
func localStoragePathHint(configured string) string {
|
||||
cwd, _ := os.Getwd()
|
||||
parts := []string{}
|
||||
if strings.TrimSpace(configured) != "" {
|
||||
parts = append(parts, fmt.Sprintf("configured=%q", strings.TrimSpace(configured)))
|
||||
}
|
||||
if cwd != "" {
|
||||
parts = append(parts, fmt.Sprintf("cwd=%q", cwd))
|
||||
}
|
||||
if _, err := os.Stat("/.dockerenv"); err == nil {
|
||||
parts = append(parts, "docker=host paths must be bind-mounted into the container")
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
return " (" + strings.Join(parts, ", ") + ")"
|
||||
}
|
||||
|
||||
func decodeRel(id string) (string, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" || id == "/" {
|
||||
|
||||
@@ -58,6 +58,159 @@ func TestListEncodesRelativePathsAndStreamURLResolvesFile(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLResolvesHTTPSTRM(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
strmPath := filepath.Join(root, "movie.strm")
|
||||
target := "https://media.example/clip.mp4?token=abc"
|
||||
if err := os.WriteFile(strmPath, []byte("\ufeff\n "+target+"\n"), 0o644); err != nil {
|
||||
t.Fatalf("write strm: %v", err)
|
||||
}
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
|
||||
link, err := drv.StreamURL(context.Background(), encodeRel("movie.strm"))
|
||||
if err != nil {
|
||||
t.Fatalf("stream url: %v", err)
|
||||
}
|
||||
if link.URL != target {
|
||||
t.Fatalf("url = %q, want %q", link.URL, target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLResolvesRelativeLocalSTRM(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(root, "links"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir links: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(root, "media"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir media: %v", err)
|
||||
}
|
||||
videoPath := filepath.Join(root, "media", "clip.mp4")
|
||||
if err := os.WriteFile(videoPath, []byte("video"), 0o644); err != nil {
|
||||
t.Fatalf("write video: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "links", "movie.strm"), []byte("../media/clip.mp4\n"), 0o644); err != nil {
|
||||
t.Fatalf("write strm: %v", err)
|
||||
}
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
|
||||
link, err := drv.StreamURL(context.Background(), encodeRel("links/movie.strm"))
|
||||
if err != nil {
|
||||
t.Fatalf("stream url: %v", err)
|
||||
}
|
||||
if link.URL != videoPath {
|
||||
t.Fatalf("url = %q, want %q", link.URL, videoPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRejectsInvalidSTRMTargets(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(t *testing.T, root string) string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
setup: func(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "empty.strm"), []byte("\n \r\n"))
|
||||
return "empty.strm"
|
||||
},
|
||||
want: "empty strm target",
|
||||
},
|
||||
{
|
||||
name: "escapes root",
|
||||
setup: func(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
writeLocalStorageTestFile(t, filepath.Join(filepath.Dir(root), "outside.mp4"), []byte("video"))
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "escape.strm"), []byte("../outside.mp4\n"))
|
||||
return "escape.strm"
|
||||
},
|
||||
want: "escapes root",
|
||||
},
|
||||
{
|
||||
name: "nested",
|
||||
setup: func(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "nested.strm"), []byte("https://media.example/clip.mp4\n"))
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "outer.strm"), []byte("nested.strm\n"))
|
||||
return "outer.strm"
|
||||
},
|
||||
want: "nested strm target",
|
||||
},
|
||||
{
|
||||
name: "unsupported scheme",
|
||||
setup: func(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "ftp.strm"), []byte("ftp://media.example/clip.mp4\n"))
|
||||
return "ftp.strm"
|
||||
},
|
||||
want: "unsupported strm target scheme",
|
||||
},
|
||||
{
|
||||
name: "too large",
|
||||
setup: func(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "large.strm"), []byte(strings.Repeat("x", maxSTRMBytes+1)))
|
||||
return "large.strm"
|
||||
},
|
||||
want: "strm file is too large",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
rel := tt.setup(t, root)
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
|
||||
_, err := drv.StreamURL(context.Background(), encodeRel(rel))
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("error = %v, want contain %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRejectsSTRMTargetEscapingRootThroughSymlink(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
writeLocalStorageTestFile(t, filepath.Join(outside, "secret.mp4"), []byte("secret"))
|
||||
if err := os.MkdirAll(filepath.Join(root, "links"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir links: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(root, "real"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir real: %v", err)
|
||||
}
|
||||
if err := os.Symlink(outside, filepath.Join(root, "real", "outside")); err != nil {
|
||||
t.Fatalf("symlink: %v", err)
|
||||
}
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "links", "movie.strm"), []byte("../real/outside/secret.mp4\n"))
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
|
||||
_, err := drv.StreamURL(context.Background(), encodeRel("links/movie.strm"))
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "strm target escapes root") {
|
||||
t.Fatalf("error = %v, want strm target escapes root", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRejectsSymlinkFileIDEscapingRoot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
writeLocalStorageTestFile(t, filepath.Join(outside, "secret.mp4"), []byte("secret"))
|
||||
if err := os.Symlink(filepath.Join(outside, "secret.mp4"), filepath.Join(root, "link.mp4")); err != nil {
|
||||
t.Fatalf("symlink: %v", err)
|
||||
}
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
|
||||
_, err := drv.StreamURL(context.Background(), encodeRel("link.mp4"))
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "path escapes root") {
|
||||
t.Fatalf("error = %v, want path escapes root", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRejectsEscapingID(t *testing.T) {
|
||||
drv := New(Config{ID: "local", RootPath: t.TempDir()})
|
||||
escaped := base64.RawURLEncoding.EncodeToString([]byte("../secret.mp4"))
|
||||
@@ -70,13 +223,73 @@ func TestStreamURLRejectsEscapingID(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestInitRequiresExistingDirectory(t *testing.T) {
|
||||
drv := New(Config{ID: "local", RootPath: filepath.Join(t.TempDir(), "missing")})
|
||||
missing := filepath.Join(t.TempDir(), "missing")
|
||||
drv := New(Config{ID: "local", RootPath: missing})
|
||||
|
||||
err := drv.Init(context.Background())
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "stat root") {
|
||||
t.Fatalf("error = %v, want stat root failure", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), missing) || !strings.Contains(err.Error(), "configured=") {
|
||||
t.Fatalf("error = %v, want diagnostic path details", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathForIDAllowsRootPathSlash(t *testing.T) {
|
||||
drv := New(Config{ID: "local", RootPath: string(os.PathSeparator)})
|
||||
childID := encodeRel("tmp")
|
||||
|
||||
path, rel, err := drv.pathForID(childID)
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("pathForID: %v", err)
|
||||
}
|
||||
if rel != "tmp" {
|
||||
t.Fatalf("rel = %q, want tmp", rel)
|
||||
}
|
||||
if path != filepath.Join(string(os.PathSeparator), "tmp") {
|
||||
t.Fatalf("path = %q, want /tmp", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerPersistsLocalStorageSTRM(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(root, "collection"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir collection: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "collection", "clip.strm"), []byte("https://media.example/clip.mp4\n"), 0o644); err != nil {
|
||||
t.Fatalf("write strm: %v", err)
|
||||
}
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
sc := scanner.New(cat, drv, []string{".strm"}, nil, nil)
|
||||
stats, err := sc.Run(ctx, drv.RootID())
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Added != 1 {
|
||||
t.Fatalf("added = %d, want 1", stats.Added)
|
||||
}
|
||||
|
||||
fileID := encodeRel("collection/clip.strm")
|
||||
got, err := cat.GetVideo(ctx, Kind+"-local-"+fileID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.Ext != "strm" || got.FileID != fileID || got.Category != "collection" {
|
||||
t.Fatalf("video = %#v, want local strm video in collection", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerPersistsLocalStorageVideo(t *testing.T) {
|
||||
@@ -117,3 +330,10 @@ func TestScannerPersistsLocalStorageVideo(t *testing.T) {
|
||||
t.Fatalf("video = %#v, want local drive video in collection", got)
|
||||
}
|
||||
}
|
||||
|
||||
func writeLocalStorageTestFile(t *testing.T, path string, data []byte) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,6 +149,10 @@ func sleepContext(ctx context.Context, d time.Duration) error {
|
||||
}
|
||||
|
||||
func isTransient115ListError(err error) bool {
|
||||
return isTransient115UpstreamError(err)
|
||||
}
|
||||
|
||||
func isTransient115UpstreamError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
@@ -248,11 +252,11 @@ func (d *Driver) streamURLWithUA(ctx context.Context, fileID string, ua string)
|
||||
// 需要先拿到 pickCode
|
||||
f, err := d.client.GetFile(fileID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("115 get file: %w", err)
|
||||
return nil, wrap115StreamTransientError("115 get file", err)
|
||||
}
|
||||
info, ua, err := d.downloadInfo(f.PickCode, ua)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("115 download url: %w", err)
|
||||
return nil, wrap115StreamTransientError("115 download url", err)
|
||||
}
|
||||
if info == nil || info.Url.Url == "" {
|
||||
return nil, errors.New("115 download url: empty")
|
||||
@@ -288,6 +292,18 @@ func (d *Driver) downloadInfo(pickCode string, ua string) (*sdk.DownloadInfo, st
|
||||
return info, ua, nil
|
||||
}
|
||||
|
||||
func wrap115StreamTransientError(op string, err error) error {
|
||||
wrapped := fmt.Errorf("%s: %w", op, err)
|
||||
if !isTransient115UpstreamError(err) {
|
||||
return wrapped
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: "p115",
|
||||
RetryAfter: p115ListCooldown,
|
||||
Err: wrapped,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
res, err := d.UploadAndReportSha1(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
|
||||
@@ -10,6 +10,9 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestIsTransient115ListError(t *testing.T) {
|
||||
@@ -34,6 +37,42 @@ func TestIsTransient115ListError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrap115StreamTransientError(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
wantRateLimit bool
|
||||
}{
|
||||
{name: "unexpected", err: errors.New("unexpected error"), wantRateLimit: true},
|
||||
{name: "405 blocked", err: errors.New("405 request has been blocked"), wantRateLimit: true},
|
||||
{name: "429", err: errors.New("429 too many requests"), wantRateLimit: true},
|
||||
{name: "blocked", err: errors.New("blocked by waf"), wantRateLimit: true},
|
||||
{name: "auth", err: errors.New("invalid credential"), wantRateLimit: false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := wrap115StreamTransientError("115 get file", tc.err)
|
||||
var rateLimit *drives.RateLimitError
|
||||
isRateLimit := errors.As(got, &rateLimit)
|
||||
if isRateLimit != tc.wantRateLimit {
|
||||
t.Fatalf("rate limit = %v, want %v; err=%v", isRateLimit, tc.wantRateLimit, got)
|
||||
}
|
||||
if !strings.Contains(got.Error(), "115 get file") {
|
||||
t.Fatalf("err = %v, want operation prefix", got)
|
||||
}
|
||||
if tc.wantRateLimit {
|
||||
if rateLimit.Provider != "p115" {
|
||||
t.Fatalf("provider = %q, want p115", rateLimit.Provider)
|
||||
}
|
||||
if rateLimit.RetryAfter != 10*time.Minute {
|
||||
t.Fatalf("retry after = %s, want 10m", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBufferAndHashSha1 验证 bufferAndHashSha1:
|
||||
//
|
||||
// - 把 reader 的全部字节落到 tmp 文件
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,487 @@
|
||||
package p123
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestStreamURLResolvesDownloadInfoRedirect(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var downloadReferer string
|
||||
var download *httptest.Server
|
||||
download = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/resolve":
|
||||
downloadReferer = r.Header.Get("Referer")
|
||||
http.Redirect(w, r, download.URL+"/cdn/video.mp4", http.StatusFound)
|
||||
case "/cdn/video.mp4":
|
||||
t.Fatalf("driver followed redirect unexpectedly")
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer download.Close()
|
||||
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/api/user/sign_in":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 200,
|
||||
"data": map[string]string{"token": "token-1"},
|
||||
})
|
||||
case "/b/api/user/info":
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer token-1" {
|
||||
t.Fatalf("Authorization = %q, want bearer token", got)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{}})
|
||||
case "/b/api/file/list/new":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"Next": "-1",
|
||||
"Total": 1,
|
||||
"InfoList": []map[string]any{
|
||||
{
|
||||
"FileName": "video.mp4",
|
||||
"Size": 1234,
|
||||
"UpdateAt": "2026-01-02 03:04:05",
|
||||
"FileId": 100,
|
||||
"Type": 0,
|
||||
"Etag": "ABCDEF",
|
||||
"S3KeyFlag": "flag-1",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
case "/b/api/file/download_info":
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode download_info body: %v", err)
|
||||
}
|
||||
if got := body["fileName"]; got != "video.mp4" {
|
||||
t.Fatalf("fileName = %#v, want cached file metadata", got)
|
||||
}
|
||||
if got := body["etag"]; got != "ABCDEF" {
|
||||
t.Fatalf("etag = %#v, want cached etag", got)
|
||||
}
|
||||
entryURL := download.URL + "/entry?params=" + base64.StdEncoding.EncodeToString([]byte(download.URL+"/resolve"))
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]string{"DownloadUrl": entryURL},
|
||||
})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
var savedToken string
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
Username: "user@example.com",
|
||||
Password: "secret",
|
||||
MainAPIBaseURL: api.URL + "/b/api",
|
||||
LoginAPIBaseURL: api.URL + "/api",
|
||||
OnTokenUpdate: func(access string) {
|
||||
savedToken = access
|
||||
},
|
||||
})
|
||||
if err := d.Init(ctx); err != nil {
|
||||
t.Fatalf("Init() error = %v", err)
|
||||
}
|
||||
if savedToken != "token-1" {
|
||||
t.Fatalf("saved token = %q, want token-1", savedToken)
|
||||
}
|
||||
if _, err := d.List(ctx, d.RootID()); err != nil {
|
||||
t.Fatalf("List() error = %v", err)
|
||||
}
|
||||
|
||||
link, err := d.StreamURL(ctx, "100")
|
||||
if err != nil {
|
||||
t.Fatalf("StreamURL() error = %v", err)
|
||||
}
|
||||
if got := link.URL; got != download.URL+"/cdn/video.mp4" {
|
||||
t.Fatalf("URL = %q, want final CDN URL", got)
|
||||
}
|
||||
if got := link.Headers.Get("Referer"); !strings.HasPrefix(got, download.URL) {
|
||||
t.Fatalf("Referer = %q, want original download host", got)
|
||||
}
|
||||
if downloadReferer != defaultReferer {
|
||||
t.Fatalf("resolve Referer = %q, want %q", downloadReferer, defaultReferer)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitUsesAccessTokenWithoutLogin(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/api/user/sign_in":
|
||||
t.Fatalf("driver should not password-login when access_token is configured")
|
||||
case "/b/api/user/info":
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer token-1" {
|
||||
t.Fatalf("Authorization = %q, want bearer token", got)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{}})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "Bearer token-1",
|
||||
MainAPIBaseURL: api.URL + "/b/api",
|
||||
LoginAPIBaseURL: api.URL + "/api",
|
||||
})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("Init() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginRiskErrorSuggestsAccessToken(t *testing.T) {
|
||||
err := loginError("当前账号存在境外登录风险,请使用短信验证码或者微信进行登录。")
|
||||
if err == nil || !strings.Contains(err.Error(), "access_token") {
|
||||
t.Fatalf("loginError() = %v, want access_token guidance", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestCode429ReturnsRateLimitError(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Retry-After", "2")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 429,
|
||||
"message": "请求太频繁",
|
||||
})
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "token-1",
|
||||
MainAPIBaseURL: api.URL,
|
||||
})
|
||||
_, err := d.request(context.Background(), endpointFileList, http.MethodGet, nil, nil)
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != 2*time.Second {
|
||||
t.Fatalf("RetryAfter = %s, want 2s", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListCoolsDownAndRetriesRateLimit(t *testing.T) {
|
||||
var listCalls int
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/file/list/new" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
listCalls++
|
||||
if listCalls == 1 {
|
||||
w.Header().Set("Retry-After", "1")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 429,
|
||||
"message": "请求太频繁",
|
||||
})
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"Next": "-1",
|
||||
"Total": 1,
|
||||
"InfoList": []map[string]any{
|
||||
{
|
||||
"FileName": "video.mp4",
|
||||
"Size": 1234,
|
||||
"UpdateAt": "2026-01-02 03:04:05",
|
||||
"FileId": 100,
|
||||
"Type": 0,
|
||||
"Etag": "ABCDEF",
|
||||
"S3KeyFlag": "flag-1",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "token-1",
|
||||
MainAPIBaseURL: api.URL,
|
||||
})
|
||||
entries, err := d.List(context.Background(), d.RootID())
|
||||
if err != nil {
|
||||
t.Fatalf("List() error = %v", err)
|
||||
}
|
||||
if listCalls != 2 {
|
||||
t.Fatalf("list calls = %d, want 2", listCalls)
|
||||
}
|
||||
if len(entries) != 1 || entries[0].ID != "100" {
|
||||
t.Fatalf("entries = %#v, want one file", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDownloadURL429ReturnsRateLimitError(t *testing.T) {
|
||||
download := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Retry-After", "3")
|
||||
http.Error(w, "too many requests", http.StatusTooManyRequests)
|
||||
}))
|
||||
defer download.Close()
|
||||
|
||||
d := New(Config{ID: "123-main"})
|
||||
_, err := d.resolveDownloadURL(context.Background(), download.URL)
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != 3*time.Second {
|
||||
t.Fatalf("RetryAfter = %s, want 3s", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAndReportHashUsesPresignedPUTAndComplete(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
body := []byte("video bytes for 123 upload")
|
||||
wantMD5 := fmt.Sprintf("%x", md5.Sum(body))
|
||||
|
||||
var putBody []byte
|
||||
upload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPut {
|
||||
t.Fatalf("upload method = %s, want PUT", r.Method)
|
||||
}
|
||||
if r.ContentLength != int64(len(body)) {
|
||||
t.Fatalf("ContentLength = %d, want %d", r.ContentLength, len(body))
|
||||
}
|
||||
got, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read upload body: %v", err)
|
||||
}
|
||||
putBody = got
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer upload.Close()
|
||||
|
||||
var uploadRequest map[string]any
|
||||
var uploadURLRequest map[string]any
|
||||
var completeRequest map[string]any
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/file/upload_request":
|
||||
if err := json.NewDecoder(r.Body).Decode(&uploadRequest); err != nil {
|
||||
t.Fatalf("decode upload_request: %v", err)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"FileId": 9001,
|
||||
"Bucket": "bucket-1",
|
||||
"Key": "key-1",
|
||||
"StorageNode": "node-1",
|
||||
"UploadId": "upload-1",
|
||||
},
|
||||
})
|
||||
case "/file/s3_upload_object/auth":
|
||||
if err := json.NewDecoder(r.Body).Decode(&uploadURLRequest); err != nil {
|
||||
t.Fatalf("decode s3 auth: %v", err)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"presignedUrls": map[string]string{
|
||||
"1": upload.URL + "/part-1",
|
||||
},
|
||||
},
|
||||
})
|
||||
case "/file/upload_complete/v2":
|
||||
if err := json.NewDecoder(r.Body).Decode(&completeRequest); err != nil {
|
||||
t.Fatalf("decode complete: %v", err)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{}})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "token-1",
|
||||
MainAPIBaseURL: api.URL,
|
||||
})
|
||||
res, err := d.UploadAndReportHash(ctx, "parent-1", "video.mp4", bytes.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("UploadAndReportHash() error = %v", err)
|
||||
}
|
||||
if res.FileID != "9001" {
|
||||
t.Fatalf("FileID = %q, want 9001", res.FileID)
|
||||
}
|
||||
if res.Hash != wantMD5 {
|
||||
t.Fatalf("Hash = %q, want %q", res.Hash, wantMD5)
|
||||
}
|
||||
if res.Size != int64(len(body)) {
|
||||
t.Fatalf("Size = %d, want %d", res.Size, len(body))
|
||||
}
|
||||
if !bytes.Equal(putBody, body) {
|
||||
t.Fatalf("PUT body = %q, want %q", putBody, body)
|
||||
}
|
||||
if uploadRequest["etag"] != wantMD5 {
|
||||
t.Fatalf("upload etag = %#v, want %q", uploadRequest["etag"], wantMD5)
|
||||
}
|
||||
if uploadRequest["fileName"] != "video.mp4" || uploadRequest["parentFileId"] != "parent-1" {
|
||||
t.Fatalf("upload request = %#v, want fileName and parentFileId", uploadRequest)
|
||||
}
|
||||
if uploadURLRequest["partNumberStart"].(float64) != 1 || uploadURLRequest["partNumberEnd"].(float64) != 2 {
|
||||
t.Fatalf("s3 auth request = %#v, want part range 1..2", uploadURLRequest)
|
||||
}
|
||||
if completeRequest["fileId"].(float64) != 9001 || completeRequest["fileSize"].(float64) != float64(len(body)) {
|
||||
t.Fatalf("complete request = %#v, want file id and size", completeRequest)
|
||||
}
|
||||
if completeRequest["isMultipart"].(bool) {
|
||||
t.Fatalf("complete isMultipart = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAndReportHashReuseSkipsPUTAndComplete(t *testing.T) {
|
||||
body := []byte("reused body")
|
||||
var presignedCalled bool
|
||||
var completeCalled bool
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/file/upload_request":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"FileId": 7001,
|
||||
"Reuse": true,
|
||||
},
|
||||
})
|
||||
case "/file/s3_upload_object/auth", "/file/s3_repare_upload_parts_batch":
|
||||
presignedCalled = true
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0})
|
||||
case "/file/upload_complete/v2":
|
||||
completeCalled = true
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "token-1",
|
||||
MainAPIBaseURL: api.URL,
|
||||
})
|
||||
res, err := d.UploadAndReportHash(context.Background(), "parent-1", "reused.mp4", bytes.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("UploadAndReportHash() error = %v", err)
|
||||
}
|
||||
if res.FileID != "7001" {
|
||||
t.Fatalf("FileID = %q, want 7001", res.FileID)
|
||||
}
|
||||
if presignedCalled {
|
||||
t.Fatal("reuse upload should not request presigned URLs")
|
||||
}
|
||||
if completeCalled {
|
||||
t.Fatal("reuse upload should not call upload_complete")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadPresignedPUT429ReturnsRateLimitError(t *testing.T) {
|
||||
upload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Retry-After", "4")
|
||||
http.Error(w, "too many requests", http.StatusTooManyRequests)
|
||||
}))
|
||||
defer upload.Close()
|
||||
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/file/upload_request":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"FileId": 9001,
|
||||
"Bucket": "bucket-1",
|
||||
"Key": "key-1",
|
||||
"StorageNode": "node-1",
|
||||
"UploadId": "upload-1",
|
||||
},
|
||||
})
|
||||
case "/file/s3_upload_object/auth":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"presignedUrls": map[string]string{"1": upload.URL},
|
||||
},
|
||||
})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "token-1",
|
||||
MainAPIBaseURL: api.URL,
|
||||
})
|
||||
_, err := d.UploadAndReportHash(context.Background(), "parent-1", "limited.mp4", strings.NewReader("limited"), int64(len("limited")))
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != 4*time.Second {
|
||||
t.Fatalf("RetryAfter = %s, want 4s", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameSendsExpectedBody(t *testing.T) {
|
||||
var renameRequest map[string]any
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/file/rename" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&renameRequest); err != nil {
|
||||
t.Fatalf("decode rename: %v", err)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{}})
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "token-1",
|
||||
MainAPIBaseURL: api.URL,
|
||||
})
|
||||
if err := d.Rename(context.Background(), "9001", "new name.mp4"); err != nil {
|
||||
t.Fatalf("Rename() error = %v", err)
|
||||
}
|
||||
if renameRequest["driveId"].(float64) != 0 || renameRequest["fileId"] != "9001" || renameRequest["fileName"] != "new name.mp4" {
|
||||
t.Fatalf("rename request = %#v, want driveId/fileId/fileName", renameRequest)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
package p123
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-resty/resty/v2"
|
||||
"github.com/skip2/go-qrcode"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultUserAPIBase = "https://user.123pan.cn/api"
|
||||
defaultQRLoginPage = "https://www.123pan.com/wx-app-login.html"
|
||||
defaultQRReferer = "https://user.123pan.com/centerlogin"
|
||||
defaultQROrigin = "https://user.123pan.com"
|
||||
defaultQRUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36"
|
||||
|
||||
endpointQRCodeGenerate = "/user/qr-code/generate"
|
||||
endpointQRCodeResult = "/user/qr-code/result"
|
||||
endpointQRCodeWXCode = "/user/qr-code/wx_code"
|
||||
)
|
||||
|
||||
type QRConfig struct {
|
||||
UserAPIBaseURL string
|
||||
HTTPClient *http.Client
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type QRClient struct {
|
||||
userAPIBase string
|
||||
client *resty.Client
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type QRCodeSession struct {
|
||||
LoginUUID string `json:"loginUuid"`
|
||||
UniID string `json:"uniID"`
|
||||
QRCodeURL string `json:"qrCodeUrl"`
|
||||
QRImageDataURL string `json:"qrImageDataUrl"`
|
||||
ExpiresAt string `json:"expiresAt,omitempty"`
|
||||
}
|
||||
|
||||
type QRCodeStatus struct {
|
||||
LoginStatus int `json:"loginStatus"`
|
||||
StatusText string `json:"statusText"`
|
||||
ScanPlatform int `json:"scanPlatform,omitempty"`
|
||||
PlatformText string `json:"platformText,omitempty"`
|
||||
AccessToken string `json:"accessToken,omitempty"`
|
||||
}
|
||||
|
||||
func NewQRClient(c QRConfig) *QRClient {
|
||||
userAPIBase := strings.TrimRight(strings.TrimSpace(c.UserAPIBaseURL), "/")
|
||||
if userAPIBase == "" {
|
||||
userAPIBase = defaultUserAPIBase
|
||||
}
|
||||
httpClient := c.HTTPClient
|
||||
if httpClient == nil {
|
||||
httpClient = &http.Client{Timeout: 20 * time.Second}
|
||||
}
|
||||
now := c.Now
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
return &QRClient{
|
||||
userAPIBase: userAPIBase,
|
||||
client: resty.NewWithClient(httpClient).
|
||||
SetTimeout(20*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
now: now,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *QRClient) Generate(ctx context.Context) (QRCodeSession, error) {
|
||||
loginUUID, err := newLoginUUID()
|
||||
if err != nil {
|
||||
return QRCodeSession{}, err
|
||||
}
|
||||
var resp qrGenerateResp
|
||||
res, err := c.request(ctx, loginUUID).
|
||||
SetResult(&resp).
|
||||
Get(c.userAPIBase + endpointQRCodeGenerate)
|
||||
if err != nil {
|
||||
return QRCodeSession{}, err
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return QRCodeSession{}, qrAPIError(resp.Message, res.StatusCode(), resp.Code)
|
||||
}
|
||||
uniID := strings.TrimSpace(resp.Data.UniID)
|
||||
if uniID == "" {
|
||||
return QRCodeSession{}, errors.New("123pan qr: empty uniID")
|
||||
}
|
||||
qrURL := buildQRLoginURL(resp.Data.URL, uniID)
|
||||
png, err := qrcode.Encode(qrURL, qrcode.Medium, 220)
|
||||
if err != nil {
|
||||
return QRCodeSession{}, err
|
||||
}
|
||||
return QRCodeSession{
|
||||
LoginUUID: loginUUID,
|
||||
UniID: uniID,
|
||||
QRCodeURL: qrURL,
|
||||
QRImageDataURL: "data:image/png;base64," + base64.StdEncoding.EncodeToString(png),
|
||||
ExpiresAt: c.now().Add(5 * time.Minute).Format(time.RFC3339),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *QRClient) Poll(ctx context.Context, loginUUID, uniID string) (QRCodeStatus, error) {
|
||||
loginUUID = strings.TrimSpace(loginUUID)
|
||||
uniID = strings.TrimSpace(uniID)
|
||||
if loginUUID == "" {
|
||||
return QRCodeStatus{}, errors.New("loginUuid is required")
|
||||
}
|
||||
if uniID == "" {
|
||||
return QRCodeStatus{}, errors.New("uniID is required")
|
||||
}
|
||||
var resp qrResultResp
|
||||
res, err := c.request(ctx, loginUUID).
|
||||
SetQueryParam("uniID", uniID).
|
||||
SetResult(&resp).
|
||||
Get(c.userAPIBase + endpointQRCodeResult)
|
||||
if err != nil {
|
||||
return QRCodeStatus{}, err
|
||||
}
|
||||
if resp.Code != 0 && resp.Code != 200 {
|
||||
return QRCodeStatus{}, qrAPIError(resp.Message, res.StatusCode(), resp.Code)
|
||||
}
|
||||
if resp.Code == 200 {
|
||||
resp.Data.LoginStatus = 3
|
||||
if resp.Data.ScanPlatform == 0 {
|
||||
resp.Data.ScanPlatform = resp.Data.LoginType
|
||||
}
|
||||
}
|
||||
status := QRCodeStatus{
|
||||
LoginStatus: resp.Data.LoginStatus,
|
||||
StatusText: qrLoginStatusText(resp.Data.LoginStatus),
|
||||
ScanPlatform: resp.Data.ScanPlatform,
|
||||
PlatformText: qrScanPlatformText(resp.Data.ScanPlatform),
|
||||
}
|
||||
if status.LoginStatus != 3 {
|
||||
return status, nil
|
||||
}
|
||||
if token := resp.TokenValue(); token != "" {
|
||||
status.AccessToken = normalizeAccessToken(token)
|
||||
return status, nil
|
||||
}
|
||||
if resp.Data.ScanPlatform == 4 {
|
||||
token, err := c.finishWechatLogin(ctx, loginUUID, uniID)
|
||||
if err != nil {
|
||||
return QRCodeStatus{}, err
|
||||
}
|
||||
status.AccessToken = normalizeAccessToken(token)
|
||||
return status, nil
|
||||
}
|
||||
return QRCodeStatus{}, errors.New("123pan qr: confirmed login returned empty token")
|
||||
}
|
||||
|
||||
func (c *QRClient) finishWechatLogin(ctx context.Context, loginUUID, uniID string) (string, error) {
|
||||
var wxResp qrWXCodeResp
|
||||
res, err := c.request(ctx, loginUUID).
|
||||
SetBody(map[string]string{"uniID": uniID}).
|
||||
SetResult(&wxResp).
|
||||
Post(c.userAPIBase + endpointQRCodeWXCode)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if wxResp.Code != 0 {
|
||||
return "", qrAPIError(wxResp.Message, res.StatusCode(), wxResp.Code)
|
||||
}
|
||||
wxCode := strings.TrimSpace(wxResp.WXCode())
|
||||
if wxCode == "" {
|
||||
return "", errors.New("123pan qr: empty wechat code")
|
||||
}
|
||||
var signIn loginResp
|
||||
res, err = c.request(ctx, loginUUID).
|
||||
SetBody(map[string]any{
|
||||
"from": "web",
|
||||
"wechat_code": wxCode,
|
||||
"type": 4,
|
||||
}).
|
||||
SetResult(&signIn).
|
||||
Post(c.userAPIBase + endpointSignIn)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if signIn.Code != 200 && signIn.Code != 0 {
|
||||
return "", qrAPIError(signIn.Message, res.StatusCode(), signIn.Code)
|
||||
}
|
||||
token := strings.TrimSpace(signIn.Data.Token)
|
||||
if token == "" {
|
||||
return "", errors.New("123pan qr: empty token")
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func (c *QRClient) request(ctx context.Context, loginUUID string) *resty.Request {
|
||||
return c.client.R().
|
||||
SetContext(ctx).
|
||||
SetHeaders(map[string]string{
|
||||
"Content-Type": "application/json;charset=UTF-8",
|
||||
"platform": defaultPlatform,
|
||||
"App-Version": defaultAppVersion,
|
||||
"LoginUuid": loginUUID,
|
||||
"Referer": defaultQRReferer,
|
||||
"Origin": defaultQROrigin,
|
||||
"User-Agent": defaultQRUserAgent,
|
||||
})
|
||||
}
|
||||
|
||||
func buildQRLoginURL(raw, uniID string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
raw = defaultQRLoginPage
|
||||
}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return defaultQRLoginPage + "?env=production&uniID=" + url.QueryEscape(uniID) + "&source=123pan&type=login"
|
||||
}
|
||||
q := u.Query()
|
||||
q.Set("env", "production")
|
||||
q.Set("uniID", uniID)
|
||||
q.Set("source", "123pan")
|
||||
q.Set("type", "login")
|
||||
u.RawQuery = q.Encode()
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func newLoginUUID() (string, error) {
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
b[6] = (b[6] & 0x0f) | 0x40
|
||||
b[8] = (b[8] & 0x3f) | 0x80
|
||||
parts := []string{
|
||||
hex.EncodeToString(b[0:4]),
|
||||
hex.EncodeToString(b[4:6]),
|
||||
hex.EncodeToString(b[6:8]),
|
||||
hex.EncodeToString(b[8:10]),
|
||||
hex.EncodeToString(b[10:16]),
|
||||
}
|
||||
return strings.Join(parts, "-"), nil
|
||||
}
|
||||
|
||||
func qrAPIError(message string, httpStatus, apiCode int) error {
|
||||
message = strings.TrimSpace(message)
|
||||
if message == "" {
|
||||
message = fmt.Sprintf("HTTP %d code=%d", httpStatus, apiCode)
|
||||
}
|
||||
return errors.New(message)
|
||||
}
|
||||
|
||||
func qrLoginStatusText(status int) string {
|
||||
switch status {
|
||||
case 0:
|
||||
return "等待扫码"
|
||||
case 1:
|
||||
return "已扫码,等待确认"
|
||||
case 2:
|
||||
return "已拒绝"
|
||||
case 3:
|
||||
return "已确认"
|
||||
case 4:
|
||||
return "已过期"
|
||||
default:
|
||||
return "未知状态"
|
||||
}
|
||||
}
|
||||
|
||||
func qrScanPlatformText(platform int) string {
|
||||
switch platform {
|
||||
case 4:
|
||||
return "微信"
|
||||
case 7:
|
||||
return "123 云盘 App"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
package p123
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestQRCodeGenerateBuildsImage(t *testing.T) {
|
||||
var seenLoginUUID string
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/api/user/qr-code/generate" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
seenLoginUUID = r.Header.Get("LoginUuid")
|
||||
if seenLoginUUID == "" {
|
||||
t.Fatalf("missing LoginUuid header")
|
||||
}
|
||||
if r.Header.Get("platform") != defaultPlatform {
|
||||
t.Fatalf("platform header = %q, want %q", r.Header.Get("platform"), defaultPlatform)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"message": "ok",
|
||||
"data": map[string]string{
|
||||
"uniID": "uni-1",
|
||||
"url": "https://www.123pan.com/wx-app-login.html",
|
||||
},
|
||||
})
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{UserAPIBaseURL: api.URL + "/api"}).Generate(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
if got.LoginUUID != seenLoginUUID {
|
||||
t.Fatalf("loginUuid = %q, want header %q", got.LoginUUID, seenLoginUUID)
|
||||
}
|
||||
if got.UniID != "uni-1" {
|
||||
t.Fatalf("uniID = %q, want uni-1", got.UniID)
|
||||
}
|
||||
if !strings.Contains(got.QRCodeURL, "uniID=uni-1") || !strings.Contains(got.QRCodeURL, "type=login") {
|
||||
t.Fatalf("qrCodeUrl = %q, want login params", got.QRCodeURL)
|
||||
}
|
||||
if !strings.HasPrefix(got.QRImageDataURL, "data:image/png;base64,") {
|
||||
t.Fatalf("qrImageDataUrl missing png data url prefix")
|
||||
}
|
||||
if got.ExpiresAt == "" {
|
||||
t.Fatalf("expiresAt is empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQRCodePollCompletesWechatLogin(t *testing.T) {
|
||||
var wxCodeRequested bool
|
||||
var signInRequested bool
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.Header.Get("LoginUuid") != "login-1" {
|
||||
t.Fatalf("LoginUuid = %q, want login-1", r.Header.Get("LoginUuid"))
|
||||
}
|
||||
switch r.URL.Path {
|
||||
case "/api/user/qr-code/result":
|
||||
if r.URL.Query().Get("uniID") != "uni-1" {
|
||||
t.Fatalf("uniID = %q, want uni-1", r.URL.Query().Get("uniID"))
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"loginStatus": 3,
|
||||
"scanPlatform": 4,
|
||||
},
|
||||
})
|
||||
case "/api/user/qr-code/wx_code":
|
||||
wxCodeRequested = true
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode wx_code body: %v", err)
|
||||
}
|
||||
if body["uniID"] != "uni-1" {
|
||||
t.Fatalf("wx_code uniID = %q, want uni-1", body["uniID"])
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]string{"wxCode": "wx-code-1"},
|
||||
})
|
||||
case "/api/user/sign_in":
|
||||
signInRequested = true
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode sign_in body: %v", err)
|
||||
}
|
||||
if body["wechat_code"] != "wx-code-1" {
|
||||
t.Fatalf("wechat_code = %#v, want wx-code-1", body["wechat_code"])
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 200,
|
||||
"data": map[string]string{"token": "Bearer token-1"},
|
||||
})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{UserAPIBaseURL: api.URL + "/api"}).Poll(context.Background(), "login-1", "uni-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Poll() error = %v", err)
|
||||
}
|
||||
if !wxCodeRequested || !signInRequested {
|
||||
t.Fatalf("wechat completion calls wx=%v signIn=%v, want both", wxCodeRequested, signInRequested)
|
||||
}
|
||||
if got.LoginStatus != 3 || got.AccessToken != "token-1" || got.PlatformText != "微信" {
|
||||
t.Fatalf("status = %#v, want confirmed wechat token", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQRCodePollUsesAppToken(t *testing.T) {
|
||||
var wxCodeRequested bool
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/api/user/qr-code/result":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"loginStatus": 3,
|
||||
"scanPlatform": 7,
|
||||
"token": "app-token",
|
||||
},
|
||||
})
|
||||
case "/api/user/qr-code/wx_code":
|
||||
wxCodeRequested = true
|
||||
http.Error(w, "unexpected wx_code", http.StatusInternalServerError)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{UserAPIBaseURL: api.URL + "/api"}).Poll(context.Background(), "login-1", "uni-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Poll() error = %v", err)
|
||||
}
|
||||
if wxCodeRequested {
|
||||
t.Fatalf("wx_code should not be called when app token is already returned")
|
||||
}
|
||||
if got.AccessToken != "app-token" || got.PlatformText != "123 云盘 App" {
|
||||
t.Fatalf("status = %#v, want app token", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQRCodePollUsesOfficialAppSuccessCode(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/api/user/qr-code/result" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 200,
|
||||
"data": map[string]any{
|
||||
"login_type": 7,
|
||||
"token": "app-token",
|
||||
},
|
||||
})
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{UserAPIBaseURL: api.URL + "/api"}).Poll(context.Background(), "login-1", "uni-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Poll() error = %v", err)
|
||||
}
|
||||
if got.LoginStatus != 3 || got.ScanPlatform != 7 || got.AccessToken != "app-token" {
|
||||
t.Fatalf("status = %#v, want official app success token", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package p123
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type apiEnvelope struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
type loginResp struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
Token string `json:"token"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type qrGenerateResp struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
UniID string `json:"uniID"`
|
||||
URL string `json:"url"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type qrResultResp struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
LoginStatus int `json:"loginStatus"`
|
||||
ScanPlatform int `json:"scanPlatform"`
|
||||
LoginType int `json:"login_type"`
|
||||
Token string `json:"token"`
|
||||
AccessToken string `json:"accessToken"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func (r qrResultResp) TokenValue() string {
|
||||
if strings.TrimSpace(r.Data.Token) != "" {
|
||||
return r.Data.Token
|
||||
}
|
||||
return r.Data.AccessToken
|
||||
}
|
||||
|
||||
type qrWXCodeResp struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
WXCodeLower string `json:"wxCode"`
|
||||
WXCodeTitle string `json:"WxCode"`
|
||||
Code string `json:"code"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func (r qrWXCodeResp) WXCode() string {
|
||||
if r.Data.WXCodeLower != "" {
|
||||
return r.Data.WXCodeLower
|
||||
}
|
||||
if r.Data.WXCodeTitle != "" {
|
||||
return r.Data.WXCodeTitle
|
||||
}
|
||||
return r.Data.Code
|
||||
}
|
||||
|
||||
type fileListResp struct {
|
||||
Data struct {
|
||||
Next string `json:"Next"`
|
||||
Total int `json:"Total"`
|
||||
InfoList []panFile `json:"InfoList"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type panFile struct {
|
||||
FileName string `json:"FileName"`
|
||||
Size int64 `json:"Size"`
|
||||
UpdateAt flexibleTime `json:"UpdateAt"`
|
||||
FileID int64 `json:"FileId"`
|
||||
Type int `json:"Type"`
|
||||
Etag string `json:"Etag"`
|
||||
S3KeyFlag string `json:"S3KeyFlag"`
|
||||
}
|
||||
|
||||
type cachedFile struct {
|
||||
file panFile
|
||||
parentID string
|
||||
}
|
||||
|
||||
type downloadInfoResp struct {
|
||||
Data struct {
|
||||
DownloadURL string `json:"DownloadUrl"`
|
||||
DownloadURLLower string `json:"downloadUrl"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func (r downloadInfoResp) URL() string {
|
||||
if r.Data.DownloadURL != "" {
|
||||
return r.Data.DownloadURL
|
||||
}
|
||||
return r.Data.DownloadURLLower
|
||||
}
|
||||
|
||||
type redirectResp struct {
|
||||
Data struct {
|
||||
RedirectURL string `json:"redirect_url"`
|
||||
RedirectURLCamel string `json:"redirectUrl"`
|
||||
RedirectURLTitle string `json:"RedirectUrl"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func (r redirectResp) URL() string {
|
||||
if r.Data.RedirectURL != "" {
|
||||
return r.Data.RedirectURL
|
||||
}
|
||||
if r.Data.RedirectURLCamel != "" {
|
||||
return r.Data.RedirectURLCamel
|
||||
}
|
||||
return r.Data.RedirectURLTitle
|
||||
}
|
||||
|
||||
type mkdirResp struct {
|
||||
Data struct {
|
||||
FileID int64 `json:"FileId"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type uploadResp struct {
|
||||
Data struct {
|
||||
AccessKeyID string `json:"AccessKeyId"`
|
||||
Bucket string `json:"Bucket"`
|
||||
Key string `json:"Key"`
|
||||
SecretAccessKey string `json:"SecretAccessKey"`
|
||||
SessionToken string `json:"SessionToken"`
|
||||
FileID int64 `json:"FileId"`
|
||||
Reuse bool `json:"Reuse"`
|
||||
EndPoint string `json:"EndPoint"`
|
||||
StorageNode string `json:"StorageNode"`
|
||||
UploadID string `json:"UploadId"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type s3PreSignedURLsResp struct {
|
||||
Data struct {
|
||||
PreSignedURLs map[string]string `json:"presignedUrls"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type flexibleTime struct {
|
||||
t time.Time
|
||||
}
|
||||
|
||||
func (t *flexibleTime) UnmarshalJSON(data []byte) error {
|
||||
if string(data) == "null" || string(data) == `""` {
|
||||
return nil
|
||||
}
|
||||
var s string
|
||||
if err := json.Unmarshal(data, &s); err == nil {
|
||||
t.t = parseTimeString(s)
|
||||
return nil
|
||||
}
|
||||
var n int64
|
||||
if err := json.Unmarshal(data, &n); err == nil {
|
||||
if n > 1_000_000_000_000 {
|
||||
t.t = time.UnixMilli(n)
|
||||
} else {
|
||||
t.t = time.Unix(n, 0)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t flexibleTime) Time() time.Time {
|
||||
return t.t
|
||||
}
|
||||
|
||||
func parseTimeString(s string) time.Time {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
for _, layout := range []string{
|
||||
time.RFC3339Nano,
|
||||
time.RFC3339,
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02T15:04:05",
|
||||
} {
|
||||
if parsed, err := time.ParseInLocation(layout, s, time.FixedZone("UTC+8", 8*3600)); err == nil {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
if n, err := strconv.ParseInt(s, 10, 64); err == nil {
|
||||
if n > 1_000_000_000_000 {
|
||||
return time.UnixMilli(n)
|
||||
}
|
||||
return time.Unix(n, 0)
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
@@ -263,7 +263,7 @@ func (d *Driver) findChildDir(ctx context.Context, parent, name string) (string,
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// ---------- 上传(第一版不实现,走本地 teaser 兜底) ----------
|
||||
// ---------- 上传(第一版不实现,走本地预览视频兜底) ----------
|
||||
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -20,6 +21,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
"golang.org/x/net/proxy"
|
||||
)
|
||||
|
||||
// 默认 author/tag 标签,便于在前端筛选 spider91 来源的视频。
|
||||
@@ -59,8 +62,14 @@ type CrawlerConfig struct {
|
||||
// DownloadTimeout 限制单条视频/封面下载的耗时。
|
||||
DownloadTimeout time.Duration
|
||||
|
||||
// OnNewVideo 是新视频成功入库后的回调,用于触发 teaser worker。
|
||||
// OnNewVideo 是新视频成功入库后的回调,用于触发预览视频 worker。
|
||||
OnNewVideo func(v *catalog.Video)
|
||||
// OnProgress 在抓取统计变化时触发,用于后台管理页展示实时进度。
|
||||
OnProgress func(progress CrawlProgress)
|
||||
// OnCheckedVideo 在 Python 爬虫开始检查一个列表页视频时触发。
|
||||
OnCheckedVideo func()
|
||||
// OnExtractedVideo 在 Python 爬虫提取到一个新视频直链时触发。
|
||||
OnExtractedVideo func()
|
||||
}
|
||||
|
||||
// Crawler 把 Python 爬虫产出包装成 catalog 入库流程。
|
||||
@@ -79,29 +88,123 @@ func NewCrawler(cfg CrawlerConfig) *Crawler {
|
||||
cfg.DownloadTimeout = 30 * time.Minute
|
||||
}
|
||||
if cfg.HTTPClient == nil {
|
||||
// 选 proxy 函数:显式 ProxyURL > 环境变量 > 直连
|
||||
proxyFn := http.ProxyFromEnvironment
|
||||
if strings.TrimSpace(cfg.ProxyURL) != "" {
|
||||
if u, err := url.Parse(cfg.ProxyURL); err == nil {
|
||||
proxyFn = http.ProxyURL(u)
|
||||
} else {
|
||||
log.Printf("[spider91] invalid proxy URL %q, falling back to env: %v", cfg.ProxyURL, err)
|
||||
}
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
ResponseHeaderTimeout: 60 * time.Second,
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
}
|
||||
if err := configureExplicitProxy(transport, cfg.ProxyURL); err != nil {
|
||||
log.Printf("[spider91] invalid configured proxy URL, falling back to env: %v", err)
|
||||
}
|
||||
cfg.HTTPClient = &http.Client{
|
||||
// 不限制总下载时长,靠 ctx 控制;只挡 dial / handshake / header
|
||||
Timeout: 0,
|
||||
Transport: &http.Transport{
|
||||
Proxy: proxyFn,
|
||||
ResponseHeaderTimeout: 60 * time.Second,
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
},
|
||||
Timeout: 0,
|
||||
Transport: transport,
|
||||
}
|
||||
}
|
||||
return &Crawler{cfg: cfg}
|
||||
}
|
||||
|
||||
func configureExplicitProxy(transport *http.Transport, raw string) error {
|
||||
proxyURL := strings.TrimSpace(raw)
|
||||
if proxyURL == "" {
|
||||
return nil
|
||||
}
|
||||
u, err := url.Parse(proxyURL)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return fmt.Errorf("invalid proxy URL")
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https":
|
||||
transport.Proxy = http.ProxyURL(u)
|
||||
transport.DialContext = nil
|
||||
return nil
|
||||
case "socks5", "socks5h":
|
||||
dialContext, err := socksProxyDialContext(u)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
transport.Proxy = nil
|
||||
transport.DialContext = dialContext
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("unsupported proxy scheme %q", u.Scheme)
|
||||
}
|
||||
}
|
||||
|
||||
func socksProxyDialContext(proxyURL *url.URL) (func(context.Context, string, string) (net.Conn, error), error) {
|
||||
var auth *proxy.Auth
|
||||
if proxyURL.User != nil {
|
||||
username := proxyURL.User.Username()
|
||||
password, _ := proxyURL.User.Password()
|
||||
auth = &proxy.Auth{User: username, Password: password}
|
||||
}
|
||||
dialer, err := proxy.SOCKS5("tcp", proxyURL.Host, auth, &net.Dialer{Timeout: 60 * time.Second})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remoteDNS := strings.EqualFold(proxyURL.Scheme, "socks5h")
|
||||
return func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
target := addr
|
||||
if !remoteDNS {
|
||||
resolved, err := resolveSocksTarget(ctx, addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target = resolved
|
||||
}
|
||||
if ctxDialer, ok := dialer.(proxy.ContextDialer); ok {
|
||||
return ctxDialer.DialContext(ctx, network, target)
|
||||
}
|
||||
type result struct {
|
||||
conn net.Conn
|
||||
err error
|
||||
}
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
conn, err := dialer.Dial(network, target)
|
||||
ch <- result{conn: conn, err: err}
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case res := <-ch:
|
||||
return res.conn, res.err
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveSocksTarget(ctx context.Context, addr string) (string, error) {
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil || net.ParseIP(host) != nil {
|
||||
return addr, nil
|
||||
}
|
||||
ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
ip := selectSocksTargetIP(ips)
|
||||
if ip == nil {
|
||||
return "", fmt.Errorf("resolve %s: no address", host)
|
||||
}
|
||||
return net.JoinHostPort(ip.String(), port), nil
|
||||
}
|
||||
|
||||
func selectSocksTargetIP(ips []net.IPAddr) net.IP {
|
||||
for _, addr := range ips {
|
||||
if ip4 := addr.IP.To4(); ip4 != nil {
|
||||
return ip4
|
||||
}
|
||||
}
|
||||
for _, addr := range ips {
|
||||
if addr.IP != nil {
|
||||
return addr.IP
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CrawlResult 汇总一次 RunOnce 的结果。
|
||||
type CrawlResult struct {
|
||||
// TargetNew 是本次 RunOnce 的目标新增数(来自 drive.Credentials.target_new)。
|
||||
@@ -122,6 +225,16 @@ type CrawlResult struct {
|
||||
SeenFile string
|
||||
}
|
||||
|
||||
// CrawlProgress 是 RunOnce 过程中可安全对外发布的实时计数。
|
||||
type CrawlProgress struct {
|
||||
TargetNew int
|
||||
TotalEntries int
|
||||
NewVideos int
|
||||
Skipped int
|
||||
Failed int
|
||||
SeenSnapshot int
|
||||
}
|
||||
|
||||
// spiderVideoEntry 对应 spider_91porn.py 输出 JSON 中的单条视频。
|
||||
type spiderVideoEntry struct {
|
||||
Title string `json:"title"`
|
||||
@@ -139,7 +252,7 @@ type spiderVideoEntry struct {
|
||||
// 3. Go 端 bufio.Scanner 按行读:每行立即下载视频和封面、入库。
|
||||
// 这样 "Python 翻页找下一个" 与 "Go 下载当前一个" 在时间上重叠,缩短整轮耗时;
|
||||
// 更重要的是不会让前几个下载耽误后面签名链接 e= 过期。
|
||||
// 4. 全部消费完 + 子进程退出 → 返回 CrawlResult。teaser 不在此处入队,
|
||||
// 4. 全部消费完 + 子进程退出 → 返回 CrawlResult。预览视频不在此处入队,
|
||||
// 由调用方 (App.runSpider91Crawl) 在 RunOnce 后统一调 enqueueDriveGeneration。
|
||||
//
|
||||
// targetNew <= 0 会被规范化成 spider91DefaultTargetNew(15)。
|
||||
@@ -169,6 +282,20 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
|
||||
result := &CrawlResult{TargetNew: targetNew, StartedAt: time.Now()}
|
||||
defer func() { result.FinishedAt = time.Now() }()
|
||||
emitProgress := func() {
|
||||
if c.cfg.OnProgress == nil {
|
||||
return
|
||||
}
|
||||
c.cfg.OnProgress(CrawlProgress{
|
||||
TargetNew: result.TargetNew,
|
||||
TotalEntries: result.TotalEntries,
|
||||
NewVideos: result.NewVideos,
|
||||
Skipped: result.Skipped,
|
||||
Failed: result.Failed,
|
||||
SeenSnapshot: result.SeenSnapshot,
|
||||
})
|
||||
}
|
||||
emitProgress()
|
||||
|
||||
// 1. 准备 .crawl/ 目录 + 已知源视频 ID 列表
|
||||
//
|
||||
@@ -194,6 +321,7 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
return result, fmt.Errorf("spider91 crawler: build seen list: %w", err)
|
||||
}
|
||||
result.SeenSnapshot = seenCount
|
||||
emitProgress()
|
||||
|
||||
// 2-3. 启动 Python 爬虫(流式 stdout 协议),并边读边处理。
|
||||
//
|
||||
@@ -224,9 +352,11 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
continue
|
||||
}
|
||||
result.TotalEntries++
|
||||
emitProgress()
|
||||
sourceID := sourceIDForItem(item)
|
||||
if sourceID == "" || strings.TrimSpace(item.VideoURL) == "" {
|
||||
result.Failed++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
if result.NewVideos >= targetNew {
|
||||
@@ -234,16 +364,31 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
break
|
||||
}
|
||||
videoID := buildVideoID(c.cfg.Driver.ID(), sourceID)
|
||||
deleted, err := c.cfg.Catalog.IsVideoDeleted(ctx, videoID)
|
||||
if err != nil {
|
||||
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s check deleted: %v", c.cfg.Driver.ID(), item.Viewkey, sourceID, err)
|
||||
result.Failed++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
if deleted {
|
||||
result.Skipped++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
if existing, _ := c.cfg.Catalog.GetVideo(ctx, videoID); existing != nil {
|
||||
result.Skipped++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
if perr := c.processOne(ctx, videoID, item); perr != nil {
|
||||
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s failed: %v", c.cfg.Driver.ID(), item.Viewkey, sourceID, perr)
|
||||
result.Failed++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
result.NewVideos++
|
||||
emitProgress()
|
||||
}
|
||||
if scerr := scanner.Err(); scerr != nil {
|
||||
log.Printf("[spider91] drive=%s stdout scan: %v", c.cfg.Driver.ID(), scerr)
|
||||
@@ -324,6 +469,16 @@ func (c *Crawler) startSpiderTargetNew(ctx context.Context, targetNew int, seenP
|
||||
if c.cfg.WorkDir != "" {
|
||||
cmd.Dir = c.cfg.WorkDir
|
||||
}
|
||||
if proxyURL := strings.TrimSpace(c.cfg.ProxyURL); proxyURL != "" {
|
||||
cmd.Env = append(os.Environ(),
|
||||
"HTTP_PROXY="+proxyURL,
|
||||
"HTTPS_PROXY="+proxyURL,
|
||||
"http_proxy="+proxyURL,
|
||||
"https_proxy="+proxyURL,
|
||||
"NO_PROXY=",
|
||||
"no_proxy=",
|
||||
)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("stdout pipe: %w", err)
|
||||
@@ -341,12 +496,12 @@ func (c *Crawler) startSpiderTargetNew(ctx context.Context, targetNew int, seenP
|
||||
return nil, nil, fmt.Errorf("start: %w", err)
|
||||
}
|
||||
// stderr 转发到 backend log。子进程退出时 reader 自动 EOF,goroutine 自然结束。
|
||||
go forwardSpiderLog(c.cfg.Driver.ID(), stderr)
|
||||
go forwardSpiderLog(c.cfg.Driver.ID(), stderr, c.cfg.OnCheckedVideo, c.cfg.OnExtractedVideo)
|
||||
return cmd, stdout, nil
|
||||
}
|
||||
|
||||
// forwardSpiderLog 把 Python stderr 逐行转发到 backend log,便于调试。
|
||||
func forwardSpiderLog(driveID string, r io.Reader) {
|
||||
func forwardSpiderLog(driveID string, r io.Reader, onCheckedVideo func(), onExtractedVideo func()) {
|
||||
scanner := bufio.NewScanner(r)
|
||||
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
@@ -355,9 +510,23 @@ func forwardSpiderLog(driveID string, r io.Reader) {
|
||||
continue
|
||||
}
|
||||
log.Printf("[spider91:py] drive=%s %s", driveID, line)
|
||||
if onCheckedVideo != nil && isSpider91CheckedVideoLogLine(line) {
|
||||
onCheckedVideo()
|
||||
}
|
||||
if onExtractedVideo != nil && isSpider91ExtractedVideoLogLine(line) {
|
||||
onExtractedVideo()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isSpider91CheckedVideoLogLine(line string) bool {
|
||||
return checkedVideoLogRE.MatchString(line)
|
||||
}
|
||||
|
||||
func isSpider91ExtractedVideoLogLine(line string) bool {
|
||||
return strings.Contains(line, "[OK] 成功提取视频直链")
|
||||
}
|
||||
|
||||
// processOne 处理单个 91 源视频:下载视频 + 封面 + 复制封面 + 入库。
|
||||
// 任一步失败会清理已写入的临时文件,不留半成品。
|
||||
func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVideoEntry) error {
|
||||
@@ -419,7 +588,7 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
log.Printf("[spider91] drive=%s mkdir common thumbs: %v", c.cfg.Driver.ID(), err)
|
||||
thumbReady = false
|
||||
} else {
|
||||
dst := filepath.Join(c.cfg.CommonThumbDir, videoID+".jpg")
|
||||
dst := mediaasset.ThumbnailPathInDir(c.cfg.CommonThumbDir, videoID)
|
||||
if err := copyFileAtomic(thumbPath, dst); err != nil {
|
||||
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s copy thumb to common dir: %v", c.cfg.Driver.ID(), viewkey, sourceID, err)
|
||||
thumbReady = false
|
||||
@@ -427,6 +596,17 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
}
|
||||
}
|
||||
|
||||
title := strings.TrimSpace(item.Title)
|
||||
if title == "" {
|
||||
title = sourceID
|
||||
}
|
||||
tags := []string{DefaultTag}
|
||||
if matched, err := c.cfg.Catalog.MatchTags(ctx, title+" "+DefaultAuthor); err == nil {
|
||||
tags = mergeCatalogTags(tags, matched)
|
||||
} else {
|
||||
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s match tags: %v", c.cfg.Driver.ID(), viewkey, sourceID, err)
|
||||
}
|
||||
|
||||
// 入库
|
||||
now := time.Now()
|
||||
v := &catalog.Video{
|
||||
@@ -434,9 +614,9 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
DriveID: c.cfg.Driver.ID(),
|
||||
FileID: videoFile,
|
||||
FileName: videoFile,
|
||||
Title: strings.TrimSpace(item.Title),
|
||||
Title: title,
|
||||
Author: DefaultAuthor,
|
||||
Tags: []string{DefaultTag},
|
||||
Tags: tags,
|
||||
Ext: strings.TrimPrefix(videoExt, "."),
|
||||
Quality: "HD",
|
||||
Size: videoSize,
|
||||
@@ -445,9 +625,6 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if v.Title == "" {
|
||||
v.Title = sourceID
|
||||
}
|
||||
if thumbReady {
|
||||
// 设了 ThumbnailURL 后 thumb worker 会跳过这条视频,
|
||||
// 不再尝试用 ffmpeg 抽帧(封面已经是网站原图)。
|
||||
@@ -722,9 +899,10 @@ func spider91CookieHeader(cookies []*http.Cookie) string {
|
||||
}
|
||||
|
||||
var (
|
||||
strencode2RE = regexp.MustCompile(`strencode2\(["']([^"']+)["']\)`)
|
||||
srcAttrRE = regexp.MustCompile(`src=['"]([^'"]+)['"]`)
|
||||
mp4URLRE = regexp.MustCompile(`https?://[^\s"'<>]+\.mp4[^\s"'<>]*`)
|
||||
checkedVideoLogRE = regexp.MustCompile(`处理视频\s+\d+/\d+:`)
|
||||
strencode2RE = regexp.MustCompile(`strencode2\(["']([^"']+)["']\)`)
|
||||
srcAttrRE = regexp.MustCompile(`src=['"]([^'"]+)['"]`)
|
||||
mp4URLRE = regexp.MustCompile(`https?://[^\s"'<>]+\.mp4[^\s"'<>]*`)
|
||||
)
|
||||
|
||||
func parseSpider91VideoURL(html string) string {
|
||||
@@ -888,6 +1066,26 @@ func copyFileAtomic(src, dst string) error {
|
||||
return os.Rename(tmp, dst)
|
||||
}
|
||||
|
||||
func mergeCatalogTags(lists ...[]string) []string {
|
||||
out := []string{}
|
||||
seen := map[string]bool{}
|
||||
for _, list := range lists {
|
||||
for _, tag := range list {
|
||||
tag = strings.TrimSpace(tag)
|
||||
if tag == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(tag)
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, tag)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BuildVideoID 给定 driveID + 91 源视频 ID,按统一规则生成 catalog 中 videos.id。
|
||||
// 与 scanner 用法一致:<kind>-<driveID>-<fileID>。
|
||||
func BuildVideoID(driveID, sourceID string) string {
|
||||
|
||||
@@ -3,6 +3,8 @@ package spider91
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
@@ -53,7 +55,7 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
|
||||
// 同时仍写 --output 文件作归档。
|
||||
videoEntries := []map[string]string{
|
||||
{
|
||||
"title": "Video One",
|
||||
"title": "Video One 口交",
|
||||
"thumb_url": srv.URL + "/thumb/not-120001.jpg",
|
||||
"video_url": srv.URL + "/videos/120001.mp4",
|
||||
"viewkey": "vk-001",
|
||||
@@ -94,6 +96,9 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert drive: %v", err)
|
||||
}
|
||||
if _, err := cat.CreateTagAndClassify(context.Background(), "Video One", nil, "user"); err != nil {
|
||||
t.Fatalf("create user tag: %v", err)
|
||||
}
|
||||
|
||||
var newVideos []*catalog.Video
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
@@ -188,6 +193,17 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
|
||||
if !hasDefaultTag {
|
||||
t.Fatalf("video %s tags = %v, want contain %q", videoID, v.Tags, DefaultTag)
|
||||
}
|
||||
if sourceID == "120001" {
|
||||
if !containsString(v.Tags, "口交") {
|
||||
t.Fatalf("video %s tags = %v, want contain built-in tag 口交", videoID, v.Tags)
|
||||
}
|
||||
if !containsString(v.Tags, "Video One") {
|
||||
t.Fatalf("video %s tags = %v, want contain user tag Video One", videoID, v.Tags)
|
||||
}
|
||||
}
|
||||
if sourceID == "120002" && (containsString(v.Tags, "口交") || containsString(v.Tags, "Video One")) {
|
||||
t.Fatalf("video %s tags = %v, should not inherit tags from other spider91 videos", videoID, v.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
// 7. 第二次 RunOnce:源视频 ID 已存在 → 全部 skipped,无新文件下载
|
||||
@@ -233,6 +249,108 @@ func TestCrawlerRunOnceMissingScript(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerPassesProxyToSpiderProcess(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell-based fake script only on unix")
|
||||
}
|
||||
|
||||
tmp := t.TempDir()
|
||||
scriptPath := filepath.Join(tmp, "print_proxy_env.sh")
|
||||
script := `#!/bin/sh
|
||||
printf 'HTTP_PROXY=%s\n' "$HTTP_PROXY"
|
||||
printf 'HTTPS_PROXY=%s\n' "$HTTPS_PROXY"
|
||||
printf 'http_proxy=%s\n' "$http_proxy"
|
||||
printf 'https_proxy=%s\n' "$https_proxy"
|
||||
printf 'NO_PROXY=%s\n' "$NO_PROXY"
|
||||
printf 'no_proxy=%s\n' "$no_proxy"
|
||||
`
|
||||
if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
|
||||
proxyURL := "socks5h://proxy.local:1080"
|
||||
drv := New(Config{ID: "proxy-drive", RootDir: filepath.Join(tmp, "proxy-drive")})
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
PythonPath: "sh",
|
||||
ScriptPath: scriptPath,
|
||||
ProxyURL: proxyURL,
|
||||
})
|
||||
cmd, stdout, err := c.startSpiderTargetNew(
|
||||
context.Background(),
|
||||
1,
|
||||
filepath.Join(tmp, "seen.txt"),
|
||||
filepath.Join(tmp, "out.json"),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("startSpiderTargetNew: %v", err)
|
||||
}
|
||||
raw, err := io.ReadAll(stdout)
|
||||
if err != nil {
|
||||
t.Fatalf("read stdout: %v", err)
|
||||
}
|
||||
if err := cmd.Wait(); err != nil {
|
||||
t.Fatalf("wait: %v", err)
|
||||
}
|
||||
|
||||
want := strings.Join([]string{
|
||||
"HTTP_PROXY=" + proxyURL,
|
||||
"HTTPS_PROXY=" + proxyURL,
|
||||
"http_proxy=" + proxyURL,
|
||||
"https_proxy=" + proxyURL,
|
||||
"NO_PROXY=",
|
||||
"no_proxy=",
|
||||
}, "\n") + "\n"
|
||||
if string(raw) != want {
|
||||
t.Fatalf("proxy env = %q, want %q", string(raw), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigureExplicitProxySupportsSocksSchemes(t *testing.T) {
|
||||
for _, raw := range []string{
|
||||
"socks5://127.0.0.1:1080",
|
||||
"socks5h://proxy-user:proxy-pass@127.0.0.1:1080",
|
||||
} {
|
||||
t.Run(raw, func(t *testing.T) {
|
||||
transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
|
||||
if err := configureExplicitProxy(transport, raw); err != nil {
|
||||
t.Fatalf("configureExplicitProxy: %v", err)
|
||||
}
|
||||
if transport.Proxy != nil {
|
||||
t.Fatalf("Transport.Proxy should be nil for SOCKS proxy")
|
||||
}
|
||||
if transport.DialContext == nil {
|
||||
t.Fatalf("Transport.DialContext should be set for SOCKS proxy")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
|
||||
if err := configureExplicitProxy(transport, "http://127.0.0.1:7890"); err != nil {
|
||||
t.Fatalf("configureExplicitProxy http: %v", err)
|
||||
}
|
||||
if transport.Proxy == nil {
|
||||
t.Fatalf("Transport.Proxy should be set for HTTP proxy")
|
||||
}
|
||||
if transport.DialContext != nil {
|
||||
t.Fatalf("Transport.DialContext should not be set for HTTP proxy")
|
||||
}
|
||||
|
||||
if err := configureExplicitProxy(&http.Transport{}, "ftp://127.0.0.1:21"); err == nil {
|
||||
t.Fatalf("expected unsupported proxy scheme error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSocksTargetIPPrefersIPv4(t *testing.T) {
|
||||
got := selectSocksTargetIP([]net.IPAddr{
|
||||
{IP: net.ParseIP("2606:4700:20::681a:229")},
|
||||
{IP: net.ParseIP("104.26.3.41")},
|
||||
})
|
||||
if got == nil || got.String() != "104.26.3.41" {
|
||||
t.Fatalf("selectSocksTargetIP = %v, want IPv4 104.26.3.41", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCrawlerThumbDownloadFailureMarksStatusFailed 验证:网站封面下载失败时
|
||||
// crawler 把 thumbnail_status 显式标 'failed',避免后续封面补队列一直重复
|
||||
// 捞到这条 spider91 视频。
|
||||
@@ -589,6 +707,18 @@ func TestSpider91CookieHeader(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpider91ProgressLogLineClassifiers(t *testing.T) {
|
||||
if !isSpider91CheckedVideoLogLine("[2026-06-08 16:49:17] 处理视频 3/24: 标题") {
|
||||
t.Fatal("checked video log line was not recognized")
|
||||
}
|
||||
if isSpider91CheckedVideoLogLine("[2026-06-08 16:49:17] [页 2] 发现 24 个视频") {
|
||||
t.Fatal("page summary log line should not count as checked video")
|
||||
}
|
||||
if !isSpider91ExtractedVideoLogLine("[2026-06-08 16:49:39] [OK] 成功提取视频直链") {
|
||||
t.Fatal("extracted video log line was not recognized")
|
||||
}
|
||||
}
|
||||
|
||||
func spider91DetailHTML(videoURL string) string {
|
||||
fragment := `<video><source src="` + videoURL + `" type="video/mp4"></video>`
|
||||
return `document.write(strencode2("` + url.PathEscape(fragment) + `"));`
|
||||
@@ -658,3 +788,12 @@ func buildFakeSpiderScript(entries []map[string]string) string {
|
||||
sb.WriteString("fi\n")
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func containsString(values []string, want string) bool {
|
||||
for _, value := range values {
|
||||
if value == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -138,7 +138,7 @@ func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error)
|
||||
|
||||
// StreamURL 返回本地视频文件路径,给 ffmpeg / 上层服务使用。
|
||||
// 注意:proxy.serve 不能直接处理本地路径,回放要走 api.handleSpider91Video。
|
||||
// teaser/封面 worker 通过 localPreviewLink 兜底走本地文件,刚好兼容 path 形式的 URL。
|
||||
// 预览视频/封面 worker 通过 localPreviewLink 兜底走本地文件,刚好兼容 path 形式的 URL。
|
||||
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
path, err := d.VideoPath(fileID)
|
||||
if err != nil {
|
||||
|
||||
@@ -327,11 +327,65 @@ func readHTTPRange(ctx context.Context, hc *http.Client, link *drives.StreamLink
|
||||
return data, nil
|
||||
}
|
||||
}
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
|
||||
if remoteRangeResponseLooksRateLimited(link.URL, resp.StatusCode, body) {
|
||||
return nil, &drives.RateLimitError{
|
||||
Provider: "fingerprint",
|
||||
RetryAfter: parseRetryAfter(resp.Header.Get("Retry-After")),
|
||||
Err: fmt.Errorf("remote sample rate limited: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(body))),
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("fingerprint: range request got status=%d for bytes=%d-%d", resp.StatusCode, r.start, end)
|
||||
}
|
||||
return io.ReadAll(io.LimitReader(resp.Body, r.length))
|
||||
}
|
||||
|
||||
func remoteRangeResponseLooksRateLimited(rawURL string, status int, body []byte) bool {
|
||||
if status == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
text := strings.ToLower(strings.TrimSpace(string(body)))
|
||||
compact := compactRemoteRangeErrorText(text)
|
||||
if strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "quota exceeded") ||
|
||||
strings.Contains(text, "download quota") ||
|
||||
strings.Contains(text, "sharing rate") ||
|
||||
strings.Contains(text, "daily limit") ||
|
||||
strings.Contains(text, "user rate") ||
|
||||
strings.Contains(text, "usage limit") ||
|
||||
strings.Contains(compact, "ratelimitexceeded") ||
|
||||
strings.Contains(compact, "userratelimitexceeded") ||
|
||||
strings.Contains(compact, "dailylimitexceeded") ||
|
||||
strings.Contains(compact, "downloadquotaexceeded") ||
|
||||
strings.Contains(compact, "sharingratelimitexceeded") ||
|
||||
strings.Contains(compact, "quotaexceeded") ||
|
||||
strings.Contains(compact, "toomanyrequests") ||
|
||||
strings.Contains(compact, "usagelimits") {
|
||||
return true
|
||||
}
|
||||
if status == http.StatusForbidden && isGoogleDriveMediaURL(rawURL) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isGoogleDriveMediaURL(rawURL string) bool {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Host)
|
||||
path := strings.ToLower(u.Path)
|
||||
return strings.Contains(host, "googleapis.com") && strings.Contains(path, "/drive/")
|
||||
}
|
||||
|
||||
func compactRemoteRangeErrorText(text string) string {
|
||||
replacer := strings.NewReplacer("_", "", "-", "", " ", "", ".", "", ":", "")
|
||||
return replacer.Replace(strings.ToLower(strings.TrimSpace(text)))
|
||||
}
|
||||
|
||||
func parseRetryAfter(raw string) time.Duration {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
|
||||
@@ -2,6 +2,7 @@ package fingerprint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -85,6 +86,33 @@ func TestComputeRemoteUsesRangeSamples(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeRemoteGoogleQuotaExceededReturnsRateLimit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Retry-After", "60")
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
_, _ = w.Write([]byte(`{"error":{"code":403,"message":"The download quota for this file has been exceeded.","errors":[{"domain":"usageLimits","reason":"downloadQuotaExceeded","message":"The download quota for this file has been exceeded."}]}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
drv := &fakeDrive{paths: map[string]string{"remote": srv.URL + "/drive/v3/files/file-1?alt=media"}}
|
||||
_, err := Compute(ctx, drv, &catalog.Video{ID: "remote", FileID: "remote", Size: 1024 * 1024}, Config{
|
||||
SampleSizeBytes: 4,
|
||||
FullHashMaxSize: 8,
|
||||
HTTPClient: srv.Client(),
|
||||
}, srv.Client())
|
||||
if err == nil {
|
||||
t.Fatal("compute succeeded, want rate limit")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != time.Minute {
|
||||
t.Fatalf("retry after = %s, want 1m", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeDrive struct {
|
||||
paths map[string]string
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package mediaasset
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const maxPlainStemBytes = 180
|
||||
const maxLegacyFilenameBytes = 255
|
||||
|
||||
func PreviewPath(localDir, videoID string) string {
|
||||
return filepath.Join(localDir, PreviewFilename(videoID))
|
||||
}
|
||||
|
||||
func ThumbnailPath(localDir, videoID string) string {
|
||||
return ThumbnailPathInDir(filepath.Join(localDir, "thumbs"), videoID)
|
||||
}
|
||||
|
||||
func ThumbnailPathInDir(thumbDir, videoID string) string {
|
||||
return filepath.Join(thumbDir, ThumbnailFilename(videoID))
|
||||
}
|
||||
|
||||
func PreviewPathCandidates(localDir, videoID string) []string {
|
||||
return pathCandidates(localDir, videoID, ".mp4", "")
|
||||
}
|
||||
|
||||
func ThumbnailPathCandidates(localDir, videoID string) []string {
|
||||
return pathCandidates(localDir, videoID, ".jpg", "thumbs")
|
||||
}
|
||||
|
||||
func PreviewFilename(videoID string) string {
|
||||
return safeFilename(videoID, ".mp4")
|
||||
}
|
||||
|
||||
func ThumbnailFilename(videoID string) string {
|
||||
return safeFilename(videoID, ".jpg")
|
||||
}
|
||||
|
||||
func pathCandidates(localDir, videoID, ext, subdir string) []string {
|
||||
safe := safeFilename(videoID, ext)
|
||||
legacy := videoID + ext
|
||||
base := localDir
|
||||
if subdir != "" {
|
||||
base = filepath.Join(base, subdir)
|
||||
}
|
||||
out := []string{filepath.Join(base, safe)}
|
||||
if legacy != safe && isPlainSafeStem(videoID) && len([]byte(legacy)) <= maxLegacyFilenameBytes {
|
||||
out = append(out, filepath.Join(base, legacy))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func safeFilename(videoID, ext string) string {
|
||||
if isPlainSafeStem(videoID) && len([]byte(videoID))+len(ext) <= maxPlainStemBytes {
|
||||
return videoID + ext
|
||||
}
|
||||
sum := sha256.Sum256([]byte(videoID))
|
||||
return "v-" + hex.EncodeToString(sum[:]) + ext
|
||||
}
|
||||
|
||||
func isPlainSafeStem(value string) bool {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" || value == "." || value == ".." {
|
||||
return false
|
||||
}
|
||||
return !strings.ContainsAny(value, `/\`+"\x00")
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package mediaasset
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFilenamesKeepShortSafeIDs(t *testing.T) {
|
||||
if got := ThumbnailFilename("video-1"); got != "video-1.jpg" {
|
||||
t.Fatalf("thumbnail filename = %q, want video-1.jpg", got)
|
||||
}
|
||||
if got := PreviewFilename("video-1"); got != "video-1.mp4" {
|
||||
t.Fatalf("preview filename = %q, want video-1.mp4", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilenamesHashLongOrUnsafeIDs(t *testing.T) {
|
||||
longID := "localstorage-" + strings.Repeat("x", 240)
|
||||
got := ThumbnailFilename(longID)
|
||||
if !strings.HasPrefix(got, "v-") || !strings.HasSuffix(got, ".jpg") {
|
||||
t.Fatalf("thumbnail filename = %q, want hashed jpg", got)
|
||||
}
|
||||
if len([]byte(got)) >= len([]byte(longID+".jpg")) {
|
||||
t.Fatalf("thumbnail filename = %q should be shorter than original id", got)
|
||||
}
|
||||
|
||||
unsafe := ThumbnailFilename("dir/video")
|
||||
if unsafe == "dir/video.jpg" || strings.ContainsAny(unsafe, `/\`) {
|
||||
t.Fatalf("unsafe thumbnail filename = %q, want hashed single filename", unsafe)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailPathCandidatesIncludeLegacyForHashedIDs(t *testing.T) {
|
||||
localDir := t.TempDir()
|
||||
mediumID := "localstorage-" + strings.Repeat("x", 190)
|
||||
got := ThumbnailPathCandidates(localDir, mediumID)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("candidates = %#v, want hashed and legacy paths", got)
|
||||
}
|
||||
if got[0] != ThumbnailPath(localDir, mediumID) {
|
||||
t.Fatalf("first candidate = %q, want safe path %q", got[0], ThumbnailPath(localDir, mediumID))
|
||||
}
|
||||
if filepath.Base(got[1]) != mediumID+".jpg" {
|
||||
t.Fatalf("legacy candidate = %q, want original id jpg", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailPathCandidatesSkipOverlongLegacy(t *testing.T) {
|
||||
localDir := t.TempDir()
|
||||
longID := "localstorage-" + strings.Repeat("x", 240)
|
||||
got := ThumbnailPathCandidates(localDir, longID)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("candidates = %#v, want only hashed path for overlong id", got)
|
||||
}
|
||||
}
|
||||
@@ -5,11 +5,11 @@
|
||||
// "扫描所有网盘"):
|
||||
//
|
||||
// Phase 1: for each non-spider91 cloud drive
|
||||
// scan + delete-detection + enqueue thumb + enqueue teaser
|
||||
// wait until all thumb / teaser queues are idle
|
||||
// scan + delete-detection + enqueue thumb + enqueue preview video
|
||||
// wait until all thumb / preview-video queues are idle
|
||||
// Phase 2: if any spider91 drive configured
|
||||
// crawl + enqueue teaser for new videos
|
||||
// wait until teaser queues are idle
|
||||
// crawl + enqueue preview video for new videos
|
||||
// wait until preview-video queues are idle
|
||||
// Phase 3: spider91 → cloud migration (single sweep, captcha cooldown still
|
||||
// honored within this call)
|
||||
// Phase 4: cleanup duplicate local preview/thumbnail assets after sampled
|
||||
@@ -76,10 +76,10 @@ type Config struct {
|
||||
ListSpider91Drives func(ctx context.Context) []string
|
||||
|
||||
// RunSpider91Crawl synchronously runs one crawl cycle (downloads + thumbs +
|
||||
// teaser enqueue) for a single spider91 drive.
|
||||
// preview-video enqueue) for a single spider91 drive.
|
||||
RunSpider91Crawl func(ctx context.Context, driveID string)
|
||||
|
||||
// WaitPreviewQueuesIdle blocks until both the thumbnail and teaser queues
|
||||
// WaitPreviewQueuesIdle blocks until both the thumbnail and preview-video queues
|
||||
// across all drives are drained (queue empty + no in-flight task). It must
|
||||
// honor ctx cancellation.
|
||||
WaitPreviewQueuesIdle func(ctx context.Context) error
|
||||
@@ -115,6 +115,7 @@ type Runner struct {
|
||||
queued bool
|
||||
startedAt time.Time
|
||||
lastFinishedAt time.Time
|
||||
currentCancel context.CancelFunc
|
||||
}
|
||||
|
||||
// New constructs a Runner. cfg is shallow-copied; defaults are applied.
|
||||
@@ -175,6 +176,28 @@ func (r *Runner) TriggerNow() bool {
|
||||
}
|
||||
}
|
||||
|
||||
// StopCurrent cancels the currently running pipeline and drops one queued
|
||||
// manual trigger, if present. It returns true when there was something to stop.
|
||||
func (r *Runner) StopCurrent() bool {
|
||||
r.stateMu.Lock()
|
||||
wasRunning := r.running
|
||||
wasQueued := r.queued
|
||||
cancel := r.currentCancel
|
||||
r.queued = false
|
||||
r.stateMu.Unlock()
|
||||
|
||||
if wasQueued {
|
||||
select {
|
||||
case <-r.trigger:
|
||||
default:
|
||||
}
|
||||
}
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
return wasRunning || wasQueued || cancel != nil
|
||||
}
|
||||
|
||||
func (r *Runner) Status() Status {
|
||||
r.stateMu.Lock()
|
||||
running := r.running
|
||||
@@ -232,14 +255,25 @@ func shouldRun(now time.Time, lastRunDate string) bool {
|
||||
//
|
||||
// 流水线没有总耗时上限:一直跑到 ctx 取消(进程退出)或所有 phase 完成。
|
||||
func (r *Runner) runPipelineLocked(ctx context.Context, manual bool) {
|
||||
if manual {
|
||||
r.stateMu.Lock()
|
||||
queued := r.queued
|
||||
r.stateMu.Unlock()
|
||||
if !queued {
|
||||
log.Printf("[nightly] manual trigger was canceled before start")
|
||||
return
|
||||
}
|
||||
}
|
||||
if !r.runMu.TryLock() {
|
||||
log.Printf("[nightly] another pipeline is already running, skipping this trigger")
|
||||
return
|
||||
}
|
||||
|
||||
started := r.cfg.Now()
|
||||
r.markStarted(started)
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
r.markStarted(started, cancel)
|
||||
defer func() {
|
||||
cancel()
|
||||
r.markFinished(r.cfg.Now())
|
||||
r.runMu.Unlock()
|
||||
}()
|
||||
@@ -250,7 +284,7 @@ func (r *Runner) runPipelineLocked(ctx context.Context, manual bool) {
|
||||
}
|
||||
log.Printf("[nightly] pipeline (%s) start", mode)
|
||||
|
||||
r.runPipeline(ctx)
|
||||
r.runPipeline(runCtx)
|
||||
|
||||
finished := r.cfg.Now()
|
||||
log.Printf("[nightly] pipeline (%s) finish; took=%s", mode, finished.Sub(started).Round(time.Second))
|
||||
@@ -264,12 +298,13 @@ func (r *Runner) runPipelineLocked(ctx context.Context, manual bool) {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) markStarted(started time.Time) {
|
||||
func (r *Runner) markStarted(started time.Time, cancel context.CancelFunc) {
|
||||
r.stateMu.Lock()
|
||||
defer r.stateMu.Unlock()
|
||||
r.running = true
|
||||
r.queued = false
|
||||
r.startedAt = started
|
||||
r.currentCancel = cancel
|
||||
}
|
||||
|
||||
func (r *Runner) markFinished(finished time.Time) {
|
||||
@@ -278,6 +313,7 @@ func (r *Runner) markFinished(finished time.Time) {
|
||||
r.running = false
|
||||
r.startedAt = time.Time{}
|
||||
r.lastFinishedAt = finished
|
||||
r.currentCancel = nil
|
||||
}
|
||||
|
||||
// runPipeline executes the three phases. It returns when the pipeline finishes
|
||||
|
||||
@@ -395,6 +395,61 @@ func TestStatusTracksQueuedRunningAndFinished(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopCurrentCancelsRunningPipeline(t *testing.T) {
|
||||
scanStarted := make(chan struct{})
|
||||
scanCanceled := make(chan struct{})
|
||||
var startedOnce sync.Once
|
||||
r := New(Config{
|
||||
Settings: newStubSettings(),
|
||||
ListScanTargets: func(context.Context) []string {
|
||||
return []string{"drive"}
|
||||
},
|
||||
RunScan: func(ctx context.Context, _ string) {
|
||||
startedOnce.Do(func() { close(scanStarted) })
|
||||
<-ctx.Done()
|
||||
close(scanCanceled)
|
||||
},
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go r.Run(ctx)
|
||||
|
||||
if !r.TriggerNow() {
|
||||
t.Fatal("TriggerNow should queue a manual run")
|
||||
}
|
||||
select {
|
||||
case <-scanStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("pipeline did not start")
|
||||
}
|
||||
|
||||
if !r.StopCurrent() {
|
||||
t.Fatal("StopCurrent should report a running pipeline")
|
||||
}
|
||||
select {
|
||||
case <-scanCanceled:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("StopCurrent did not cancel pipeline context")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopCurrentDropsQueuedTrigger(t *testing.T) {
|
||||
r := New(Config{Settings: newStubSettings()})
|
||||
if !r.TriggerNow() {
|
||||
t.Fatal("TriggerNow should queue a manual run")
|
||||
}
|
||||
if !r.StopCurrent() {
|
||||
t.Fatal("StopCurrent should report a queued pipeline")
|
||||
}
|
||||
if got := r.Status(); got.State != "idle" || got.Running || got.Queued {
|
||||
t.Fatalf("status = %#v, want idle after dropping queued trigger", got)
|
||||
}
|
||||
if !r.TriggerNow() {
|
||||
t.Fatal("TriggerNow should accept a new request after queued stop")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTriggerNowAcceptsOnlyOneConcurrentRequest(t *testing.T) {
|
||||
r := New(Config{Settings: newStubSettings()})
|
||||
|
||||
|
||||
@@ -21,15 +21,16 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
FFmpegPath string
|
||||
FFprobePath string
|
||||
DurationSeconds int // 兼容旧配置;当前 teaser 每段固定 3 秒
|
||||
DurationSeconds int // 兼容旧配置;当前预览视频每段固定 3 秒
|
||||
Width int
|
||||
Segments int // 兼容旧配置;当前 30 秒及以上视频固定使用 4 段
|
||||
LocalDir string // 本地 teaser 和封面目录
|
||||
LocalDir string // 本地预览视频和封面目录
|
||||
}
|
||||
|
||||
type Generator struct {
|
||||
@@ -236,23 +237,43 @@ func appendUniqueStart(starts []float64, start, eachSec float64) []float64 {
|
||||
return append(starts, start)
|
||||
}
|
||||
|
||||
// thumbnailOffsets 选封面抽帧的时间点(秒)。独立于 teaser。
|
||||
func thumbnailOffsets() []float64 {
|
||||
return []float64{5, 1, 0}
|
||||
// thumbnailOffsets 选封面抽帧的时间点(秒)。独立于预览视频。
|
||||
// 默认取视频中间帧;时长未知时退回早期帧。
|
||||
func thumbnailOffsets(duration float64) []float64 {
|
||||
if duration <= 0 {
|
||||
return []float64{5, 1, 0}
|
||||
}
|
||||
mid := duration / 2
|
||||
out := []float64{mid}
|
||||
for _, fallback := range []float64{5, 1, 0} {
|
||||
if !containsOffset(out, fallback) {
|
||||
out = append(out, fallback)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func containsOffset(offsets []float64, target float64) bool {
|
||||
for _, offset := range offsets {
|
||||
if math.Abs(offset-target) < 0.01 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// --- 封面 ---
|
||||
|
||||
// GenerateThumbnail 抽一张 jpg 封面。默认从第 5 秒抽帧,失败时回退到更早时间点。
|
||||
// GenerateThumbnail 抽一张 jpg 封面。默认从视频中间抽帧,失败时回退到更早时间点。
|
||||
func (g *Generator) GenerateThumbnail(ctx context.Context, link *drives.StreamLink, videoID string, duration float64) (string, error) {
|
||||
dir := filepath.Join(g.cfg.LocalDir, "thumbs")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
dst := filepath.Join(dir, videoID+".jpg")
|
||||
dst := mediaasset.ThumbnailPath(g.cfg.LocalDir, videoID)
|
||||
|
||||
var lastErr error
|
||||
offsets := thumbnailOffsets()
|
||||
offsets := thumbnailOffsets(duration)
|
||||
for i, offset := range offsets {
|
||||
if i > 0 {
|
||||
_ = os.Remove(dst)
|
||||
@@ -363,9 +384,9 @@ func (g *Generator) Probe(ctx context.Context, link *drives.StreamLink) (float64
|
||||
return strconv.ParseFloat(raw, 64)
|
||||
}
|
||||
|
||||
// --- Teaser ---
|
||||
// --- 预览视频 ---
|
||||
|
||||
// Generate 拉取 teaser 到本地临时文件,返回路径。
|
||||
// Generate 拉取预览视频到本地临时文件,返回路径。
|
||||
// 根据 Config.Segments 和视频时长决定是单段还是多段拼接。
|
||||
func (g *Generator) Generate(ctx context.Context, link *drives.StreamLink, duration float64) (string, error) {
|
||||
return g.generate(ctx, duration, func(int) (*drives.StreamLink, error) {
|
||||
@@ -946,7 +967,10 @@ func ffmpegOutputLooksRateLimited(output []byte) bool {
|
||||
|
||||
// MoveToLocal 把临时文件改名到稳定位置,返回最终路径
|
||||
func (g *Generator) MoveToLocal(tmpPath, videoID string) (string, error) {
|
||||
dst := filepath.Join(g.cfg.LocalDir, videoID+".mp4")
|
||||
if err := os.MkdirAll(g.cfg.LocalDir, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
dst := mediaasset.PreviewPath(g.cfg.LocalDir, videoID)
|
||||
if err := os.Rename(tmpPath, dst); err != nil {
|
||||
// 跨盘 rename 可能失败,fallback 到 copy
|
||||
if cerr := copyFile(tmpPath, dst); cerr != nil {
|
||||
@@ -1338,12 +1362,19 @@ func (w *ThumbWorker) Run(ctx context.Context) {
|
||||
|
||||
func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
defer w.queue.release(v)
|
||||
w.activity.start(v)
|
||||
if w.Catalog == nil || v == nil || v.ID == "" {
|
||||
return
|
||||
}
|
||||
current, err := w.Catalog.GetVideo(ctx, v.ID)
|
||||
if err != nil || current.Hidden {
|
||||
return
|
||||
}
|
||||
w.activity.start(current)
|
||||
defer w.activity.done()
|
||||
if !waitForRateLimitCooldown(ctx, &w.rateLimit, "preview", w.Drive) {
|
||||
return
|
||||
}
|
||||
w.process(ctx, v)
|
||||
w.process(ctx, current)
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
@@ -1396,11 +1427,14 @@ func (w *Worker) skipIfRateLimited(v *catalog.Video) bool {
|
||||
}
|
||||
|
||||
func (w *Worker) pauseForRateLimit(err error, step, title string) bool {
|
||||
_, ok := drives.RateLimitRetryAfter(err)
|
||||
wait, ok := drives.RateLimitRetryAfter(err)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), defaultGenerationRateLimitCooldown)
|
||||
if wait <= 0 {
|
||||
wait = defaultGenerationRateLimitCooldown
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), wait)
|
||||
log.Printf("[preview] drive=%s rate-limited until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err)
|
||||
return true
|
||||
}
|
||||
@@ -1429,11 +1463,14 @@ func (w *ThumbWorker) skipIfRateLimited(v *catalog.Video) bool {
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) pauseForRateLimit(err error, step, title string) bool {
|
||||
_, ok := drives.RateLimitRetryAfter(err)
|
||||
wait, ok := drives.RateLimitRetryAfter(err)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), defaultGenerationRateLimitCooldown)
|
||||
if wait <= 0 {
|
||||
wait = defaultGenerationRateLimitCooldown
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), wait)
|
||||
log.Printf("[thumb] drive=%s rate-limited until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err)
|
||||
return true
|
||||
}
|
||||
@@ -1486,7 +1523,7 @@ func driveErrorShouldCooldown(d drives.Drive, err error) bool {
|
||||
strings.Contains(text, "request has been blocked") ||
|
||||
strings.Contains(text, "访问被阻断")
|
||||
case "pikpak":
|
||||
// PikPak 在 teaser / 封面生成阶段(取链或拉直链字节)可能命中:
|
||||
// PikPak 在预览视频 / 封面生成阶段(取链或拉直链字节)可能命中:
|
||||
// - error_code=10 操作频繁
|
||||
// - HTTP 429 / 5xx / 509 限流和服务端不可用
|
||||
// - 通用文本:rate limit / too many requests / blocked
|
||||
@@ -1504,29 +1541,99 @@ func driveErrorShouldCooldown(d drives.Drive, err error) bool {
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "blocked") ||
|
||||
strings.Contains(text, "moov atom not found") ||
|
||||
strings.Contains(text, "partial file") ||
|
||||
strings.Contains(text, "service unavailable")
|
||||
case "p123":
|
||||
// 123 云盘直链解析 / ffmpeg 读取阶段可能返回 429、5xx,或 WAF 类
|
||||
// blocked / 访问阻断文本。命中时冷却,避免封面和预览视频生成连续打接口。
|
||||
text := strings.ToLower(err.Error())
|
||||
return strings.Contains(text, "请求太频繁") ||
|
||||
strings.Contains(text, "请求过于频繁") ||
|
||||
strings.Contains(text, "请求频繁") ||
|
||||
strings.Contains(text, "操作频繁") ||
|
||||
strings.Contains(text, "频率限制") ||
|
||||
strings.Contains(text, "请求次数过多") ||
|
||||
strings.Contains(text, "429") ||
|
||||
strings.Contains(text, "http 500") ||
|
||||
strings.Contains(text, "http 502") ||
|
||||
strings.Contains(text, "http 503") ||
|
||||
strings.Contains(text, "http 504") ||
|
||||
strings.Contains(text, "server returned 403") ||
|
||||
strings.Contains(text, "403 forbidden") ||
|
||||
strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "blocked") ||
|
||||
strings.Contains(text, "访问被阻断") ||
|
||||
strings.Contains(text, "service unavailable")
|
||||
case "googledrive":
|
||||
// Google Drive 下载/取样阶段常把频控和配额问题包装成 403,
|
||||
// 具体标识在 error.errors[].reason/message 里(OpenList 也按该结构解析)。
|
||||
// ffmpeg/ffprobe 只能看到 stderr 文本时,按这些 reason/文本兜底冷却。
|
||||
text := strings.ToLower(err.Error())
|
||||
return googleDriveMediaErrorShouldCooldown(text)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func googleDriveMediaErrorShouldCooldown(text string) bool {
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
compact := compactGoogleDriveErrorText(text)
|
||||
return strings.Contains(text, "server returned 403") ||
|
||||
strings.Contains(text, "403 forbidden") ||
|
||||
strings.Contains(text, "server returned 429") ||
|
||||
strings.Contains(text, "http 429") ||
|
||||
strings.Contains(text, "http 500") ||
|
||||
strings.Contains(text, "http 502") ||
|
||||
strings.Contains(text, "http 503") ||
|
||||
strings.Contains(text, "http 504") ||
|
||||
strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "quota exceeded") ||
|
||||
strings.Contains(text, "download quota") ||
|
||||
strings.Contains(text, "sharing rate") ||
|
||||
strings.Contains(text, "daily limit") ||
|
||||
strings.Contains(text, "user rate") ||
|
||||
strings.Contains(text, "usage limit") ||
|
||||
strings.Contains(text, "service unavailable") ||
|
||||
strings.Contains(compact, "ratelimitexceeded") ||
|
||||
strings.Contains(compact, "userratelimitexceeded") ||
|
||||
strings.Contains(compact, "dailylimitexceeded") ||
|
||||
strings.Contains(compact, "downloadquotaexceeded") ||
|
||||
strings.Contains(compact, "sharingratelimitexceeded") ||
|
||||
strings.Contains(compact, "quotaexceeded") ||
|
||||
strings.Contains(compact, "toomanyrequests") ||
|
||||
strings.Contains(compact, "usagelimits")
|
||||
}
|
||||
|
||||
func compactGoogleDriveErrorText(text string) string {
|
||||
replacer := strings.NewReplacer("_", "", "-", "", " ", "", ".", "", ":", "")
|
||||
return replacer.Replace(strings.ToLower(strings.TrimSpace(text)))
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) bool {
|
||||
if w.skipIfRateLimited(v) {
|
||||
return false
|
||||
}
|
||||
if w.Catalog == nil || v == nil || v.ID == "" {
|
||||
return false
|
||||
}
|
||||
queued := v
|
||||
current := v
|
||||
if loaded, err := w.Catalog.GetVideo(ctx, v.ID); err == nil {
|
||||
if loaded.PreviewLocal == "" {
|
||||
loaded.PreviewLocal = queued.PreviewLocal
|
||||
}
|
||||
current = loaded
|
||||
v = loaded
|
||||
if loaded.ThumbnailURL != "" && loaded.DurationSeconds > 0 {
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
|
||||
return false
|
||||
}
|
||||
loaded, err := w.Catalog.GetVideo(ctx, v.ID)
|
||||
if err != nil || loaded.Hidden {
|
||||
return false
|
||||
}
|
||||
if loaded.PreviewLocal == "" {
|
||||
loaded.PreviewLocal = queued.PreviewLocal
|
||||
}
|
||||
current := loaded
|
||||
v = loaded
|
||||
if loaded.ThumbnailURL != "" && loaded.DurationSeconds > 0 {
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
|
||||
return false
|
||||
}
|
||||
if current.ThumbnailURL != "" {
|
||||
durationBackfillFailed := false
|
||||
@@ -1553,6 +1660,11 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) bool {
|
||||
return false
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "pending"})
|
||||
if isSpider91OriginVideo(v) {
|
||||
log.Printf("[thumb] skip %s: spider91-origin video must use crawled thumbnail", v.Title)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
|
||||
return false
|
||||
}
|
||||
link, err := w.streamLink(ctx, v)
|
||||
if err != nil {
|
||||
if w.pauseForRecoverableError(ctx, v, err, "streamURL") {
|
||||
@@ -1618,17 +1730,26 @@ func (w *ThumbWorker) probeDuration(ctx context.Context, v *catalog.Video, link
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) generateThumbnailFromLink(ctx context.Context, v *catalog.Video, link *drives.StreamLink) error {
|
||||
if _, err := w.Gen.GenerateThumbnail(ctx, link, v.ID, 0); err != nil {
|
||||
local, err := w.Gen.GenerateThumbnail(ctx, link, v.ID, float64(v.DurationSeconds))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
if err := w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
ThumbnailURL: "/p/thumb/" + v.ID,
|
||||
ThumbnailStatus: "ready",
|
||||
})
|
||||
}); err != nil {
|
||||
_ = os.Remove(local)
|
||||
log.Printf("[thumb] update %s after generate: %v", v.Title, err)
|
||||
return nil
|
||||
}
|
||||
log.Printf("[thumb] ready %s", v.Title)
|
||||
return nil
|
||||
}
|
||||
|
||||
func isSpider91OriginVideo(v *catalog.Video) bool {
|
||||
return v != nil && strings.HasPrefix(v.ID, "spider91-")
|
||||
}
|
||||
|
||||
func localPreviewLink(v *catalog.Video) (*drives.StreamLink, bool) {
|
||||
if v.PreviewLocal == "" {
|
||||
return nil, false
|
||||
@@ -1677,7 +1798,7 @@ func (w *Worker) process(ctx context.Context, v *catalog.Video) {
|
||||
}
|
||||
}
|
||||
|
||||
// 2) teaser
|
||||
// 2) 预览视频
|
||||
tmp, err := w.generateTeaser(ctx, v, link, duration)
|
||||
if err != nil {
|
||||
if w.pauseForRecoverableError(err, "generate", v.Title) {
|
||||
@@ -1695,7 +1816,11 @@ func (w *Worker) process(ctx context.Context, v *catalog.Video) {
|
||||
}
|
||||
|
||||
removePreviousLocalTeaser(v.PreviewLocal, local)
|
||||
w.Catalog.UpdatePreview(ctx, v.ID, local, "ready")
|
||||
if err := w.Catalog.UpdatePreview(ctx, v.ID, local, "ready"); err != nil {
|
||||
removePreviousLocalTeaser(local, "")
|
||||
log.Printf("[preview] update %s after generate: %v", v.Title, err)
|
||||
return
|
||||
}
|
||||
log.Printf("[preview] ready %s (duration=%.1fs)", v.Title, duration)
|
||||
}
|
||||
|
||||
|
||||
@@ -168,16 +168,29 @@ func TestMediumAndLongVideosStillRequirePlannedTeaserSegments(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailOffsetsUseFiveSecondsWithEarlyFallbacks(t *testing.T) {
|
||||
got := thumbnailOffsets()
|
||||
want := []float64{5, 1, 0}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("offsets = %#v, want %#v", got, want)
|
||||
func TestThumbnailOffsetsPreferMiddleFrame(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
duration float64
|
||||
want []float64
|
||||
}{
|
||||
{name: "unknown duration", duration: 0, want: []float64{5, 1, 0}},
|
||||
{name: "long video", duration: 2804.9, want: []float64{1402.45, 5, 1, 0}},
|
||||
{name: "short video", duration: 8.9, want: []float64{4.45, 5, 1, 0}},
|
||||
{name: "middle equals fallback", duration: 10, want: []float64{5, 1, 0}},
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("offset[%d] = %.2f, want %.2f", i, got[i], want[i])
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := thumbnailOffsets(tt.duration)
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("offsets = %#v, want %#v", got, tt.want)
|
||||
}
|
||||
for i := range tt.want {
|
||||
if math.Abs(got[i]-tt.want[i]) > 0.001 {
|
||||
t.Fatalf("offset[%d] = %.2f, want %.2f", i, got[i], tt.want[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -39,8 +39,8 @@ func TestThumbWorkerUpdatesThumbnailAndDurationWithoutChangingPreviewStatus(t *t
|
||||
if gen.thumbnailVideoID != video.ID {
|
||||
t.Fatalf("thumbnail video id = %q, want %q", gen.thumbnailVideoID, video.ID)
|
||||
}
|
||||
if gen.thumbnailDuration != 0 {
|
||||
t.Fatalf("thumbnail duration = %.1f, want fixed-offset thumbnail generation", gen.thumbnailDuration)
|
||||
if gen.thumbnailDuration != 42 {
|
||||
t.Fatalf("thumbnail duration = %.1f, want probed duration", gen.thumbnailDuration)
|
||||
}
|
||||
if gen.probeCalls != 1 {
|
||||
t.Fatalf("probe calls = %d, want 1 for thumbnail generation", gen.probeCalls)
|
||||
@@ -89,6 +89,35 @@ func TestThumbWorkerBackfillsDurationWhenThumbnailAlreadyExists(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerDoesNotGenerateThumbnailForSpider91OriginVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "spider91-91-spider-1200001")
|
||||
|
||||
gen := &fakeThumbGenerator{probeDuration: 42}
|
||||
drv := &previewFakeDrive{kind: "pikpak"}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
worker.process(ctx, video)
|
||||
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want empty when crawled spider91 thumbnail is missing", got.ThumbnailURL)
|
||||
}
|
||||
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list failed thumbnails: %v", err)
|
||||
}
|
||||
if len(failed) != 1 || failed[0].ID != video.ID {
|
||||
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
|
||||
}
|
||||
if gen.probeCalls != 0 || gen.generateCalls != 0 {
|
||||
t.Fatalf("generator calls probe=%d generate=%d, want no ffmpeg work for spider91-origin thumbnail", gen.probeCalls, gen.generateCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerSkipsDurationBackfillWhenExistingThumbnailCannotBeProbed(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-existing-thumbnail-probe-fails")
|
||||
@@ -413,7 +442,7 @@ func TestPreviewWorkerRateLimitLeavesCurrentPendingAndSkipsNextVideo(t *testing.
|
||||
if gen.generateCalls != 1 {
|
||||
t.Fatalf("generate calls = %d, want 1", gen.generateCalls)
|
||||
}
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 5*time.Minute)
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 2*time.Hour)
|
||||
|
||||
gen.generateErr = nil
|
||||
worker.process(ctx, &second)
|
||||
@@ -429,7 +458,7 @@ func TestPreviewWorkerRateLimitLeavesCurrentPendingAndSkipsNextVideo(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerRateLimitCoolsDownFiveMinutes(t *testing.T) {
|
||||
func TestThumbWorkerRateLimitHonorsRetryAfter(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-rate-limit")
|
||||
|
||||
@@ -453,7 +482,7 @@ func TestThumbWorkerRateLimitCoolsDownFiveMinutes(t *testing.T) {
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want unchanged after rate limit", got.ThumbnailURL)
|
||||
}
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 5*time.Minute)
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 2*time.Hour)
|
||||
}
|
||||
|
||||
func TestThumbWorkerP115TransientErrorFailsAfterRetryLimit(t *testing.T) {
|
||||
@@ -563,6 +592,35 @@ func TestThumbWorkerRequeuesP115TransientErrorBeforeRetryLimit(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerPikPakMoovAtomErrorFailsWithoutCooldown(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-pikpak-missing-moov")
|
||||
|
||||
mediaErr := errors.New("ffprobe: exit status 1, stderr: moov atom not found Invalid data found when processing input")
|
||||
gen := &fakeThumbGenerator{
|
||||
probeErr: mediaErr,
|
||||
generateErr: mediaErr,
|
||||
}
|
||||
drv := &previewFakeDrive{kind: "pikpak"}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
worker.process(ctx, video)
|
||||
|
||||
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list failed thumbnails: %v", err)
|
||||
}
|
||||
if len(failed) != 1 || failed[0].ID != video.ID {
|
||||
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
|
||||
}
|
||||
if !worker.Status().CooldownUntil.IsZero() {
|
||||
t.Fatalf("cooldown until = %s, want no cooldown for invalid PikPak MP4", worker.Status().CooldownUntil)
|
||||
}
|
||||
if gen.generateCalls != 1 {
|
||||
t.Fatalf("generate calls = %d, want 1", gen.generateCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewWorkerP115TransientErrorKeepsVideoPending(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "preview-p115-transient")
|
||||
@@ -587,6 +645,40 @@ func TestPreviewWorkerP115TransientErrorKeepsVideoPending(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestP123TransientErrorsShouldCooldown(t *testing.T) {
|
||||
drv := &previewFakeDrive{kind: "p123"}
|
||||
for _, err := range []error{
|
||||
errors.New("Server returned 403 Forbidden"),
|
||||
errors.New("请求太频繁"),
|
||||
errors.New("http 503 service unavailable"),
|
||||
} {
|
||||
if !driveErrorShouldCooldown(drv, err) {
|
||||
t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err)
|
||||
}
|
||||
}
|
||||
if driveErrorShouldCooldown(drv, errors.New("invalid credential")) {
|
||||
t.Fatal("invalid credential should not trigger p123 cooldown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoogleDriveMediaErrorsShouldCooldown(t *testing.T) {
|
||||
drv := &previewFakeDrive{kind: "googledrive"}
|
||||
for _, err := range []error{
|
||||
errors.New("google drive api error: usageLimits userRateLimitExceeded"),
|
||||
errors.New("ffmpeg: Server returned 403 Forbidden"),
|
||||
errors.New("downloadQuotaExceeded: The download quota for this file has been exceeded"),
|
||||
errors.New("sharingRateLimitExceeded"),
|
||||
errors.New("http 503 service unavailable"),
|
||||
} {
|
||||
if !driveErrorShouldCooldown(drv, err) {
|
||||
t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err)
|
||||
}
|
||||
}
|
||||
if driveErrorShouldCooldown(drv, errors.New("invalid credentials")) {
|
||||
t.Fatal("invalid credentials should not trigger googledrive cooldown")
|
||||
}
|
||||
}
|
||||
|
||||
func assertCooldownAround(t *testing.T, until time.Time, before time.Time, want time.Duration) {
|
||||
t.Helper()
|
||||
if until.IsZero() {
|
||||
|
||||
@@ -147,13 +147,15 @@ func (p *Proxy) ServeStream(w http.ResponseWriter, r *http.Request, driveID, fil
|
||||
// CDN 不校验请求头,直连可获得最佳带宽并避免占用 backend 出站
|
||||
// - onedrive:Microsoft Graph 返回的 @microsoft.graph.downloadUrl 是短期
|
||||
// 免鉴权下载 URL,不需要后端继续代传视频字节
|
||||
// - p123:123 云盘 download_info 返回的下载页会再跳 CDN;driver 已在后端
|
||||
// 先解出最终 Location,浏览器可直接 302 到该短期地址
|
||||
//
|
||||
// 其余网盘(如沃盘 / 夸克等)仍走反代,因为它们的下载
|
||||
// 链接通常需要随请求带上后端持有的 Cookie / Authorization / Range
|
||||
// 的特殊处理,浏览器拿不到这些上下文。
|
||||
func shouldRedirect(d drives.Drive) bool {
|
||||
switch d.Kind() {
|
||||
case "p115", "pikpak", "onedrive":
|
||||
case "p115", "pikpak", "onedrive", "p123":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -214,7 +216,7 @@ func (p *Proxy) serve(w http.ResponseWriter, r *http.Request, link *drives.Strea
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
// ServeLocal 服务本地 teaser 文件
|
||||
// ServeLocal 服务本地预览视频文件
|
||||
func (p *Proxy) ServeLocal(w http.ResponseWriter, r *http.Request, path string) {
|
||||
http.ServeFile(w, r, path)
|
||||
}
|
||||
|
||||
@@ -176,6 +176,31 @@ func TestServeStreamRedirectsOneDrive(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamRedirectsP123(t *testing.T) {
|
||||
reg := NewRegistry()
|
||||
drv := &proxyFakeSimpleDrive{
|
||||
kind: "p123",
|
||||
url: "https://cdn.123pan.example/video.mp4",
|
||||
}
|
||||
reg.Set("p123", drv)
|
||||
|
||||
p := New(reg)
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/p123/file-1", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
p.ServeStream(rr, req, "p123", "file-1")
|
||||
|
||||
if rr.Code != http.StatusFound {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusFound)
|
||||
}
|
||||
if got := rr.Header().Get("Location"); got != "https://cdn.123pan.example/video.mp4" {
|
||||
t.Fatalf("Location = %q", got)
|
||||
}
|
||||
if drv.calls != 1 {
|
||||
t.Fatalf("link calls = %d, want 1", drv.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamServesLocalFilePath(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "video.mp4")
|
||||
if err := os.WriteFile(path, []byte("0123456789"), 0o644); err != nil {
|
||||
|
||||
@@ -23,8 +23,10 @@ type Scanner struct {
|
||||
//
|
||||
// nil / 空集合 → 行为等同于不跳过任何目录。
|
||||
SkipDirIDs map[string]struct{}
|
||||
// 回调:新视频被加入后触发 teaser 生成
|
||||
// 回调:新视频被加入后触发预览视频生成
|
||||
OnNewVideo func(v *catalog.Video)
|
||||
// OnProgress 在扫描进度变化时触发。回调只应读取 Stats 里的计数,不应修改 map 字段。
|
||||
OnProgress func(stats Stats)
|
||||
// ProgressInterval 控制扫描内部 heartbeat 的最小输出间隔。
|
||||
// 0 → 默认 30s;< 0 → 关闭 heartbeat(仅留外层 start / done 两行)。
|
||||
// heartbeat 单行格式:
|
||||
@@ -91,6 +93,9 @@ func (s *Scanner) Run(ctx context.Context, startDirID string) (Stats, error) {
|
||||
driveID = s.Drive.ID()
|
||||
}
|
||||
progress := func(currentDir string) {
|
||||
if s.OnProgress != nil {
|
||||
s.OnProgress(stats)
|
||||
}
|
||||
if interval < 0 {
|
||||
return
|
||||
}
|
||||
@@ -127,8 +132,11 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
}
|
||||
|
||||
for _, e := range entries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if e.IsDir {
|
||||
// 跳过 previews 目录,避免扫到自己生成的 teaser
|
||||
// 跳过 previews 目录,避免扫到自己生成的预览视频
|
||||
if strings.EqualFold(e.Name, "previews") {
|
||||
continue
|
||||
}
|
||||
@@ -137,13 +145,15 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
continue
|
||||
}
|
||||
if err := s.walk(ctx, e.ID, e.Name, stats, progress); err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return ctxErr
|
||||
}
|
||||
stats.Errors++
|
||||
log.Printf("[scanner] walk %s error: %v", e.Name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
stats.Scanned++
|
||||
ext := strings.ToLower(path.Ext(e.Name))
|
||||
if !s.Exts[ext] {
|
||||
continue
|
||||
@@ -151,9 +161,22 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
if e.Size <= 0 {
|
||||
continue
|
||||
}
|
||||
stats.Scanned++
|
||||
progress(dirName)
|
||||
stats.SeenFileIDs[e.ID] = struct{}{}
|
||||
|
||||
id := s.Drive.Kind() + "-" + s.Drive.ID() + "-" + e.ID
|
||||
if deleted, err := s.Catalog.IsDeletedVideoCandidate(ctx, id, s.Drive.ID(), e.ID, e.Hash, e.Name, e.Size); err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return ctxErr
|
||||
}
|
||||
stats.Errors++
|
||||
log.Printf("[scanner] check deleted video %s error: %v", id, err)
|
||||
continue
|
||||
} else if deleted {
|
||||
continue
|
||||
}
|
||||
|
||||
parsed := Parse(e.Name)
|
||||
if parsed.Title == "" {
|
||||
parsed.Title = strings.TrimSuffix(e.Name, ext)
|
||||
@@ -162,11 +185,20 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
if matched, err := s.Catalog.MatchTags(ctx, e.Name+" "+dirName+" "+parsed.Author); err == nil {
|
||||
tags = mergeTags(tags, matched)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if label, ok, err := s.Catalog.EnsureCollectionTag(ctx, dirName); err == nil && ok {
|
||||
tags = mergeTags(tags, []string{label})
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
existing, _ := s.Catalog.GetVideo(ctx, id)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if existing != nil {
|
||||
patch := catalog.VideoMetaPatch{}
|
||||
if e.Hash != "" && existing.ContentHash == "" {
|
||||
@@ -181,26 +213,33 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
if existing.Category == "" && dirName != "" {
|
||||
patch.Category = dirName
|
||||
}
|
||||
if existing.ThumbnailURL == "" && e.ThumbnailURL != "" {
|
||||
patch.ThumbnailURL = e.ThumbnailURL
|
||||
}
|
||||
if patch.Category != "" || patch.ThumbnailURL != "" || patch.ContentHash != "" || patch.FileName != "" {
|
||||
if patch.Category != "" || patch.ContentHash != "" || patch.FileName != "" {
|
||||
_ = s.Catalog.UpdateVideoMeta(ctx, id, patch)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if dup := s.findDuplicate(ctx, e.Hash, e.Name, e.Size, id); dup != nil {
|
||||
s.backfillDuplicateThumbnail(ctx, dup, e.ThumbnailURL)
|
||||
continue
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !sameTags(existing.Tags, tags) {
|
||||
_ = s.Catalog.SetAutoVideoTags(ctx, id, tags)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if dup := s.findDuplicate(ctx, e.Hash, e.Name, e.Size, id); dup != nil {
|
||||
s.backfillDuplicateThumbnail(ctx, dup, e.ThumbnailURL)
|
||||
continue
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
v := &catalog.Video{
|
||||
@@ -216,7 +255,6 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
Ext: strings.TrimPrefix(ext, "."),
|
||||
Quality: "HD",
|
||||
Size: e.Size,
|
||||
ThumbnailURL: e.ThumbnailURL,
|
||||
PreviewStatus: "pending",
|
||||
Category: dirName,
|
||||
PublishedAt: now,
|
||||
@@ -224,10 +262,17 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if err := s.Catalog.UpsertVideo(ctx, v); err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return ctxErr
|
||||
}
|
||||
log.Printf("[scanner] upsert %s error: %v", v.Title, err)
|
||||
continue
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
stats.Added++
|
||||
progress(dirName)
|
||||
if s.OnNewVideo != nil {
|
||||
s.OnNewVideo(v)
|
||||
}
|
||||
@@ -268,13 +313,6 @@ func (s *Scanner) findDuplicateByFileSignature(ctx context.Context, fileName str
|
||||
return dup
|
||||
}
|
||||
|
||||
func (s *Scanner) backfillDuplicateThumbnail(ctx context.Context, canonical *catalog.Video, thumbnailURL string) {
|
||||
if canonical.ThumbnailURL != "" || thumbnailURL == "" {
|
||||
return
|
||||
}
|
||||
_ = s.Catalog.UpdateVideoMeta(ctx, canonical.ID, catalog.VideoMetaPatch{ThumbnailURL: thumbnailURL})
|
||||
}
|
||||
|
||||
func sameTags(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
|
||||
@@ -3,6 +3,7 @@ package scanner
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -14,7 +15,7 @@ import (
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestRunPersistsRemoteThumbnailFromDriveEntry(t *testing.T) {
|
||||
func TestRunIgnoresRemoteThumbnailFromDriveEntry(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -50,8 +51,8 @@ func TestRunPersistsRemoteThumbnailFromDriveEntry(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "https://thumbnail.example/clip.jpg" {
|
||||
t.Fatalf("thumbnail = %q, want remote thumbnail", got.ThumbnailURL)
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want empty so local thumbnail worker regenerates it", got.ThumbnailURL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,7 +91,139 @@ func TestRunIgnoresZeroSizeVideoFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBackfillsRemoteThumbnailForExistingVideo(t *testing.T) {
|
||||
func TestRunScannedCountsOnlyVideoCandidates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := &scannerFakeDrive{
|
||||
entries: []drives.Entry{
|
||||
{ID: "file-1", Name: "clip.mp4", Size: 123},
|
||||
{ID: "file-2", Name: "notes.txt", Size: 123},
|
||||
{ID: "file-3", Name: "empty.mp4", Size: 0},
|
||||
},
|
||||
}
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
|
||||
stats, err := sc.Run(ctx, "")
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Scanned != 1 {
|
||||
t.Fatalf("scanned = %d, want one non-empty video candidate", stats.Scanned)
|
||||
}
|
||||
if stats.Added != 1 {
|
||||
t.Fatalf("added = %d, want one added video", stats.Added)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStopsWhenContextCanceledDuringFileLoop(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := &scannerFakeDrive{
|
||||
entries: []drives.Entry{
|
||||
{ID: "file-1", Name: "one.mp4", Size: 123},
|
||||
{ID: "file-2", Name: "two.mp4", Size: 123},
|
||||
{ID: "file-3", Name: "three.mp4", Size: 123},
|
||||
},
|
||||
}
|
||||
callbacks := 0
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, func(*catalog.Video) {
|
||||
callbacks++
|
||||
cancel()
|
||||
})
|
||||
|
||||
stats, err := sc.Run(ctx, "")
|
||||
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("scan error = %v, want context.Canceled", err)
|
||||
}
|
||||
if stats.Added != 1 || callbacks != 1 {
|
||||
t.Fatalf("added=%d callbacks=%d, want exactly one video before cancellation", stats.Added, callbacks)
|
||||
}
|
||||
if _, err := cat.GetVideo(context.Background(), "fake-drive-file-1"); err != nil {
|
||||
t.Fatalf("first video should be persisted before cancellation: %v", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(context.Background(), "fake-drive-file-2"); err != sql.ErrNoRows {
|
||||
t.Fatalf("second video lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(context.Background(), "fake-drive-file-3"); err != sql.ErrNoRows {
|
||||
t.Fatalf("third video lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSkipsAdminDeletedVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "fake-drive-file-1",
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
FileName: "clip.mp4",
|
||||
ContentHash: "HASH-1",
|
||||
Title: "Deleted Clip",
|
||||
Size: 123,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
if err := cat.DeleteVideoWithTombstone(ctx, "fake-drive-file-1"); err != nil {
|
||||
t.Fatalf("delete with tombstone: %v", err)
|
||||
}
|
||||
|
||||
drv := &scannerFakeDrive{
|
||||
entries: []drives.Entry{{
|
||||
ID: "file-1",
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
Hash: "hash-1",
|
||||
MimeType: "video/mp4",
|
||||
ModTime: now,
|
||||
}},
|
||||
}
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
|
||||
stats, err := sc.Run(ctx, "")
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Added != 0 {
|
||||
t.Fatalf("added = %d, want 0", stats.Added)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "fake-drive-file-1"); err != sql.ErrNoRows {
|
||||
t.Fatalf("deleted video was recreated, get error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDoesNotBackfillRemoteThumbnailForExistingVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -140,8 +273,8 @@ func TestRunBackfillsRemoteThumbnailForExistingVideo(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "https://thumbnail.example/backfilled.jpg" {
|
||||
t.Fatalf("thumbnail = %q, want backfilled remote thumbnail", got.ThumbnailURL)
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want empty so local thumbnail worker regenerates it", got.ThumbnailURL)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
// Package spider91migrate 周期性把 spider91 drive 下载到本地的视频
|
||||
// 上传到一个指定的目标 drive 目录(PikPak、115 或 OneDrive),上传成功后:
|
||||
// 上传到一个指定的目标 drive 目录(PikPak、115、123、OneDrive 或 Google Drive),上传成功后:
|
||||
//
|
||||
// - 改写 catalog 行:drive_id / file_id / content_hash 改成目标盘的;
|
||||
// 视频自身的 id 不变(仍是 spider91-<driveID>-<viewkey>),video_tags、
|
||||
// 收藏、点赞、views 等关联数据全部保留
|
||||
// - 删除本地 mp4(spider91/<id>/videos/<viewkey>.<ext>)和 thumb(spider91/<id>/thumbs/<viewkey>.jpg)
|
||||
// - 删除本地 mp4(spider91/<id>/videos/<viewkey>.<ext>)和源 thumb
|
||||
// (spider91/<id>/thumbs/<viewkey>.jpg);公共 /p/thumb/<videoID> 副本会保留
|
||||
//
|
||||
// 之后回放时,videoSource() 自动落到 /p/stream/<target>/<file_id>,
|
||||
// proxy 层走对应盘的直链 / 302 直连。
|
||||
@@ -28,19 +29,24 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/googledrive"
|
||||
"github.com/video-site/backend/internal/drives/onedrive"
|
||||
"github.com/video-site/backend/internal/drives/p115"
|
||||
"github.com/video-site/backend/internal/drives/p123"
|
||||
"github.com/video-site/backend/internal/drives/pikpak"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
)
|
||||
|
||||
// uploadTarget 是 migrator 调用目标 drive 的最小接口。任何一种"接收 spider91 上传"的
|
||||
// 网盘都要实现它;当前 PikPak 和 115 各自通过适配器满足。
|
||||
// 网盘都要实现它;当前 PikPak、115、123、OneDrive 和 Google Drive 各自通过适配器满足。
|
||||
//
|
||||
// 这一层抽象把"迁移调用方"和"具体盘的 SDK 协议"解耦:
|
||||
// - PikPak 走 GCID + OSS PutObject(pikpak.UploadResult)
|
||||
// - 115 走 SHA1 + 秒传 / OSS / 分片(p115.UploadResult)
|
||||
// - 123 走 MD5 + 秒传 / S3 预签名分片(p123.UploadResult)
|
||||
// - OneDrive 走 SHA1 + 小文件 PUT / 大文件 upload session
|
||||
// - Google Drive 走 MD5 + resumable upload session
|
||||
//
|
||||
// 各家返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
|
||||
type uploadTarget interface {
|
||||
@@ -55,7 +61,7 @@ type uploadTarget interface {
|
||||
// UploadResult 是 uploadTarget.UploadAndReportHash 的归一返回。
|
||||
//
|
||||
// FileID 目标盘上的新文件 ID;
|
||||
// Hash GCID(PikPak)或 SHA1 HEX(115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;
|
||||
// Hash GCID(PikPak)、MD5 HEX(123 / Google Drive)或 SHA1 HEX(115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;
|
||||
// Size 实际上传字节数。
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
@@ -65,7 +71,7 @@ type UploadResult struct {
|
||||
|
||||
const spider91UploadDirName = "91 Spider"
|
||||
|
||||
// pikpakAdapter / p115Adapter / onedriveAdapter 把具体 driver 包装成 uploadTarget。
|
||||
// pikpakAdapter / p115Adapter / p123Adapter / onedriveAdapter / googledriveAdapter 把具体 driver 包装成 uploadTarget。
|
||||
//
|
||||
// 之所以不让 driver 直接实现 uploadTarget:
|
||||
//
|
||||
@@ -114,6 +120,27 @@ func (a *p115Adapter) Rename(ctx context.Context, fileID, newName string) error
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
type p123Adapter struct {
|
||||
d *p123.Driver
|
||||
}
|
||||
|
||||
func (a *p123Adapter) ID() string { return a.d.ID() }
|
||||
func (a *p123Adapter) Kind() string { return a.d.Kind() }
|
||||
func (a *p123Adapter) RootID() string { return a.d.RootID() }
|
||||
func (a *p123Adapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *p123Adapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
return UploadResult{FileID: res.FileID, Hash: res.Hash, Size: res.Size}, nil
|
||||
}
|
||||
func (a *p123Adapter) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
type onedriveAdapter struct {
|
||||
d *onedrive.Driver
|
||||
}
|
||||
@@ -135,6 +162,27 @@ func (a *onedriveAdapter) Rename(ctx context.Context, fileID, newName string) er
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
type googledriveAdapter struct {
|
||||
d *googledrive.Driver
|
||||
}
|
||||
|
||||
func (a *googledriveAdapter) ID() string { return a.d.ID() }
|
||||
func (a *googledriveAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *googledriveAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *googledriveAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *googledriveAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
return UploadResult{FileID: res.FileID, Hash: res.Hash, Size: res.Size}, nil
|
||||
}
|
||||
func (a *googledriveAdapter) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
// adaptUploadTarget 把通用 drive 包装成 uploadTarget。
|
||||
// 不支持的盘 kind 返回 error;调用方静默跳过。
|
||||
func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
@@ -143,8 +191,12 @@ func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
return &pikpakAdapter{d: v}, nil
|
||||
case *p115.Driver:
|
||||
return &p115Adapter{d: v}, nil
|
||||
case *p123.Driver:
|
||||
return &p123Adapter{d: v}, nil
|
||||
case *onedrive.Driver:
|
||||
return &onedriveAdapter{d: v}, nil
|
||||
case *googledrive.Driver:
|
||||
return &googledriveAdapter{d: v}, nil
|
||||
case uploadTarget:
|
||||
// 测试或自定义实现可以直接传入;优先使用具体类型分支以拿到适配器。
|
||||
return v, nil
|
||||
@@ -175,6 +227,7 @@ type Config struct {
|
||||
// 4002 / 9)后整体进入冷却的时长。冷却期间 runOnce 直接返回,不再发起任何
|
||||
// PikPak API 请求,避免被进一步风控。0 时默认 5 分钟;< 0 关闭冷却(仅用于测试)。
|
||||
CaptchaCooldown time.Duration
|
||||
CommonThumbDir string
|
||||
OnMigrated func(videoID string)
|
||||
}
|
||||
|
||||
@@ -571,18 +624,98 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
|
||||
if err := m.cfg.Catalog.MigrateVideoToDrive(ctx, v.ID, targetDriveID, res.FileID, res.Hash); err != nil {
|
||||
return false, fmt.Errorf("catalog migrate: %w", err)
|
||||
}
|
||||
m.preserveCrawledThumbnail(ctx, src, v)
|
||||
// 同步 catalog 里的 file_name,让下次目标盘扫盘时 (file_name, size) 也能匹配上
|
||||
if err := m.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{FileName: uploadName}); err != nil {
|
||||
log.Printf("[spider91migrate] %s update file_name after migrate: %v", v.ID, err)
|
||||
}
|
||||
|
||||
// 删除本地 mp4 和 thumb(thumb 在 previews/thumbs/ 还有副本,不影响展示)
|
||||
// 删除本地 mp4 和源 thumb(公共 /p/thumb 副本已在 preserveCrawledThumbnail 中保留)。
|
||||
CleanupSpider91Local(src, v.FileID)
|
||||
|
||||
log.Printf("[spider91migrate] %s migrated to drive=%s(kind=%s) file=%s name=%q", v.ID, targetDriveID, pp.Kind(), res.FileID, uploadName)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *Migrator) preserveCrawledThumbnail(ctx context.Context, src *spider91.Driver, v *catalog.Video) {
|
||||
if m == nil || m.cfg.Catalog == nil || src == nil || v == nil || v.ID == "" || v.FileID == "" {
|
||||
return
|
||||
}
|
||||
commonDir := strings.TrimSpace(m.cfg.CommonThumbDir)
|
||||
if commonDir == "" {
|
||||
return
|
||||
}
|
||||
thumbPath, ok := findSpider91ThumbPath(src, v.FileID)
|
||||
if !ok {
|
||||
if v.ThumbnailURL == "" {
|
||||
log.Printf("[spider91migrate] %s crawled thumbnail missing before migration cleanup", v.ID)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(commonDir, 0o755); err != nil {
|
||||
log.Printf("[spider91migrate] %s mkdir common thumbs: %v", v.ID, err)
|
||||
return
|
||||
}
|
||||
dst := mediaasset.ThumbnailPathInDir(commonDir, v.ID)
|
||||
if _, err := os.Stat(dst); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
log.Printf("[spider91migrate] %s stat common thumb: %v", v.ID, err)
|
||||
return
|
||||
}
|
||||
if err := copyFileAtomic(thumbPath, dst); err != nil {
|
||||
log.Printf("[spider91migrate] %s preserve crawled thumbnail: %v", v.ID, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := m.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
ThumbnailURL: "/p/thumb/" + v.ID,
|
||||
}); err != nil {
|
||||
log.Printf("[spider91migrate] %s update crawled thumbnail url: %v", v.ID, err)
|
||||
return
|
||||
}
|
||||
v.ThumbnailURL = "/p/thumb/" + v.ID
|
||||
}
|
||||
|
||||
func findSpider91ThumbPath(src *spider91.Driver, fileID string) (string, bool) {
|
||||
thumbBase := stripExt(fileID)
|
||||
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
|
||||
thumbPath, err := src.ThumbPath(thumbBase + ext)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
info, statErr := os.Stat(thumbPath)
|
||||
if statErr == nil && info.Mode().IsRegular() && info.Size() > 0 {
|
||||
return thumbPath, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func copyFileAtomic(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
|
||||
tmp := dst + ".part"
|
||||
out, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, copyErr := io.Copy(out, in)
|
||||
closeErr := out.Close()
|
||||
if copyErr != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return copyErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return closeErr
|
||||
}
|
||||
return os.Rename(tmp, dst)
|
||||
}
|
||||
|
||||
// CleanupSpider91Local 删除已迁移视频的本地 mp4 和 thumb。
|
||||
//
|
||||
// thumb 删除是 best-effort —— 找不到就算了(spider91 thumb 文件名带后缀,
|
||||
@@ -677,7 +810,7 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driv
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// backfillFileNames 扫描目标 drive(PikPak、115 或 OneDrive)下所有 spider91-* 起始 ID 的视频,
|
||||
// backfillFileNames 扫描目标 drive(PikPak、115、123、OneDrive 或 Google Drive)下所有 spider91-* 起始 ID 的视频,
|
||||
// 对文件名不是 desiredPikPakName(...) 期望格式的,调 target.Rename 修正,
|
||||
// 并把 catalog.file_name 同步到新名字。
|
||||
//
|
||||
|
||||
@@ -14,6 +14,8 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/googledrive"
|
||||
"github.com/video-site/backend/internal/drives/p123"
|
||||
"github.com/video-site/backend/internal/drives/pikpak"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
)
|
||||
@@ -134,6 +136,19 @@ func (d *fakeP115) Kind() string { return "p115" }
|
||||
var _ drives.Drive = (*fakeP115)(nil)
|
||||
var _ uploadTarget = (*fakeP115)(nil)
|
||||
|
||||
type fakeP123 struct {
|
||||
*fakePikPak
|
||||
}
|
||||
|
||||
func newFakeP123(id, rootID string) *fakeP123 {
|
||||
return &fakeP123{fakePikPak: newFakePikPak(id, rootID)}
|
||||
}
|
||||
|
||||
func (d *fakeP123) Kind() string { return "p123" }
|
||||
|
||||
var _ drives.Drive = (*fakeP123)(nil)
|
||||
var _ uploadTarget = (*fakeP123)(nil)
|
||||
|
||||
type fakeOneDrive struct {
|
||||
*fakePikPak
|
||||
}
|
||||
@@ -339,12 +354,14 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
|
||||
|
||||
now := time.Now()
|
||||
id := writeSpider91Video(t, cat, src, "vk001", ".mp4", []byte("video bytes here"), now)
|
||||
commonThumbDir := t.TempDir()
|
||||
|
||||
m := New(Config{
|
||||
Catalog: cat,
|
||||
Registry: reg,
|
||||
GetTargetDriveID: func() string { return pp.ID() },
|
||||
KeepLatestN: -1, // 关闭"保留最新 N 个",让 1 条也能立即上传
|
||||
CommonThumbDir: commonThumbDir,
|
||||
})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
@@ -382,8 +399,15 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
|
||||
if got.ContentHash == "" {
|
||||
t.Fatalf("content_hash should be set after migration")
|
||||
}
|
||||
if got.ThumbnailURL != "/p/thumb/"+id {
|
||||
t.Fatalf("thumbnail_url = %q, want preserved crawled thumbnail URL", got.ThumbnailURL)
|
||||
}
|
||||
commonThumbPath := filepath.Join(commonThumbDir, id+".jpg")
|
||||
if data, err := os.ReadFile(commonThumbPath); err != nil || string(data) != "thumb" {
|
||||
t.Fatalf("common thumb = %q, %v; want copied crawled thumb", string(data), err)
|
||||
}
|
||||
|
||||
// 3) 本地视频和 thumb 都被删了
|
||||
// 3) 本地视频和源 thumb 都被删了;公共 /p/thumb 副本保留。
|
||||
videoPath, _ := src.VideoPath("vk001.mp4")
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local mp4 still exists or stat error %v", err)
|
||||
@@ -937,6 +961,66 @@ func TestRunOnceMigratesToP115Target(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceMigratesToP123Target(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src, _ := setupSpider91(t)
|
||||
target := newFakeP123("p123-target", "p123-root-id")
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(target)
|
||||
|
||||
now := time.Now()
|
||||
id := writeSpider91Video(t, cat, src, "vk-123-001", ".mp4", []byte("video bytes 123"), now)
|
||||
|
||||
m := New(Config{
|
||||
Catalog: cat,
|
||||
Registry: reg,
|
||||
GetTargetDriveID: func() string { return target.ID() },
|
||||
KeepLatestN: -1,
|
||||
})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
if target.uploadCalls != 1 {
|
||||
t.Fatalf("p123 upload calls = %d, want 1", target.uploadCalls)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DriveID != target.ID() {
|
||||
t.Fatalf("drive_id = %q, want %q", got.DriveID, target.ID())
|
||||
}
|
||||
wantName := "Sample vk-123-001-001.mp4"
|
||||
if _, ok := target.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("p123 did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
|
||||
}
|
||||
if gotParent := target.gotParents[wantName]; gotParent != "p123-root-id/"+spider91UploadDirName {
|
||||
t.Fatalf("p123 upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("p123 ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
if got.FileName != wantName {
|
||||
t.Fatalf("file_name = %q, want %q", got.FileName, wantName)
|
||||
}
|
||||
if got.ContentHash == "" {
|
||||
t.Fatal("content_hash should be set after p123 migration")
|
||||
}
|
||||
|
||||
videoPath, _ := src.VideoPath("vk-123-001.mp4")
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local mp4 still exists after p123 migration or stat error: %v", err)
|
||||
}
|
||||
thumbPath, _ := src.ThumbPath("vk-123-001.jpg")
|
||||
if _, err := os.Stat(thumbPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local thumb still exists after p123 migration or stat error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceMigratesToOneDriveTarget(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src, _ := setupSpider91(t)
|
||||
@@ -997,7 +1081,37 @@ func TestRunOnceMigratesToOneDriveTarget(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115 也不是 OneDrive 时,
|
||||
func TestAdaptUploadTargetSupportsP123Driver(t *testing.T) {
|
||||
d := p123.New(p123.Config{
|
||||
ID: "p123-target",
|
||||
RootID: "root-123",
|
||||
AccessToken: "token-1",
|
||||
})
|
||||
target, err := adaptUploadTarget(d)
|
||||
if err != nil {
|
||||
t.Fatalf("adaptUploadTarget() error = %v", err)
|
||||
}
|
||||
if target.ID() != "p123-target" || target.Kind() != "p123" || target.RootID() != "root-123" {
|
||||
t.Fatalf("target id/kind/root = %q/%q/%q, want p123-target/p123/root-123", target.ID(), target.Kind(), target.RootID())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdaptUploadTargetSupportsGoogleDriveDriver(t *testing.T) {
|
||||
d := googledrive.New(googledrive.Config{
|
||||
ID: "google-target",
|
||||
RootID: "root-google",
|
||||
RefreshToken: "refresh-token",
|
||||
})
|
||||
target, err := adaptUploadTarget(d)
|
||||
if err != nil {
|
||||
t.Fatalf("adaptUploadTarget() error = %v", err)
|
||||
}
|
||||
if target.ID() != "google-target" || target.Kind() != "googledrive" || target.RootID() != "root-google" {
|
||||
t.Fatalf("target id/kind/root = %q/%q/%q, want google-target/googledrive/root-google", target.ID(), target.Kind(), target.RootID())
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115、123、OneDrive 也不是 Google Drive 时,
|
||||
// resolveTarget 拒绝并返回 error,让 runOnce 静默跳过(不会做破坏性变更)。
|
||||
func TestResolveTargetRejectsUnsupportedKind(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
)
|
||||
|
||||
type VideoAssetRef struct {
|
||||
@@ -71,14 +73,15 @@ func Compute(
|
||||
continue
|
||||
}
|
||||
driveUsage := out.Drives[ref.DriveID]
|
||||
thumbPath := filepath.Join(localDir, "thumbs", ref.ID+".jpg")
|
||||
if size, exists, err := regularFileSize(thumbPath); err != nil {
|
||||
return Usage{}, err
|
||||
} else if exists {
|
||||
key := ref.DriveID + "\x00thumb\x00" + thumbPath
|
||||
if !seen[key] {
|
||||
driveUsage.ThumbnailBytes += size
|
||||
seen[key] = true
|
||||
for _, thumbPath := range mediaasset.ThumbnailPathCandidates(localDir, ref.ID) {
|
||||
if size, exists, err := regularFileSize(thumbPath); err != nil {
|
||||
return Usage{}, err
|
||||
} else if exists {
|
||||
key := ref.DriveID + "\x00thumb\x00" + thumbPath
|
||||
if !seen[key] {
|
||||
driveUsage.ThumbnailBytes += size
|
||||
seen[key] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,10 @@ package storageusage
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
)
|
||||
|
||||
func TestComputeCountsLocalThumbnailsAndTeasersByDrive(t *testing.T) {
|
||||
@@ -13,6 +16,8 @@ func TestComputeCountsLocalThumbnailsAndTeasersByDrive(t *testing.T) {
|
||||
}
|
||||
writeSizedFile(t, filepath.Join(localDir, "thumbs", "video-a.jpg"), 3)
|
||||
writeSizedFile(t, filepath.Join(localDir, "thumbs", "video-b.jpg"), 5)
|
||||
longID := "localstorage-" + strings.Repeat("x", 240)
|
||||
writeSizedFile(t, mediaasset.ThumbnailPath(localDir, longID), 13)
|
||||
teaserA := filepath.Join(localDir, "video-a.mp4")
|
||||
teaserB := filepath.Join(localDir, "video-b.mp4")
|
||||
writeSizedFile(t, teaserA, 7)
|
||||
@@ -24,6 +29,7 @@ func TestComputeCountsLocalThumbnailsAndTeasersByDrive(t *testing.T) {
|
||||
{ID: "video-a", DriveID: "drive-a", PreviewLocal: teaserA},
|
||||
{ID: "video-a-copy", DriveID: "drive-a", PreviewLocal: teaserA},
|
||||
{ID: "video-b", DriveID: "drive-b", PreviewLocal: teaserB},
|
||||
{ID: longID, DriveID: "drive-b"},
|
||||
{ID: "outside", DriveID: "drive-b", PreviewLocal: outside},
|
||||
{ID: "unknown-drive-video", DriveID: "missing", PreviewLocal: teaserB},
|
||||
}, []string{"drive-a", "drive-b"}, func(string) (DiskStats, error) {
|
||||
@@ -41,11 +47,11 @@ func TestComputeCountsLocalThumbnailsAndTeasersByDrive(t *testing.T) {
|
||||
t.Fatalf("drive-a usage = %#v, want thumbnails=3 teaser=7 total=10", driveA)
|
||||
}
|
||||
driveB := got.Drives["drive-b"]
|
||||
if driveB.ThumbnailBytes != 5 || driveB.TeaserBytes != 11 || driveB.TotalBytes != 16 {
|
||||
t.Fatalf("drive-b usage = %#v, want thumbnails=5 teaser=11 total=16", driveB)
|
||||
if driveB.ThumbnailBytes != 18 || driveB.TeaserBytes != 11 || driveB.TotalBytes != 29 {
|
||||
t.Fatalf("drive-b usage = %#v, want thumbnails=18 teaser=11 total=29", driveB)
|
||||
}
|
||||
if got.ThumbnailBytes != 8 || got.TeaserBytes != 18 || got.TotalBytes != 26 {
|
||||
t.Fatalf("totals = %#v, want thumbnails=8 teaser=18 total=26", got)
|
||||
if got.ThumbnailBytes != 21 || got.TeaserBytes != 18 || got.TotalBytes != 39 {
|
||||
t.Fatalf("totals = %#v, want thumbnails=21 teaser=18 total=39", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
// Copyright 2018 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package socks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
noDeadline = time.Time{}
|
||||
aLongTimeAgo = time.Unix(1, 0)
|
||||
)
|
||||
|
||||
func (d *Dialer) connect(ctx context.Context, c net.Conn, address string) (_ net.Addr, ctxErr error) {
|
||||
host, port, err := splitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if deadline, ok := ctx.Deadline(); ok && !deadline.IsZero() {
|
||||
c.SetDeadline(deadline)
|
||||
defer c.SetDeadline(noDeadline)
|
||||
}
|
||||
if ctx != context.Background() {
|
||||
errCh := make(chan error, 1)
|
||||
done := make(chan struct{})
|
||||
defer func() {
|
||||
close(done)
|
||||
if ctxErr == nil {
|
||||
ctxErr = <-errCh
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
c.SetDeadline(aLongTimeAgo)
|
||||
errCh <- ctx.Err()
|
||||
case <-done:
|
||||
errCh <- nil
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
b := make([]byte, 0, 6+len(host)) // the size here is just an estimate
|
||||
b = append(b, Version5)
|
||||
if len(d.AuthMethods) == 0 || d.Authenticate == nil {
|
||||
b = append(b, 1, byte(AuthMethodNotRequired))
|
||||
} else {
|
||||
ams := d.AuthMethods
|
||||
if len(ams) > 255 {
|
||||
return nil, errors.New("too many authentication methods")
|
||||
}
|
||||
b = append(b, byte(len(ams)))
|
||||
for _, am := range ams {
|
||||
b = append(b, byte(am))
|
||||
}
|
||||
}
|
||||
if _, ctxErr = c.Write(b); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if _, ctxErr = io.ReadFull(c, b[:2]); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
if b[0] != Version5 {
|
||||
return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0])))
|
||||
}
|
||||
am := AuthMethod(b[1])
|
||||
if am == AuthMethodNoAcceptableMethods {
|
||||
return nil, errors.New("no acceptable authentication methods")
|
||||
}
|
||||
if d.Authenticate != nil {
|
||||
if ctxErr = d.Authenticate(ctx, c, am); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
b = b[:0]
|
||||
b = append(b, Version5, byte(d.cmd), 0)
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
b = append(b, AddrTypeIPv4)
|
||||
b = append(b, ip4...)
|
||||
} else if ip6 := ip.To16(); ip6 != nil {
|
||||
b = append(b, AddrTypeIPv6)
|
||||
b = append(b, ip6...)
|
||||
} else {
|
||||
return nil, errors.New("unknown address type")
|
||||
}
|
||||
} else {
|
||||
if len(host) > 255 {
|
||||
return nil, errors.New("FQDN too long")
|
||||
}
|
||||
b = append(b, AddrTypeFQDN)
|
||||
b = append(b, byte(len(host)))
|
||||
b = append(b, host...)
|
||||
}
|
||||
b = append(b, byte(port>>8), byte(port))
|
||||
if _, ctxErr = c.Write(b); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if _, ctxErr = io.ReadFull(c, b[:4]); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
if b[0] != Version5 {
|
||||
return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0])))
|
||||
}
|
||||
if cmdErr := Reply(b[1]); cmdErr != StatusSucceeded {
|
||||
return nil, errors.New("unknown error " + cmdErr.String())
|
||||
}
|
||||
if b[2] != 0 {
|
||||
return nil, errors.New("non-zero reserved field")
|
||||
}
|
||||
l := 2
|
||||
var a Addr
|
||||
switch b[3] {
|
||||
case AddrTypeIPv4:
|
||||
l += net.IPv4len
|
||||
a.IP = make(net.IP, net.IPv4len)
|
||||
case AddrTypeIPv6:
|
||||
l += net.IPv6len
|
||||
a.IP = make(net.IP, net.IPv6len)
|
||||
case AddrTypeFQDN:
|
||||
if _, err := io.ReadFull(c, b[:1]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l += int(b[0])
|
||||
default:
|
||||
return nil, errors.New("unknown address type " + strconv.Itoa(int(b[3])))
|
||||
}
|
||||
if cap(b) < l {
|
||||
b = make([]byte, l)
|
||||
} else {
|
||||
b = b[:l]
|
||||
}
|
||||
if _, ctxErr = io.ReadFull(c, b); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
if a.IP != nil {
|
||||
copy(a.IP, b)
|
||||
} else {
|
||||
a.Name = string(b[:len(b)-2])
|
||||
}
|
||||
a.Port = int(b[len(b)-2])<<8 | int(b[len(b)-1])
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
func splitHostPort(address string) (string, int, error) {
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
portnum, err := strconv.Atoi(port)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if 1 > portnum || portnum > 0xffff {
|
||||
return "", 0, errors.New("port number out of range " + port)
|
||||
}
|
||||
return host, portnum, nil
|
||||
}
|
||||
+317
@@ -0,0 +1,317 @@
|
||||
// Copyright 2018 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package socks provides a SOCKS version 5 client implementation.
|
||||
//
|
||||
// SOCKS protocol version 5 is defined in RFC 1928.
|
||||
// Username/Password authentication for SOCKS version 5 is defined in
|
||||
// RFC 1929.
|
||||
package socks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// A Command represents a SOCKS command.
|
||||
type Command int
|
||||
|
||||
func (cmd Command) String() string {
|
||||
switch cmd {
|
||||
case CmdConnect:
|
||||
return "socks connect"
|
||||
case cmdBind:
|
||||
return "socks bind"
|
||||
default:
|
||||
return "socks " + strconv.Itoa(int(cmd))
|
||||
}
|
||||
}
|
||||
|
||||
// An AuthMethod represents a SOCKS authentication method.
|
||||
type AuthMethod int
|
||||
|
||||
// A Reply represents a SOCKS command reply code.
|
||||
type Reply int
|
||||
|
||||
func (code Reply) String() string {
|
||||
switch code {
|
||||
case StatusSucceeded:
|
||||
return "succeeded"
|
||||
case 0x01:
|
||||
return "general SOCKS server failure"
|
||||
case 0x02:
|
||||
return "connection not allowed by ruleset"
|
||||
case 0x03:
|
||||
return "network unreachable"
|
||||
case 0x04:
|
||||
return "host unreachable"
|
||||
case 0x05:
|
||||
return "connection refused"
|
||||
case 0x06:
|
||||
return "TTL expired"
|
||||
case 0x07:
|
||||
return "command not supported"
|
||||
case 0x08:
|
||||
return "address type not supported"
|
||||
default:
|
||||
return "unknown code: " + strconv.Itoa(int(code))
|
||||
}
|
||||
}
|
||||
|
||||
// Wire protocol constants.
|
||||
const (
|
||||
Version5 = 0x05
|
||||
|
||||
AddrTypeIPv4 = 0x01
|
||||
AddrTypeFQDN = 0x03
|
||||
AddrTypeIPv6 = 0x04
|
||||
|
||||
CmdConnect Command = 0x01 // establishes an active-open forward proxy connection
|
||||
cmdBind Command = 0x02 // establishes a passive-open forward proxy connection
|
||||
|
||||
AuthMethodNotRequired AuthMethod = 0x00 // no authentication required
|
||||
AuthMethodUsernamePassword AuthMethod = 0x02 // use username/password
|
||||
AuthMethodNoAcceptableMethods AuthMethod = 0xff // no acceptable authentication methods
|
||||
|
||||
StatusSucceeded Reply = 0x00
|
||||
)
|
||||
|
||||
// An Addr represents a SOCKS-specific address.
|
||||
// Either Name or IP is used exclusively.
|
||||
type Addr struct {
|
||||
Name string // fully-qualified domain name
|
||||
IP net.IP
|
||||
Port int
|
||||
}
|
||||
|
||||
func (a *Addr) Network() string { return "socks" }
|
||||
|
||||
func (a *Addr) String() string {
|
||||
if a == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
port := strconv.Itoa(a.Port)
|
||||
if a.IP == nil {
|
||||
return net.JoinHostPort(a.Name, port)
|
||||
}
|
||||
return net.JoinHostPort(a.IP.String(), port)
|
||||
}
|
||||
|
||||
// A Conn represents a forward proxy connection.
|
||||
type Conn struct {
|
||||
net.Conn
|
||||
|
||||
boundAddr net.Addr
|
||||
}
|
||||
|
||||
// BoundAddr returns the address assigned by the proxy server for
|
||||
// connecting to the command target address from the proxy server.
|
||||
func (c *Conn) BoundAddr() net.Addr {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
return c.boundAddr
|
||||
}
|
||||
|
||||
// A Dialer holds SOCKS-specific options.
|
||||
type Dialer struct {
|
||||
cmd Command // either CmdConnect or cmdBind
|
||||
proxyNetwork string // network between a proxy server and a client
|
||||
proxyAddress string // proxy server address
|
||||
|
||||
// ProxyDial specifies the optional dial function for
|
||||
// establishing the transport connection.
|
||||
ProxyDial func(context.Context, string, string) (net.Conn, error)
|
||||
|
||||
// AuthMethods specifies the list of request authentication
|
||||
// methods.
|
||||
// If empty, SOCKS client requests only AuthMethodNotRequired.
|
||||
AuthMethods []AuthMethod
|
||||
|
||||
// Authenticate specifies the optional authentication
|
||||
// function. It must be non-nil when AuthMethods is not empty.
|
||||
// It must return an error when the authentication is failed.
|
||||
Authenticate func(context.Context, io.ReadWriter, AuthMethod) error
|
||||
}
|
||||
|
||||
// DialContext connects to the provided address on the provided
|
||||
// network.
|
||||
//
|
||||
// The returned error value may be a net.OpError. When the Op field of
|
||||
// net.OpError contains "socks", the Source field contains a proxy
|
||||
// server address and the Addr field contains a command target
|
||||
// address.
|
||||
//
|
||||
// See func Dial of the net package of standard library for a
|
||||
// description of the network and address parameters.
|
||||
func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
if err := d.validateTarget(network, address); err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
if ctx == nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")}
|
||||
}
|
||||
var err error
|
||||
var c net.Conn
|
||||
if d.ProxyDial != nil {
|
||||
c, err = d.ProxyDial(ctx, d.proxyNetwork, d.proxyAddress)
|
||||
} else {
|
||||
var dd net.Dialer
|
||||
c, err = dd.DialContext(ctx, d.proxyNetwork, d.proxyAddress)
|
||||
}
|
||||
if err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
a, err := d.connect(ctx, c, address)
|
||||
if err != nil {
|
||||
c.Close()
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
return &Conn{Conn: c, boundAddr: a}, nil
|
||||
}
|
||||
|
||||
// DialWithConn initiates a connection from SOCKS server to the target
|
||||
// network and address using the connection c that is already
|
||||
// connected to the SOCKS server.
|
||||
//
|
||||
// It returns the connection's local address assigned by the SOCKS
|
||||
// server.
|
||||
func (d *Dialer) DialWithConn(ctx context.Context, c net.Conn, network, address string) (net.Addr, error) {
|
||||
if err := d.validateTarget(network, address); err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
if ctx == nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")}
|
||||
}
|
||||
a, err := d.connect(ctx, c, address)
|
||||
if err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// Dial connects to the provided address on the provided network.
|
||||
//
|
||||
// Unlike DialContext, it returns a raw transport connection instead
|
||||
// of a forward proxy connection.
|
||||
//
|
||||
// Deprecated: Use DialContext or DialWithConn instead.
|
||||
func (d *Dialer) Dial(network, address string) (net.Conn, error) {
|
||||
if err := d.validateTarget(network, address); err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
var err error
|
||||
var c net.Conn
|
||||
if d.ProxyDial != nil {
|
||||
c, err = d.ProxyDial(context.Background(), d.proxyNetwork, d.proxyAddress)
|
||||
} else {
|
||||
c, err = net.Dial(d.proxyNetwork, d.proxyAddress)
|
||||
}
|
||||
if err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
if _, err := d.DialWithConn(context.Background(), c, network, address); err != nil {
|
||||
c.Close()
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (d *Dialer) validateTarget(network, address string) error {
|
||||
switch network {
|
||||
case "tcp", "tcp6", "tcp4":
|
||||
default:
|
||||
return errors.New("network not implemented")
|
||||
}
|
||||
switch d.cmd {
|
||||
case CmdConnect, cmdBind:
|
||||
default:
|
||||
return errors.New("command not implemented")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Dialer) pathAddrs(address string) (proxy, dst net.Addr, err error) {
|
||||
for i, s := range []string{d.proxyAddress, address} {
|
||||
host, port, err := splitHostPort(s)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
a := &Addr{Port: port}
|
||||
a.IP = net.ParseIP(host)
|
||||
if a.IP == nil {
|
||||
a.Name = host
|
||||
}
|
||||
if i == 0 {
|
||||
proxy = a
|
||||
} else {
|
||||
dst = a
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// NewDialer returns a new Dialer that dials through the provided
|
||||
// proxy server's network and address.
|
||||
func NewDialer(network, address string) *Dialer {
|
||||
return &Dialer{proxyNetwork: network, proxyAddress: address, cmd: CmdConnect}
|
||||
}
|
||||
|
||||
const (
|
||||
authUsernamePasswordVersion = 0x01
|
||||
authStatusSucceeded = 0x00
|
||||
)
|
||||
|
||||
// UsernamePassword are the credentials for the username/password
|
||||
// authentication method.
|
||||
type UsernamePassword struct {
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
// Authenticate authenticates a pair of username and password with the
|
||||
// proxy server.
|
||||
func (up *UsernamePassword) Authenticate(ctx context.Context, rw io.ReadWriter, auth AuthMethod) error {
|
||||
switch auth {
|
||||
case AuthMethodNotRequired:
|
||||
return nil
|
||||
case AuthMethodUsernamePassword:
|
||||
if len(up.Username) == 0 || len(up.Username) > 255 || len(up.Password) > 255 {
|
||||
return errors.New("invalid username/password")
|
||||
}
|
||||
b := []byte{authUsernamePasswordVersion}
|
||||
b = append(b, byte(len(up.Username)))
|
||||
b = append(b, up.Username...)
|
||||
b = append(b, byte(len(up.Password)))
|
||||
b = append(b, up.Password...)
|
||||
// TODO(mikio): handle IO deadlines and cancelation if
|
||||
// necessary
|
||||
if _, err := rw.Write(b); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.ReadFull(rw, b[:2]); err != nil {
|
||||
return err
|
||||
}
|
||||
if b[0] != authUsernamePasswordVersion {
|
||||
return errors.New("invalid username/password version")
|
||||
}
|
||||
if b[1] != authStatusSucceeded {
|
||||
return errors.New("username/password authentication failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errors.New("unsupported authentication method " + strconv.Itoa(int(auth)))
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
// Copyright 2019 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
)
|
||||
|
||||
// A ContextDialer dials using a context.
|
||||
type ContextDialer interface {
|
||||
DialContext(ctx context.Context, network, address string) (net.Conn, error)
|
||||
}
|
||||
|
||||
// Dial works like DialContext on net.Dialer but using a dialer returned by FromEnvironment.
|
||||
//
|
||||
// The passed ctx is only used for returning the Conn, not the lifetime of the Conn.
|
||||
//
|
||||
// Custom dialers (registered via RegisterDialerType) that do not implement ContextDialer
|
||||
// can leak a goroutine for as long as it takes the underlying Dialer implementation to timeout.
|
||||
//
|
||||
// A Conn returned from a successful Dial after the context has been cancelled will be immediately closed.
|
||||
func Dial(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
d := FromEnvironment()
|
||||
if xd, ok := d.(ContextDialer); ok {
|
||||
return xd.DialContext(ctx, network, address)
|
||||
}
|
||||
return dialContext(ctx, d, network, address)
|
||||
}
|
||||
|
||||
// WARNING: this can leak a goroutine for as long as the underlying Dialer implementation takes to timeout
|
||||
// A Conn returned from a successful Dial after the context has been cancelled will be immediately closed.
|
||||
func dialContext(ctx context.Context, d Dialer, network, address string) (net.Conn, error) {
|
||||
var (
|
||||
conn net.Conn
|
||||
done = make(chan struct{}, 1)
|
||||
err error
|
||||
)
|
||||
go func() {
|
||||
conn, err = d.Dial(network, address)
|
||||
close(done)
|
||||
if conn != nil && ctx.Err() != nil {
|
||||
conn.Close()
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
case <-done:
|
||||
}
|
||||
return conn, err
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
)
|
||||
|
||||
type direct struct{}
|
||||
|
||||
// Direct implements Dialer by making network connections directly using net.Dial or net.DialContext.
|
||||
var Direct = direct{}
|
||||
|
||||
var (
|
||||
_ Dialer = Direct
|
||||
_ ContextDialer = Direct
|
||||
)
|
||||
|
||||
// Dial directly invokes net.Dial with the supplied parameters.
|
||||
func (direct) Dial(network, addr string) (net.Conn, error) {
|
||||
return net.Dial(network, addr)
|
||||
}
|
||||
|
||||
// DialContext instantiates a net.Dialer and invokes its DialContext receiver with the supplied parameters.
|
||||
func (direct) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, network, addr)
|
||||
}
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A PerHost directs connections to a default Dialer unless the host name
|
||||
// requested matches one of a number of exceptions.
|
||||
type PerHost struct {
|
||||
def, bypass Dialer
|
||||
|
||||
bypassNetworks []*net.IPNet
|
||||
bypassIPs []net.IP
|
||||
bypassZones []string
|
||||
bypassHosts []string
|
||||
}
|
||||
|
||||
// NewPerHost returns a PerHost Dialer that directs connections to either
|
||||
// defaultDialer or bypass, depending on whether the connection matches one of
|
||||
// the configured rules.
|
||||
func NewPerHost(defaultDialer, bypass Dialer) *PerHost {
|
||||
return &PerHost{
|
||||
def: defaultDialer,
|
||||
bypass: bypass,
|
||||
}
|
||||
}
|
||||
|
||||
// Dial connects to the address addr on the given network through either
|
||||
// defaultDialer or bypass.
|
||||
func (p *PerHost) Dial(network, addr string) (c net.Conn, err error) {
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return p.dialerForRequest(host).Dial(network, addr)
|
||||
}
|
||||
|
||||
// DialContext connects to the address addr on the given network through either
|
||||
// defaultDialer or bypass.
|
||||
func (p *PerHost) DialContext(ctx context.Context, network, addr string) (c net.Conn, err error) {
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := p.dialerForRequest(host)
|
||||
if x, ok := d.(ContextDialer); ok {
|
||||
return x.DialContext(ctx, network, addr)
|
||||
}
|
||||
return dialContext(ctx, d, network, addr)
|
||||
}
|
||||
|
||||
func (p *PerHost) dialerForRequest(host string) Dialer {
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
for _, net := range p.bypassNetworks {
|
||||
if net.Contains(ip) {
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
for _, bypassIP := range p.bypassIPs {
|
||||
if bypassIP.Equal(ip) {
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
return p.def
|
||||
}
|
||||
|
||||
for _, zone := range p.bypassZones {
|
||||
if strings.HasSuffix(host, zone) {
|
||||
return p.bypass
|
||||
}
|
||||
if host == zone[1:] {
|
||||
// For a zone ".example.com", we match "example.com"
|
||||
// too.
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
for _, bypassHost := range p.bypassHosts {
|
||||
if bypassHost == host {
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
return p.def
|
||||
}
|
||||
|
||||
// AddFromString parses a string that contains comma-separated values
|
||||
// specifying hosts that should use the bypass proxy. Each value is either an
|
||||
// IP address, a CIDR range, a zone (*.example.com) or a host name
|
||||
// (localhost). A best effort is made to parse the string and errors are
|
||||
// ignored.
|
||||
func (p *PerHost) AddFromString(s string) {
|
||||
hosts := strings.Split(s, ",")
|
||||
for _, host := range hosts {
|
||||
host = strings.TrimSpace(host)
|
||||
if len(host) == 0 {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(host, "/") {
|
||||
// We assume that it's a CIDR address like 127.0.0.0/8
|
||||
if _, net, err := net.ParseCIDR(host); err == nil {
|
||||
p.AddNetwork(net)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
p.AddIP(ip)
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(host, "*.") {
|
||||
p.AddZone(host[1:])
|
||||
continue
|
||||
}
|
||||
p.AddHost(host)
|
||||
}
|
||||
}
|
||||
|
||||
// AddIP specifies an IP address that will use the bypass proxy. Note that
|
||||
// this will only take effect if a literal IP address is dialed. A connection
|
||||
// to a named host will never match an IP.
|
||||
func (p *PerHost) AddIP(ip net.IP) {
|
||||
p.bypassIPs = append(p.bypassIPs, ip)
|
||||
}
|
||||
|
||||
// AddNetwork specifies an IP range that will use the bypass proxy. Note that
|
||||
// this will only take effect if a literal IP address is dialed. A connection
|
||||
// to a named host will never match.
|
||||
func (p *PerHost) AddNetwork(net *net.IPNet) {
|
||||
p.bypassNetworks = append(p.bypassNetworks, net)
|
||||
}
|
||||
|
||||
// AddZone specifies a DNS suffix that will use the bypass proxy. A zone of
|
||||
// "example.com" matches "example.com" and all of its subdomains.
|
||||
func (p *PerHost) AddZone(zone string) {
|
||||
zone = strings.TrimSuffix(zone, ".")
|
||||
if !strings.HasPrefix(zone, ".") {
|
||||
zone = "." + zone
|
||||
}
|
||||
p.bypassZones = append(p.bypassZones, zone)
|
||||
}
|
||||
|
||||
// AddHost specifies a host name that will use the bypass proxy.
|
||||
func (p *PerHost) AddHost(host string) {
|
||||
host = strings.TrimSuffix(host, ".")
|
||||
p.bypassHosts = append(p.bypassHosts, host)
|
||||
}
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package proxy provides support for a variety of protocols to proxy network
|
||||
// data.
|
||||
package proxy // import "golang.org/x/net/proxy"
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// A Dialer is a means to establish a connection.
|
||||
// Custom dialers should also implement ContextDialer.
|
||||
type Dialer interface {
|
||||
// Dial connects to the given address via the proxy.
|
||||
Dial(network, addr string) (c net.Conn, err error)
|
||||
}
|
||||
|
||||
// Auth contains authentication parameters that specific Dialers may require.
|
||||
type Auth struct {
|
||||
User, Password string
|
||||
}
|
||||
|
||||
// FromEnvironment returns the dialer specified by the proxy-related
|
||||
// variables in the environment and makes underlying connections
|
||||
// directly.
|
||||
func FromEnvironment() Dialer {
|
||||
return FromEnvironmentUsing(Direct)
|
||||
}
|
||||
|
||||
// FromEnvironmentUsing returns the dialer specify by the proxy-related
|
||||
// variables in the environment and makes underlying connections
|
||||
// using the provided forwarding Dialer (for instance, a *net.Dialer
|
||||
// with desired configuration).
|
||||
func FromEnvironmentUsing(forward Dialer) Dialer {
|
||||
allProxy := allProxyEnv.Get()
|
||||
if len(allProxy) == 0 {
|
||||
return forward
|
||||
}
|
||||
|
||||
proxyURL, err := url.Parse(allProxy)
|
||||
if err != nil {
|
||||
return forward
|
||||
}
|
||||
proxy, err := FromURL(proxyURL, forward)
|
||||
if err != nil {
|
||||
return forward
|
||||
}
|
||||
|
||||
noProxy := noProxyEnv.Get()
|
||||
if len(noProxy) == 0 {
|
||||
return proxy
|
||||
}
|
||||
|
||||
perHost := NewPerHost(proxy, forward)
|
||||
perHost.AddFromString(noProxy)
|
||||
return perHost
|
||||
}
|
||||
|
||||
// proxySchemes is a map from URL schemes to a function that creates a Dialer
|
||||
// from a URL with such a scheme.
|
||||
var proxySchemes map[string]func(*url.URL, Dialer) (Dialer, error)
|
||||
|
||||
// RegisterDialerType takes a URL scheme and a function to generate Dialers from
|
||||
// a URL with that scheme and a forwarding Dialer. Registered schemes are used
|
||||
// by FromURL.
|
||||
func RegisterDialerType(scheme string, f func(*url.URL, Dialer) (Dialer, error)) {
|
||||
if proxySchemes == nil {
|
||||
proxySchemes = make(map[string]func(*url.URL, Dialer) (Dialer, error))
|
||||
}
|
||||
proxySchemes[scheme] = f
|
||||
}
|
||||
|
||||
// FromURL returns a Dialer given a URL specification and an underlying
|
||||
// Dialer for it to make network requests.
|
||||
func FromURL(u *url.URL, forward Dialer) (Dialer, error) {
|
||||
var auth *Auth
|
||||
if u.User != nil {
|
||||
auth = new(Auth)
|
||||
auth.User = u.User.Username()
|
||||
if p, ok := u.User.Password(); ok {
|
||||
auth.Password = p
|
||||
}
|
||||
}
|
||||
|
||||
switch u.Scheme {
|
||||
case "socks5", "socks5h":
|
||||
addr := u.Hostname()
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
port = "1080"
|
||||
}
|
||||
return SOCKS5("tcp", net.JoinHostPort(addr, port), auth, forward)
|
||||
}
|
||||
|
||||
// If the scheme doesn't match any of the built-in schemes, see if it
|
||||
// was registered by another package.
|
||||
if proxySchemes != nil {
|
||||
if f, ok := proxySchemes[u.Scheme]; ok {
|
||||
return f(u, forward)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.New("proxy: unknown scheme: " + u.Scheme)
|
||||
}
|
||||
|
||||
var (
|
||||
allProxyEnv = &envOnce{
|
||||
names: []string{"ALL_PROXY", "all_proxy"},
|
||||
}
|
||||
noProxyEnv = &envOnce{
|
||||
names: []string{"NO_PROXY", "no_proxy"},
|
||||
}
|
||||
)
|
||||
|
||||
// envOnce looks up an environment variable (optionally by multiple
|
||||
// names) once. It mitigates expensive lookups on some platforms
|
||||
// (e.g. Windows).
|
||||
// (Borrowed from net/http/transport.go)
|
||||
type envOnce struct {
|
||||
names []string
|
||||
once sync.Once
|
||||
val string
|
||||
}
|
||||
|
||||
func (e *envOnce) Get() string {
|
||||
e.once.Do(e.init)
|
||||
return e.val
|
||||
}
|
||||
|
||||
func (e *envOnce) init() {
|
||||
for _, n := range e.names {
|
||||
e.val = os.Getenv(n)
|
||||
if e.val != "" {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reset is used by tests
|
||||
func (e *envOnce) reset() {
|
||||
e.once = sync.Once{}
|
||||
e.val = ""
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
|
||||
"golang.org/x/net/internal/socks"
|
||||
)
|
||||
|
||||
// SOCKS5 returns a Dialer that makes SOCKSv5 connections to the given
|
||||
// address with an optional username and password.
|
||||
// See RFC 1928 and RFC 1929.
|
||||
func SOCKS5(network, address string, auth *Auth, forward Dialer) (Dialer, error) {
|
||||
d := socks.NewDialer(network, address)
|
||||
if forward != nil {
|
||||
if f, ok := forward.(ContextDialer); ok {
|
||||
d.ProxyDial = func(ctx context.Context, network string, address string) (net.Conn, error) {
|
||||
return f.DialContext(ctx, network, address)
|
||||
}
|
||||
} else {
|
||||
d.ProxyDial = func(ctx context.Context, network string, address string) (net.Conn, error) {
|
||||
return dialContext(ctx, forward, network, address)
|
||||
}
|
||||
}
|
||||
}
|
||||
if auth != nil {
|
||||
up := socks.UsernamePassword{
|
||||
Username: auth.User,
|
||||
Password: auth.Password,
|
||||
}
|
||||
d.AuthMethods = []socks.AuthMethod{
|
||||
socks.AuthMethodNotRequired,
|
||||
socks.AuthMethodUsernamePassword,
|
||||
}
|
||||
d.Authenticate = up.Authenticate
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
Vendored
+2
@@ -67,6 +67,8 @@ github.com/skip2/go-qrcode/reedsolomon
|
||||
golang.org/x/crypto/curve25519
|
||||
# golang.org/x/net v0.27.0
|
||||
## explicit; go 1.18
|
||||
golang.org/x/net/internal/socks
|
||||
golang.org/x/net/proxy
|
||||
golang.org/x/net/publicsuffix
|
||||
# golang.org/x/sys v0.30.0
|
||||
## explicit; go 1.18
|
||||
|
||||
@@ -49,7 +49,7 @@ Common overrides:
|
||||
FRONTEND_PORT=9191 Public web port
|
||||
FRONTEND_HOST=0.0.0.0 Public web bind address
|
||||
GO_VERSION=1.23.12
|
||||
INSTALL_DEPS=0 Do not install missing Node/Go/ffmpeg
|
||||
INSTALL_DEPS=0 Do not install missing Node/Go/ffmpeg/Python runtime deps
|
||||
CONFIGURE_UFW=0 Do not open UFW port automatically
|
||||
DEPLOY_USER=<user> Service user; defaults to sudo user or root
|
||||
|
||||
@@ -130,7 +130,25 @@ apt_install() {
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
log "installing base packages"
|
||||
apt-get update
|
||||
apt-get install -y ca-certificates curl git ffmpeg openssl iproute2 build-essential
|
||||
apt-get install -y ca-certificates curl git ffmpeg openssl iproute2 build-essential \
|
||||
python3 python3-requests python3-bs4 python3-lxml python3-socks
|
||||
}
|
||||
|
||||
verify_spider91_python_deps() {
|
||||
command -v python3 >/dev/null 2>&1 || die "python3 is required for 91Spider"
|
||||
python3 - <<'PY' || die "missing Python modules for 91Spider: requests, bs4, lxml, socks"
|
||||
import importlib.util
|
||||
import sys
|
||||
|
||||
missing = [
|
||||
name
|
||||
for name in ("requests", "bs4", "lxml", "socks")
|
||||
if importlib.util.find_spec(name) is None
|
||||
]
|
||||
if missing:
|
||||
print("missing Python modules: " + ", ".join(missing), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
install_node() {
|
||||
@@ -182,6 +200,7 @@ install_dependencies() {
|
||||
install_go
|
||||
command -v ffmpeg >/dev/null 2>&1 || die "ffmpeg is required"
|
||||
command -v ffprobe >/dev/null 2>&1 || die "ffprobe is required"
|
||||
verify_spider91_python_deps
|
||||
}
|
||||
|
||||
ensure_ownership() {
|
||||
@@ -315,8 +334,8 @@ EOF
|
||||
}
|
||||
|
||||
open_firewall_port() {
|
||||
[[ "$CONFIGURE_UFW" == "1" ]] || return
|
||||
command -v ufw >/dev/null 2>&1 || return
|
||||
[[ "$CONFIGURE_UFW" == "1" ]] || return 0
|
||||
command -v ufw >/dev/null 2>&1 || return 0
|
||||
if ufw status 2>/dev/null | grep -qi "Status: active"; then
|
||||
log "UFW is active; allowing ${FRONTEND_PORT}/tcp"
|
||||
ufw allow "${FRONTEND_PORT}/tcp"
|
||||
@@ -359,7 +378,9 @@ install_or_update() {
|
||||
open_firewall_port
|
||||
restart_services
|
||||
show_status
|
||||
[[ "$mode" == "install" ]] && show_summary
|
||||
if [[ "$mode" == "install" ]]; then
|
||||
show_summary
|
||||
fi
|
||||
}
|
||||
|
||||
uninstall_services() {
|
||||
|
||||
+26
-5
@@ -122,6 +122,27 @@ asset_name() {
|
||||
printf '%s-linux-%s.tar.gz' "$APP_NAME" "$ARCH"
|
||||
}
|
||||
|
||||
verify_runtime_deps() {
|
||||
local cmd
|
||||
for cmd in curl tar ffmpeg ffprobe openssl python3; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || die "missing command: $cmd"
|
||||
done
|
||||
|
||||
python3 - <<'PY' || die "missing Python modules for 91Spider: requests, bs4, lxml, socks"
|
||||
import importlib.util
|
||||
import sys
|
||||
|
||||
missing = [
|
||||
name
|
||||
for name in ("requests", "bs4", "lxml", "socks")
|
||||
if importlib.util.find_spec(name) is None
|
||||
]
|
||||
if missing:
|
||||
print("missing Python modules: " + ", ".join(missing), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
install_deps() {
|
||||
if [[ "$INSTALL_DEPS" != "1" ]]; then
|
||||
return
|
||||
@@ -130,13 +151,12 @@ install_deps() {
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
log "installing runtime dependencies"
|
||||
apt-get update
|
||||
apt-get install -y ca-certificates curl tar ffmpeg openssl iproute2 python3 python3-requests python3-bs4 python3-lxml
|
||||
apt-get install -y ca-certificates curl tar ffmpeg openssl iproute2 python3 python3-requests python3-bs4 python3-lxml python3-socks
|
||||
verify_runtime_deps
|
||||
return
|
||||
fi
|
||||
|
||||
for cmd in curl tar ffmpeg ffprobe openssl; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || die "missing command: $cmd"
|
||||
done
|
||||
verify_runtime_deps
|
||||
}
|
||||
|
||||
check_system() {
|
||||
@@ -689,13 +709,14 @@ update_app() {
|
||||
exec_latest_manager_update
|
||||
fi
|
||||
|
||||
install_deps
|
||||
|
||||
if should_skip_update; then
|
||||
log "already up to date; skipped app update"
|
||||
return 0
|
||||
fi
|
||||
|
||||
check_disk_space
|
||||
install_deps
|
||||
|
||||
local backup
|
||||
backup="$(mktemp -d)"
|
||||
|
||||
Generated
+38
-2
@@ -1,13 +1,16 @@
|
||||
{
|
||||
"name": "video-site",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.5",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "video-site",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.5",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"artplayer": "^5.4.0",
|
||||
"hls.js": "^1.6.16",
|
||||
"lucide-react": "0.453.0",
|
||||
"react": "18.3.1",
|
||||
"react-dom": "18.3.1",
|
||||
@@ -474,6 +477,15 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/artplayer": {
|
||||
"version": "5.4.0",
|
||||
"resolved": "https://registry.npmjs.org/artplayer/-/artplayer-5.4.0.tgz",
|
||||
"integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"option-validator": "^2.0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/csstype": {
|
||||
"version": "3.2.3",
|
||||
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz",
|
||||
@@ -524,12 +536,27 @@
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/hls.js": {
|
||||
"version": "1.6.16",
|
||||
"resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.6.16.tgz",
|
||||
"integrity": "sha512-VSIRpLfRwlAAdGL4wiTucx2ScRipo0ed1FBatWkyt832jC4CReKstga6yIhYVwGu9LOBjuX9wzmRMeQdBJtzEA==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/js-tokens": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
|
||||
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/kind-of": {
|
||||
"version": "6.0.3",
|
||||
"resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
|
||||
"integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/lightningcss": {
|
||||
"version": "1.32.0",
|
||||
"resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz",
|
||||
@@ -831,6 +858,15 @@
|
||||
"node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/option-validator": {
|
||||
"version": "2.0.6",
|
||||
"resolved": "https://registry.npmjs.org/option-validator/-/option-validator-2.0.6.tgz",
|
||||
"integrity": "sha512-tmZDan2LRIRQyhUGvkff68/O0R8UmF+Btmiiz0SmSw2ng3CfPZB9wJlIjHpe/MKUZqyIZkVIXCrwr1tIN+0Dzg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"kind-of": "^6.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/picocolors": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
|
||||
|
||||
+4
-1
@@ -1,7 +1,8 @@
|
||||
{
|
||||
"name": "video-site",
|
||||
"private": true,
|
||||
"version": "0.1.0",
|
||||
"license": "MIT",
|
||||
"version": "0.1.5",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
@@ -12,6 +13,8 @@
|
||||
"test": "node --import tsx --test tests/*.test.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"artplayer": "^5.4.0",
|
||||
"hls.js": "^1.6.16",
|
||||
"lucide-react": "0.453.0",
|
||||
"react": "18.3.1",
|
||||
"react-dom": "18.3.1",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useState } from "react";
|
||||
import { useEffect, useState } from "react";
|
||||
import { NavLink, Outlet, useNavigate } from "react-router-dom";
|
||||
import {
|
||||
HardDrive,
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
Tags,
|
||||
Palette,
|
||||
RefreshCw,
|
||||
MoreVertical,
|
||||
} from "lucide-react";
|
||||
import * as api from "./api";
|
||||
import { useAuth } from "./AuthContext";
|
||||
@@ -19,6 +20,18 @@ export function AdminLayout() {
|
||||
const navigate = useNavigate();
|
||||
const { show } = useToast();
|
||||
const [checkingUpdate, setCheckingUpdate] = useState(false);
|
||||
const [mobileMenuOpen, setMobileMenuOpen] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!mobileMenuOpen) return;
|
||||
function onKeyDown(e: KeyboardEvent) {
|
||||
if (e.key === "Escape") {
|
||||
setMobileMenuOpen(false);
|
||||
}
|
||||
}
|
||||
document.addEventListener("keydown", onKeyDown);
|
||||
return () => document.removeEventListener("keydown", onKeyDown);
|
||||
}, [mobileMenuOpen]);
|
||||
|
||||
async function handleCheckUpdate() {
|
||||
if (checkingUpdate) return;
|
||||
@@ -114,7 +127,31 @@ export function AdminLayout() {
|
||||
退出登录
|
||||
</button>
|
||||
</div>
|
||||
<button
|
||||
className="admin-sidebar__mobile-menu"
|
||||
onClick={() => setMobileMenuOpen((v) => !v)}
|
||||
aria-label="更多操作"
|
||||
>
|
||||
<MoreVertical size={18} />
|
||||
</button>
|
||||
</aside>
|
||||
{mobileMenuOpen && (
|
||||
<div className="admin-sidebar__mobile-overlay" onClick={() => setMobileMenuOpen(false)} />
|
||||
)}
|
||||
<div className={`admin-sidebar__mobile-panel${mobileMenuOpen ? " is-open" : ""}`}>
|
||||
<button
|
||||
className="admin-sidebar__check-update"
|
||||
onClick={() => { handleCheckUpdate(); setMobileMenuOpen(false); }}
|
||||
disabled={checkingUpdate}
|
||||
>
|
||||
<RefreshCw size={14} />
|
||||
{checkingUpdate ? "检查中" : "检查更新"}
|
||||
</button>
|
||||
<button className="admin-sidebar__logout" onClick={() => { handleLogout(); setMobileMenuOpen(false); }}>
|
||||
<LogOut size={14} />
|
||||
退出登录
|
||||
</button>
|
||||
</div>
|
||||
<main className="admin-main">
|
||||
<Outlet />
|
||||
</main>
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import { AlertTriangle } from "lucide-react";
|
||||
import { Modal } from "./Modal";
|
||||
|
||||
type ConfirmModalProps = {
|
||||
open: boolean;
|
||||
title: string;
|
||||
message: string;
|
||||
details?: string[];
|
||||
confirmText?: string;
|
||||
cancelText?: string;
|
||||
danger?: boolean;
|
||||
centerMessage?: boolean;
|
||||
modalClassName?: string;
|
||||
loading?: boolean;
|
||||
onCancel: () => void;
|
||||
onConfirm: () => void;
|
||||
};
|
||||
|
||||
export function ConfirmModal({
|
||||
open,
|
||||
title,
|
||||
message,
|
||||
details,
|
||||
confirmText = "确认",
|
||||
cancelText = "取消",
|
||||
danger = false,
|
||||
centerMessage = false,
|
||||
modalClassName = "",
|
||||
loading = false,
|
||||
onCancel,
|
||||
onConfirm,
|
||||
}: ConfirmModalProps) {
|
||||
return (
|
||||
<Modal
|
||||
open={open}
|
||||
title={title}
|
||||
onClose={onCancel}
|
||||
className={modalClassName}
|
||||
footer={
|
||||
<>
|
||||
<button type="button" className="admin-btn" onClick={onCancel} disabled={loading}>
|
||||
{cancelText}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`admin-btn${danger ? " is-danger" : " is-primary"}`}
|
||||
onClick={onConfirm}
|
||||
disabled={loading}
|
||||
>
|
||||
{loading ? "处理中..." : confirmText}
|
||||
</button>
|
||||
</>
|
||||
}
|
||||
>
|
||||
<div className={`admin-confirm${centerMessage ? " is-message-centered" : ""}`}>
|
||||
<div className={`admin-confirm__icon${danger ? " is-danger" : ""}`} aria-hidden={centerMessage}>
|
||||
<AlertTriangle size={20} />
|
||||
</div>
|
||||
<div className="admin-confirm__content">
|
||||
<p className="admin-confirm__message">{message}</p>
|
||||
{details && details.length > 0 && (
|
||||
<ul className="admin-confirm__list">
|
||||
{details.map((item) => (
|
||||
<li key={item}>{item}</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
+461
-1101
File diff suppressed because it is too large
Load Diff
+16
-4
@@ -16,6 +16,7 @@ export function LoginPage() {
|
||||
const navigate = useNavigate();
|
||||
const location = useLocation();
|
||||
const { show } = useToast();
|
||||
const passwordMismatch = setupRequired === true && p2.length > 0 && p !== p2;
|
||||
|
||||
useEffect(() => {
|
||||
let active = true;
|
||||
@@ -83,8 +84,9 @@ export function LoginPage() {
|
||||
</h1>
|
||||
<div className="admin-form">
|
||||
<div className="admin-form__row">
|
||||
<label>用户名</label>
|
||||
<label htmlFor="admin-login-username">用户名</label>
|
||||
<input
|
||||
id="admin-login-username"
|
||||
autoFocus
|
||||
value={u}
|
||||
onChange={(e) => setU(e.target.value)}
|
||||
@@ -92,8 +94,9 @@ export function LoginPage() {
|
||||
/>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>密码</label>
|
||||
<label htmlFor="admin-login-password">密码</label>
|
||||
<input
|
||||
id="admin-login-password"
|
||||
type="password"
|
||||
value={p}
|
||||
onChange={(e) => setP(e.target.value)}
|
||||
@@ -102,19 +105,28 @@ export function LoginPage() {
|
||||
</div>
|
||||
{setupRequired && (
|
||||
<div className="admin-form__row">
|
||||
<label>确认密码</label>
|
||||
<label htmlFor="admin-login-password-confirm">确认密码</label>
|
||||
<input
|
||||
id="admin-login-password-confirm"
|
||||
type="password"
|
||||
value={p2}
|
||||
onChange={(e) => setP2(e.target.value)}
|
||||
autoComplete="new-password"
|
||||
className={passwordMismatch ? "is-invalid" : undefined}
|
||||
aria-invalid={passwordMismatch ? "true" : undefined}
|
||||
aria-describedby={passwordMismatch ? "admin-login-password-confirm-error" : undefined}
|
||||
/>
|
||||
{passwordMismatch && (
|
||||
<div className="admin-form__error" id="admin-login-password-confirm-error">
|
||||
密码不一致
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
<button
|
||||
className="admin-btn is-primary"
|
||||
type="submit"
|
||||
disabled={loading || !u || !p || (setupRequired && !p2)}
|
||||
disabled={loading || !u || !p || (setupRequired && (!p2 || passwordMismatch))}
|
||||
>
|
||||
{loading
|
||||
? setupRequired
|
||||
|
||||
+97
-4
@@ -1,4 +1,4 @@
|
||||
import { ReactNode } from "react";
|
||||
import { useEffect, useId, useRef, ReactNode } from "react";
|
||||
import { X } from "lucide-react";
|
||||
|
||||
type Props = {
|
||||
@@ -7,9 +7,72 @@ type Props = {
|
||||
onClose: () => void;
|
||||
children: ReactNode;
|
||||
footer?: ReactNode;
|
||||
className?: string;
|
||||
};
|
||||
|
||||
export function Modal({ open, title, onClose, children, footer }: Props) {
|
||||
export function Modal({ open, title, onClose, children, footer, className = "" }: Props) {
|
||||
const dialogRef = useRef<HTMLDivElement>(null);
|
||||
const onCloseRef = useRef(onClose);
|
||||
const titleId = useId();
|
||||
|
||||
useEffect(() => {
|
||||
onCloseRef.current = onClose;
|
||||
}, [onClose]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) return;
|
||||
const previousFocus =
|
||||
document.activeElement instanceof HTMLElement ? document.activeElement : null;
|
||||
|
||||
function onKeyDown(e: KeyboardEvent) {
|
||||
const dialog = dialogRef.current;
|
||||
if (!dialog || !isTopDialog(dialog)) return;
|
||||
|
||||
if (e.key === "Escape") {
|
||||
e.preventDefault();
|
||||
onCloseRef.current();
|
||||
return;
|
||||
}
|
||||
|
||||
if (e.key !== "Tab") return;
|
||||
|
||||
const focusable = getFocusableElements(dialog);
|
||||
if (focusable.length === 0) {
|
||||
e.preventDefault();
|
||||
dialog.focus();
|
||||
return;
|
||||
}
|
||||
|
||||
const first = focusable[0];
|
||||
const last = focusable[focusable.length - 1];
|
||||
const current = document.activeElement;
|
||||
|
||||
if (e.shiftKey && current === first) {
|
||||
e.preventDefault();
|
||||
last.focus();
|
||||
} else if (!e.shiftKey && current === last) {
|
||||
e.preventDefault();
|
||||
first.focus();
|
||||
}
|
||||
}
|
||||
|
||||
const focusTimer = window.setTimeout(() => {
|
||||
const dialog = dialogRef.current;
|
||||
if (!dialog || !isTopDialog(dialog)) return;
|
||||
const first = getFocusableElements(dialog)[0];
|
||||
(first ?? dialog).focus();
|
||||
}, 0);
|
||||
|
||||
document.addEventListener("keydown", onKeyDown);
|
||||
return () => {
|
||||
window.clearTimeout(focusTimer);
|
||||
document.removeEventListener("keydown", onKeyDown);
|
||||
if (previousFocus?.isConnected) {
|
||||
previousFocus.focus();
|
||||
}
|
||||
};
|
||||
}, [open]);
|
||||
|
||||
if (!open) return null;
|
||||
return (
|
||||
<div
|
||||
@@ -18,10 +81,18 @@ export function Modal({ open, title, onClose, children, footer }: Props) {
|
||||
if (e.target === e.currentTarget) onClose();
|
||||
}}
|
||||
>
|
||||
<div className="admin-modal" role="dialog" aria-modal="true">
|
||||
<div
|
||||
ref={dialogRef}
|
||||
className={`admin-modal${className ? ` ${className}` : ""}`}
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
aria-labelledby={titleId}
|
||||
tabIndex={-1}
|
||||
>
|
||||
<div className="admin-modal__header">
|
||||
<span>{title}</span>
|
||||
<span id={titleId}>{title}</span>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={onClose}
|
||||
aria-label="关闭"
|
||||
@@ -36,3 +107,25 @@ export function Modal({ open, title, onClose, children, footer }: Props) {
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function getFocusableElements(root: HTMLElement): HTMLElement[] {
|
||||
const selectors = [
|
||||
"a[href]",
|
||||
"button:not([disabled])",
|
||||
"textarea:not([disabled])",
|
||||
"input:not([disabled])",
|
||||
"select:not([disabled])",
|
||||
"[tabindex]:not([tabindex='-1'])",
|
||||
].join(",");
|
||||
|
||||
return Array.from(root.querySelectorAll<HTMLElement>(selectors)).filter(
|
||||
(el) => !el.hasAttribute("disabled") && el.getAttribute("aria-hidden") !== "true"
|
||||
);
|
||||
}
|
||||
|
||||
function isTopDialog(dialog: HTMLElement): boolean {
|
||||
const dialogs = Array.from(
|
||||
document.querySelectorAll<HTMLElement>('[role="dialog"][aria-modal="true"]')
|
||||
);
|
||||
return dialogs[dialogs.length - 1] === dialog;
|
||||
}
|
||||
|
||||
@@ -8,11 +8,7 @@ export function RequireAuth({ children }: { children: ReactNode }) {
|
||||
const location = useLocation();
|
||||
|
||||
if (status === "loading") {
|
||||
return (
|
||||
<div className="admin-loading-screen">
|
||||
加载中...
|
||||
</div>
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (status === "guest") {
|
||||
|
||||
+103
-47
@@ -2,18 +2,26 @@ import { useEffect, useMemo, useState } from "react";
|
||||
import { CheckSquare, Film, Plus, RefreshCw, Search, Tags, Trash2 } from "lucide-react";
|
||||
import * as api from "./api";
|
||||
import { useToast } from "./ToastContext";
|
||||
import { ConfirmModal } from "./ConfirmModal";
|
||||
|
||||
const DESKTOP_TAGS_PAGE_SIZE = 25;
|
||||
const MOBILE_TAGS_PAGE_SIZE = 8;
|
||||
const TAGS_MOBILE_QUERY = "(max-width: 640px)";
|
||||
|
||||
type DeleteConfirmState =
|
||||
| { kind: "single"; tag: api.AdminTag }
|
||||
| { kind: "bulk"; ids: number[] }
|
||||
| null;
|
||||
|
||||
export function TagsPage() {
|
||||
const [tags, setTags] = useState<api.AdminTag[]>([]);
|
||||
const [label, setLabel] = useState("");
|
||||
const [aliases, setAliases] = useState("");
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [loadError, setLoadError] = useState("");
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [deletingId, setDeletingId] = useState<number | null>(null);
|
||||
const [deleteConfirm, setDeleteConfirm] = useState<DeleteConfirmState>(null);
|
||||
const [searchQuery, setSearchQuery] = useState("");
|
||||
const [filterSource, setFilterSource] = useState<string>("all");
|
||||
const [selectMode, setSelectMode] = useState(false);
|
||||
@@ -25,10 +33,13 @@ export function TagsPage() {
|
||||
|
||||
async function refresh() {
|
||||
setLoading(true);
|
||||
setLoadError("");
|
||||
try {
|
||||
setTags(await api.listTags());
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "加载标签失败", "error");
|
||||
const message = e instanceof Error ? e.message : "加载标签失败";
|
||||
setLoadError(message);
|
||||
show(message, "error");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
@@ -55,21 +66,9 @@ export function TagsPage() {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleDelete(tag: api.AdminTag) {
|
||||
function handleDelete(tag: api.AdminTag) {
|
||||
if (tag.source === "system") return;
|
||||
if (!window.confirm(`确定删除标签「${tag.label}」吗?此操作会从所有视频上移除该标签。`)) {
|
||||
return;
|
||||
}
|
||||
setDeletingId(tag.id);
|
||||
try {
|
||||
const r = await api.deleteTag(tag.id);
|
||||
show(`已删除标签,并从 ${r.removedVideos} 个视频移除`, "success");
|
||||
await refresh();
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "删除标签失败", "error");
|
||||
} finally {
|
||||
setDeletingId(null);
|
||||
}
|
||||
setDeleteConfirm({ kind: "single", tag });
|
||||
}
|
||||
|
||||
function toggleSelectMode() {
|
||||
@@ -88,24 +87,40 @@ export function TagsPage() {
|
||||
async function handleBulkDelete() {
|
||||
const ids = [...selected];
|
||||
if (ids.length === 0) return;
|
||||
if (!window.confirm(`确定删除选中的 ${ids.length} 个标签吗?此操作会从所有视频上移除这些标签。`)) {
|
||||
setDeleteConfirm({ kind: "bulk", ids });
|
||||
}
|
||||
|
||||
async function confirmDelete() {
|
||||
if (!deleteConfirm) return;
|
||||
|
||||
if (deleteConfirm.kind === "single") {
|
||||
const tag = deleteConfirm.tag;
|
||||
setDeletingId(tag.id);
|
||||
try {
|
||||
const r = await api.deleteTag(tag.id);
|
||||
show(`已删除标签,并从 ${r.removedVideos} 个视频移除`, "success");
|
||||
setDeleteConfirm(null);
|
||||
await refresh();
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "删除标签失败", "error");
|
||||
} finally {
|
||||
setDeletingId(null);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const ids = deleteConfirm.ids;
|
||||
setBulkDeleting(true);
|
||||
try {
|
||||
let ok = 0;
|
||||
for (const id of ids) {
|
||||
try {
|
||||
await api.deleteTag(id);
|
||||
ok += 1;
|
||||
} catch {
|
||||
/* 统计失败数,继续删除其余标签 */
|
||||
}
|
||||
}
|
||||
const results = await Promise.allSettled(
|
||||
ids.map((id) => api.deleteTag(id))
|
||||
);
|
||||
const ok = results.filter((r) => r.status === "fulfilled").length;
|
||||
const failed = ids.length - ok;
|
||||
show(failed ? `已删除 ${ok} 个,${failed} 个失败` : `已删除 ${ok} 个标签`, failed ? "error" : "success");
|
||||
setSelected(new Set());
|
||||
setSelectMode(false);
|
||||
setDeleteConfirm(null);
|
||||
await refresh();
|
||||
} finally {
|
||||
setBulkDeleting(false);
|
||||
@@ -188,7 +203,7 @@ export function TagsPage() {
|
||||
<section>
|
||||
<header className="admin-page__header">
|
||||
<h1 className="admin-page__title">标签管理</h1>
|
||||
<button className="admin-btn" onClick={refresh}>
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 刷新
|
||||
</button>
|
||||
</header>
|
||||
@@ -200,34 +215,39 @@ export function TagsPage() {
|
||||
<div className="admin-card__title">
|
||||
<Plus size={15} /> 新增分类标签
|
||||
</div>
|
||||
<div className="admin-form">
|
||||
<form
|
||||
className="admin-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
handleCreate();
|
||||
}}
|
||||
>
|
||||
<div className="admin-form__row">
|
||||
<label>标签名</label>
|
||||
<label htmlFor="admin-tag-label">标签名</label>
|
||||
<input
|
||||
id="admin-tag-label"
|
||||
value={label}
|
||||
onChange={(e) => setLabel(e.target.value)}
|
||||
placeholder="例如:清纯"
|
||||
/>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>别名</label>
|
||||
<label htmlFor="admin-tag-aliases">别名</label>
|
||||
<input
|
||||
id="admin-tag-aliases"
|
||||
value={aliases}
|
||||
onChange={(e) => setAliases(e.target.value)}
|
||||
placeholder="逗号分隔,例如:纯欲, 清新"
|
||||
/>
|
||||
<div className="admin-form__help">
|
||||
新增后会按别名和标签名匹配已有视频的标题、作者和目录并自动归类。
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
type="submit"
|
||||
className="admin-btn is-primary"
|
||||
onClick={handleCreate}
|
||||
disabled={saving || !label.trim()}
|
||||
>
|
||||
<Plus size={13} /> {saving ? "添加中..." : "添加并自动归类"}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div className="admin-card">
|
||||
@@ -253,6 +273,7 @@ export function TagsPage() {
|
||||
<div className="admin-tags-search">
|
||||
<Search className="admin-tags-search__icon" size={14} />
|
||||
<input
|
||||
aria-label="搜索标签名或别名"
|
||||
type="text"
|
||||
value={searchQuery}
|
||||
onChange={(e) => setSearchQuery(e.target.value)}
|
||||
@@ -336,7 +357,18 @@ export function TagsPage() {
|
||||
)}
|
||||
|
||||
{loading ? (
|
||||
<div className="admin-empty">加载中...</div>
|
||||
<div className="admin-loading-state">
|
||||
<RefreshCw size={20} className="admin-spin" />
|
||||
<span>加载中...</span>
|
||||
</div>
|
||||
) : loadError ? (
|
||||
<div className="admin-error-state">
|
||||
<strong>标签加载失败</strong>
|
||||
<span>{loadError}</span>
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 重试
|
||||
</button>
|
||||
</div>
|
||||
) : filteredTags.length === 0 ? (
|
||||
<div className="admin-card admin-empty">
|
||||
没有找到匹配的标签。
|
||||
@@ -347,21 +379,18 @@ export function TagsPage() {
|
||||
{pagedTags.map((tag) => {
|
||||
const selectable = selectMode && tag.source !== "system";
|
||||
const isSelected = selected.has(tag.id);
|
||||
return (
|
||||
<div
|
||||
key={tag.id}
|
||||
className={`admin-tag-card${selectable ? " is-selectable" : ""}${
|
||||
selectable && isSelected ? " is-selected" : ""
|
||||
}`}
|
||||
onClick={selectable ? () => toggleSelect(tag.id) : undefined}
|
||||
>
|
||||
const cardClass = `admin-tag-card${selectable ? " is-selectable" : ""}${
|
||||
selectable && isSelected ? " is-selected" : ""
|
||||
}`;
|
||||
const cardContent = (
|
||||
<>
|
||||
<div className="admin-tag-card__head">
|
||||
{selectable && (
|
||||
<input
|
||||
type="checkbox"
|
||||
className="admin-tag-card__check"
|
||||
checked={isSelected}
|
||||
readOnly
|
||||
onChange={() => toggleSelect(tag.id)}
|
||||
/>
|
||||
)}
|
||||
<span className="admin-tag-card__title">{tag.label}</span>
|
||||
@@ -380,7 +409,7 @@ export function TagsPage() {
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="admin-tag-card__footer">
|
||||
<div className="admin-tag-card__footer">
|
||||
<span className="admin-tag-card__count">
|
||||
<Film size={13} />
|
||||
<strong>{tag.count}</strong> 视频
|
||||
@@ -399,8 +428,17 @@ export function TagsPage() {
|
||||
<span>{deletingId === tag.id ? "删除中" : "删除"}</span>
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
return selectable ? (
|
||||
<label key={tag.id} className={cardClass}>
|
||||
{cardContent}
|
||||
</label>
|
||||
) : (
|
||||
<div key={tag.id} className={cardClass}>
|
||||
{cardContent}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
@@ -449,6 +487,24 @@ export function TagsPage() {
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<ConfirmModal
|
||||
open={!!deleteConfirm}
|
||||
title={deleteConfirm?.kind === "bulk" ? "删除选中标签" : "删除标签"}
|
||||
message={
|
||||
deleteConfirm?.kind === "bulk"
|
||||
? `确定要删除选中的 ${deleteConfirm.ids.length} 个标签吗?`
|
||||
: `确定要删除标签「${deleteConfirm?.tag.label ?? ""}」吗?`
|
||||
}
|
||||
confirmText="确认删除"
|
||||
danger
|
||||
centerMessage
|
||||
modalClassName="admin-modal--delete-confirm"
|
||||
loading={deletingId !== null || bulkDeleting}
|
||||
onCancel={() => {
|
||||
if (deletingId === null && !bulkDeleting) setDeleteConfirm(null);
|
||||
}}
|
||||
onConfirm={confirmDelete}
|
||||
/>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -103,9 +103,6 @@ export function ThemePage() {
|
||||
<div className="theme-page">
|
||||
<header className="theme-page__head">
|
||||
<h1 className="theme-page__title">主题外观</h1>
|
||||
<p className="theme-page__sub">
|
||||
切换全站主题。所有访客都会看到这里选定的主题,本设置会写入数据库永久保存。
|
||||
</p>
|
||||
</header>
|
||||
|
||||
<div className="theme-grid">
|
||||
|
||||
@@ -43,11 +43,13 @@ export function ToastProvider({ children }: { children: ReactNode }) {
|
||||
return (
|
||||
<ToastCtx.Provider value={{ show }}>
|
||||
{children}
|
||||
{items.map((t) => (
|
||||
<div key={t.id} className={`admin-toast is-${t.kind}`}>
|
||||
{t.text}
|
||||
</div>
|
||||
))}
|
||||
<div className="admin-toast-stack" role="status" aria-live="polite">
|
||||
{items.map((t) => (
|
||||
<div key={t.id} className={`admin-toast is-${t.kind}`}>
|
||||
{t.text}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</ToastCtx.Provider>
|
||||
);
|
||||
}
|
||||
|
||||
+395
-139
@@ -1,28 +1,43 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Edit, RefreshCw, Search } from "lucide-react";
|
||||
import { useEffect, useId, useState } from "react";
|
||||
import { ChevronDown, Edit, RefreshCw, Search, CheckSquare, Square, Image, Trash2 } from "lucide-react";
|
||||
import * as api from "./api";
|
||||
import { useToast } from "./ToastContext";
|
||||
import { Modal } from "./Modal";
|
||||
import { ConfirmModal } from "./ConfirmModal";
|
||||
import { formatBytes } from "./storageFormat";
|
||||
|
||||
const PAGE_SIZE = 100;
|
||||
const DESKTOP_VIDEOS_PAGE_SIZE = 50;
|
||||
const MOBILE_VIDEOS_PAGE_SIZE = 20;
|
||||
const VIDEOS_MOBILE_QUERY = "(max-width: 640px)";
|
||||
|
||||
export function VideosPage() {
|
||||
const [list, setList] = useState<api.AdminVideo[]>([]);
|
||||
const [drives, setDrives] = useState<api.AdminDrive[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [loadError, setLoadError] = useState("");
|
||||
const [keyword, setKeyword] = useState("");
|
||||
const [searchKeyword, setSearchKeyword] = useState("");
|
||||
const [driveId, setDriveId] = useState("");
|
||||
const [page, setPage] = useState(1);
|
||||
const [total, setTotal] = useState(0);
|
||||
const [editing, setEditing] = useState<api.AdminVideo | null>(null);
|
||||
const [availableTags, setAvailableTags] = useState<api.AdminTag[]>([]);
|
||||
const [selectedIds, setSelectedIds] = useState<Set<string>>(new Set());
|
||||
const [batchRegenOpen, setBatchRegenOpen] = useState(false);
|
||||
const [batchRegening, setBatchRegening] = useState(false);
|
||||
const [batchDeleteOpen, setBatchDeleteOpen] = useState(false);
|
||||
const [batchDeleting, setBatchDeleting] = useState(false);
|
||||
const [deleteTarget, setDeleteTarget] = useState<api.AdminVideo | null>(null);
|
||||
const [deleting, setDeleting] = useState(false);
|
||||
const pageSize = useVideosPageSize();
|
||||
const { show } = useToast();
|
||||
|
||||
async function refresh() {
|
||||
setLoading(true);
|
||||
setLoadError("");
|
||||
try {
|
||||
const [r, tagList, driveList] = await Promise.all([
|
||||
api.listVideos({ driveId, page, size: PAGE_SIZE }),
|
||||
api.listVideos({ driveId, page, size: pageSize, keyword: searchKeyword }),
|
||||
api.listTags(),
|
||||
api.listDrives(),
|
||||
]);
|
||||
@@ -30,8 +45,11 @@ export function VideosPage() {
|
||||
setTotal(r.total ?? 0);
|
||||
setAvailableTags(tagList);
|
||||
setDrives(driveList ?? []);
|
||||
setSelectedIds(new Set());
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "加载失败", "error");
|
||||
const message = e instanceof Error ? e.message : "加载失败";
|
||||
setLoadError(message);
|
||||
show(message, "error");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
@@ -39,164 +57,314 @@ export function VideosPage() {
|
||||
|
||||
useEffect(() => {
|
||||
refresh();
|
||||
}, [driveId, page]);
|
||||
}, [driveId, page, searchKeyword, pageSize]);
|
||||
|
||||
useEffect(() => {
|
||||
setPage(1);
|
||||
}, [pageSize]);
|
||||
|
||||
useEffect(() => {
|
||||
if (keyword === searchKeyword) return;
|
||||
const timer = window.setTimeout(() => {
|
||||
setSearchKeyword(keyword);
|
||||
setPage(1);
|
||||
}, 300);
|
||||
return () => window.clearTimeout(timer);
|
||||
}, [keyword]);
|
||||
|
||||
const driveNameMap = new Map(
|
||||
drives.map((d) => [d.id, d.name || d.id])
|
||||
);
|
||||
|
||||
const filtered = keyword.trim()
|
||||
? list.filter((v) => {
|
||||
const k = keyword.toLowerCase();
|
||||
return (
|
||||
v.title.toLowerCase().includes(k) ||
|
||||
(v.author ?? "").toLowerCase().includes(k)
|
||||
);
|
||||
})
|
||||
: list;
|
||||
const totalPages = Math.max(1, Math.ceil(total / PAGE_SIZE));
|
||||
const pageStart = total === 0 ? 0 : (page - 1) * PAGE_SIZE + 1;
|
||||
const pageEnd = Math.min(total, page * PAGE_SIZE);
|
||||
const listItems = list;
|
||||
const totalPages = Math.max(1, Math.ceil(total / pageSize));
|
||||
const pageStart = total === 0 ? 0 : (page - 1) * pageSize + 1;
|
||||
const pageEnd = Math.min(total, page * pageSize);
|
||||
const listSummary = driveId
|
||||
? `${driveNameMap.get(driveId) ?? driveId}:共 ${total} 个视频,第 ${page} / ${totalPages} 页,显示 ${pageStart}-${pageEnd}`
|
||||
: `全部网盘:共 ${total} 个视频,第 ${page} / ${totalPages} 页,显示 ${pageStart}-${pageEnd}`;
|
||||
|
||||
async function handleRegen(v: api.AdminVideo) {
|
||||
try {
|
||||
await api.regenPreview(v.id);
|
||||
show("已触发 teaser 重生", "success");
|
||||
show("已触发预览视频重生", "success");
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "触发失败", "error");
|
||||
}
|
||||
}
|
||||
|
||||
async function handleBatchRegen() {
|
||||
if (selectedIds.size === 0) return;
|
||||
setBatchRegenOpen(true);
|
||||
}
|
||||
|
||||
async function handleBatchDelete() {
|
||||
if (selectedIds.size === 0) return;
|
||||
setBatchDeleteOpen(true);
|
||||
}
|
||||
|
||||
async function confirmBatchRegen() {
|
||||
const ids = [...selectedIds];
|
||||
setBatchRegening(true);
|
||||
let success = 0;
|
||||
try {
|
||||
const results = await Promise.allSettled(
|
||||
ids.map((id) => api.regenPreview(id))
|
||||
);
|
||||
for (const r of results) {
|
||||
if (r.status === "fulfilled") success++;
|
||||
}
|
||||
show(`批量触发完成,成功 ${success} / ${ids.length} 个`, success === ids.length ? "success" : "info");
|
||||
setSelectedIds(new Set());
|
||||
setBatchRegenOpen(false);
|
||||
} finally {
|
||||
setBatchRegening(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function confirmDeleteVideo() {
|
||||
if (!deleteTarget) return;
|
||||
const target = deleteTarget;
|
||||
setDeleting(true);
|
||||
try {
|
||||
const result = await api.deleteVideo(target.id);
|
||||
setDeleteTarget(null);
|
||||
setSelectedIds((ids) => {
|
||||
const next = new Set(ids);
|
||||
next.delete(target.id);
|
||||
return next;
|
||||
});
|
||||
show(result.deletedSource ? "已删除视频,并清理 91Spider 源文件" : "已删除视频", "success");
|
||||
if (listItems.length === 1 && page > 1) {
|
||||
setPage((p) => Math.max(1, p - 1));
|
||||
} else {
|
||||
refresh();
|
||||
}
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "删除失败", "error");
|
||||
} finally {
|
||||
setDeleting(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function confirmBatchDelete() {
|
||||
const ids = [...selectedIds];
|
||||
if (ids.length === 0) return;
|
||||
setBatchDeleting(true);
|
||||
try {
|
||||
let success = 0;
|
||||
let deletedSources = 0;
|
||||
for (const id of ids) {
|
||||
try {
|
||||
const result = await api.deleteVideo(id);
|
||||
success++;
|
||||
if (result.deletedSource) deletedSources++;
|
||||
} catch {
|
||||
// Keep deleting the rest of the selected videos; report aggregate failure below.
|
||||
}
|
||||
}
|
||||
const failed = ids.length - success;
|
||||
if (failed === 0) {
|
||||
const extra = deletedSources > 0 ? `,其中 ${deletedSources} 个清理了 91Spider 源文件` : "";
|
||||
show(`批量删除完成,成功 ${success} 个${extra}`, "success");
|
||||
} else {
|
||||
show(`批量删除完成,成功 ${success} / ${ids.length} 个,失败 ${failed} 个`, success > 0 ? "info" : "error");
|
||||
}
|
||||
setSelectedIds(new Set());
|
||||
setBatchDeleteOpen(false);
|
||||
if (success >= listItems.length && page > 1) {
|
||||
setPage((p) => Math.max(1, p - 1));
|
||||
} else {
|
||||
refresh();
|
||||
}
|
||||
} finally {
|
||||
setBatchDeleting(false);
|
||||
}
|
||||
}
|
||||
|
||||
const toggleSelectAll = () => {
|
||||
if (selectedIds.size === listItems.length && listItems.length > 0) {
|
||||
setSelectedIds(new Set());
|
||||
} else {
|
||||
setSelectedIds(new Set(listItems.map(v => v.id)));
|
||||
}
|
||||
};
|
||||
|
||||
const toggleSelect = (id: string) => {
|
||||
const next = new Set(selectedIds);
|
||||
if (next.has(id)) next.delete(id);
|
||||
else next.add(id);
|
||||
setSelectedIds(next);
|
||||
};
|
||||
|
||||
function handleSearchSubmit(e: React.FormEvent) {
|
||||
e.preventDefault();
|
||||
setSearchKeyword(keyword);
|
||||
setPage(1);
|
||||
}
|
||||
|
||||
return (
|
||||
<section>
|
||||
<header className="admin-page__header">
|
||||
<h1 className="admin-page__title">视频管理</h1>
|
||||
<div className="admin-page__actions admin-videos-filter">
|
||||
<select
|
||||
className="admin-videos-filter__select"
|
||||
value={driveId}
|
||||
onChange={(e) => {
|
||||
setDriveId(e.target.value);
|
||||
setPage(1);
|
||||
}}
|
||||
>
|
||||
<option value="">全部网盘</option>
|
||||
{drives.map((d) => (
|
||||
<option key={d.id} value={d.id}>
|
||||
{d.name || d.id}(已生成 {d.teaserReadyCount ?? 0},待生成{" "}
|
||||
{d.teaserPendingCount ?? 0})
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<div className="admin-videos-filter__search">
|
||||
<div className="admin-videos-filter__select-wrap">
|
||||
<select
|
||||
className="admin-videos-filter__select"
|
||||
value={driveId}
|
||||
onChange={(e) => {
|
||||
setDriveId(e.target.value);
|
||||
setPage(1);
|
||||
}}
|
||||
>
|
||||
<option value="">全部网盘</option>
|
||||
{drives.map((d) => (
|
||||
<option key={d.id} value={d.id}>
|
||||
{d.name || d.id}(已生成 {d.teaserReadyCount ?? 0},待生成{" "}
|
||||
{d.teaserPendingCount ?? 0})
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<ChevronDown size={15} className="admin-videos-filter__select-icon" aria-hidden="true" />
|
||||
</div>
|
||||
<form className="admin-videos-filter__search" onSubmit={handleSearchSubmit}>
|
||||
<Search size={14} className="admin-videos-filter__search-icon" />
|
||||
<input
|
||||
aria-label="搜索标题或作者"
|
||||
value={keyword}
|
||||
onChange={(e) => setKeyword(e.target.value)}
|
||||
placeholder="搜索标题 / 作者"
|
||||
/>
|
||||
</div>
|
||||
<button className="admin-btn" onClick={refresh}>
|
||||
</form>
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 刷新
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{drives.length > 0 && (
|
||||
<div className="admin-drive-teasers" aria-label="网盘 Teaser 统计">
|
||||
{drives.map((d) => (
|
||||
<button
|
||||
key={d.id}
|
||||
type="button"
|
||||
className={`admin-drive-teaser${
|
||||
driveId === d.id ? " is-active" : ""
|
||||
}`}
|
||||
onClick={() => {
|
||||
setDriveId(d.id);
|
||||
setPage(1);
|
||||
}}
|
||||
>
|
||||
<span className="admin-drive-teaser__name">{d.name || d.id}</span>
|
||||
<span className="admin-drive-teaser__metric is-ready">
|
||||
已生成 {d.teaserReadyCount ?? 0}
|
||||
</span>
|
||||
<span className="admin-drive-teaser__metric is-pending">
|
||||
待生成 {d.teaserPendingCount ?? 0}
|
||||
</span>
|
||||
{(d.teaserFailedCount ?? 0) > 0 && (
|
||||
<span className="admin-drive-teaser__metric is-failed">
|
||||
失败 {d.teaserFailedCount}
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!loading && (
|
||||
<div className="admin-videos-summary">
|
||||
{driveId
|
||||
? `${driveNameMap.get(driveId) ?? driveId}:共 ${total} 个视频,第 ${page} / ${totalPages} 页,显示 ${pageStart}-${pageEnd}`
|
||||
: `全部网盘:共 ${total} 个视频,第 ${page} / ${totalPages} 页,显示 ${pageStart}-${pageEnd}`}
|
||||
<div className="admin-videos-list-toolbar">
|
||||
<div className="admin-videos-summary">{listSummary}</div>
|
||||
{selectedIds.size > 0 && (
|
||||
<div className="admin-videos-bulk-actions">
|
||||
<span className="admin-videos-bulk-actions__count">
|
||||
已选择 {selectedIds.size} 项
|
||||
</span>
|
||||
<button type="button" className="admin-btn is-primary admin-videos-bulk-actions__btn" onClick={handleBatchRegen}>
|
||||
<RefreshCw size={13} /> 批量重生预览视频
|
||||
</button>
|
||||
<button type="button" className="admin-btn is-danger admin-videos-bulk-actions__btn" onClick={handleBatchDelete}>
|
||||
<Trash2 size={13} /> 批量删除
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{loading ? (
|
||||
<div className="admin-empty">加载中...</div>
|
||||
) : filtered.length === 0 ? (
|
||||
<div className="admin-card admin-empty">
|
||||
{driveId
|
||||
? "这个网盘下还没有可显示的视频。可以在「网盘管理」里触发重扫。"
|
||||
: "还没有视频。先在「网盘管理」里配置好盘并触发扫描。"}
|
||||
<div className="admin-loading-state">
|
||||
<RefreshCw size={20} className="admin-spin" />
|
||||
<span>加载中...</span>
|
||||
</div>
|
||||
) : loadError ? (
|
||||
<div className="admin-error-state">
|
||||
<strong>视频加载失败</strong>
|
||||
<span>{loadError}</span>
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 重试
|
||||
</button>
|
||||
</div>
|
||||
) : listItems.length === 0 ? (
|
||||
<div className="admin-empty-state">
|
||||
<div className="admin-empty-state__icon">
|
||||
<Image size={48} />
|
||||
</div>
|
||||
<div className="admin-empty-state__text">
|
||||
{driveId
|
||||
? "这个网盘下还没有可显示的视频,或未匹配到搜索结果。"
|
||||
: "还没有视频。先在「网盘管理」里配置好盘并触发扫描,或调整搜索词。"}
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<table className="admin-table">
|
||||
<table className="admin-table is-selectable admin-videos-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th className="is-checkbox" style={{ width: '40px' }}>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-table-checkbox-btn"
|
||||
onClick={toggleSelectAll}
|
||||
aria-label={selectedIds.size > 0 && selectedIds.size === listItems.length ? "清空当前页选择" : "选择当前页视频"}
|
||||
>
|
||||
{selectedIds.size > 0 && selectedIds.size === listItems.length ? <CheckSquare size={16} /> : <Square size={16} />}
|
||||
</button>
|
||||
</th>
|
||||
<th>标题</th>
|
||||
<th>作者</th>
|
||||
<th>标签</th>
|
||||
<th>时长</th>
|
||||
<th>Teaser</th>
|
||||
<th>预览视频</th>
|
||||
<th>来源</th>
|
||||
<th className="is-actions">操作</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{filtered.map((v) => (
|
||||
<tr key={v.id}>
|
||||
<td data-label="标题">
|
||||
<div className="admin-video-title">{v.title}</div>
|
||||
{fileMeta(v) && (
|
||||
<div className="admin-video-filemeta">
|
||||
{fileMeta(v)}
|
||||
</div>
|
||||
)}
|
||||
{listItems.map((v) => (
|
||||
<tr key={v.id} className={selectedIds.has(v.id) ? "is-selected" : ""}>
|
||||
<td className="is-checkbox">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-table-checkbox-btn"
|
||||
onClick={() => toggleSelect(v.id)}
|
||||
aria-label={`${selectedIds.has(v.id) ? "取消选择" : "选择"}视频 ${v.title}`}
|
||||
>
|
||||
{selectedIds.has(v.id) ? <CheckSquare size={16} color="var(--accent)" /> : <Square size={16} color="var(--border-strong)" />}
|
||||
</button>
|
||||
</td>
|
||||
<td data-label="作者">{v.author || <span className="admin-text-faint">—</span>}</td>
|
||||
<td data-label="标签">
|
||||
<div className="admin-pills">
|
||||
{(v.tags ?? []).map((t) => (
|
||||
<span key={t} className="admin-pill">
|
||||
{t}
|
||||
</span>
|
||||
))}
|
||||
<td data-label="标题">
|
||||
<div className="admin-video-title-cell">
|
||||
<div className="admin-video-thumb-wrap" aria-hidden="true">
|
||||
{v.thumbnailUrl ? (
|
||||
<img className="admin-video-thumb" src={v.thumbnailUrl} alt="" />
|
||||
) : (
|
||||
<div className="admin-video-thumb-placeholder">
|
||||
<Image size={14} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="admin-video-title-body">
|
||||
<div className="admin-video-title">{v.title}</div>
|
||||
{fileMeta(v) && (
|
||||
<div className="admin-video-filemeta">{fileMeta(v)}</div>
|
||||
)}
|
||||
{(v.tags ?? []).length > 0 && (
|
||||
<div className="admin-pills admin-video-title-tags">
|
||||
{(v.tags ?? []).map((t) => (
|
||||
<span key={t} className="admin-pill">{t}</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<VideoFileMetaPills video={v} />
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
<td data-label="作者">{v.author || <span className="admin-text-faint">—</span>}</td>
|
||||
<td data-label="时长">{formatDur(v.durationSeconds)}</td>
|
||||
<td data-label="Teaser">
|
||||
<td data-label="预览视频">
|
||||
<PreviewStatus s={v.previewStatus} />
|
||||
</td>
|
||||
<td data-label="来源" className="admin-mono-cell">
|
||||
{driveNameMap.get(v.driveId) ?? v.driveId}
|
||||
</td>
|
||||
<td className="is-actions" data-label="操作">
|
||||
<button className="admin-btn" onClick={() => setEditing(v)}>
|
||||
<Edit size={13} /> 编辑
|
||||
<button type="button" className="admin-btn" onClick={() => setEditing(v)} title="编辑视频">
|
||||
<Edit size={13} />
|
||||
</button>{" "}
|
||||
<button className="admin-btn" onClick={() => handleRegen(v)}>
|
||||
<RefreshCw size={13} /> 重生 teaser
|
||||
<button type="button" className="admin-btn" onClick={() => handleRegen(v)} title="重生预览视频">
|
||||
<RefreshCw size={13} />
|
||||
</button>{" "}
|
||||
<button type="button" className="admin-btn is-danger" onClick={() => setDeleteTarget(v)} title="删除视频">
|
||||
<Trash2 size={13} />
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
@@ -205,6 +373,7 @@ export function VideosPage() {
|
||||
</table>
|
||||
<div className="admin-table-pagination">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage(1)}
|
||||
disabled={page <= 1}
|
||||
@@ -212,6 +381,7 @@ export function VideosPage() {
|
||||
首页
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage((p) => Math.max(1, p - 1))}
|
||||
disabled={page <= 1}
|
||||
@@ -219,9 +389,10 @@ export function VideosPage() {
|
||||
上一页
|
||||
</button>
|
||||
<span className="admin-table-pagination__info">
|
||||
第 {page} / {totalPages} 页,每页 {PAGE_SIZE} 个
|
||||
第 {page} / {totalPages} 页,每页 {pageSize} 个
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage((p) => Math.min(totalPages, p + 1))}
|
||||
disabled={page >= totalPages}
|
||||
@@ -229,6 +400,7 @@ export function VideosPage() {
|
||||
下一页
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage(totalPages)}
|
||||
disabled={page >= totalPages}
|
||||
@@ -250,6 +422,45 @@ export function VideosPage() {
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
<ConfirmModal
|
||||
open={batchRegenOpen}
|
||||
title="批量重生预览视频"
|
||||
message={`确定要为当前页选中的 ${selectedIds.size} 个视频重新生成预览视频吗?`}
|
||||
confirmText="确认重生"
|
||||
loading={batchRegening}
|
||||
onCancel={() => {
|
||||
if (!batchRegening) setBatchRegenOpen(false);
|
||||
}}
|
||||
onConfirm={confirmBatchRegen}
|
||||
/>
|
||||
<ConfirmModal
|
||||
open={deleteTarget !== null}
|
||||
title="删除视频"
|
||||
message={deleteTarget ? `确定要删除「${deleteTarget.title}」吗?` : ""}
|
||||
confirmText="删除视频"
|
||||
danger
|
||||
centerMessage
|
||||
modalClassName="admin-modal--delete-confirm"
|
||||
loading={deleting}
|
||||
onCancel={() => {
|
||||
if (!deleting) setDeleteTarget(null);
|
||||
}}
|
||||
onConfirm={confirmDeleteVideo}
|
||||
/>
|
||||
<ConfirmModal
|
||||
open={batchDeleteOpen}
|
||||
title="批量删除视频"
|
||||
message={`确定要删除当前页选中的 ${selectedIds.size} 个视频吗?`}
|
||||
confirmText="批量删除"
|
||||
danger
|
||||
centerMessage
|
||||
modalClassName="admin-modal--delete-confirm"
|
||||
loading={batchDeleting}
|
||||
onCancel={() => {
|
||||
if (!batchDeleting) setBatchDeleteOpen(false);
|
||||
}}
|
||||
onConfirm={confirmBatchDelete}
|
||||
/>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -261,6 +472,27 @@ function PreviewStatus({ s }: { s: string }) {
|
||||
return <span className="admin-status is-pending">待生成</span>;
|
||||
}
|
||||
|
||||
function VideoFileMetaPills({ video }: { video: api.AdminVideo }) {
|
||||
const parts = fileMetaParts(video);
|
||||
const category = (video.category ?? "").trim();
|
||||
if (parts.length === 0 && !category) return null;
|
||||
|
||||
return (
|
||||
<div className="admin-video-filemeta-pills" aria-label="视频文件信息">
|
||||
{parts.map((part, index) => (
|
||||
<span key={`${part}-${index}`} className="admin-video-filemeta-pill">
|
||||
{part}
|
||||
</span>
|
||||
))}
|
||||
{category && (
|
||||
<span className="admin-video-filemeta-pill is-category">
|
||||
{category}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function formatDur(sec: number): string {
|
||||
if (!sec) return "—";
|
||||
const m = Math.floor(sec / 60);
|
||||
@@ -268,6 +500,26 @@ function formatDur(sec: number): string {
|
||||
return `${m.toString().padStart(2, "0")}:${s.toString().padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
function useVideosPageSize() {
|
||||
const [pageSize, setPageSize] = useState(() =>
|
||||
window.matchMedia(VIDEOS_MOBILE_QUERY).matches
|
||||
? MOBILE_VIDEOS_PAGE_SIZE
|
||||
: DESKTOP_VIDEOS_PAGE_SIZE
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const media = window.matchMedia(VIDEOS_MOBILE_QUERY);
|
||||
const update = () => {
|
||||
setPageSize(media.matches ? MOBILE_VIDEOS_PAGE_SIZE : DESKTOP_VIDEOS_PAGE_SIZE);
|
||||
};
|
||||
update();
|
||||
media.addEventListener("change", update);
|
||||
return () => media.removeEventListener("change", update);
|
||||
}, []);
|
||||
|
||||
return pageSize;
|
||||
}
|
||||
|
||||
function EditVideoModal({
|
||||
video,
|
||||
availableTags,
|
||||
@@ -279,6 +531,7 @@ function EditVideoModal({
|
||||
onClose: () => void;
|
||||
onSaved: () => void;
|
||||
}) {
|
||||
const idPrefix = useId();
|
||||
const [title, setTitle] = useState(video.title);
|
||||
const [author, setAuthor] = useState(video.author ?? "");
|
||||
const [selectedTags, setSelectedTags] = useState(video.tags ?? []);
|
||||
@@ -321,10 +574,10 @@ function EditVideoModal({
|
||||
onClose={onClose}
|
||||
footer={
|
||||
<>
|
||||
<button className="admin-btn" onClick={onClose}>
|
||||
<button type="button" className="admin-btn" onClick={onClose}>
|
||||
取消
|
||||
</button>
|
||||
<button className="admin-btn is-primary" onClick={handleSave} disabled={saving}>
|
||||
<button type="button" className="admin-btn is-primary" onClick={handleSave} disabled={saving}>
|
||||
{saving ? "保存中..." : "保存"}
|
||||
</button>
|
||||
</>
|
||||
@@ -332,15 +585,15 @@ function EditVideoModal({
|
||||
>
|
||||
<div className="admin-form">
|
||||
<div className="admin-form__row">
|
||||
<label>标题</label>
|
||||
<input value={title} onChange={(e) => setTitle(e.target.value)} />
|
||||
<label htmlFor={`${idPrefix}-video-title`}>标题</label>
|
||||
<input id={`${idPrefix}-video-title`} value={title} onChange={(e) => setTitle(e.target.value)} />
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>作者</label>
|
||||
<input value={author} onChange={(e) => setAuthor(e.target.value)} />
|
||||
<label htmlFor={`${idPrefix}-video-author`}>作者</label>
|
||||
<input id={`${idPrefix}-video-author`} value={author} onChange={(e) => setAuthor(e.target.value)} />
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>标签</label>
|
||||
<div className="admin-form__label">标签</div>
|
||||
<div className="admin-tag-picker">
|
||||
{availableTags.map((tag) => (
|
||||
<label key={tag.id} className="admin-check">
|
||||
@@ -356,36 +609,49 @@ function EditVideoModal({
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>分类</label>
|
||||
<input value={category} onChange={(e) => setCategory(e.target.value)} />
|
||||
<label htmlFor={`${idPrefix}-video-category`}>分类</label>
|
||||
<input id={`${idPrefix}-video-category`} value={category} onChange={(e) => setCategory(e.target.value)} />
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>徽标(逗号分隔,例如 精选, 原创)</label>
|
||||
<input value={badges} onChange={(e) => setBadges(e.target.value)} />
|
||||
<label htmlFor={`${idPrefix}-video-badges`}>徽标(逗号分隔,例如 精选, 原创)</label>
|
||||
<input id={`${idPrefix}-video-badges`} value={badges} onChange={(e) => setBadges(e.target.value)} />
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>质量</label>
|
||||
<select value={quality} onChange={(e) => setQuality(e.target.value)}>
|
||||
<label htmlFor={`${idPrefix}-video-quality`}>质量</label>
|
||||
<select id={`${idPrefix}-video-quality`} value={quality} onChange={(e) => setQuality(e.target.value)}>
|
||||
<option value="">未设置</option>
|
||||
<option value="HD">HD</option>
|
||||
<option value="SD">SD</option>
|
||||
</select>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>时长(秒)</label>
|
||||
<label htmlFor={`${idPrefix}-video-duration`}>时长(秒)</label>
|
||||
<input
|
||||
id={`${idPrefix}-video-duration`}
|
||||
value={durationSec}
|
||||
onChange={(e) => setDurationSec(e.target.value)}
|
||||
inputMode="numeric"
|
||||
/>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>封面 URL</label>
|
||||
<input value={thumbnail} onChange={(e) => setThumbnail(e.target.value)} />
|
||||
<label htmlFor={`${idPrefix}-video-thumbnail`}>封面 URL</label>
|
||||
<div className="admin-thumbnail-preview">
|
||||
<input id={`${idPrefix}-video-thumbnail`} value={thumbnail} onChange={(e) => setThumbnail(e.target.value)} />
|
||||
{thumbnail && (
|
||||
<img
|
||||
src={thumbnail}
|
||||
alt="封面预览"
|
||||
className="admin-thumbnail-img"
|
||||
onError={(e) => (e.currentTarget.style.display = 'none')}
|
||||
onLoad={(e) => (e.currentTarget.style.display = 'block')}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>描述</label>
|
||||
<label htmlFor={`${idPrefix}-video-description`}>描述</label>
|
||||
<textarea
|
||||
id={`${idPrefix}-video-description`}
|
||||
value={description}
|
||||
onChange={(e) => setDescription(e.target.value)}
|
||||
/>
|
||||
@@ -395,7 +661,7 @@ function EditVideoModal({
|
||||
<dd>{video.driveId}</dd>
|
||||
<dt>文件信息</dt>
|
||||
<dd>{fileMeta(video) || "—"}</dd>
|
||||
<dt>Teaser</dt>
|
||||
<dt>预览视频</dt>
|
||||
<dd>
|
||||
<PreviewStatus s={video.previewStatus} />
|
||||
</dd>
|
||||
@@ -415,12 +681,15 @@ function EditVideoModal({
|
||||
}
|
||||
|
||||
function fileMeta(v: api.AdminVideo): string {
|
||||
const parts = [
|
||||
return fileMetaParts(v).join(" · ");
|
||||
}
|
||||
|
||||
function fileMetaParts(v: api.AdminVideo): string[] {
|
||||
return [
|
||||
normalizeExt(v.ext),
|
||||
v.quality,
|
||||
formatBytes(v.size),
|
||||
v.size > 0 ? formatBytes(v.size) : "",
|
||||
].filter(Boolean);
|
||||
return parts.join(" · ");
|
||||
}
|
||||
|
||||
function normalizeExt(ext: string): string {
|
||||
@@ -428,19 +697,6 @@ function normalizeExt(ext: string): string {
|
||||
return value ? value.toUpperCase() : "";
|
||||
}
|
||||
|
||||
function formatBytes(size: number): string {
|
||||
if (!size || size <= 0) return "";
|
||||
const units = ["B", "KB", "MB", "GB", "TB"];
|
||||
let value = size;
|
||||
let unit = 0;
|
||||
while (value >= 1024 && unit < units.length - 1) {
|
||||
value /= 1024;
|
||||
unit += 1;
|
||||
}
|
||||
const digits = unit === 0 || value >= 100 ? 0 : 1;
|
||||
return `${value.toFixed(digits)} ${units[unit]}`;
|
||||
}
|
||||
|
||||
function splitList(s: string): string[] {
|
||||
return s
|
||||
.split(/[,,、\s]+/)
|
||||
|
||||
+81
-13
@@ -77,13 +77,13 @@ export function checkUpdate() {
|
||||
|
||||
export type AdminDrive = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
kind: "quark" | "p115" | "p123" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
name: string;
|
||||
rootId: string;
|
||||
status: string;
|
||||
lastError?: string;
|
||||
hasCredential: boolean;
|
||||
/** 当前是否给该盘生成 teaser/封面(per-drive 开关,替代旧的全局 preview.enabled)。 */
|
||||
/** 当前是否给该盘生成预览视频/封面(per-drive 开关,替代旧的全局 preview.enabled)。 */
|
||||
teaserEnabled: boolean;
|
||||
/**
|
||||
* 用户在 admin 配置的"扫描跳过目录"集合(drive 侧目录 fileID 列表)。
|
||||
@@ -93,6 +93,11 @@ export type AdminDrive = {
|
||||
skipDirIds: string[];
|
||||
// spider91 上次成功爬取时间(unix 秒);其它 kind 留空。
|
||||
lastCrawlAt?: number;
|
||||
// spider91 专用代理地址;仅后台管理接口返回,用于编辑表单回显。
|
||||
spider91Proxy?: string;
|
||||
// Google Drive 是否使用 OpenList 在线续期 API;未配置时后端按 true 返回。
|
||||
googleDriveUseOnlineAPI?: boolean;
|
||||
scanGenerationStatus?: DriveGenerationStatus;
|
||||
thumbnailGenerationStatus?: DriveGenerationStatus;
|
||||
previewGenerationStatus?: DriveGenerationStatus;
|
||||
fingerprintGenerationStatus?: DriveGenerationStatus;
|
||||
@@ -113,6 +118,8 @@ export type DriveGenerationStatus = {
|
||||
currentTitle?: string;
|
||||
queueLength: number;
|
||||
cooldownUntil?: string;
|
||||
scannedCount: number;
|
||||
addedCount: number;
|
||||
};
|
||||
|
||||
export function listDrives() {
|
||||
@@ -137,7 +144,7 @@ export function getDriveStorage() {
|
||||
|
||||
export type UpsertDriveInput = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
kind: "quark" | "p115" | "p123" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
name: string;
|
||||
rootId: string;
|
||||
credentials: Record<string, string>;
|
||||
@@ -156,23 +163,62 @@ export function upsertDrive(body: UpsertDriveInput) {
|
||||
});
|
||||
}
|
||||
|
||||
export function deleteDrive(id: string) {
|
||||
return request<{ ok: boolean }>(`/drives/${encodeURIComponent(id)}`, {
|
||||
export type DeleteDriveInput = {
|
||||
deleteVideos: true;
|
||||
};
|
||||
|
||||
export function deleteDrive(id: string, body: DeleteDriveInput) {
|
||||
return request<{ ok: boolean; deletedVideos: number }>(`/drives/${encodeURIComponent(id)}`, {
|
||||
method: "DELETE",
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
export function rescan(id: string) {
|
||||
return request<{ ok: boolean }>(
|
||||
return request<{ ok: boolean; accepted: boolean; message?: string; status?: NightlyJobStatus }>(
|
||||
`/drives/${encodeURIComponent(id)}/rescan`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
export function stopDriveTasks(id: string) {
|
||||
return request<{ ok: boolean; stopped: boolean }>(
|
||||
`/drives/${encodeURIComponent(id)}/tasks/stop`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
export type P123QRSession = {
|
||||
loginUuid: string;
|
||||
uniID: string;
|
||||
qrCodeUrl: string;
|
||||
qrImageDataUrl: string;
|
||||
expiresAt?: string;
|
||||
};
|
||||
|
||||
export type P123QRStatus = {
|
||||
loginStatus: number;
|
||||
statusText: string;
|
||||
scanPlatform?: number;
|
||||
platformText?: string;
|
||||
accessToken?: string;
|
||||
};
|
||||
|
||||
export function startP123QRLogin() {
|
||||
return request<P123QRSession>("/drives/p123/qr", { method: "POST" });
|
||||
}
|
||||
|
||||
export function getP123QRStatus(uniID: string, loginUuid: string) {
|
||||
const qs = new URLSearchParams({ loginUuid });
|
||||
return request<P123QRStatus>(
|
||||
`/drives/p123/qr/${encodeURIComponent(uniID)}?${qs.toString()}`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 切换某个云盘的 teaser 生成开关。点击网盘列表里行内的 toggle 按钮时调用。
|
||||
* 切换某个云盘的预览视频生成开关。点击网盘列表里行内的 toggle 按钮时调用。
|
||||
*
|
||||
* 后端会写 catalog.drives.teaser_enabled,并在从关到开时立刻补扫该盘 pending teaser;
|
||||
* 后端会写 catalog.drives.teaser_enabled,并在从关到开时立刻补扫该盘 pending 预览视频;
|
||||
* 关闭分支不补做任何事,新的入队判断会自动停。
|
||||
*/
|
||||
export function setDriveTeaserEnabled(id: string, enabled: boolean) {
|
||||
@@ -231,7 +277,7 @@ export function regenFailedPreviews(id: string) {
|
||||
|
||||
/**
|
||||
* 触发某 drive 下所有 thumbnail_status=failed 的封面重新入队生成。
|
||||
* 与 regenFailedPreviews 行为对称(一个管 teaser,一个管封面)。
|
||||
* 与 regenFailedPreviews 行为对称(一个管预览视频,一个管封面)。
|
||||
*
|
||||
* 后端立即返回 202;实际状态变化在下次 listDrives 拉到的 thumbnailFailedCount /
|
||||
* thumbnailGenerationStatus 字段里观察。
|
||||
@@ -243,6 +289,13 @@ export function regenFailedThumbnails(id: string) {
|
||||
);
|
||||
}
|
||||
|
||||
export function regenFailedFingerprints(id: string) {
|
||||
return request<{ ok: boolean }>(
|
||||
`/drives/${encodeURIComponent(id)}/fingerprints/failed/regenerate`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- Videos ----------
|
||||
|
||||
export type AdminVideo = {
|
||||
@@ -276,11 +329,12 @@ export type AdminVideoList = {
|
||||
size: number;
|
||||
};
|
||||
|
||||
export function listVideos(params: { driveId?: string; page?: number; size?: number } = {}) {
|
||||
export function listVideos(params: { driveId?: string; page?: number; size?: number; keyword?: string } = {}) {
|
||||
const qs = new URLSearchParams();
|
||||
if (params.driveId) qs.set("driveId", params.driveId);
|
||||
if (params.page) qs.set("page", String(params.page));
|
||||
if (params.size) qs.set("size", String(params.size));
|
||||
if (params.keyword) qs.set("keyword", params.keyword);
|
||||
const suffix = qs.toString() ? `?${qs.toString()}` : "";
|
||||
return request<AdminVideoList>(`/videos${suffix}`);
|
||||
}
|
||||
@@ -304,6 +358,13 @@ export function updateVideo(id: string, body: UpdateVideoInput) {
|
||||
});
|
||||
}
|
||||
|
||||
export function deleteVideo(id: string) {
|
||||
return request<{ ok: boolean; deletedSource: boolean }>(
|
||||
`/videos/${encodeURIComponent(id)}`,
|
||||
{ method: "DELETE" }
|
||||
);
|
||||
}
|
||||
|
||||
export function regenPreview(id: string) {
|
||||
return request<{ ok: boolean }>(
|
||||
`/videos/${encodeURIComponent(id)}/regen-preview`,
|
||||
@@ -346,9 +407,9 @@ export type Theme = "dark" | "pink";
|
||||
export type Settings = {
|
||||
theme: Theme;
|
||||
/**
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115 或 onedrive drive)。
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115、p123、onedrive 或 googledrive drive)。
|
||||
* - 空字符串:本地保存,不上传到云盘。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, onedrive}。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, p123, onedrive, googledrive}。
|
||||
*/
|
||||
spider91UploadDriveId: string;
|
||||
};
|
||||
@@ -392,8 +453,15 @@ export function getNightlyJobStatus() {
|
||||
}
|
||||
|
||||
export function runNightlyJob() {
|
||||
return request<{ ok: boolean; accepted: boolean; status: NightlyJobStatus }>(
|
||||
return request<{ ok: boolean; accepted: boolean; status: NightlyJobStatus; message?: string }>(
|
||||
"/jobs/nightly/run",
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
export function stopAllTasks() {
|
||||
return request<{ ok: boolean; stoppedDrives: number; status: NightlyJobStatus }>(
|
||||
"/tasks/stop",
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { Trash2 } from "lucide-react";
|
||||
import * as api from "../api";
|
||||
import { Modal } from "../Modal";
|
||||
|
||||
export function DeleteDriveModal({
|
||||
drive,
|
||||
deleting,
|
||||
onCancel,
|
||||
onConfirm,
|
||||
}: {
|
||||
drive: api.AdminDrive | null;
|
||||
deleting: boolean;
|
||||
onCancel: () => void;
|
||||
onConfirm: () => void;
|
||||
}) {
|
||||
const name = drive?.name || drive?.id || "";
|
||||
const isSpider91 = drive?.kind === "spider91";
|
||||
const title = isSpider91 ? "删除 91Spider" : "删除存储";
|
||||
const primaryText = deleting ? "删除中..." : "确认删除";
|
||||
|
||||
return (
|
||||
<Modal
|
||||
open={!!drive}
|
||||
title={title}
|
||||
onClose={onCancel}
|
||||
className="admin-modal--delete-confirm"
|
||||
footer={
|
||||
<>
|
||||
<button className="admin-btn" onClick={onCancel} disabled={deleting}>
|
||||
取消
|
||||
</button>
|
||||
<button className="admin-btn is-danger" onClick={onConfirm} disabled={deleting}>
|
||||
<Trash2 size={13} />
|
||||
{primaryText}
|
||||
</button>
|
||||
</>
|
||||
}
|
||||
>
|
||||
<div className="admin-confirm is-message-centered">
|
||||
<div className="admin-confirm__content">
|
||||
<p className="admin-confirm__message">{`确定要删除「${name}」吗?`}</p>
|
||||
</div>
|
||||
</div>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,319 @@
|
||||
import { PlayCircle, Power, PowerOff, RotateCcw } from "lucide-react";
|
||||
import * as api from "../api";
|
||||
import { formatBytes } from "../storageFormat";
|
||||
import {
|
||||
generationStateLabel,
|
||||
generationStateClass,
|
||||
generationDetail,
|
||||
generationTitle,
|
||||
} from "./constants";
|
||||
|
||||
export function StorageSummary({ storage }: { storage: api.AdminDriveStorage }) {
|
||||
return (
|
||||
<section className="admin-card admin-storage-summary" aria-label="本地媒体存储">
|
||||
<div className="admin-storage-summary__metric">
|
||||
<span>封面占用</span>
|
||||
<strong>{formatBytes(storage.thumbnailBytes)}</strong>
|
||||
</div>
|
||||
<div className="admin-storage-summary__metric">
|
||||
<span>预览视频占用</span>
|
||||
<strong>{formatBytes(storage.teaserBytes)}</strong>
|
||||
</div>
|
||||
<div className="admin-storage-summary__metric">
|
||||
<span>本地媒体合计</span>
|
||||
<strong>{formatBytes(storage.totalBytes)}</strong>
|
||||
</div>
|
||||
<div className="admin-storage-summary__metric">
|
||||
<span>磁盘可用</span>
|
||||
<strong>{formatBytes(storage.availableBytes)}</strong>
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
export function GenerationCounts({
|
||||
ready,
|
||||
pending,
|
||||
failed,
|
||||
durationPending,
|
||||
}: {
|
||||
ready?: number;
|
||||
pending?: number;
|
||||
failed?: number;
|
||||
durationPending?: number;
|
||||
}) {
|
||||
return (
|
||||
<div className="admin-generation-counts">
|
||||
<span className="admin-drive-teaser__metric is-ready">
|
||||
就绪 {ready ?? 0}
|
||||
</span>
|
||||
<span className="admin-drive-teaser__metric is-pending">
|
||||
待生成 {pending ?? 0}
|
||||
</span>
|
||||
<span className="admin-drive-teaser__metric is-failed">
|
||||
失败 {failed ?? 0}
|
||||
</span>
|
||||
{(durationPending ?? 0) > 0 && (
|
||||
<span className="admin-drive-teaser__metric">
|
||||
待补时长 {durationPending}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function GenerationStatusLine({
|
||||
label,
|
||||
status,
|
||||
}: {
|
||||
label: string;
|
||||
status?: api.DriveGenerationStatus;
|
||||
}) {
|
||||
const state = status?.state || "idle";
|
||||
const queueLength = status?.queueLength ?? 0;
|
||||
const detail = generationDetail(status);
|
||||
const title = generationTitle(status, detail);
|
||||
const countText = queueLength > 0 ? `${label === "封面" ? "待处理" : "队列"} ${queueLength}` : "";
|
||||
|
||||
return (
|
||||
<div className="admin-generation-row" title={title}>
|
||||
<span className="admin-generation-kind">{label}</span>
|
||||
<span className={`admin-status admin-generation-state is-${generationStateClass(state)}`}>
|
||||
{generationStateLabel(state)}
|
||||
</span>
|
||||
{(detail || queueLength > 0) && (
|
||||
<span className="admin-generation-detail">
|
||||
{[detail, countText].filter(Boolean).join(" / ")}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function StatusTag({
|
||||
kind,
|
||||
status,
|
||||
error,
|
||||
hasCred,
|
||||
}: {
|
||||
kind: string;
|
||||
status: string;
|
||||
error?: string;
|
||||
hasCred: boolean;
|
||||
}) {
|
||||
if (kind !== "spider91" && !hasCred) {
|
||||
return <span className="admin-status is-pending">未配置凭证</span>;
|
||||
}
|
||||
if (status === "ok") {
|
||||
if (kind === "spider91") {
|
||||
return <span className="admin-status is-ok">已就绪</span>;
|
||||
}
|
||||
return <span className="admin-status is-ok">已连接</span>;
|
||||
}
|
||||
if (status === "error")
|
||||
return (
|
||||
<span className="admin-status is-error" title={error}>
|
||||
错误
|
||||
</span>
|
||||
);
|
||||
return <span className="admin-status">{status || "未连接"}</span>;
|
||||
}
|
||||
|
||||
export function DriveCardMetrics({ d }: { d: api.AdminDrive }) {
|
||||
return (
|
||||
<div className="admin-drive-card__info">
|
||||
<div className="admin-drive-card__metric">
|
||||
<span>封面数 (就绪/失败)</span>
|
||||
<strong>
|
||||
{d.thumbnailReadyCount ?? 0}
|
||||
<span style={{ fontSize: "11px", fontWeight: "normal", color: "var(--text-faint)" }}>
|
||||
{" "}/ {d.thumbnailFailedCount ?? 0}
|
||||
</span>
|
||||
</strong>
|
||||
</div>
|
||||
<div className="admin-drive-card__metric">
|
||||
<span>预览视频数 (就绪/失败)</span>
|
||||
<strong>
|
||||
{d.teaserReadyCount ?? 0}
|
||||
<span style={{ fontSize: "11px", fontWeight: "normal", color: "var(--text-faint)" }}>
|
||||
{" "}/ {d.teaserFailedCount ?? 0}
|
||||
</span>
|
||||
</strong>
|
||||
</div>
|
||||
<div className="admin-drive-card__metric">
|
||||
<span>视频指纹数 (就绪/失败)</span>
|
||||
<strong>
|
||||
{d.fingerprintReadyCount ?? 0}
|
||||
<span style={{ fontSize: "11px", fontWeight: "normal", color: "var(--text-faint)" }}>
|
||||
{" "}/ {d.fingerprintFailedCount ?? 0}
|
||||
</span>
|
||||
</strong>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function DriveGenerationPanel({
|
||||
d,
|
||||
regenFailedId,
|
||||
regenFailedThumbId,
|
||||
regenFailedFingerprintId,
|
||||
togglingTeaserId,
|
||||
onToggleTeaser,
|
||||
onRegenFailed,
|
||||
onRegenFailedThumbnails,
|
||||
onRegenFailedFingerprints,
|
||||
}: {
|
||||
d: api.AdminDrive;
|
||||
regenFailedId: string;
|
||||
regenFailedThumbId: string;
|
||||
regenFailedFingerprintId: string;
|
||||
togglingTeaserId: string;
|
||||
onToggleTeaser: () => void;
|
||||
onRegenFailed: () => void;
|
||||
onRegenFailedThumbnails: () => void;
|
||||
onRegenFailedFingerprints: () => void;
|
||||
}) {
|
||||
const canQueueThumbnails =
|
||||
(d.thumbnailFailedCount ?? 0) > 0 ||
|
||||
(d.thumbnailPendingCount ?? 0) > 0 ||
|
||||
(d.thumbnailDurationPendingCount ?? 0) > 0;
|
||||
const canQueuePreviews =
|
||||
(d.teaserFailedCount ?? 0) > 0 || (d.teaserPendingCount ?? 0) > 0;
|
||||
const canQueueFingerprints =
|
||||
(d.fingerprintFailedCount ?? 0) > 0 || (d.fingerprintPendingCount ?? 0) > 0;
|
||||
|
||||
return (
|
||||
<div className="admin-detail-card">
|
||||
<header className="admin-detail-card__title">
|
||||
<div className="admin-detail-card__title-left">
|
||||
<PlayCircle size={16} />
|
||||
<span>生成状态</span>
|
||||
</div>
|
||||
<div className="admin-detail-actions-inline">
|
||||
<button
|
||||
className={`admin-btn ${d.teaserEnabled ? "is-success" : ""}`}
|
||||
onClick={onToggleTeaser}
|
||||
disabled={togglingTeaserId === d.id}
|
||||
style={{ padding: "4px 10px", fontSize: "11px" }}
|
||||
>
|
||||
{d.teaserEnabled ? <Power size={11} /> : <PowerOff size={11} />}
|
||||
<span>{d.teaserEnabled ? "预览视频:开" : "预览视频:关"}</span>
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div className="admin-gen-columns">
|
||||
<DriveGenCol
|
||||
label={d.kind === "spider91" ? "抓取" : "扫盘"}
|
||||
status={d.scanGenerationStatus}
|
||||
showCounts={false}
|
||||
/>
|
||||
<DriveGenCol
|
||||
label="封面"
|
||||
status={d.thumbnailGenerationStatus}
|
||||
ready={d.thumbnailReadyCount}
|
||||
pending={d.thumbnailPendingCount}
|
||||
failed={d.thumbnailFailedCount}
|
||||
extra={d.thumbnailDurationPendingCount}
|
||||
/>
|
||||
<DriveGenCol
|
||||
label="预览视频"
|
||||
status={d.previewGenerationStatus}
|
||||
ready={d.teaserReadyCount}
|
||||
pending={d.teaserPendingCount}
|
||||
failed={d.teaserFailedCount}
|
||||
/>
|
||||
<DriveGenCol
|
||||
label="视频指纹"
|
||||
status={d.fingerprintGenerationStatus}
|
||||
ready={d.fingerprintReadyCount}
|
||||
pending={d.fingerprintPendingCount}
|
||||
failed={d.fingerprintFailedCount}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="admin-detail-actions">
|
||||
<button
|
||||
className="admin-btn"
|
||||
disabled={!canQueueThumbnails || regenFailedThumbId === d.id}
|
||||
onClick={onRegenFailedThumbnails}
|
||||
>
|
||||
<RotateCcw size={13} />
|
||||
<span>{(d.thumbnailFailedCount ?? 0) > 0 ? "重试失败封面" : "继续生成封面"}</span>
|
||||
</button>
|
||||
<button
|
||||
className="admin-btn"
|
||||
disabled={!canQueuePreviews || regenFailedId === d.id}
|
||||
onClick={onRegenFailed}
|
||||
>
|
||||
<RotateCcw size={13} />
|
||||
<span>{(d.teaserFailedCount ?? 0) > 0 ? "重试失败预览视频" : "继续生成预览视频"}</span>
|
||||
</button>
|
||||
<button
|
||||
className="admin-btn"
|
||||
disabled={!canQueueFingerprints || regenFailedFingerprintId === d.id}
|
||||
onClick={onRegenFailedFingerprints}
|
||||
>
|
||||
<RotateCcw size={13} />
|
||||
<span>{(d.fingerprintFailedCount ?? 0) > 0 ? "重试失败指纹" : "继续生成指纹"}</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function DriveGenCol({
|
||||
label,
|
||||
status,
|
||||
ready,
|
||||
pending,
|
||||
failed,
|
||||
extra,
|
||||
showCounts = true,
|
||||
}: {
|
||||
label: string;
|
||||
status?: api.DriveGenerationStatus;
|
||||
ready?: number;
|
||||
pending?: number;
|
||||
failed?: number;
|
||||
extra?: number;
|
||||
showCounts?: boolean;
|
||||
}) {
|
||||
const state = status?.state || "idle";
|
||||
const detail = generationDetail(status);
|
||||
const title = generationTitle(status, detail);
|
||||
const stateLabel = label === "抓取" && state === "scanning" ? "抓取中" : generationStateLabel(state);
|
||||
const showScanProgress = !showCounts && (state === "scanning" || (status?.scannedCount ?? 0) > 0 || (status?.addedCount ?? 0) > 0);
|
||||
const scannedLabel = label === "抓取" ? "已抓取" : "已扫描";
|
||||
return (
|
||||
<div className="admin-gen-col">
|
||||
<div className="admin-gen-col__head">
|
||||
<span className="admin-gen-col__label">{label}</span>
|
||||
<span
|
||||
className={`admin-status admin-generation-state is-${generationStateClass(state)}`}
|
||||
title={title || undefined}
|
||||
>
|
||||
{stateLabel}
|
||||
</span>
|
||||
</div>
|
||||
{detail && <div className="admin-gen-col__detail">{detail}</div>}
|
||||
{showScanProgress && (
|
||||
<div className="admin-gen-col__counts admin-gen-col__counts--scan">
|
||||
<div className="admin-gen-col__count"><span>{scannedLabel}</span><strong>{status?.scannedCount ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>预计新增</span><strong>{status?.addedCount ?? 0}</strong></div>
|
||||
</div>
|
||||
)}
|
||||
{showCounts && (
|
||||
<div className="admin-gen-col__counts">
|
||||
<div className="admin-gen-col__count"><span>就绪</span><strong>{ready ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>待生成</span><strong>{pending ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>失败</span><strong>{failed ?? 0}</strong></div>
|
||||
{(extra ?? 0) > 0 && (
|
||||
<div className="admin-gen-col__count"><span>待补时长</span><strong>{extra}</strong></div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
import { useId, useMemo, useState } from "react";
|
||||
import { ArrowLeft, ChevronDown } from "lucide-react";
|
||||
import { P123QRCodeLogin } from "./P123QRCodeLogin";
|
||||
import { Spider91UploadTargetField } from "./Spider91UploadTargetField";
|
||||
import {
|
||||
FormState,
|
||||
Kind,
|
||||
credentialFields,
|
||||
credentialHelp,
|
||||
usesRootDirectoryID,
|
||||
rootIdPlaceholder,
|
||||
} from "./constants";
|
||||
import * as api from "../api";
|
||||
|
||||
type DriveOption = {
|
||||
kind: Kind;
|
||||
label: string;
|
||||
abbr: string;
|
||||
desc: string;
|
||||
};
|
||||
|
||||
const DRIVE_OPTIONS: DriveOption[] = [
|
||||
{ kind: "p115", label: "115 网盘", abbr: "115", desc: "302直链,不占带宽" },
|
||||
{ kind: "p123", label: "123 云盘", abbr: "123", desc: "扫码登录,302直链" },
|
||||
{ kind: "pikpak", label: "PikPak", abbr: "Pk", desc: "302直链,稳定快速" },
|
||||
{ kind: "onedrive", label: "OneDrive", abbr: "OD", desc: "302直链,微软网盘" },
|
||||
{ kind: "googledrive", label: "Google Drive", abbr: "GD", desc: "服务器中转模式" },
|
||||
{ kind: "localstorage", label: "本地存储", abbr: "Lo", desc: "本机文件目录" },
|
||||
{ kind: "spider91", label: "91 爬虫", abbr: "91", desc: "自动抓取热门视频" },
|
||||
{ kind: "quark", label: "夸克网盘", abbr: "Qk", desc: "302直链" },
|
||||
{ kind: "wopan", label: "联通沃盘", abbr: "Wo", desc: "302直链" },
|
||||
];
|
||||
|
||||
export function DriveForm({
|
||||
form,
|
||||
onChange,
|
||||
isEdit,
|
||||
uploadTargets,
|
||||
nameError,
|
||||
onNameBlur,
|
||||
onBack,
|
||||
}: {
|
||||
form: FormState;
|
||||
onChange: (f: FormState) => void;
|
||||
isEdit: boolean;
|
||||
uploadTargets: api.AdminDrive[];
|
||||
nameError?: string;
|
||||
onNameBlur?: () => void;
|
||||
onBack?: () => void;
|
||||
}) {
|
||||
const idPrefix = useId();
|
||||
const fields = useMemo(() => credentialFields(form.kind, form.creds), [form.kind, form.creds]);
|
||||
const help = credentialHelp(form.kind, isEdit);
|
||||
const [step, setStep] = useState<"type" | "form">(isEdit ? "form" : "type");
|
||||
const nameId = `${idPrefix}-drive-name`;
|
||||
const rootId = `${idPrefix}-drive-root`;
|
||||
|
||||
function set<K extends keyof FormState>(k: K, v: FormState[K]) {
|
||||
onChange({ ...form, [k]: v });
|
||||
}
|
||||
function setCred(k: string, v: string) {
|
||||
onChange({ ...form, creds: { ...form.creds, [k]: v } });
|
||||
}
|
||||
function setKind(v: Kind) {
|
||||
onChange({
|
||||
...form,
|
||||
kind: v,
|
||||
rootId: "",
|
||||
creds: {},
|
||||
});
|
||||
}
|
||||
function selectType(kind: Kind) {
|
||||
setKind(kind);
|
||||
setStep("form");
|
||||
}
|
||||
function goBack() {
|
||||
setStep("type");
|
||||
onChange({
|
||||
...form,
|
||||
name: "",
|
||||
rootId: "",
|
||||
creds: {},
|
||||
});
|
||||
onBack?.();
|
||||
}
|
||||
|
||||
const selectedOption = DRIVE_OPTIONS.find((o) => o.kind === form.kind);
|
||||
|
||||
if (step === "type" && !isEdit) {
|
||||
return (
|
||||
<div className="admin-drive-type-picker">
|
||||
<div className="admin-drive-type-grid">
|
||||
{DRIVE_OPTIONS.map((opt) => (
|
||||
<button
|
||||
key={opt.kind}
|
||||
type="button"
|
||||
className="admin-drive-type-card"
|
||||
data-kind={opt.kind}
|
||||
onClick={() => selectType(opt.kind)}
|
||||
>
|
||||
<span className="admin-drive-type-card__icon" data-kind={opt.kind}>
|
||||
{opt.abbr}
|
||||
</span>
|
||||
<span className="admin-drive-type-card__label">{opt.label}</span>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="admin-form">
|
||||
{!isEdit && selectedOption && (
|
||||
<div className="admin-drive-selected-bar" data-kind={form.kind}>
|
||||
<span className="admin-drive-selected-bar__icon" data-kind={form.kind}>
|
||||
{selectedOption.abbr}
|
||||
</span>
|
||||
<div className="admin-drive-selected-bar__text">
|
||||
<span className="admin-drive-selected-bar__name">{selectedOption.label}</span>
|
||||
<span className="admin-drive-selected-bar__desc">{selectedOption.desc}</span>
|
||||
</div>
|
||||
<button type="button" className="admin-drive-selected-bar__back" onClick={goBack}>
|
||||
<ArrowLeft size={12} /> 重选类型
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="admin-form__section">
|
||||
<div className="admin-form__row">
|
||||
<label htmlFor={nameId}>名称 *</label>
|
||||
<input
|
||||
id={nameId}
|
||||
value={form.name}
|
||||
onChange={(e) => set("name", e.target.value)}
|
||||
onBlur={onNameBlur}
|
||||
placeholder="给这个盘起个名字"
|
||||
className={nameError ? "is-invalid" : undefined}
|
||||
aria-invalid={nameError ? "true" : undefined}
|
||||
aria-describedby={nameError ? `${nameId}-error` : undefined}
|
||||
/>
|
||||
{nameError && (
|
||||
<div className="admin-form__error" id={`${nameId}-error`}>
|
||||
{nameError}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{usesRootDirectoryID(form.kind) && (
|
||||
<div className="admin-form__row">
|
||||
<label htmlFor={rootId}>根目录 ID</label>
|
||||
<input
|
||||
id={rootId}
|
||||
value={form.rootId}
|
||||
onChange={(e) => set("rootId", e.target.value)}
|
||||
placeholder={rootIdPlaceholder(form.kind)}
|
||||
/>
|
||||
<div className="admin-form__help">
|
||||
留空时使用该网盘类型的默认根目录
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{(help || fields.length > 0) && (
|
||||
<div className="admin-form__section">
|
||||
<h3 className="admin-form__section-label">凭证配置</h3>
|
||||
|
||||
{help && (
|
||||
<div className="admin-form__help admin-form__help--lead">
|
||||
{help}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{form.kind === "p123" && (
|
||||
<P123QRCodeLogin
|
||||
onToken={(token) => setCred("access_token", token)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{fields.map((f) => (
|
||||
<div key={f.key} className="admin-form__row">
|
||||
{f.type === "select" ? (
|
||||
<>
|
||||
<label htmlFor={`${idPrefix}-credential-${f.key}`}>
|
||||
{f.label}
|
||||
{f.required && " *"}
|
||||
</label>
|
||||
<div className="admin-form-select-wrap">
|
||||
<select
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
className="admin-form-select"
|
||||
value={form.creds[f.key] ?? f.defaultValue ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
>
|
||||
{(f.options ?? []).map((option) => (
|
||||
<option key={option.value} value={option.value}>
|
||||
{option.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<ChevronDown size={15} className="admin-form-select__icon" aria-hidden="true" />
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<label htmlFor={`${idPrefix}-credential-${f.key}`}>
|
||||
{f.label}
|
||||
{f.required && " *"}
|
||||
</label>
|
||||
{f.multiline ? (
|
||||
<textarea
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
value={form.creds[f.key] ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
placeholder={f.placeholder}
|
||||
required={f.required && !isEdit}
|
||||
/>
|
||||
) : (
|
||||
<input
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
type={credentialInputType(f.key)}
|
||||
value={form.creds[f.key] ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
placeholder={f.placeholder}
|
||||
required={f.required && !isEdit}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
{f.help && <div className="admin-form__help">{f.help}</div>}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{form.kind === "spider91" && (
|
||||
<div className="admin-form__section">
|
||||
<h3 className="admin-form__section-label">上传设置</h3>
|
||||
<Spider91UploadTargetField
|
||||
value={form.spider91UploadDriveId}
|
||||
onChange={(v) => set("spider91UploadDriveId", v)}
|
||||
uploadTargets={uploadTargets}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function credentialInputType(key: string): string {
|
||||
return /password|token|secret/i.test(key) ? "password" : "text";
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { QrCode } from "lucide-react";
|
||||
import * as api from "../api";
|
||||
import { useToast } from "../ToastContext";
|
||||
|
||||
function p123QRStatusClass(
|
||||
status: api.P123QRStatus | null,
|
||||
completed: boolean,
|
||||
error: string
|
||||
): string {
|
||||
if (completed || status?.loginStatus === 3) return "is-ok";
|
||||
if (error || status?.loginStatus === 2 || status?.loginStatus === 4) {
|
||||
return "is-error";
|
||||
}
|
||||
return "is-pending";
|
||||
}
|
||||
|
||||
export function P123QRCodeLogin({ onToken }: { onToken: (token: string) => void }) {
|
||||
const { show } = useToast();
|
||||
const [session, setSession] = useState<api.P123QRSession | null>(null);
|
||||
const [status, setStatus] = useState<api.P123QRStatus | null>(null);
|
||||
const [starting, setStarting] = useState(false);
|
||||
const [pollingError, setPollingError] = useState("");
|
||||
const [completed, setCompleted] = useState(false);
|
||||
|
||||
async function start() {
|
||||
setStarting(true);
|
||||
setPollingError("");
|
||||
setCompleted(false);
|
||||
setStatus(null);
|
||||
try {
|
||||
const next = await api.startP123QRLogin();
|
||||
setSession(next);
|
||||
} catch (e) {
|
||||
setSession(null);
|
||||
show(e instanceof Error ? e.message : "生成二维码失败", "error");
|
||||
} finally {
|
||||
setStarting(false);
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (!session || completed) return;
|
||||
const activeSession = session;
|
||||
let stopped = false;
|
||||
let inFlight = false;
|
||||
let timer: number | undefined;
|
||||
|
||||
async function poll() {
|
||||
if (stopped || inFlight) return;
|
||||
inFlight = true;
|
||||
try {
|
||||
const next = await api.getP123QRStatus(activeSession.uniID, activeSession.loginUuid);
|
||||
if (stopped) return;
|
||||
setStatus(next);
|
||||
setPollingError("");
|
||||
if (next.accessToken) {
|
||||
stopped = true;
|
||||
if (timer) window.clearInterval(timer);
|
||||
setCompleted(true);
|
||||
onToken(next.accessToken);
|
||||
show("扫码成功,已填入 access_token,保存后生效", "success");
|
||||
return;
|
||||
}
|
||||
if (next.loginStatus === 2 || next.loginStatus === 4) {
|
||||
stopped = true;
|
||||
if (timer) window.clearInterval(timer);
|
||||
}
|
||||
} catch (e) {
|
||||
if (stopped) return;
|
||||
setPollingError(e instanceof Error ? e.message : "查询扫码状态失败");
|
||||
} finally {
|
||||
inFlight = false;
|
||||
}
|
||||
}
|
||||
|
||||
poll();
|
||||
timer = window.setInterval(poll, 1800);
|
||||
return () => {
|
||||
stopped = true;
|
||||
if (timer) window.clearInterval(timer);
|
||||
};
|
||||
}, [session, completed, onToken, show]);
|
||||
|
||||
const statusText = completed
|
||||
? "已获取 token"
|
||||
: pollingError || status?.statusText || (session ? "等待扫码" : "未生成二维码");
|
||||
const statusClass = p123QRStatusClass(status, completed, pollingError);
|
||||
const platform = status?.platformText ? ` · ${status.platformText}` : "";
|
||||
|
||||
return (
|
||||
<div className="admin-form__row">
|
||||
<label>扫码登录</label>
|
||||
<div className="admin-p123-qr">
|
||||
<div className="admin-p123-qr__actions">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={start}
|
||||
disabled={starting}
|
||||
>
|
||||
<QrCode size={14} />
|
||||
{starting ? "生成中..." : session ? "重新生成二维码" : "生成二维码"}
|
||||
</button>
|
||||
<span className={`admin-status ${statusClass}`}>
|
||||
{statusText}
|
||||
{platform}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{session && (
|
||||
<div className="admin-p123-qr__body">
|
||||
<img
|
||||
className="admin-p123-qr__image"
|
||||
src={session.qrImageDataUrl}
|
||||
alt="123 云盘扫码登录二维码"
|
||||
/>
|
||||
<div className="admin-p123-qr__meta">
|
||||
<div className="admin-form__help">
|
||||
使用微信或 123 云盘 App 扫码并确认登录;确认后系统会自动填入 access_token。
|
||||
</div>
|
||||
{session.expiresAt && (
|
||||
<div className="admin-form__help">
|
||||
过期时间:{new Date(session.expiresAt).toLocaleTimeString("zh-CN", {
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
{(status?.loginStatus === 2 || status?.loginStatus === 4) && (
|
||||
<div className="admin-form__help">
|
||||
当前二维码{status.loginStatus === 2 ? "已被拒绝" : "已过期"},请重新生成。
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import { ChevronDown, ChevronRight } from "lucide-react";
|
||||
import * as api from "../api";
|
||||
import { useToast } from "../ToastContext";
|
||||
import { kindLabel } from "./constants";
|
||||
|
||||
type SkipDirsPanelProps = {
|
||||
drive: api.AdminDrive;
|
||||
onSaved: (saved: { id: string; skipDirIds: string[] }) => void;
|
||||
};
|
||||
|
||||
export function SkipDirsPanel({ drive, onSaved }: SkipDirsPanelProps) {
|
||||
const { show } = useToast();
|
||||
const [selected, setSelected] = useState<Set<string>>(
|
||||
() => new Set(drive.skipDirIds ?? [])
|
||||
);
|
||||
const [saving, setSaving] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
setSelected(new Set(drive.skipDirIds ?? []));
|
||||
}, [drive.id, drive.skipDirIds]);
|
||||
|
||||
const toggle = useCallback((id: string) => {
|
||||
setSelected((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (next.has(id)) {
|
||||
next.delete(id);
|
||||
} else {
|
||||
next.add(id);
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}, []);
|
||||
|
||||
async function handleSave() {
|
||||
setSaving(true);
|
||||
try {
|
||||
const ids = Array.from(selected);
|
||||
const resp = await api.setDriveSkipDirIds(drive.id, ids);
|
||||
onSaved({ id: drive.id, skipDirIds: resp.skipDirIds });
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "保存失败", "error");
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
}
|
||||
|
||||
const selectedList = useMemo(() => Array.from(selected), [selected]);
|
||||
|
||||
return (
|
||||
<div className="admin-detail-card">
|
||||
<header className="admin-detail-card__title">
|
||||
<div className="admin-detail-card__title-left">
|
||||
<span>扫描跳过目录</span>
|
||||
</div>
|
||||
<button
|
||||
className="admin-btn is-primary"
|
||||
onClick={handleSave}
|
||||
disabled={saving}
|
||||
style={{ padding: "4px 10px", fontSize: "12px", height: "auto" }}
|
||||
>
|
||||
{saving ? "保存中..." : `保存更改 (${selectedList.length})`}
|
||||
</button>
|
||||
</header>
|
||||
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: "12px" }}>
|
||||
<p className="admin-text-faint" style={{ margin: 0, fontSize: "12px", lineHeight: "1.5" }}>
|
||||
勾选要在扫描时跳过的目录。命中目录及其全部子目录都不会被递归扫描。下次扫描生效。
|
||||
</p>
|
||||
|
||||
<SelectedDirsChips
|
||||
drive={drive}
|
||||
selected={selectedList}
|
||||
onRemove={toggle}
|
||||
/>
|
||||
|
||||
<div className="admin-detail-tree-container">
|
||||
<DirTreeNode
|
||||
driveId={drive.id}
|
||||
id=""
|
||||
name={drive.name || drive.id}
|
||||
depth={0}
|
||||
initiallyOpen
|
||||
ancestorSkipped={false}
|
||||
selected={selected}
|
||||
onToggle={toggle}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function SelectedDirsChips({
|
||||
drive,
|
||||
selected,
|
||||
onRemove,
|
||||
}: {
|
||||
drive: api.AdminDrive;
|
||||
selected: string[];
|
||||
onRemove: (id: string) => void;
|
||||
}) {
|
||||
if (selected.length === 0) {
|
||||
return (
|
||||
<div
|
||||
className="admin-text-faint"
|
||||
style={{ fontSize: "13px", padding: "6px 0" }}
|
||||
>
|
||||
当前未勾选任何跳过目录({kindLabel[drive.kind] ?? drive.kind}{" "}
|
||||
将完整扫描)。
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<div style={{ display: "flex", flexWrap: "wrap", gap: "6px" }}>
|
||||
{selected.map((id) => (
|
||||
<span
|
||||
key={id}
|
||||
className="admin-mono-cell"
|
||||
style={{
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
gap: "6px",
|
||||
padding: "3px 10px",
|
||||
border: "1px solid var(--border-subtle)",
|
||||
borderRadius: "999px",
|
||||
fontSize: "12px",
|
||||
}}
|
||||
title="点击 × 移除"
|
||||
>
|
||||
{id}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onRemove(id)}
|
||||
style={{
|
||||
border: "none",
|
||||
background: "transparent",
|
||||
cursor: "pointer",
|
||||
color: "var(--text-secondary)",
|
||||
padding: 0,
|
||||
lineHeight: 1,
|
||||
fontSize: "14px",
|
||||
}}
|
||||
aria-label={`移除 ${id}`}
|
||||
>
|
||||
×
|
||||
</button>
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
type DirTreeNodeProps = {
|
||||
driveId: string;
|
||||
id: string;
|
||||
name: string;
|
||||
depth: number;
|
||||
initiallyOpen?: boolean;
|
||||
ancestorSkipped: boolean;
|
||||
selected: Set<string>;
|
||||
onToggle: (id: string) => void;
|
||||
};
|
||||
|
||||
function DirTreeNode({
|
||||
driveId,
|
||||
id,
|
||||
name,
|
||||
depth,
|
||||
initiallyOpen,
|
||||
ancestorSkipped,
|
||||
selected,
|
||||
onToggle,
|
||||
}: DirTreeNodeProps) {
|
||||
const [open, setOpen] = useState(!!initiallyOpen);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [loaded, setLoaded] = useState(false);
|
||||
const [children, setChildren] = useState<api.DriveDirEntry[]>([]);
|
||||
const [error, setError] = useState("");
|
||||
|
||||
const isRoot = depth === 0;
|
||||
const isSelected = id !== "" && selected.has(id);
|
||||
const dimmed = ancestorSkipped;
|
||||
|
||||
const loadChildren = useCallback(async () => {
|
||||
if (loaded || loading) return;
|
||||
setLoading(true);
|
||||
setError("");
|
||||
try {
|
||||
const data = await api.listDriveDirChildren(driveId, id || undefined);
|
||||
setChildren(data ?? []);
|
||||
setLoaded(true);
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : "加载失败");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [driveId, id, loaded, loading]);
|
||||
|
||||
useEffect(() => {
|
||||
if (open && !loaded) {
|
||||
void loadChildren();
|
||||
}
|
||||
}, [open, loaded, loadChildren]);
|
||||
|
||||
function handleToggleOpen() {
|
||||
setOpen((v) => !v);
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
paddingLeft: depth === 0 ? 0 : 16,
|
||||
opacity: dimmed && !isSelected ? 0.55 : 1,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: "6px",
|
||||
padding: "4px 6px",
|
||||
borderRadius: "4px",
|
||||
background: isSelected ? "var(--accent-soft, rgba(255,140,0,0.12))" : "transparent",
|
||||
}}
|
||||
>
|
||||
{!isRoot ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleToggleOpen}
|
||||
style={{
|
||||
border: "none",
|
||||
background: "transparent",
|
||||
cursor: "pointer",
|
||||
padding: 0,
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
}}
|
||||
aria-label={open ? "折叠" : "展开"}
|
||||
>
|
||||
{open ? <ChevronDown size={14} /> : <ChevronRight size={14} />}
|
||||
</button>
|
||||
) : (
|
||||
<span style={{ width: 14, display: "inline-block" }} />
|
||||
)}
|
||||
|
||||
{!isRoot && (
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isSelected}
|
||||
onChange={() => onToggle(id)}
|
||||
aria-label={`跳过目录 ${name}`}
|
||||
/>
|
||||
)}
|
||||
|
||||
<span
|
||||
style={{
|
||||
fontSize: "13px",
|
||||
cursor: isRoot ? "default" : "pointer",
|
||||
userSelect: "none",
|
||||
fontWeight: isRoot ? 600 : 400,
|
||||
}}
|
||||
onClick={isRoot ? undefined : handleToggleOpen}
|
||||
>
|
||||
{name}
|
||||
{isRoot ? " (根目录)" : ""}
|
||||
</span>
|
||||
{!isRoot && (
|
||||
<span
|
||||
className="admin-mono-cell admin-text-faint"
|
||||
style={{ fontSize: "11px", marginLeft: "6px" }}
|
||||
>
|
||||
{id}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{open && (
|
||||
<div>
|
||||
{loading && (
|
||||
<div className="admin-text-faint" style={{ fontSize: "12px", padding: "4px 28px" }}>
|
||||
加载中...
|
||||
</div>
|
||||
)}
|
||||
{error && (
|
||||
<div style={{ fontSize: "12px", padding: "4px 28px", color: "var(--danger, #d33)" }}>
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
{loaded && !error && children.length === 0 && (
|
||||
<div className="admin-text-faint" style={{ fontSize: "12px", padding: "4px 28px" }}>
|
||||
(无子目录)
|
||||
</div>
|
||||
)}
|
||||
{children.map((child) => (
|
||||
<DirTreeNode
|
||||
key={child.id}
|
||||
driveId={driveId}
|
||||
id={child.id}
|
||||
name={child.name}
|
||||
depth={depth + 1}
|
||||
ancestorSkipped={ancestorSkipped || isSelected}
|
||||
selected={selected}
|
||||
onToggle={onToggle}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { useId } from "react";
|
||||
import { ChevronDown } from "lucide-react";
|
||||
import { kindLabel } from "./constants";
|
||||
import * as api from "../api";
|
||||
|
||||
export function Spider91UploadTargetField({
|
||||
value,
|
||||
onChange,
|
||||
uploadTargets,
|
||||
}: {
|
||||
value: string;
|
||||
onChange: (v: string) => void;
|
||||
uploadTargets: api.AdminDrive[];
|
||||
}) {
|
||||
const targetId = useId();
|
||||
|
||||
return (
|
||||
<div className="admin-form__row">
|
||||
<label htmlFor={targetId}>视频上传目标</label>
|
||||
<div className="admin-form-select-wrap">
|
||||
<select
|
||||
id={targetId}
|
||||
className="admin-form-select"
|
||||
value={value}
|
||||
onChange={(e) => onChange(e.target.value)}
|
||||
>
|
||||
<option value="">本地保存,不上传</option>
|
||||
{uploadTargets.map((d) => (
|
||||
<option key={d.id} value={d.id}>
|
||||
{kindLabel[d.kind] ?? d.kind} · {d.name || d.id}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<ChevronDown size={15} className="admin-form-select__icon" aria-hidden="true" />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
export type Kind = "quark" | "p115" | "p123" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
|
||||
export const kindAbbr: Record<string, string> = {
|
||||
quark: "Qk",
|
||||
p115: "115",
|
||||
p123: "123",
|
||||
pikpak: "Pk",
|
||||
wopan: "Wo",
|
||||
onedrive: "OD",
|
||||
googledrive: "GD",
|
||||
localstorage: "Lo",
|
||||
spider91: "91",
|
||||
};
|
||||
|
||||
export function driveKindAbbr(kind: string): string {
|
||||
const explicit = kindAbbr[kind];
|
||||
if (explicit) return explicit;
|
||||
|
||||
const trimmed = kind.trim();
|
||||
if (!trimmed) return "??";
|
||||
const compact = trimmed.replace(/[^a-zA-Z0-9]+/g, "");
|
||||
return (compact || trimmed).slice(0, 2).toUpperCase();
|
||||
}
|
||||
|
||||
export const kindLabel: Record<string, string> = {
|
||||
quark: "夸克网盘",
|
||||
p115: "115 网盘",
|
||||
p123: "123 云盘",
|
||||
pikpak: "PikPak",
|
||||
wopan: "联通沃盘",
|
||||
onedrive: "OneDrive",
|
||||
googledrive: "Google Drive",
|
||||
localstorage: "本地存储",
|
||||
spider91: "91 爬虫",
|
||||
};
|
||||
|
||||
export type FormState = {
|
||||
id: string;
|
||||
kind: Kind;
|
||||
name: string;
|
||||
rootId: string;
|
||||
creds: Record<string, string>;
|
||||
spider91UploadDriveId: string;
|
||||
};
|
||||
|
||||
export const emptyForm: FormState = {
|
||||
id: "",
|
||||
kind: "p115",
|
||||
name: "",
|
||||
rootId: "",
|
||||
creds: {},
|
||||
spider91UploadDriveId: "",
|
||||
};
|
||||
|
||||
export const idleNightlyStatus = {
|
||||
state: "idle" as const,
|
||||
running: false,
|
||||
queued: false,
|
||||
};
|
||||
|
||||
export function nightlyButtonText(status: { running: boolean; queued: boolean }, triggering: boolean) {
|
||||
if (triggering) return "触发中...";
|
||||
if (status.running) return "扫描运行中";
|
||||
if (status.queued) return "扫描已排队";
|
||||
return "扫描所有网盘";
|
||||
}
|
||||
|
||||
export function nightlyBusyText(status: { running: boolean; queued: boolean }) {
|
||||
if (status.running || status.queued) return "当前有全量扫描任务正在进行,请稍后重试";
|
||||
return "";
|
||||
}
|
||||
|
||||
export function generationStateLabel(state: string): string {
|
||||
if (state === "scanning") return "扫盘中";
|
||||
if (state === "generating") return "生成中";
|
||||
if (state === "cooling") return "冷却中";
|
||||
if (state === "queued") return "排队中";
|
||||
return "空闲";
|
||||
}
|
||||
|
||||
export function generationStateClass(state: string): string {
|
||||
if (state === "scanning" || state === "generating" || state === "cooling" || state === "queued") {
|
||||
if (state === "scanning") return "generating";
|
||||
return state;
|
||||
}
|
||||
return "idle";
|
||||
}
|
||||
|
||||
export function generationDetail(status?: { state: string; cooldownUntil?: string; currentTitle?: string }): string {
|
||||
if (!status) return "";
|
||||
if (status.state === "cooling" && status.cooldownUntil) {
|
||||
return `剩余 ${formatCooldownRemaining(status.cooldownUntil)}`;
|
||||
}
|
||||
if (status.currentTitle) {
|
||||
return status.currentTitle;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
export function generationTitle(status: { state: string; cooldownUntil?: string; currentTitle?: string } | undefined, detail: string): string | undefined {
|
||||
if (!status) return detail || undefined;
|
||||
if (status.state === "cooling" && status.cooldownUntil) {
|
||||
return `冷却至 ${formatClock(status.cooldownUntil)}`;
|
||||
}
|
||||
return status.currentTitle || detail || undefined;
|
||||
}
|
||||
|
||||
export function formatCooldownRemaining(value: string): string {
|
||||
const d = new Date(value);
|
||||
if (Number.isNaN(d.getTime())) return value;
|
||||
const totalSeconds = Math.max(0, Math.ceil((d.getTime() - Date.now()) / 1000));
|
||||
const hours = Math.floor(totalSeconds / 3600);
|
||||
const minutes = Math.floor((totalSeconds % 3600) / 60);
|
||||
const seconds = totalSeconds % 60;
|
||||
if (hours > 0) return `${hours}小时${minutes}分`;
|
||||
if (minutes > 0) return `${minutes}分${seconds}秒`;
|
||||
return `${seconds}秒`;
|
||||
}
|
||||
|
||||
export function formatClock(value: string): string {
|
||||
const d = new Date(value);
|
||||
if (Number.isNaN(d.getTime())) return value;
|
||||
return d.toLocaleTimeString("zh-CN", { hour: "2-digit", minute: "2-digit" });
|
||||
}
|
||||
|
||||
export function defaultRootId(kind: Kind): string {
|
||||
if (kind === "pikpak") return "";
|
||||
if (kind === "onedrive") return "root";
|
||||
if (kind === "googledrive") return "root";
|
||||
if (kind === "localstorage") return "/";
|
||||
if (kind === "spider91") return "/";
|
||||
return "0";
|
||||
}
|
||||
|
||||
export function usesRootDirectoryID(kind: Kind): boolean {
|
||||
return kind !== "localstorage" && kind !== "spider91";
|
||||
}
|
||||
|
||||
export function rootIdPlaceholder(kind: Kind): string {
|
||||
const rootId = defaultRootId(kind);
|
||||
return rootId ? `默认:${rootId}` : "留空表示根目录";
|
||||
}
|
||||
|
||||
export function credentialHelp(kind: Kind, isEdit: boolean): string {
|
||||
const note = isEdit ? "如不修改凭证,留空即可,保存时会沿用旧值。" : "";
|
||||
switch (kind) {
|
||||
case "quark":
|
||||
return `在 pan.quark.cn 登录后,F12 → Network → 任意请求 → Request Headers 里复制整段 Cookie 粘贴到下方。${note}`;
|
||||
case "p115":
|
||||
return `登录 115.com 后复制 Cookie,形如 "UID=...; CID=...; SEID=...; KID=..."。${note}`;
|
||||
case "p123":
|
||||
return `推荐使用扫码登录自动获取 access_token;账号密码登录被 123 云盘风控拦截时,也可以只填写 access_token。播放走 302 跳转到 123 云盘返回的短期 CDN 地址。${note}`;
|
||||
case "pikpak":
|
||||
return `填写 PikPak 账号和密码即可。平台、设备 ID、验证码 token 和 refresh token 会由服务端自动处理并保存。${note}`;
|
||||
case "wopan":
|
||||
return `需要 access_token 和 refresh_token。后续会加扫码/短信登录入口,第一版只能手工粘贴。${note}`;
|
||||
case "onedrive":
|
||||
return `按 OpenList 默认应用在线挂载,只需要 refresh_token;保存时会自动刷新并保存 token。${note}`;
|
||||
case "googledrive":
|
||||
return isEdit
|
||||
? "请参考OpenList文档中关于谷歌云盘的配置方法;如不修改凭证,留空即可,保存时会沿用旧值"
|
||||
: "请参考OpenList文档中关于谷歌云盘的配置方法";
|
||||
case "localstorage":
|
||||
return `填写服务器可访问的本地目录绝对路径,例如 /mnt/videos。系统会扫描该目录及子目录中的视频文件和 .strm 文件;.strm 可指向 HTTP/HTTPS 直链,或指向本地存储根目录内的真实视频路径。Docker 部署时请填写容器内路径。${note}`;
|
||||
case "spider91":
|
||||
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;可按服务器网络情况单独配置代理。后续流水线会把较早的视频上传到你选择的 115 / PikPak / OneDrive 目标盘。";
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
export type CredentialField = {
|
||||
key: string;
|
||||
label: string;
|
||||
placeholder: string;
|
||||
type?: "text" | "select";
|
||||
options?: Array<{ value: string; label: string }>;
|
||||
multiline?: boolean;
|
||||
required?: boolean;
|
||||
defaultValue?: string;
|
||||
help?: string;
|
||||
};
|
||||
|
||||
export function credentialBoolValue(value: string | undefined, defaultValue: boolean): boolean {
|
||||
const normalized = (value ?? "").trim().toLowerCase();
|
||||
if (normalized === "") return defaultValue;
|
||||
if (normalized === "true" || normalized === "1" || normalized === "yes" || normalized === "on") return true;
|
||||
if (normalized === "false" || normalized === "0" || normalized === "no" || normalized === "off") return false;
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
export function googleDriveUsesOnlineAPI(creds: Record<string, string> = {}): boolean {
|
||||
return credentialBoolValue(creds.use_online_api, true);
|
||||
}
|
||||
|
||||
export function credentialFields(kind: Kind, creds: Record<string, string> = {}): CredentialField[] {
|
||||
switch (kind) {
|
||||
case "quark":
|
||||
return [
|
||||
{
|
||||
key: "cookie",
|
||||
label: "Cookie",
|
||||
placeholder: "__pus=...; __puus=...; ...",
|
||||
multiline: true,
|
||||
required: true,
|
||||
},
|
||||
];
|
||||
case "p115":
|
||||
return [
|
||||
{
|
||||
key: "cookie",
|
||||
label: "Cookie",
|
||||
placeholder: "UID=xxx; CID=xxx; SEID=xxx; KID=xxx",
|
||||
multiline: true,
|
||||
required: true,
|
||||
},
|
||||
];
|
||||
case "p123":
|
||||
return [
|
||||
{
|
||||
key: "username",
|
||||
label: "用户名 / 邮箱(可选)",
|
||||
placeholder: "user@example.com",
|
||||
},
|
||||
{
|
||||
key: "password",
|
||||
label: "密码(可选)",
|
||||
placeholder: "123 云盘密码",
|
||||
},
|
||||
{
|
||||
key: "access_token",
|
||||
label: "access_token(推荐用于风控场景)",
|
||||
placeholder: "Bearer eyJ... 或直接粘贴 token",
|
||||
multiline: true,
|
||||
help: "扫码成功后会自动填入该字段;如果 token 过期,重新扫码后保存即可。",
|
||||
},
|
||||
];
|
||||
case "pikpak":
|
||||
return [
|
||||
{
|
||||
key: "username",
|
||||
label: "用户名 / 邮箱",
|
||||
placeholder: "user@example.com",
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
key: "password",
|
||||
label: "密码",
|
||||
placeholder: "PikPak 密码",
|
||||
required: true,
|
||||
},
|
||||
];
|
||||
case "wopan":
|
||||
return [
|
||||
{
|
||||
key: "access_token",
|
||||
label: "access_token",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
key: "refresh_token",
|
||||
label: "refresh_token",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
key: "family_id",
|
||||
label: "family_id(家庭空间可选)",
|
||||
placeholder: "留空走个人空间",
|
||||
},
|
||||
];
|
||||
case "onedrive":
|
||||
return [
|
||||
{
|
||||
key: "refresh_token",
|
||||
label: "refresh_token",
|
||||
placeholder: "OpenList OneDrive refresh_token",
|
||||
multiline: true,
|
||||
required: true,
|
||||
},
|
||||
];
|
||||
case "googledrive":
|
||||
return [
|
||||
{
|
||||
key: "use_online_api",
|
||||
label: "认证方式",
|
||||
placeholder: "",
|
||||
type: "select",
|
||||
defaultValue: "true",
|
||||
options: [
|
||||
{ value: "true", label: "OpenList 在线 API" },
|
||||
{ value: "false", label: "自建 Google OAuth 客户端" },
|
||||
],
|
||||
},
|
||||
{
|
||||
key: "refresh_token",
|
||||
label: "refresh_token",
|
||||
placeholder: "OpenList Google Drive refresh_token",
|
||||
multiline: true,
|
||||
required: true,
|
||||
},
|
||||
...(googleDriveUsesOnlineAPI(creds)
|
||||
? []
|
||||
: [
|
||||
{
|
||||
key: "client_id",
|
||||
label: "客户端 ID",
|
||||
placeholder: "xxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.apps.googleusercontent.com",
|
||||
required: true,
|
||||
help: "Google Cloud Console 中 OAuth 2.0 客户端的 Client ID",
|
||||
},
|
||||
{
|
||||
key: "client_secret",
|
||||
label: "客户端密钥",
|
||||
placeholder: "Google OAuth client secret",
|
||||
required: true,
|
||||
help: "Google Cloud Console 中同一个 OAuth 客户端的 Client Secret",
|
||||
},
|
||||
]),
|
||||
];
|
||||
case "localstorage":
|
||||
return [
|
||||
{
|
||||
key: "path",
|
||||
label: "本地目录路径",
|
||||
placeholder: "/mnt/videos",
|
||||
required: true,
|
||||
help: "路径必须是后端服务器上的已有目录;保存后可手动重扫,系统会递归扫描支持的视频格式。",
|
||||
},
|
||||
];
|
||||
case "spider91":
|
||||
return [
|
||||
{
|
||||
key: "proxy",
|
||||
label: "代理地址(可选)",
|
||||
placeholder: "http://127.0.0.1:7890",
|
||||
help: "支持 http://、https://、socks5://、socks5h://代理",
|
||||
},
|
||||
];
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { ReactNode } from "react";
|
||||
import { ReactNode, useEffect, useState } from "react";
|
||||
import { TopBar } from "./TopBar";
|
||||
import { MainNav } from "./MainNav";
|
||||
import { SubNav } from "./SubNav";
|
||||
@@ -7,14 +7,84 @@ import { BackToTop } from "./BackToTop";
|
||||
|
||||
type Props = {
|
||||
children: ReactNode;
|
||||
mobileAutoHideNav?: boolean;
|
||||
};
|
||||
|
||||
export function AppShell({ children }: Props) {
|
||||
const MOBILE_NAV_QUERY = "(max-width: 768px)";
|
||||
const SCROLL_DELTA_THRESHOLD = 6;
|
||||
const HIDE_AFTER_SCROLL_Y = 56;
|
||||
|
||||
export function AppShell({ children, mobileAutoHideNav = false }: Props) {
|
||||
const [mobileNavHidden, setMobileNavHidden] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!mobileAutoHideNav) {
|
||||
setMobileNavHidden(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const mediaQuery = window.matchMedia(MOBILE_NAV_QUERY);
|
||||
let lastScrollY = Math.max(window.scrollY, 0);
|
||||
let ticking = false;
|
||||
|
||||
const showNav = () => setMobileNavHidden(false);
|
||||
|
||||
const updateNavVisibility = () => {
|
||||
ticking = false;
|
||||
const currentScrollY = Math.max(window.scrollY, 0);
|
||||
|
||||
if (!mediaQuery.matches || currentScrollY <= 0) {
|
||||
showNav();
|
||||
lastScrollY = currentScrollY;
|
||||
return;
|
||||
}
|
||||
|
||||
const delta = currentScrollY - lastScrollY;
|
||||
if (Math.abs(delta) < SCROLL_DELTA_THRESHOLD) return;
|
||||
|
||||
if (delta > 0 && currentScrollY > HIDE_AFTER_SCROLL_Y) {
|
||||
setMobileNavHidden(true);
|
||||
} else if (delta < 0) {
|
||||
showNav();
|
||||
}
|
||||
|
||||
lastScrollY = currentScrollY;
|
||||
};
|
||||
|
||||
const handleScroll = () => {
|
||||
if (ticking) return;
|
||||
ticking = true;
|
||||
window.requestAnimationFrame(updateNavVisibility);
|
||||
};
|
||||
|
||||
const handleMediaChange = () => {
|
||||
lastScrollY = Math.max(window.scrollY, 0);
|
||||
showNav();
|
||||
};
|
||||
|
||||
handleMediaChange();
|
||||
window.addEventListener("scroll", handleScroll, { passive: true });
|
||||
mediaQuery.addEventListener("change", handleMediaChange);
|
||||
|
||||
return () => {
|
||||
window.removeEventListener("scroll", handleScroll);
|
||||
mediaQuery.removeEventListener("change", handleMediaChange);
|
||||
};
|
||||
}, [mobileAutoHideNav]);
|
||||
|
||||
const className = [
|
||||
"app-shell",
|
||||
mobileAutoHideNav ? "app-shell--mobile-auto-hide-nav" : "",
|
||||
mobileNavHidden ? "is-mobile-nav-hidden" : "",
|
||||
].filter(Boolean).join(" ");
|
||||
|
||||
return (
|
||||
<div className="app-shell">
|
||||
<TopBar />
|
||||
<MainNav />
|
||||
<SubNav />
|
||||
<div className={className}>
|
||||
<div className="app-shell__nav-stack">
|
||||
<TopBar />
|
||||
<MainNav />
|
||||
<SubNav />
|
||||
</div>
|
||||
<main className="app-shell__main">{children}</main>
|
||||
<Footer />
|
||||
<BackToTop />
|
||||
|
||||
@@ -38,13 +38,11 @@ export function RecommendedRail({ videos }: Props) {
|
||||
return (
|
||||
<aside className="vd-rail" aria-label="推荐视频">
|
||||
<header className="vd-rail__head">
|
||||
<span className="vd-rail__head-bar" aria-hidden="true" />
|
||||
<div className="vd-rail__head-text">
|
||||
<h2 className="vd-rail__head-title">推荐视频</h2>
|
||||
<span className="vd-rail__head-sub">
|
||||
根据当前视频 · {videos.length} 条
|
||||
</span>
|
||||
</div>
|
||||
<span className="vd-rail__head-icon" aria-hidden="true">
|
||||
<span />
|
||||
<span />
|
||||
</span>
|
||||
<h2 className="vd-rail__head-title">推荐视频</h2>
|
||||
</header>
|
||||
<ul className="vd-rail__list">
|
||||
{videos.map((v) => (
|
||||
|
||||
@@ -10,11 +10,14 @@ export function TagCloud() {
|
||||
|
||||
useEffect(() => {
|
||||
let active = true;
|
||||
fetchTags().then((list) => {
|
||||
if (active) setTags(list);
|
||||
});
|
||||
const timer = window.setTimeout(() => {
|
||||
fetchTags().then((list) => {
|
||||
if (active) setTags(list);
|
||||
});
|
||||
}, 500);
|
||||
return () => {
|
||||
active = false;
|
||||
window.clearTimeout(timer);
|
||||
};
|
||||
}, []);
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useState } from "react";
|
||||
import { useEffect, useState } from "react";
|
||||
import { EyeOff, ThumbsDown, ThumbsUp } from "lucide-react";
|
||||
import type { VideoDetail } from "@/types";
|
||||
import { formatCount } from "@/lib/format";
|
||||
@@ -12,11 +12,11 @@ type Props = {
|
||||
/**
|
||||
* 视频操作工具条。
|
||||
* - 整体是一张浮起的圆角玻璃卡,比上一版的横线分隔更"成体"。
|
||||
* - 点赞 + 点踩组成一个胶囊(中间一道竖线分隔),两侧分别带计数。
|
||||
* - 点赞 + 点踩是两个独立按钮。
|
||||
* - "不再显示" 单独成一个次要按钮,hover 时露出 danger 色。
|
||||
*
|
||||
* 功能没变:
|
||||
* - 后端只有点赞接口(POST /api/video/:id/like),点踩仅本地 state。
|
||||
* - 后端只有点赞计数接口,点踩仅本地 state。
|
||||
* - 失败回滚已经处理。
|
||||
*/
|
||||
export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
@@ -25,11 +25,20 @@ export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
const [bursting, setBursting] = useState(false);
|
||||
const [liked, setLiked] = useState(false);
|
||||
const [disliked, setDisliked] = useState(false);
|
||||
const [likeSubmitted, setLikeSubmitted] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
setLikes(video.likes ?? 0);
|
||||
setDislikes(video.dislikes ?? 0);
|
||||
setBursting(false);
|
||||
setLiked(false);
|
||||
setDisliked(false);
|
||||
setLikeSubmitted(false);
|
||||
}, [video.id, video.likes, video.dislikes]);
|
||||
|
||||
async function handleLike() {
|
||||
if (liked) return;
|
||||
setLiked(true);
|
||||
setLikes((n) => n + 1);
|
||||
setBursting(true);
|
||||
window.setTimeout(() => setBursting(false), 320);
|
||||
|
||||
@@ -38,6 +47,11 @@ export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
setDislikes((n) => Math.max(0, n - 1));
|
||||
}
|
||||
|
||||
if (likeSubmitted) return;
|
||||
|
||||
setLikeSubmitted(true);
|
||||
setLikes((n) => n + 1);
|
||||
|
||||
try {
|
||||
const res = await fetch(
|
||||
`/api/video/${encodeURIComponent(video.id)}/like`,
|
||||
@@ -51,6 +65,7 @@ export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
} catch {
|
||||
setLikes((n) => Math.max(0, n - 1));
|
||||
setLiked(false);
|
||||
setLikeSubmitted(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,7 +79,6 @@ export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
setDislikes((n) => n + 1);
|
||||
if (liked) {
|
||||
setLiked(false);
|
||||
setLikes((n) => Math.max(0, n - 1));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,7 +97,6 @@ export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
<ThumbsUp size={18} fill={liked ? "currentColor" : "none"} />
|
||||
<span className="vd-actions__count">{formatCount(likes)}</span>
|
||||
</button>
|
||||
<span className="vd-actions__divider" aria-hidden="true" />
|
||||
<button
|
||||
type="button"
|
||||
className={`vd-actions__pill vd-actions__dislike${
|
||||
|
||||
@@ -12,6 +12,7 @@ import { PreviewVideo } from "./PreviewVideo";
|
||||
|
||||
type Props = {
|
||||
video: VideoItem;
|
||||
priority?: boolean;
|
||||
};
|
||||
|
||||
const HOVER_DELAY_MS = 300;
|
||||
@@ -24,7 +25,7 @@ function useActivePreviewId(): string | null {
|
||||
);
|
||||
}
|
||||
|
||||
export function VideoCard({ video }: Props) {
|
||||
export function VideoCard({ video, priority = false }: Props) {
|
||||
const [previewState, setPreviewState] = useState<PreviewState>("idle");
|
||||
const [shouldRenderPreview, setShouldRenderPreview] = useState(false);
|
||||
const [progress, setProgress] = useState(0); // 0~1
|
||||
@@ -204,7 +205,7 @@ export function VideoCard({ video }: Props) {
|
||||
className="thumb-image"
|
||||
src={thumbnailSrc}
|
||||
alt={video.title}
|
||||
loading="lazy"
|
||||
loading={priority ? "eager" : "lazy"}
|
||||
onError={handleThumbnailError}
|
||||
/>
|
||||
|
||||
@@ -291,6 +292,7 @@ function sourceKindFromLabel(label: string): string {
|
||||
const value = label.toLowerCase();
|
||||
if (value.includes("夸克") || value.includes("quark")) return "quark";
|
||||
if (value.includes("115") || value.includes("p115")) return "p115";
|
||||
if (value.includes("123") || value.includes("p123")) return "p123";
|
||||
if (value.includes("pikpak")) return "pikpak";
|
||||
if (value.includes("沃盘") || value.includes("wopan") || value.includes("联通")) return "wopan";
|
||||
if (value.includes("onedrive") || value.includes("one drive")) return "onedrive";
|
||||
|
||||
@@ -6,6 +6,7 @@ type Props = {
|
||||
loading?: boolean;
|
||||
compact?: boolean;
|
||||
emptyText?: string;
|
||||
priorityCount?: number;
|
||||
skeletonCount?: number;
|
||||
};
|
||||
|
||||
@@ -14,6 +15,7 @@ export function VideoGrid({
|
||||
loading,
|
||||
compact,
|
||||
emptyText = "暂时没有视频",
|
||||
priorityCount = 0,
|
||||
skeletonCount = 8,
|
||||
}: Props) {
|
||||
if (loading) {
|
||||
@@ -32,8 +34,8 @@ export function VideoGrid({
|
||||
|
||||
return (
|
||||
<div className={`video-grid ${compact ? "is-compact" : ""}`}>
|
||||
{(videos ?? []).map((v) => (
|
||||
<VideoCard key={v.id} video={v} />
|
||||
{(videos ?? []).map((v, index) => (
|
||||
<VideoCard key={v.id} video={v} priority={index < priorityCount} />
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useMemo, useState } from "react";
|
||||
import { Hash, Pencil, X } from "lucide-react";
|
||||
import { Pencil, Tag, X } from "lucide-react";
|
||||
import type { TagItem, VideoDetail } from "@/types";
|
||||
|
||||
type Props = {
|
||||
@@ -17,7 +17,7 @@ type Props = {
|
||||
* 视觉上和上一版的"两张分离卡"相比,整体感更强:
|
||||
* - 一张大卡内分两个小区块,区块之间用细分隔线
|
||||
* - 简介区块加 "简介" 标题前缀
|
||||
* - 标签区块加 # 图标暗示
|
||||
* - 标签区块加标签轮廓图标暗示
|
||||
*/
|
||||
export function VideoInfoPanel({
|
||||
video,
|
||||
@@ -99,7 +99,7 @@ export function VideoInfoPanel({
|
||||
<div className="vd-info__tags">
|
||||
<div className="vd-info__section-head">
|
||||
<span className="vd-info__section-title">
|
||||
<Hash size={14} aria-hidden="true" />
|
||||
<Tag size={15} strokeWidth={2} aria-hidden="true" />
|
||||
标签
|
||||
</span>
|
||||
{onTagsChange && (
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { CalendarDays, Clock3, Eye } from "lucide-react";
|
||||
import type { VideoDetail } from "@/types";
|
||||
import { formatCount } from "@/lib/format";
|
||||
|
||||
@@ -9,12 +10,11 @@ type Props = {
|
||||
* 详情页标题块。
|
||||
*
|
||||
* 视觉:
|
||||
* - 标题:大、粗、最高两行
|
||||
* - meta:作者首字头像 + 名字 + 一组小胶囊(来源、画质、时长、观看数、发布时间)
|
||||
* - meta:一组小胶囊(来源、画质、时长、观看数、发布时间)
|
||||
* 每个胶囊有自己的语义色彩,避免传统 "·" 分隔列表的列表感。
|
||||
* - 标题:大、粗、最高两行,位于 meta 下方
|
||||
*/
|
||||
export function VideoMetaHeader({ video }: Props) {
|
||||
const author = (video.author ?? "").trim();
|
||||
const source = (video.sourceLabel ?? "").trim();
|
||||
const quality = (video.quality ?? "").trim();
|
||||
const duration = (video.duration ?? "").trim();
|
||||
@@ -23,20 +23,7 @@ export function VideoMetaHeader({ video }: Props) {
|
||||
|
||||
return (
|
||||
<header className="vd-header">
|
||||
<h1 className="vd-header__title" title={video.title}>
|
||||
{video.title}
|
||||
</h1>
|
||||
|
||||
<div className="vd-header__row">
|
||||
{author && (
|
||||
<div className="vd-author" aria-label={`作者 ${author}`}>
|
||||
<span className="vd-author__avatar" aria-hidden="true">
|
||||
{author.slice(0, 1)}
|
||||
</span>
|
||||
<span className="vd-author__name">{author}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<ul className="vd-meta" aria-label="视频信息">
|
||||
{source && (
|
||||
<li className="vd-meta__chip" data-tone={sourceKind || "neutral"}>
|
||||
@@ -52,13 +39,28 @@ export function VideoMetaHeader({ video }: Props) {
|
||||
{quality}
|
||||
</li>
|
||||
)}
|
||||
{duration && <li className="vd-meta__chip">{duration}</li>}
|
||||
<li className="vd-meta__chip">
|
||||
{duration && (
|
||||
<li className="vd-meta__chip vd-meta__chip--plain">
|
||||
<Clock3 size={14} aria-hidden="true" />
|
||||
{duration}
|
||||
</li>
|
||||
)}
|
||||
<li className="vd-meta__chip vd-meta__chip--plain">
|
||||
<Eye size={14} aria-hidden="true" />
|
||||
<strong>{formatCount(video.views)}</strong> 次观看
|
||||
</li>
|
||||
{published && <li className="vd-meta__chip">{published}</li>}
|
||||
{published && (
|
||||
<li className="vd-meta__chip vd-meta__chip--plain">
|
||||
<CalendarDays size={14} aria-hidden="true" />
|
||||
{published}
|
||||
</li>
|
||||
)}
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<h1 className="vd-header__title" title={video.title}>
|
||||
{video.title}
|
||||
</h1>
|
||||
</header>
|
||||
);
|
||||
}
|
||||
@@ -68,6 +70,7 @@ function sourceKindFromLabel(label: string): string {
|
||||
const value = label.toLowerCase();
|
||||
if (value.includes("夸克") || value.includes("quark")) return "quark";
|
||||
if (value.includes("115") || value.includes("p115")) return "p115";
|
||||
if (value.includes("123") || value.includes("p123")) return "p123";
|
||||
if (value.includes("pikpak")) return "pikpak";
|
||||
if (value.includes("沃盘") || value.includes("wopan") || value.includes("联通"))
|
||||
return "wopan";
|
||||
|
||||
+1451
-133
File diff suppressed because it is too large
Load Diff
+30
-4
@@ -1,14 +1,19 @@
|
||||
import type { VideoDetail, VideoItem } from "@/types";
|
||||
|
||||
// 真实后端接口调用。未配置网盘时,各接口返回空数据。
|
||||
export function fetchHomeVideos(): Promise<VideoItem[]> {
|
||||
return apiGet<VideoItem[]>("/api/home").catch(() => []);
|
||||
export function fetchHomeVideos(excludeIds?: string[]): Promise<VideoItem[]> {
|
||||
const qs = new URLSearchParams();
|
||||
for (const id of excludeIds ?? []) {
|
||||
if (id.trim()) qs.append("exclude", id.trim());
|
||||
}
|
||||
const suffix = qs.toString() ? `?${qs.toString()}` : "";
|
||||
return apiGet<VideoItem[]>(`/api/home${suffix}`).catch(() => []);
|
||||
}
|
||||
|
||||
export function fetchListing(
|
||||
page: number,
|
||||
pageSize: number,
|
||||
params?: { q?: string; tag?: string; cat?: string; sort?: string }
|
||||
params?: { q?: string; tag?: string; cat?: string; sort?: string; includeTotal?: boolean }
|
||||
): Promise<{ items: VideoItem[]; total: number }> {
|
||||
const qs = new URLSearchParams({
|
||||
page: String(page),
|
||||
@@ -18,6 +23,7 @@ export function fetchListing(
|
||||
if (params?.tag) qs.set("tag", params.tag);
|
||||
if (params?.cat) qs.set("cat", params.cat);
|
||||
if (params?.sort) qs.set("sort", params.sort);
|
||||
if (params?.includeTotal === false) qs.set("count", "false");
|
||||
return apiGet<{ items: VideoItem[]; total: number }>(
|
||||
`/api/list?${qs.toString()}`
|
||||
).catch(() => ({ items: [], total: 0 }));
|
||||
@@ -73,8 +79,28 @@ export function uploadVideo(input: UploadVideoInput): Promise<VideoItem> {
|
||||
|
||||
export type TagItem = { id: string; label: string; count?: number };
|
||||
|
||||
const TAG_CACHE_TTL_MS = 30_000;
|
||||
let cachedTags: TagItem[] | null = null;
|
||||
let cachedTagsAt = 0;
|
||||
let pendingTags: Promise<TagItem[]> | null = null;
|
||||
|
||||
export function fetchTags(): Promise<TagItem[]> {
|
||||
return apiGet<TagItem[]>("/api/tags").catch(() => []);
|
||||
const now = Date.now();
|
||||
if (cachedTags && now - cachedTagsAt < TAG_CACHE_TTL_MS) {
|
||||
return Promise.resolve(cachedTags);
|
||||
}
|
||||
if (pendingTags) return pendingTags;
|
||||
pendingTags = apiGet<TagItem[]>("/api/tags")
|
||||
.then((tags) => {
|
||||
cachedTags = tags;
|
||||
cachedTagsAt = Date.now();
|
||||
return tags;
|
||||
})
|
||||
.catch(() => cachedTags ?? [])
|
||||
.finally(() => {
|
||||
pendingTags = null;
|
||||
});
|
||||
return pendingTags;
|
||||
}
|
||||
|
||||
/** 短视频模式单条记录。比 VideoItem 多 videoSrc / poster。 */
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user