mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-15 16:55:42 +08:00
Compare commits
29 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b8388eba59 | |||
| 76782f3801 | |||
| 1ae1408fb6 | |||
| 738406162a | |||
| 0f111b846d | |||
| 4dd9015bd7 | |||
| 84fbb6f51c | |||
| 992b20da93 | |||
| 1770693666 | |||
| 177041633a | |||
| ae324d3752 | |||
| 7f1e4eaa29 | |||
| 811d87cc27 | |||
| e4408f5655 | |||
| e93c906921 | |||
| 96e423b952 | |||
| a8ccc19e9e | |||
| 7ddf33d726 | |||
| c1355385e1 | |||
| ec5a01b6aa | |||
| 71d4a16db1 | |||
| 940e5dd76d | |||
| e826c05d5c | |||
| 3465b9e837 | |||
| d33c1b1b20 | |||
| 5fc8e9ebb7 | |||
| dc7d2a5de3 | |||
| 2f2bfbfcdc | |||
| 9def08b0c5 |
+15
@@ -35,3 +35,18 @@ tmp/
|
||||
91VideoSpider/__pycache__/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Local scratch images
|
||||
/*.png
|
||||
/*.jpg
|
||||
/*.jpeg
|
||||
/*.gif
|
||||
/*.webp
|
||||
/*.bmp
|
||||
/*.ico
|
||||
/image.jpg
|
||||
/image003.jpg
|
||||
/image004.jpg
|
||||
/image005.png
|
||||
/image006.png
|
||||
/image02.png
|
||||
|
||||
@@ -12,6 +12,9 @@
|
||||
pip install requests beautifulsoup4 lxml PySocks
|
||||
|
||||
使用方法:
|
||||
# 作为 video-site-91 通用爬虫脚本运行(后台会自动这样调用)
|
||||
python spider_91porn.py --job /path/to/job.json
|
||||
|
||||
# 全量爬取(默认行为,从 page=1 一直爬到末尾,写到 OUTPUT_FILE)
|
||||
python spider_91porn.py
|
||||
|
||||
@@ -22,6 +25,7 @@
|
||||
python spider_91porn.py --target-new 15 --seen-viewkeys-file /tmp/seen.txt --output /tmp/new.json
|
||||
|
||||
CLI 参数:
|
||||
--job FILE crawler.v1 job JSON 路径;后台爬虫管理会使用此模式
|
||||
--page N 只爬第 N 页,配合 --output 用于手动调试
|
||||
--target-new N 从 page 1 起翻页直到凑够 N 个新视频(不在 seen 列表里的)
|
||||
--seen-viewkeys-file FILE 每行一个已知 viewkey 或 mp4 源 ID,命中即跳过;与 --target-new 配合使用
|
||||
@@ -37,6 +41,8 @@ CLI 参数:
|
||||
- OUTPUT_FILE : 输出文件名
|
||||
|
||||
输出格式 (JSON):
|
||||
--job 模式下 stdout 输出 crawler.v1 JSON Lines,日志全部写到 stderr。
|
||||
手动运行模式仍会写传统 JSON 文件:
|
||||
{
|
||||
"videos": [
|
||||
{
|
||||
@@ -77,8 +83,8 @@ from datetime import datetime
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except ImportError:
|
||||
print("错误: 缺少依赖库 beautifulsoup4")
|
||||
print("请运行: pip install beautifulsoup4 lxml")
|
||||
print("错误: 缺少依赖库 beautifulsoup4", file=sys.stderr)
|
||||
print("请运行: pip install beautifulsoup4 lxml", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@@ -148,9 +154,24 @@ OUTPUT_FILE = "91porn_videos.json"
|
||||
MAX_PAGES = None # 设置为 None 爬取所有页,或设置整数如 5 只爬前5页
|
||||
RESUME = True # 是否跳过输出文件中已存在的 viewkey (断点续爬)
|
||||
MAX_EMPTY_PAGES = 2 # 连续空页数达到此值时停止爬取
|
||||
CRAWLER_NAME = "91Porn"
|
||||
CRAWLER_PROTOCOL = "crawler.v1"
|
||||
# ===================================================
|
||||
|
||||
|
||||
def crawler_source_id(raw: str) -> str:
|
||||
"""Return a backend-safe source_id, preserving existing numeric 91 IDs."""
|
||||
value = str(raw or "").strip()
|
||||
if not value:
|
||||
return ""
|
||||
safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", value).strip("._-")
|
||||
return safe[:160]
|
||||
|
||||
|
||||
def write_jsonl(event: dict):
|
||||
print(json.dumps(event, ensure_ascii=False), flush=True)
|
||||
|
||||
|
||||
class Porn91Spider:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -163,6 +184,7 @@ class Porn91Spider:
|
||||
target_new: int = None,
|
||||
seen_viewkeys: list = None,
|
||||
stream_output: bool = False,
|
||||
stream_protocol: str = "legacy",
|
||||
):
|
||||
"""
|
||||
构造函数。所有参数都有默认值,等同于使用脚本顶部的全局配置。
|
||||
@@ -198,6 +220,7 @@ class Porn91Spider:
|
||||
# (配合 backend Go 端 bufio.Scanner 实时消费,下载一个就开始下一个)。
|
||||
# 开启后所有 log 都走 stderr。
|
||||
self.stream_output = bool(stream_output)
|
||||
self.stream_protocol = stream_protocol or "legacy"
|
||||
|
||||
# 添加重试适配器
|
||||
try:
|
||||
@@ -263,7 +286,28 @@ class Porn91Spider:
|
||||
if not self.stream_output:
|
||||
return
|
||||
try:
|
||||
print(json.dumps(video, ensure_ascii=False), flush=True)
|
||||
if self.stream_protocol == "crawler.v1":
|
||||
source_id = crawler_source_id(video.get("source_id") or video.get("viewkey") or "")
|
||||
item = {
|
||||
"title": video.get("title") or "",
|
||||
"detail_url": video.get("detail_url") or "",
|
||||
"author": "91porn",
|
||||
"tags": ["91porn"],
|
||||
"media_url": video.get("video_url") or "",
|
||||
"thumbnail_url": video.get("thumb_url") or "",
|
||||
"headers": {
|
||||
"Referer": video.get("detail_url") or BASE_URL,
|
||||
},
|
||||
}
|
||||
if source_id:
|
||||
item["source_id"] = source_id
|
||||
event = {
|
||||
"type": "item",
|
||||
"item": item,
|
||||
}
|
||||
write_jsonl(event)
|
||||
else:
|
||||
print(json.dumps(video, ensure_ascii=False), flush=True)
|
||||
except Exception as e:
|
||||
# stdout 异常基本只在管道断开时发生(消费方进程死了);
|
||||
# 写到 stderr 让 backend 看到,然后让 crawl 循环自己 break。
|
||||
@@ -697,8 +741,9 @@ class Porn91Spider:
|
||||
except Exception as e:
|
||||
self.log(f"保存文件失败: {e}")
|
||||
# 尝试输出到控制台作为备份
|
||||
print("\n--- 备份输出 ---")
|
||||
print(json.dumps(output_data, ensure_ascii=False, indent=2))
|
||||
backup_out = sys.stderr if self.stream_output else sys.stdout
|
||||
print("\n--- 备份输出 ---", file=backup_out, flush=True)
|
||||
print(json.dumps(output_data, ensure_ascii=False, indent=2), file=backup_out, flush=True)
|
||||
|
||||
def _print_summary(self):
|
||||
"""
|
||||
@@ -751,6 +796,84 @@ def print_help():
|
||||
""")
|
||||
|
||||
|
||||
def run_job(job_path: str):
|
||||
"""Run as a crawler.v1 script plugin.
|
||||
|
||||
The Go host passes a job JSON file and expects stdout JSONL events. Logs go
|
||||
to stderr so stdout stays machine-readable.
|
||||
"""
|
||||
with open(job_path, "r", encoding="utf-8") as f:
|
||||
job = json.load(f)
|
||||
|
||||
if job.get("protocol") != CRAWLER_PROTOCOL:
|
||||
raise ValueError(f"unsupported crawler protocol: {job.get('protocol')!r}")
|
||||
if job.get("mode") not in ("", None, "crawl"):
|
||||
raise ValueError(f"unsupported crawler mode: {job.get('mode')!r}")
|
||||
|
||||
try:
|
||||
target_new = int(job.get("target_new") or 15)
|
||||
except (TypeError, ValueError):
|
||||
target_new = 15
|
||||
if target_new <= 0:
|
||||
target_new = 15
|
||||
seen_file = job.get("seen_source_ids_file") or ""
|
||||
output_dir = job.get("output_dir") or os.getcwd()
|
||||
run_id = job.get("run_id") or datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
output_file = os.path.join(output_dir, f"spider91-{run_id}.json")
|
||||
|
||||
network = job.get("network") if isinstance(job.get("network"), dict) else {}
|
||||
proxy_url = str(network.get("proxy_url") or "").strip()
|
||||
if proxy_url:
|
||||
os.environ["HTTP_PROXY"] = proxy_url
|
||||
os.environ["HTTPS_PROXY"] = proxy_url
|
||||
os.environ["http_proxy"] = proxy_url
|
||||
os.environ["https_proxy"] = proxy_url
|
||||
os.environ["NO_PROXY"] = ""
|
||||
os.environ["no_proxy"] = ""
|
||||
|
||||
seen_viewkeys = []
|
||||
if seen_file:
|
||||
try:
|
||||
with open(seen_file, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
seen_viewkeys.append(line)
|
||||
except FileNotFoundError:
|
||||
print(f"警告: seen_source_ids_file 不存在: {seen_file}", file=sys.stderr, flush=True)
|
||||
except Exception as e:
|
||||
print(f"警告: 读取 seen_source_ids_file 失败: {e}", file=sys.stderr, flush=True)
|
||||
|
||||
prefer_ipv4_for_plain_socks5_proxy()
|
||||
spider = Porn91Spider(
|
||||
output_file=output_file,
|
||||
start_page=1,
|
||||
max_pages=None,
|
||||
resume=False,
|
||||
quiet=True,
|
||||
target_new=target_new,
|
||||
seen_viewkeys=seen_viewkeys,
|
||||
stream_output=True,
|
||||
stream_protocol="crawler.v1",
|
||||
)
|
||||
try:
|
||||
spider.crawl()
|
||||
done = {
|
||||
"type": "done",
|
||||
"stats": {
|
||||
"emitted": spider.processed_videos,
|
||||
"failed": spider.failed_videos,
|
||||
"skipped": spider.skipped_videos,
|
||||
},
|
||||
}
|
||||
write_jsonl(done)
|
||||
except KeyboardInterrupt:
|
||||
spider.log("\n用户中断,正在保存已爬取的数据...")
|
||||
spider._save_results()
|
||||
raise
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) > 1 and sys.argv[1] in ('-h', '--help', 'help'):
|
||||
print_help()
|
||||
@@ -778,8 +901,14 @@ def main():
|
||||
parser.add_argument("--stream-output", action="store_true",
|
||||
help="流式模式:每解析一条视频直链就立即把它作为一行 JSON 写到 stdout 并 flush;"
|
||||
"日志改走 stderr。配合 backend 边读边下载使用。")
|
||||
parser.add_argument("--job", type=str, default=None,
|
||||
help="crawler.v1 job JSON 路径;作为通用脚本爬虫运行。")
|
||||
|
||||
args, _ = parser.parse_known_args()
|
||||
if args.job:
|
||||
run_job(args.job)
|
||||
return
|
||||
|
||||
cli_out = sys.stderr if args.stream_output else sys.stdout
|
||||
prefer_ipv4_for_plain_socks5_proxy()
|
||||
|
||||
|
||||
@@ -48,7 +48,6 @@ WORKDIR /opt/video-site-91
|
||||
COPY --from=backend /out/server ./server
|
||||
COPY --from=frontend /app/dist ./dist
|
||||
COPY backend/config.example.yaml ./config.example.yaml
|
||||
COPY 91VideoSpider/ ./91VideoSpider/
|
||||
COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
@@ -20,13 +20,13 @@
|
||||
|
||||
## 功能特性
|
||||
|
||||
- **多后端支持** — 兼容 115 云盘、PikPak 云盘、123云盘、OneDrive、Google Drive 和本地存储
|
||||
- **低带宽播放** — 115 云盘、PikPak 云盘、123云盘、OneDrive 都支持302模式,在线播放视频时,不占用服务器带宽,播放体验不受服务器带宽影响;Google Drive 不支持302模式,走服务器中转,观看体验会受服务器带宽影响
|
||||
- **多后端支持** — 兼容 115 云盘、PikPak 云盘、123网盘、联通网盘、OneDrive、Google Drive 和本地存储
|
||||
- **低带宽播放** — 115 云盘、PikPak 云盘、123网盘、联通网盘、OneDrive 支持302模式,在线播放视频时,不占用服务器带宽,播放体验不受服务器带宽影响;Google Drive 不支持302模式,走服务器中转,观看体验会受服务器带宽影响
|
||||
- **封面 & 预览片段** — 自动为每个视频生成封面图和预览片段,首页快速选片
|
||||
- **91 爬虫** — 内置爬虫,支持抓取 91 本月最热视频
|
||||
- **双主题** — 黑黄经典主题 / 粉白清新主题,随时切换
|
||||
- **爬虫脚本** — 项目支持导入自定义脚本,但是有一些规范,具体可以参考 [SpiderFor91](https://github.com/Just-Spider/SpiderFor91),项目不再内置任何爬虫脚本
|
||||
- **短视频模式** — 一键切换抖音风格,沉浸刷片
|
||||
- **低资源占用** — 2C2G 服务器稳定运行,主要性能消耗就是封面图和预览视频的生成
|
||||
- **兼容性转码** — 对 AVI、WMV、RMVB 等浏览器无法直接播放的视频,可在后台手动转码为 H.264 MP4
|
||||
- **视频管理** — 删除或隐藏的视频进入黑名单,支持在后台移出黑名单后重新扫盘入库
|
||||
|
||||
---
|
||||
|
||||
@@ -84,6 +84,14 @@ sudo bash install.sh
|
||||
|
||||
> `video-site-91` 为等效别名,两者可互换使用。
|
||||
|
||||
**已部署用户升级:**
|
||||
|
||||
```bash
|
||||
91 update
|
||||
```
|
||||
|
||||
升级会保留现有 `config.yaml`、数据库、封面、预览、上传文件和爬虫数据。脚本会自动安装或检查 `ffmpeg` / `ffprobe` 等运行依赖,并在新版本启动失败时回滚到升级前文件。
|
||||
|
||||
**自定义端口:**
|
||||
|
||||
```bash
|
||||
@@ -155,6 +163,7 @@ docker compose up -d # 更新并重启
|
||||
```
|
||||
|
||||
> 所有配置、数据库、封面、预览及上传文件均保存在 `./data/` 目录下。
|
||||
> 从旧版本升级 Docker 部署时,执行 `docker compose pull && docker compose up -d` 即可;`./data/` 不会被镜像更新覆盖。
|
||||
|
||||
---
|
||||
|
||||
@@ -180,14 +189,6 @@ docker compose up -d # 更新并重启
|
||||
|
||||
---
|
||||
|
||||
## 更多文档
|
||||
|
||||
| 文档 | 内容 |
|
||||
|------|------|
|
||||
| [backend/README.md](backend/README.md) | 后端实现、接口说明、网盘字段 |
|
||||
|
||||
---
|
||||
|
||||
## 使用须知
|
||||
|
||||
本项目面向**个人私有部署**,请仅接入你有权访问和管理的内容,并遵守对应网盘、站点的服务条款及所在地法律法规。
|
||||
@@ -196,6 +197,14 @@ docker compose up -d # 更新并重启
|
||||
|
||||
---
|
||||
|
||||
## PR提交规范
|
||||
欢迎大家提交PR,一起来完善这个项目,但是这里要说明一下PR提交的规范
|
||||
1. 一个PR的功能改动要单一,不建议一个PR修改了大量功能。单个PR单个功能修改,这样也更容易Merge
|
||||
2. 完善项目的PR比新增功能的PR更容易Merge(例如:例如你发现开发者没有实现爬取的视频上传到某个网盘,并且你有这个需求,此时你可以实现一下这个功能然后提交PR,也感谢你为开发者分担工作量)
|
||||
3. 新增功能的PR不容易Merge,因为某些功能的需求可能不是所有人都需要的,如果一味的不断增加功能,会让项目变得过于庞大。当然如果你肯定你的新功能和想法很好,并且相信将会对于项目有很大的改善,那么热烈欢迎你的PR
|
||||
|
||||
---
|
||||
|
||||
## 许可证
|
||||
|
||||
本项目基于 [MIT License](LICENSE) 开源。
|
||||
|
||||
+9
-9
@@ -2,7 +2,7 @@
|
||||
|
||||
视频聚合站的 Go 后端。提供三件事:
|
||||
|
||||
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive / Google Drive / 本地存储)
|
||||
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通网盘 / OneDrive / Google Drive / 本地存储)
|
||||
2. 视频元数据目录(SQLite)+ 扫描 + 预览视频预生成
|
||||
3. REST API(前台)+ 管理后台 + 直链代理
|
||||
4. 标签池、视频隐藏、按网盘统计和详情页来源网盘类型展示能力
|
||||
@@ -19,7 +19,7 @@ internal/
|
||||
quark/ 夸克(自己实现,参考 OpenList quark_uc)
|
||||
p115/ 115(壳子 + SheltonZhu/115driver)
|
||||
pikpak/ PikPak(自己实现,参考 OpenList pikpak)
|
||||
wopan/ 联通沃盘(壳子 + OpenListTeam/wopan-sdk-go)
|
||||
wopan/ 联通网盘(壳子 + OpenListTeam/wopan-sdk-go)
|
||||
onedrive/ OneDrive(OpenList 在线续期 + Microsoft Graph 文件接口)
|
||||
googledrive/ Google Drive(OpenList 在线续期 + Google Drive API;播放走后端代理)
|
||||
localstorage/ 本地目录扫描(服务器已有视频目录)
|
||||
@@ -109,7 +109,7 @@ go run ./cmd/server 后端 9192
|
||||
| pikpak | `username`、`password`(token、验证码和设备 ID 由服务端自动处理并保存) |
|
||||
| wopan | `access_token`、`refresh_token`,可选 `family_id` |
|
||||
| onedrive | `refresh_token` |
|
||||
| googledrive | `refresh_token` |
|
||||
| googledrive | 默认只需 `refresh_token`;自建 OAuth 客户端模式还需 `use_online_api=false`、`client_id`、`client_secret` |
|
||||
| localstorage | `path`(服务器上的已有视频目录,如 `/mnt/videos`) |
|
||||
|
||||
### PikPak 速度说明
|
||||
@@ -120,7 +120,7 @@ go run ./cmd/server 后端 9192
|
||||
|
||||
OneDrive 按 OpenList 默认应用方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。后台新建 OneDrive 时只需要填 OpenList 代刷得到的 `refresh_token`;服务端会默认挂载根目录并自动回写新 token。
|
||||
|
||||
Google Drive 按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/renewapi` 刷新 token。后台新建 Google Drive 时只需要填 OpenList Google Drive 获取到的 `refresh_token`。Google Drive 下载地址必须携带 `Authorization` 头,浏览器不能直接 302 使用,所以本站会由后端代理 `/p/stream` 播放,不加入零带宽 302 白名单。
|
||||
Google Drive 默认按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/renewapi` 刷新 token。后台新建 Google Drive 时只需要填 OpenList Google Drive 获取到的 `refresh_token`。如果不想依赖 OpenList 在线 API,可以关闭“使用 OpenList 在线续期 API”,并填写同一个 Google OAuth 客户端授权得到的 `refresh_token`、`client_id`、`client_secret`,服务端会直接请求 Google OAuth token 接口续期。Google Drive 下载地址必须携带 `Authorization` 头,浏览器不能直接 302 使用,所以本站会由后端代理 `/p/stream` 播放,不加入零带宽 302 白名单。
|
||||
|
||||
## 文件名约定
|
||||
|
||||
@@ -145,18 +145,18 @@ Google Drive 按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/ren
|
||||
|
||||
1. 同一网盘同一文件按 `(drive_id, file_id)` 形成稳定视频 ID,重复扫描只更新同一行。
|
||||
2. 扫描时优先按网盘侧 `content_hash` 去重;没有 hash 时退化为 `file_name + size_bytes`。
|
||||
3. 扫描、爬虫、本地上传或服务启动挂载网盘后,后台指纹 worker 会异步读取视频的少量 Range 片段,生成 `sampled_sha256`。前台列表、首页、搜索、推荐会按 `size_bytes + sampled_sha256` 只展示最早入库的 canonical 视频。
|
||||
3. 扫描、本地上传或服务启动挂载网盘后,后台指纹 worker 会异步读取视频的少量 Range 片段,生成 `sampled_sha256`。前台列表、首页、搜索、推荐会按 `size_bytes + sampled_sha256` 只展示最早入库的 canonical 视频。
|
||||
|
||||
`sampled_sha256` 是文件级去重:适合识别同一个视频文件被复制到 115 / PikPak / OneDrive 等不同网盘的情况。它不会删除任何网盘文件,也不用于识别转码、裁剪、加水印后的同源视频。
|
||||
`sampled_sha256` 是文件级去重:适合识别同一个视频文件被复制到 115 / PikPak / OneDrive / Google Drive 等不同网盘的情况。它不会删除任何网盘文件,也不用于识别转码、裁剪、加水印后的同源视频。
|
||||
|
||||
封面和预览视频仍然优先生成,不等待指纹完成。夜间流水线最后会做一次重复资产清理:对 `size_bytes + sampled_sha256` 命中的非 canonical 视频,只删除本机生成的重复封面和预览视频,并把对应字段重置为 `pending`。网盘原文件和视频元数据记录不会被删除;如果 canonical 视频以后被移除,这些重复项会重新进入生成队列。
|
||||
|
||||
## 管理能力
|
||||
|
||||
- `/admin/drives`:新增、编辑、删除网盘,触发扫描。
|
||||
- `/admin/videos`:按网盘筛选视频,每页 100 条分页,查看各网盘预览视频统计,编辑标题/作者/分类/标签,单条或全量重生预览视频。
|
||||
- `/admin/videos`:按网盘筛选视频,每页 100 条分页,查看各网盘预览视频统计,编辑标题/作者/分类/标签,单条或全量重生预览视频;拉黑视频页可查看被删除或被隐藏的视频,并支持移出黑名单后在下次扫盘重新入库。
|
||||
- `/admin/tags`:新增标签并用内置规则自动匹配已有视频;删除非系统标签时会从所有视频上同步移除该标签。
|
||||
- 播放页视频信息会展示来源网盘类型;同时提供“不再展示”,点击后会把视频标记为全局隐藏。隐藏视频不会再出现在首页、列表、搜索、相关推荐和详情接口中。目前没有管理后台恢复入口,如需恢复可把数据库里对应视频的 `hidden` 字段改回 `0`。
|
||||
- 播放页视频信息会展示来源网盘类型,并提供删除入口。被删除或被隐藏的视频会进入黑名单,不会再出现在首页、列表、搜索和详情接口中;在后台移出黑名单后,会在下次扫盘时重新发现并入库。
|
||||
|
||||
## 预览视频生成
|
||||
|
||||
@@ -170,7 +170,7 @@ ffmpeg -ss <起点> -headers "UA/Cookie/Referer" -i <直链> \
|
||||
|
||||
当前策略是每段固定 3 秒;30 秒以下最多 3 段,30 秒及以上固定 4 段;长视频在 20% 到 80% 区间均匀取段。生成的预览视频和封面都只保存在本地 `data/previews/`,不会回写到网盘;旧数据中的 `preview_file_id` 会被忽略。
|
||||
|
||||
服务启动或网盘重新挂载时,如果预览视频开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 扫盘和直链生成预览视频 / 封面时可能触发 Microsoft Graph 429、`TooManyRequests`、`activityLimitReached` 或 throttled 文本;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。扫盘阶段会按 `Retry-After` 或默认冷却时间等待后继续当前目录。
|
||||
服务启动或网盘重新挂载时,如果预览视频开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 扫盘和直链生成预览视频 / 封面时可能触发 Microsoft Graph 429、`TooManyRequests`、`activityLimitReached` 或 throttled 文本;Google Drive 可能返回 429、`usageLimits`、`userRateLimitExceeded`、`downloadQuotaExceeded` 等限制标识。后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。扫盘阶段会按 `Retry-After` 或默认冷却时间等待后继续当前目录。
|
||||
|
||||
前端卡片的 `previewSrc` 统一指向 `/p/preview/<videoID>`,后端只从本地 `preview_local` 文件读取。
|
||||
|
||||
|
||||
+963
-185
File diff suppressed because it is too large
Load Diff
@@ -40,6 +40,7 @@ func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
|
||||
reg.Set("p115-one", &spider91UploadTargetFakeDrive{id: "p115-one", kind: "p115"})
|
||||
reg.Set("p123-one", &spider91UploadTargetFakeDrive{id: "p123-one", kind: "p123"})
|
||||
reg.Set("onedrive-one", &spider91UploadTargetFakeDrive{id: "onedrive-one", kind: "onedrive"})
|
||||
reg.Set("wopan-one", &spider91UploadTargetFakeDrive{id: "wopan-one", kind: "wopan"})
|
||||
|
||||
app := &App{registry: reg}
|
||||
if got := app.Spider91UploadDriveID(); got != "" {
|
||||
@@ -61,6 +62,11 @@ func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
|
||||
t.Fatalf("explicit onedrive upload target = %q, want onedrive-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "wopan-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "wopan-one" {
|
||||
t.Fatalf("explicit wopan upload target = %q, want wopan-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "missing"
|
||||
if got := app.Spider91UploadDriveID(); got != "" {
|
||||
t.Fatalf("missing upload target = %q, want empty", got)
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -13,6 +14,7 @@ import (
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/config"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/scriptcrawler"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/fingerprint"
|
||||
"github.com/video-site/backend/internal/preview"
|
||||
@@ -260,6 +262,7 @@ func TestStopDriveTasksCancelsQueuedTasksAndReplacesWorkers(t *testing.T) {
|
||||
"drive-id": func() { close(oldCanceled) },
|
||||
},
|
||||
scanQueued: map[string]bool{"drive-id": true},
|
||||
scanProgress: map[string]driveScanProgress{"drive-id": {Scanned: 8, Added: 2}},
|
||||
fingerprintQueueing: map[string]bool{"drive-id": true},
|
||||
}
|
||||
taskCtx, done := app.registerDriveTaskContext(ctx, "drive-id")
|
||||
@@ -279,6 +282,9 @@ func TestStopDriveTasksCancelsQueuedTasksAndReplacesWorkers(t *testing.T) {
|
||||
if app.scanQueued["drive-id"] {
|
||||
t.Fatal("scan queue marker was not cleared")
|
||||
}
|
||||
if _, ok := app.scanProgress["drive-id"]; ok {
|
||||
t.Fatal("scan progress marker was not cleared")
|
||||
}
|
||||
if app.fingerprintQueueing["drive-id"] {
|
||||
t.Fatal("fingerprint queue marker was not cleared")
|
||||
}
|
||||
@@ -304,6 +310,196 @@ func TestStopDriveTasksCancelsQueuedTasksAndReplacesWorkers(t *testing.T) {
|
||||
newCancel()
|
||||
}
|
||||
|
||||
func TestScheduleScanRejectsDriveWithActiveGenerationWork(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
thumbWorker := preview.NewThumbWorker(&serverFakeTeaserGenerator{}, nil, &serverFakeDrive{})
|
||||
if !thumbWorker.Enqueue(&catalog.Video{ID: "busy-video", DriveID: "drive-id", Title: "Busy Video"}) {
|
||||
t.Fatal("failed to enqueue busy thumbnail task")
|
||||
}
|
||||
app := &App{
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{"drive-id": thumbWorker},
|
||||
}
|
||||
|
||||
if app.scheduleScan(ctx, "drive-id") {
|
||||
t.Fatal("scheduleScan accepted a drive with active generation work")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleScanRunsDifferentDrivesConcurrently(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
seedDriveWithTeaser(t, cat, "drive-a", true)
|
||||
seedDriveWithTeaser(t, cat, "drive-b", true)
|
||||
|
||||
started := make(chan string, 2)
|
||||
release := make(chan struct{})
|
||||
registry := proxy.NewRegistry()
|
||||
registry.Set("drive-a", &serverBlockingListDrive{id: "drive-a", started: started, release: release})
|
||||
registry.Set("drive-b", &serverBlockingListDrive{id: "drive-b", started: started, release: release})
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{
|
||||
Scanner: config.Scanner{VideoExtensions: []string{".mp4"}},
|
||||
},
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
}
|
||||
|
||||
if !app.scheduleScan(ctx, "drive-a") {
|
||||
t.Fatal("scheduleScan drive-a was rejected")
|
||||
}
|
||||
if !app.scheduleScan(ctx, "drive-b") {
|
||||
t.Fatal("scheduleScan drive-b was rejected")
|
||||
}
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
deadline := time.After(time.Second)
|
||||
for len(seen) < 2 {
|
||||
select {
|
||||
case id := <-started:
|
||||
seen[id] = struct{}{}
|
||||
case <-deadline:
|
||||
close(release)
|
||||
t.Fatalf("started drives = %#v, want both drives before releasing List", seen)
|
||||
}
|
||||
}
|
||||
close(release)
|
||||
}
|
||||
|
||||
func TestDriveGenerationStatusIncludesScanState(t *testing.T) {
|
||||
app := &App{
|
||||
scanQueued: map[string]bool{"drive-id": true},
|
||||
scanProgress: map[string]driveScanProgress{"drive-id": {Scanned: 12, Added: 3}},
|
||||
}
|
||||
|
||||
status := app.driveGenerationStatuses()["drive-id"].Scan
|
||||
if status.State != "scanning" {
|
||||
t.Fatalf("scan status = %#v, want scanning", status)
|
||||
}
|
||||
if status.ScannedCount != 12 || status.AddedCount != 3 {
|
||||
t.Fatalf("scan counts = scanned %d added %d, want 12 and 3", status.ScannedCount, status.AddedCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSpider91MigrationAfterManualCrawlRequiresConfiguredUploadTarget(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
registry := proxy.NewRegistry()
|
||||
migrator := &serverFakeSpider91MigrationRunner{}
|
||||
app := &App{
|
||||
registry: registry,
|
||||
spider91Migrator: migrator,
|
||||
workers: map[string]*preview.Worker{},
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{},
|
||||
fingerprintWorkers: map[string]*fingerprint.Worker{},
|
||||
}
|
||||
|
||||
app.runSpider91MigrationAfterManualCrawl(ctx, "91spider")
|
||||
if migrator.called != 0 {
|
||||
t.Fatalf("migration called without upload target")
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "pikpak"
|
||||
app.runSpider91MigrationAfterManualCrawl(ctx, "91spider")
|
||||
if migrator.called != 0 {
|
||||
t.Fatalf("migration called when upload target is not attached")
|
||||
}
|
||||
|
||||
registry.Set("pikpak", &serverFakeKindDrive{id: "pikpak", kind: "pikpak"})
|
||||
app.runSpider91MigrationAfterManualCrawl(ctx, "91spider")
|
||||
if migrator.called != 1 {
|
||||
t.Fatalf("migration calls = %d, want 1", migrator.called)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleCrawlerUploadMigrationRunsForConfiguredCrawler(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "crawler-truvaze",
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Truvaze",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"script_path": "/tmp/Truvaze.py",
|
||||
"upload_drive_id": "pikpak",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed crawler: %v", err)
|
||||
}
|
||||
registry := proxy.NewRegistry()
|
||||
registry.Set("crawler-truvaze", &serverFakeKindDrive{id: "crawler-truvaze", kind: scriptcrawler.Kind})
|
||||
migrator := &serverFakeSpider91MigrationRunner{}
|
||||
app := &App{
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
spider91Migrator: migrator,
|
||||
workers: map[string]*preview.Worker{},
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{},
|
||||
fingerprintWorkers: map[string]*fingerprint.Worker{},
|
||||
}
|
||||
|
||||
if !app.scheduleCrawlerUploadMigration(ctx, "crawler-truvaze") {
|
||||
t.Fatal("scheduleCrawlerUploadMigration returned false, want true")
|
||||
}
|
||||
deadline := time.After(time.Second)
|
||||
for migrator.called == 0 {
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatalf("migration calls = %d, want 1", migrator.called)
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleCrawlerUploadMigrationSkipsWithoutUploadTarget(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "crawler-local",
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Local Only",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{"script_path": "/tmp/local.py"},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed crawler: %v", err)
|
||||
}
|
||||
migrator := &serverFakeSpider91MigrationRunner{}
|
||||
app := &App{cat: cat, registry: proxy.NewRegistry(), spider91Migrator: migrator}
|
||||
|
||||
if app.scheduleCrawlerUploadMigration(ctx, "crawler-local") {
|
||||
t.Fatal("scheduleCrawlerUploadMigration returned true without upload target")
|
||||
}
|
||||
if migrator.called != 0 {
|
||||
t.Fatalf("migration calls = %d, want 0", migrator.called)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriveGenerationStatusUsesWorkerQueueNotPendingCatalogRows(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -491,7 +687,9 @@ func TestNightlyTargetsComeFromCatalogBeforeDriveAttach(t *testing.T) {
|
||||
for _, d := range []*catalog.Drive{
|
||||
{ID: "115", Kind: "p115", Name: "115", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "pikpak", Kind: "pikpak", Name: "PikPak", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "91-spider", Kind: "spider91", Name: "91 Spider", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "91-legacy", Kind: "spider91", Name: "91 Legacy", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "91-crawler", Kind: scriptcrawler.Kind, Name: "91 Spider", RootID: "/", Credentials: map[string]string{"script_path": "/tmp/crawler.py"}, TeaserEnabled: true},
|
||||
{ID: "91-crawler-deleted", Kind: scriptcrawler.Kind, Name: "Deleted Spider", RootID: "/", Credentials: map[string]string{}, TeaserEnabled: true},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
t.Fatalf("seed drive %s: %v", d.ID, err)
|
||||
@@ -504,8 +702,47 @@ func TestNightlyTargetsComeFromCatalogBeforeDriveAttach(t *testing.T) {
|
||||
t.Fatalf("scan target ids = %#v, want 115 and pikpak from catalog", scanIDs)
|
||||
}
|
||||
spiderIDs := app.listSpider91DriveIDs(ctx)
|
||||
if len(spiderIDs) != 1 || spiderIDs[0] != "91-spider" {
|
||||
t.Fatalf("spider91 ids = %#v, want catalog spider drive", spiderIDs)
|
||||
if len(spiderIDs) != 1 || spiderIDs[0] != "91-crawler" {
|
||||
t.Fatalf("spider91 ids = %#v, want crawler-page script drive", spiderIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAttachDriveRejectsLegacySpider91Storage(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
d := &catalog.Drive{
|
||||
ID: "91-legacy",
|
||||
Kind: spider91.Kind,
|
||||
Name: "91 Legacy",
|
||||
RootID: "/",
|
||||
TeaserEnabled: true,
|
||||
}
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
app := &App{cat: cat, registry: proxy.NewRegistry()}
|
||||
err = app.attachDrive(ctx, d)
|
||||
if err == nil || !strings.Contains(err.Error(), "爬虫管理") {
|
||||
t.Fatalf("attach err = %v, want crawler management guidance", err)
|
||||
}
|
||||
if _, ok := app.registry.Get(d.ID); ok {
|
||||
t.Fatal("legacy spider91 drive should not be registered")
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, d.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.Status != "error" || !strings.Contains(got.LastError, "爬虫管理") {
|
||||
t.Fatalf("status/error = %q/%q, want deprecated error", got.Status, got.LastError)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -918,7 +1155,6 @@ func TestCleanupDriveVideosForDeleteRemovesRowsAndGeneratedAssetsOnly(t *testing
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
fingerprintWorkers: make(map[string]*fingerprint.Worker),
|
||||
spider91Crawlers: make(map[string]*spider91.Crawler),
|
||||
}
|
||||
removed, err := app.cleanupDriveVideosForDelete(ctx, "local-main")
|
||||
if err != nil {
|
||||
@@ -1007,7 +1243,7 @@ func TestDeleteVideoRemovesGeneratedAssetsKeepsLocalOriginalAndTombstones(t *tes
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
}
|
||||
result, err := app.deleteVideo(ctx, "localstorage-local-main-file")
|
||||
result, err := app.deleteVideo(ctx, "localstorage-local-main-file", false)
|
||||
if err != nil {
|
||||
t.Fatalf("delete video: %v", err)
|
||||
}
|
||||
@@ -1034,6 +1270,126 @@ func TestDeleteVideoRemovesGeneratedAssetsKeepsLocalOriginalAndTombstones(t *tes
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteVideoRemovesSourceFileWhenRequested(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
localDir := filepath.Join(root, "previews")
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
previewPath := filepath.Join(localDir, "video-with-source.mp4")
|
||||
thumbPath := filepath.Join(localDir, "thumbs", "video-with-source.jpg")
|
||||
for _, path := range []string{previewPath, thumbPath} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("file"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "video-with-source",
|
||||
DriveID: "source-drive",
|
||||
FileID: "source-file",
|
||||
FileName: "clip.mp4",
|
||||
Title: "Source File",
|
||||
PreviewLocal: previewPath,
|
||||
PreviewStatus: "ready",
|
||||
ThumbnailURL: "/p/thumb/video-with-source",
|
||||
Size: 123,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
registry := proxy.NewRegistry()
|
||||
drv := &serverRemovableFakeDrive{id: "source-drive"}
|
||||
registry.Set(drv.ID(), drv)
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
}
|
||||
result, err := app.deleteVideo(ctx, "video-with-source", true)
|
||||
if err != nil {
|
||||
t.Fatalf("delete video: %v", err)
|
||||
}
|
||||
if !result.OK || !result.DeletedSource {
|
||||
t.Fatalf("delete result = %#v, want source deleted", result)
|
||||
}
|
||||
if got, want := drv.removedFileID, "source-file"; got != want {
|
||||
t.Fatalf("removed source fileID = %q, want %q", got, want)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "video-with-source"); err != sql.ErrNoRows {
|
||||
t.Fatalf("deleted video lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
for _, path := range []string{previewPath, thumbPath} {
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("generated asset %s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteVideoUsesSourceRemoverWithCatalogMetadata(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "video-with-rich-source",
|
||||
DriveID: "source-drive",
|
||||
FileID: "source-fid",
|
||||
ParentID: "parent-dir",
|
||||
FileName: "clip.mp4",
|
||||
Title: "Source File",
|
||||
Size: 123,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
registry := proxy.NewRegistry()
|
||||
drv := &serverSourceRemovableFakeDrive{id: "source-drive"}
|
||||
registry.Set(drv.ID(), drv)
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: filepath.Join(t.TempDir(), "previews")}},
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
}
|
||||
result, err := app.deleteVideo(ctx, "video-with-rich-source", true)
|
||||
if err != nil {
|
||||
t.Fatalf("delete video: %v", err)
|
||||
}
|
||||
if !result.OK || !result.DeletedSource {
|
||||
t.Fatalf("delete result = %#v, want source deleted", result)
|
||||
}
|
||||
if drv.fallbackRemoveCalled {
|
||||
t.Fatal("fallback Remove was called, want SourceRemover")
|
||||
}
|
||||
want := drives.SourceFile{
|
||||
FileID: "source-fid",
|
||||
ParentID: "parent-dir",
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
}
|
||||
if drv.removedSource != want {
|
||||
t.Fatalf("removed source = %#v, want %#v", drv.removedSource, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteVideoRemovesSpider91SourceFile(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
@@ -1090,7 +1446,7 @@ func TestDeleteVideoRemovesSpider91SourceFile(t *testing.T) {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
result, err := app.deleteVideo(ctx, "spider91-spider-main-source")
|
||||
result, err := app.deleteVideo(ctx, "spider91-spider-main-source", true)
|
||||
if err != nil {
|
||||
t.Fatalf("delete spider video: %v", err)
|
||||
}
|
||||
@@ -1198,7 +1554,6 @@ func TestCleanupDriveVideosForDeleteSpider91RemovesCrawledDirAndOriginRecords(t
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
fingerprintWorkers: make(map[string]*fingerprint.Worker),
|
||||
spider91Crawlers: make(map[string]*spider91.Crawler),
|
||||
}
|
||||
removed, err := app.cleanupDriveVideosForDelete(ctx, driveID)
|
||||
if err != nil {
|
||||
@@ -1496,6 +1851,103 @@ func (d *serverFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
}
|
||||
func (d *serverFakeDrive) RootID() string { return "root" }
|
||||
|
||||
type serverFakeKindDrive struct {
|
||||
serverFakeDrive
|
||||
id string
|
||||
kind string
|
||||
}
|
||||
|
||||
func (d *serverFakeKindDrive) Kind() string { return d.kind }
|
||||
func (d *serverFakeKindDrive) ID() string { return d.id }
|
||||
|
||||
type serverRemovableFakeDrive struct {
|
||||
serverFakeDrive
|
||||
id string
|
||||
removedFileID string
|
||||
}
|
||||
|
||||
func (d *serverRemovableFakeDrive) Kind() string { return "fake-removable" }
|
||||
func (d *serverRemovableFakeDrive) ID() string { return d.id }
|
||||
func (d *serverRemovableFakeDrive) Remove(ctx context.Context, fileID string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
d.removedFileID = fileID
|
||||
return nil
|
||||
}
|
||||
|
||||
type serverSourceRemovableFakeDrive struct {
|
||||
serverFakeDrive
|
||||
id string
|
||||
removedSource drives.SourceFile
|
||||
fallbackRemoveCalled bool
|
||||
}
|
||||
|
||||
func (d *serverSourceRemovableFakeDrive) Kind() string { return "fake-source-removable" }
|
||||
func (d *serverSourceRemovableFakeDrive) ID() string { return d.id }
|
||||
func (d *serverSourceRemovableFakeDrive) RemoveSource(ctx context.Context, source drives.SourceFile) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
d.removedSource = source
|
||||
return nil
|
||||
}
|
||||
func (d *serverSourceRemovableFakeDrive) Remove(ctx context.Context, fileID string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
d.fallbackRemoveCalled = true
|
||||
return nil
|
||||
}
|
||||
|
||||
type serverFakeSpider91MigrationRunner struct {
|
||||
called int
|
||||
}
|
||||
|
||||
func (r *serverFakeSpider91MigrationRunner) RunOnce(context.Context) error {
|
||||
r.called++
|
||||
return nil
|
||||
}
|
||||
|
||||
type serverBlockingListDrive struct {
|
||||
id string
|
||||
started chan string
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (d *serverBlockingListDrive) Kind() string { return "fake" }
|
||||
func (d *serverBlockingListDrive) ID() string { return d.id }
|
||||
func (d *serverBlockingListDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *serverBlockingListDrive) List(ctx context.Context, _ string) ([]drives.Entry, error) {
|
||||
if d.started != nil {
|
||||
select {
|
||||
case d.started <- d.id:
|
||||
default:
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-d.release:
|
||||
return nil, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
func (d *serverBlockingListDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *serverBlockingListDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
|
||||
return &drives.StreamLink{URL: "https://video.example/clip.mp4"}, nil
|
||||
}
|
||||
func (d *serverBlockingListDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *serverBlockingListDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *serverBlockingListDrive) RootID() string { return "root" }
|
||||
|
||||
type serverFingerprintFakeDrive struct {
|
||||
serverFakeDrive
|
||||
path string
|
||||
|
||||
@@ -36,11 +36,8 @@ scanner:
|
||||
video_extensions: [".mp4", ".mkv", ".mov", ".webm", ".avi", ".strm"]
|
||||
|
||||
nightly:
|
||||
# 凌晨流水线触发整点(0-23),默认 1 即每天 01:00。流程:
|
||||
# Phase 1 扫所有非 spider91 / 非 localupload 网盘 → 检测新增 / 删除
|
||||
# → 入队封面和预览视频 → 等所有队列 idle
|
||||
# Phase 2 spider91 爬虫(如配置)→ 入队预览视频 → 等队列 idle
|
||||
# Phase 3 spider91 → 云盘迁移(一次性 sweep)
|
||||
# 凌晨流水线触发整点(0-23),默认 1 即每天 01:00。
|
||||
# 运行时会统一编排扫描、媒体资产生成和后续清理任务。
|
||||
cron_hour: 1
|
||||
# 单次流水线总耗时上限(软超时);超过后当前 phase 跑完不启动后续 phase。
|
||||
max_duration: 6h
|
||||
@@ -74,6 +71,11 @@ preview:
|
||||
# root_id: "root"
|
||||
# params:
|
||||
# refresh_token: "..."
|
||||
# # 默认 use_online_api=true,会使用 OpenList 在线续期 API。
|
||||
# # 如需使用自己创建的 Google OAuth 客户端,取消下面三行注释:
|
||||
# # use_online_api: "false"
|
||||
# # client_id: "..."
|
||||
# # client_secret: "..."
|
||||
# 本地存储示例:
|
||||
# - id: "local-media"
|
||||
# kind: "localstorage"
|
||||
|
||||
+1047
-13
File diff suppressed because it is too large
Load Diff
+1143
-88
File diff suppressed because it is too large
Load Diff
+113
-44
@@ -11,6 +11,7 @@ import (
|
||||
"io"
|
||||
"math/rand/v2"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
@@ -54,6 +55,10 @@ type Server struct {
|
||||
LocalDir string
|
||||
UploadDir string
|
||||
OnVideoUploaded func(*catalog.Video)
|
||||
// OnHideVideo 处理前台「不再展示」。隐藏机制已废弃,改走拉黑逻辑:
|
||||
// 删除库中记录 + 本地封面/预览,保留网盘源文件,并写黑名单墓碑
|
||||
// (扫盘不再入库)。未注入时回退为旧的 hidden 标记。
|
||||
OnHideVideo func(ctx context.Context, videoID string) error
|
||||
|
||||
tagCacheMu sync.Mutex
|
||||
tagCacheUntil time.Time
|
||||
@@ -146,7 +151,7 @@ func (s *Server) RegisterRoutes(r chi.Router, a *auth.Authenticator) {
|
||||
r.Post("/api/shorts/next", s.handleShortsNext)
|
||||
|
||||
// 代理路由同样需要鉴权,防止绕过
|
||||
r.Get("/p/stream/{driveID}/{fileID}", s.handleStream)
|
||||
r.Get("/p/stream/{driveID}/*", s.handleStream)
|
||||
r.Get("/p/upload/{videoID}", s.handleUploadedVideo)
|
||||
r.Get("/p/spider91/{videoID}", s.handleSpider91Video)
|
||||
r.Get("/p/preview/{videoID}", s.handlePreview)
|
||||
@@ -313,7 +318,7 @@ func (s *Server) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleVideoDetail(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
id := routeParam(r, "id")
|
||||
v, err := s.Catalog.GetVideo(r.Context(), id)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusNotFound, err)
|
||||
@@ -343,7 +348,7 @@ func (s *Server) handleVideoDetail(w http.ResponseWriter, r *http.Request) {
|
||||
VideoSrc: s.videoSource(v),
|
||||
Poster: thumbnailURL(v),
|
||||
Description: v.Description,
|
||||
EmbedURL: fmt.Sprintf(`<iframe src="/embed/%s" width="640" height="360" frameborder="0" allowfullscreen></iframe>`, v.ID),
|
||||
EmbedURL: fmt.Sprintf(`<iframe src="/embed/%s" width="640" height="360" frameborder="0" allowfullscreen></iframe>`, pathSegment(v.ID)),
|
||||
AuthorProfile: AuthorProfile{
|
||||
ID: "author-" + v.Author,
|
||||
Name: v.Author,
|
||||
@@ -525,11 +530,9 @@ func (s *Server) handleTags(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// shortsNextReq 客户端把当前轮已看过的 video id 列表传上来。
|
||||
// PreferredFromVideoID 来自短视频页最近一次点赞成功的视频,用于优先推荐相似标签。
|
||||
type shortsNextReq struct {
|
||||
SeenIDs []string `json:"seenIds"`
|
||||
Count int `json:"count"`
|
||||
PreferredFromVideoID string `json:"preferredFromVideoId"`
|
||||
SeenIDs []string `json:"seenIds"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// ShortsItemDTO 是短视频流单条的精简结构。比 VideoDTO 多 videoSrc / poster,
|
||||
@@ -547,8 +550,8 @@ type ShortsItemDTO struct {
|
||||
// - 服务器从未在 seenIds 中的可见视频里随机抽至多 count 条返回
|
||||
// - 当返回数量 < count 且小于全库可见总数时,说明本轮即将结束,
|
||||
// 返回 roundComplete=true,前端应在用户看完返回的这些后清空本地已看记录开新一轮
|
||||
// - 当 seenIds 已经覆盖全库时,本接口直接返回新一轮的随机一批
|
||||
// (传 seenIds=[] 即可让客户端在轮次完成后重新开始)
|
||||
// - 当 seenIds 真实覆盖当前全部可见视频时,本接口直接返回新一轮的随机一批
|
||||
// (不能仅看 seenIds 长度,里面可能有隐藏、删除或历史脏 ID)
|
||||
func (s *Server) handleShortsNext(w http.ResponseWriter, r *http.Request) {
|
||||
var body shortsNextReq
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
|
||||
@@ -569,22 +572,18 @@ func (s *Server) handleShortsNext(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// 如果客户端已看记录已经 ≥ 全库,则视为新一轮,直接忽略 seenIds
|
||||
exclude := body.SeenIDs
|
||||
if total > 0 && len(exclude) >= total {
|
||||
exclude = nil
|
||||
}
|
||||
|
||||
var items []*catalog.Video
|
||||
if strings.TrimSpace(body.PreferredFromVideoID) != "" {
|
||||
items, err = s.Catalog.RandomVideosForPreferredVideoExcluding(r.Context(), body.PreferredFromVideoID, exclude, count)
|
||||
} else {
|
||||
items, err = s.Catalog.RandomVideosExcluding(r.Context(), exclude, count)
|
||||
}
|
||||
items, err := s.Catalog.RandomVideosExcluding(r.Context(), body.SeenIDs, count)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
if total > 0 && len(items) == 0 && len(body.SeenIDs) > 0 {
|
||||
items, err = s.Catalog.RandomVideosExcluding(r.Context(), nil, count)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 注入 sourceLabel 以便前端展示来源网盘
|
||||
driveLabels := make(map[string]string)
|
||||
@@ -622,7 +621,7 @@ type updateVideoTagsReq struct {
|
||||
}
|
||||
|
||||
func (s *Server) handleUpdateVideoTags(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
id := routeParam(r, "id")
|
||||
var body updateVideoTagsReq
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
writeErr(w, http.StatusBadRequest, err)
|
||||
@@ -645,7 +644,7 @@ func (s *Server) handleUpdateVideoTags(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleLike(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
id := routeParam(r, "id")
|
||||
likes, err := s.Catalog.IncrementLike(r.Context(), id)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
@@ -657,7 +656,7 @@ func (s *Server) handleLike(w http.ResponseWriter, r *http.Request) {
|
||||
// handleUnlike 取消点赞:likes - 1(保底 0)。
|
||||
// 短视频模式中爱心按钮点击切换状态时使用。
|
||||
func (s *Server) handleUnlike(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
id := routeParam(r, "id")
|
||||
likes, err := s.Catalog.DecrementLike(r.Context(), id)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
@@ -671,7 +670,7 @@ func (s *Server) handleUnlike(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleView(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
id := routeParam(r, "id")
|
||||
views, err := s.Catalog.IncrementView(r.Context(), id)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
@@ -685,8 +684,15 @@ func (s *Server) handleView(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleHideVideo(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
if err := s.Catalog.HideVideo(r.Context(), id); err != nil {
|
||||
id := routeParam(r, "id")
|
||||
var err error
|
||||
if s.OnHideVideo != nil {
|
||||
// 走拉黑逻辑:删记录 + 删本地封面/预览 + 写墓碑,保留网盘源文件。
|
||||
err = s.OnHideVideo(r.Context(), id)
|
||||
} else {
|
||||
err = s.Catalog.HideVideo(r.Context(), id)
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
writeErr(w, http.StatusNotFound, err)
|
||||
return
|
||||
@@ -802,12 +808,12 @@ func (s *Server) handleUploadVideo(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleStream(w http.ResponseWriter, r *http.Request) {
|
||||
driveID := chi.URLParam(r, "driveID")
|
||||
fileID := chi.URLParam(r, "fileID")
|
||||
driveID := routeParam(r, "driveID")
|
||||
fileID := routeWildcardParam(r, "*")
|
||||
s.Proxy.ServeStream(w, r, driveID, fileID)
|
||||
}
|
||||
func (s *Server) handleUploadedVideo(w http.ResponseWriter, r *http.Request) {
|
||||
videoID := chi.URLParam(r, "videoID")
|
||||
videoID := routeParam(r, "videoID")
|
||||
v, err := s.Catalog.GetVideo(r.Context(), videoID)
|
||||
if err != nil || v.Hidden || v.DriveID != localUploadDriveID {
|
||||
http.NotFound(w, r)
|
||||
@@ -831,7 +837,7 @@ func (s *Server) handleUploadedVideo(w http.ResponseWriter, r *http.Request) {
|
||||
// 路径形如 /p/spider91/<videoID>,videoID = "spider91-<driveID>-<sourceID>"。
|
||||
// 通过 catalog 拿到 file_id("<sourceID>.mp4"),再让 driver 解析到绝对路径并 ServeFile。
|
||||
func (s *Server) handleSpider91Video(w http.ResponseWriter, r *http.Request) {
|
||||
videoID := chi.URLParam(r, "videoID")
|
||||
videoID := routeParam(r, "videoID")
|
||||
v, err := s.Catalog.GetVideo(r.Context(), videoID)
|
||||
if err != nil || v.Hidden {
|
||||
http.NotFound(w, r)
|
||||
@@ -866,7 +872,7 @@ func (s *Server) handleSpider91Video(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) {
|
||||
videoID := chi.URLParam(r, "videoID")
|
||||
videoID := routeParam(r, "videoID")
|
||||
v, err := s.Catalog.GetVideo(r.Context(), videoID)
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
@@ -891,7 +897,7 @@ func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleThumb(w http.ResponseWriter, r *http.Request) {
|
||||
videoID := chi.URLParam(r, "videoID")
|
||||
videoID := routeParam(r, "videoID")
|
||||
var clean string
|
||||
for _, path := range mediaasset.ThumbnailPathCandidates(s.LocalDir, videoID) {
|
||||
candidate := filepath.Clean(path)
|
||||
@@ -926,7 +932,7 @@ func mapVideo(v *catalog.Video) VideoDTO {
|
||||
}
|
||||
return VideoDTO{
|
||||
ID: v.ID,
|
||||
Href: "/video/" + v.ID,
|
||||
Href: "/video/" + pathSegment(v.ID),
|
||||
Title: v.Title,
|
||||
Thumbnail: thumbnailURL(v),
|
||||
PreviewSrc: previewURL(v),
|
||||
@@ -948,7 +954,7 @@ func mapVideo(v *catalog.Video) VideoDTO {
|
||||
}
|
||||
|
||||
func previewURL(v *catalog.Video) string {
|
||||
base := "/p/preview/" + v.ID
|
||||
base := "/p/preview/" + pathSegment(v.ID)
|
||||
if v.UpdatedAt.IsZero() {
|
||||
return base
|
||||
}
|
||||
@@ -956,9 +962,12 @@ func previewURL(v *catalog.Video) string {
|
||||
}
|
||||
|
||||
func thumbnailURL(v *catalog.Video) string {
|
||||
base := "/p/thumb/" + v.ID
|
||||
base := "/p/thumb/" + pathSegment(v.ID)
|
||||
if v.ThumbnailURL != "" {
|
||||
base = v.ThumbnailURL
|
||||
if thumbnailURLMatchesVideoID(base, v.ID) {
|
||||
base = "/p/thumb/" + pathSegment(v.ID)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(base, "/p/thumb/") || v.UpdatedAt.IsZero() {
|
||||
return base
|
||||
@@ -966,25 +975,85 @@ func thumbnailURL(v *catalog.Video) string {
|
||||
return base + "?v=" + strconv.FormatInt(v.UpdatedAt.UnixMilli(), 10)
|
||||
}
|
||||
|
||||
// transcodedSource 在视频有就绪的浏览器兼容性转码产物时返回产物的播放地址。
|
||||
// 产物和原始文件在同一个 drive 上,走同一条 /p/stream 代理/302 链路。
|
||||
func transcodedSource(v *catalog.Video) (string, bool) {
|
||||
if v.TranscodeStatus == "ready" && v.TranscodedFileID != "" && v.DriveID != localUploadDriveID {
|
||||
return fmt.Sprintf("/p/stream/%s/%s", pathSegment(v.DriveID), pathSegment(v.TranscodedFileID)), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func (s *Server) videoSource(v *catalog.Video) string {
|
||||
if v.DriveID == localUploadDriveID {
|
||||
return "/p/upload/" + v.ID
|
||||
return "/p/upload/" + pathSegment(v.ID)
|
||||
}
|
||||
if s.Proxy != nil && s.Proxy.Registry != nil {
|
||||
if d, ok := s.Proxy.Registry.Get(v.DriveID); ok && d.Kind() == spider91.Kind {
|
||||
return "/p/spider91/" + v.ID
|
||||
if d, ok := s.Proxy.Registry.Get(v.DriveID); ok {
|
||||
switch d.Kind() {
|
||||
case spider91.Kind:
|
||||
return "/p/spider91/" + pathSegment(v.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("/p/stream/%s/%s", v.DriveID, v.FileID)
|
||||
if src, ok := transcodedSource(v); ok {
|
||||
return src
|
||||
}
|
||||
return fmt.Sprintf("/p/stream/%s/%s", pathSegment(v.DriveID), pathSegment(v.FileID))
|
||||
}
|
||||
|
||||
// videoSource 兼容旧调用点,没有 server context 时按之前逻辑回退到 /p/stream。
|
||||
// 内部新增的代码请使用 (*Server).videoSource。
|
||||
func videoSource(v *catalog.Video) string {
|
||||
if v.DriveID == localUploadDriveID {
|
||||
return "/p/upload/" + v.ID
|
||||
return "/p/upload/" + pathSegment(v.ID)
|
||||
}
|
||||
return fmt.Sprintf("/p/stream/%s/%s", v.DriveID, v.FileID)
|
||||
if src, ok := transcodedSource(v); ok {
|
||||
return src
|
||||
}
|
||||
return fmt.Sprintf("/p/stream/%s/%s", pathSegment(v.DriveID), pathSegment(v.FileID))
|
||||
}
|
||||
|
||||
func pathSegment(value string) string {
|
||||
return url.PathEscape(value)
|
||||
}
|
||||
|
||||
func routeParam(r *http.Request, key string) string {
|
||||
value := chi.URLParam(r, key)
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
if decoded, err := url.PathUnescape(value); err == nil {
|
||||
return decoded
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func routeWildcardParam(r *http.Request, key string) string {
|
||||
value := chi.URLParam(r, key)
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
value = strings.TrimPrefix(value, "/")
|
||||
if decoded, err := url.PathUnescape(value); err == nil {
|
||||
return decoded
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func thumbnailURLMatchesVideoID(value, videoID string) bool {
|
||||
if !strings.HasPrefix(value, "/p/thumb/") {
|
||||
return false
|
||||
}
|
||||
tail := strings.TrimPrefix(value, "/p/thumb/")
|
||||
if idx := strings.IndexByte(tail, '?'); idx >= 0 {
|
||||
tail = tail[:idx]
|
||||
}
|
||||
if tail == videoID {
|
||||
return true
|
||||
}
|
||||
decoded, err := url.PathUnescape(tail)
|
||||
return err == nil && decoded == videoID
|
||||
}
|
||||
|
||||
func driveKindLabel(kind string) string {
|
||||
@@ -994,11 +1063,11 @@ func driveKindLabel(kind string) string {
|
||||
case "p115":
|
||||
return "115 网盘"
|
||||
case "p123":
|
||||
return "123 云盘"
|
||||
return "123网盘"
|
||||
case "pikpak":
|
||||
return "PikPak"
|
||||
case "wopan":
|
||||
return "联通沃盘"
|
||||
return "联通网盘"
|
||||
case "onedrive":
|
||||
return "OneDrive"
|
||||
case "googledrive":
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
)
|
||||
@@ -66,6 +68,68 @@ func TestVideoSourceKeepsDirectStreamForMp4(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoURLsEscapePathSegments(t *testing.T) {
|
||||
updated := time.UnixMilli(1778863000123)
|
||||
v := &catalog.Video{
|
||||
ID: "wopan-drive-fid/with space",
|
||||
DriveID: "drive-1",
|
||||
FileID: "fid/with space",
|
||||
Title: "Video",
|
||||
UpdatedAt: updated,
|
||||
}
|
||||
|
||||
dto := mapVideo(v)
|
||||
if dto.Href != "/video/wopan-drive-fid%2Fwith%20space" {
|
||||
t.Fatalf("href = %q, want escaped video id", dto.Href)
|
||||
}
|
||||
if dto.PreviewSrc != "/p/preview/wopan-drive-fid%2Fwith%20space?v=1778863000123" {
|
||||
t.Fatalf("preview = %q, want escaped video id", dto.PreviewSrc)
|
||||
}
|
||||
if dto.Thumbnail != "/p/thumb/wopan-drive-fid%2Fwith%20space?v=1778863000123" {
|
||||
t.Fatalf("thumbnail = %q, want escaped video id", dto.Thumbnail)
|
||||
}
|
||||
if got := videoSource(v); got != "/p/stream/drive-1/fid%2Fwith%20space" {
|
||||
t.Fatalf("video source = %q, want escaped file id", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailURLRewritesStoredLocalURLForUnsafeVideoID(t *testing.T) {
|
||||
got := thumbnailURL(&catalog.Video{
|
||||
ID: "wopan-drive-fid/with space",
|
||||
ThumbnailURL: "/p/thumb/wopan-drive-fid/with space",
|
||||
UpdatedAt: time.UnixMilli(1778863000123),
|
||||
})
|
||||
|
||||
if got != "/p/thumb/wopan-drive-fid%2Fwith%20space?v=1778863000123" {
|
||||
t.Fatalf("thumbnail URL = %q, want escaped local URL", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStreamDecodesEscapedWildcardFileID(t *testing.T) {
|
||||
local := filepath.Join(t.TempDir(), "video.mp4")
|
||||
if err := os.WriteFile(local, []byte("ok"), 0o644); err != nil {
|
||||
t.Fatalf("write local video: %v", err)
|
||||
}
|
||||
drv := &apiStreamFakeDrive{localPath: local}
|
||||
reg := proxy.NewRegistry()
|
||||
reg.Set("drive-1", drv)
|
||||
srv := &Server{Proxy: proxy.New(reg)}
|
||||
|
||||
router := chi.NewRouter()
|
||||
router.Get("/p/stream/{driveID}/*", srv.handleStream)
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/drive-1/fid%2Fwith%20space", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
router.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if drv.fileID != "fid/with space" {
|
||||
t.Fatalf("fileID = %q, want decoded original", drv.fileID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoSourceUsesLocalUploadRoute(t *testing.T) {
|
||||
v := &catalog.Video{
|
||||
ID: "video-1",
|
||||
@@ -100,6 +164,49 @@ func TestPreviewURLFallsBackWithoutUpdatedAt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleVideoDetailDecodesEscapedVideoID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "wopan-drive-fid/with space",
|
||||
DriveID: "drive-1",
|
||||
FileID: "fid/with space",
|
||||
Title: "Video",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
router := chi.NewRouter()
|
||||
router.Get("/api/video/{id}", (&Server{Catalog: cat}).handleVideoDetail)
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/video/wopan-drive-fid%2Fwith%20space", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
router.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got VideoDetailDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.ID != "wopan-drive-fid/with space" {
|
||||
t.Fatalf("id = %q, want original video id", got.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailURLVersionsLocalGeneratedThumbnails(t *testing.T) {
|
||||
got := thumbnailURL(&catalog.Video{
|
||||
ID: "video-1",
|
||||
@@ -703,7 +810,7 @@ func TestHandleTagsReturnsUnifiedTagPool(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleShortsNextUsesPreferredVideoLeastPopulatedTag(t *testing.T) {
|
||||
func TestHandleShortsNextReturnsRandomBatchExcludingSeen(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -727,7 +834,7 @@ func TestHandleShortsNextUsesPreferredVideoLeastPopulatedTag(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/shorts/next", strings.NewReader(`{"seenIds":["current"],"count":3,"preferredFromVideoId":"current"}`))
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/shorts/next", strings.NewReader(`{"seenIds":["current"],"count":3}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&Server{Catalog: cat}).handleShortsNext(rr, req)
|
||||
|
||||
@@ -750,10 +857,7 @@ func TestHandleShortsNextUsesPreferredVideoLeastPopulatedTag(t *testing.T) {
|
||||
t.Fatalf("total = %d, want 4", got.Total)
|
||||
}
|
||||
if got.RoundComplete {
|
||||
t.Fatalf("roundComplete = true, want false with fallback-filled batch")
|
||||
}
|
||||
if !containsString(ids, "rare-1") {
|
||||
t.Fatalf("ids = %#v, want rare-1 from least populated tag", ids)
|
||||
t.Fatalf("roundComplete = true, want false with a full remaining batch")
|
||||
}
|
||||
if containsString(ids, "current") {
|
||||
t.Fatalf("ids = %#v, should exclude current", ids)
|
||||
@@ -761,6 +865,76 @@ func TestHandleShortsNextUsesPreferredVideoLeastPopulatedTag(t *testing.T) {
|
||||
if len(ids) != 3 {
|
||||
t.Fatalf("ids = %#v, want 3 items", ids)
|
||||
}
|
||||
for _, want := range []string{"common-1", "common-2", "rare-1"} {
|
||||
if !containsString(ids, want) {
|
||||
t.Fatalf("ids = %#v, want remaining id %s", ids, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleShortsNextDoesNotResetForStaleSeenIDs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*catalog.Video{
|
||||
{ID: "seen-1", DriveID: "drive", FileID: "f-seen-1", Title: "seen 1", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "fresh-1", DriveID: "drive", FileID: "f-fresh-1", Title: "fresh 1", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "fresh-2", DriveID: "drive", FileID: "f-fresh-2", Title: "fresh 2", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "hidden-1", DriveID: "drive", FileID: "f-hidden-1", Title: "hidden 1", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
if err := cat.HideVideo(ctx, "hidden-1"); err != nil {
|
||||
t.Fatalf("hide hidden-1: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/shorts/next", strings.NewReader(`{"seenIds":["seen-1","hidden-1","deleted-stale"],"count":3}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&Server{Catalog: cat}).handleShortsNext(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
Items []ShortsItemDTO `json:"items"`
|
||||
Total int `json:"total"`
|
||||
RoundComplete bool `json:"roundComplete"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
ids := make([]string, 0, len(got.Items))
|
||||
for _, item := range got.Items {
|
||||
ids = append(ids, item.ID)
|
||||
}
|
||||
if got.Total != 3 {
|
||||
t.Fatalf("total = %d, want 3", got.Total)
|
||||
}
|
||||
if !got.RoundComplete {
|
||||
t.Fatalf("roundComplete = false, want true after returning all unviewed visible videos")
|
||||
}
|
||||
if containsString(ids, "seen-1") || containsString(ids, "hidden-1") {
|
||||
t.Fatalf("ids = %#v, should not reset and return seen or hidden videos", ids)
|
||||
}
|
||||
for _, want := range []string{"fresh-1", "fresh-2"} {
|
||||
if !containsString(ids, want) {
|
||||
t.Fatalf("ids = %#v, want %s", ids, want)
|
||||
}
|
||||
}
|
||||
if len(ids) != 2 {
|
||||
t.Fatalf("ids = %#v, want exactly the two unviewed visible videos", ids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpdateVideoTagsRejectsUnknownTags(t *testing.T) {
|
||||
@@ -1084,6 +1258,37 @@ func sameStringSet(a, b []string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
type apiStreamFakeDrive struct {
|
||||
localPath string
|
||||
fileID string
|
||||
}
|
||||
|
||||
func (d *apiStreamFakeDrive) Kind() string { return "fake" }
|
||||
func (d *apiStreamFakeDrive) ID() string { return "drive-1" }
|
||||
func (d *apiStreamFakeDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *apiStreamFakeDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *apiStreamFakeDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *apiStreamFakeDrive) StreamURL(_ context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
d.fileID = fileID
|
||||
return &drives.StreamLink{
|
||||
URL: d.localPath,
|
||||
Expires: time.Now().Add(time.Minute),
|
||||
}, nil
|
||||
}
|
||||
func (d *apiStreamFakeDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *apiStreamFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *apiStreamFakeDrive) RootID() string { return "root" }
|
||||
|
||||
func requestWithVideoID(method, target, videoID string, body *strings.Reader) *http.Request {
|
||||
return requestWithRouteParam(method, target, "id", videoID, body)
|
||||
}
|
||||
|
||||
+519
-195
@@ -20,6 +20,15 @@ type Catalog struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
type CrawlerAssetCounts struct {
|
||||
Total int
|
||||
Local int
|
||||
Migrated int
|
||||
Thumbnail DriveThumbnailCounts
|
||||
Teaser DriveTeaserCounts
|
||||
Fingerprint DriveFingerprintCounts
|
||||
}
|
||||
|
||||
func Open(path string) (*Catalog, error) {
|
||||
db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
@@ -42,43 +51,54 @@ func (c *Catalog) Close() error { return c.db.Close() }
|
||||
// ---------- Video ----------
|
||||
|
||||
type Video struct {
|
||||
ID string `json:"id"`
|
||||
DriveID string `json:"driveId"`
|
||||
FileID string `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
ContentHash string `json:"contentHash"`
|
||||
SampledSHA256 string `json:"sampledSha256"`
|
||||
FingerprintStatus string `json:"fingerprintStatus"`
|
||||
FingerprintError string `json:"fingerprintError"`
|
||||
ParentID string `json:"parentId"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Tags []string `json:"tags"`
|
||||
DurationSeconds int `json:"durationSeconds"`
|
||||
Size int64 `json:"size"`
|
||||
Ext string `json:"ext"`
|
||||
Quality string `json:"quality"`
|
||||
ThumbnailURL string `json:"thumbnailUrl"`
|
||||
PreviewFileID string `json:"previewFileId"`
|
||||
PreviewLocal string `json:"previewLocal"`
|
||||
PreviewStatus string `json:"previewStatus"`
|
||||
Views int `json:"views"`
|
||||
Favorites int `json:"favorites"`
|
||||
Comments int `json:"comments"`
|
||||
Likes int `json:"likes"`
|
||||
Dislikes int `json:"dislikes"`
|
||||
Category string `json:"category"`
|
||||
Hidden bool `json:"hidden"`
|
||||
Badges []string `json:"badges"`
|
||||
Description string `json:"description"`
|
||||
PublishedAt time.Time `json:"publishedAt"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
ID string `json:"id"`
|
||||
DriveID string `json:"driveId"`
|
||||
FileID string `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
ContentHash string `json:"contentHash"`
|
||||
SampledSHA256 string `json:"sampledSha256"`
|
||||
FingerprintStatus string `json:"fingerprintStatus"`
|
||||
FingerprintError string `json:"fingerprintError"`
|
||||
ParentID string `json:"parentId"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Tags []string `json:"tags"`
|
||||
DurationSeconds int `json:"durationSeconds"`
|
||||
Size int64 `json:"size"`
|
||||
Ext string `json:"ext"`
|
||||
Quality string `json:"quality"`
|
||||
ThumbnailURL string `json:"thumbnailUrl"`
|
||||
PreviewFileID string `json:"previewFileId"`
|
||||
PreviewLocal string `json:"previewLocal"`
|
||||
PreviewStatus string `json:"previewStatus"`
|
||||
// TranscodeStatus:浏览器兼容性转码状态。
|
||||
// ''=未检测 / pending=已入队 / ready=已转码 / skipped=无需转码 / failed=失败。
|
||||
TranscodeStatus string `json:"transcodeStatus"`
|
||||
TranscodeError string `json:"transcodeError"`
|
||||
TranscodedFileID string `json:"transcodedFileId"`
|
||||
TranscodedSize int64 `json:"transcodedSize"`
|
||||
Views int `json:"views"`
|
||||
Favorites int `json:"favorites"`
|
||||
Comments int `json:"comments"`
|
||||
Likes int `json:"likes"`
|
||||
Dislikes int `json:"dislikes"`
|
||||
Category string `json:"category"`
|
||||
Hidden bool `json:"hidden"`
|
||||
Badges []string `json:"badges"`
|
||||
Description string `json:"description"`
|
||||
PublishedAt time.Time `json:"publishedAt"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
func (c *Catalog) UpsertVideo(ctx context.Context, v *Video) error {
|
||||
existed := c.videoExists(ctx, v.ID)
|
||||
v.ContentHash = normalizeContentHash(v.ContentHash)
|
||||
v.SampledSHA256 = normalizeContentHash(v.SampledSHA256)
|
||||
fingerprintStatus := nullableStatus(v.FingerprintStatus)
|
||||
if v.SampledSHA256 != "" && (v.FingerprintStatus == "" || v.FingerprintStatus == "pending") {
|
||||
fingerprintStatus = "ready"
|
||||
}
|
||||
tagsJSON, _ := json.Marshal(v.Tags)
|
||||
badgesJSON, _ := json.Marshal(v.Badges)
|
||||
now := time.Now().UnixMilli()
|
||||
@@ -89,13 +109,13 @@ func (c *Catalog) UpsertVideo(ctx context.Context, v *Video) error {
|
||||
|
||||
_, err := c.db.ExecContext(ctx, `
|
||||
INSERT INTO videos (
|
||||
id, drive_id, file_id, file_name, content_hash, parent_id, title, author, tags,
|
||||
id, drive_id, file_id, file_name, content_hash, sampled_sha256, fingerprint_status, fingerprint_error, parent_id, title, author, tags,
|
||||
duration_seconds, size_bytes, ext, quality, thumbnail_url, thumbnail_status,
|
||||
preview_file_id, preview_local, preview_status,
|
||||
views, favorites, comments, likes, dislikes,
|
||||
category, hidden, badges, description, published_at, created_at, updated_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, CASE WHEN COALESCE(?, '') != '' THEN 'ready' ELSE 'pending' END,
|
||||
?, ?, ?,
|
||||
?, ?, ?, ?, ?,
|
||||
@@ -114,15 +134,18 @@ ON CONFLICT(id) DO UPDATE SET
|
||||
ELSE videos.content_hash
|
||||
END,
|
||||
sampled_sha256 = CASE
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN ''
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN excluded.sampled_sha256
|
||||
WHEN excluded.sampled_sha256 != '' THEN excluded.sampled_sha256
|
||||
ELSE videos.sampled_sha256
|
||||
END,
|
||||
fingerprint_status = CASE
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN 'pending'
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN COALESCE(excluded.fingerprint_status, 'pending')
|
||||
WHEN excluded.sampled_sha256 != '' THEN COALESCE(excluded.fingerprint_status, 'ready')
|
||||
ELSE COALESCE(videos.fingerprint_status, 'pending')
|
||||
END,
|
||||
fingerprint_error = CASE
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN ''
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN COALESCE(excluded.fingerprint_error, '')
|
||||
WHEN excluded.sampled_sha256 != '' THEN COALESCE(excluded.fingerprint_error, '')
|
||||
ELSE COALESCE(videos.fingerprint_error, '')
|
||||
END,
|
||||
duration_seconds= excluded.duration_seconds,
|
||||
@@ -143,7 +166,7 @@ ON CONFLICT(id) DO UPDATE SET
|
||||
description = excluded.description,
|
||||
updated_at = excluded.updated_at
|
||||
`,
|
||||
v.ID, v.DriveID, v.FileID, v.FileName, v.ContentHash, v.ParentID, v.Title, v.Author, string(tagsJSON),
|
||||
v.ID, v.DriveID, v.FileID, v.FileName, v.ContentHash, v.SampledSHA256, fingerprintStatus, v.FingerprintError, v.ParentID, v.Title, v.Author, string(tagsJSON),
|
||||
v.DurationSeconds, v.Size, v.Ext, v.Quality, v.ThumbnailURL, v.ThumbnailURL,
|
||||
v.PreviewFileID, v.PreviewLocal, nullableStatus(v.PreviewStatus),
|
||||
v.Views, v.Favorites, v.Comments, v.Likes, v.Dislikes,
|
||||
@@ -173,6 +196,84 @@ func (c *Catalog) UpdatePreview(ctx context.Context, id, previewLocal, status st
|
||||
return err
|
||||
}
|
||||
|
||||
// transcodeCandidateWhereSQL 圈定"可能需要浏览器兼容性转码"的视频:
|
||||
// mp4/webm/m4v 默认浏览器可播不进候选;strm 是远程引用没有本体。
|
||||
// 其余扩展名都先入候选,由转码 worker probe 实际编码后决定转码还是跳过
|
||||
// (skipped)。failed 也保留在候选里,重新点开始转码时会自动重试。
|
||||
const transcodeCandidateWhereSQL = `COALESCE(ext, '') NOT IN ('mp4', 'webm', 'm4v', 'strm')
|
||||
AND COALESCE(transcode_status, '') IN ('', 'pending', 'failed')`
|
||||
|
||||
// ListTranscodeCandidates 列出某盘所有转码候选视频。limit<=0 表示不限制。
|
||||
func (c *Catalog) ListTranscodeCandidates(ctx context.Context, driveID string, limit int) ([]*Video, error) {
|
||||
query := `SELECT ` + allVideoCols + ` FROM videos
|
||||
WHERE drive_id = ? AND ` + transcodeCandidateWhereSQL + `
|
||||
ORDER BY created_at ASC, id ASC`
|
||||
args := []any{driveID}
|
||||
if limit > 0 {
|
||||
query += ` LIMIT ?`
|
||||
args = append(args, limit)
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// UpdateVideoTranscode 写回单条视频的转码结果。
|
||||
// status=ready 时 transcodedFileID/transcodedSize 指向转码产物;
|
||||
// 其它 status 调用方应传空值,本函数会按传入值原样覆盖。
|
||||
func (c *Catalog) UpdateVideoTranscode(ctx context.Context, id, status, errMsg, transcodedFileID string, transcodedSize int64) error {
|
||||
_, err := c.db.ExecContext(ctx,
|
||||
`UPDATE videos SET transcode_status = ?, transcode_error = ?, transcoded_file_id = ?, transcoded_size = ?, updated_at = ? WHERE id = ?`,
|
||||
status, errMsg, transcodedFileID, transcodedSize, time.Now().UnixMilli(), id)
|
||||
return err
|
||||
}
|
||||
|
||||
// DriveTranscodeCounts 是单盘的转码进度统计。
|
||||
type DriveTranscodeCounts struct {
|
||||
// Pending 是仍在候选集合里、还没有出结果的数量(含从未检测过的)。
|
||||
Pending int
|
||||
Ready int
|
||||
Failed int
|
||||
Skipped int
|
||||
}
|
||||
|
||||
func (c *Catalog) CountTranscodesByDrive(ctx context.Context) (map[string]DriveTranscodeCounts, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT drive_id,
|
||||
COUNT(CASE WHEN COALESCE(ext, '') NOT IN ('mp4', 'webm', 'm4v', 'strm')
|
||||
AND COALESCE(transcode_status, '') IN ('', 'pending') THEN 1 END) AS pending_count,
|
||||
COUNT(CASE WHEN COALESCE(transcode_status, '') = 'ready' THEN 1 END) AS ready_count,
|
||||
COUNT(CASE WHEN COALESCE(transcode_status, '') = 'failed' THEN 1 END) AS failed_count,
|
||||
COUNT(CASE WHEN COALESCE(transcode_status, '') = 'skipped' THEN 1 END) AS skipped_count
|
||||
FROM videos
|
||||
GROUP BY drive_id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make(map[string]DriveTranscodeCounts)
|
||||
for rows.Next() {
|
||||
var driveID string
|
||||
var counts DriveTranscodeCounts
|
||||
if err := rows.Scan(&driveID, &counts.Pending, &counts.Ready, &counts.Failed, &counts.Skipped); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[driveID] = counts
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) HideVideo(ctx context.Context, id string) error {
|
||||
res, err := c.db.ExecContext(ctx,
|
||||
`UPDATE videos SET hidden = 1, updated_at = ? WHERE id = ?`,
|
||||
@@ -186,6 +287,27 @@ func (c *Catalog) HideVideo(ctx context.Context, id string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListHiddenVideos 返回所有被标记隐藏(hidden=1)的视频。
|
||||
// 仅用于一次性把历史「隐藏」视频迁移为黑名单墓碑——隐藏机制已废弃,
|
||||
// 前台「不再展示」改走拉黑逻辑。
|
||||
func (c *Catalog) ListHiddenVideos(ctx context.Context) ([]*Video, error) {
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos WHERE COALESCE(hidden, 0) = 1`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// MigrateVideoToDrive 把 catalog 里 id=videoID 这条视频迁移到另一个 drive。
|
||||
// 用于 spider91 → PikPak 的迁移:上传成功后改写 drive_id / file_id /
|
||||
// content_hash,保留视频自身的 id(spider91-<driveID>-<sourceID>),这样
|
||||
@@ -706,12 +828,27 @@ func (c *Catalog) ListVideoFileIDsByDrive(ctx context.Context, driveID string) (
|
||||
// 用途:crawler 把这个集合写到 seen 文件,让 Python/Go 跳过已爬过的视频,
|
||||
// 配合 --target-new 真正凑出 N 个未爬过的视频。
|
||||
func (c *Catalog) ListSpider91Viewkeys(ctx context.Context, driveID string) ([]string, error) {
|
||||
prefix := "spider91-" + driveID + "-"
|
||||
return c.ListCrawlerSourceIDs(ctx, "spider91", driveID)
|
||||
}
|
||||
|
||||
// ListCrawlerSourceIDs lists source IDs that were already imported by a
|
||||
// crawler-like drive. It reads both videos and deleted_videos so explicit admin
|
||||
// deletions remain tombstoned for future crawler runs.
|
||||
func (c *Catalog) ListCrawlerSourceIDs(ctx context.Context, kind, driveID string) ([]string, error) {
|
||||
kind = strings.TrimSpace(kind)
|
||||
driveID = strings.TrimSpace(driveID)
|
||||
if kind == "" || driveID == "" {
|
||||
return nil, nil
|
||||
}
|
||||
prefix := kind + "-" + driveID + "-"
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT SUBSTR(id, ?) FROM videos WHERE id LIKE ? || '%'
|
||||
UNION
|
||||
SELECT SUBSTR(id, ?) FROM deleted_videos WHERE id LIKE ? || '%'`,
|
||||
len(prefix)+1, prefix, len(prefix)+1, prefix)
|
||||
SELECT SUBSTR(id, ?) FROM deleted_videos WHERE id LIKE ? || '%'
|
||||
UNION
|
||||
SELECT source_id FROM crawler_seen_sources
|
||||
WHERE kind = ? AND drive_id = ? AND status IN ('imported', 'duplicate')`,
|
||||
len(prefix)+1, prefix, len(prefix)+1, prefix, kind, driveID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -729,6 +866,47 @@ func (c *Catalog) ListSpider91Viewkeys(ctx context.Context, driveID string) ([]s
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// MarkCrawlerSourceSeen records the outcome for a crawler source item. Duplicate
|
||||
// source IDs are included in future seen files so scripts can skip them before
|
||||
// the backend downloads the same duplicate content again.
|
||||
func (c *Catalog) MarkCrawlerSourceSeen(ctx context.Context, kind, driveID, sourceID, status, canonicalVideoID, sampledSHA256 string, size int64) error {
|
||||
kind = strings.TrimSpace(kind)
|
||||
driveID = strings.TrimSpace(driveID)
|
||||
sourceID = strings.TrimSpace(sourceID)
|
||||
status = strings.TrimSpace(status)
|
||||
if kind == "" || driveID == "" || sourceID == "" {
|
||||
return nil
|
||||
}
|
||||
switch status {
|
||||
case "imported", "duplicate":
|
||||
default:
|
||||
return fmt.Errorf("catalog: unsupported crawler source status %q", status)
|
||||
}
|
||||
sampledSHA256 = normalizeContentHash(sampledSHA256)
|
||||
if size < 0 {
|
||||
size = 0
|
||||
}
|
||||
now := time.Now().UnixMilli()
|
||||
_, err := c.db.ExecContext(ctx, `
|
||||
INSERT INTO crawler_seen_sources (
|
||||
kind, drive_id, source_id, status, canonical_video_id, sampled_sha256, size_bytes, first_seen_at, last_seen_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(kind, drive_id, source_id) DO UPDATE SET
|
||||
status = excluded.status,
|
||||
canonical_video_id = excluded.canonical_video_id,
|
||||
sampled_sha256 = CASE
|
||||
WHEN excluded.sampled_sha256 != '' THEN excluded.sampled_sha256
|
||||
ELSE crawler_seen_sources.sampled_sha256
|
||||
END,
|
||||
size_bytes = CASE
|
||||
WHEN excluded.size_bytes > 0 THEN excluded.size_bytes
|
||||
ELSE crawler_seen_sources.size_bytes
|
||||
END,
|
||||
last_seen_at = excluded.last_seen_at`,
|
||||
kind, driveID, sourceID, status, strings.TrimSpace(canonicalVideoID), sampledSHA256, size, now, now)
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteVideoWithTombstone records that an administrator explicitly deleted a
|
||||
// video, then removes the visible catalog row. The tombstone is used by
|
||||
// scanners/crawlers to avoid importing the same source file again.
|
||||
@@ -825,6 +1003,92 @@ func (c *Catalog) DeleteVideo(ctx context.Context, id string) error {
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeletedVideo 是黑名单(墓碑)表里的一条记录。原始视频行已删除,
|
||||
// 这里只保留扫盘去重和后台展示需要的最小字段;没有 title/封面/作者。
|
||||
type DeletedVideo struct {
|
||||
ID string `json:"id"`
|
||||
DriveID string `json:"driveId"`
|
||||
FileID string `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
Size int64 `json:"size"`
|
||||
DeletedAt int64 `json:"deletedAt"` // unix 毫秒
|
||||
}
|
||||
|
||||
// ListDeletedVideos 分页列出黑名单视频,按拉黑时间倒序。
|
||||
// keyword 非空时按文件名模糊匹配。
|
||||
func (c *Catalog) ListDeletedVideos(ctx context.Context, keyword string, page, size int) ([]*DeletedVideo, int, error) {
|
||||
if size <= 0 {
|
||||
size = 50
|
||||
}
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
var where []string
|
||||
var args []any
|
||||
if kw := strings.TrimSpace(keyword); kw != "" {
|
||||
where = append(where, "file_name LIKE ?")
|
||||
args = append(args, "%"+kw+"%")
|
||||
}
|
||||
whereSQL := ""
|
||||
if len(where) > 0 {
|
||||
whereSQL = " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
|
||||
var total int
|
||||
if err := c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM deleted_videos`+whereSQL, args...).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
offset := (page - 1) * size
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT id, COALESCE(drive_id, ''), COALESCE(file_id, ''), COALESCE(file_name, ''), COALESCE(size_bytes, 0), deleted_at
|
||||
FROM deleted_videos`+whereSQL+`
|
||||
ORDER BY deleted_at DESC
|
||||
LIMIT ? OFFSET ?`,
|
||||
append(args, size, offset)...)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []*DeletedVideo
|
||||
for rows.Next() {
|
||||
v := &DeletedVideo{}
|
||||
if err := rows.Scan(&v.ID, &v.DriveID, &v.FileID, &v.FileName, &v.Size, &v.DeletedAt); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, total, rows.Err()
|
||||
}
|
||||
|
||||
// RemoveDeletedVideo 把视频移出黑名单(删除墓碑)。移除后该视频会在
|
||||
// 下次扫盘/凌晨流水线时被重新发现并入库,本函数不主动触发扫描。
|
||||
func (c *Catalog) RemoveDeletedVideo(ctx context.Context, id string) error {
|
||||
res, err := c.db.ExecContext(ctx, `DELETE FROM deleted_videos WHERE id = ?`, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rows, err := res.RowsAffected(); err == nil && rows == 0 {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// VideoManagementCounts 返回后台视频管理两个标签的计数:
|
||||
// current=当前可见(与「当前视频」页一致的去重+在线盘+hidden=0 口径),
|
||||
// blacklisted=黑名单墓碑总数。
|
||||
func (c *Catalog) VideoManagementCounts(ctx context.Context) (current, blacklisted int, err error) {
|
||||
currentSQL := `SELECT COUNT(*) FROM videos WHERE COALESCE(hidden, 0) = 0 AND ` + activeDriveWhereSQL + ` AND ` + uniqueVideoWhereSQL
|
||||
if err = c.db.QueryRowContext(ctx, currentSQL).Scan(¤t); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if err = c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM deleted_videos`).Scan(&blacklisted); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return current, blacklisted, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) IsVideoDeleted(ctx context.Context, id string) (bool, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
@@ -901,6 +1165,103 @@ func (c *Catalog) FindVideoByFileSignature(ctx context.Context, fileName string,
|
||||
return scanVideo(row)
|
||||
}
|
||||
|
||||
// FindEquivalentVideo returns the earliest visible video that represents the
|
||||
// same content as source by strong hash or sampled fingerprint, regardless of
|
||||
// which drive currently owns it.
|
||||
func (c *Catalog) FindEquivalentVideo(ctx context.Context, source *Video) (*Video, error) {
|
||||
if source == nil {
|
||||
return nil, sql.ErrNoRows
|
||||
}
|
||||
where, args, ok := equivalentVideoLookupWhere(source)
|
||||
if !ok {
|
||||
return nil, sql.ErrNoRows
|
||||
}
|
||||
args = append([]any{source.ID}, args...)
|
||||
row := c.db.QueryRowContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE id != ?
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
AND COALESCE(file_id, '') != ''
|
||||
AND (`+where+`)
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT 1`, args...)
|
||||
return scanVideo(row)
|
||||
}
|
||||
|
||||
// FindEquivalentVideoOnDrive returns a visible video on driveID that represents
|
||||
// the same content as source by strong hash or sampled fingerprint.
|
||||
func (c *Catalog) FindEquivalentVideoOnDrive(ctx context.Context, source *Video, driveID string) (*Video, error) {
|
||||
driveID = strings.TrimSpace(driveID)
|
||||
if source == nil || driveID == "" {
|
||||
return nil, sql.ErrNoRows
|
||||
}
|
||||
where, args, ok := equivalentVideoLookupWhere(source)
|
||||
if !ok {
|
||||
return nil, sql.ErrNoRows
|
||||
}
|
||||
args = append([]any{driveID, source.ID}, args...)
|
||||
row := c.db.QueryRowContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND id != ?
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
AND COALESCE(file_id, '') != ''
|
||||
AND (`+where+`)
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT 1`, args...)
|
||||
return scanVideo(row)
|
||||
}
|
||||
|
||||
// HasReadyEquivalentPreview reports whether another visible row for the same
|
||||
// content already has a ready preview video.
|
||||
func (c *Catalog) HasReadyEquivalentPreview(ctx context.Context, source *Video) (bool, error) {
|
||||
if source == nil {
|
||||
return false, nil
|
||||
}
|
||||
where, args, ok := equivalentVideoLookupWhere(source)
|
||||
if !ok {
|
||||
return false, nil
|
||||
}
|
||||
args = append([]any{source.ID}, args...)
|
||||
var found int
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
`SELECT 1 FROM videos
|
||||
WHERE id != ?
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
AND COALESCE(preview_status, 'pending') = 'ready'
|
||||
AND (`+where+`)
|
||||
LIMIT 1`, args...).Scan(&found)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func equivalentVideoLookupWhere(source *Video) (string, []any, bool) {
|
||||
if source == nil {
|
||||
return "", nil, false
|
||||
}
|
||||
var parts []string
|
||||
var args []any
|
||||
if hash := normalizeContentHash(source.ContentHash); hash != "" {
|
||||
parts = append(parts, "(COALESCE(content_hash, '') != '' AND content_hash = ?)")
|
||||
args = append(args, hash)
|
||||
}
|
||||
if source.Size > 0 {
|
||||
if sampled := normalizeContentHash(source.SampledSHA256); sampled != "" {
|
||||
parts = append(parts, "(size_bytes = ? AND COALESCE(sampled_sha256, '') != '' AND sampled_sha256 = ?)")
|
||||
args = append(args, source.Size, sampled)
|
||||
}
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "", nil, false
|
||||
}
|
||||
return strings.Join(parts, " OR "), args, true
|
||||
}
|
||||
|
||||
func (c *Catalog) ListVideosNeedingFingerprint(ctx context.Context, driveID string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10000
|
||||
@@ -1172,160 +1533,6 @@ func cleanVideoIDs(ids []string) []string {
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func cleanTagLabels(labels []string) []string {
|
||||
seen := make(map[string]struct{}, len(labels))
|
||||
cleaned := make([]string, 0, len(labels))
|
||||
for _, label := range labels {
|
||||
label = strings.TrimSpace(label)
|
||||
if label == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(label)
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
cleaned = append(cleaned, label)
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func (c *Catalog) LeastPopulatedVisibleUniqueTag(ctx context.Context, labels []string) (string, error) {
|
||||
cleaned := cleanTagLabels(labels)
|
||||
bestLabel := ""
|
||||
bestCount := 0
|
||||
for _, label := range cleaned {
|
||||
var count int
|
||||
if err := c.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*)
|
||||
FROM videos
|
||||
WHERE COALESCE(hidden, 0) = 0
|
||||
AND `+activeDriveWhereSQL+`
|
||||
AND `+uniqueVideoWhereSQL+`
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
JOIN tags t ON t.id = vt.tag_id
|
||||
WHERE vt.video_id = videos.id
|
||||
AND t.label = ? COLLATE NOCASE
|
||||
)`,
|
||||
label,
|
||||
).Scan(&count); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if count == 0 {
|
||||
continue
|
||||
}
|
||||
if bestLabel == "" || count < bestCount {
|
||||
bestLabel = label
|
||||
bestCount = count
|
||||
}
|
||||
}
|
||||
return bestLabel, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) RandomVideosByTagExcluding(ctx context.Context, tag string, excludeIDs []string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
tag = strings.TrimSpace(tag)
|
||||
if tag == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
cleaned := cleanVideoIDs(excludeIDs)
|
||||
args := make([]any, 0, len(cleaned)+2)
|
||||
args = append(args, tag)
|
||||
whereSQL := `WHERE COALESCE(hidden, 0) = 0
|
||||
AND ` + activeDriveWhereSQL + `
|
||||
AND ` + uniqueVideoWhereSQL + `
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
JOIN tags t ON t.id = vt.tag_id
|
||||
WHERE vt.video_id = videos.id
|
||||
AND t.label = ? COLLATE NOCASE
|
||||
)`
|
||||
if len(cleaned) > 0 {
|
||||
placeholders := strings.Repeat("?,", len(cleaned))
|
||||
placeholders = placeholders[:len(placeholders)-1]
|
||||
whereSQL += " AND id NOT IN (" + placeholders + ")"
|
||||
for _, id := range cleaned {
|
||||
args = append(args, id)
|
||||
}
|
||||
}
|
||||
args = append(args, limit)
|
||||
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos `+whereSQL+`
|
||||
ORDER BY RANDOM() LIMIT ?`,
|
||||
args...,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) RandomVideosForPreferredVideoExcluding(ctx context.Context, preferredVideoID string, excludeIDs []string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
preferredVideoID = strings.TrimSpace(preferredVideoID)
|
||||
if preferredVideoID == "" {
|
||||
return c.RandomVideosExcluding(ctx, excludeIDs, limit)
|
||||
}
|
||||
|
||||
preferredExclude := append([]string{}, excludeIDs...)
|
||||
preferredExclude = append(preferredExclude, preferredVideoID)
|
||||
|
||||
preferred, err := c.GetVideo(ctx, preferredVideoID)
|
||||
if err != nil || preferred == nil || preferred.Hidden || len(preferred.Tags) == 0 {
|
||||
return c.RandomVideosExcluding(ctx, preferredExclude, limit)
|
||||
}
|
||||
tag, err := c.LeastPopulatedVisibleUniqueTag(ctx, preferred.Tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tag == "" {
|
||||
return c.RandomVideosExcluding(ctx, preferredExclude, limit)
|
||||
}
|
||||
|
||||
items, err := c.RandomVideosByTagExcluding(ctx, tag, preferredExclude, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(items) >= limit {
|
||||
return items, nil
|
||||
}
|
||||
|
||||
mergedExclude := make([]string, 0, len(preferredExclude)+len(items))
|
||||
mergedExclude = append(mergedExclude, preferredExclude...)
|
||||
for _, item := range items {
|
||||
if item != nil {
|
||||
mergedExclude = append(mergedExclude, item.ID)
|
||||
}
|
||||
}
|
||||
fallback, err := c.RandomVideosExcluding(ctx, mergedExclude, limit-len(items))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(items, fallback...), nil
|
||||
}
|
||||
|
||||
type DriveTeaserCounts struct {
|
||||
Ready int
|
||||
Pending int
|
||||
@@ -1443,6 +1650,121 @@ func (c *Catalog) CountFingerprintsByDrive(ctx context.Context) (map[string]Driv
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) CountCrawlerAssets(ctx context.Context, crawlerID string, prefixes []string) (CrawlerAssetCounts, error) {
|
||||
var out CrawlerAssetCounts
|
||||
crawlerID = strings.TrimSpace(crawlerID)
|
||||
prefixes = cleanCrawlerIDPrefixes(prefixes)
|
||||
if crawlerID == "" || len(prefixes) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
where := make([]string, 0, len(prefixes))
|
||||
args := make([]any, 0, 2+len(prefixes))
|
||||
args = append(args, crawlerID, crawlerID)
|
||||
for range prefixes {
|
||||
where = append(where, "id LIKE ? ESCAPE '\\'")
|
||||
}
|
||||
for _, prefix := range prefixes {
|
||||
args = append(args, escapeSQLLike(prefix)+"%")
|
||||
}
|
||||
query := `SELECT
|
||||
COUNT(*) AS total_count,
|
||||
COUNT(CASE WHEN drive_id = ? THEN 1 END) AS local_count,
|
||||
COUNT(CASE WHEN drive_id != ? THEN 1 END) AS migrated_count,
|
||||
COUNT(CASE WHEN EXISTS (
|
||||
SELECT 1 FROM videos AS asset_dup
|
||||
WHERE ` + crawlerAssetEquivalentSQL("asset_dup", "videos") + `
|
||||
AND COALESCE(asset_dup.thumbnail_url, '') != ''
|
||||
) THEN 1 END) AS thumbnail_ready_count,
|
||||
COUNT(CASE WHEN NOT EXISTS (
|
||||
SELECT 1 FROM videos AS asset_dup
|
||||
WHERE ` + crawlerAssetEquivalentSQL("asset_dup", "videos") + `
|
||||
AND COALESCE(asset_dup.thumbnail_url, '') != ''
|
||||
)
|
||||
AND COALESCE(thumbnail_url, '') = ''
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('failed', 'skipped') THEN 1 END) AS thumbnail_pending_count,
|
||||
COUNT(CASE WHEN NOT EXISTS (
|
||||
SELECT 1 FROM videos AS asset_dup
|
||||
WHERE ` + crawlerAssetEquivalentSQL("asset_dup", "videos") + `
|
||||
AND COALESCE(asset_dup.thumbnail_url, '') != ''
|
||||
)
|
||||
AND COALESCE(thumbnail_url, '') = ''
|
||||
AND COALESCE(thumbnail_status, 'pending') = 'failed' THEN 1 END) AS thumbnail_failed_count,
|
||||
COUNT(CASE WHEN EXISTS (
|
||||
SELECT 1 FROM videos AS asset_dup
|
||||
WHERE ` + crawlerAssetEquivalentSQL("asset_dup", "videos") + `
|
||||
AND COALESCE(asset_dup.preview_status, 'pending') = 'ready'
|
||||
) THEN 1 END) AS teaser_ready_count,
|
||||
COUNT(CASE WHEN NOT EXISTS (
|
||||
SELECT 1 FROM videos AS asset_dup
|
||||
WHERE ` + crawlerAssetEquivalentSQL("asset_dup", "videos") + `
|
||||
AND COALESCE(asset_dup.preview_status, 'pending') = 'ready'
|
||||
)
|
||||
AND COALESCE(preview_status, 'pending') = 'pending' THEN 1 END) AS teaser_pending_count,
|
||||
COUNT(CASE WHEN NOT EXISTS (
|
||||
SELECT 1 FROM videos AS asset_dup
|
||||
WHERE ` + crawlerAssetEquivalentSQL("asset_dup", "videos") + `
|
||||
AND COALESCE(asset_dup.preview_status, 'pending') = 'ready'
|
||||
)
|
||||
AND COALESCE(preview_status, 'pending') = 'failed' THEN 1 END) AS teaser_failed_count,
|
||||
COUNT(CASE WHEN COALESCE(sampled_sha256, '') != ''
|
||||
OR COALESCE(fingerprint_status, 'pending') = 'ready' THEN 1 END) AS fingerprint_ready_count,
|
||||
COUNT(CASE WHEN size_bytes > 0
|
||||
AND COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = 'pending' THEN 1 END) AS fingerprint_pending_count,
|
||||
COUNT(CASE WHEN COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = 'failed' THEN 1 END) AS fingerprint_failed_count
|
||||
FROM videos
|
||||
WHERE COALESCE(hidden, 0) = 0
|
||||
AND (` + strings.Join(where, " OR ") + `)`
|
||||
err := c.db.QueryRowContext(ctx, query, args...).Scan(
|
||||
&out.Total,
|
||||
&out.Local,
|
||||
&out.Migrated,
|
||||
&out.Thumbnail.Ready,
|
||||
&out.Thumbnail.Pending,
|
||||
&out.Thumbnail.Failed,
|
||||
&out.Teaser.Ready,
|
||||
&out.Teaser.Pending,
|
||||
&out.Teaser.Failed,
|
||||
&out.Fingerprint.Ready,
|
||||
&out.Fingerprint.Pending,
|
||||
&out.Fingerprint.Failed,
|
||||
)
|
||||
return out, err
|
||||
}
|
||||
|
||||
func crawlerAssetEquivalentSQL(candidateAlias, sourceAlias string) string {
|
||||
return fmt.Sprintf(`(%[1]s.id = %[2]s.id
|
||||
OR (COALESCE(%[2]s.content_hash, '') != ''
|
||||
AND %[1]s.content_hash = %[2]s.content_hash)
|
||||
OR (%[2]s.size_bytes > 0
|
||||
AND COALESCE(%[2]s.sampled_sha256, '') != ''
|
||||
AND %[1]s.size_bytes = %[2]s.size_bytes
|
||||
AND %[1]s.sampled_sha256 = %[2]s.sampled_sha256))`, candidateAlias, sourceAlias)
|
||||
}
|
||||
|
||||
func cleanCrawlerIDPrefixes(prefixes []string) []string {
|
||||
out := make([]string, 0, len(prefixes))
|
||||
seen := map[string]bool{}
|
||||
for _, prefix := range prefixes {
|
||||
prefix = strings.TrimSpace(prefix)
|
||||
if prefix == "" || seen[prefix] {
|
||||
continue
|
||||
}
|
||||
seen[prefix] = true
|
||||
out = append(out, prefix)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func escapeSQLLike(raw string) string {
|
||||
raw = strings.ReplaceAll(raw, `\`, `\\`)
|
||||
raw = strings.ReplaceAll(raw, `%`, `\%`)
|
||||
raw = strings.ReplaceAll(raw, `_`, `\_`)
|
||||
return raw
|
||||
}
|
||||
|
||||
func (c *Catalog) CountVideosNeedingFingerprint(ctx context.Context, driveID string) (int, error) {
|
||||
var count int
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
@@ -1880,6 +2202,7 @@ COALESCE(sampled_sha256, ''), COALESCE(fingerprint_status, 'pending'), COALESCE(
|
||||
COALESCE(parent_id, ''), title, COALESCE(author, ''), COALESCE(tags, '[]'),
|
||||
duration_seconds, size_bytes, COALESCE(ext, ''), COALESCE(quality, ''), COALESCE(thumbnail_url, ''),
|
||||
COALESCE(preview_file_id, ''), COALESCE(preview_local, ''), COALESCE(preview_status, 'pending'),
|
||||
COALESCE(transcode_status, ''), COALESCE(transcode_error, ''), COALESCE(transcoded_file_id, ''), COALESCE(transcoded_size, 0),
|
||||
views, favorites, comments, likes, dislikes,
|
||||
COALESCE(category, ''), COALESCE(hidden, 0), COALESCE(badges, '[]'), COALESCE(description, ''),
|
||||
published_at, created_at, updated_at
|
||||
@@ -1951,6 +2274,7 @@ func scanVideo(row rowScanner) (*Video, error) {
|
||||
&v.ParentID, &v.Title, &v.Author, &tagsJSON,
|
||||
&v.DurationSeconds, &v.Size, &v.Ext, &v.Quality, &v.ThumbnailURL,
|
||||
&v.PreviewFileID, &v.PreviewLocal, &v.PreviewStatus,
|
||||
&v.TranscodeStatus, &v.TranscodeError, &v.TranscodedFileID, &v.TranscodedSize,
|
||||
&v.Views, &v.Favorites, &v.Comments, &v.Likes, &v.Dislikes,
|
||||
&v.Category, &hidden, &badgesJSON, &v.Description,
|
||||
&publishedAt, &createdAt, &updatedAt,
|
||||
|
||||
@@ -22,6 +22,10 @@ CREATE TABLE IF NOT EXISTS videos (
|
||||
preview_file_id TEXT, -- deprecated: 旧版回写网盘后的预览视频 file id
|
||||
preview_local TEXT, -- 本地预览视频路径(兜底)
|
||||
preview_status TEXT DEFAULT 'pending', -- pending / ready / failed
|
||||
transcode_status TEXT DEFAULT '', -- '' / pending / ready / skipped / failed(浏览器兼容性转码)
|
||||
transcode_error TEXT DEFAULT '',
|
||||
transcoded_file_id TEXT DEFAULT '', -- 转码产物在同一 drive 上的 fileID,播放源优先用它
|
||||
transcoded_size INTEGER DEFAULT 0,
|
||||
views INTEGER DEFAULT 0,
|
||||
favorites INTEGER DEFAULT 0,
|
||||
comments INTEGER DEFAULT 0,
|
||||
@@ -89,6 +93,24 @@ CREATE INDEX IF NOT EXISTS idx_deleted_videos_drive_hash
|
||||
CREATE INDEX IF NOT EXISTS idx_deleted_videos_drive_signature
|
||||
ON deleted_videos(drive_id, file_name, size_bytes);
|
||||
|
||||
-- 爬虫来源记录。用于把已确认重复的 source_id 写回 seen 列表,
|
||||
-- 避免后续爬虫反复下载同一个候选视频。
|
||||
CREATE TABLE IF NOT EXISTS crawler_seen_sources (
|
||||
kind TEXT NOT NULL,
|
||||
drive_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'imported', -- imported / duplicate
|
||||
canonical_video_id TEXT NOT NULL DEFAULT '',
|
||||
sampled_sha256 TEXT NOT NULL DEFAULT '',
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
first_seen_at INTEGER NOT NULL,
|
||||
last_seen_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (kind, drive_id, source_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_crawler_seen_sources_drive
|
||||
ON crawler_seen_sources(kind, drive_id, status);
|
||||
|
||||
-- 网盘账户
|
||||
CREATE TABLE IF NOT EXISTS drives (
|
||||
id TEXT PRIMARY KEY,
|
||||
|
||||
@@ -165,171 +165,3 @@ func TestRandomVideosWithReadyThumbnailsExcluding(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoChoosesLeastPopulatedTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "current", DriveID: "drive", FileID: "f-current", Title: "current", Tags: []string{"common", "rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-1", DriveID: "drive", FileID: "f-common-1", Title: "common 1", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-2", DriveID: "drive", FileID: "f-common-2", Title: "common 2", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "rare-1", DriveID: "drive", FileID: "f-rare-1", Title: "rare 1", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
tag, err := cat.LeastPopulatedVisibleUniqueTag(ctx, []string{"common", "rare"})
|
||||
if err != nil {
|
||||
t.Fatalf("least populated tag: %v", err)
|
||||
}
|
||||
if tag != "rare" {
|
||||
t.Fatalf("least populated tag = %q, want rare", tag)
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosForPreferredVideoExcluding(ctx, "current", []string{"current"}, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("random preferred: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID != "rare-1" {
|
||||
t.Fatalf("preferred result = %#v, want rare-1", videoIDs(got))
|
||||
}
|
||||
|
||||
got, err = cat.RandomVideosForPreferredVideoExcluding(ctx, "current", nil, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("random preferred without explicit exclude: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID == "current" {
|
||||
t.Fatalf("preferred result without explicit exclude = %#v, should not return current", videoIDs(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoFallsBackToFillBatch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "current", DriveID: "drive", FileID: "f-current", Title: "current", Tags: []string{"common", "rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-1", DriveID: "drive", FileID: "f-common-1", Title: "common 1", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-2", DriveID: "drive", FileID: "f-common-2", Title: "common 2", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "rare-1", DriveID: "drive", FileID: "f-rare-1", Title: "rare 1", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "hidden-rare", DriveID: "drive", FileID: "f-hidden-rare", Title: "hidden rare", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
if err := cat.HideVideo(ctx, "hidden-rare"); err != nil {
|
||||
t.Fatalf("hide hidden-rare: %v", err)
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosForPreferredVideoExcluding(ctx, "current", []string{"current"}, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("random preferred: %v", err)
|
||||
}
|
||||
ids := videoIDs(got)
|
||||
if len(ids) != 3 {
|
||||
t.Fatalf("result ids = %#v, want 3 items", ids)
|
||||
}
|
||||
for _, excluded := range []string{"current", "hidden-rare"} {
|
||||
if hasVideoID(ids, excluded) {
|
||||
t.Fatalf("result ids = %#v, should not include %s", ids, excluded)
|
||||
}
|
||||
}
|
||||
if !hasVideoID(ids, "rare-1") {
|
||||
t.Fatalf("result ids = %#v, want rare-1 from least populated tag", ids)
|
||||
}
|
||||
if len(uniqueVideoIDs(ids)) != len(ids) {
|
||||
t.Fatalf("result ids = %#v, want no duplicates", ids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoFallbacksWhenPreferenceUnavailable(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "untagged", DriveID: "drive", FileID: "f-untagged", Title: "untagged", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "visible-1", DriveID: "drive", FileID: "f-visible-1", Title: "visible 1", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "visible-2", DriveID: "drive", FileID: "f-visible-2", Title: "visible 2", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosForPreferredVideoExcluding(ctx, "missing", []string{"untagged"}, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("random missing preferred: %v", err)
|
||||
}
|
||||
if !sameVideoIDSet(videoIDs(got), []string{"visible-1", "visible-2"}) {
|
||||
t.Fatalf("missing preferred ids = %#v, want visible fallback videos", videoIDs(got))
|
||||
}
|
||||
|
||||
got, err = cat.RandomVideosForPreferredVideoExcluding(ctx, "untagged", []string{"untagged"}, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("random untagged preferred: %v", err)
|
||||
}
|
||||
if !sameVideoIDSet(videoIDs(got), []string{"visible-1", "visible-2"}) {
|
||||
t.Fatalf("untagged preferred ids = %#v, want visible fallback videos", videoIDs(got))
|
||||
}
|
||||
}
|
||||
|
||||
func videoIDs(videos []*Video) []string {
|
||||
ids := make([]string, 0, len(videos))
|
||||
for _, v := range videos {
|
||||
ids = append(ids, v.ID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func hasVideoID(ids []string, want string) bool {
|
||||
for _, id := range ids {
|
||||
if id == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func uniqueVideoIDs(ids []string) map[string]struct{} {
|
||||
seen := make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
seen[id] = struct{}{}
|
||||
}
|
||||
return seen
|
||||
}
|
||||
|
||||
func sameVideoIDSet(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
seen := make(map[string]int, len(a))
|
||||
for _, value := range a {
|
||||
seen[value]++
|
||||
}
|
||||
for _, value := range b {
|
||||
if seen[value] == 0 {
|
||||
return false
|
||||
}
|
||||
seen[value]--
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -66,6 +66,21 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "thumbnail_failures", "INTEGER DEFAULT 0"); err != nil {
|
||||
return err
|
||||
}
|
||||
// videos.transcode_*:浏览器兼容性转码状态。
|
||||
// status:''=未检测 / pending=已入队 / ready=已转码 / skipped=检测后无需转码 / failed=失败。
|
||||
// transcoded_file_id 指向转码产物在同一 drive 上的 fileID,播放源优先使用它。
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "transcode_status", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "transcode_error", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "transcoded_file_id", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "transcoded_size", "INTEGER DEFAULT 0"); err != nil {
|
||||
return err
|
||||
}
|
||||
// drives.teaser_enabled:每盘预览视频开关,替代旧的全局 preview.enabled。
|
||||
// 升级路径:直接让 ALTER TABLE 的 DEFAULT 1 兜底 —— 每个现存 drive 都默认开启,
|
||||
// 不读旧的 settings.preview.enabled 字段。这样老用户即便之前关过全局开关,
|
||||
@@ -297,7 +312,7 @@ UPDATE videos
|
||||
}
|
||||
|
||||
func (c *Catalog) clearRemoteP123ThumbnailsOnce(ctx context.Context) error {
|
||||
// 123 云盘列表返回的缩略图尺寸和稳定性都不适合作为站内封面;清空历史写入的
|
||||
// 123网盘列表返回的缩略图尺寸和稳定性都不适合作为站内封面;清空历史写入的
|
||||
// 远程 URL,让封面 worker 统一从视频直链抽帧生成本地 /p/thumb/<id>。
|
||||
const markerKey = "videos.p123.remote_thumbnails_cleared"
|
||||
marker, err := c.GetSetting(ctx, markerKey, "")
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestListHiddenVideosForMigration 验证:隐藏的视频不进可见列表,
|
||||
// 但能被 ListHiddenVideos 拿到(供一次性迁移为墓碑)。
|
||||
func TestListHiddenVideosForMigration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, id := range []string{"v1", "v2", "v3"} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id, DriveID: "drive", FileID: "f-" + id, Title: id,
|
||||
PublishedAt: now, CreatedAt: now, UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
if err := cat.HideVideo(ctx, "v2"); err != nil {
|
||||
t.Fatalf("hide v2: %v", err)
|
||||
}
|
||||
|
||||
visible, total, err := cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 50})
|
||||
if err != nil {
|
||||
t.Fatalf("list visible: %v", err)
|
||||
}
|
||||
if total != 2 || len(visible) != 2 {
|
||||
t.Fatalf("visible total/len = %d/%d, want 2/2", total, len(visible))
|
||||
}
|
||||
for _, v := range visible {
|
||||
if v.ID == "v2" {
|
||||
t.Fatalf("hidden v2 leaked into visible list")
|
||||
}
|
||||
}
|
||||
|
||||
hidden, err := cat.ListHiddenVideos(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list hidden: %v", err)
|
||||
}
|
||||
if len(hidden) != 1 || hidden[0].ID != "v2" {
|
||||
t.Fatalf("ListHiddenVideos = %v, want only v2", hidden)
|
||||
}
|
||||
|
||||
current, blacklisted, err := cat.VideoManagementCounts(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("counts: %v", err)
|
||||
}
|
||||
if current != 2 || blacklisted != 0 {
|
||||
t.Fatalf("counts = current %d blacklisted %d, want 2/0", current, blacklisted)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBlacklistListAndRemove 验证墓碑表的列出、关键字过滤和移除。
|
||||
func TestBlacklistListAndRemove(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
seed := []struct{ id, file string }{
|
||||
{"d1", "movie-alpha.avi"},
|
||||
{"d2", "movie-beta.mp4"},
|
||||
{"d3", "clip-gamma.wmv"},
|
||||
}
|
||||
for _, s := range seed {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: s.id, DriveID: "drive", FileID: "f-" + s.id, FileName: s.file,
|
||||
Title: s.id, PublishedAt: now, CreatedAt: now, UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", s.id, err)
|
||||
}
|
||||
if err := cat.DeleteVideoWithTombstone(ctx, s.id); err != nil {
|
||||
t.Fatalf("tombstone %s: %v", s.id, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, total, err := cat.ListDeletedVideos(ctx, "", 1, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list deleted: %v", err)
|
||||
}
|
||||
if total != 3 || len(items) != 3 {
|
||||
t.Fatalf("deleted total/len = %d/%d, want 3/3", total, len(items))
|
||||
}
|
||||
|
||||
// 关键字过滤
|
||||
filtered, ftotal, err := cat.ListDeletedVideos(ctx, "movie", 1, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list deleted filtered: %v", err)
|
||||
}
|
||||
if ftotal != 2 || len(filtered) != 2 {
|
||||
t.Fatalf("filtered total/len = %d/%d, want 2/2", ftotal, len(filtered))
|
||||
}
|
||||
|
||||
// 移出黑名单
|
||||
if err := cat.RemoveDeletedVideo(ctx, "d1"); err != nil {
|
||||
t.Fatalf("remove d1: %v", err)
|
||||
}
|
||||
if deleted, err := cat.IsVideoDeleted(ctx, "d1"); err != nil || deleted {
|
||||
t.Fatalf("d1 should no longer be blacklisted (deleted=%v err=%v)", deleted, err)
|
||||
}
|
||||
_, total, err = cat.ListDeletedVideos(ctx, "", 1, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list deleted after remove: %v", err)
|
||||
}
|
||||
if total != 2 {
|
||||
t.Fatalf("deleted total after remove = %d, want 2", total)
|
||||
}
|
||||
|
||||
if err := cat.RemoveDeletedVideo(ctx, "does-not-exist"); err == nil {
|
||||
t.Fatalf("remove missing id should return error")
|
||||
}
|
||||
|
||||
// counts: 删完一个还剩 2 个黑名单;可见视频已全部被墓碑删除
|
||||
current, blacklisted, err := cat.VideoManagementCounts(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("counts: %v", err)
|
||||
}
|
||||
if current != 0 || blacklisted != 2 {
|
||||
t.Fatalf("counts = current %d blacklisted %d, want 0/2", current, blacklisted)
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,17 @@
|
||||
package googledrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -21,10 +28,13 @@ import (
|
||||
const (
|
||||
Kind = "googledrive"
|
||||
defaultAPIBaseURL = "https://www.googleapis.com/drive/v3"
|
||||
defaultUploadAPIURL = "https://www.googleapis.com/upload/drive/v3"
|
||||
defaultOAuthURL = "https://www.googleapis.com/oauth2/v4/token"
|
||||
defaultRenewAPIURL = "https://api.oplist.org/googleui/renewapi"
|
||||
defaultListInterval = 1 * time.Second
|
||||
defaultListCooldown = 5 * time.Minute
|
||||
defaultLinkCooldown = 5 * time.Minute
|
||||
uploadChunkSize = int64(8 * 1024 * 1024)
|
||||
|
||||
filesListFields = "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken"
|
||||
fileInfoFields = "id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum"
|
||||
@@ -41,13 +51,19 @@ type Driver struct {
|
||||
renewAPIURL string
|
||||
oauthURL string
|
||||
apiBaseURL string
|
||||
uploadBaseURL string
|
||||
client *resty.Client
|
||||
httpClient *http.Client
|
||||
onTokenUpdate func(access, refresh string)
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
listInterval time.Duration
|
||||
listCooldown time.Duration
|
||||
|
||||
linkCooldownMu sync.Mutex
|
||||
linkCooldownUntil time.Time
|
||||
linkCooldownDuration time.Duration
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -61,6 +77,7 @@ type Config struct {
|
||||
RenewAPIURL string
|
||||
OAuthURL string
|
||||
APIBaseURL string
|
||||
UploadAPIURL string
|
||||
|
||||
OnTokenUpdate func(access, refresh string)
|
||||
}
|
||||
@@ -82,6 +99,10 @@ func New(c Config) *Driver {
|
||||
if apiBaseURL == "" {
|
||||
apiBaseURL = defaultAPIBaseURL
|
||||
}
|
||||
uploadBaseURL := strings.TrimRight(strings.TrimSpace(c.UploadAPIURL), "/")
|
||||
if uploadBaseURL == "" {
|
||||
uploadBaseURL = deriveUploadBaseURL(apiBaseURL)
|
||||
}
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
rootID: rootID,
|
||||
@@ -93,15 +114,34 @@ func New(c Config) *Driver {
|
||||
renewAPIURL: renewAPIURL,
|
||||
oauthURL: oauthURL,
|
||||
apiBaseURL: apiBaseURL,
|
||||
uploadBaseURL: uploadBaseURL,
|
||||
onTokenUpdate: c.OnTokenUpdate,
|
||||
client: resty.New().
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
listInterval: defaultListInterval,
|
||||
listCooldown: defaultListCooldown,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 0,
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
},
|
||||
listInterval: defaultListInterval,
|
||||
listCooldown: defaultListCooldown,
|
||||
linkCooldownDuration: defaultLinkCooldown,
|
||||
}
|
||||
}
|
||||
|
||||
func deriveUploadBaseURL(apiBaseURL string) string {
|
||||
apiBaseURL = strings.TrimRight(strings.TrimSpace(apiBaseURL), "/")
|
||||
if apiBaseURL == "" || apiBaseURL == defaultAPIBaseURL {
|
||||
return defaultUploadAPIURL
|
||||
}
|
||||
if strings.HasSuffix(apiBaseURL, "/drive/v3") {
|
||||
return strings.TrimSuffix(apiBaseURL, "/drive/v3") + "/upload/drive/v3"
|
||||
}
|
||||
return apiBaseURL
|
||||
}
|
||||
|
||||
func (d *Driver) Kind() string { return Kind }
|
||||
func (d *Driver) ID() string { return d.id }
|
||||
func (d *Driver) RootID() string { return d.rootID }
|
||||
@@ -209,8 +249,19 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
if fileID == "" {
|
||||
return nil, errors.New("googledrive stream: empty file id")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := d.linkCooldownError(time.Now()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := d.Stat(ctx, fileID); err != nil {
|
||||
return nil, fmt.Errorf("googledrive stream: %w", err)
|
||||
err = fmt.Errorf("googledrive stream: %w", err)
|
||||
if wait, ok := drives.RateLimitRetryAfter(err); ok {
|
||||
until := d.pauseLinkCooldown(wait)
|
||||
log.Printf("[googledrive] stream link cooling down drive=%s until=%s err=%v", d.id, until.Format(time.RFC3339), err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
u := d.fileURL(fileID) + "?alt=media&acknowledgeAbuse=true&supportsAllDrives=true"
|
||||
return &drives.StreamLink{
|
||||
@@ -222,12 +273,396 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
func (d *Driver) linkCooldownError(now time.Time) error {
|
||||
d.linkCooldownMu.Lock()
|
||||
defer d.linkCooldownMu.Unlock()
|
||||
if d.linkCooldownUntil.IsZero() {
|
||||
return nil
|
||||
}
|
||||
if !now.Before(d.linkCooldownUntil) {
|
||||
d.linkCooldownUntil = time.Time{}
|
||||
return nil
|
||||
}
|
||||
wait := d.linkCooldownUntil.Sub(now)
|
||||
if wait <= 0 {
|
||||
return nil
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: wait,
|
||||
Err: fmt.Errorf("googledrive stream link cooling down until %s", d.linkCooldownUntil.Format(time.RFC3339)),
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
func (d *Driver) pauseLinkCooldown(wait time.Duration) time.Time {
|
||||
if wait <= 0 {
|
||||
wait = d.linkCooldownDuration
|
||||
}
|
||||
if wait <= 0 {
|
||||
wait = defaultLinkCooldown
|
||||
}
|
||||
until := time.Now().Add(wait)
|
||||
d.linkCooldownMu.Lock()
|
||||
if until.After(d.linkCooldownUntil) {
|
||||
d.linkCooldownUntil = until
|
||||
} else {
|
||||
until = d.linkCooldownUntil
|
||||
}
|
||||
d.linkCooldownMu.Unlock()
|
||||
return until
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
res, err := d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return res.FileID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
parentID, name, err := d.normalizeUploadArgs(parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
sessionURL, err := d.createUploadSession(ctx, parentID, name, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if strings.TrimSpace(sessionURL) == "" {
|
||||
return UploadResult{}, errors.New("googledrive upload session: empty upload url")
|
||||
}
|
||||
|
||||
hasher := md5.New()
|
||||
var item driveFile
|
||||
var copied int64
|
||||
if size == 0 {
|
||||
completed, err := d.putUploadSessionChunkWithRetry(ctx, sessionURL, 0, 0, nil, hasher)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if completed != nil {
|
||||
item = *completed
|
||||
}
|
||||
} else {
|
||||
chunkSize := uploadChunkSize
|
||||
if chunkSize <= 0 {
|
||||
chunkSize = 8 * 1024 * 1024
|
||||
}
|
||||
if chunkSize > int64(math.MaxInt32) {
|
||||
chunkSize = int64(math.MaxInt32)
|
||||
}
|
||||
buf := make([]byte, int(chunkSize))
|
||||
for copied < size {
|
||||
partSize := minInt64(chunkSize, size-copied)
|
||||
chunk := buf[:int(partSize)]
|
||||
n, err := io.ReadFull(r, chunk)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return UploadResult{}, fmt.Errorf("googledrive upload: size mismatch: declared %d, copied %d", size, copied+int64(n))
|
||||
}
|
||||
return UploadResult{}, fmt.Errorf("googledrive upload: read body: %w", err)
|
||||
}
|
||||
chunk = chunk[:n]
|
||||
completed, err := d.putUploadSessionChunkWithRetry(ctx, sessionURL, copied, size, chunk, hasher)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if completed != nil {
|
||||
item = *completed
|
||||
}
|
||||
copied += int64(n)
|
||||
}
|
||||
}
|
||||
|
||||
hashHex := hex.EncodeToString(hasher.Sum(nil))
|
||||
if item.ID == "" {
|
||||
fileID, err := d.findUploadedFileID(ctx, parentID, name, hashHex)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
item.ID = fileID
|
||||
}
|
||||
return UploadResult{FileID: item.ID, Hash: hashHex, Size: copied}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) normalizeUploadArgs(parentID, name string, r io.Reader, size int64) (string, string, error) {
|
||||
if r == nil {
|
||||
return "", "", errors.New("googledrive upload: body is required")
|
||||
}
|
||||
if size < 0 {
|
||||
return "", "", fmt.Errorf("googledrive upload: invalid size %d", size)
|
||||
}
|
||||
parentID = strings.TrimSpace(parentID)
|
||||
if parentID == "" || parentID == "/" {
|
||||
parentID = d.rootID
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return "", "", errors.New("googledrive upload: empty file name")
|
||||
}
|
||||
return parentID, name, nil
|
||||
}
|
||||
|
||||
func (d *Driver) createUploadSession(ctx context.Context, parentID, name string, size int64) (string, error) {
|
||||
return d.createUploadSessionOnce(ctx, parentID, name, size, true)
|
||||
}
|
||||
|
||||
func (d *Driver) createUploadSessionOnce(ctx context.Context, parentID, name string, size int64, retry bool) (string, error) {
|
||||
var apiErr apiErrorResp
|
||||
res, err := d.client.R().
|
||||
SetContext(ctx).
|
||||
SetHeader("Authorization", "Bearer "+d.accessToken).
|
||||
SetHeader("X-Upload-Content-Type", mimeType(driveFile{Name: name})).
|
||||
SetHeader("X-Upload-Content-Length", strconv.FormatInt(size, 10)).
|
||||
SetQueryParams(map[string]string{
|
||||
"uploadType": "resumable",
|
||||
"supportsAllDrives": "true",
|
||||
"fields": fileInfoFields,
|
||||
}).
|
||||
SetBody(map[string]any{
|
||||
"name": name,
|
||||
"parents": []string{parentID},
|
||||
}).
|
||||
SetError(&apiErr).
|
||||
Post(d.uploadFilesURL())
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("googledrive upload session: %w", err)
|
||||
}
|
||||
if isGoogleRateLimit(res, apiErr.Error) {
|
||||
return "", googleRateLimitError(res, apiErr.Error.Message)
|
||||
}
|
||||
if apiErr.Error.Code != 0 {
|
||||
if apiErr.Error.Code == http.StatusUnauthorized && retry {
|
||||
if err := d.refresh(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return d.createUploadSessionOnce(ctx, parentID, name, size, false)
|
||||
}
|
||||
return "", googleAPIError(apiErr.Error)
|
||||
}
|
||||
if res.IsError() {
|
||||
return "", fmt.Errorf("googledrive upload session: status=%d body=%s", res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return strings.TrimSpace(res.Header().Get("Location")), nil
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunkWithRetry(ctx context.Context, uploadURL string, start, total int64, data []byte, hasher hash.Hash) (*driveFile, error) {
|
||||
var last error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if attempt > 0 {
|
||||
if err := sleepContext(ctx, time.Duration(attempt)*time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
item, retryable, err := d.putUploadSessionChunk(ctx, uploadURL, start, total, data)
|
||||
if err == nil {
|
||||
if hasher != nil && len(data) > 0 {
|
||||
_, _ = hasher.Write(data)
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
last = err
|
||||
if !retryable {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if last == nil {
|
||||
last = errors.New("googledrive upload session: retry attempts exhausted")
|
||||
}
|
||||
return nil, last
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunk(ctx context.Context, uploadURL string, start, total int64, data []byte) (*driveFile, bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
req.ContentLength = int64(len(data))
|
||||
req.Header.Set("Authorization", "Bearer "+d.accessToken)
|
||||
req.Header.Set("Content-Length", strconv.Itoa(len(data)))
|
||||
if total == 0 {
|
||||
req.Header.Set("Content-Range", "bytes */0")
|
||||
} else {
|
||||
end := start + int64(len(data)) - 1
|
||||
req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, total))
|
||||
}
|
||||
hc := d.httpClient
|
||||
if hc == nil {
|
||||
hc = http.DefaultClient
|
||||
}
|
||||
res, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, true, fmt.Errorf("googledrive upload session: put chunk: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK, http.StatusCreated:
|
||||
var item driveFile
|
||||
if err := json.NewDecoder(res.Body).Decode(&item); err != nil {
|
||||
return nil, false, fmt.Errorf("googledrive upload session: decode completed file: %w", err)
|
||||
}
|
||||
return &item, false, nil
|
||||
case http.StatusPermanentRedirect:
|
||||
return nil, false, nil
|
||||
case http.StatusUnauthorized:
|
||||
if err := d.refresh(ctx); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return nil, true, fmt.Errorf("googledrive upload session: unauthorized")
|
||||
default:
|
||||
body, _ := io.ReadAll(io.LimitReader(res.Body, 64*1024))
|
||||
var apiErr apiErrorResp
|
||||
_ = json.Unmarshal(body, &apiErr)
|
||||
if isGoogleUploadHTTPRateLimit(res.StatusCode, res.Header, body, apiErr.Error) {
|
||||
return nil, false, googleUploadRateLimitError(res.StatusCode, res.Header, body, apiErr.Error.Message)
|
||||
}
|
||||
retryable := res.StatusCode == http.StatusTooManyRequests || (res.StatusCode >= 500 && res.StatusCode <= 504)
|
||||
return nil, retryable, fmt.Errorf("googledrive upload session: status=%d body=%s", res.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
currentID := d.rootID
|
||||
for _, name := range splitPath(pathFromRoot) {
|
||||
childID, err := d.findChildDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if childID == "" {
|
||||
childID, err = d.makeDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
currentID = childID
|
||||
}
|
||||
return currentID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) findChildDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
entries, err := d.List(ctx, parentID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir && e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
var item driveFile
|
||||
err := d.request(ctx, d.filesURL(), http.MethodPost, func(req *resty.Request) {
|
||||
req.SetQueryParam("fields", fileInfoFields)
|
||||
req.SetBody(map[string]any{
|
||||
"name": name,
|
||||
"parents": []string{parentID},
|
||||
"mimeType": "application/vnd.google-apps.folder",
|
||||
})
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("googledrive mkdir %s: %w", name, err)
|
||||
}
|
||||
if item.ID == "" {
|
||||
return "", fmt.Errorf("googledrive mkdir %s: empty file id", name)
|
||||
}
|
||||
return item.ID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("googledrive rename: empty file id")
|
||||
}
|
||||
newName = strings.TrimSpace(newName)
|
||||
if newName == "" {
|
||||
return errors.New("googledrive rename: empty new name")
|
||||
}
|
||||
var item driveFile
|
||||
err := d.request(ctx, d.fileURL(fileID), http.MethodPatch, func(req *resty.Request) {
|
||||
req.SetQueryParam("fields", fileInfoFields)
|
||||
req.SetBody(map[string]string{"name": newName})
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return fmt.Errorf("googledrive rename: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("googledrive remove: empty file id")
|
||||
}
|
||||
if err := d.request(ctx, d.fileURL(fileID), http.MethodDelete, nil, nil); err != nil {
|
||||
return fmt.Errorf("googledrive remove: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) findUploadedFileID(ctx context.Context, parentID, name, md5Hex string) (string, error) {
|
||||
entries, err := d.List(ctx, parentID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("googledrive upload verify: %w", err)
|
||||
}
|
||||
var hashHit string
|
||||
for _, e := range entries {
|
||||
if e.IsDir {
|
||||
continue
|
||||
}
|
||||
if !strings.EqualFold(e.Hash, md5Hex) {
|
||||
continue
|
||||
}
|
||||
if e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
if hashHit == "" {
|
||||
hashHit = e.ID
|
||||
}
|
||||
}
|
||||
if hashHit != "" {
|
||||
return hashHit, nil
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir && e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("googledrive upload: uploaded file %q not found in parent %q", name, parentID)
|
||||
}
|
||||
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
func isGoogleUploadHTTPRateLimit(status int, header http.Header, body []byte, apiErr apiErrorBody) bool {
|
||||
if status == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if status == http.StatusForbidden && strings.TrimSpace(header.Get("Retry-After")) != "" {
|
||||
return true
|
||||
}
|
||||
if isGoogleRateLimit(nil, apiErr) {
|
||||
return true
|
||||
}
|
||||
return googleLimitText(string(body))
|
||||
}
|
||||
|
||||
func googleUploadRateLimitError(status int, header http.Header, body []byte, message string) error {
|
||||
if strings.TrimSpace(message) == "" {
|
||||
message = "google drive upload rate limited"
|
||||
}
|
||||
bodyText := strings.TrimSpace(string(body))
|
||||
if bodyText != "" {
|
||||
message = fmt.Sprintf("%s: status=%d body=%s", message, status, bodyText)
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: parseRetryAfterHeader(header.Get("Retry-After")),
|
||||
Err: errors.New(message),
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) refresh(ctx context.Context) error {
|
||||
@@ -288,6 +723,26 @@ func (d *Driver) applyToken(out tokenResp) {
|
||||
}
|
||||
|
||||
func tokenResponseError(prefix string, res *resty.Response, out tokenResp, requireRefresh bool) error {
|
||||
if isGoogleTokenRateLimit(res, out) {
|
||||
message := strings.TrimSpace(out.Text)
|
||||
if message == "" {
|
||||
message = strings.TrimSpace(out.ErrorDescription)
|
||||
}
|
||||
if message == "" {
|
||||
message = strings.TrimSpace(out.Error)
|
||||
}
|
||||
if message == "" {
|
||||
message = "google drive token refresh rate limited"
|
||||
}
|
||||
if res != nil && strings.TrimSpace(res.String()) != "" {
|
||||
message = fmt.Sprintf("%s: status=%d body=%s", message, res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: parseRetryAfter(res),
|
||||
Err: fmt.Errorf("%s: %s", prefix, message),
|
||||
}
|
||||
}
|
||||
if out.Text != "" {
|
||||
return fmt.Errorf("%s: %s", prefix, out.Text)
|
||||
}
|
||||
@@ -380,6 +835,10 @@ func (d *Driver) filesURL() string {
|
||||
return d.apiBaseURL + "/files"
|
||||
}
|
||||
|
||||
func (d *Driver) uploadFilesURL() string {
|
||||
return d.uploadBaseURL + "/files"
|
||||
}
|
||||
|
||||
func (d *Driver) fileURL(fileID string) string {
|
||||
return d.filesURL() + "/" + url.PathEscape(fileID)
|
||||
}
|
||||
@@ -444,18 +903,85 @@ func isGoogleRateLimit(res *resty.Response, body apiErrorBody) bool {
|
||||
if res != nil && res.StatusCode() == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if res != nil && res.StatusCode() == http.StatusForbidden && strings.TrimSpace(res.Header().Get("Retry-After")) != "" {
|
||||
return true
|
||||
}
|
||||
if body.Code == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
for _, e := range body.Errors {
|
||||
reason := strings.ToLower(strings.TrimSpace(e.Reason))
|
||||
switch reason {
|
||||
case "ratelimitexceeded", "userratelimitexceeded", "downloadquotaexceeded", "sharingratelimitexceeded":
|
||||
if googleLimitReason(e.Reason) || googleLimitText(e.Message) {
|
||||
return true
|
||||
}
|
||||
domain := compactGoogleLimitText(e.Domain)
|
||||
if domain == "usagelimits" && (body.Code == http.StatusForbidden || body.Code == http.StatusTooManyRequests) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
msg := strings.ToLower(body.Message)
|
||||
return strings.Contains(msg, "rate limit") || strings.Contains(msg, "too many requests") || strings.Contains(msg, "quota exceeded")
|
||||
return googleLimitText(body.Message)
|
||||
}
|
||||
|
||||
func isGoogleTokenRateLimit(res *resty.Response, out tokenResp) bool {
|
||||
if res != nil {
|
||||
if res.StatusCode() == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if res.StatusCode() == http.StatusForbidden && strings.TrimSpace(res.Header().Get("Retry-After")) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return googleLimitText(out.Text) ||
|
||||
googleLimitText(out.Error) ||
|
||||
googleLimitText(out.ErrorDescription)
|
||||
}
|
||||
|
||||
func googleLimitReason(reason string) bool {
|
||||
switch compactGoogleLimitText(reason) {
|
||||
case "ratelimitexceeded",
|
||||
"userratelimitexceeded",
|
||||
"dailylimitexceeded",
|
||||
"dailylimitexceededunreg",
|
||||
"downloadquotaexceeded",
|
||||
"sharingratelimitexceeded",
|
||||
"quotaexceeded",
|
||||
"uploadlimitexceeded",
|
||||
"storagelimitexceeded",
|
||||
"storagequotaexceeded":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func googleLimitText(text string) bool {
|
||||
text = strings.ToLower(strings.TrimSpace(text))
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
compact := compactGoogleLimitText(text)
|
||||
if strings.Contains(compact, "ratelimitexceeded") ||
|
||||
strings.Contains(compact, "userratelimitexceeded") ||
|
||||
strings.Contains(compact, "dailylimitexceeded") ||
|
||||
strings.Contains(compact, "downloadquotaexceeded") ||
|
||||
strings.Contains(compact, "sharingratelimitexceeded") ||
|
||||
strings.Contains(compact, "quotaexceeded") ||
|
||||
strings.Contains(compact, "toomanyrequests") {
|
||||
return true
|
||||
}
|
||||
return strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "quota exceeded") ||
|
||||
strings.Contains(text, "download quota") ||
|
||||
strings.Contains(text, "sharing rate") ||
|
||||
strings.Contains(text, "daily limit") ||
|
||||
strings.Contains(text, "user rate") ||
|
||||
strings.Contains(text, "usage limit")
|
||||
}
|
||||
|
||||
func compactGoogleLimitText(text string) string {
|
||||
text = strings.ToLower(strings.TrimSpace(text))
|
||||
replacer := strings.NewReplacer("_", "", "-", "", " ", "", ".", "", ":", "")
|
||||
return replacer.Replace(text)
|
||||
}
|
||||
|
||||
func googleRateLimitError(res *resty.Response, message string) error {
|
||||
@@ -486,7 +1012,11 @@ func parseRetryAfter(res *resty.Response) time.Duration {
|
||||
if res == nil {
|
||||
return 0
|
||||
}
|
||||
raw := strings.TrimSpace(res.Header().Get("Retry-After"))
|
||||
return parseRetryAfterHeader(res.Header().Get("Retry-After"))
|
||||
}
|
||||
|
||||
func parseRetryAfterHeader(raw string) time.Duration {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return 0
|
||||
}
|
||||
@@ -502,4 +1032,19 @@ func parseRetryAfter(res *resty.Response) time.Duration {
|
||||
return 0
|
||||
}
|
||||
|
||||
func splitPath(p string) []string {
|
||||
p = strings.Trim(p, "/")
|
||||
if p == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(p, "/")
|
||||
}
|
||||
|
||||
func minInt64(a, b int64) int64 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
|
||||
@@ -2,11 +2,18 @@ package googledrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestInitUsesOnlineRenewAPI(t *testing.T) {
|
||||
@@ -131,6 +138,134 @@ func TestStreamURLReturnsAuthenticatedMediaLinkWithoutRedirectRequirement(t *tes
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAndReportHashUsesResumableSession(t *testing.T) {
|
||||
body := "hello google drive"
|
||||
wantHash := md5.Sum([]byte(body))
|
||||
var sawSession bool
|
||||
var sawUpload bool
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/upload/drive/v3/files":
|
||||
sawSession = true
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("session Authorization = %q", got)
|
||||
}
|
||||
if got := r.URL.Query().Get("uploadType"); got != "resumable" {
|
||||
t.Fatalf("uploadType = %q", got)
|
||||
}
|
||||
if got := r.Header.Get("X-Upload-Content-Length"); got != "18" {
|
||||
t.Fatalf("X-Upload-Content-Length = %q", got)
|
||||
}
|
||||
var meta struct {
|
||||
Name string `json:"name"`
|
||||
Parents []string `json:"parents"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&meta); err != nil {
|
||||
t.Fatalf("decode session metadata: %v", err)
|
||||
}
|
||||
if meta.Name != "clip.mp4" || len(meta.Parents) != 1 || meta.Parents[0] != "parent-1" {
|
||||
t.Fatalf("metadata = %+v", meta)
|
||||
}
|
||||
w.Header().Set("Location", srv.URL+"/upload/session/1")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case "/upload/session/1":
|
||||
sawUpload = true
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("upload Authorization = %q", got)
|
||||
}
|
||||
if got := r.Header.Get("Content-Range"); got != "bytes 0-17/18" {
|
||||
t.Fatalf("Content-Range = %q", got)
|
||||
}
|
||||
gotBody, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read upload body: %v", err)
|
||||
}
|
||||
if string(gotBody) != body {
|
||||
t.Fatalf("upload body = %q", string(gotBody))
|
||||
}
|
||||
writeTestJSONStatus(w, http.StatusCreated, driveFile{
|
||||
ID: "file-uploaded",
|
||||
Name: "clip.mp4",
|
||||
Size: "18",
|
||||
MD5Checksum: hex.EncodeToString(wantHash[:]),
|
||||
})
|
||||
default:
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", APIBaseURL: srv.URL + "/drive/v3"})
|
||||
d.accessToken = "access"
|
||||
res, err := d.UploadAndReportHash(context.Background(), "parent-1", "clip.mp4", strings.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("UploadAndReportHash() error = %v", err)
|
||||
}
|
||||
if !sawSession || !sawUpload {
|
||||
t.Fatalf("saw session/upload = %v/%v, want both", sawSession, sawUpload)
|
||||
}
|
||||
if res.FileID != "file-uploaded" || res.Size != int64(len(body)) || res.Hash != hex.EncodeToString(wantHash[:]) {
|
||||
t.Fatalf("upload result = %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDirAndRenameUseGoogleDriveFileAPI(t *testing.T) {
|
||||
var madeDir bool
|
||||
var renamed bool
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/drive/v3/files":
|
||||
writeTestJSON(w, filesResp{})
|
||||
case r.Method == http.MethodPost && r.URL.Path == "/drive/v3/files":
|
||||
madeDir = true
|
||||
var meta struct {
|
||||
Name string `json:"name"`
|
||||
Parents []string `json:"parents"`
|
||||
MimeType string `json:"mimeType"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&meta); err != nil {
|
||||
t.Fatalf("decode mkdir body: %v", err)
|
||||
}
|
||||
if meta.Name != "91 Spider" || len(meta.Parents) != 1 || meta.Parents[0] != "root" || meta.MimeType != "application/vnd.google-apps.folder" {
|
||||
t.Fatalf("mkdir body = %+v", meta)
|
||||
}
|
||||
writeTestJSON(w, driveFile{ID: "folder-91", Name: "91 Spider", MimeType: "application/vnd.google-apps.folder"})
|
||||
case r.Method == http.MethodPatch && r.URL.Path == "/drive/v3/files/file-1":
|
||||
renamed = true
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode rename body: %v", err)
|
||||
}
|
||||
if body["name"] != "new-name.mp4" {
|
||||
t.Fatalf("rename body = %+v", body)
|
||||
}
|
||||
writeTestJSON(w, driveFile{ID: "file-1", Name: "new-name.mp4"})
|
||||
default:
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", RootID: "root", APIBaseURL: srv.URL + "/drive/v3"})
|
||||
d.accessToken = "access"
|
||||
d.listInterval = -1
|
||||
|
||||
dirID, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureDir() error = %v", err)
|
||||
}
|
||||
if dirID != "folder-91" || !madeDir {
|
||||
t.Fatalf("dirID/madeDir = %q/%v, want folder-91/true", dirID, madeDir)
|
||||
}
|
||||
if err := d.Rename(context.Background(), "file-1", "new-name.mp4"); err != nil {
|
||||
t.Fatalf("Rename() error = %v", err)
|
||||
}
|
||||
if !renamed {
|
||||
t.Fatal("rename endpoint was not called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestRefreshesOnUnauthorized(t *testing.T) {
|
||||
var fileCalls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -179,6 +314,88 @@ func TestRequestRefreshesOnUnauthorized(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateLimitReasonsFollowGoogleDriveErrorShape(t *testing.T) {
|
||||
reasons := []string{
|
||||
"rateLimitExceeded",
|
||||
"userRateLimitExceeded",
|
||||
"dailyLimitExceeded",
|
||||
"dailyLimitExceededUnreg",
|
||||
"downloadQuotaExceeded",
|
||||
"sharingRateLimitExceeded",
|
||||
"quotaExceeded",
|
||||
}
|
||||
for _, reason := range reasons {
|
||||
body := apiErrorBody{
|
||||
Code: http.StatusForbidden,
|
||||
Message: "google drive quota or rate limited",
|
||||
Errors: []struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
LocationType string `json:"location_type"`
|
||||
Location string `json:"location"`
|
||||
}{
|
||||
{Domain: "usageLimits", Reason: reason, Message: reason},
|
||||
},
|
||||
}
|
||||
if !isGoogleRateLimit(nil, body) {
|
||||
t.Fatalf("reason %q not treated as rate limit", reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRateLimitStartsSharedLinkCooldown(t *testing.T) {
|
||||
var calls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
w.Header().Set("Retry-After", "120")
|
||||
writeTestJSONStatus(w, http.StatusForbidden, apiErrorResp{Error: apiErrorBody{
|
||||
Code: http.StatusForbidden,
|
||||
Message: "User rate limit exceeded.",
|
||||
Errors: []struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
LocationType string `json:"location_type"`
|
||||
Location string `json:"location"`
|
||||
}{
|
||||
{Domain: "usageLimits", Reason: "userRateLimitExceeded", Message: "User rate limit exceeded."},
|
||||
},
|
||||
}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", APIBaseURL: srv.URL})
|
||||
d.accessToken = "access"
|
||||
d.linkCooldownDuration = time.Hour
|
||||
|
||||
_, err := d.StreamURL(context.Background(), "file-1")
|
||||
if err == nil {
|
||||
t.Fatal("first StreamURL succeeded, want rate limit")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("first error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != 2*time.Minute {
|
||||
t.Fatalf("retry after = %s, want 2m", rateLimit.RetryAfter)
|
||||
}
|
||||
|
||||
_, err = d.StreamURL(context.Background(), "file-1")
|
||||
if err == nil {
|
||||
t.Fatal("second StreamURL succeeded during cooldown")
|
||||
}
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("second error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("remote calls = %d, want 1; second call should use shared cooldown", calls)
|
||||
}
|
||||
if rateLimit.RetryAfter <= 0 || rateLimit.RetryAfter > 2*time.Minute {
|
||||
t.Fatalf("second retry after = %s, want remaining cooldown", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestJSON(w http.ResponseWriter, v any) {
|
||||
writeTestJSONStatus(w, http.StatusOK, v)
|
||||
}
|
||||
|
||||
@@ -42,8 +42,16 @@ type apiErrorBody struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Errors []struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
LocationType string `json:"location_type"`
|
||||
Location string `json:"location"`
|
||||
} `json:"errors"`
|
||||
}
|
||||
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
Hash string
|
||||
Size int64
|
||||
}
|
||||
|
||||
@@ -40,6 +40,27 @@ type Drive interface {
|
||||
RootID() string
|
||||
}
|
||||
|
||||
// Remover is an optional drive capability. It mirrors OpenList's optional
|
||||
// Remove interface: callers must type-assert before deleting a source file.
|
||||
type Remover interface {
|
||||
Remove(ctx context.Context, fileID string) error
|
||||
}
|
||||
|
||||
// SourceFile carries the catalog metadata available when an administrator
|
||||
// requests deletion of the original source file.
|
||||
type SourceFile struct {
|
||||
FileID string
|
||||
ParentID string
|
||||
Name string
|
||||
Size int64
|
||||
}
|
||||
|
||||
// SourceRemover is an optional, richer removal capability for providers whose
|
||||
// playback ID is not the same ID required by their delete API.
|
||||
type SourceRemover interface {
|
||||
RemoveSource(ctx context.Context, source SourceFile) error
|
||||
}
|
||||
|
||||
type Entry struct {
|
||||
ID string
|
||||
Name string
|
||||
|
||||
@@ -23,17 +23,24 @@ const maxSTRMBytes = 64 * 1024
|
||||
type Config struct {
|
||||
ID string
|
||||
RootPath string
|
||||
// STRMAllowOutsideRoot 允许 .strm 指向存储根目录之外的本地路径。
|
||||
// 默认关闭:strm 等于可以让 /p/stream 读到服务器上的任意文件,只有
|
||||
// 管理员明确知道自己在做什么(例如 strm 库与 rclone 挂载目录分离)
|
||||
// 时才应打开。
|
||||
STRMAllowOutsideRoot bool
|
||||
}
|
||||
|
||||
type Driver struct {
|
||||
id string
|
||||
rootPath string
|
||||
id string
|
||||
rootPath string
|
||||
strmAllowOutsideRoot bool
|
||||
}
|
||||
|
||||
func New(c Config) *Driver {
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
rootPath: c.RootPath,
|
||||
id: c.ID,
|
||||
rootPath: c.RootPath,
|
||||
strmAllowOutsideRoot: c.STRMAllowOutsideRoot,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -230,8 +237,8 @@ func (d *Driver) localSTRMLink(strmPath, target string) (*drives.StreamLink, err
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !within {
|
||||
return nil, errors.New("localstorage: strm target escapes root")
|
||||
if !within && !d.strmAllowOutsideRoot {
|
||||
return nil, errors.New("localstorage: strm target escapes root (enable strm_allow_outside_root to allow)")
|
||||
}
|
||||
if strings.EqualFold(filepath.Ext(p), ".strm") || strings.EqualFold(filepath.Ext(realPath), ".strm") {
|
||||
return nil, errors.New("localstorage: nested strm target is not supported")
|
||||
@@ -257,6 +264,39 @@ func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
p, rel, err := d.pathForID(fileID)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if rel == "" {
|
||||
return errors.New("localstorage: refusing to remove root")
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return errors.New("localstorage: refusing to remove directory")
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return errors.New("localstorage: refusing to remove non-regular file")
|
||||
}
|
||||
if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) root() (string, error) {
|
||||
raw := strings.TrimSpace(d.rootPath)
|
||||
if raw == "" {
|
||||
@@ -276,6 +316,8 @@ func (d *Driver) root() (string, error) {
|
||||
return filepath.Abs(raw)
|
||||
}
|
||||
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
func (d *Driver) pathForID(id string) (string, string, error) {
|
||||
root, err := d.root()
|
||||
if err != nil {
|
||||
|
||||
@@ -195,6 +195,46 @@ func TestStreamURLRejectsSTRMTargetEscapingRootThroughSymlink(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLAllowsSTRMTargetOutsideRootWhenEnabled(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
target := filepath.Join(outside, "movie.mp4")
|
||||
writeLocalStorageTestFile(t, target, []byte("movie-data"))
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "movie.strm"), []byte(target+"\n"))
|
||||
|
||||
// 默认关闭:根目录外的目标仍被拒绝
|
||||
strict := New(Config{ID: "local", RootPath: root})
|
||||
if _, err := strict.StreamURL(context.Background(), encodeRel("movie.strm")); err == nil || !strings.Contains(err.Error(), "strm target escapes root") {
|
||||
t.Fatalf("default error = %v, want strm target escapes root", err)
|
||||
}
|
||||
|
||||
// 开启 strm_allow_outside_root 后放行
|
||||
relaxed := New(Config{ID: "local", RootPath: root, STRMAllowOutsideRoot: true})
|
||||
link, err := relaxed.StreamURL(context.Background(), encodeRel("movie.strm"))
|
||||
if err != nil {
|
||||
t.Fatalf("StreamURL with allow-outside-root: %v", err)
|
||||
}
|
||||
resolved, err := filepath.EvalSymlinks(target)
|
||||
if err != nil {
|
||||
t.Fatalf("eval target: %v", err)
|
||||
}
|
||||
if link.URL != resolved {
|
||||
t.Fatalf("link url = %q, want %q", link.URL, resolved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLAllowOutsideRootStillRejectsNestedSTRM(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
writeLocalStorageTestFile(t, filepath.Join(outside, "inner.strm"), []byte("http://example.com/v.mp4\n"))
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "movie.strm"), []byte(filepath.Join(outside, "inner.strm")+"\n"))
|
||||
|
||||
drv := New(Config{ID: "local", RootPath: root, STRMAllowOutsideRoot: true})
|
||||
if _, err := drv.StreamURL(context.Background(), encodeRel("movie.strm")); err == nil || !strings.Contains(err.Error(), "nested strm") {
|
||||
t.Fatalf("error = %v, want nested strm rejection", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRejectsSymlinkFileIDEscapingRoot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
|
||||
@@ -78,12 +78,38 @@ func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
path, err := d.uploadPath(fileID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return errors.New("localupload: refusing to remove directory")
|
||||
}
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) RootID() string { return d.uploadDir() }
|
||||
|
||||
func (d *Driver) uploadDir() string {
|
||||
return d.uploadDirPath
|
||||
}
|
||||
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
func (d *Driver) uploadPath(fileID string) (string, error) {
|
||||
if strings.TrimSpace(fileID) == "" || filepath.Base(fileID) != fileID {
|
||||
return "", errors.New("invalid upload file id")
|
||||
|
||||
@@ -501,6 +501,17 @@ func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("onedrive remove: empty file id")
|
||||
}
|
||||
if err := d.request(ctx, d.itemURL(fileID), http.MethodDelete, nil, nil); err != nil {
|
||||
return fmt.Errorf("onedrive remove: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) request(ctx context.Context, rawURL, method string, configure func(*resty.Request), out any) error {
|
||||
return d.requestOnce(ctx, rawURL, method, configure, out, true)
|
||||
}
|
||||
@@ -741,3 +752,4 @@ func guessMime(name string) string {
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
@@ -461,6 +461,23 @@ func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
if d.client == nil {
|
||||
return errors.New("p115 remove: driver not initialized")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("p115 remove: empty fileID")
|
||||
}
|
||||
if err := d.client.Delete(fileID); err != nil {
|
||||
return fmt.Errorf("p115 remove: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// bufferAndHashSha1 把 r 全量复制到一个临时文件,同时计算 SHA1。
|
||||
// 返回临时文件(位置在末尾,需调用方 Seek 回 0)、SHA1 hex 大写、实际字节数。
|
||||
//
|
||||
@@ -563,3 +580,4 @@ func guessMime(name string) string {
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
@@ -42,6 +42,7 @@ const (
|
||||
endpointDownloadInfo = "/file/download_info"
|
||||
endpointMkdir = "/file/upload_request"
|
||||
endpointRename = "/file/rename"
|
||||
endpointTrash = "/file/trash"
|
||||
endpointUpload = "/file/upload_request"
|
||||
endpointS3Auth = "/file/s3_upload_object/auth"
|
||||
endpointS3Parts = "/file/s3_repare_upload_parts_batch"
|
||||
@@ -259,8 +260,8 @@ func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader,
|
||||
|
||||
// UploadResult 是 UploadAndReportHash 的返回值。
|
||||
//
|
||||
// FileID 是 123 云盘分配的新文件 ID;Hash 是本次上传的 MD5 HEX(小写),
|
||||
// 与 123 云盘列表返回的 Etag 一致;Size 是实际上传字节数。
|
||||
// FileID 是 123网盘分配的新文件 ID;Hash 是本次上传的 MD5 HEX(小写),
|
||||
// 与 123网盘列表返回的 Etag 一致;Size 是实际上传字节数。
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
Hash string
|
||||
@@ -269,7 +270,7 @@ type UploadResult struct {
|
||||
|
||||
// UploadAndReportHash 把 r 上传到 parentID 目录下的指定文件名,返回新文件元数据。
|
||||
//
|
||||
// 123 云盘 Web 上传协议需要先计算文件 MD5 作为 etag 申请 upload_request。
|
||||
// 123网盘 Web 上传协议需要先计算文件 MD5 作为 etag 申请 upload_request。
|
||||
// 命中 Reuse 时服务端已经秒传;否则用返回的 S3 预签名 URL 分片 PUT,最后
|
||||
// 调 upload_complete/v2 完成。
|
||||
func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
@@ -522,7 +523,7 @@ func (d *Driver) cacheUploadedFile(fileID, parentID, name, md5Hex string, size i
|
||||
}, parentID)
|
||||
}
|
||||
|
||||
// Rename 调用 123 云盘 Web API 把指定 fileID 重命名为 newName。
|
||||
// Rename 调用 123网盘 Web API 把指定 fileID 重命名为 newName。
|
||||
func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
@@ -545,6 +546,32 @@ func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("123pan remove: empty file id")
|
||||
}
|
||||
f, _, err := d.findFile(ctx, fileID)
|
||||
if err != nil {
|
||||
if strings.Contains(strings.ToLower(err.Error()), "not found") {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("123pan remove metadata: %w", err)
|
||||
}
|
||||
body := map[string]any{
|
||||
"driveId": 0,
|
||||
"operation": true,
|
||||
"fileTrashInfoList": []panFile{f},
|
||||
}
|
||||
if _, err := d.request(ctx, endpointTrash, http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(body)
|
||||
}, nil); err != nil {
|
||||
return fmt.Errorf("123pan remove: %w", err)
|
||||
}
|
||||
d.removeCachedFile(fileID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
parts := splitPath(pathFromRoot)
|
||||
currentID := d.rootID
|
||||
@@ -583,7 +610,7 @@ func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, er
|
||||
if resp.Data.FileID != 0 {
|
||||
return strconv.FormatInt(resp.Data.FileID, 10), nil
|
||||
}
|
||||
// 123 云盘创建目录的返回字段不稳定;创建成功但没回 fileId 时回读父目录确认。
|
||||
// 123网盘创建目录的返回字段不稳定;创建成功但没回 fileId 时回读父目录确认。
|
||||
childID, err := d.findChildDir(ctx, parentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -942,6 +969,12 @@ func (d *Driver) renameCachedFile(fileID, newName string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) removeCachedFile(fileID string) {
|
||||
d.fileMu.Lock()
|
||||
delete(d.files, fileID)
|
||||
d.fileMu.Unlock()
|
||||
}
|
||||
|
||||
func (d *Driver) cachedFile(fileID string) (panFile, string, bool) {
|
||||
d.fileMu.RLock()
|
||||
defer d.fileMu.RUnlock()
|
||||
@@ -1008,7 +1041,7 @@ func loginError(message string) error {
|
||||
message = strings.TrimSpace(message)
|
||||
if strings.Contains(message, "境外登录风险") ||
|
||||
(strings.Contains(message, "短信验证码") && strings.Contains(message, "微信")) {
|
||||
return errors.New("123pan login: 账号密码登录被 123 云盘风控拦截,请在浏览器完成短信/微信验证后复制 access_token,并在后台编辑该 123 云盘时只填写 access_token")
|
||||
return errors.New("123pan login: 账号密码登录被 123网盘风控拦截,请在浏览器完成短信/微信验证后复制 access_token,并在后台编辑该 123网盘时只填写 access_token")
|
||||
}
|
||||
if message == "" {
|
||||
message = "login failed"
|
||||
@@ -1111,3 +1144,4 @@ func guessMime(name string) string {
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
@@ -278,7 +278,7 @@ func qrScanPlatformText(platform int) string {
|
||||
case 4:
|
||||
return "微信"
|
||||
case 7:
|
||||
return "123 云盘 App"
|
||||
return "123网盘 App"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -150,7 +150,7 @@ func TestQRCodePollUsesAppToken(t *testing.T) {
|
||||
if wxCodeRequested {
|
||||
t.Fatalf("wx_code should not be called when app token is already returned")
|
||||
}
|
||||
if got.AccessToken != "app-token" || got.PlatformText != "123 云盘 App" {
|
||||
if got.AccessToken != "app-token" || got.PlatformText != "123网盘 App" {
|
||||
t.Fatalf("status = %#v, want app token", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"path"
|
||||
@@ -43,8 +44,9 @@ type Driver struct {
|
||||
algorithms []string
|
||||
userAgent string
|
||||
|
||||
client *resty.Client
|
||||
onTokenUpdate func(access, refresh, captcha, deviceID string)
|
||||
client *resty.Client
|
||||
onTokenUpdate func(access, refresh, captcha, deviceID string)
|
||||
uploadToOSSFunc func(context.Context, *s3Params, io.Reader) error
|
||||
|
||||
// captchaMu serializes captcha-token refreshes triggered by 4002 / 9
|
||||
// recovery in requestOnce. Without it, N concurrent callers all hitting
|
||||
@@ -354,6 +356,19 @@ func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("pikpak remove: empty file id")
|
||||
}
|
||||
if err := d.request(ctx, filesURL+":batchTrash", http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(map[string]any{"ids": []string{fileID}})
|
||||
}, nil); err != nil {
|
||||
return fmt.Errorf("pikpak remove: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
currentID := d.rootID
|
||||
for _, name := range splitPath(pathFromRoot) {
|
||||
@@ -563,3 +578,4 @@ func ParseBoolDefault(raw string, def bool) bool {
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
@@ -6,7 +6,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -26,7 +29,7 @@ import (
|
||||
// - 未命中:resumable.params 含 S3 兼容凭证(access_key / secret /
|
||||
// bucket / endpoint / key / security_token)
|
||||
//
|
||||
// 3. 用 Aliyun OSS SDK PutObject 把字节传到 endpoint+bucket+key
|
||||
// 3. 用 Aliyun OSS SDK PutObject 把字节传到 PikPak 返回的临时 OSS endpoint
|
||||
//
|
||||
// 4. PikPak 服务端轮询 OSS,发现完成后把 resp.File.ID 标记为可用;
|
||||
// 所以 Upload 完成后直接返回 resp.File.ID 即可(一开始就有,
|
||||
@@ -39,6 +42,9 @@ const (
|
||||
// spider91 视频通常 ~100MiB,远低于该值。超过则需走 multipart,
|
||||
// 当前未实现,遇到会显式报错。
|
||||
maxSinglePutSize = 5*1024*1024*1024 - 1
|
||||
// 首次上传失败后最多再重试 3 次。每次重试都会重新申请 PikPak
|
||||
// upload session,以避开偶发不可解析/不可达的临时上传 endpoint。
|
||||
pikpakUploadMaxAttempts = 4
|
||||
)
|
||||
|
||||
// uploadTaskData 是 POST /drive/v1/files 的响应结构。
|
||||
@@ -129,13 +135,49 @@ func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string,
|
||||
_ = os.Remove(tmp.Name())
|
||||
}()
|
||||
|
||||
// 2) 申请上传会话。
|
||||
result := UploadResult{Hash: gcidHex, Size: actualSize}
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= pikpakUploadMaxAttempts; attempt++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
|
||||
resp, err := d.requestUploadSession(ctx, parentID, name, actualSize, gcidHex)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("pikpak upload: request session: %w", err)
|
||||
if !shouldRetryPikPakUploadAttempt(lastErr, attempt) {
|
||||
return UploadResult{}, lastErr
|
||||
}
|
||||
d.logUploadRetry(name, attempt, lastErr)
|
||||
if err := pikpakSleepContext(ctx, pikpakUploadRetryDelay(attempt)); err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
out, err := d.completeUploadAttempt(ctx, tmp, parentID, name, result, resp)
|
||||
if err == nil {
|
||||
return out, nil
|
||||
}
|
||||
lastErr = err
|
||||
if !shouldRetryPikPakUploadAttempt(lastErr, attempt) {
|
||||
return UploadResult{}, lastErr
|
||||
}
|
||||
d.logUploadRetry(name, attempt, lastErr)
|
||||
if err := pikpakSleepContext(ctx, pikpakUploadRetryDelay(attempt)); err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
}
|
||||
return UploadResult{}, lastErr
|
||||
}
|
||||
|
||||
func (d *Driver) requestUploadSession(ctx context.Context, parentID, name string, size int64, gcidHex string) (uploadTaskData, error) {
|
||||
var resp uploadTaskData
|
||||
if err := d.request(ctx, filesURL, http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(map[string]any{
|
||||
"kind": "drive#file",
|
||||
"name": name,
|
||||
"size": actualSize,
|
||||
"size": size,
|
||||
"hash": gcidHex,
|
||||
"upload_type": "UPLOAD_TYPE_RESUMABLE",
|
||||
"objProvider": map[string]any{"provider": "UPLOAD_TYPE_UNKNOWN"},
|
||||
@@ -143,12 +185,13 @@ func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string,
|
||||
"folder_type": "NORMAL",
|
||||
})
|
||||
}, &resp); err != nil {
|
||||
return UploadResult{}, fmt.Errorf("pikpak upload: request session: %w", err)
|
||||
return uploadTaskData{}, err
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
result := UploadResult{Hash: gcidHex, Size: actualSize}
|
||||
|
||||
// 3) 命中秒传:服务端已经知道这个 hash,直接返回新文件 ID。
|
||||
func (d *Driver) completeUploadAttempt(ctx context.Context, tmp *os.File, parentID, name string, result UploadResult, resp uploadTaskData) (UploadResult, error) {
|
||||
// 命中秒传:服务端已经知道这个 hash,直接返回新文件 ID。
|
||||
if resp.Resumable == nil {
|
||||
if resp.File.ID != "" {
|
||||
result.FileID = resp.File.ID
|
||||
@@ -163,7 +206,7 @@ func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string,
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// 4) 未命中秒传:把字节传到 S3 兼容存储。
|
||||
// 未命中秒传:把字节传到 S3 兼容存储。
|
||||
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
|
||||
return UploadResult{}, fmt.Errorf("pikpak upload: seek tmp: %w", err)
|
||||
}
|
||||
@@ -171,7 +214,7 @@ func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string,
|
||||
return UploadResult{}, fmt.Errorf("pikpak upload: oss put: %w", err)
|
||||
}
|
||||
|
||||
// 5) 拿到 fileID。优先走响应里的预分配 ID;为空就回查目录。
|
||||
// 拿到 fileID。优先走响应里的预分配 ID;为空就回查目录。
|
||||
if resp.File.ID != "" {
|
||||
result.FileID = resp.File.ID
|
||||
return result, nil
|
||||
@@ -184,6 +227,58 @@ func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string,
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func shouldRetryPikPakUploadAttempt(err error, attempt int) bool {
|
||||
return attempt < pikpakUploadMaxAttempts && isRetryablePikPakUploadError(err)
|
||||
}
|
||||
|
||||
func pikpakUploadRetryDelay(attempt int) time.Duration {
|
||||
if attempt <= 0 {
|
||||
return 0
|
||||
}
|
||||
return time.Duration(attempt) * time.Second
|
||||
}
|
||||
|
||||
func (d *Driver) logUploadRetry(name string, attempt int, err error) {
|
||||
log.Printf("[pikpak] upload retry drive=%s name=%q next_attempt=%d/%d err=%v",
|
||||
d.id, name, attempt+1, pikpakUploadMaxAttempts, err)
|
||||
}
|
||||
|
||||
func isRetryablePikPakUploadError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return false
|
||||
}
|
||||
var serviceErr oss.ServiceError
|
||||
if errors.As(err, &serviceErr) {
|
||||
return serviceErr.StatusCode == http.StatusTooManyRequests || serviceErr.StatusCode >= 500
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) {
|
||||
return true
|
||||
}
|
||||
text := strings.ToLower(err.Error())
|
||||
return strings.Contains(text, "no such host") ||
|
||||
strings.Contains(text, "temporary failure in name resolution") ||
|
||||
strings.Contains(text, "server misbehaving") ||
|
||||
strings.Contains(text, "connection reset") ||
|
||||
strings.Contains(text, "connection refused") ||
|
||||
strings.Contains(text, "broken pipe") ||
|
||||
strings.Contains(text, "eof") ||
|
||||
strings.Contains(text, "i/o timeout") ||
|
||||
strings.Contains(text, "tls handshake timeout") ||
|
||||
strings.Contains(text, "http 429") ||
|
||||
strings.Contains(text, "http 500") ||
|
||||
strings.Contains(text, "http 502") ||
|
||||
strings.Contains(text, "http 503") ||
|
||||
strings.Contains(text, "http 504") ||
|
||||
strings.Contains(text, "http 509") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "temporarily unavailable") ||
|
||||
strings.Contains(text, "service unavailable")
|
||||
}
|
||||
|
||||
// bufferAndHashGCID 把 r 复制到一个临时文件,同时计算 GCID。
|
||||
// 返回临时文件(位置在末尾,需要调用方 Seek 回 0)、GCID hex 大写、实际写入字节数。
|
||||
//
|
||||
@@ -215,10 +310,13 @@ func bufferAndHashGCID(r io.Reader, size int64) (*os.File, string, int64, error)
|
||||
//
|
||||
// 参数复用 PikPak 的临时凭证;必须带 Security Token 头部 + UserAgent,与 OpenList 一致。
|
||||
func (d *Driver) uploadToOSS(ctx context.Context, p *s3Params, body io.Reader) error {
|
||||
if d.uploadToOSSFunc != nil {
|
||||
return d.uploadToOSSFunc(ctx, p, body)
|
||||
}
|
||||
if p == nil {
|
||||
return errors.New("pikpak upload: nil s3 params")
|
||||
}
|
||||
client, err := oss.New(p.Endpoint, p.AccessKeyID, p.AccessKeySecret)
|
||||
client, err := newPikPakOSSClient(p)
|
||||
if err != nil {
|
||||
return fmt.Errorf("oss client: %w", err)
|
||||
}
|
||||
@@ -235,6 +333,44 @@ func (d *Driver) uploadToOSS(ctx context.Context, p *s3Params, body io.Reader) e
|
||||
)
|
||||
}
|
||||
|
||||
func newPikPakOSSClient(p *s3Params, options ...oss.ClientOption) (*oss.Client, error) {
|
||||
if p == nil {
|
||||
return nil, errors.New("pikpak upload: nil s3 params")
|
||||
}
|
||||
clientOptions := make([]oss.ClientOption, 0, len(options)+1)
|
||||
if isPikPakCNAMEEndpoint(p.Endpoint) {
|
||||
clientOptions = append(clientOptions, oss.UseCname(true))
|
||||
}
|
||||
clientOptions = append(clientOptions, options...)
|
||||
return oss.New(p.Endpoint, p.AccessKeyID, p.AccessKeySecret, clientOptions...)
|
||||
}
|
||||
|
||||
func isPikPakCNAMEEndpoint(endpoint string) bool {
|
||||
host := endpointHost(endpoint)
|
||||
if host == "" {
|
||||
return false
|
||||
}
|
||||
host = strings.TrimSuffix(strings.ToLower(host), ".")
|
||||
return host != "mypikpak.com" && host != "mypikpak.net" &&
|
||||
(strings.HasSuffix(host, ".mypikpak.com") || strings.HasSuffix(host, ".mypikpak.net"))
|
||||
}
|
||||
|
||||
func endpointHost(endpoint string) string {
|
||||
endpoint = strings.TrimSpace(endpoint)
|
||||
if endpoint == "" {
|
||||
return ""
|
||||
}
|
||||
if u, err := url.Parse(endpoint); err == nil && u.Host != "" {
|
||||
endpoint = u.Host
|
||||
} else if idx := strings.IndexByte(endpoint, '/'); idx >= 0 {
|
||||
endpoint = endpoint[:idx]
|
||||
}
|
||||
if host, _, err := net.SplitHostPort(endpoint); err == nil {
|
||||
endpoint = host
|
||||
}
|
||||
return strings.Trim(endpoint, "[]")
|
||||
}
|
||||
|
||||
type readerWithCtx struct {
|
||||
ctx context.Context
|
||||
r io.Reader
|
||||
|
||||
@@ -6,12 +6,15 @@ import (
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||||
"github.com/go-resty/resty/v2"
|
||||
)
|
||||
|
||||
@@ -181,6 +184,95 @@ func TestUploadInstantSuccessFallsBackToListWhenFileIDMissing(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadRetriesWithNewSessionWhenOSSEndpointDNSFails(t *testing.T) {
|
||||
sessionRequests := 0
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
t.Errorf("method = %q, want POST", r.Method)
|
||||
}
|
||||
sessionRequests++
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(fmt.Sprintf(`{
|
||||
"upload_type": "UPLOAD_TYPE_RESUMABLE",
|
||||
"resumable": {
|
||||
"kind": "drive#resumable",
|
||||
"provider": "UPLOAD_TYPE_UNKNOWN",
|
||||
"params": {
|
||||
"access_key_id": "ak",
|
||||
"access_key_secret": "sk",
|
||||
"bucket": "bucket",
|
||||
"endpoint": "https://vip-lixian-%02d.upload-a10b.mypikpak.com",
|
||||
"key": "object-key-%02d",
|
||||
"security_token": "token"
|
||||
}
|
||||
},
|
||||
"file": {"id": "retry-file-%02d", "name": "retry.mp4", "kind": "drive#file"}
|
||||
}`, sessionRequests, sessionRequests, sessionRequests)))
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
uploadAttempts := 0
|
||||
var uploaded []byte
|
||||
d.uploadToOSSFunc = func(_ context.Context, _ *s3Params, body io.Reader) error {
|
||||
uploadAttempts++
|
||||
if uploadAttempts == 1 {
|
||||
return &net.DNSError{Err: "no such host", Name: "vip-lixian-01.upload-a10b.mypikpak.com"}
|
||||
}
|
||||
var err error
|
||||
uploaded, err = io.ReadAll(body)
|
||||
return err
|
||||
}
|
||||
|
||||
payload := []byte("retry payload body")
|
||||
id, err := d.Upload(context.Background(), "parent-id", "retry.mp4", bytes.NewReader(payload), int64(len(payload)))
|
||||
if err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
if id != "retry-file-02" {
|
||||
t.Fatalf("file id = %q, want retry-file-02 from the second session", id)
|
||||
}
|
||||
if sessionRequests != 2 {
|
||||
t.Fatalf("session requests = %d, want 2", sessionRequests)
|
||||
}
|
||||
if uploadAttempts != 2 {
|
||||
t.Fatalf("upload attempts = %d, want 2", uploadAttempts)
|
||||
}
|
||||
if !bytes.Equal(uploaded, payload) {
|
||||
t.Fatalf("uploaded body = %q, want %q", string(uploaded), string(payload))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPikPakOSSClientUsesCNAMEForPikPakUploadEndpoint(t *testing.T) {
|
||||
params := &s3Params{
|
||||
AccessKeyID: "ak",
|
||||
AccessKeySecret: "sk",
|
||||
Bucket: "vip-lixian-07",
|
||||
Endpoint: "http://upload-a10b.mypikpak.com",
|
||||
Key: "upload_tmp/object-key",
|
||||
}
|
||||
client, err := newPikPakOSSClient(params)
|
||||
if err != nil {
|
||||
t.Fatalf("new oss client: %v", err)
|
||||
}
|
||||
bucket, err := client.Bucket(params.Bucket)
|
||||
if err != nil {
|
||||
t.Fatalf("bucket: %v", err)
|
||||
}
|
||||
signed, err := bucket.SignURL(params.Key, oss.HTTPPut, 60)
|
||||
if err != nil {
|
||||
t.Fatalf("sign url: %v", err)
|
||||
}
|
||||
if strings.Contains(signed, "vip-lixian-07.upload-a10b.mypikpak.com") {
|
||||
t.Fatalf("signed url uses invalid bucket-prefixed PikPak host: %s", signed)
|
||||
}
|
||||
if !strings.Contains(signed, "http://upload-a10b.mypikpak.com/upload_tmp%2Fobject-key") {
|
||||
t.Fatalf("signed url = %s, want PikPak endpoint host with object key path", signed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadRejectsInvalidArguments(t *testing.T) {
|
||||
d := New(Config{ID: "x", Username: "u", Password: "p", Platform: "web"})
|
||||
cases := []struct {
|
||||
|
||||
@@ -16,23 +16,23 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
defaultUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) quark-cloud-drive/2.5.20 Chrome/100.0.4896.160 Electron/18.3.5.4-b478491100 Safari/537.36 Channel/pckk_other_ch"
|
||||
defaultUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) quark-cloud-drive/2.5.20 Chrome/100.0.4896.160 Electron/18.3.5.4-b478491100 Safari/537.36 Channel/pckk_other_ch"
|
||||
defaultReferer = "https://pan.quark.cn"
|
||||
defaultAPI = "https://drive.quark.cn/1/clouddrive"
|
||||
defaultPR = "ucpro"
|
||||
)
|
||||
|
||||
type Driver struct {
|
||||
id string
|
||||
cookie string
|
||||
rootID string
|
||||
ua string
|
||||
referer string
|
||||
apiBase string
|
||||
pr string
|
||||
client *resty.Client
|
||||
onCookieUpdate func(string)
|
||||
useTranscodingAddress bool
|
||||
id string
|
||||
cookie string
|
||||
rootID string
|
||||
ua string
|
||||
referer string
|
||||
apiBase string
|
||||
pr string
|
||||
client *resty.Client
|
||||
onCookieUpdate func(string)
|
||||
useTranscodingAddress bool
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -60,7 +60,7 @@ func New(c Config) *Driver {
|
||||
onCookieUpdate: c.OnCookieUpdate,
|
||||
}
|
||||
d.client = resty.New().
|
||||
SetTimeout(30 * time.Second).
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*").
|
||||
SetHeader("Referer", d.referer).
|
||||
SetHeader("User-Agent", d.ua)
|
||||
@@ -269,6 +269,22 @@ func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader,
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("quark remove: empty file id")
|
||||
}
|
||||
body := map[string]any{
|
||||
"action_type": 1,
|
||||
"exclude_fids": []string{},
|
||||
"filelist": []string{fileID},
|
||||
}
|
||||
if err := d.request(ctx, "/file/delete", http.MethodPost, nil, body, nil); err != nil {
|
||||
return fmt.Errorf("quark remove: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------- helpers ----------
|
||||
|
||||
func fileToEntry(f *file, parentID string) drives.Entry {
|
||||
@@ -343,3 +359,4 @@ func setCookieValue(cookie, key, value string) string {
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,653 @@
|
||||
package scriptcrawler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/fingerprint"
|
||||
)
|
||||
|
||||
const (
|
||||
scriptCrawlerDuplicateBytes = "duplicate-video-bytes"
|
||||
scriptCrawlerUniqueBytes = "unique-video-bytes"
|
||||
)
|
||||
|
||||
func writeScriptCrawlerFFprobeStub(t *testing.T, dir string, ok bool) string {
|
||||
t.Helper()
|
||||
name := "ffprobe-ok.sh"
|
||||
body := "#!/bin/sh\necho video\nexit 0\n"
|
||||
if !ok {
|
||||
name = "ffprobe-fail.sh"
|
||||
body = "#!/bin/sh\necho 'moov atom not found' >&2\nexit 1\n"
|
||||
}
|
||||
path := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
|
||||
t.Fatalf("write ffprobe stub: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func writeScriptCrawlerFFmpegStub(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, "ffmpeg-hls.sh")
|
||||
body := "#!/bin/sh\nout=\"\"\nfor arg do out=\"$arg\"; done\nprintf 'hls-video-bytes' > \"$out\"\n"
|
||||
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
|
||||
t.Fatalf("write ffmpeg stub: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceImportsLocalFileAndSkipsExisting(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
dummyScript := filepath.Join(tmp, "helper-script")
|
||||
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
|
||||
t.Fatalf("write dummy script: %v", err)
|
||||
}
|
||||
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
|
||||
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
|
||||
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
|
||||
t.Fatalf("write helper wrapper: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
CrawlerName: "Demo Crawler",
|
||||
PythonPath: wrapper,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
ScriptPath: dummyScript,
|
||||
})
|
||||
res, err := c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 || res.Skipped != 0 || res.Failed != 0 {
|
||||
t.Fatalf("result = new:%d skipped:%d failed:%d, want 1/0/0", res.NewVideos, res.Skipped, res.Failed)
|
||||
}
|
||||
v, err := cat.GetVideo(ctx, BuildVideoID("demo", "abc-123"))
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if v.Title != "Imported From Helper" || v.FileID != "abc-123.mp4" || v.Size == 0 {
|
||||
t.Fatalf("video = title:%q file:%q size:%d", v.Title, v.FileID, v.Size)
|
||||
}
|
||||
if !hasString(v.Tags, "Demo Crawler") {
|
||||
t.Fatalf("video tags = %#v, want crawler name tag", v.Tags)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(drv.VideosDir(), "abc-123.mp4")); err != nil {
|
||||
t.Fatalf("video file not copied: %v", err)
|
||||
}
|
||||
|
||||
res, err = c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("second run: %v", err)
|
||||
}
|
||||
if res.NewVideos != 0 || res.Skipped != 1 {
|
||||
t.Fatalf("second result = new:%d skipped:%d, want 0/1", res.NewVideos, res.Skipped)
|
||||
}
|
||||
if res.SeenSnapshot != 1 {
|
||||
t.Fatalf("seen snapshot = %d, want 1", res.SeenSnapshot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceUsesSourceKindNamespace(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
dummyScript := filepath.Join(tmp, "helper-script")
|
||||
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
|
||||
t.Fatalf("write dummy script: %v", err)
|
||||
}
|
||||
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
|
||||
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
|
||||
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
|
||||
t.Fatalf("write helper wrapper: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
SourceKind: "spider91",
|
||||
PythonPath: wrapper,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
ScriptPath: dummyScript,
|
||||
})
|
||||
res, err := c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 || res.SeenSnapshot != 0 {
|
||||
t.Fatalf("result = new:%d seen:%d, want 1/0", res.NewVideos, res.SeenSnapshot)
|
||||
}
|
||||
videoID := BuildVideoIDForKind("spider91", "demo", "abc-123")
|
||||
if _, err := cat.GetVideo(ctx, videoID); err != nil {
|
||||
t.Fatalf("get source-kind video: %v", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, BuildVideoID("demo", "abc-123")); err == nil {
|
||||
t.Fatalf("default namespace video unexpectedly exists")
|
||||
}
|
||||
|
||||
res, err = c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("second run: %v", err)
|
||||
}
|
||||
if res.NewVideos != 0 || res.Skipped != 1 || res.SeenSnapshot != 1 {
|
||||
t.Fatalf("second result = new:%d skipped:%d seen:%d, want 0/1/1", res.NewVideos, res.Skipped, res.SeenSnapshot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOncePassesAbsoluteJobPathsWhenWorkDirDiffers(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
t.Chdir(tmp)
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join("data", "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
scriptDir := filepath.Join(tmp, "scripts")
|
||||
if err := os.MkdirAll(scriptDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir script dir: %v", err)
|
||||
}
|
||||
dummyScript := filepath.Join(scriptDir, "helper-script")
|
||||
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
|
||||
t.Fatalf("write dummy script: %v", err)
|
||||
}
|
||||
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
|
||||
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
|
||||
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
|
||||
t.Fatalf("write helper wrapper: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_ASSERT_ABS", "1")
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
PythonPath: wrapper,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
ScriptPath: dummyScript,
|
||||
WorkDir: scriptDir,
|
||||
})
|
||||
res, err := c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 || res.Skipped != 0 || res.Failed != 0 {
|
||||
t.Fatalf("result = new:%d skipped:%d failed:%d, want 1/0/0", res.NewVideos, res.Skipped, res.Failed)
|
||||
}
|
||||
if !filepath.IsAbs(res.JobFile) || !filepath.IsAbs(res.SeenFile) {
|
||||
t.Fatalf("result paths should be absolute: job=%q seen=%q", res.JobFile, res.SeenFile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceImportsSimpleMediaURLWithoutSourceID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/video.mp4" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte("simple-video-bytes"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
dummyScript := filepath.Join(tmp, "helper-script")
|
||||
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
|
||||
t.Fatalf("write dummy script: %v", err)
|
||||
}
|
||||
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
|
||||
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
|
||||
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
|
||||
t.Fatalf("write helper wrapper: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_SIMPLE", "1")
|
||||
t.Setenv("GO_SCRIPTCRAWLER_MEDIA_URL", srv.URL+"/video.mp4?token=first")
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
PythonPath: wrapper,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
ScriptPath: dummyScript,
|
||||
HTTPClient: srv.Client(),
|
||||
})
|
||||
res, err := c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 || res.Skipped != 0 || res.Failed != 0 {
|
||||
t.Fatalf("result = new:%d skipped:%d failed:%d, want 1/0/0", res.NewVideos, res.Skipped, res.Failed)
|
||||
}
|
||||
videos, err := cat.ListVideosByDrive(ctx, "demo")
|
||||
if err != nil {
|
||||
t.Fatalf("list videos: %v", err)
|
||||
}
|
||||
if len(videos) != 1 {
|
||||
t.Fatalf("videos = %d, want 1", len(videos))
|
||||
}
|
||||
v := videos[0]
|
||||
if !strings.HasPrefix(v.ID, BuildVideoID("demo", "auto-")) {
|
||||
t.Fatalf("video id = %q, want generated auto source id", v.ID)
|
||||
}
|
||||
if v.Title != "Simple Protocol Video" || v.Ext != "mp4" || v.ThumbnailURL != "" || v.Size == 0 {
|
||||
t.Fatalf("video = title:%q ext:%q thumb:%q size:%d", v.Title, v.Ext, v.ThumbnailURL, v.Size)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(drv.VideosDir(), v.FileID)); err != nil {
|
||||
t.Fatalf("video file not downloaded: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_SCRIPTCRAWLER_MEDIA_URL", srv.URL+"/video.mp4?token=second")
|
||||
res, err = c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("second run: %v", err)
|
||||
}
|
||||
if res.NewVideos != 0 || res.Skipped != 1 {
|
||||
t.Fatalf("second result = new:%d skipped:%d, want 0/1", res.NewVideos, res.Skipped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceSkipsFingerprintDuplicateAndContinues(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
|
||||
seedFile := "seed-canonical.mp4"
|
||||
if err := os.WriteFile(filepath.Join(drv.VideosDir(), seedFile), []byte(scriptCrawlerDuplicateBytes), 0o644); err != nil {
|
||||
t.Fatalf("write seed video: %v", err)
|
||||
}
|
||||
seed := &catalog.Video{
|
||||
ID: "seed-for-hash",
|
||||
DriveID: drv.ID(),
|
||||
FileID: seedFile,
|
||||
Title: "Seed",
|
||||
Size: int64(len(scriptCrawlerDuplicateBytes)),
|
||||
PublishedAt: time.Now(),
|
||||
}
|
||||
sampled, err := fingerprint.Compute(ctx, drv, seed, fingerprint.Config{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("compute seed fingerprint: %v", err)
|
||||
}
|
||||
_ = os.Remove(filepath.Join(drv.VideosDir(), seedFile))
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "existing-canonical",
|
||||
DriveID: "other-drive",
|
||||
FileID: "existing.mp4",
|
||||
FileName: "existing.mp4",
|
||||
Title: "Existing Canonical",
|
||||
Size: int64(len(scriptCrawlerDuplicateBytes)),
|
||||
Ext: "mp4",
|
||||
SampledSHA256: sampled,
|
||||
FingerprintStatus: "ready",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed canonical video: %v", err)
|
||||
}
|
||||
|
||||
dummyScript := filepath.Join(tmp, "helper-script")
|
||||
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
|
||||
t.Fatalf("write dummy script: %v", err)
|
||||
}
|
||||
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
|
||||
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
|
||||
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
|
||||
t.Fatalf("write helper wrapper: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_DUP_UNIQUE", "1")
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
PythonPath: wrapper,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
ScriptPath: dummyScript,
|
||||
})
|
||||
res, err := c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 || res.Skipped != 1 || res.Failed != 0 || res.TotalEntries != 2 {
|
||||
t.Fatalf("result = total:%d new:%d skipped:%d failed:%d, want 2/1/1/0", res.TotalEntries, res.NewVideos, res.Skipped, res.Failed)
|
||||
}
|
||||
if res.CandidateBudget <= res.TargetNew {
|
||||
t.Fatalf("candidate budget = %d, target = %d; want expanded budget", res.CandidateBudget, res.TargetNew)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, BuildVideoID("demo", "dup-source")); err == nil {
|
||||
t.Fatal("duplicate candidate should not be imported")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(drv.VideosDir(), "dup-source.mp4")); !os.IsNotExist(err) {
|
||||
t.Fatalf("duplicate local file stat = %v, want removed", err)
|
||||
}
|
||||
v, err := cat.GetVideo(ctx, BuildVideoID("demo", "unique-source"))
|
||||
if err != nil {
|
||||
t.Fatalf("unique video should be imported: %v", err)
|
||||
}
|
||||
if v.SampledSHA256 == "" || v.FingerprintStatus != "ready" {
|
||||
t.Fatalf("unique fingerprint = %q status=%q, want ready sampled fingerprint", v.SampledSHA256, v.FingerprintStatus)
|
||||
}
|
||||
seen, err := cat.ListCrawlerSourceIDs(ctx, Kind, "demo")
|
||||
if err != nil {
|
||||
t.Fatalf("list seen source ids: %v", err)
|
||||
}
|
||||
seenSet := map[string]bool{}
|
||||
for _, id := range seen {
|
||||
seenSet[id] = true
|
||||
}
|
||||
if !seenSet["dup-source"] || !seenSet["unique-source"] {
|
||||
t.Fatalf("seen ids = %#v, want duplicate and imported source ids", seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceRejectsInvalidDownloadedVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
dummyScript := filepath.Join(tmp, "helper-script")
|
||||
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
|
||||
t.Fatalf("write dummy script: %v", err)
|
||||
}
|
||||
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
|
||||
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
|
||||
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
|
||||
t.Fatalf("write helper wrapper: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
CrawlerName: "Demo Crawler",
|
||||
PythonPath: wrapper,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, false),
|
||||
ScriptPath: dummyScript,
|
||||
})
|
||||
res, err := c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if res.NewVideos != 0 || res.Skipped != 0 || res.Failed != 1 || res.TotalEntries != 1 {
|
||||
t.Fatalf("result = total:%d new:%d skipped:%d failed:%d, want 1/0/0/1", res.TotalEntries, res.NewVideos, res.Skipped, res.Failed)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, BuildVideoID("demo", "abc-123")); err == nil {
|
||||
t.Fatal("invalid video should not be imported")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(drv.VideosDir(), "abc-123.mp4")); !os.IsNotExist(err) {
|
||||
t.Fatalf("invalid local video stat = %v, want removed", err)
|
||||
}
|
||||
seen, err := cat.ListCrawlerSourceIDs(ctx, Kind, "demo")
|
||||
if err != nil {
|
||||
t.Fatalf("list seen source ids: %v", err)
|
||||
}
|
||||
if len(seen) != 0 {
|
||||
t.Fatalf("seen ids = %#v, want none for invalid video", seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceDownloadsHLSMediaURL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
dummyScript := filepath.Join(tmp, "helper-script")
|
||||
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
|
||||
t.Fatalf("write dummy script: %v", err)
|
||||
}
|
||||
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
|
||||
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
|
||||
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
|
||||
t.Fatalf("write helper wrapper: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HLS", "1")
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
CrawlerName: "Demo Crawler",
|
||||
PythonPath: wrapper,
|
||||
FFmpegPath: writeScriptCrawlerFFmpegStub(t, tmp),
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
ScriptPath: dummyScript,
|
||||
})
|
||||
res, err := c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 || res.Skipped != 0 || res.Failed != 0 {
|
||||
t.Fatalf("result = new:%d skipped:%d failed:%d, want 1/0/0", res.NewVideos, res.Skipped, res.Failed)
|
||||
}
|
||||
v, err := cat.GetVideo(ctx, BuildVideoID("demo", "hls-source"))
|
||||
if err != nil {
|
||||
t.Fatalf("get hls video: %v", err)
|
||||
}
|
||||
if v.FileID != "hls-source.mp4" || v.Size != int64(len("hls-video-bytes")) {
|
||||
t.Fatalf("video file=%q size=%d, want hls-source.mp4 size %d", v.FileID, v.Size, len("hls-video-bytes"))
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(drv.VideosDir(), "hls-source.mp4"))
|
||||
if err != nil {
|
||||
t.Fatalf("read hls output: %v", err)
|
||||
}
|
||||
if string(data) != "hls-video-bytes" {
|
||||
t.Fatalf("hls output = %q", string(data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptCrawlerHelperProcess(t *testing.T) {
|
||||
if os.Getenv("GO_WANT_SCRIPTCRAWLER_HELPER") != "1" {
|
||||
return
|
||||
}
|
||||
args := os.Args
|
||||
jobPath := ""
|
||||
for i := 0; i < len(args)-1; i++ {
|
||||
if args[i] == "--job" {
|
||||
jobPath = args[i+1]
|
||||
break
|
||||
}
|
||||
}
|
||||
if jobPath == "" {
|
||||
fmt.Fprintln(os.Stderr, "missing --job")
|
||||
os.Exit(2)
|
||||
}
|
||||
data, err := os.ReadFile(jobPath)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
var job Job
|
||||
if err := json.Unmarshal(data, &job); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
if os.Getenv("GO_WANT_SCRIPTCRAWLER_ASSERT_ABS") == "1" {
|
||||
if !filepath.IsAbs(jobPath) || !filepath.IsAbs(job.SeenSourceIDsFile) || !filepath.IsAbs(job.OutputDir) {
|
||||
fmt.Fprintf(os.Stderr, "expected absolute paths, got job=%q seen=%q output=%q\n", jobPath, job.SeenSourceIDsFile, job.OutputDir)
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
if os.Getenv("GO_WANT_SCRIPTCRAWLER_SIMPLE") == "1" {
|
||||
event := map[string]any{
|
||||
"title": "Simple Protocol Video",
|
||||
"media_url": os.Getenv("GO_SCRIPTCRAWLER_MEDIA_URL"),
|
||||
}
|
||||
_ = json.NewEncoder(os.Stdout).Encode(event)
|
||||
os.Exit(0)
|
||||
}
|
||||
if os.Getenv("GO_WANT_SCRIPTCRAWLER_HLS") == "1" {
|
||||
event := Event{
|
||||
Type: "item",
|
||||
Item: Item{
|
||||
SourceID: "hls-source",
|
||||
Title: "HLS Protocol Video",
|
||||
Author: "helper",
|
||||
Media: MediaRef{
|
||||
URL: "https://media.example.test/video.m3u8",
|
||||
Headers: map[string]string{
|
||||
"Referer": "https://example.test/",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_ = json.NewEncoder(os.Stdout).Encode(event)
|
||||
os.Exit(0)
|
||||
}
|
||||
if os.Getenv("GO_WANT_SCRIPTCRAWLER_DUP_UNIQUE") == "1" {
|
||||
duplicateFile := filepath.Join(job.OutputDir, "duplicate.mp4")
|
||||
if err := os.WriteFile(duplicateFile, []byte(scriptCrawlerDuplicateBytes), 0o644); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
uniqueFile := filepath.Join(job.OutputDir, "unique.mp4")
|
||||
if err := os.WriteFile(uniqueFile, []byte(scriptCrawlerUniqueBytes), 0o644); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
for _, event := range []Event{
|
||||
{
|
||||
Type: "item",
|
||||
Item: Item{
|
||||
SourceID: "dup-source",
|
||||
Title: "Duplicate Candidate",
|
||||
Author: "helper",
|
||||
Media: MediaRef{LocalFile: duplicateFile},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: "item",
|
||||
Item: Item{
|
||||
SourceID: "unique-source",
|
||||
Title: "Unique Candidate",
|
||||
Author: "helper",
|
||||
Media: MediaRef{LocalFile: uniqueFile},
|
||||
},
|
||||
},
|
||||
} {
|
||||
_ = json.NewEncoder(os.Stdout).Encode(event)
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
localFile := filepath.Join(job.OutputDir, "helper.mp4")
|
||||
if err := os.WriteFile(localFile, []byte("helper-video"), 0o644); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
event := Event{
|
||||
Type: "item",
|
||||
Item: Item{
|
||||
SourceID: "abc-123",
|
||||
Title: "Imported From Helper",
|
||||
Author: "helper",
|
||||
Media: MediaRef{LocalFile: localFile},
|
||||
},
|
||||
}
|
||||
_ = json.NewEncoder(os.Stdout).Encode(event)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
func hasString(values []string, want string) bool {
|
||||
for _, value := range values {
|
||||
if value == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
// Package scriptcrawler provides a generic local drive for script-based
|
||||
// crawlers. A crawler script discovers videos; the Go runner downloads them
|
||||
// into this drive and the existing preview/fingerprint workers consume them
|
||||
// through the normal drives.Drive interface.
|
||||
package scriptcrawler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
const Kind = "scriptcrawler"
|
||||
|
||||
type Config struct {
|
||||
ID string
|
||||
RootDir string
|
||||
}
|
||||
|
||||
type Driver struct {
|
||||
id string
|
||||
rootDir string
|
||||
}
|
||||
|
||||
func New(c Config) *Driver {
|
||||
return &Driver{id: c.ID, rootDir: c.RootDir}
|
||||
}
|
||||
|
||||
func (d *Driver) Kind() string { return Kind }
|
||||
|
||||
func (d *Driver) ID() string { return d.id }
|
||||
|
||||
func (d *Driver) RootID() string { return "/" }
|
||||
|
||||
func (d *Driver) Init(context.Context) error {
|
||||
if strings.TrimSpace(d.id) == "" {
|
||||
return errors.New("scriptcrawler: empty drive id")
|
||||
}
|
||||
if strings.TrimSpace(d.rootDir) == "" {
|
||||
return errors.New("scriptcrawler: empty root dir")
|
||||
}
|
||||
for _, sub := range []string{"videos", "thumbs", "output", ".crawl"} {
|
||||
if err := os.MkdirAll(filepath.Join(d.rootDir, sub), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) RootDir() string { return d.rootDir }
|
||||
|
||||
func (d *Driver) VideosDir() string { return filepath.Join(d.rootDir, "videos") }
|
||||
|
||||
func (d *Driver) ThumbsDir() string { return filepath.Join(d.rootDir, "thumbs") }
|
||||
|
||||
func (d *Driver) OutputDir() string { return filepath.Join(d.rootDir, "output") }
|
||||
|
||||
func (d *Driver) CrawlDir() string { return filepath.Join(d.rootDir, ".crawl") }
|
||||
|
||||
func (d *Driver) VideoPath(fileID string) (string, error) {
|
||||
return safeJoin(d.VideosDir(), fileID)
|
||||
}
|
||||
|
||||
func (d *Driver) ThumbPath(fileID string) (string, error) {
|
||||
return safeJoin(d.ThumbsDir(), fileID)
|
||||
}
|
||||
|
||||
func (d *Driver) OutputPath(fileName string) (string, error) {
|
||||
return safeJoin(d.OutputDir(), fileName)
|
||||
}
|
||||
|
||||
func (d *Driver) List(context.Context, string) ([]drives.Entry, error) {
|
||||
entries, err := os.ReadDir(d.VideosDir())
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
out := make([]drives.Entry, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, drives.Entry{
|
||||
ID: e.Name(),
|
||||
Name: e.Name(),
|
||||
Size: info.Size(),
|
||||
IsDir: false,
|
||||
ModTime: info.ModTime(),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
|
||||
path, err := d.VideoPath(fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &drives.Entry{
|
||||
ID: fileID,
|
||||
Name: fileID,
|
||||
Size: info.Size(),
|
||||
IsDir: info.IsDir(),
|
||||
ModTime: info.ModTime(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
path, err := d.VideoPath(fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.IsDir() || info.Size() == 0 {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return &drives.StreamLink{
|
||||
URL: path,
|
||||
Expires: time.Now().Add(24 * time.Hour),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
videoPath, err := d.VideoPath(fileID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Stat(videoPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
removeThumbCandidates(d.ThumbPath, strings.TrimSuffix(fileID, filepath.Ext(fileID)))
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return errors.New("scriptcrawler: refusing to remove directory")
|
||||
}
|
||||
if err := os.Remove(videoPath); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
removeThumbCandidates(d.ThumbPath, strings.TrimSuffix(fileID, filepath.Ext(fileID)))
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeThumbCandidates(pathFor func(string) (string, error), stem string) {
|
||||
stem = strings.TrimSpace(stem)
|
||||
if stem == "" {
|
||||
return
|
||||
}
|
||||
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
|
||||
path, err := pathFor(stem + ext)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}
|
||||
|
||||
func safeJoin(root, fileID string) (string, error) {
|
||||
id := strings.TrimSpace(fileID)
|
||||
if id == "" || filepath.Base(id) != id {
|
||||
return "", errors.New("scriptcrawler: invalid file id")
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return "", errors.New("scriptcrawler: empty root")
|
||||
}
|
||||
rootAbs, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
pathAbs, err := filepath.Abs(filepath.Join(rootAbs, id))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if pathAbs != rootAbs && !strings.HasPrefix(pathAbs, rootAbs+string(os.PathSeparator)) {
|
||||
return "", errors.New("scriptcrawler: file id escapes root")
|
||||
}
|
||||
return pathAbs, nil
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
@@ -0,0 +1,386 @@
|
||||
package scriptcrawler
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DryRun 在不入库的前提下试跑一个爬虫脚本:临时目录里生成 job.json,
|
||||
// 启动脚本进程,拿到第一条(或前 MaxItems 条)item 事件后立即停止,
|
||||
// 再对视频直链做一次小范围探测,验证脚本"能不能爬取到视频"。
|
||||
// 用于后台导入脚本后的"测试脚本"按钮。
|
||||
|
||||
const (
|
||||
defaultDryRunTimeout = 2 * time.Minute
|
||||
dryRunLogTailLines = 60
|
||||
dryRunMediaProbeLimit = 20 * time.Second
|
||||
dryRunStopGrace = 100 * time.Millisecond
|
||||
)
|
||||
|
||||
type DryRunConfig struct {
|
||||
PythonPath string
|
||||
ScriptPath string
|
||||
ProxyURL string
|
||||
ConfigJSON string
|
||||
// MaxItems 收到多少条 item 后停止脚本,默认 1。
|
||||
MaxItems int
|
||||
// Timeout 整个试跑的硬上限,默认 2 分钟。
|
||||
Timeout time.Duration
|
||||
// SkipMediaProbe 跳过视频直链可达性探测(单测注入用)。
|
||||
SkipMediaProbe bool
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
type DryRunItem struct {
|
||||
Title string `json:"title"`
|
||||
SourceID string `json:"sourceId,omitempty"`
|
||||
MediaURL string `json:"mediaUrl,omitempty"`
|
||||
MediaLocalFile string `json:"mediaLocalFile,omitempty"`
|
||||
ThumbnailURL string `json:"thumbnailUrl,omitempty"`
|
||||
DetailURL string `json:"detailUrl,omitempty"`
|
||||
}
|
||||
|
||||
type DryRunMediaCheck struct {
|
||||
OK bool `json:"ok"`
|
||||
Status int `json:"status,omitempty"`
|
||||
ContentType string `json:"contentType,omitempty"`
|
||||
ContentLength int64 `json:"contentLengthBytes,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type DryRunResult struct {
|
||||
OK bool `json:"ok"`
|
||||
Items []DryRunItem `json:"items"`
|
||||
MediaCheck *DryRunMediaCheck `json:"mediaCheck,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Log []string `json:"log,omitempty"`
|
||||
DurationMs int64 `json:"durationMs"`
|
||||
}
|
||||
|
||||
func DryRun(ctx context.Context, cfg DryRunConfig) *DryRunResult {
|
||||
started := time.Now()
|
||||
result := &DryRunResult{Items: []DryRunItem{}}
|
||||
defer func() { result.DurationMs = time.Since(started).Milliseconds() }()
|
||||
|
||||
scriptPath := strings.TrimSpace(cfg.ScriptPath)
|
||||
if scriptPath == "" {
|
||||
result.Error = "脚本路径为空,请先导入脚本"
|
||||
return result
|
||||
}
|
||||
if _, err := os.Stat(scriptPath); err != nil {
|
||||
result.Error = fmt.Sprintf("脚本不存在: %v", err)
|
||||
return result
|
||||
}
|
||||
pythonPath := strings.TrimSpace(cfg.PythonPath)
|
||||
if pythonPath == "" {
|
||||
pythonPath = "python3"
|
||||
}
|
||||
maxItems := cfg.MaxItems
|
||||
if maxItems <= 0 {
|
||||
maxItems = 1
|
||||
}
|
||||
timeout := cfg.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = defaultDryRunTimeout
|
||||
}
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "crawler-dryrun-")
|
||||
if err != nil {
|
||||
result.Error = fmt.Sprintf("创建临时目录失败: %v", err)
|
||||
return result
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
outputDir := filepath.Join(tmpDir, "output")
|
||||
if err := os.MkdirAll(outputDir, 0o755); err != nil {
|
||||
result.Error = fmt.Sprintf("创建输出目录失败: %v", err)
|
||||
return result
|
||||
}
|
||||
seenPath := filepath.Join(tmpDir, "seen.txt")
|
||||
if err := os.WriteFile(seenPath, nil, 0o644); err != nil {
|
||||
result.Error = fmt.Sprintf("写入 seen 文件失败: %v", err)
|
||||
return result
|
||||
}
|
||||
|
||||
configJSON := json.RawMessage([]byte("{}"))
|
||||
if raw := strings.TrimSpace(cfg.ConfigJSON); raw != "" {
|
||||
if !json.Valid([]byte(raw)) {
|
||||
result.Error = "自定义配置必须是合法 JSON"
|
||||
return result
|
||||
}
|
||||
configJSON = json.RawMessage(raw)
|
||||
}
|
||||
job := Job{
|
||||
Protocol: "crawler.v1",
|
||||
Mode: "crawl",
|
||||
RunID: "dryrun-" + started.UTC().Format("20060102T150405Z"),
|
||||
CrawlerID: "dryrun",
|
||||
TargetNew: maxItems,
|
||||
SeenSourceIDsFile: seenPath,
|
||||
OutputDir: outputDir,
|
||||
Config: configJSON,
|
||||
Network: JobNetwork{ProxyURL: strings.TrimSpace(cfg.ProxyURL)},
|
||||
}
|
||||
jobPath := filepath.Join(tmpDir, "job.json")
|
||||
jobData, err := json.MarshalIndent(job, "", " ")
|
||||
if err != nil {
|
||||
result.Error = fmt.Sprintf("生成 job 文件失败: %v", err)
|
||||
return result
|
||||
}
|
||||
if err := os.WriteFile(jobPath, jobData, 0o600); err != nil {
|
||||
result.Error = fmt.Sprintf("写入 job 文件失败: %v", err)
|
||||
return result
|
||||
}
|
||||
|
||||
runCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(runCtx, pythonPath, scriptPath, "--job", jobPath)
|
||||
cmd.Dir = filepath.Dir(scriptPath)
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
cmd.Cancel = func() error {
|
||||
return killDryRunProcess(cmd)
|
||||
}
|
||||
// 超时或提前 kill 后,脚本派生的子进程可能仍持有 stdout/stderr 管道;
|
||||
// WaitDelay 强制在宽限期后关闭管道,避免读取端永久阻塞。
|
||||
cmd.WaitDelay = 3 * time.Second
|
||||
if proxyURL := strings.TrimSpace(cfg.ProxyURL); proxyURL != "" {
|
||||
cmd.Env = append(os.Environ(),
|
||||
"HTTP_PROXY="+proxyURL,
|
||||
"HTTPS_PROXY="+proxyURL,
|
||||
"http_proxy="+proxyURL,
|
||||
"https_proxy="+proxyURL,
|
||||
"NO_PROXY=",
|
||||
"no_proxy=",
|
||||
)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
result.Error = fmt.Sprintf("启动脚本失败: %v", err)
|
||||
return result
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
_ = stdout.Close()
|
||||
result.Error = fmt.Sprintf("启动脚本失败: %v", err)
|
||||
return result
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
_ = stdout.Close()
|
||||
_ = stderr.Close()
|
||||
result.Error = fmt.Sprintf("启动脚本失败: %v", err)
|
||||
return result
|
||||
}
|
||||
|
||||
// stderr 是脚本日志,保留尾部若干行用于排错回显。
|
||||
var logMu sync.Mutex
|
||||
logTail := make([]string, 0, dryRunLogTailLines)
|
||||
stderrDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(stderrDone)
|
||||
scanner := bufio.NewScanner(stderr)
|
||||
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
logMu.Lock()
|
||||
if len(logTail) >= dryRunLogTailLines {
|
||||
logTail = logTail[1:]
|
||||
}
|
||||
logTail = append(logTail, line)
|
||||
logMu.Unlock()
|
||||
}
|
||||
}()
|
||||
|
||||
items := []DryRunItem{}
|
||||
var firstMediaHeaders map[string]string
|
||||
parseFailures := 0
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
scanner.Buffer(make([]byte, 64*1024), 4*1024*1024)
|
||||
for scanner.Scan() {
|
||||
if runCtx.Err() != nil {
|
||||
break
|
||||
}
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var event Event
|
||||
if err := json.Unmarshal([]byte(line), &event); err != nil {
|
||||
parseFailures++
|
||||
continue
|
||||
}
|
||||
eventType := strings.ToLower(strings.TrimSpace(event.Type))
|
||||
item := event.normalizedItem()
|
||||
if eventType == "" && item.hasPayload() {
|
||||
eventType = "item"
|
||||
}
|
||||
if eventType != "item" {
|
||||
continue
|
||||
}
|
||||
normalized, _, err := normalizeItemForImport(item)
|
||||
if err != nil {
|
||||
result.Error = fmt.Sprintf("item 字段不完整: %v", err)
|
||||
continue
|
||||
}
|
||||
mediaURL := strings.TrimSpace(normalized.Media.URL)
|
||||
if len(items) == 0 {
|
||||
firstMediaHeaders = normalized.Media.Headers
|
||||
}
|
||||
items = append(items, DryRunItem{
|
||||
Title: strings.TrimSpace(normalized.Title),
|
||||
SourceID: strings.TrimSpace(item.SourceID),
|
||||
MediaURL: mediaURL,
|
||||
MediaLocalFile: strings.TrimSpace(normalized.Media.LocalFile),
|
||||
ThumbnailURL: strings.TrimSpace(normalized.Thumbnail.URL),
|
||||
DetailURL: strings.TrimSpace(normalized.DetailURL),
|
||||
})
|
||||
if len(items) >= maxItems {
|
||||
break
|
||||
}
|
||||
}
|
||||
// 拿够了就停掉脚本,避免它继续翻页。给已经自然结束的脚本一个很短
|
||||
// 的宽限期,让 stderr 日志先被管道读完,避免 dry-run 回显偶发为空。
|
||||
waitDone := make(chan struct{})
|
||||
go func() {
|
||||
_ = cmd.Wait()
|
||||
close(waitDone)
|
||||
}()
|
||||
select {
|
||||
case <-waitDone:
|
||||
case <-time.After(dryRunStopGrace):
|
||||
_ = killDryRunProcess(cmd)
|
||||
<-waitDone
|
||||
}
|
||||
<-stderrDone
|
||||
|
||||
logMu.Lock()
|
||||
result.Log = append([]string{}, logTail...)
|
||||
logMu.Unlock()
|
||||
result.Items = items
|
||||
|
||||
if len(items) == 0 {
|
||||
if result.Error == "" {
|
||||
switch {
|
||||
case runCtx.Err() != nil && ctx.Err() == nil:
|
||||
result.Error = fmt.Sprintf("测试超时(%s),脚本没有输出任何视频", timeout)
|
||||
case parseFailures > 0:
|
||||
result.Error = "脚本 stdout 不是合法的 crawler.v1 JSON Lines(日志应输出到 stderr)"
|
||||
default:
|
||||
result.Error = "脚本退出但没有输出任何视频"
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
result.Error = ""
|
||||
|
||||
first := items[0]
|
||||
switch {
|
||||
case cfg.SkipMediaProbe:
|
||||
result.OK = true
|
||||
case first.MediaLocalFile != "":
|
||||
// 脚本自己下载到 output_dir 的模式:试跑用的是临时目录,
|
||||
// 文件已随目录清理,能输出合法 local_file 即视为通过。
|
||||
result.OK = true
|
||||
default:
|
||||
check := probeMediaURL(ctx, cfg, first, firstMediaHeaders)
|
||||
result.MediaCheck = check
|
||||
result.OK = check.OK
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func killDryRunProcess(cmd *exec.Cmd) error {
|
||||
if cmd == nil || cmd.Process == nil {
|
||||
return nil
|
||||
}
|
||||
if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil {
|
||||
if err == syscall.ESRCH {
|
||||
return nil
|
||||
}
|
||||
return cmd.Process.Kill()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// probeMediaURL 对视频直链发一个 Range: bytes=0-0 的小请求,
|
||||
// 验证直链可达(带上脚本给的防盗链 headers 和代理)。
|
||||
func probeMediaURL(ctx context.Context, cfg DryRunConfig, item DryRunItem, mediaHeaders map[string]string) *DryRunMediaCheck {
|
||||
check := &DryRunMediaCheck{}
|
||||
if item.MediaURL == "" {
|
||||
check.Error = "item 没有视频直链"
|
||||
return check
|
||||
}
|
||||
|
||||
client := cfg.HTTPClient
|
||||
if client == nil {
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
ResponseHeaderTimeout: dryRunMediaProbeLimit,
|
||||
}
|
||||
if err := configureExplicitProxy(transport, cfg.ProxyURL); err != nil {
|
||||
check.Error = fmt.Sprintf("代理配置无效: %v", err)
|
||||
return check
|
||||
}
|
||||
client = &http.Client{Transport: transport}
|
||||
}
|
||||
|
||||
probeCtx, cancel := context.WithTimeout(ctx, dryRunMediaProbeLimit)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(probeCtx, http.MethodGet, item.MediaURL, nil)
|
||||
if err != nil {
|
||||
check.Error = fmt.Sprintf("视频直链无效: %v", err)
|
||||
return check
|
||||
}
|
||||
req.Header.Set("User-Agent", defaultUserAgent)
|
||||
req.Header.Set("Range", "bytes=0-0")
|
||||
if item.DetailURL != "" {
|
||||
req.Header.Set("Referer", item.DetailURL)
|
||||
}
|
||||
for k, v := range mediaHeaders {
|
||||
k = strings.TrimSpace(k)
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
check.Error = fmt.Sprintf("视频直链请求失败: %v", err)
|
||||
return check
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
check.Status = resp.StatusCode
|
||||
check.ContentType = resp.Header.Get("Content-Type")
|
||||
if cr := resp.Header.Get("Content-Range"); cr != "" {
|
||||
// Content-Range: bytes 0-0/12345 → 取总大小
|
||||
if idx := strings.LastIndex(cr, "/"); idx >= 0 {
|
||||
var total int64
|
||||
if _, err := fmt.Sscanf(cr[idx+1:], "%d", &total); err == nil {
|
||||
check.ContentLength = total
|
||||
}
|
||||
}
|
||||
}
|
||||
if check.ContentLength == 0 && resp.StatusCode == http.StatusOK {
|
||||
check.ContentLength = resp.ContentLength
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||
check.Error = fmt.Sprintf("视频直链返回 HTTP %d", resp.StatusCode)
|
||||
return check
|
||||
}
|
||||
check.OK = true
|
||||
return check
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package scriptcrawler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func writeDryRunScript(t *testing.T, body string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "crawler.sh")
|
||||
if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body), 0o755); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestDryRunCollectsFirstItem(t *testing.T) {
|
||||
script := writeDryRunScript(t, `
|
||||
echo '[log] fetching list page' >&2
|
||||
echo '{"type":"item","item":{"title":"Test Video","media_url":"https://cdn.example.test/v.mp4","source_id":"123","thumbnail_url":"https://cdn.example.test/t.jpg"}}'
|
||||
echo '{"type":"done","stats":{"emitted":1}}'
|
||||
`)
|
||||
result := DryRun(context.Background(), DryRunConfig{
|
||||
PythonPath: "/bin/sh",
|
||||
ScriptPath: script,
|
||||
SkipMediaProbe: true,
|
||||
})
|
||||
if !result.OK {
|
||||
t.Fatalf("ok = false, error = %q, log = %v", result.Error, result.Log)
|
||||
}
|
||||
if len(result.Items) != 1 {
|
||||
t.Fatalf("items = %d, want 1", len(result.Items))
|
||||
}
|
||||
item := result.Items[0]
|
||||
if item.Title != "Test Video" || item.MediaURL != "https://cdn.example.test/v.mp4" || item.SourceID != "123" {
|
||||
t.Fatalf("item = %+v", item)
|
||||
}
|
||||
if len(result.Log) == 0 || !strings.Contains(result.Log[0], "fetching list page") {
|
||||
t.Fatalf("log tail = %v, want stderr captured", result.Log)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDryRunProbesMediaURL(t *testing.T) {
|
||||
var gotRange, gotReferer string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotRange = r.Header.Get("Range")
|
||||
gotReferer = r.Header.Get("Referer")
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
w.Header().Set("Content-Range", "bytes 0-0/4096")
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write([]byte("x"))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
script := writeDryRunScript(t, fmt.Sprintf(
|
||||
`echo '{"type":"item","title":"Probe Video","media_url":"%s/v.mp4","detail_url":"https://example.test/view"}'`,
|
||||
srv.URL,
|
||||
))
|
||||
result := DryRun(context.Background(), DryRunConfig{
|
||||
PythonPath: "/bin/sh",
|
||||
ScriptPath: script,
|
||||
})
|
||||
if !result.OK {
|
||||
t.Fatalf("ok = false, error = %q, mediaCheck = %+v", result.Error, result.MediaCheck)
|
||||
}
|
||||
if result.MediaCheck == nil || !result.MediaCheck.OK {
|
||||
t.Fatalf("mediaCheck = %+v, want ok", result.MediaCheck)
|
||||
}
|
||||
if result.MediaCheck.Status != http.StatusPartialContent || result.MediaCheck.ContentLength != 4096 {
|
||||
t.Fatalf("mediaCheck = %+v, want 206 with total 4096", result.MediaCheck)
|
||||
}
|
||||
if gotRange != "bytes=0-0" || gotReferer != "https://example.test/view" {
|
||||
t.Fatalf("probe headers range=%q referer=%q", gotRange, gotReferer)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDryRunReportsBrokenMediaURL(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "forbidden", http.StatusForbidden)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
script := writeDryRunScript(t, fmt.Sprintf(
|
||||
`echo '{"type":"item","title":"Dead Link","media_url":"%s/v.mp4"}'`,
|
||||
srv.URL,
|
||||
))
|
||||
result := DryRun(context.Background(), DryRunConfig{
|
||||
PythonPath: "/bin/sh",
|
||||
ScriptPath: script,
|
||||
})
|
||||
if result.OK {
|
||||
t.Fatal("ok = true, want false for HTTP 403 media url")
|
||||
}
|
||||
if result.MediaCheck == nil || result.MediaCheck.OK || result.MediaCheck.Status != http.StatusForbidden {
|
||||
t.Fatalf("mediaCheck = %+v, want failed 403", result.MediaCheck)
|
||||
}
|
||||
if len(result.Items) != 1 {
|
||||
t.Fatalf("items = %d, want item still returned for debugging", len(result.Items))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDryRunRejectsNonJSONStdout(t *testing.T) {
|
||||
script := writeDryRunScript(t, `echo 'plain text progress output'`)
|
||||
result := DryRun(context.Background(), DryRunConfig{
|
||||
PythonPath: "/bin/sh",
|
||||
ScriptPath: script,
|
||||
SkipMediaProbe: true,
|
||||
})
|
||||
if result.OK {
|
||||
t.Fatal("ok = true, want false for non-JSON stdout")
|
||||
}
|
||||
if !strings.Contains(result.Error, "JSON Lines") {
|
||||
t.Fatalf("error = %q, want JSON Lines hint", result.Error)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDryRunTimesOut(t *testing.T) {
|
||||
script := writeDryRunScript(t, `sleep 30`)
|
||||
start := time.Now()
|
||||
result := DryRun(context.Background(), DryRunConfig{
|
||||
PythonPath: "/bin/sh",
|
||||
ScriptPath: script,
|
||||
Timeout: 2 * time.Second,
|
||||
SkipMediaProbe: true,
|
||||
})
|
||||
if result.OK {
|
||||
t.Fatal("ok = true, want false on timeout")
|
||||
}
|
||||
if !strings.Contains(result.Error, "超时") {
|
||||
t.Fatalf("error = %q, want timeout message", result.Error)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed > 10*time.Second {
|
||||
t.Fatalf("dry run took %s, script was not killed", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDryRunMissingScript(t *testing.T) {
|
||||
result := DryRun(context.Background(), DryRunConfig{
|
||||
PythonPath: "/bin/sh",
|
||||
ScriptPath: filepath.Join(t.TempDir(), "missing.py"),
|
||||
})
|
||||
if result.OK || result.Error == "" {
|
||||
t.Fatalf("result = %+v, want error for missing script", result)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package scriptcrawler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const maxCrawlerNameRunes = 80
|
||||
|
||||
type Metadata struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
func ReadMetadata(scriptPath string) (Metadata, error) {
|
||||
scriptPath = strings.TrimSpace(scriptPath)
|
||||
if scriptPath == "" {
|
||||
return Metadata{}, errors.New("脚本路径为空")
|
||||
}
|
||||
if filepath.Ext(scriptPath) != ".py" {
|
||||
return Metadata{}, errors.New("目前只支持 .py 爬虫脚本")
|
||||
}
|
||||
data, err := os.ReadFile(scriptPath)
|
||||
if err != nil {
|
||||
return Metadata{}, fmt.Errorf("读取脚本失败: %w", err)
|
||||
}
|
||||
return ExtractMetadata(string(data))
|
||||
}
|
||||
|
||||
func ExtractMetadata(source string) (Metadata, error) {
|
||||
for _, line := range strings.Split(source, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" || strings.HasPrefix(trimmed, "#") {
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(trimmed, "CRAWLER_NAME") {
|
||||
continue
|
||||
}
|
||||
left, right, ok := strings.Cut(trimmed, "=")
|
||||
if !ok || strings.TrimSpace(left) != "CRAWLER_NAME" {
|
||||
continue
|
||||
}
|
||||
name, ok := parsePythonStringLiteral(right)
|
||||
if !ok {
|
||||
return Metadata{}, errors.New(`CRAWLER_NAME 必须是字符串字面量,例如 CRAWLER_NAME = "示例爬虫"`)
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return Metadata{}, errors.New("CRAWLER_NAME 不能为空")
|
||||
}
|
||||
if len([]rune(name)) > maxCrawlerNameRunes {
|
||||
return Metadata{}, fmt.Errorf("CRAWLER_NAME 不能超过 %d 个字符", maxCrawlerNameRunes)
|
||||
}
|
||||
return Metadata{Name: name}, nil
|
||||
}
|
||||
return Metadata{}, errors.New(`脚本必须声明 CRAWLER_NAME,例如 CRAWLER_NAME = "示例爬虫"`)
|
||||
}
|
||||
|
||||
func parsePythonStringLiteral(raw string) (string, bool) {
|
||||
s := strings.TrimSpace(raw)
|
||||
if s == "" {
|
||||
return "", false
|
||||
}
|
||||
rawString := false
|
||||
for len(s) > 0 {
|
||||
switch s[0] {
|
||||
case 'r', 'R':
|
||||
rawString = true
|
||||
s = strings.TrimSpace(s[1:])
|
||||
case 'u', 'U', 'b', 'B':
|
||||
s = strings.TrimSpace(s[1:])
|
||||
default:
|
||||
goto parseQuote
|
||||
}
|
||||
}
|
||||
|
||||
parseQuote:
|
||||
if len(s) < 2 || (s[0] != '"' && s[0] != '\'') {
|
||||
return "", false
|
||||
}
|
||||
quote := s[0]
|
||||
var b strings.Builder
|
||||
escaped := false
|
||||
for i := 1; i < len(s); i++ {
|
||||
ch := s[i]
|
||||
if escaped {
|
||||
switch {
|
||||
case rawString:
|
||||
b.WriteByte('\\')
|
||||
b.WriteByte(ch)
|
||||
case ch == 'n':
|
||||
b.WriteByte('\n')
|
||||
case ch == 'r':
|
||||
b.WriteByte('\r')
|
||||
case ch == 't':
|
||||
b.WriteByte('\t')
|
||||
case ch == '\\' || ch == quote || ch == '"' || ch == '\'':
|
||||
b.WriteByte(ch)
|
||||
default:
|
||||
b.WriteByte(ch)
|
||||
}
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
if ch == '\\' {
|
||||
escaped = true
|
||||
continue
|
||||
}
|
||||
if ch == quote {
|
||||
return b.String(), true
|
||||
}
|
||||
b.WriteByte(ch)
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package scriptcrawler
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestExtractMetadataReadsCrawlerName(t *testing.T) {
|
||||
meta, err := ExtractMetadata(`
|
||||
# comment
|
||||
CRAWLER_NAME = "示例爬虫"
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("extract metadata: %v", err)
|
||||
}
|
||||
if meta.Name != "示例爬虫" {
|
||||
t.Fatalf("name = %q", meta.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractMetadataRejectsMissingCrawlerName(t *testing.T) {
|
||||
_, err := ExtractMetadata(`print("hello")`)
|
||||
if err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "CRAWLER_NAME") {
|
||||
t.Fatalf("error = %v, want CRAWLER_NAME guidance", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractMetadataRejectsEmptyCrawlerName(t *testing.T) {
|
||||
_, err := ExtractMetadata(`CRAWLER_NAME = " "`)
|
||||
if err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "不能为空") {
|
||||
t.Fatalf("error = %v, want empty-name error", err)
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,12 @@ type CrawlerConfig struct {
|
||||
|
||||
// OnNewVideo 是新视频成功入库后的回调,用于触发预览视频 worker。
|
||||
OnNewVideo func(v *catalog.Video)
|
||||
// OnProgress 在抓取统计变化时触发,用于后台管理页展示实时进度。
|
||||
OnProgress func(progress CrawlProgress)
|
||||
// OnCheckedVideo 在 Python 爬虫开始检查一个列表页视频时触发。
|
||||
OnCheckedVideo func()
|
||||
// OnExtractedVideo 在 Python 爬虫提取到一个新视频直链时触发。
|
||||
OnExtractedVideo func()
|
||||
}
|
||||
|
||||
// Crawler 把 Python 爬虫产出包装成 catalog 入库流程。
|
||||
@@ -219,6 +225,16 @@ type CrawlResult struct {
|
||||
SeenFile string
|
||||
}
|
||||
|
||||
// CrawlProgress 是 RunOnce 过程中可安全对外发布的实时计数。
|
||||
type CrawlProgress struct {
|
||||
TargetNew int
|
||||
TotalEntries int
|
||||
NewVideos int
|
||||
Skipped int
|
||||
Failed int
|
||||
SeenSnapshot int
|
||||
}
|
||||
|
||||
// spiderVideoEntry 对应 spider_91porn.py 输出 JSON 中的单条视频。
|
||||
type spiderVideoEntry struct {
|
||||
Title string `json:"title"`
|
||||
@@ -266,6 +282,20 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
|
||||
result := &CrawlResult{TargetNew: targetNew, StartedAt: time.Now()}
|
||||
defer func() { result.FinishedAt = time.Now() }()
|
||||
emitProgress := func() {
|
||||
if c.cfg.OnProgress == nil {
|
||||
return
|
||||
}
|
||||
c.cfg.OnProgress(CrawlProgress{
|
||||
TargetNew: result.TargetNew,
|
||||
TotalEntries: result.TotalEntries,
|
||||
NewVideos: result.NewVideos,
|
||||
Skipped: result.Skipped,
|
||||
Failed: result.Failed,
|
||||
SeenSnapshot: result.SeenSnapshot,
|
||||
})
|
||||
}
|
||||
emitProgress()
|
||||
|
||||
// 1. 准备 .crawl/ 目录 + 已知源视频 ID 列表
|
||||
//
|
||||
@@ -291,6 +321,7 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
return result, fmt.Errorf("spider91 crawler: build seen list: %w", err)
|
||||
}
|
||||
result.SeenSnapshot = seenCount
|
||||
emitProgress()
|
||||
|
||||
// 2-3. 启动 Python 爬虫(流式 stdout 协议),并边读边处理。
|
||||
//
|
||||
@@ -321,9 +352,11 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
continue
|
||||
}
|
||||
result.TotalEntries++
|
||||
emitProgress()
|
||||
sourceID := sourceIDForItem(item)
|
||||
if sourceID == "" || strings.TrimSpace(item.VideoURL) == "" {
|
||||
result.Failed++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
if result.NewVideos >= targetNew {
|
||||
@@ -335,22 +368,27 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
if err != nil {
|
||||
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s check deleted: %v", c.cfg.Driver.ID(), item.Viewkey, sourceID, err)
|
||||
result.Failed++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
if deleted {
|
||||
result.Skipped++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
if existing, _ := c.cfg.Catalog.GetVideo(ctx, videoID); existing != nil {
|
||||
result.Skipped++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
if perr := c.processOne(ctx, videoID, item); perr != nil {
|
||||
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s failed: %v", c.cfg.Driver.ID(), item.Viewkey, sourceID, perr)
|
||||
result.Failed++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
result.NewVideos++
|
||||
emitProgress()
|
||||
}
|
||||
if scerr := scanner.Err(); scerr != nil {
|
||||
log.Printf("[spider91] drive=%s stdout scan: %v", c.cfg.Driver.ID(), scerr)
|
||||
@@ -458,12 +496,12 @@ func (c *Crawler) startSpiderTargetNew(ctx context.Context, targetNew int, seenP
|
||||
return nil, nil, fmt.Errorf("start: %w", err)
|
||||
}
|
||||
// stderr 转发到 backend log。子进程退出时 reader 自动 EOF,goroutine 自然结束。
|
||||
go forwardSpiderLog(c.cfg.Driver.ID(), stderr)
|
||||
go forwardSpiderLog(c.cfg.Driver.ID(), stderr, c.cfg.OnCheckedVideo, c.cfg.OnExtractedVideo)
|
||||
return cmd, stdout, nil
|
||||
}
|
||||
|
||||
// forwardSpiderLog 把 Python stderr 逐行转发到 backend log,便于调试。
|
||||
func forwardSpiderLog(driveID string, r io.Reader) {
|
||||
func forwardSpiderLog(driveID string, r io.Reader, onCheckedVideo func(), onExtractedVideo func()) {
|
||||
scanner := bufio.NewScanner(r)
|
||||
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
@@ -472,9 +510,23 @@ func forwardSpiderLog(driveID string, r io.Reader) {
|
||||
continue
|
||||
}
|
||||
log.Printf("[spider91:py] drive=%s %s", driveID, line)
|
||||
if onCheckedVideo != nil && isSpider91CheckedVideoLogLine(line) {
|
||||
onCheckedVideo()
|
||||
}
|
||||
if onExtractedVideo != nil && isSpider91ExtractedVideoLogLine(line) {
|
||||
onExtractedVideo()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isSpider91CheckedVideoLogLine(line string) bool {
|
||||
return checkedVideoLogRE.MatchString(line)
|
||||
}
|
||||
|
||||
func isSpider91ExtractedVideoLogLine(line string) bool {
|
||||
return strings.Contains(line, "[OK] 成功提取视频直链")
|
||||
}
|
||||
|
||||
// processOne 处理单个 91 源视频:下载视频 + 封面 + 复制封面 + 入库。
|
||||
// 任一步失败会清理已写入的临时文件,不留半成品。
|
||||
func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVideoEntry) error {
|
||||
@@ -847,9 +899,10 @@ func spider91CookieHeader(cookies []*http.Cookie) string {
|
||||
}
|
||||
|
||||
var (
|
||||
strencode2RE = regexp.MustCompile(`strencode2\(["']([^"']+)["']\)`)
|
||||
srcAttrRE = regexp.MustCompile(`src=['"]([^'"]+)['"]`)
|
||||
mp4URLRE = regexp.MustCompile(`https?://[^\s"'<>]+\.mp4[^\s"'<>]*`)
|
||||
checkedVideoLogRE = regexp.MustCompile(`处理视频\s+\d+/\d+:`)
|
||||
strencode2RE = regexp.MustCompile(`strencode2\(["']([^"']+)["']\)`)
|
||||
srcAttrRE = regexp.MustCompile(`src=['"]([^'"]+)['"]`)
|
||||
mp4URLRE = regexp.MustCompile(`https?://[^\s"'<>]+\.mp4[^\s"'<>]*`)
|
||||
)
|
||||
|
||||
func parseSpider91VideoURL(html string) string {
|
||||
|
||||
@@ -707,6 +707,18 @@ func TestSpider91CookieHeader(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpider91ProgressLogLineClassifiers(t *testing.T) {
|
||||
if !isSpider91CheckedVideoLogLine("[2026-06-08 16:49:17] 处理视频 3/24: 标题") {
|
||||
t.Fatal("checked video log line was not recognized")
|
||||
}
|
||||
if isSpider91CheckedVideoLogLine("[2026-06-08 16:49:17] [页 2] 发现 24 个视频") {
|
||||
t.Fatal("page summary log line should not count as checked video")
|
||||
}
|
||||
if !isSpider91ExtractedVideoLogLine("[2026-06-08 16:49:39] [OK] 成功提取视频直链") {
|
||||
t.Fatal("extracted video log line was not recognized")
|
||||
}
|
||||
}
|
||||
|
||||
func spider91DetailHTML(videoURL string) string {
|
||||
fragment := `<video><source src="` + videoURL + `" type="video/mp4"></video>`
|
||||
return `document.write(strencode2("` + url.PathEscape(fragment) + `"));`
|
||||
|
||||
@@ -167,6 +167,46 @@ func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, er
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
videoPath, err := d.VideoPath(fileID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Stat(videoPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
removeThumbCandidates(d.ThumbPath, strings.TrimSuffix(fileID, filepath.Ext(fileID)))
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return errors.New("spider91: refusing to remove directory")
|
||||
}
|
||||
if err := os.Remove(videoPath); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
removeThumbCandidates(d.ThumbPath, strings.TrimSuffix(fileID, filepath.Ext(fileID)))
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeThumbCandidates(pathFor func(string) (string, error), stem string) {
|
||||
stem = strings.TrimSpace(stem)
|
||||
if stem == "" {
|
||||
return
|
||||
}
|
||||
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
|
||||
path, err := pathFor(stem + ext)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}
|
||||
|
||||
// safeJoin 把 fileID 拼到 root 下,保证最终路径不会逃出 root。
|
||||
// fileID 必须是单纯的文件名(不含 / 或 .. 等组件)。
|
||||
func safeJoin(root, fileID string) (string, error) {
|
||||
@@ -192,3 +232,4 @@ func safeJoin(root, fileID string) (string, error) {
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
@@ -2,19 +2,23 @@ package wopan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
sdk "github.com/OpenListTeam/wopan-sdk-go"
|
||||
"github.com/go-resty/resty/v2"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
// Driver 封装联通沃盘
|
||||
// Driver 封装联通网盘
|
||||
type Driver struct {
|
||||
id string
|
||||
rootID string
|
||||
@@ -23,6 +27,14 @@ type Driver struct {
|
||||
refreshToken string
|
||||
client *sdk.WoClient
|
||||
onTokenUpdate func(access, refresh string)
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
listInterval time.Duration
|
||||
listCooldown time.Duration
|
||||
|
||||
fileIDMu sync.RWMutex
|
||||
fidToID map[string]string
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -47,6 +59,9 @@ func New(c Config) *Driver {
|
||||
accessToken: c.AccessToken,
|
||||
refreshToken: c.RefreshToken,
|
||||
onTokenUpdate: c.OnTokenUpdate,
|
||||
listInterval: 800 * time.Millisecond,
|
||||
listCooldown: 5 * time.Minute,
|
||||
fidToID: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,15 +93,41 @@ func (d *Driver) spaceType() string {
|
||||
}
|
||||
|
||||
func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error) {
|
||||
d.listMu.Lock()
|
||||
defer d.listMu.Unlock()
|
||||
|
||||
var result []drives.Entry
|
||||
pageNum := 0
|
||||
pageSize := 100
|
||||
for {
|
||||
data, err := d.client.QueryAllFiles(d.spaceType(), dirID, pageNum, pageSize, 0, d.familyID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wopan list: %w", err)
|
||||
var data *sdk.QueryAllFilesData
|
||||
for attempt := 0; ; attempt++ {
|
||||
if err := d.waitForListSlotLocked(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var err error
|
||||
data, err = d.client.QueryAllFiles(d.spaceType(), dirID, pageNum, pageSize, 0, d.familyID, func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
err = wopanRequestError("list", err)
|
||||
wait, ok := drives.RateLimitRetryAfter(err)
|
||||
if !ok {
|
||||
return nil, err
|
||||
}
|
||||
if wait <= 0 {
|
||||
wait = d.listCooldown
|
||||
}
|
||||
log.Printf("[wopan] list cooling down drive=%s dir=%s page=%d cooldown=%s attempt=%d err=%v",
|
||||
d.id, dirID, pageNum, wait, attempt+1, err)
|
||||
if err := sleepContext(ctx, wait); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for _, f := range data.Files {
|
||||
d.rememberFileID(f)
|
||||
result = append(result, fileToEntry(f, dirID))
|
||||
}
|
||||
if len(data.Files) < pageSize {
|
||||
@@ -103,9 +144,11 @@ func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error)
|
||||
}
|
||||
|
||||
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
data, err := d.client.GetDownloadUrlV2([]string{fileID})
|
||||
data, err := d.client.GetDownloadUrlV2([]string{fileID}, func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wopan download url: %w", err)
|
||||
return nil, wopanRequestError("download url", err)
|
||||
}
|
||||
if len(data.List) == 0 {
|
||||
return nil, fmt.Errorf("wopan download url: empty response")
|
||||
@@ -142,9 +185,151 @@ func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader,
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("wopan upload: %w", err)
|
||||
}
|
||||
if fid != "" {
|
||||
if objectID, err := d.findDeleteFileIDInParent(ctx, parentID, drives.SourceFile{
|
||||
FileID: fid,
|
||||
Name: name,
|
||||
Size: size,
|
||||
}); err == nil {
|
||||
d.rememberFIDMapping(fid, objectID)
|
||||
} else {
|
||||
log.Printf("[wopan] upload drive=%s parent=%s fid=%s resolve object id: %v", d.id, parentID, fid, err)
|
||||
}
|
||||
}
|
||||
return fid, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
if d.client == nil {
|
||||
return fmt.Errorf("wopan rename: driver not initialized")
|
||||
}
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return fmt.Errorf("wopan rename: empty file id")
|
||||
}
|
||||
newName = strings.TrimSpace(newName)
|
||||
if newName == "" {
|
||||
return fmt.Errorf("wopan rename: empty new name")
|
||||
}
|
||||
renameID := fileID
|
||||
if cached := d.cachedDeleteFileID(fileID); cached != "" {
|
||||
renameID = cached
|
||||
}
|
||||
if err := d.client.RenameFileOrDirectory(d.spaceType(), 1, renameID, newName, d.familyID, func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
}); err != nil {
|
||||
return wopanRequestError("rename", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
if d.client == nil {
|
||||
return fmt.Errorf("wopan remove: driver not initialized")
|
||||
}
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return fmt.Errorf("wopan remove: empty file id")
|
||||
}
|
||||
deleteID := fileID
|
||||
if cached := d.cachedDeleteFileID(fileID); cached != "" {
|
||||
deleteID = cached
|
||||
}
|
||||
if err := d.deleteFileByObjectID(ctx, deleteID); err != nil {
|
||||
return fmt.Errorf("wopan remove: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) RemoveSource(ctx context.Context, source drives.SourceFile) error {
|
||||
if d.client == nil {
|
||||
return fmt.Errorf("wopan remove: driver not initialized")
|
||||
}
|
||||
fileID := strings.TrimSpace(source.FileID)
|
||||
if fileID == "" {
|
||||
return fmt.Errorf("wopan remove: empty file id")
|
||||
}
|
||||
deleteID, err := d.resolveDeleteFileID(ctx, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := d.deleteFileByObjectID(ctx, deleteID); err != nil {
|
||||
return fmt.Errorf("wopan remove: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) deleteFileByObjectID(ctx context.Context, fileID string) error {
|
||||
if err := d.client.DeleteFile(d.spaceType(), nil, []string{fileID}, func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) resolveDeleteFileID(ctx context.Context, source drives.SourceFile) (string, error) {
|
||||
fileID := strings.TrimSpace(source.FileID)
|
||||
if fileID == "" {
|
||||
return "", fmt.Errorf("wopan remove: empty file id")
|
||||
}
|
||||
if cached := d.cachedDeleteFileID(fileID); cached != "" {
|
||||
return cached, nil
|
||||
}
|
||||
parentID := strings.TrimSpace(source.ParentID)
|
||||
if parentID == "" {
|
||||
return fileID, nil
|
||||
}
|
||||
return d.findDeleteFileIDInParent(ctx, parentID, source)
|
||||
}
|
||||
|
||||
func (d *Driver) findDeleteFileIDInParent(ctx context.Context, parentID string, source drives.SourceFile) (string, error) {
|
||||
d.listMu.Lock()
|
||||
defer d.listMu.Unlock()
|
||||
|
||||
pageNum := 0
|
||||
pageSize := 100
|
||||
for {
|
||||
var data *sdk.QueryAllFilesData
|
||||
for attempt := 0; ; attempt++ {
|
||||
if err := d.waitForListSlotLocked(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var err error
|
||||
data, err = d.client.QueryAllFiles(d.spaceType(), parentID, pageNum, pageSize, 0, d.familyID, func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
err = wopanRequestError("resolve delete id", err)
|
||||
wait, ok := drives.RateLimitRetryAfter(err)
|
||||
if !ok {
|
||||
return "", err
|
||||
}
|
||||
if wait <= 0 {
|
||||
wait = d.listCooldown
|
||||
}
|
||||
log.Printf("[wopan] resolve delete id cooling down drive=%s parent=%s page=%d cooldown=%s attempt=%d err=%v",
|
||||
d.id, parentID, pageNum, wait, attempt+1, err)
|
||||
if err := sleepContext(ctx, wait); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
for _, f := range data.Files {
|
||||
d.rememberFileID(f)
|
||||
if id, ok := deleteFileIDFromWopanFile(f, source); ok {
|
||||
return id, nil
|
||||
}
|
||||
}
|
||||
if len(data.Files) < pageSize {
|
||||
break
|
||||
}
|
||||
pageNum++
|
||||
}
|
||||
return "", fmt.Errorf("wopan remove: source file %q not found under parent %q", source.FileID, parentID)
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
parts := splitPath(pathFromRoot)
|
||||
currentID := d.rootID
|
||||
@@ -154,9 +339,11 @@ func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, er
|
||||
return "", err
|
||||
}
|
||||
if childID == "" {
|
||||
resp, err := d.client.CreateDirectory(d.spaceType(), currentID, name, d.familyID)
|
||||
resp, err := d.client.CreateDirectory(d.spaceType(), currentID, name, d.familyID, func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("wopan mkdir %s: %w", name, err)
|
||||
return "", wopanRequestError("mkdir "+name, err)
|
||||
}
|
||||
childID = resp.Id
|
||||
}
|
||||
@@ -190,9 +377,12 @@ func fileToEntry(f *sdk.File, parentID string) drives.Entry {
|
||||
mod, _ := time.Parse("2006-01-02 15:04:05", f.CreateTime)
|
||||
name := f.Name
|
||||
isDir := f.Type == 0
|
||||
id := f.Fid
|
||||
id := f.Id
|
||||
if !isDir && f.Fid != "" {
|
||||
id = f.Fid
|
||||
}
|
||||
if id == "" {
|
||||
id = f.Id
|
||||
id = f.Fid
|
||||
}
|
||||
if isDir && !strings.HasSuffix(name, "/") {
|
||||
// 不改 name,只标志
|
||||
@@ -208,6 +398,156 @@ func fileToEntry(f *sdk.File, parentID string) drives.Entry {
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) rememberFileID(f *sdk.File) {
|
||||
if f == nil || f.Type == 0 {
|
||||
return
|
||||
}
|
||||
objectID := strings.TrimSpace(f.Id)
|
||||
fid := strings.TrimSpace(f.Fid)
|
||||
if objectID == "" {
|
||||
return
|
||||
}
|
||||
d.fileIDMu.Lock()
|
||||
if d.fidToID == nil {
|
||||
d.fidToID = make(map[string]string)
|
||||
}
|
||||
d.fidToID[objectID] = objectID
|
||||
if fid != "" {
|
||||
d.fidToID[fid] = objectID
|
||||
}
|
||||
d.fileIDMu.Unlock()
|
||||
}
|
||||
|
||||
func (d *Driver) rememberFIDMapping(fid, objectID string) {
|
||||
fid = strings.TrimSpace(fid)
|
||||
objectID = strings.TrimSpace(objectID)
|
||||
if fid == "" || objectID == "" {
|
||||
return
|
||||
}
|
||||
d.fileIDMu.Lock()
|
||||
if d.fidToID == nil {
|
||||
d.fidToID = make(map[string]string)
|
||||
}
|
||||
d.fidToID[fid] = objectID
|
||||
d.fidToID[objectID] = objectID
|
||||
d.fileIDMu.Unlock()
|
||||
}
|
||||
|
||||
func (d *Driver) cachedDeleteFileID(fileID string) string {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return ""
|
||||
}
|
||||
d.fileIDMu.RLock()
|
||||
defer d.fileIDMu.RUnlock()
|
||||
return strings.TrimSpace(d.fidToID[fileID])
|
||||
}
|
||||
|
||||
func deleteFileIDFromWopanFile(f *sdk.File, source drives.SourceFile) (string, bool) {
|
||||
if f == nil || f.Type == 0 {
|
||||
return "", false
|
||||
}
|
||||
sourceID := strings.TrimSpace(source.FileID)
|
||||
if sourceID == "" {
|
||||
return "", false
|
||||
}
|
||||
objectID := strings.TrimSpace(f.Id)
|
||||
fid := strings.TrimSpace(f.Fid)
|
||||
if objectID == "" {
|
||||
return "", false
|
||||
}
|
||||
if sourceID != objectID && sourceID != fid {
|
||||
return "", false
|
||||
}
|
||||
return objectID, true
|
||||
}
|
||||
|
||||
func (d *Driver) waitForListSlotLocked(ctx context.Context) error {
|
||||
if d.listInterval <= 0 || d.lastListAt.IsZero() {
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
next := d.lastListAt.Add(d.listInterval)
|
||||
now := time.Now()
|
||||
if now.Before(next) {
|
||||
if err := sleepContext(ctx, next.Sub(now)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, d time.Duration) error {
|
||||
if d <= 0 {
|
||||
return ctx.Err()
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func wopanRequestError(step string, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
wrapped := fmt.Errorf("wopan %s: %w", step, err)
|
||||
if isWopanRateLimitError(err) {
|
||||
return &drives.RateLimitError{
|
||||
Provider: "wopan",
|
||||
Err: wrapped,
|
||||
}
|
||||
}
|
||||
return wrapped
|
||||
}
|
||||
|
||||
func isWopanRateLimitError(err error) bool {
|
||||
if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return false
|
||||
}
|
||||
text := strings.ToLower(strings.TrimSpace(err.Error()))
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(text, "status: 429") ||
|
||||
strings.Contains(text, "status 429") ||
|
||||
strings.Contains(text, "http status: 429") ||
|
||||
strings.Contains(text, "status: 500") ||
|
||||
strings.Contains(text, "status 500") ||
|
||||
strings.Contains(text, "status: 502") ||
|
||||
strings.Contains(text, "status 502") ||
|
||||
strings.Contains(text, "status: 503") ||
|
||||
strings.Contains(text, "status 503") ||
|
||||
strings.Contains(text, "status: 504") ||
|
||||
strings.Contains(text, "status 504") ||
|
||||
strings.Contains(text, "status: 509") ||
|
||||
strings.Contains(text, "status 509") ||
|
||||
strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "rate-limit") ||
|
||||
strings.Contains(text, "throttl") ||
|
||||
strings.Contains(text, "blocked") ||
|
||||
strings.Contains(text, "request has been blocked") ||
|
||||
strings.Contains(text, "操作频繁") ||
|
||||
strings.Contains(text, "请求频繁") ||
|
||||
strings.Contains(text, "请求太频繁") ||
|
||||
strings.Contains(text, "请求过于频繁") ||
|
||||
strings.Contains(text, "频率限制") ||
|
||||
strings.Contains(text, "请求次数过多") ||
|
||||
strings.Contains(text, "系统繁忙") ||
|
||||
strings.Contains(text, "服务繁忙") ||
|
||||
strings.Contains(text, "稍后再试") ||
|
||||
strings.Contains(text, "稍后重试") ||
|
||||
strings.Contains(text, "访问被阻断") ||
|
||||
strings.Contains(text, "风控")
|
||||
}
|
||||
|
||||
func guessMime(name string) string {
|
||||
ext := strings.ToLower(path.Ext(name))
|
||||
switch ext {
|
||||
@@ -229,3 +569,5 @@ func guessMime(name string) string {
|
||||
|
||||
// 确保实现接口
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
var _ drives.SourceRemover = (*Driver)(nil)
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
package wopan
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
sdk "github.com/OpenListTeam/wopan-sdk-go"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestFileToEntryUsesDirectoryIDAndFileFID(t *testing.T) {
|
||||
dir := fileToEntry(&sdk.File{
|
||||
Id: "dir-object-id",
|
||||
Fid: "0",
|
||||
Type: 0,
|
||||
Name: "collection",
|
||||
}, "root")
|
||||
if !dir.IsDir {
|
||||
t.Fatal("directory entry IsDir = false")
|
||||
}
|
||||
if dir.ID != "dir-object-id" {
|
||||
t.Fatalf("directory id = %q, want object id", dir.ID)
|
||||
}
|
||||
|
||||
file := fileToEntry(&sdk.File{
|
||||
Id: "file-object-id",
|
||||
Fid: "fid/with/slash",
|
||||
Type: 1,
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
}, "dir-object-id")
|
||||
if file.IsDir {
|
||||
t.Fatal("file entry IsDir = true")
|
||||
}
|
||||
if file.ID != "fid/with/slash" {
|
||||
t.Fatalf("file id = %q, want fid for download", file.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteFileIDFromWopanFileUsesObjectIDForFID(t *testing.T) {
|
||||
got, ok := deleteFileIDFromWopanFile(&sdk.File{
|
||||
Id: "file-object-id",
|
||||
Fid: "fid/with/slash",
|
||||
Type: 1,
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
}, drives.SourceFile{
|
||||
FileID: "fid/with/slash",
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
})
|
||||
if !ok {
|
||||
t.Fatal("delete file id not resolved")
|
||||
}
|
||||
if got != "file-object-id" {
|
||||
t.Fatalf("delete file id = %q, want object id", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteFileIDFromWopanFileAcceptsObjectID(t *testing.T) {
|
||||
got, ok := deleteFileIDFromWopanFile(&sdk.File{
|
||||
Id: "file-object-id",
|
||||
Fid: "fid-1",
|
||||
Type: 1,
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
}, drives.SourceFile{
|
||||
FileID: "file-object-id",
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
})
|
||||
if !ok {
|
||||
t.Fatal("delete file id not resolved")
|
||||
}
|
||||
if got != "file-object-id" {
|
||||
t.Fatalf("delete file id = %q, want object id", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteFileIDFromWopanFileRejectsIDMismatch(t *testing.T) {
|
||||
if _, ok := deleteFileIDFromWopanFile(&sdk.File{
|
||||
Id: "file-object-id",
|
||||
Fid: "fid-1",
|
||||
Type: 1,
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
}, drives.SourceFile{
|
||||
FileID: "other-fid",
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
}); ok {
|
||||
t.Fatal("delete file id resolved despite id mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWopanRequestErrorWrapsRateLimit(t *testing.T) {
|
||||
err := wopanRequestError("list", errors.New("request failed with status: 429 Too Many Requests"))
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.Provider != "wopan" {
|
||||
t.Fatalf("provider = %q, want wopan", rateLimit.Provider)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWopanRequestErrorLeavesNormalErrors(t *testing.T) {
|
||||
err := wopanRequestError("download url", errors.New("invalid access token"))
|
||||
var rateLimit *drives.RateLimitError
|
||||
if errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want non-rate-limit error", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
package wopan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-resty/resty/v2"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultQRCodeAPIBase = "https://panservice.mail.wo.cn/wohome/open/v1/QRCode"
|
||||
defaultQRCodeClient = "1001000021"
|
||||
)
|
||||
|
||||
type QRConfig struct {
|
||||
APIBaseURL string
|
||||
HTTPClient *http.Client
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type QRClient struct {
|
||||
apiBase string
|
||||
client *resty.Client
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type QRCodeSession struct {
|
||||
UUID string `json:"uuid"`
|
||||
QRImageDataURL string `json:"qrImageDataUrl"`
|
||||
ExpiresAt string `json:"expiresAt,omitempty"`
|
||||
}
|
||||
|
||||
type QRCodeStatus struct {
|
||||
State int `json:"state"`
|
||||
StatusText string `json:"statusText"`
|
||||
AccessToken string `json:"accessToken,omitempty"`
|
||||
RefreshToken string `json:"refreshToken,omitempty"`
|
||||
FamilyID string `json:"familyID,omitempty"`
|
||||
}
|
||||
|
||||
func NewQRClient(c QRConfig) *QRClient {
|
||||
apiBase := strings.TrimRight(strings.TrimSpace(c.APIBaseURL), "/")
|
||||
if apiBase == "" {
|
||||
apiBase = defaultQRCodeAPIBase
|
||||
}
|
||||
httpClient := c.HTTPClient
|
||||
if httpClient == nil {
|
||||
httpClient = &http.Client{Timeout: 20 * time.Second}
|
||||
}
|
||||
now := c.Now
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
return &QRClient{
|
||||
apiBase: apiBase,
|
||||
client: resty.NewWithClient(httpClient).
|
||||
SetTimeout(20*time.Second).
|
||||
SetHeader("Accept", "application/json"),
|
||||
now: now,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *QRClient) Generate(ctx context.Context) (QRCodeSession, error) {
|
||||
var envelope qrEnvelope
|
||||
res, err := c.request(ctx).
|
||||
SetResult(&envelope).
|
||||
Get(c.apiBase + "/generate")
|
||||
if err != nil {
|
||||
return QRCodeSession{}, err
|
||||
}
|
||||
if res.IsError() {
|
||||
return QRCodeSession{}, qrAPIError(envelope.message(), res.StatusCode())
|
||||
}
|
||||
|
||||
var result qrGenerateResult
|
||||
if err := decodeResult(envelope.Result, &result); err != nil {
|
||||
return QRCodeSession{}, err
|
||||
}
|
||||
result.UUID = strings.TrimSpace(result.UUID)
|
||||
result.Image = strings.TrimSpace(result.Image)
|
||||
if result.UUID == "" {
|
||||
return QRCodeSession{}, errors.New("wopan qr: empty uuid")
|
||||
}
|
||||
if result.Image == "" {
|
||||
return QRCodeSession{}, errors.New("wopan qr: empty image")
|
||||
}
|
||||
return QRCodeSession{
|
||||
UUID: result.UUID,
|
||||
QRImageDataURL: qrImageDataURL(result.Image),
|
||||
ExpiresAt: c.now().Add(60 * time.Second).Format(time.RFC3339),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *QRClient) Poll(ctx context.Context, uuid string) (QRCodeStatus, error) {
|
||||
uuid = strings.TrimSpace(uuid)
|
||||
if uuid == "" {
|
||||
return QRCodeStatus{}, errors.New("uuid is required")
|
||||
}
|
||||
|
||||
var envelope qrEnvelope
|
||||
res, err := c.request(ctx).
|
||||
SetQueryParam("uuid", uuid).
|
||||
SetResult(&envelope).
|
||||
Get(c.apiBase + "/query")
|
||||
if err != nil {
|
||||
return QRCodeStatus{}, err
|
||||
}
|
||||
if res.IsError() {
|
||||
return QRCodeStatus{}, qrAPIError(envelope.message(), res.StatusCode())
|
||||
}
|
||||
|
||||
result, err := decodeResultMap(envelope.Result)
|
||||
if err != nil {
|
||||
return QRCodeStatus{}, err
|
||||
}
|
||||
state := intValue(result["state"])
|
||||
status := QRCodeStatus{
|
||||
State: state,
|
||||
StatusText: qrStateText(state),
|
||||
}
|
||||
if state != 3 {
|
||||
return status, nil
|
||||
}
|
||||
|
||||
status.AccessToken = findStringByKeys(result, "access_token", "accessToken", "token", "tokenValue")
|
||||
status.RefreshToken = findStringByKeys(result, "refresh_token", "refreshToken")
|
||||
status.FamilyID = findStringByKeys(result, "family_id", "familyId", "familyID", "defaultFamilyId", "defaultHomeId", "homeId")
|
||||
if status.AccessToken == "" || status.RefreshToken == "" {
|
||||
missing := make([]string, 0, 2)
|
||||
if status.AccessToken == "" {
|
||||
missing = append(missing, "access_token")
|
||||
}
|
||||
if status.RefreshToken == "" {
|
||||
missing = append(missing, "refresh_token")
|
||||
}
|
||||
return QRCodeStatus{}, fmt.Errorf("wopan qr: login succeeded but missing %s; available keys: %s",
|
||||
strings.Join(missing, ", "), strings.Join(collectJSONKeys(result), ", "))
|
||||
}
|
||||
return status, nil
|
||||
}
|
||||
|
||||
func (c *QRClient) request(ctx context.Context) *resty.Request {
|
||||
return c.client.R().
|
||||
SetContext(ctx).
|
||||
SetHeaders(map[string]string{
|
||||
"client-id": defaultQRCodeClient,
|
||||
"x-yp-client-id": defaultQRCodeClient,
|
||||
"Accept": "application/json",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
})
|
||||
}
|
||||
|
||||
type qrEnvelope struct {
|
||||
Meta qrMeta `json:"meta"`
|
||||
Result json.RawMessage `json:"result"`
|
||||
Code any `json:"code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Msg string `json:"msg,omitempty"`
|
||||
}
|
||||
|
||||
type qrMeta struct {
|
||||
Code any `json:"code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Msg string `json:"msg,omitempty"`
|
||||
}
|
||||
|
||||
type qrGenerateResult struct {
|
||||
UUID string `json:"uuid"`
|
||||
Image string `json:"image"`
|
||||
}
|
||||
|
||||
func (e qrEnvelope) message() string {
|
||||
for _, s := range []string{e.Message, e.Msg, e.Meta.Message, e.Meta.Msg} {
|
||||
if strings.TrimSpace(s) != "" {
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func decodeResult(raw json.RawMessage, dst any) error {
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return errors.New("wopan qr: empty result")
|
||||
}
|
||||
if err := json.Unmarshal(raw, dst); err != nil {
|
||||
return fmt.Errorf("wopan qr: decode result: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func decodeResultMap(raw json.RawMessage) (map[string]any, error) {
|
||||
var result map[string]any
|
||||
if err := decodeResult(raw, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if result == nil {
|
||||
return nil, errors.New("wopan qr: empty result")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func qrImageDataURL(image string) string {
|
||||
image = strings.TrimSpace(image)
|
||||
if strings.HasPrefix(strings.ToLower(image), "data:image/") {
|
||||
return image
|
||||
}
|
||||
return "data:image/png;base64," + image
|
||||
}
|
||||
|
||||
func qrAPIError(message string, httpStatus int) error {
|
||||
message = strings.TrimSpace(message)
|
||||
if message == "" {
|
||||
message = fmt.Sprintf("HTTP %d", httpStatus)
|
||||
}
|
||||
return errors.New(message)
|
||||
}
|
||||
|
||||
func qrStateText(state int) string {
|
||||
switch state {
|
||||
case 1:
|
||||
return "等待扫码"
|
||||
case 2:
|
||||
return "已扫码,请在联通网盘 App 确认"
|
||||
case 3:
|
||||
return "登录成功"
|
||||
case 4:
|
||||
return "二维码已过期"
|
||||
default:
|
||||
return "未知状态"
|
||||
}
|
||||
}
|
||||
|
||||
func intValue(v any) int {
|
||||
switch x := v.(type) {
|
||||
case int:
|
||||
return x
|
||||
case int64:
|
||||
return int(x)
|
||||
case float64:
|
||||
return int(x)
|
||||
case json.Number:
|
||||
n, _ := x.Int64()
|
||||
return int(n)
|
||||
case string:
|
||||
n, _ := strconv.Atoi(strings.TrimSpace(x))
|
||||
return n
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func findStringByKeys(v any, keys ...string) string {
|
||||
targets := make(map[string]struct{}, len(keys))
|
||||
for _, key := range keys {
|
||||
targets[normalizeJSONKey(key)] = struct{}{}
|
||||
}
|
||||
return findStringByNormalizedKeys(v, targets)
|
||||
}
|
||||
|
||||
func findStringByNormalizedKeys(v any, targets map[string]struct{}) string {
|
||||
switch x := v.(type) {
|
||||
case map[string]any:
|
||||
for key, value := range x {
|
||||
if _, ok := targets[normalizeJSONKey(key)]; ok {
|
||||
if s := stringValue(value); s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, value := range x {
|
||||
if s := findStringByNormalizedKeys(value, targets); s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, value := range x {
|
||||
if s := findStringByNormalizedKeys(value, targets); s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func stringValue(v any) string {
|
||||
switch x := v.(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(x)
|
||||
case int:
|
||||
return strconv.Itoa(x)
|
||||
case int64:
|
||||
return strconv.FormatInt(x, 10)
|
||||
case float64:
|
||||
if x == float64(int64(x)) {
|
||||
return strconv.FormatInt(int64(x), 10)
|
||||
}
|
||||
return strconv.FormatFloat(x, 'f', -1, 64)
|
||||
case json.Number:
|
||||
return strings.TrimSpace(x.String())
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeJSONKey(key string) string {
|
||||
key = strings.ToLower(strings.TrimSpace(key))
|
||||
key = strings.ReplaceAll(key, "_", "")
|
||||
key = strings.ReplaceAll(key, "-", "")
|
||||
key = strings.ReplaceAll(key, " ", "")
|
||||
return key
|
||||
}
|
||||
|
||||
func collectJSONKeys(v any) []string {
|
||||
seen := map[string]struct{}{}
|
||||
var walk func(any)
|
||||
walk = func(value any) {
|
||||
switch x := value.(type) {
|
||||
case map[string]any:
|
||||
for key, child := range x {
|
||||
if strings.TrimSpace(key) != "" {
|
||||
seen[key] = struct{}{}
|
||||
}
|
||||
walk(child)
|
||||
}
|
||||
case []any:
|
||||
for _, child := range x {
|
||||
walk(child)
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(v)
|
||||
|
||||
keys := make([]string, 0, len(seen))
|
||||
for key := range seen {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
if len(keys) > 16 {
|
||||
keys = append(keys[:16], "...")
|
||||
}
|
||||
return keys
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package wopan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestQRCodeGenerateUsesServiceImage(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/QRCode/generate" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if r.Header.Get("client-id") != defaultQRCodeClient {
|
||||
t.Fatalf("client-id = %q, want %q", r.Header.Get("client-id"), defaultQRCodeClient)
|
||||
}
|
||||
if r.Header.Get("x-yp-client-id") != defaultQRCodeClient {
|
||||
t.Fatalf("x-yp-client-id = %q, want %q", r.Header.Get("x-yp-client-id"), defaultQRCodeClient)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"meta": map[string]string{"code": "0000", "message": "ok"},
|
||||
"result": map[string]string{
|
||||
"uuid": "uuid-1",
|
||||
"image": "iVBORw0KGgo=",
|
||||
},
|
||||
})
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{APIBaseURL: api.URL + "/QRCode"}).Generate(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
if got.UUID != "uuid-1" {
|
||||
t.Fatalf("uuid = %q, want uuid-1", got.UUID)
|
||||
}
|
||||
if got.QRImageDataURL != "data:image/png;base64,iVBORw0KGgo=" {
|
||||
t.Fatalf("qrImageDataUrl = %q, want PNG data URL", got.QRImageDataURL)
|
||||
}
|
||||
if got.ExpiresAt == "" {
|
||||
t.Fatalf("expiresAt is empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQRCodePollPending(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/QRCode/query" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if r.URL.Query().Get("uuid") != "uuid-1" {
|
||||
t.Fatalf("uuid query = %q, want uuid-1", r.URL.Query().Get("uuid"))
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"meta": map[string]string{"code": "0000", "message": "ok"},
|
||||
"result": map[string]any{
|
||||
"state": 1,
|
||||
"token": nil,
|
||||
"refreshToken": nil,
|
||||
},
|
||||
})
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{APIBaseURL: api.URL + "/QRCode"}).Poll(context.Background(), "uuid-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Poll() error = %v", err)
|
||||
}
|
||||
if got.State != 1 || got.StatusText != "等待扫码" || got.AccessToken != "" || got.RefreshToken != "" {
|
||||
t.Fatalf("status = %#v, want pending without tokens", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQRCodePollSuccessMapsTokenFields(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/QRCode/query" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"meta": map[string]string{"code": "0000", "message": "ok"},
|
||||
"result": map[string]any{
|
||||
"state": 3,
|
||||
"token": "access-1",
|
||||
"refreshToken": "refresh-1",
|
||||
},
|
||||
})
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{APIBaseURL: api.URL + "/QRCode"}).Poll(context.Background(), "uuid-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Poll() error = %v", err)
|
||||
}
|
||||
if got.State != 3 || got.AccessToken != "access-1" || got.RefreshToken != "refresh-1" {
|
||||
t.Fatalf("status = %#v, want token and refreshToken mapped", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQRCodePollSuccessReportsMissingTokenKeys(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"meta": map[string]string{"code": "0000", "message": "ok"},
|
||||
"result": map[string]any{
|
||||
"state": 3,
|
||||
"user": map[string]string{"name": "demo"},
|
||||
},
|
||||
})
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
_, err := NewQRClient(QRConfig{APIBaseURL: api.URL + "/QRCode"}).Poll(context.Background(), "uuid-1")
|
||||
if err == nil {
|
||||
t.Fatal("Poll() error is nil, want missing token error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "missing access_token, refresh_token") ||
|
||||
!strings.Contains(err.Error(), "available keys") {
|
||||
t.Fatalf("error = %q, want missing token keys", err.Error())
|
||||
}
|
||||
}
|
||||
@@ -149,6 +149,28 @@ func (w *Worker) Status() TaskStatus {
|
||||
return status
|
||||
}
|
||||
|
||||
// WaitIdle blocks until the fingerprint queue is empty and no item is being processed.
|
||||
func (w *Worker) WaitIdle(ctx context.Context) error {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
if w.queue.lengthExcluding("") == 0 {
|
||||
return nil
|
||||
}
|
||||
ticker := time.NewTicker(200 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
if w.queue.lengthExcluding("") == 0 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
defer w.queue.release(v.ID)
|
||||
if w.Catalog == nil || w.Drive == nil || v == nil || v.ID == "" {
|
||||
@@ -327,11 +349,96 @@ func readHTTPRange(ctx context.Context, hc *http.Client, link *drives.StreamLink
|
||||
return data, nil
|
||||
}
|
||||
}
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
|
||||
if remoteRangeResponseLooksRateLimited(link.URL, resp.StatusCode, body) {
|
||||
return nil, &drives.RateLimitError{
|
||||
Provider: "fingerprint",
|
||||
RetryAfter: parseRetryAfter(resp.Header.Get("Retry-After")),
|
||||
Err: fmt.Errorf("remote sample rate limited: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(body))),
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("fingerprint: range request got status=%d for bytes=%d-%d", resp.StatusCode, r.start, end)
|
||||
}
|
||||
return io.ReadAll(io.LimitReader(resp.Body, r.length))
|
||||
}
|
||||
|
||||
func remoteRangeResponseLooksRateLimited(rawURL string, status int, body []byte) bool {
|
||||
if status == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if isWopanMediaURL(rawURL) && (status == http.StatusForbidden || status == http.StatusTooManyRequests ||
|
||||
status == http.StatusInternalServerError || status == http.StatusBadGateway ||
|
||||
status == http.StatusServiceUnavailable || status == http.StatusGatewayTimeout ||
|
||||
status == 509) {
|
||||
return true
|
||||
}
|
||||
text := strings.ToLower(strings.TrimSpace(string(body)))
|
||||
compact := compactRemoteRangeErrorText(text)
|
||||
if strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "quota exceeded") ||
|
||||
strings.Contains(text, "操作频繁") ||
|
||||
strings.Contains(text, "请求频繁") ||
|
||||
strings.Contains(text, "请求太频繁") ||
|
||||
strings.Contains(text, "请求过于频繁") ||
|
||||
strings.Contains(text, "频率限制") ||
|
||||
strings.Contains(text, "请求次数过多") ||
|
||||
strings.Contains(text, "系统繁忙") ||
|
||||
strings.Contains(text, "服务繁忙") ||
|
||||
strings.Contains(text, "稍后再试") ||
|
||||
strings.Contains(text, "稍后重试") ||
|
||||
strings.Contains(text, "访问被阻断") ||
|
||||
strings.Contains(text, "风控") ||
|
||||
strings.Contains(text, "download quota") ||
|
||||
strings.Contains(text, "sharing rate") ||
|
||||
strings.Contains(text, "daily limit") ||
|
||||
strings.Contains(text, "user rate") ||
|
||||
strings.Contains(text, "usage limit") ||
|
||||
strings.Contains(compact, "ratelimitexceeded") ||
|
||||
strings.Contains(compact, "userratelimitexceeded") ||
|
||||
strings.Contains(compact, "dailylimitexceeded") ||
|
||||
strings.Contains(compact, "downloadquotaexceeded") ||
|
||||
strings.Contains(compact, "sharingratelimitexceeded") ||
|
||||
strings.Contains(compact, "quotaexceeded") ||
|
||||
strings.Contains(compact, "toomanyrequests") ||
|
||||
strings.Contains(compact, "usagelimits") {
|
||||
return true
|
||||
}
|
||||
if status == http.StatusForbidden && isGoogleDriveMediaURL(rawURL) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isWopanMediaURL(rawURL string) bool {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
path := strings.ToLower(u.Path)
|
||||
return (strings.HasSuffix(host, "pan.wo.cn") ||
|
||||
strings.HasSuffix(host, "smartont.net") ||
|
||||
strings.Contains(host, "wo.cn")) &&
|
||||
strings.Contains(path, "/openapi/download")
|
||||
}
|
||||
|
||||
func isGoogleDriveMediaURL(rawURL string) bool {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Host)
|
||||
path := strings.ToLower(u.Path)
|
||||
return strings.Contains(host, "googleapis.com") && strings.Contains(path, "/drive/")
|
||||
}
|
||||
|
||||
func compactRemoteRangeErrorText(text string) string {
|
||||
replacer := strings.NewReplacer("_", "", "-", "", " ", "", ".", "", ":", "")
|
||||
return replacer.Replace(strings.ToLower(strings.TrimSpace(text)))
|
||||
}
|
||||
|
||||
func parseRetryAfter(raw string) time.Duration {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
|
||||
@@ -2,6 +2,7 @@ package fingerprint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -85,6 +86,51 @@ func TestComputeRemoteUsesRangeSamples(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeRemoteGoogleQuotaExceededReturnsRateLimit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Retry-After", "60")
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
_, _ = w.Write([]byte(`{"error":{"code":403,"message":"The download quota for this file has been exceeded.","errors":[{"domain":"usageLimits","reason":"downloadQuotaExceeded","message":"The download quota for this file has been exceeded."}]}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
drv := &fakeDrive{paths: map[string]string{"remote": srv.URL + "/drive/v3/files/file-1?alt=media"}}
|
||||
_, err := Compute(ctx, drv, &catalog.Video{ID: "remote", FileID: "remote", Size: 1024 * 1024}, Config{
|
||||
SampleSizeBytes: 4,
|
||||
FullHashMaxSize: 8,
|
||||
HTTPClient: srv.Client(),
|
||||
}, srv.Client())
|
||||
if err == nil {
|
||||
t.Fatal("compute succeeded, want rate limit")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != time.Minute {
|
||||
t.Fatalf("retry after = %s, want 1m", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWopanRemoteRangeErrorsLookRateLimited(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
rawURL string
|
||||
status int
|
||||
}{
|
||||
{rawURL: "https://gxdownload.pan.wo.cn:8445/openapi/download?fid=encoded", status: http.StatusForbidden},
|
||||
{rawURL: "https://du.smartont.net:8445/openapi/download?fid=encoded", status: http.StatusServiceUnavailable},
|
||||
{rawURL: "https://du.smartont.net:8445/openapi/download?fid=encoded", status: 509},
|
||||
} {
|
||||
if !remoteRangeResponseLooksRateLimited(tc.rawURL, tc.status, nil) {
|
||||
t.Fatalf("remoteRangeResponseLooksRateLimited(%q, %d) = false, want true", tc.rawURL, tc.status)
|
||||
}
|
||||
}
|
||||
if remoteRangeResponseLooksRateLimited("https://example.com/video.mp4", http.StatusForbidden, nil) {
|
||||
t.Fatal("generic 403 should not be treated as wopan rate limit")
|
||||
}
|
||||
}
|
||||
|
||||
type fakeDrive struct {
|
||||
paths map[string]string
|
||||
}
|
||||
|
||||
@@ -1427,11 +1427,17 @@ func (w *Worker) skipIfRateLimited(v *catalog.Video) bool {
|
||||
}
|
||||
|
||||
func (w *Worker) pauseForRateLimit(err error, step, title string) bool {
|
||||
_, ok := drives.RateLimitRetryAfter(err)
|
||||
wait, ok := drives.RateLimitRetryAfter(err)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), defaultGenerationRateLimitCooldown)
|
||||
if wait <= 0 {
|
||||
wait = w.RateLimitCooldown
|
||||
if wait <= 0 {
|
||||
wait = defaultGenerationRateLimitCooldown
|
||||
}
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), wait)
|
||||
log.Printf("[preview] drive=%s rate-limited until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err)
|
||||
return true
|
||||
}
|
||||
@@ -1460,11 +1466,17 @@ func (w *ThumbWorker) skipIfRateLimited(v *catalog.Video) bool {
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) pauseForRateLimit(err error, step, title string) bool {
|
||||
_, ok := drives.RateLimitRetryAfter(err)
|
||||
wait, ok := drives.RateLimitRetryAfter(err)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), defaultGenerationRateLimitCooldown)
|
||||
if wait <= 0 {
|
||||
wait = w.RateLimitCooldown
|
||||
if wait <= 0 {
|
||||
wait = defaultGenerationRateLimitCooldown
|
||||
}
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), wait)
|
||||
log.Printf("[thumb] drive=%s rate-limited until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err)
|
||||
return true
|
||||
}
|
||||
@@ -1538,7 +1550,7 @@ func driveErrorShouldCooldown(d drives.Drive, err error) bool {
|
||||
strings.Contains(text, "partial file") ||
|
||||
strings.Contains(text, "service unavailable")
|
||||
case "p123":
|
||||
// 123 云盘直链解析 / ffmpeg 读取阶段可能返回 429、5xx,或 WAF 类
|
||||
// 123网盘直链解析 / ffmpeg 读取阶段可能返回 429、5xx,或 WAF 类
|
||||
// blocked / 访问阻断文本。命中时冷却,避免封面和预览视频生成连续打接口。
|
||||
text := strings.ToLower(err.Error())
|
||||
return strings.Contains(text, "请求太频繁") ||
|
||||
@@ -1560,10 +1572,91 @@ func driveErrorShouldCooldown(d drives.Drive, err error) bool {
|
||||
strings.Contains(text, "blocked") ||
|
||||
strings.Contains(text, "访问被阻断") ||
|
||||
strings.Contains(text, "service unavailable")
|
||||
case "wopan":
|
||||
// 联通网盘的取链接口和下载直链都可能返回"操作频繁"、429、5xx
|
||||
// 或 WAF 阻断文本。封面/预览失败时先冷却,避免持续触发风控。
|
||||
text := strings.ToLower(err.Error())
|
||||
return strings.Contains(text, "请求太频繁") ||
|
||||
strings.Contains(text, "请求过于频繁") ||
|
||||
strings.Contains(text, "请求频繁") ||
|
||||
strings.Contains(text, "操作频繁") ||
|
||||
strings.Contains(text, "频率限制") ||
|
||||
strings.Contains(text, "请求次数过多") ||
|
||||
strings.Contains(text, "系统繁忙") ||
|
||||
strings.Contains(text, "服务繁忙") ||
|
||||
strings.Contains(text, "稍后再试") ||
|
||||
strings.Contains(text, "稍后重试") ||
|
||||
strings.Contains(text, "429") ||
|
||||
strings.Contains(text, "http 500") ||
|
||||
strings.Contains(text, "http 502") ||
|
||||
strings.Contains(text, "http 503") ||
|
||||
strings.Contains(text, "http 504") ||
|
||||
strings.Contains(text, "http 509") ||
|
||||
strings.Contains(text, "server returned 403") ||
|
||||
strings.Contains(text, "403 forbidden") ||
|
||||
strings.Contains(text, "server returned 429") ||
|
||||
strings.Contains(text, "server returned 500") ||
|
||||
strings.Contains(text, "server returned 502") ||
|
||||
strings.Contains(text, "server returned 503") ||
|
||||
strings.Contains(text, "server returned 504") ||
|
||||
strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "rate-limit") ||
|
||||
strings.Contains(text, "throttl") ||
|
||||
strings.Contains(text, "blocked") ||
|
||||
strings.Contains(text, "request has been blocked") ||
|
||||
strings.Contains(text, "访问被阻断") ||
|
||||
strings.Contains(text, "风控") ||
|
||||
strings.Contains(text, "service unavailable")
|
||||
case "googledrive":
|
||||
// Google Drive 下载/取样阶段常把频控和配额问题包装成 403,
|
||||
// 具体标识在 error.errors[].reason/message 里(OpenList 也按该结构解析)。
|
||||
// ffmpeg/ffprobe 只能看到 stderr 文本时,按这些 reason/文本兜底冷却。
|
||||
text := strings.ToLower(err.Error())
|
||||
return googleDriveMediaErrorShouldCooldown(text)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func googleDriveMediaErrorShouldCooldown(text string) bool {
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
compact := compactGoogleDriveErrorText(text)
|
||||
return strings.Contains(text, "server returned 403") ||
|
||||
strings.Contains(text, "403 forbidden") ||
|
||||
strings.Contains(text, "server returned 429") ||
|
||||
strings.Contains(text, "http 429") ||
|
||||
strings.Contains(text, "http 500") ||
|
||||
strings.Contains(text, "http 502") ||
|
||||
strings.Contains(text, "http 503") ||
|
||||
strings.Contains(text, "http 504") ||
|
||||
strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "quota exceeded") ||
|
||||
strings.Contains(text, "download quota") ||
|
||||
strings.Contains(text, "sharing rate") ||
|
||||
strings.Contains(text, "daily limit") ||
|
||||
strings.Contains(text, "user rate") ||
|
||||
strings.Contains(text, "usage limit") ||
|
||||
strings.Contains(text, "service unavailable") ||
|
||||
strings.Contains(compact, "ratelimitexceeded") ||
|
||||
strings.Contains(compact, "userratelimitexceeded") ||
|
||||
strings.Contains(compact, "dailylimitexceeded") ||
|
||||
strings.Contains(compact, "downloadquotaexceeded") ||
|
||||
strings.Contains(compact, "sharingratelimitexceeded") ||
|
||||
strings.Contains(compact, "quotaexceeded") ||
|
||||
strings.Contains(compact, "toomanyrequests") ||
|
||||
strings.Contains(compact, "usagelimits")
|
||||
}
|
||||
|
||||
func compactGoogleDriveErrorText(text string) string {
|
||||
replacer := strings.NewReplacer("_", "", "-", "", " ", "", ".", "", ":", "")
|
||||
return replacer.Replace(strings.ToLower(strings.TrimSpace(text)))
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) bool {
|
||||
if w.skipIfRateLimited(v) {
|
||||
return false
|
||||
|
||||
@@ -442,7 +442,7 @@ func TestPreviewWorkerRateLimitLeavesCurrentPendingAndSkipsNextVideo(t *testing.
|
||||
if gen.generateCalls != 1 {
|
||||
t.Fatalf("generate calls = %d, want 1", gen.generateCalls)
|
||||
}
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 5*time.Minute)
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 2*time.Hour)
|
||||
|
||||
gen.generateErr = nil
|
||||
worker.process(ctx, &second)
|
||||
@@ -458,7 +458,7 @@ func TestPreviewWorkerRateLimitLeavesCurrentPendingAndSkipsNextVideo(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerRateLimitCoolsDownFiveMinutes(t *testing.T) {
|
||||
func TestThumbWorkerRateLimitHonorsRetryAfter(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-rate-limit")
|
||||
|
||||
@@ -482,7 +482,7 @@ func TestThumbWorkerRateLimitCoolsDownFiveMinutes(t *testing.T) {
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want unchanged after rate limit", got.ThumbnailURL)
|
||||
}
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 5*time.Minute)
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 2*time.Hour)
|
||||
}
|
||||
|
||||
func TestThumbWorkerP115TransientErrorFailsAfterRetryLimit(t *testing.T) {
|
||||
@@ -661,6 +661,41 @@ func TestP123TransientErrorsShouldCooldown(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWopanTransientErrorsShouldCooldown(t *testing.T) {
|
||||
drv := &previewFakeDrive{kind: "wopan"}
|
||||
for _, err := range []error{
|
||||
errors.New("ffmpeg: Server returned 403 Forbidden"),
|
||||
errors.New("wopan download url: request failed with status: 429 Too Many Requests"),
|
||||
errors.New("操作频繁,请稍后重试"),
|
||||
errors.New("http 503 service unavailable"),
|
||||
} {
|
||||
if !driveErrorShouldCooldown(drv, err) {
|
||||
t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err)
|
||||
}
|
||||
}
|
||||
if driveErrorShouldCooldown(drv, errors.New("invalid access token")) {
|
||||
t.Fatal("invalid access token should not trigger wopan cooldown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoogleDriveMediaErrorsShouldCooldown(t *testing.T) {
|
||||
drv := &previewFakeDrive{kind: "googledrive"}
|
||||
for _, err := range []error{
|
||||
errors.New("google drive api error: usageLimits userRateLimitExceeded"),
|
||||
errors.New("ffmpeg: Server returned 403 Forbidden"),
|
||||
errors.New("downloadQuotaExceeded: The download quota for this file has been exceeded"),
|
||||
errors.New("sharingRateLimitExceeded"),
|
||||
errors.New("http 503 service unavailable"),
|
||||
} {
|
||||
if !driveErrorShouldCooldown(drv, err) {
|
||||
t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err)
|
||||
}
|
||||
}
|
||||
if driveErrorShouldCooldown(drv, errors.New("invalid credentials")) {
|
||||
t.Fatal("invalid credentials should not trigger googledrive cooldown")
|
||||
}
|
||||
}
|
||||
|
||||
func assertCooldownAround(t *testing.T, until time.Time, before time.Time, want time.Duration) {
|
||||
t.Helper()
|
||||
if until.IsZero() {
|
||||
|
||||
@@ -147,15 +147,17 @@ func (p *Proxy) ServeStream(w http.ResponseWriter, r *http.Request, driveID, fil
|
||||
// CDN 不校验请求头,直连可获得最佳带宽并避免占用 backend 出站
|
||||
// - onedrive:Microsoft Graph 返回的 @microsoft.graph.downloadUrl 是短期
|
||||
// 免鉴权下载 URL,不需要后端继续代传视频字节
|
||||
// - p123:123 云盘 download_info 返回的下载页会再跳 CDN;driver 已在后端
|
||||
// - p123:123网盘 download_info 返回的下载页会再跳 CDN;driver 已在后端
|
||||
// 先解出最终 Location,浏览器可直接 302 到该短期地址
|
||||
// - wopan:联通网盘 GetDownloadUrlV2 返回的是短期直链,OpenList 也是直接
|
||||
// 将该 URL 交给客户端使用;不需要后端持续代传视频字节
|
||||
//
|
||||
// 其余网盘(如沃盘 / 夸克等)仍走反代,因为它们的下载
|
||||
// 其余网盘(如夸克等)仍走反代,因为它们的下载
|
||||
// 链接通常需要随请求带上后端持有的 Cookie / Authorization / Range
|
||||
// 的特殊处理,浏览器拿不到这些上下文。
|
||||
func shouldRedirect(d drives.Drive) bool {
|
||||
switch d.Kind() {
|
||||
case "p115", "pikpak", "onedrive", "p123":
|
||||
case "p115", "pikpak", "onedrive", "p123", "wopan":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
|
||||
@@ -201,6 +201,31 @@ func TestServeStreamRedirectsP123(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamRedirectsWopan(t *testing.T) {
|
||||
reg := NewRegistry()
|
||||
drv := &proxyFakeSimpleDrive{
|
||||
kind: "wopan",
|
||||
url: "https://du.smartont.net:8445/openapi/download?fid=encoded",
|
||||
}
|
||||
reg.Set("wopan", drv)
|
||||
|
||||
p := New(reg)
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/wopan/file-1", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
p.ServeStream(rr, req, "wopan", "file-1")
|
||||
|
||||
if rr.Code != http.StatusFound {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusFound)
|
||||
}
|
||||
if got := rr.Header().Get("Location"); got != "https://du.smartont.net:8445/openapi/download?fid=encoded" {
|
||||
t.Fatalf("Location = %q", got)
|
||||
}
|
||||
if drv.calls != 1 {
|
||||
t.Fatalf("link calls = %d, want 1", drv.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamServesLocalFilePath(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "video.mp4")
|
||||
if err := os.WriteFile(path, []byte("0123456789"), 0o644); err != nil {
|
||||
|
||||
@@ -2,6 +2,7 @@ package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"log"
|
||||
"path"
|
||||
@@ -25,6 +26,8 @@ type Scanner struct {
|
||||
SkipDirIDs map[string]struct{}
|
||||
// 回调:新视频被加入后触发预览视频生成
|
||||
OnNewVideo func(v *catalog.Video)
|
||||
// OnProgress 在扫描进度变化时触发。回调只应读取 Stats 里的计数,不应修改 map 字段。
|
||||
OnProgress func(stats Stats)
|
||||
// ProgressInterval 控制扫描内部 heartbeat 的最小输出间隔。
|
||||
// 0 → 默认 30s;< 0 → 关闭 heartbeat(仅留外层 start / done 两行)。
|
||||
// heartbeat 单行格式:
|
||||
@@ -91,6 +94,9 @@ func (s *Scanner) Run(ctx context.Context, startDirID string) (Stats, error) {
|
||||
driveID = s.Drive.ID()
|
||||
}
|
||||
progress := func(currentDir string) {
|
||||
if s.OnProgress != nil {
|
||||
s.OnProgress(stats)
|
||||
}
|
||||
if interval < 0 {
|
||||
return
|
||||
}
|
||||
@@ -149,7 +155,6 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
continue
|
||||
}
|
||||
|
||||
stats.Scanned++
|
||||
ext := strings.ToLower(path.Ext(e.Name))
|
||||
if !s.Exts[ext] {
|
||||
continue
|
||||
@@ -157,9 +162,11 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
if e.Size <= 0 {
|
||||
continue
|
||||
}
|
||||
stats.Scanned++
|
||||
progress(dirName)
|
||||
stats.SeenFileIDs[e.ID] = struct{}{}
|
||||
|
||||
id := s.Drive.Kind() + "-" + s.Drive.ID() + "-" + e.ID
|
||||
id := s.Drive.Kind() + "-" + s.Drive.ID() + "-" + videoIDFilePart(e.ID)
|
||||
if deleted, err := s.Catalog.IsDeletedVideoCandidate(ctx, id, s.Drive.ID(), e.ID, e.Hash, e.Name, e.Size); err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return ctxErr
|
||||
@@ -266,6 +273,7 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
return err
|
||||
}
|
||||
stats.Added++
|
||||
progress(dirName)
|
||||
if s.OnNewVideo != nil {
|
||||
s.OnNewVideo(v)
|
||||
}
|
||||
@@ -332,3 +340,10 @@ func mergeTags(lists ...[]string) []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func videoIDFilePart(fileID string) string {
|
||||
if !strings.ContainsAny(fileID, `/\`+"\x00") {
|
||||
return fileID
|
||||
}
|
||||
return "b64_" + base64.RawURLEncoding.EncodeToString([]byte(fileID))
|
||||
}
|
||||
|
||||
@@ -91,6 +91,84 @@ func TestRunIgnoresZeroSizeVideoFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunScannedCountsOnlyVideoCandidates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := &scannerFakeDrive{
|
||||
entries: []drives.Entry{
|
||||
{ID: "file-1", Name: "clip.mp4", Size: 123},
|
||||
{ID: "file-2", Name: "notes.txt", Size: 123},
|
||||
{ID: "file-3", Name: "empty.mp4", Size: 0},
|
||||
},
|
||||
}
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
|
||||
stats, err := sc.Run(ctx, "")
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Scanned != 1 {
|
||||
t.Fatalf("scanned = %d, want one non-empty video candidate", stats.Scanned)
|
||||
}
|
||||
if stats.Added != 1 {
|
||||
t.Fatalf("added = %d, want one added video", stats.Added)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUsesPathSafeVideoIDForUnsafeFileID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := &scannerFakeDrive{
|
||||
entries: []drives.Entry{{
|
||||
ID: "fid/with space",
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
}},
|
||||
}
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
|
||||
stats, err := sc.Run(ctx, "")
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Added != 1 {
|
||||
t.Fatalf("added = %d, want 1", stats.Added)
|
||||
}
|
||||
if _, ok := stats.SeenFileIDs["fid/with space"]; !ok {
|
||||
t.Fatalf("seen file ids = %#v, want original file id", stats.SeenFileIDs)
|
||||
}
|
||||
|
||||
wantID := "fake-drive-b64_ZmlkL3dpdGggc3BhY2U"
|
||||
got, err := cat.GetVideo(ctx, wantID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video %s: %v", wantID, err)
|
||||
}
|
||||
if strings.Contains(got.ID, "/") {
|
||||
t.Fatalf("video id = %q, must not contain slash", got.ID)
|
||||
}
|
||||
if got.FileID != "fid/with space" {
|
||||
t.Fatalf("file id = %q, want original", got.FileID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStopsWhenContextCanceledDuringFileLoop(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Package spider91migrate 周期性把 spider91 drive 下载到本地的视频
|
||||
// 上传到一个指定的目标 drive 目录(PikPak、115、123 或 OneDrive),上传成功后:
|
||||
// 上传到一个指定的目标 drive 目录(PikPak、115、123、OneDrive、Google Drive 或联通网盘),上传成功后:
|
||||
//
|
||||
// - 改写 catalog 行:drive_id / file_id / content_hash 改成目标盘的;
|
||||
// 视频自身的 id 不变(仍是 spider91-<driveID>-<viewkey>),video_tags、
|
||||
@@ -16,6 +16,7 @@ package spider91migrate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -29,22 +30,27 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/googledrive"
|
||||
"github.com/video-site/backend/internal/drives/onedrive"
|
||||
"github.com/video-site/backend/internal/drives/p115"
|
||||
"github.com/video-site/backend/internal/drives/p123"
|
||||
"github.com/video-site/backend/internal/drives/pikpak"
|
||||
"github.com/video-site/backend/internal/drives/scriptcrawler"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/drives/wopan"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
)
|
||||
|
||||
// uploadTarget 是 migrator 调用目标 drive 的最小接口。任何一种"接收 spider91 上传"的
|
||||
// 网盘都要实现它;当前 PikPak、115、123 和 OneDrive 各自通过适配器满足。
|
||||
// 网盘都要实现它;当前 PikPak、115、123、OneDrive、Google Drive 和联通网盘各自通过适配器满足。
|
||||
//
|
||||
// 这一层抽象把"迁移调用方"和"具体盘的 SDK 协议"解耦:
|
||||
// - PikPak 走 GCID + OSS PutObject(pikpak.UploadResult)
|
||||
// - 115 走 SHA1 + 秒传 / OSS / 分片(p115.UploadResult)
|
||||
// - 123 走 MD5 + 秒传 / S3 预签名分片(p123.UploadResult)
|
||||
// - OneDrive 走 SHA1 + 小文件 PUT / 大文件 upload session
|
||||
// - Google Drive 走 MD5 + resumable upload session
|
||||
// - 联通网盘 走 SDK Upload2C,当前上游不返回内容 hash
|
||||
//
|
||||
// 各家返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
|
||||
type uploadTarget interface {
|
||||
@@ -56,10 +62,21 @@ type uploadTarget interface {
|
||||
Rename(ctx context.Context, fileID, newName string) error
|
||||
}
|
||||
|
||||
// Spider91LocalSource is the local source interface used by the migration
|
||||
// worker. Legacy spider91.Driver and the new scriptcrawler.Driver both satisfy
|
||||
// it when they are mounted for the Spider91 built-in crawler.
|
||||
type Spider91LocalSource interface {
|
||||
drives.Drive
|
||||
VideosDir() string
|
||||
ThumbsDir() string
|
||||
VideoPath(fileID string) (string, error)
|
||||
ThumbPath(fileID string) (string, error)
|
||||
}
|
||||
|
||||
// UploadResult 是 uploadTarget.UploadAndReportHash 的归一返回。
|
||||
//
|
||||
// FileID 目标盘上的新文件 ID;
|
||||
// Hash GCID(PikPak)、MD5 HEX(123)或 SHA1 HEX(115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;
|
||||
// Hash GCID(PikPak)、MD5 HEX(123 / Google Drive)或 SHA1 HEX(115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;联通网盘暂为空;
|
||||
// Size 实际上传字节数。
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
@@ -67,9 +84,33 @@ type UploadResult struct {
|
||||
Size int64
|
||||
}
|
||||
|
||||
const spider91UploadDirName = "91 Spider"
|
||||
type UploadProgress struct {
|
||||
DriveID string
|
||||
State string
|
||||
CurrentTitle string
|
||||
QueueLength int
|
||||
DoneCount int
|
||||
TotalCount int
|
||||
}
|
||||
|
||||
// pikpakAdapter / p115Adapter / p123Adapter / onedriveAdapter 把具体 driver 包装成 uploadTarget。
|
||||
const (
|
||||
spider91UploadDirName = "91 Spider"
|
||||
scriptCrawlerUploadRootDirName = "Script Crawlers"
|
||||
)
|
||||
|
||||
type migrationPlan struct {
|
||||
source Spider91LocalSource
|
||||
row *catalog.Drive
|
||||
sourceKinds []string
|
||||
targetDriveID string
|
||||
target uploadTarget
|
||||
uploadDir string
|
||||
keepLatestN int
|
||||
requireAssetsReady bool
|
||||
legacyBackfill bool
|
||||
}
|
||||
|
||||
// pikpakAdapter / p115Adapter / p123Adapter / onedriveAdapter / googledriveAdapter / wopanAdapter 把具体 driver 包装成 uploadTarget。
|
||||
//
|
||||
// 之所以不让 driver 直接实现 uploadTarget:
|
||||
//
|
||||
@@ -160,6 +201,48 @@ func (a *onedriveAdapter) Rename(ctx context.Context, fileID, newName string) er
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
type googledriveAdapter struct {
|
||||
d *googledrive.Driver
|
||||
}
|
||||
|
||||
func (a *googledriveAdapter) ID() string { return a.d.ID() }
|
||||
func (a *googledriveAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *googledriveAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *googledriveAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *googledriveAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
return UploadResult{FileID: res.FileID, Hash: res.Hash, Size: res.Size}, nil
|
||||
}
|
||||
func (a *googledriveAdapter) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
type wopanAdapter struct {
|
||||
d *wopan.Driver
|
||||
}
|
||||
|
||||
func (a *wopanAdapter) ID() string { return a.d.ID() }
|
||||
func (a *wopanAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *wopanAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *wopanAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *wopanAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
fileID, err := a.d.Upload(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
return UploadResult{FileID: fileID, Size: size}, nil
|
||||
}
|
||||
func (a *wopanAdapter) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
// adaptUploadTarget 把通用 drive 包装成 uploadTarget。
|
||||
// 不支持的盘 kind 返回 error;调用方静默跳过。
|
||||
func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
@@ -172,6 +255,10 @@ func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
return &p123Adapter{d: v}, nil
|
||||
case *onedrive.Driver:
|
||||
return &onedriveAdapter{d: v}, nil
|
||||
case *googledrive.Driver:
|
||||
return &googledriveAdapter{d: v}, nil
|
||||
case *wopan.Driver:
|
||||
return &wopanAdapter{d: v}, nil
|
||||
case uploadTarget:
|
||||
// 测试或自定义实现可以直接传入;优先使用具体类型分支以拿到适配器。
|
||||
return v, nil
|
||||
@@ -201,9 +288,10 @@ type Config struct {
|
||||
// CaptchaCooldown 是迁移 worker 在遇到 PikPak captcha 错误(error_code
|
||||
// 4002 / 9)后整体进入冷却的时长。冷却期间 runOnce 直接返回,不再发起任何
|
||||
// PikPak API 请求,避免被进一步风控。0 时默认 5 分钟;< 0 关闭冷却(仅用于测试)。
|
||||
CaptchaCooldown time.Duration
|
||||
CommonThumbDir string
|
||||
OnMigrated func(videoID string)
|
||||
CaptchaCooldown time.Duration
|
||||
CommonThumbDir string
|
||||
OnMigrated func(videoID string)
|
||||
OnUploadProgress func(UploadProgress)
|
||||
}
|
||||
|
||||
type Migrator struct {
|
||||
@@ -332,59 +420,79 @@ func (m *Migrator) runOnce(ctx context.Context) {
|
||||
log.Printf("[spider91migrate] captcha cooldown ended at %s, resuming migration", until.Format(time.RFC3339))
|
||||
}
|
||||
|
||||
target, pp, err := m.resolveTarget()
|
||||
if err != nil {
|
||||
// 没目标就静默 —— 用户选择了本地保存,或还没配 115/PikPak drive。
|
||||
plans := m.migrationPlans(ctx)
|
||||
if len(plans) == 0 {
|
||||
// 没目标就静默 —— 用户选择了本地保存,或目标盘还没挂载。
|
||||
return
|
||||
}
|
||||
|
||||
migrated := 0
|
||||
for _, src := range m.spider91Drives() {
|
||||
backfillTargets := map[string]uploadTarget{}
|
||||
for _, plan := range plans {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return
|
||||
}
|
||||
n, err := m.migrateDrive(ctx, src, target, pp)
|
||||
n, err := m.migrateDrive(ctx, plan)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] drive=%s migrate batch error: %v", src.ID(), err)
|
||||
log.Printf("[spider91migrate] drive=%s migrate batch error: %v", plan.source.ID(), err)
|
||||
}
|
||||
migrated += n
|
||||
if active, _ := m.inCooldown(); active {
|
||||
if migrated > 0 {
|
||||
log.Printf("[spider91migrate] migrated %d video(s) to drive=%s", migrated, target)
|
||||
log.Printf("[spider91migrate] migrated %d video(s)", migrated)
|
||||
}
|
||||
return
|
||||
}
|
||||
if plan.legacyBackfill {
|
||||
backfillTargets[plan.targetDriveID] = plan.target
|
||||
}
|
||||
}
|
||||
if migrated > 0 {
|
||||
log.Printf("[spider91migrate] migrated %d video(s) to drive=%s", migrated, target)
|
||||
log.Printf("[spider91migrate] migrated %d video(s)", migrated)
|
||||
}
|
||||
|
||||
// 收尾:扫每个 spider91 drive 的本地目录,把 catalog 已经迁到别处但本地
|
||||
// 收尾:扫每个本地爬虫 drive 的 videos 目录,把 catalog 已经迁到别处但本地
|
||||
// 仍有残留的孤儿文件清掉。这是纯防御性兜底——正常路径下 migrateDrive
|
||||
// 已经在迁移成功后立刻 CleanupSpider91Local,不会留孤儿。
|
||||
for _, src := range m.spider91Drives() {
|
||||
for _, plan := range plans {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return
|
||||
}
|
||||
deleted, err := m.cleanupOldLocalVideos(ctx, src)
|
||||
deleted, err := m.cleanupOldLocalVideos(ctx, plan)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] cleanup drive=%s: %v", src.ID(), err)
|
||||
log.Printf("[spider91migrate] cleanup drive=%s: %v", plan.source.ID(), err)
|
||||
}
|
||||
if deleted > 0 {
|
||||
log.Printf("[spider91migrate] cleanup drive=%s deleted %d orphan local file(s)", src.ID(), deleted)
|
||||
log.Printf("[spider91migrate] cleanup drive=%s deleted %d orphan local file(s)", plan.source.ID(), deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// 回填:把已迁移到 PikPak 的 spider91-* 视频里文件名仍是旧格式
|
||||
// (比如刚迁完没改、或人工导入)的统一改成方案 B 期望的格式。
|
||||
// 这一步幂等:已经是期望格式的不会再调 Rename。
|
||||
if renamed, err := m.backfillFileNames(ctx, target, pp); err != nil {
|
||||
log.Printf("[spider91migrate] backfill names: %v", err)
|
||||
} else if renamed > 0 {
|
||||
log.Printf("[spider91migrate] backfilled %d %s file name(s) to desired format", renamed, m.targetKindForLog())
|
||||
for targetDriveID, pp := range backfillTargets {
|
||||
if renamed, err := m.backfillFileNames(ctx, targetDriveID, pp); err != nil {
|
||||
log.Printf("[spider91migrate] backfill names: %v", err)
|
||||
} else if renamed > 0 {
|
||||
log.Printf("[spider91migrate] backfilled %d %s file name(s) to desired format", renamed, pp.Kind())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Migrator) reportUploadProgress(progress UploadProgress) {
|
||||
if m == nil || m.cfg.OnUploadProgress == nil {
|
||||
return
|
||||
}
|
||||
progress.DriveID = strings.TrimSpace(progress.DriveID)
|
||||
if progress.DriveID == "" {
|
||||
return
|
||||
}
|
||||
if progress.State == "" {
|
||||
progress.State = "idle"
|
||||
}
|
||||
m.cfg.OnUploadProgress(progress)
|
||||
}
|
||||
|
||||
// targetKindForLog 把当前目标盘 kind 转成对人友好的简称,用于日志。
|
||||
// 解析失败时回退 "target"。
|
||||
func (m *Migrator) targetKindForLog() string {
|
||||
@@ -409,9 +517,17 @@ func (m *Migrator) resolveTarget() (string, uploadTarget, error) {
|
||||
return "", nil, errors.New("no target getter")
|
||||
}
|
||||
id := m.cfg.GetTargetDriveID()
|
||||
return m.resolveTargetID(id)
|
||||
}
|
||||
|
||||
func (m *Migrator) resolveTargetID(id string) (string, uploadTarget, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
return "", nil, errors.New("target drive not configured")
|
||||
}
|
||||
if m.cfg.Registry == nil {
|
||||
return "", nil, errors.New("registry not configured")
|
||||
}
|
||||
d, ok := m.cfg.Registry.Get(id)
|
||||
if !ok {
|
||||
return "", nil, fmt.Errorf("target drive %q not in registry", id)
|
||||
@@ -423,33 +539,142 @@ func (m *Migrator) resolveTarget() (string, uploadTarget, error) {
|
||||
return id, t, nil
|
||||
}
|
||||
|
||||
// spider91Drives 返回当前注册的所有 spider91 driver。
|
||||
func (m *Migrator) spider91Drives() []*spider91.Driver {
|
||||
func (m *Migrator) migrationPlans(ctx context.Context) []migrationPlan {
|
||||
if m == nil || m.cfg.Catalog == nil || m.cfg.Registry == nil {
|
||||
return nil
|
||||
}
|
||||
all := m.cfg.Registry.All()
|
||||
out := make([]*spider91.Driver, 0, len(all))
|
||||
out := make([]migrationPlan, 0, len(all))
|
||||
for _, d := range all {
|
||||
if d.Kind() != spider91.Kind {
|
||||
if d == nil {
|
||||
continue
|
||||
}
|
||||
if sd, ok := d.(*spider91.Driver); ok {
|
||||
src, ok := d.(Spider91LocalSource)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
row, err := m.cfg.Catalog.GetDrive(ctx, d.ID())
|
||||
if (err != nil || row == nil) && d.Kind() == spider91.Kind {
|
||||
row = &catalog.Drive{ID: d.ID(), Kind: spider91.Kind, RootID: "/"}
|
||||
}
|
||||
if row == nil {
|
||||
continue
|
||||
}
|
||||
switch row.Kind {
|
||||
case scriptcrawler.Kind:
|
||||
targetID := strings.TrimSpace(row.Credentials["upload_drive_id"])
|
||||
if targetID == "" {
|
||||
continue
|
||||
}
|
||||
resolvedID, target, err := m.resolveTargetID(targetID)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] crawler=%s upload target=%q unavailable: %v", row.ID, targetID, err)
|
||||
continue
|
||||
}
|
||||
out = append(out, migrationPlan{
|
||||
source: src,
|
||||
row: row,
|
||||
sourceKinds: crawlerSourceKindsForRow(row),
|
||||
targetDriveID: resolvedID,
|
||||
target: target,
|
||||
uploadDir: scriptCrawlerUploadDir(row.ID),
|
||||
keepLatestN: 0,
|
||||
requireAssetsReady: true,
|
||||
})
|
||||
case spider91.Kind:
|
||||
if m.cfg.GetTargetDriveID == nil {
|
||||
continue
|
||||
}
|
||||
targetID := strings.TrimSpace(m.cfg.GetTargetDriveID())
|
||||
if targetID == "" {
|
||||
continue
|
||||
}
|
||||
resolvedID, target, err := m.resolveTargetID(targetID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, migrationPlan{
|
||||
source: src,
|
||||
row: row,
|
||||
sourceKinds: []string{spider91.Kind},
|
||||
targetDriveID: resolvedID,
|
||||
target: target,
|
||||
uploadDir: spider91UploadDirName,
|
||||
keepLatestN: m.cfg.KeepLatestN,
|
||||
legacyBackfill: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func crawlerSourceKindsForRow(d *catalog.Drive) []string {
|
||||
kinds := []string{scriptcrawler.Kind}
|
||||
if d != nil && strings.EqualFold(strings.TrimSpace(d.Credentials["builtin"]), spider91.Kind) {
|
||||
kinds = append(kinds, spider91.Kind)
|
||||
}
|
||||
return kinds
|
||||
}
|
||||
|
||||
func scriptCrawlerUploadDir(driveID string) string {
|
||||
driveID = sanitizeUploadDirSegment(driveID)
|
||||
if driveID == "" {
|
||||
driveID = "crawler"
|
||||
}
|
||||
return scriptCrawlerUploadRootDirName + "/" + driveID
|
||||
}
|
||||
|
||||
func sanitizeUploadDirSegment(raw string) string {
|
||||
clean := sanitizeTitle(raw)
|
||||
clean = strings.Trim(clean, "/")
|
||||
if clean == "." || clean == ".." {
|
||||
return ""
|
||||
}
|
||||
return clean
|
||||
}
|
||||
|
||||
// spider91Drives 返回当前注册的所有 Spider91 来源本地爬虫 driver。
|
||||
func (m *Migrator) spider91Drives(ctx context.Context) []Spider91LocalSource {
|
||||
all := m.cfg.Registry.All()
|
||||
out := make([]Spider91LocalSource, 0, len(all))
|
||||
for _, d := range all {
|
||||
if !m.isSpider91SourceDrive(ctx, d) {
|
||||
continue
|
||||
}
|
||||
if sd, ok := d.(Spider91LocalSource); ok {
|
||||
out = append(out, sd)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// migrateDrive 对单个 spider91 drive 跑一批迁移;返回成功迁移的条数。
|
||||
//
|
||||
// 策略(与"本地缓存最新 N 个"语义一致):
|
||||
// - 列出 spider91 drive 本地 videos/ 目录所有 mp4 文件,按 mtime 降序排
|
||||
// - 跳过最新 KeepLatestN 个:这些是用户希望保留在本地的最新爬取
|
||||
// - 对剩下的(更旧)逐个处理:
|
||||
// - 还没迁移(drive_id 仍是 src.ID())→ 上传到目标盘 + 改 catalog + 删本地
|
||||
// - 已经迁移过但本地还有残留 → 仅删本地(兜底)
|
||||
//
|
||||
// KeepLatestN < 0 时不保护任何本地文件,全部尝试迁移(旧行为,主要给测试用)。
|
||||
func (m *Migrator) migrateDrive(ctx context.Context, src *spider91.Driver, targetDriveID string, pp uploadTarget) (int, error) {
|
||||
keepN := m.cfg.KeepLatestN
|
||||
func (m *Migrator) isSpider91SourceDrive(ctx context.Context, d drives.Drive) bool {
|
||||
if d == nil {
|
||||
return false
|
||||
}
|
||||
if d.Kind() == spider91.Kind {
|
||||
return true
|
||||
}
|
||||
if d.Kind() != scriptcrawler.Kind || m.cfg.Catalog == nil {
|
||||
return false
|
||||
}
|
||||
row, err := m.cfg.Catalog.GetDrive(ctx, d.ID())
|
||||
if err != nil || row == nil {
|
||||
return false
|
||||
}
|
||||
if row.Kind == spider91.Kind {
|
||||
return true
|
||||
}
|
||||
return row.Kind == scriptcrawler.Kind && strings.EqualFold(strings.TrimSpace(row.Credentials["builtin"]), spider91.Kind)
|
||||
}
|
||||
|
||||
// migrateDrive 对单个本地爬虫 drive 跑一批迁移;返回成功迁移的条数。
|
||||
func (m *Migrator) migrateDrive(ctx context.Context, plan migrationPlan) (int, error) {
|
||||
src := plan.source
|
||||
if src == nil || plan.target == nil || plan.targetDriveID == "" {
|
||||
return 0, nil
|
||||
}
|
||||
keepN := plan.keepLatestN
|
||||
if keepN < 0 {
|
||||
keepN = 0
|
||||
}
|
||||
@@ -479,28 +704,46 @@ func (m *Migrator) migrateDrive(ctx context.Context, src *spider91.Driver, targe
|
||||
files = append(files, localFile{name: e.Name(), modTime: info.ModTime()})
|
||||
}
|
||||
|
||||
// 本地数量没超过 keepN 时不动任何文件 —— 这条是 KeepLatestN 语义的核心
|
||||
if m.cfg.KeepLatestN >= 0 && len(files) <= keepN {
|
||||
if plan.keepLatestN >= 0 && len(files) <= keepN {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// 按 mtime 降序:最新的排前面,保留前 keepN 个
|
||||
sort.Slice(files, func(i, j int) bool { return files[i].modTime.After(files[j].modTime) })
|
||||
|
||||
// 候选 = 跳过最新 keepN 个之外的(更旧的)。KeepLatestN < 0 时 candidates=files。
|
||||
skip := keepN
|
||||
if m.cfg.KeepLatestN < 0 {
|
||||
if plan.keepLatestN < 0 {
|
||||
skip = 0
|
||||
}
|
||||
candidates := files
|
||||
if skip < len(files) {
|
||||
candidates = files[skip:]
|
||||
} else {
|
||||
m.reportUploadProgress(UploadProgress{DriveID: src.ID(), State: "idle"})
|
||||
return 0, nil
|
||||
}
|
||||
totalCandidates := len(candidates)
|
||||
m.reportUploadProgress(UploadProgress{
|
||||
DriveID: src.ID(),
|
||||
State: "uploading",
|
||||
QueueLength: totalCandidates,
|
||||
TotalCount: totalCandidates,
|
||||
})
|
||||
defer m.reportUploadProgress(UploadProgress{DriveID: src.ID(), State: "idle"})
|
||||
|
||||
localVideos, err := m.cfg.Catalog.ListVideosByDriveID(ctx, src.ID(), 100000)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("list local catalog videos: %w", err)
|
||||
}
|
||||
byFileID := make(map[string]*catalog.Video, len(localVideos))
|
||||
for _, v := range localVideos {
|
||||
if v != nil && strings.TrimSpace(v.FileID) != "" {
|
||||
byFileID[v.FileID] = v
|
||||
}
|
||||
}
|
||||
|
||||
migrated := 0
|
||||
for _, f := range candidates {
|
||||
processed := 0
|
||||
for index, f := range candidates {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return migrated, err
|
||||
}
|
||||
@@ -508,21 +751,87 @@ func (m *Migrator) migrateDrive(ctx context.Context, src *spider91.Driver, targe
|
||||
break
|
||||
}
|
||||
|
||||
viewkey := stripExt(f.name)
|
||||
videoID := "spider91-" + src.ID() + "-" + viewkey
|
||||
v, err := m.cfg.Catalog.GetVideo(ctx, videoID)
|
||||
if err != nil || v == nil {
|
||||
// 找不到 catalog 行:保险起见保留本地,让管理员可见
|
||||
v := m.findVideoForLocalFile(ctx, plan, f.name, byFileID)
|
||||
if v == nil {
|
||||
processed++
|
||||
m.reportUploadProgress(UploadProgress{
|
||||
DriveID: src.ID(),
|
||||
State: "uploading",
|
||||
QueueLength: maxInt(totalCandidates-processed, 0),
|
||||
DoneCount: processed,
|
||||
TotalCount: totalCandidates,
|
||||
})
|
||||
continue
|
||||
}
|
||||
m.reportUploadProgress(UploadProgress{
|
||||
DriveID: src.ID(),
|
||||
State: "uploading",
|
||||
CurrentTitle: v.Title,
|
||||
QueueLength: maxInt(totalCandidates-index-1, 0),
|
||||
DoneCount: processed,
|
||||
TotalCount: totalCandidates,
|
||||
})
|
||||
|
||||
if v.DriveID != src.ID() {
|
||||
// catalog 已迁移到别的 drive,但本地还有残留 → 兜底删本地
|
||||
CleanupSpider91Local(src, v.FileID)
|
||||
CleanupSpider91Local(src, f.name)
|
||||
processed++
|
||||
m.reportUploadProgress(UploadProgress{
|
||||
DriveID: src.ID(),
|
||||
State: "uploading",
|
||||
QueueLength: maxInt(totalCandidates-processed, 0),
|
||||
DoneCount: processed,
|
||||
TotalCount: totalCandidates,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
ok, err := m.migrateOne(ctx, v, src, targetDriveID, pp)
|
||||
if targetDuplicate, err := m.cfg.Catalog.FindEquivalentVideoOnDrive(ctx, v, plan.targetDriveID); err != nil {
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
log.Printf("[spider91migrate] %s find target duplicate: %v", v.ID, err)
|
||||
}
|
||||
} else if targetDuplicate != nil {
|
||||
ok, err := m.bindToExistingTarget(ctx, v, targetDuplicate, plan)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] %s: %v", v.ID, err)
|
||||
continue
|
||||
}
|
||||
if ok {
|
||||
migrated++
|
||||
if m.cfg.OnMigrated != nil {
|
||||
m.cfg.OnMigrated(v.ID)
|
||||
}
|
||||
}
|
||||
processed++
|
||||
m.reportUploadProgress(UploadProgress{
|
||||
DriveID: src.ID(),
|
||||
State: "uploading",
|
||||
QueueLength: maxInt(totalCandidates-processed, 0),
|
||||
DoneCount: processed,
|
||||
TotalCount: totalCandidates,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
if plan.requireAssetsReady {
|
||||
ready, err := m.crawlerVideoAssetsReady(ctx, v)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] %s check generated assets: %v", v.ID, err)
|
||||
continue
|
||||
}
|
||||
if !ready {
|
||||
processed++
|
||||
m.reportUploadProgress(UploadProgress{
|
||||
DriveID: src.ID(),
|
||||
State: "uploading",
|
||||
QueueLength: maxInt(totalCandidates-processed, 0),
|
||||
DoneCount: processed,
|
||||
TotalCount: totalCandidates,
|
||||
})
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
ok, err := m.migrateOne(ctx, v, plan)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] %s: %v", v.ID, err)
|
||||
// captcha 错误(4002 / 9)说明 PikPak 当前正拒绝我们;继续在
|
||||
@@ -542,14 +851,64 @@ func (m *Migrator) migrateDrive(ctx context.Context, src *spider91.Driver, targe
|
||||
m.cfg.OnMigrated(v.ID)
|
||||
}
|
||||
}
|
||||
processed++
|
||||
m.reportUploadProgress(UploadProgress{
|
||||
DriveID: src.ID(),
|
||||
State: "uploading",
|
||||
QueueLength: maxInt(totalCandidates-processed, 0),
|
||||
DoneCount: processed,
|
||||
TotalCount: totalCandidates,
|
||||
})
|
||||
}
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
// migrateOne 把单条 spider91 视频上传到目标盘并改写 catalog。
|
||||
func maxInt(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (m *Migrator) findVideoForLocalFile(ctx context.Context, plan migrationPlan, localFile string, byFileID map[string]*catalog.Video) *catalog.Video {
|
||||
if v := byFileID[localFile]; v != nil {
|
||||
return v
|
||||
}
|
||||
sourceID := stripExt(localFile)
|
||||
driveID := ""
|
||||
if plan.source != nil {
|
||||
driveID = plan.source.ID()
|
||||
}
|
||||
for _, kind := range plan.sourceKinds {
|
||||
id := scriptcrawler.BuildVideoIDForKind(kind, driveID, sourceID)
|
||||
v, err := m.cfg.Catalog.GetVideo(ctx, id)
|
||||
if err == nil && v != nil {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Migrator) crawlerVideoAssetsReady(ctx context.Context, v *catalog.Video) (bool, error) {
|
||||
if v == nil {
|
||||
return false, nil
|
||||
}
|
||||
fingerprintReady := strings.EqualFold(strings.TrimSpace(v.FingerprintStatus), "ready") || strings.TrimSpace(v.SampledSHA256) != ""
|
||||
if !fingerprintReady {
|
||||
return false, nil
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(v.PreviewStatus), "ready") {
|
||||
return true, nil
|
||||
}
|
||||
return m.cfg.Catalog.HasReadyEquivalentPreview(ctx, v)
|
||||
}
|
||||
|
||||
// migrateOne 把单条本地爬虫视频上传到目标盘并改写 catalog。
|
||||
// 返回 (true, nil) 表示真的迁了一条;(false, nil) 表示跳过(本地文件已不在等);
|
||||
// (false, err) 表示真出错。
|
||||
func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider91.Driver, targetDriveID string, pp uploadTarget) (bool, error) {
|
||||
func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, plan migrationPlan) (bool, error) {
|
||||
src := plan.source
|
||||
pp := plan.target
|
||||
path, err := src.VideoPath(v.FileID)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("resolve local path: %w", err)
|
||||
@@ -573,20 +932,11 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// 上传到目标盘 rootID 下的固定 "91 Spider" 子目录。若用户把目标盘 rootID
|
||||
// 配成某个自定义目录,这里会在该自定义目录下查找/创建 "91 Spider"。
|
||||
// 上传名走 desiredPikPakName 算出来的方案 B 格式:
|
||||
//
|
||||
// <sanitized title>-<viewkey 后 8 位>.<ext>
|
||||
//
|
||||
// 这样网盘 Web 端列出来的文件名能直接看出是哪个视频,
|
||||
// 又用 viewkey 后 8 位避免同标题撞名。所有目标盘共用同一格式,
|
||||
// 简化前端 / catalog 的认知。
|
||||
parent, err := pp.EnsureDir(ctx, spider91UploadDirName)
|
||||
parent, err := pp.EnsureDir(ctx, plan.uploadDir)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s ensure %q dir: %w", pp.Kind(), spider91UploadDirName, err)
|
||||
return false, fmt.Errorf("%s ensure %q dir: %w", pp.Kind(), plan.uploadDir, err)
|
||||
}
|
||||
uploadName := desiredPikPakName(v.Title, extractViewKey(v.ID), v.Ext)
|
||||
uploadName := desiredPikPakName(v.Title, sourceIDForUploadName(v, plan), v.Ext)
|
||||
res, err := pp.UploadAndReportHash(ctx, parent, uploadName, f, info.Size())
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s upload: %w", pp.Kind(), err)
|
||||
@@ -596,7 +946,7 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
|
||||
}
|
||||
|
||||
// 事务性改写 catalog 行:drive_id / file_id / content_hash
|
||||
if err := m.cfg.Catalog.MigrateVideoToDrive(ctx, v.ID, targetDriveID, res.FileID, res.Hash); err != nil {
|
||||
if err := m.cfg.Catalog.MigrateVideoToDrive(ctx, v.ID, plan.targetDriveID, res.FileID, res.Hash); err != nil {
|
||||
return false, fmt.Errorf("catalog migrate: %w", err)
|
||||
}
|
||||
m.preserveCrawledThumbnail(ctx, src, v)
|
||||
@@ -608,11 +958,60 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
|
||||
// 删除本地 mp4 和源 thumb(公共 /p/thumb 副本已在 preserveCrawledThumbnail 中保留)。
|
||||
CleanupSpider91Local(src, v.FileID)
|
||||
|
||||
log.Printf("[spider91migrate] %s migrated to drive=%s(kind=%s) file=%s name=%q", v.ID, targetDriveID, pp.Kind(), res.FileID, uploadName)
|
||||
log.Printf("[spider91migrate] %s migrated to drive=%s(kind=%s) file=%s name=%q", v.ID, plan.targetDriveID, pp.Kind(), res.FileID, uploadName)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *Migrator) preserveCrawledThumbnail(ctx context.Context, src *spider91.Driver, v *catalog.Video) {
|
||||
func (m *Migrator) bindToExistingTarget(ctx context.Context, v, target *catalog.Video, plan migrationPlan) (bool, error) {
|
||||
if v == nil || target == nil || plan.source == nil {
|
||||
return false, nil
|
||||
}
|
||||
if plan.targetDriveID == "" || target.FileID == "" {
|
||||
return false, nil
|
||||
}
|
||||
if err := m.cfg.Catalog.MigrateVideoToDrive(ctx, v.ID, plan.targetDriveID, target.FileID, firstNonEmpty(target.ContentHash, v.ContentHash)); err != nil {
|
||||
return false, fmt.Errorf("catalog bind existing target: %w", err)
|
||||
}
|
||||
if target.FileName != "" {
|
||||
if err := m.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{FileName: target.FileName}); err != nil {
|
||||
log.Printf("[spider91migrate] %s update file_name after duplicate bind: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
m.preserveCrawledThumbnail(ctx, plan.source, v)
|
||||
CleanupSpider91Local(plan.source, v.FileID)
|
||||
log.Printf("[spider91migrate] %s bound to existing drive=%s(kind=%s) file=%s duplicate=%s", v.ID, plan.targetDriveID, plan.target.Kind(), target.FileID, target.ID)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func sourceIDForUploadName(v *catalog.Video, plan migrationPlan) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
if plan.legacyBackfill {
|
||||
return extractViewKey(v.ID)
|
||||
}
|
||||
for _, kind := range plan.sourceKinds {
|
||||
prefix := kind + "-" + plan.source.ID() + "-"
|
||||
if strings.HasPrefix(v.ID, prefix) {
|
||||
return strings.TrimPrefix(v.ID, prefix)
|
||||
}
|
||||
}
|
||||
if v.FileID != "" {
|
||||
return stripExt(v.FileID)
|
||||
}
|
||||
return extractViewKey(v.ID)
|
||||
}
|
||||
|
||||
func (m *Migrator) preserveCrawledThumbnail(ctx context.Context, src Spider91LocalSource, v *catalog.Video) {
|
||||
if m == nil || m.cfg.Catalog == nil || src == nil || v == nil || v.ID == "" || v.FileID == "" {
|
||||
return
|
||||
}
|
||||
@@ -651,7 +1050,7 @@ func (m *Migrator) preserveCrawledThumbnail(ctx context.Context, src *spider91.D
|
||||
v.ThumbnailURL = "/p/thumb/" + v.ID
|
||||
}
|
||||
|
||||
func findSpider91ThumbPath(src *spider91.Driver, fileID string) (string, bool) {
|
||||
func findSpider91ThumbPath(src Spider91LocalSource, fileID string) (string, bool) {
|
||||
thumbBase := stripExt(fileID)
|
||||
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
|
||||
thumbPath, err := src.ThumbPath(thumbBase + ext)
|
||||
@@ -697,7 +1096,7 @@ func copyFileAtomic(src, dst string) error {
|
||||
// 我们不知道具体是 .jpg 还是别的,逐个尝试常见后缀)。
|
||||
//
|
||||
// 暴露成包级函数方便 cleanup 模块复用(任务 6)。
|
||||
func CleanupSpider91Local(src *spider91.Driver, fileID string) {
|
||||
func CleanupSpider91Local(src Spider91LocalSource, fileID string) {
|
||||
videoPath, err := src.VideoPath(fileID)
|
||||
if err == nil {
|
||||
if err := os.Remove(videoPath); err != nil && !os.IsNotExist(err) {
|
||||
@@ -734,7 +1133,11 @@ func stripExt(name string) string {
|
||||
// 找到孤儿。
|
||||
//
|
||||
// 返回实际删除的文件个数。
|
||||
func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driver) (int, error) {
|
||||
func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, plan migrationPlan) (int, error) {
|
||||
src := plan.source
|
||||
if src == nil {
|
||||
return 0, nil
|
||||
}
|
||||
entries, err := os.ReadDir(src.VideosDir())
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
@@ -751,18 +1154,13 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driv
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
viewkey := stripExt(e.Name())
|
||||
videoID := "spider91-" + src.ID() + "-" + viewkey
|
||||
v, err := m.cfg.Catalog.GetVideo(ctx, videoID)
|
||||
if err != nil || v == nil {
|
||||
// 找不到 catalog 行:保险起见保留,等管理员处理
|
||||
v := m.findVideoForLocalFile(ctx, plan, e.Name(), nil)
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if v.DriveID == src.ID() {
|
||||
// 还没迁移,归 migrateDrive 管,不在这里动
|
||||
continue
|
||||
}
|
||||
// 已迁移到别的 drive 但本地还有 → 删
|
||||
path, perr := src.VideoPath(e.Name())
|
||||
if perr != nil {
|
||||
continue
|
||||
@@ -785,7 +1183,7 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driv
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// backfillFileNames 扫描目标 drive(PikPak、115、123 或 OneDrive)下所有 spider91-* 起始 ID 的视频,
|
||||
// backfillFileNames 扫描目标 drive(PikPak、115、123、OneDrive、Google Drive 或联通网盘)下所有 spider91-* 起始 ID 的视频,
|
||||
// 对文件名不是 desiredPikPakName(...) 期望格式的,调 target.Rename 修正,
|
||||
// 并把 catalog.file_name 同步到新名字。
|
||||
//
|
||||
|
||||
@@ -14,9 +14,12 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/googledrive"
|
||||
"github.com/video-site/backend/internal/drives/p123"
|
||||
"github.com/video-site/backend/internal/drives/pikpak"
|
||||
"github.com/video-site/backend/internal/drives/scriptcrawler"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/drives/wopan"
|
||||
)
|
||||
|
||||
// fakeRegistry 是 Registry 接口的最小实现。
|
||||
@@ -342,6 +345,81 @@ func writeSpider91Video(t *testing.T, cat *catalog.Catalog, d *spider91.Driver,
|
||||
return id
|
||||
}
|
||||
|
||||
func setupScriptCrawler(t *testing.T, id string) *scriptcrawler.Driver {
|
||||
t.Helper()
|
||||
d := scriptcrawler.New(scriptcrawler.Config{ID: id, RootDir: t.TempDir()})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("scriptcrawler init: %v", err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func seedScriptCrawlerDrive(t *testing.T, cat *catalog.Catalog, d *scriptcrawler.Driver, uploadDriveID string) {
|
||||
t.Helper()
|
||||
if err := cat.UpsertDrive(context.Background(), &catalog.Drive{
|
||||
ID: d.ID(),
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Script Crawler",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"script_path": "/tmp/crawler.py",
|
||||
"upload_drive_id": uploadDriveID,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed scriptcrawler drive: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeScriptCrawlerVideo(t *testing.T, cat *catalog.Catalog, d *scriptcrawler.Driver, sourceID, ext string, content []byte, readyAssets bool) string {
|
||||
t.Helper()
|
||||
fileID := sourceID + ext
|
||||
path, err := d.VideoPath(fileID)
|
||||
if err != nil {
|
||||
t.Fatalf("video path: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, content, 0o644); err != nil {
|
||||
t.Fatalf("write video: %v", err)
|
||||
}
|
||||
thumbPath, err := d.ThumbPath(sourceID + ".jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("thumb path: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(thumbPath, []byte("thumb"), 0o644); err != nil {
|
||||
t.Fatalf("write thumb: %v", err)
|
||||
}
|
||||
now := time.Now()
|
||||
id := scriptcrawler.BuildVideoID(d.ID(), sourceID)
|
||||
previewStatus := "pending"
|
||||
if readyAssets {
|
||||
previewStatus = "ready"
|
||||
}
|
||||
v := &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: d.ID(),
|
||||
FileID: fileID,
|
||||
FileName: fileID,
|
||||
Title: "Crawler " + sourceID,
|
||||
Author: "tester",
|
||||
Ext: strings.TrimPrefix(ext, "."),
|
||||
Quality: "HD",
|
||||
Size: int64(len(content)),
|
||||
ThumbnailURL: "/p/thumb/" + id,
|
||||
PreviewStatus: previewStatus,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if err := cat.UpsertVideo(context.Background(), v); err != nil {
|
||||
t.Fatalf("upsert scriptcrawler video: %v", err)
|
||||
}
|
||||
if readyAssets {
|
||||
if err := cat.UpdateVideoFingerprint(context.Background(), id, "sampled-"+sourceID, "ready", ""); err != nil {
|
||||
t.Fatalf("mark fingerprint ready: %v", err)
|
||||
}
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src, _ := setupSpider91(t)
|
||||
@@ -417,6 +495,174 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceMigratesReadyScriptCrawlerVideoToConfiguredUploadDrive(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src := setupScriptCrawler(t, "crawler-alpha")
|
||||
pp := newFakePikPak("pikpak-target", "pikpak-root-id")
|
||||
seedScriptCrawlerDrive(t, cat, src, pp.ID())
|
||||
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(pp)
|
||||
|
||||
id := writeScriptCrawlerVideo(t, cat, src, "source-with-dash-001", ".mp4", []byte("script video bytes"), true)
|
||||
commonThumbDir := t.TempDir()
|
||||
|
||||
m := New(Config{
|
||||
Catalog: cat,
|
||||
Registry: reg,
|
||||
CommonThumbDir: commonThumbDir,
|
||||
})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
if pp.uploadCalls != 1 {
|
||||
t.Fatalf("upload calls = %d, want 1", pp.uploadCalls)
|
||||
}
|
||||
wantDir := "Script Crawlers/crawler-alpha"
|
||||
if len(pp.ensureCalls) != 1 || pp.ensureCalls[0] != wantDir {
|
||||
t.Fatalf("ensure calls = %#v, want %q", pp.ensureCalls, wantDir)
|
||||
}
|
||||
wantName := desiredPikPakName("Crawler source-with-dash-001", "source-with-dash-001", "mp4")
|
||||
if gotParent := pp.gotParents[wantName]; gotParent != "pikpak-root-id/"+wantDir {
|
||||
t.Fatalf("upload parent = %q, want root/%s", gotParent, wantDir)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("get migrated video: %v", err)
|
||||
}
|
||||
if got.DriveID != pp.ID() {
|
||||
t.Fatalf("drive_id = %q, want %q", got.DriveID, pp.ID())
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want remote upload id", got.FileID)
|
||||
}
|
||||
if got.FileName != wantName {
|
||||
t.Fatalf("file_name = %q, want %q", got.FileName, wantName)
|
||||
}
|
||||
if got.PreviewStatus != "ready" || got.FingerprintStatus != "ready" || got.SampledSHA256 == "" {
|
||||
t.Fatalf("generated assets not preserved after migration: preview=%q fingerprint=%q sampled=%q", got.PreviewStatus, got.FingerprintStatus, got.SampledSHA256)
|
||||
}
|
||||
videoPath, _ := src.VideoPath("source-with-dash-001.mp4")
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local scriptcrawler video still exists or stat error %v", err)
|
||||
}
|
||||
thumbPath, _ := src.ThumbPath("source-with-dash-001.jpg")
|
||||
if _, err := os.Stat(thumbPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local scriptcrawler thumb still exists or stat error %v", err)
|
||||
}
|
||||
commonThumbPath := filepath.Join(commonThumbDir, id+".jpg")
|
||||
if data, err := os.ReadFile(commonThumbPath); err != nil || string(data) != "thumb" {
|
||||
t.Fatalf("common thumb = %q, %v; want copied crawled thumb", string(data), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceSkipsScriptCrawlerVideoUntilPreviewAndFingerprintReady(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src := setupScriptCrawler(t, "crawler-beta")
|
||||
pp := newFakePikPak("pikpak-target", "pikpak-root-id")
|
||||
seedScriptCrawlerDrive(t, cat, src, pp.ID())
|
||||
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(pp)
|
||||
|
||||
id := writeScriptCrawlerVideo(t, cat, src, "pending-assets", ".mp4", []byte("script video bytes"), false)
|
||||
m := New(Config{Catalog: cat, Registry: reg})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
if pp.uploadCalls != 0 {
|
||||
t.Fatalf("upload calls = %d, want 0 while generated assets are pending", pp.uploadCalls)
|
||||
}
|
||||
got, err := cat.GetVideo(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DriveID != src.ID() {
|
||||
t.Fatalf("drive_id = %q, want local crawler drive %q", got.DriveID, src.ID())
|
||||
}
|
||||
videoPath, _ := src.VideoPath("pending-assets.mp4")
|
||||
if _, err := os.Stat(videoPath); err != nil {
|
||||
t.Fatalf("local video should remain while assets pending: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceBindsScriptCrawlerDuplicateToExistingTargetWithoutUpload(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src := setupScriptCrawler(t, "crawler-duplicate")
|
||||
pp := newFakePikPak("pikpak-target", "pikpak-root-id")
|
||||
seedScriptCrawlerDrive(t, cat, src, pp.ID())
|
||||
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(pp)
|
||||
|
||||
content := []byte("duplicate script video bytes")
|
||||
id := writeScriptCrawlerVideo(t, cat, src, "duplicate-source", ".mp4", content, false)
|
||||
sampled := "same-sampled-fingerprint"
|
||||
if err := cat.UpdateVideoFingerprint(context.Background(), id, sampled, "ready", ""); err != nil {
|
||||
t.Fatalf("mark source fingerprint ready: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
target := &catalog.Video{
|
||||
ID: "pikpak-existing-duplicate",
|
||||
DriveID: pp.ID(),
|
||||
FileID: "existing-target-file",
|
||||
FileName: "existing-target-name.mp4",
|
||||
ContentHash: "existing-content-hash",
|
||||
Title: "Existing duplicate",
|
||||
Ext: "mp4",
|
||||
Size: int64(len(content)),
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now.Add(-time.Hour),
|
||||
CreatedAt: now.Add(-time.Hour),
|
||||
UpdatedAt: now.Add(-time.Hour),
|
||||
}
|
||||
if err := cat.UpsertVideo(context.Background(), target); err != nil {
|
||||
t.Fatalf("upsert existing target: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(context.Background(), target.ID, sampled, "ready", ""); err != nil {
|
||||
t.Fatalf("mark target fingerprint ready: %v", err)
|
||||
}
|
||||
|
||||
commonThumbDir := t.TempDir()
|
||||
m := New(Config{Catalog: cat, Registry: reg, CommonThumbDir: commonThumbDir})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
if pp.uploadCalls != 0 {
|
||||
t.Fatalf("upload calls = %d, want 0 when equivalent target file already exists", pp.uploadCalls)
|
||||
}
|
||||
got, err := cat.GetVideo(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("get bound video: %v", err)
|
||||
}
|
||||
if got.DriveID != pp.ID() {
|
||||
t.Fatalf("drive_id = %q, want %q", got.DriveID, pp.ID())
|
||||
}
|
||||
if got.FileID != target.FileID {
|
||||
t.Fatalf("file_id = %q, want existing target file %q", got.FileID, target.FileID)
|
||||
}
|
||||
if got.FileName != target.FileName {
|
||||
t.Fatalf("file_name = %q, want existing target name %q", got.FileName, target.FileName)
|
||||
}
|
||||
if got.ContentHash != target.ContentHash {
|
||||
t.Fatalf("content_hash = %q, want %q", got.ContentHash, target.ContentHash)
|
||||
}
|
||||
videoPath, _ := src.VideoPath("duplicate-source.mp4")
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local duplicate video still exists or stat error %v", err)
|
||||
}
|
||||
thumbPath, _ := src.ThumbPath("duplicate-source.jpg")
|
||||
if _, err := os.Stat(thumbPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local duplicate thumb still exists or stat error %v", err)
|
||||
}
|
||||
commonThumbPath := filepath.Join(commonThumbDir, id+".jpg")
|
||||
if data, err := os.ReadFile(commonThumbPath); err != nil || string(data) != "thumb" {
|
||||
t.Fatalf("common thumb = %q, %v; want copied crawled thumb", string(data), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceSkipsWhenLocalFileMissing(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src, _ := setupSpider91(t)
|
||||
@@ -576,7 +822,10 @@ func TestCleanupRemovesAllAlreadyMigratedOrphans(t *testing.T) {
|
||||
GetTargetDriveID: func() string { return pp.ID() },
|
||||
})
|
||||
|
||||
deleted, err := m.cleanupOldLocalVideos(context.Background(), src)
|
||||
deleted, err := m.cleanupOldLocalVideos(context.Background(), migrationPlan{
|
||||
source: src,
|
||||
sourceKinds: []string{spider91.Kind},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("cleanup: %v", err)
|
||||
}
|
||||
@@ -598,6 +847,95 @@ func TestCleanupRemovesAllAlreadyMigratedOrphans(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceMigratesBuiltInSpider91ScriptCrawlerSource(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat := setupCatalog(t)
|
||||
src := scriptcrawler.New(scriptcrawler.Config{ID: "spider-script", RootDir: t.TempDir()})
|
||||
if err := src.Init(ctx); err != nil {
|
||||
t.Fatalf("scriptcrawler init: %v", err)
|
||||
}
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: src.ID(),
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Built-in Spider91",
|
||||
Credentials: map[string]string{
|
||||
"builtin": "spider91",
|
||||
"script_path": "/tmp/spider91.py",
|
||||
"upload_drive_id": "pikpak-target",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert source drive: %v", err)
|
||||
}
|
||||
pp := newFakePikPak("pikpak-target", "pikpak-root-id")
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(pp)
|
||||
|
||||
fileID := "vk-script.mp4"
|
||||
videoPath, err := src.VideoPath(fileID)
|
||||
if err != nil {
|
||||
t.Fatalf("video path: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(videoPath, []byte("scriptcrawler spider91 video"), 0o644); err != nil {
|
||||
t.Fatalf("write video: %v", err)
|
||||
}
|
||||
thumbPath, err := src.ThumbPath("vk-script.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("thumb path: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(thumbPath, []byte("thumb"), 0o644); err != nil {
|
||||
t.Fatalf("write thumb: %v", err)
|
||||
}
|
||||
now := time.Now()
|
||||
id := "spider91-" + src.ID() + "-vk-script"
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: src.ID(),
|
||||
FileID: fileID,
|
||||
FileName: fileID,
|
||||
Title: "Scriptcrawler Spider91",
|
||||
Author: "91porn",
|
||||
Ext: "mp4",
|
||||
Quality: "HD",
|
||||
Size: int64(len("scriptcrawler spider91 video")),
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert video: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, id, "sampled-vk-script", "ready", ""); err != nil {
|
||||
t.Fatalf("mark fingerprint ready: %v", err)
|
||||
}
|
||||
|
||||
m := New(Config{
|
||||
Catalog: cat,
|
||||
Registry: reg,
|
||||
GetTargetDriveID: func() string { return pp.ID() },
|
||||
KeepLatestN: -1,
|
||||
CommonThumbDir: t.TempDir(),
|
||||
})
|
||||
m.runOnce(ctx)
|
||||
|
||||
if pp.uploadCalls != 1 {
|
||||
t.Fatalf("upload calls = %d, want 1", pp.uploadCalls)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("get migrated video: %v", err)
|
||||
}
|
||||
if got.DriveID != pp.ID() {
|
||||
t.Fatalf("drive_id = %q, want %q", got.DriveID, pp.ID())
|
||||
}
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local video stat err = %v, want not exist", err)
|
||||
}
|
||||
if _, err := os.Stat(thumbPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local thumb stat err = %v, want not exist", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunOnceKeepsAllLocalWhenWithinKeepWindow 验证:本地文件数 ≤ KeepLatestN 时
|
||||
// 一律不上传,全部留作"最新 N"缓存。这是用户的核心需求:刚爬下来的 15 个不要立即被传走。
|
||||
func TestRunOnceKeepsAllLocalWhenWithinKeepWindow(t *testing.T) {
|
||||
@@ -1095,7 +1433,38 @@ func TestAdaptUploadTargetSupportsP123Driver(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115、123 也不是 OneDrive 时,
|
||||
func TestAdaptUploadTargetSupportsGoogleDriveDriver(t *testing.T) {
|
||||
d := googledrive.New(googledrive.Config{
|
||||
ID: "google-target",
|
||||
RootID: "root-google",
|
||||
RefreshToken: "refresh-token",
|
||||
})
|
||||
target, err := adaptUploadTarget(d)
|
||||
if err != nil {
|
||||
t.Fatalf("adaptUploadTarget() error = %v", err)
|
||||
}
|
||||
if target.ID() != "google-target" || target.Kind() != "googledrive" || target.RootID() != "root-google" {
|
||||
t.Fatalf("target id/kind/root = %q/%q/%q, want google-target/googledrive/root-google", target.ID(), target.Kind(), target.RootID())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdaptUploadTargetSupportsWopanDriver(t *testing.T) {
|
||||
d := wopan.New(wopan.Config{
|
||||
ID: "wopan-target",
|
||||
RootID: "root-wopan",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
})
|
||||
target, err := adaptUploadTarget(d)
|
||||
if err != nil {
|
||||
t.Fatalf("adaptUploadTarget() error = %v", err)
|
||||
}
|
||||
if target.ID() != "wopan-target" || target.Kind() != "wopan" || target.RootID() != "root-wopan" {
|
||||
t.Fatalf("target id/kind/root = %q/%q/%q, want wopan-target/wopan/root-wopan", target.ID(), target.Kind(), target.RootID())
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115、123、OneDrive、Google Drive 也不是联通网盘时,
|
||||
// resolveTarget 拒绝并返回 error,让 runOnce 静默跳过(不会做破坏性变更)。
|
||||
func TestResolveTargetRejectsUnsupportedKind(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
// Package transcode 实现"浏览器兼容性转码":把网盘/本地存储中浏览器
|
||||
// <video> 播不动的视频(AVI/WMV/FLV、MPEG-4 Part 2、RMVB 等)转成
|
||||
// H.264 + AAC 的 MP4,并把产物上传回同一存储,播放源切到产物文件。
|
||||
//
|
||||
// 与封面/预览生成不同,转码不会自动运行——只能由管理员在网盘管理页
|
||||
// 手动开启,也可以随时手动停止。
|
||||
package transcode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MediaInfo 是 ffprobe 探测出来的、做兼容性判定所需的最小信息。
|
||||
type MediaInfo struct {
|
||||
// FormatName 是 ffprobe 的 format_name,逗号分隔的 demuxer 别名,
|
||||
// 例如 "mov,mp4,m4a,3gp,3g2,mj2" / "avi" / "matroska,webm"。
|
||||
FormatName string
|
||||
VideoCodecs []string
|
||||
AudioCodecs []string
|
||||
}
|
||||
|
||||
// browserCompatibleVideoCodecs 是主流浏览器 <video> 普遍可解码的视频编码。
|
||||
// HEVC/H.265 只有部分平台支持,保守起见不算兼容。
|
||||
var browserCompatibleVideoCodecs = map[string]bool{
|
||||
"h264": true,
|
||||
"vp8": true,
|
||||
"vp9": true,
|
||||
"av1": true,
|
||||
}
|
||||
|
||||
// browserCompatibleAudioCodecs 是主流浏览器普遍可解码的音频编码。
|
||||
var browserCompatibleAudioCodecs = map[string]bool{
|
||||
"aac": true,
|
||||
"mp3": true,
|
||||
"opus": true,
|
||||
"vorbis": true,
|
||||
"flac": true,
|
||||
}
|
||||
|
||||
// NeedsTranscode 判断这个文件是否需要转码才能在浏览器里播放。
|
||||
// ext 是 catalog 里记录的扩展名(小写、不带点),用来区分 mkv 和 webm
|
||||
// (两者的 format_name 都是 "matroska,webm")。
|
||||
func NeedsTranscode(info MediaInfo, ext string) bool {
|
||||
if !containerCompatible(info.FormatName, ext) {
|
||||
return true
|
||||
}
|
||||
for _, codec := range info.VideoCodecs {
|
||||
if !browserCompatibleVideoCodecs[strings.ToLower(codec)] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, codec := range info.AudioCodecs {
|
||||
if !browserCompatibleAudioCodecs[strings.ToLower(codec)] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containerCompatible(formatName, ext string) bool {
|
||||
format := strings.ToLower(formatName)
|
||||
for _, name := range strings.Split(format, ",") {
|
||||
if name == "mp4" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// matroska,webm:只有真 .webm 信任为浏览器可播容器;.mkv 保守转码。
|
||||
if strings.Contains(format, "webm") && strings.EqualFold(ext, "webm") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ProbeFile 用 ffprobe 探测本地文件的容器与音视频编码。
|
||||
func ProbeFile(ctx context.Context, ffprobePath, path string) (MediaInfo, error) {
|
||||
ctx2, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx2, ffprobePath,
|
||||
"-v", "error",
|
||||
"-show_entries", "format=format_name",
|
||||
"-show_entries", "stream=codec_type,codec_name",
|
||||
"-of", "json",
|
||||
path,
|
||||
)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return MediaInfo{}, fmt.Errorf("transcode: ffprobe: %w", err)
|
||||
}
|
||||
var parsed struct {
|
||||
Format struct {
|
||||
FormatName string `json:"format_name"`
|
||||
} `json:"format"`
|
||||
Streams []struct {
|
||||
CodecType string `json:"codec_type"`
|
||||
CodecName string `json:"codec_name"`
|
||||
} `json:"streams"`
|
||||
}
|
||||
if err := json.Unmarshal(out, &parsed); err != nil {
|
||||
return MediaInfo{}, fmt.Errorf("transcode: parse ffprobe output: %w", err)
|
||||
}
|
||||
info := MediaInfo{FormatName: parsed.Format.FormatName}
|
||||
for _, s := range parsed.Streams {
|
||||
switch s.CodecType {
|
||||
case "video":
|
||||
info.VideoCodecs = append(info.VideoCodecs, s.CodecName)
|
||||
case "audio":
|
||||
info.AudioCodecs = append(info.AudioCodecs, s.CodecName)
|
||||
}
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// buildFFmpegArgs 按探测结果生成转码参数:
|
||||
// - 编码本就兼容、只是容器不行(如 AVI 里装 H.264)→ 流拷贝 remux,零质量损失;
|
||||
// - 否则视频转 H.264(裁到偶数尺寸 + yuv420p 保证兼容性)、音频转 AAC。
|
||||
//
|
||||
// 两种情况都加 +faststart 把 moov 提前,便于边下边播。
|
||||
func buildFFmpegArgs(info MediaInfo, inPath, outPath string) []string {
|
||||
args := []string{"-y", "-i", inPath}
|
||||
videoOK := true
|
||||
for _, codec := range info.VideoCodecs {
|
||||
if !browserCompatibleVideoCodecs[strings.ToLower(codec)] {
|
||||
videoOK = false
|
||||
break
|
||||
}
|
||||
}
|
||||
audioOK := true
|
||||
for _, codec := range info.AudioCodecs {
|
||||
if !browserCompatibleAudioCodecs[strings.ToLower(codec)] {
|
||||
audioOK = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if videoOK {
|
||||
args = append(args, "-c:v", "copy")
|
||||
} else {
|
||||
args = append(args,
|
||||
"-c:v", "libx264",
|
||||
"-preset", "veryfast",
|
||||
"-crf", "23",
|
||||
"-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2",
|
||||
"-pix_fmt", "yuv420p",
|
||||
)
|
||||
}
|
||||
if len(info.AudioCodecs) == 0 {
|
||||
args = append(args, "-an")
|
||||
} else if audioOK {
|
||||
args = append(args, "-c:a", "copy")
|
||||
} else {
|
||||
args = append(args, "-c:a", "aac", "-b:a", "128k")
|
||||
}
|
||||
args = append(args, "-movflags", "+faststart", "-f", "mp4", outPath)
|
||||
return args
|
||||
}
|
||||
|
||||
// TranscodeFile 把本地输入文件转成浏览器可播的 MP4 写到 outPath。
|
||||
func TranscodeFile(ctx context.Context, ffmpegPath string, info MediaInfo, inPath, outPath string) error {
|
||||
args := buildFFmpegArgs(info, inPath, outPath)
|
||||
cmd := exec.CommandContext(ctx, ffmpegPath, args...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("transcode: ffmpeg: %w: %s", err, tailOf(string(out), 400))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func tailOf(s string, n int) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[len(s)-n:]
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package transcode
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
)
|
||||
|
||||
func TestNeedsTranscode(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
info MediaInfo
|
||||
ext string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "h264 aac mp4 is compatible",
|
||||
info: MediaInfo{FormatName: "mov,mp4,m4a,3gp,3g2,mj2", VideoCodecs: []string{"h264"}, AudioCodecs: []string{"aac"}},
|
||||
ext: "mp4",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "mpeg4 in avi needs transcode",
|
||||
info: MediaInfo{FormatName: "avi", VideoCodecs: []string{"mpeg4"}, AudioCodecs: []string{"mp3"}},
|
||||
ext: "avi",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "h264 in avi needs remux",
|
||||
info: MediaInfo{FormatName: "avi", VideoCodecs: []string{"h264"}, AudioCodecs: []string{"aac"}},
|
||||
ext: "avi",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "hevc in mp4 needs transcode",
|
||||
info: MediaInfo{FormatName: "mov,mp4,m4a,3gp,3g2,mj2", VideoCodecs: []string{"hevc"}, AudioCodecs: []string{"aac"}},
|
||||
ext: "mp4",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "vp9 opus webm is compatible",
|
||||
info: MediaInfo{FormatName: "matroska,webm", VideoCodecs: []string{"vp9"}, AudioCodecs: []string{"opus"}},
|
||||
ext: "webm",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "h264 in mkv is conservative transcode",
|
||||
info: MediaInfo{FormatName: "matroska,webm", VideoCodecs: []string{"h264"}, AudioCodecs: []string{"aac"}},
|
||||
ext: "mkv",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "pcm audio in mov needs transcode",
|
||||
info: MediaInfo{FormatName: "mov,mp4,m4a,3gp,3g2,mj2", VideoCodecs: []string{"h264"}, AudioCodecs: []string{"pcm_s16le"}},
|
||||
ext: "mov",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "video only h264 mp4 is compatible",
|
||||
info: MediaInfo{FormatName: "mov,mp4,m4a,3gp,3g2,mj2", VideoCodecs: []string{"h264"}},
|
||||
ext: "mp4",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := NeedsTranscode(tc.info, tc.ext); got != tc.want {
|
||||
t.Fatalf("NeedsTranscode(%+v, %q) = %v, want %v", tc.info, tc.ext, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFFmpegArgsRemuxWhenCodecsCompatible(t *testing.T) {
|
||||
// AVI 里装 H.264+AAC:只需要换容器,应该走流拷贝
|
||||
info := MediaInfo{FormatName: "avi", VideoCodecs: []string{"h264"}, AudioCodecs: []string{"aac"}}
|
||||
args := strings.Join(buildFFmpegArgs(info, "in.avi", "out.mp4"), " ")
|
||||
if !strings.Contains(args, "-c:v copy") {
|
||||
t.Fatalf("expected video stream copy, got: %s", args)
|
||||
}
|
||||
if !strings.Contains(args, "-c:a copy") {
|
||||
t.Fatalf("expected audio stream copy, got: %s", args)
|
||||
}
|
||||
if !strings.Contains(args, "+faststart") {
|
||||
t.Fatalf("expected faststart flag, got: %s", args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFFmpegArgsTranscodesIncompatibleCodecs(t *testing.T) {
|
||||
info := MediaInfo{FormatName: "avi", VideoCodecs: []string{"mpeg4"}, AudioCodecs: []string{"wmav2"}}
|
||||
args := strings.Join(buildFFmpegArgs(info, "in.avi", "out.mp4"), " ")
|
||||
if !strings.Contains(args, "-c:v libx264") {
|
||||
t.Fatalf("expected libx264 video encode, got: %s", args)
|
||||
}
|
||||
if !strings.Contains(args, "-c:a aac") {
|
||||
t.Fatalf("expected aac audio encode, got: %s", args)
|
||||
}
|
||||
if !strings.Contains(args, "yuv420p") {
|
||||
t.Fatalf("expected yuv420p pixel format, got: %s", args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFFmpegArgsDropsAudioWhenNoAudioStream(t *testing.T) {
|
||||
info := MediaInfo{FormatName: "avi", VideoCodecs: []string{"mpeg4"}}
|
||||
args := strings.Join(buildFFmpegArgs(info, "in.avi", "out.mp4"), " ")
|
||||
if !strings.Contains(args, "-an") {
|
||||
t.Fatalf("expected -an for video without audio, got: %s", args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscodedName(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
fileName, title, id, want string
|
||||
}{
|
||||
{"www.98T.la@167.avi", "www.98T.la@167", "p115-1", "www.98T.la@167.mp4"},
|
||||
{"", "标题", "p115-2", "标题.mp4"},
|
||||
{"a/b\\c.wmv", "", "p115-3", "a_b_c.mp4"},
|
||||
} {
|
||||
v := &catalog.Video{FileName: tc.fileName, Title: tc.title, ID: tc.id}
|
||||
if got := transcodedName(v); got != tc.want {
|
||||
t.Fatalf("transcodedName(%q,%q,%q) = %q, want %q", tc.fileName, tc.title, tc.id, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
package transcode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
// DefaultTargetDirName 是转码产物在网盘上的存放目录(相对根目录)。
|
||||
// worker 第一次上传前会 EnsureDir 并把该目录加进 drive 的扫描跳过列表,
|
||||
// 避免 scanner 把转码产物当成新视频重复入库。
|
||||
const DefaultTargetDirName = "91转码"
|
||||
|
||||
type Config struct {
|
||||
FFmpegPath string
|
||||
FFprobePath string
|
||||
// WorkDir 是下载原始文件 / 写转码产物的本地临时目录。
|
||||
WorkDir string
|
||||
// TargetDirName 为空时用 DefaultTargetDirName。
|
||||
TargetDirName string
|
||||
}
|
||||
|
||||
// TaskStatus 与 preview/fingerprint worker 的状态结构对齐,供 admin 展示。
|
||||
type TaskStatus struct {
|
||||
State string
|
||||
CurrentTitle string
|
||||
QueueLength int
|
||||
DoneCount int
|
||||
TotalCount int
|
||||
}
|
||||
|
||||
// Worker 串行处理一个 drive 的转码任务。生命周期与一次"开始转码"对应:
|
||||
// Run 处理完整个候选列表(或 ctx 被取消)后即结束,不常驻。
|
||||
type Worker struct {
|
||||
cfg Config
|
||||
cat *catalog.Catalog
|
||||
drv drives.Drive
|
||||
hc *http.Client
|
||||
|
||||
mu sync.Mutex
|
||||
state string
|
||||
currentTitle string
|
||||
done int
|
||||
total int
|
||||
|
||||
targetDirOnce sync.Once
|
||||
targetDirID string
|
||||
targetDirErr error
|
||||
}
|
||||
|
||||
func NewWorker(cfg Config, cat *catalog.Catalog, drv drives.Drive) *Worker {
|
||||
if cfg.FFmpegPath == "" {
|
||||
cfg.FFmpegPath = "ffmpeg"
|
||||
}
|
||||
if cfg.FFprobePath == "" {
|
||||
cfg.FFprobePath = "ffprobe"
|
||||
}
|
||||
if cfg.TargetDirName == "" {
|
||||
cfg.TargetDirName = DefaultTargetDirName
|
||||
}
|
||||
if cfg.WorkDir == "" {
|
||||
cfg.WorkDir = os.TempDir()
|
||||
}
|
||||
return &Worker{
|
||||
cfg: cfg,
|
||||
cat: cat,
|
||||
drv: drv,
|
||||
hc: &http.Client{Timeout: 0},
|
||||
state: "idle",
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) Status() TaskStatus {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
queueLen := w.total - w.done
|
||||
if w.state == "generating" && queueLen > 0 {
|
||||
// 正在处理的那条不算"排队中"
|
||||
queueLen--
|
||||
}
|
||||
if queueLen < 0 {
|
||||
queueLen = 0
|
||||
}
|
||||
return TaskStatus{
|
||||
State: w.state,
|
||||
CurrentTitle: w.currentTitle,
|
||||
QueueLength: queueLen,
|
||||
DoneCount: w.done,
|
||||
TotalCount: w.total,
|
||||
}
|
||||
}
|
||||
|
||||
// Run 串行转码整个候选列表。ctx 取消时停在当前条目边界(正在跑的 ffmpeg
|
||||
// 会被 CommandContext 杀掉),未处理的候选保持原状态,下次开始时继续。
|
||||
func (w *Worker) Run(ctx context.Context, videos []*catalog.Video) {
|
||||
w.mu.Lock()
|
||||
w.state = "generating"
|
||||
w.total = len(videos)
|
||||
w.done = 0
|
||||
w.mu.Unlock()
|
||||
|
||||
defer func() {
|
||||
w.mu.Lock()
|
||||
w.state = "idle"
|
||||
w.currentTitle = ""
|
||||
w.mu.Unlock()
|
||||
}()
|
||||
|
||||
for _, v := range videos {
|
||||
if ctx.Err() != nil {
|
||||
log.Printf("[transcode] drive=%s canceled after %d/%d", w.drv.ID(), w.doneCount(), len(videos))
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
w.currentTitle = v.Title
|
||||
w.mu.Unlock()
|
||||
|
||||
if err := w.process(ctx, v); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
// 取消导致的失败不要写 failed,保持候选状态便于下次继续
|
||||
log.Printf("[transcode] drive=%s canceled while processing %s", w.drv.ID(), v.ID)
|
||||
return
|
||||
}
|
||||
log.Printf("[transcode] drive=%s video=%s failed: %v", w.drv.ID(), v.ID, err)
|
||||
if uerr := w.cat.UpdateVideoTranscode(context.WithoutCancel(ctx), v.ID, "failed", err.Error(), "", 0); uerr != nil {
|
||||
log.Printf("[transcode] mark failed %s: %v", v.ID, uerr)
|
||||
}
|
||||
}
|
||||
w.mu.Lock()
|
||||
w.done++
|
||||
w.mu.Unlock()
|
||||
}
|
||||
log.Printf("[transcode] drive=%s finished %d videos", w.drv.ID(), len(videos))
|
||||
}
|
||||
|
||||
func (w *Worker) doneCount() int {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.done
|
||||
}
|
||||
|
||||
func (w *Worker) process(ctx context.Context, v *catalog.Video) error {
|
||||
localPath, cleanup, err := w.fetchSource(ctx, v)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch source: %w", err)
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
info, err := ProbeFile(ctx, w.cfg.FFprobePath, localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !NeedsTranscode(info, v.Ext) {
|
||||
log.Printf("[transcode] drive=%s video=%s compatible (%s), skip", w.drv.ID(), v.ID, info.FormatName)
|
||||
return w.cat.UpdateVideoTranscode(ctx, v.ID, "skipped", "", "", 0)
|
||||
}
|
||||
|
||||
outPath := filepath.Join(w.cfg.WorkDir, sanitizeFileName(v.ID)+".transcoding.mp4")
|
||||
defer os.Remove(outPath)
|
||||
if err := TranscodeFile(ctx, w.cfg.FFmpegPath, info, localPath, outPath); err != nil {
|
||||
return err
|
||||
}
|
||||
stat, err := os.Stat(outPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat transcoded output: %w", err)
|
||||
}
|
||||
|
||||
dirID, err := w.ensureTargetDir(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ensure target dir: %w", err)
|
||||
}
|
||||
f, err := os.Open(outPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
fileID, err := w.drv.Upload(ctx, dirID, transcodedName(v), f, stat.Size())
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload transcoded file: %w", err)
|
||||
}
|
||||
log.Printf("[transcode] drive=%s video=%s ready: file=%s size=%d", w.drv.ID(), v.ID, fileID, stat.Size())
|
||||
return w.cat.UpdateVideoTranscode(ctx, v.ID, "ready", "", fileID, stat.Size())
|
||||
}
|
||||
|
||||
// fetchSource 把原始文件准备成本地路径。本地存储直接复用源路径(cleanup
|
||||
// 不删除源文件);云盘则整文件下载到 WorkDir。
|
||||
func (w *Worker) fetchSource(ctx context.Context, v *catalog.Video) (string, func(), error) {
|
||||
link, err := w.drv.StreamURL(ctx, v.FileID)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
u, err := url.Parse(link.URL)
|
||||
if isLocal := err == nil && u.Scheme != "http" && u.Scheme != "https"; isLocal {
|
||||
path := link.URL
|
||||
if err == nil && u.Scheme == "file" {
|
||||
path = u.Path
|
||||
}
|
||||
return path, func() {}, nil
|
||||
}
|
||||
|
||||
tmpPath := filepath.Join(w.cfg.WorkDir, sanitizeFileName(v.ID)+".src.tmp")
|
||||
cleanup := func() { os.Remove(tmpPath) }
|
||||
if err := w.downloadTo(ctx, link, tmpPath); err != nil {
|
||||
cleanup()
|
||||
return "", nil, err
|
||||
}
|
||||
return tmpPath, cleanup, nil
|
||||
}
|
||||
|
||||
func (w *Worker) downloadTo(ctx context.Context, link *drives.StreamLink, dst string) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, link.URL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for k, vals := range link.Headers {
|
||||
for _, val := range vals {
|
||||
req.Header.Add(k, val)
|
||||
}
|
||||
}
|
||||
res, err := w.hc.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
return fmt.Errorf("download source: HTTP %d", res.StatusCode)
|
||||
}
|
||||
f, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := io.Copy(f, res.Body); err != nil {
|
||||
return fmt.Errorf("download source: %w", err)
|
||||
}
|
||||
return f.Sync()
|
||||
}
|
||||
|
||||
// ensureTargetDir 确保网盘上的转码产物目录存在,并把它写进 drive 的扫描
|
||||
// 跳过列表(幂等),避免 scanner 把产物再当新视频收进库。
|
||||
func (w *Worker) ensureTargetDir(ctx context.Context) (string, error) {
|
||||
w.targetDirOnce.Do(func() {
|
||||
dirID, err := w.drv.EnsureDir(ctx, w.cfg.TargetDirName)
|
||||
if err != nil {
|
||||
w.targetDirErr = err
|
||||
return
|
||||
}
|
||||
w.targetDirID = dirID
|
||||
if err := w.addDirToSkipList(ctx, dirID); err != nil {
|
||||
// 跳过列表更新失败不阻塞转码,只记日志(最坏情况是 scanner
|
||||
// 之后把产物扫成新视频,可手动加跳过目录修复)。
|
||||
log.Printf("[transcode] drive=%s add skip dir %s: %v", w.drv.ID(), dirID, err)
|
||||
}
|
||||
})
|
||||
return w.targetDirID, w.targetDirErr
|
||||
}
|
||||
|
||||
func (w *Worker) addDirToSkipList(ctx context.Context, dirID string) error {
|
||||
d, err := w.cat.GetDrive(ctx, w.drv.ID())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, existing := range d.SkipDirIDs {
|
||||
if existing == dirID {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return w.cat.SetDriveSkipDirIDs(ctx, w.drv.ID(), append(d.SkipDirIDs, dirID))
|
||||
}
|
||||
|
||||
// transcodedName 生成产物文件名:原文件名去掉扩展名 + .mp4。
|
||||
func transcodedName(v *catalog.Video) string {
|
||||
base := strings.TrimSpace(v.FileName)
|
||||
if base == "" {
|
||||
base = v.Title
|
||||
}
|
||||
if base == "" {
|
||||
base = v.ID
|
||||
}
|
||||
if ext := filepath.Ext(base); ext != "" {
|
||||
base = strings.TrimSuffix(base, ext)
|
||||
}
|
||||
return sanitizeFileName(base) + ".mp4"
|
||||
}
|
||||
|
||||
// sanitizeFileName 把路径分隔符等危险字符替换掉,避免拼出意外路径。
|
||||
func sanitizeFileName(name string) string {
|
||||
replacer := strings.NewReplacer(
|
||||
"/", "_", "\\", "_", ":", "_", "*", "_", "?", "_",
|
||||
"\"", "_", "<", "_", ">", "_", "|", "_", "\x00", "_",
|
||||
)
|
||||
out := strings.TrimSpace(replacer.Replace(name))
|
||||
if out == "" {
|
||||
out = fmt.Sprintf("transcoded-%d", time.Now().UnixMilli())
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -134,9 +134,9 @@ apt_install() {
|
||||
python3 python3-requests python3-bs4 python3-lxml python3-socks
|
||||
}
|
||||
|
||||
verify_spider91_python_deps() {
|
||||
command -v python3 >/dev/null 2>&1 || die "python3 is required for 91Spider"
|
||||
python3 - <<'PY' || die "missing Python modules for 91Spider: requests, bs4, lxml, socks"
|
||||
verify_crawler_python_deps() {
|
||||
command -v python3 >/dev/null 2>&1 || die "python3 is required for crawler scripts"
|
||||
python3 - <<'PY' || die "missing Python modules for crawler scripts: requests, bs4, lxml, socks"
|
||||
import importlib.util
|
||||
import sys
|
||||
|
||||
@@ -200,7 +200,7 @@ install_dependencies() {
|
||||
install_go
|
||||
command -v ffmpeg >/dev/null 2>&1 || die "ffmpeg is required"
|
||||
command -v ffprobe >/dev/null 2>&1 || die "ffprobe is required"
|
||||
verify_spider91_python_deps
|
||||
verify_crawler_python_deps
|
||||
}
|
||||
|
||||
ensure_ownership() {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="referrer" content="no-referrer" />
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
|
||||
<meta name="description" content="91 视频站" />
|
||||
|
||||
+1
-1
@@ -128,7 +128,7 @@ verify_runtime_deps() {
|
||||
command -v "$cmd" >/dev/null 2>&1 || die "missing command: $cmd"
|
||||
done
|
||||
|
||||
python3 - <<'PY' || die "missing Python modules for 91Spider: requests, bs4, lxml, socks"
|
||||
python3 - <<'PY' || die "missing Python modules for crawler scripts: requests, bs4, lxml, socks"
|
||||
import importlib.util
|
||||
import sys
|
||||
|
||||
|
||||
Generated
+37
-2
@@ -1,14 +1,16 @@
|
||||
{
|
||||
"name": "video-site",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.7",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "video-site",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.7",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"artplayer": "^5.4.0",
|
||||
"hls.js": "^1.6.16",
|
||||
"lucide-react": "0.453.0",
|
||||
"react": "18.3.1",
|
||||
"react-dom": "18.3.1",
|
||||
@@ -475,6 +477,15 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/artplayer": {
|
||||
"version": "5.4.0",
|
||||
"resolved": "https://registry.npmjs.org/artplayer/-/artplayer-5.4.0.tgz",
|
||||
"integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"option-validator": "^2.0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/csstype": {
|
||||
"version": "3.2.3",
|
||||
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz",
|
||||
@@ -525,12 +536,27 @@
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/hls.js": {
|
||||
"version": "1.6.16",
|
||||
"resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.6.16.tgz",
|
||||
"integrity": "sha512-VSIRpLfRwlAAdGL4wiTucx2ScRipo0ed1FBatWkyt832jC4CReKstga6yIhYVwGu9LOBjuX9wzmRMeQdBJtzEA==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/js-tokens": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
|
||||
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/kind-of": {
|
||||
"version": "6.0.3",
|
||||
"resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
|
||||
"integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/lightningcss": {
|
||||
"version": "1.32.0",
|
||||
"resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz",
|
||||
@@ -832,6 +858,15 @@
|
||||
"node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/option-validator": {
|
||||
"version": "2.0.6",
|
||||
"resolved": "https://registry.npmjs.org/option-validator/-/option-validator-2.0.6.tgz",
|
||||
"integrity": "sha512-tmZDan2LRIRQyhUGvkff68/O0R8UmF+Btmiiz0SmSw2ng3CfPZB9wJlIjHpe/MKUZqyIZkVIXCrwr1tIN+0Dzg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"kind-of": "^6.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/picocolors": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
|
||||
|
||||
+3
-1
@@ -2,7 +2,7 @@
|
||||
"name": "video-site",
|
||||
"private": true,
|
||||
"license": "MIT",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.7",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
@@ -13,6 +13,8 @@
|
||||
"test": "node --import tsx --test tests/*.test.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"artplayer": "^5.4.0",
|
||||
"hls.js": "^1.6.16",
|
||||
"lucide-react": "0.453.0",
|
||||
"react": "18.3.1",
|
||||
"react-dom": "18.3.1",
|
||||
|
||||
@@ -63,8 +63,6 @@ build_package() {
|
||||
cp "$ROOT_DIR/backend/config.example.yaml" "$work/config.example.yaml"
|
||||
cp "$ROOT_DIR/install.sh" "$work/install.sh"
|
||||
cp -R "$ROOT_DIR/dist" "$work/dist"
|
||||
mkdir -p "$work/91VideoSpider"
|
||||
cp "$ROOT_DIR/91VideoSpider/spider_91porn.py" "$work/91VideoSpider/spider_91porn.py"
|
||||
|
||||
cat >"$work/README.txt" <<EOF
|
||||
$APP_NAME $VERSION
|
||||
|
||||
@@ -8,6 +8,7 @@ import { AdminLayout } from "@/admin/AdminLayout";
|
||||
import { LoginPage } from "@/admin/LoginPage";
|
||||
import { RequireAuth } from "@/admin/RequireAuth";
|
||||
import { DrivesPage } from "@/admin/DrivesPage";
|
||||
import { CrawlersPage } from "@/admin/CrawlersPage";
|
||||
import { VideosPage } from "@/admin/VideosPage";
|
||||
import { TagsPage } from "@/admin/TagsPage";
|
||||
import { ThemePage } from "@/admin/ThemePage";
|
||||
@@ -70,6 +71,7 @@ export default function App() {
|
||||
>
|
||||
<Route index element={<Navigate to="/admin/drives" replace />} />
|
||||
<Route path="drives" element={<DrivesPage />} />
|
||||
<Route path="crawlers" element={<CrawlersPage />} />
|
||||
<Route path="videos" element={<VideosPage />} />
|
||||
<Route path="tags" element={<TagsPage />} />
|
||||
<Route path="theme" element={<ThemePage />} />
|
||||
|
||||
+77
-35
@@ -14,6 +14,7 @@ import {
|
||||
import * as api from "./api";
|
||||
import { useAuth } from "./AuthContext";
|
||||
import { useToast } from "./ToastContext";
|
||||
import { SpiderIcon } from "./icons/SpiderIcon";
|
||||
|
||||
export function AdminLayout() {
|
||||
const { logout } = useAuth();
|
||||
@@ -77,41 +78,79 @@ export function AdminLayout() {
|
||||
<span className="admin-sidebar__brand-text">91后台</span>
|
||||
</div>
|
||||
<nav className="admin-nav">
|
||||
<NavLink to="/" className="admin-nav__link">
|
||||
<Home size={16} /> 返回主站
|
||||
</NavLink>
|
||||
<NavLink
|
||||
to="/admin/drives"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<HardDrive size={16} /> 网盘管理
|
||||
</NavLink>
|
||||
<NavLink
|
||||
to="/admin/videos"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<Film size={16} /> 视频管理
|
||||
</NavLink>
|
||||
<NavLink
|
||||
to="/admin/tags"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<Tags size={16} /> 标签管理
|
||||
</NavLink>
|
||||
<NavLink
|
||||
to="/admin/theme"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<Palette size={16} /> 主题外观
|
||||
</NavLink>
|
||||
<div className="admin-nav__group admin-nav__group--home">
|
||||
<span className="admin-nav__group-label">主站</span>
|
||||
<NavLink to="/" className="admin-nav__link">
|
||||
<span className="admin-nav__icon"><Home size={16} /></span>
|
||||
<span className="admin-nav__text">
|
||||
<span className="admin-nav__title">返回主站</span>
|
||||
</span>
|
||||
</NavLink>
|
||||
</div>
|
||||
<div className="admin-nav__group">
|
||||
<span className="admin-nav__group-label">资源</span>
|
||||
<NavLink
|
||||
to="/admin/drives"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<span className="admin-nav__icon"><HardDrive size={16} /></span>
|
||||
<span className="admin-nav__text">
|
||||
<span className="admin-nav__title">网盘管理</span>
|
||||
</span>
|
||||
</NavLink>
|
||||
<NavLink
|
||||
to="/admin/crawlers"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<span className="admin-nav__icon"><SpiderIcon size={16} /></span>
|
||||
<span className="admin-nav__text">
|
||||
<span className="admin-nav__title">爬虫管理</span>
|
||||
</span>
|
||||
</NavLink>
|
||||
</div>
|
||||
<div className="admin-nav__group">
|
||||
<span className="admin-nav__group-label">管理</span>
|
||||
<NavLink
|
||||
to="/admin/videos"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<span className="admin-nav__icon"><Film size={16} /></span>
|
||||
<span className="admin-nav__text">
|
||||
<span className="admin-nav__title">视频管理</span>
|
||||
</span>
|
||||
</NavLink>
|
||||
<NavLink
|
||||
to="/admin/tags"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<span className="admin-nav__icon"><Tags size={16} /></span>
|
||||
<span className="admin-nav__text">
|
||||
<span className="admin-nav__title">标签管理</span>
|
||||
</span>
|
||||
</NavLink>
|
||||
</div>
|
||||
<div className="admin-nav__group">
|
||||
<span className="admin-nav__group-label">系统</span>
|
||||
<NavLink
|
||||
to="/admin/theme"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<span className="admin-nav__icon"><Palette size={16} /></span>
|
||||
<span className="admin-nav__text">
|
||||
<span className="admin-nav__title">主题外观</span>
|
||||
</span>
|
||||
</NavLink>
|
||||
</div>
|
||||
</nav>
|
||||
<div className="admin-sidebar__footer">
|
||||
<button
|
||||
@@ -139,6 +178,9 @@ export function AdminLayout() {
|
||||
<div className="admin-sidebar__mobile-overlay" onClick={() => setMobileMenuOpen(false)} />
|
||||
)}
|
||||
<div className={`admin-sidebar__mobile-panel${mobileMenuOpen ? " is-open" : ""}`}>
|
||||
<NavLink to="/" className="admin-sidebar__home" onClick={() => setMobileMenuOpen(false)}>
|
||||
<Home size={14} /> 返回主站
|
||||
</NavLink>
|
||||
<button
|
||||
className="admin-sidebar__check-update"
|
||||
onClick={() => { handleCheckUpdate(); setMobileMenuOpen(false); }}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { ReactNode } from "react";
|
||||
import { AlertTriangle } from "lucide-react";
|
||||
import { Modal } from "./Modal";
|
||||
|
||||
@@ -12,6 +13,7 @@ type ConfirmModalProps = {
|
||||
centerMessage?: boolean;
|
||||
modalClassName?: string;
|
||||
loading?: boolean;
|
||||
children?: ReactNode;
|
||||
onCancel: () => void;
|
||||
onConfirm: () => void;
|
||||
};
|
||||
@@ -27,6 +29,7 @@ export function ConfirmModal({
|
||||
centerMessage = false,
|
||||
modalClassName = "",
|
||||
loading = false,
|
||||
children,
|
||||
onCancel,
|
||||
onConfirm,
|
||||
}: ConfirmModalProps) {
|
||||
@@ -65,6 +68,7 @@ export function ConfirmModal({
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
{children}
|
||||
</div>
|
||||
</div>
|
||||
</Modal>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+131
-26
@@ -20,6 +20,7 @@ import { formatBytes } from "./storageFormat";
|
||||
import { makeUniqueDriveId } from "./driveId";
|
||||
import {
|
||||
FormState,
|
||||
driveKindAbbr,
|
||||
kindLabel,
|
||||
emptyForm,
|
||||
idleNightlyStatus,
|
||||
@@ -38,6 +39,22 @@ import { DriveForm } from "./drive/DriveForm";
|
||||
import { DeleteDriveModal } from "./drive/DeleteDriveModal";
|
||||
import { SkipDirsPanel } from "./drive/SkipDirsPanel";
|
||||
|
||||
const DRIVE_BUSY_MESSAGE = "当前存储有正在进行的任务,请稍后重试";
|
||||
const NIGHTLY_BUSY_MESSAGE = "当前有全量扫描任务正在进行,请稍后重试";
|
||||
|
||||
function isDriveBusy(d: api.AdminDrive) {
|
||||
return [
|
||||
d.scanGenerationStatus,
|
||||
d.thumbnailGenerationStatus,
|
||||
d.previewGenerationStatus,
|
||||
d.fingerprintGenerationStatus,
|
||||
d.transcodeGenerationStatus,
|
||||
].some((status) => {
|
||||
const state = status?.state || "idle";
|
||||
return state !== "idle";
|
||||
});
|
||||
}
|
||||
|
||||
export function DrivesPage() {
|
||||
const [list, setList] = useState<api.AdminDrive[]>([]);
|
||||
const [storage, setStorage] = useState<api.AdminDriveStorage | null>(null);
|
||||
@@ -58,10 +75,12 @@ export function DrivesPage() {
|
||||
const [regenFailedThumbId, setRegenFailedThumbId] = useState("");
|
||||
const [regenFailedFingerprintId, setRegenFailedFingerprintId] = useState("");
|
||||
const [togglingTeaserId, setTogglingTeaserId] = useState("");
|
||||
const [togglingTranscodeId, setTogglingTranscodeId] = useState("");
|
||||
const [scanningAll, setScanningAll] = useState(false);
|
||||
const [stoppingAll, setStoppingAll] = useState(false);
|
||||
const [trackingNightly, setTrackingNightly] = useState(false);
|
||||
const [scanningDriveId, setScanningDriveId] = useState("");
|
||||
const [scanningDriveIds, setScanningDriveIds] = useState<Record<string, boolean>>({});
|
||||
const scanningDriveIdsRef = useRef(new Set<string>());
|
||||
const [stoppingDriveId, setStoppingDriveId] = useState("");
|
||||
const [searchParams, setSearchParams] = useSearchParams();
|
||||
const selectedDriveId = searchParams.get("drive") || null;
|
||||
@@ -75,7 +94,16 @@ export function DrivesPage() {
|
||||
: hasCreateFormChanges(form, initialForm);
|
||||
|
||||
const uploadTargets = useMemo(
|
||||
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115" || d.kind === "p123" || d.kind === "onedrive"),
|
||||
() =>
|
||||
list.filter(
|
||||
(d) =>
|
||||
d.kind === "pikpak" ||
|
||||
d.kind === "p115" ||
|
||||
d.kind === "p123" ||
|
||||
d.kind === "onedrive" ||
|
||||
d.kind === "googledrive" ||
|
||||
d.kind === "wopan"
|
||||
),
|
||||
[list]
|
||||
);
|
||||
|
||||
@@ -184,7 +212,14 @@ export function DrivesPage() {
|
||||
kind: d.kind,
|
||||
name: d.name,
|
||||
rootId: d.rootId,
|
||||
creds: d.kind === "spider91" ? { proxy: d.spider91Proxy ?? "" } : {},
|
||||
creds:
|
||||
d.kind === "spider91"
|
||||
? { proxy: d.spider91Proxy ?? "" }
|
||||
: d.kind === "googledrive"
|
||||
? { use_online_api: (d.googleDriveUseOnlineAPI ?? true) ? "true" : "false" }
|
||||
: d.kind === "localstorage"
|
||||
? { strm_allow_outside_root: (d.strmAllowOutsideRoot ?? false) ? "true" : "false" }
|
||||
: {},
|
||||
spider91UploadDriveId: settings?.spider91UploadDriveId ?? "",
|
||||
};
|
||||
setForm(nextForm);
|
||||
@@ -295,25 +330,47 @@ export function DrivesPage() {
|
||||
}
|
||||
|
||||
async function handleRescan(d: api.AdminDrive) {
|
||||
if (scanningDriveId) return;
|
||||
setScanningDriveId(d.id);
|
||||
if (d.kind === "spider91") {
|
||||
show("91Spider 不再支持通过网盘运行,请到爬虫管理添加爬虫脚本", "info");
|
||||
return;
|
||||
}
|
||||
if (nightlyBusy) {
|
||||
show(nightlyBusyText(nightlyStatus) || NIGHTLY_BUSY_MESSAGE, "info");
|
||||
return;
|
||||
}
|
||||
if (isDriveBusy(d) || scanningDriveIdsRef.current.has(d.id)) {
|
||||
show(DRIVE_BUSY_MESSAGE, "info");
|
||||
return;
|
||||
}
|
||||
scanningDriveIdsRef.current.add(d.id);
|
||||
setScanningDriveIds((prev) => ({ ...prev, [d.id]: true }));
|
||||
try {
|
||||
await api.rescan(d.id);
|
||||
if (d.kind === "spider91") {
|
||||
show("已触发抓取任务,需要 2-4 分钟,可稍后刷新视频列表查看", "success");
|
||||
} else {
|
||||
show("已触发扫描,可稍后刷新视频列表查看", "success");
|
||||
const resp = await api.rescan(d.id);
|
||||
if (!resp.accepted) {
|
||||
if (resp.status) {
|
||||
setNightlyStatus(resp.status);
|
||||
}
|
||||
show(resp.message || DRIVE_BUSY_MESSAGE, "info");
|
||||
refreshDriveList();
|
||||
return;
|
||||
}
|
||||
show("已触发扫描,可稍后刷新视频列表查看", "success");
|
||||
refreshDriveList();
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "触发失败", "error");
|
||||
} finally {
|
||||
setScanningDriveId("");
|
||||
scanningDriveIdsRef.current.delete(d.id);
|
||||
setScanningDriveIds((prev) => {
|
||||
const next = { ...prev };
|
||||
delete next[d.id];
|
||||
return next;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function handleRunNightly() {
|
||||
if (nightlyBusy) {
|
||||
show(nightlyBusyText(nightlyStatus) || "当前已有扫描所有网盘任务", "info");
|
||||
show(nightlyBusyText(nightlyStatus) || NIGHTLY_BUSY_MESSAGE, "info");
|
||||
return;
|
||||
}
|
||||
setScanningAll(true);
|
||||
@@ -324,7 +381,7 @@ export function DrivesPage() {
|
||||
setTrackingNightly(!resp.status.running);
|
||||
show("已触发扫描所有网盘,耗时较长,可在任务状态和 backend 日志观察进度", "success");
|
||||
} else {
|
||||
show("当前已有扫描所有网盘任务", "info");
|
||||
show(resp.message || NIGHTLY_BUSY_MESSAGE, "info");
|
||||
}
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "触发失败", "error");
|
||||
@@ -446,6 +503,41 @@ export function DrivesPage() {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleStartTranscode(d: api.AdminDrive) {
|
||||
setTogglingTranscodeId(d.id);
|
||||
try {
|
||||
const resp = await api.startDriveTranscode(d.id);
|
||||
if (resp.accepted) {
|
||||
show(`已开始「${d.name || d.id}」的视频转码`, "success");
|
||||
} else {
|
||||
show(resp.message || "转码任务未能开启", "info");
|
||||
}
|
||||
refreshDriveList();
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "开启失败", "error");
|
||||
} finally {
|
||||
setTogglingTranscodeId("");
|
||||
}
|
||||
}
|
||||
|
||||
async function handleStopTranscode(d: api.AdminDrive) {
|
||||
setTogglingTranscodeId(d.id);
|
||||
try {
|
||||
const resp = await api.stopDriveTranscode(d.id);
|
||||
show(
|
||||
resp.stopped
|
||||
? `已停止「${d.name || d.id}」的视频转码`
|
||||
: `「${d.name || d.id}」没有正在运行的转码任务`,
|
||||
"success"
|
||||
);
|
||||
refreshDriveList();
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "停止失败", "error");
|
||||
} finally {
|
||||
setTogglingTranscodeId("");
|
||||
}
|
||||
}
|
||||
|
||||
const selectedDrive = useMemo(() => {
|
||||
return selectedDriveId ? list.find((d) => d.id === selectedDriveId) : null;
|
||||
}, [selectedDriveId, list]);
|
||||
@@ -498,10 +590,8 @@ export function DrivesPage() {
|
||||
)}
|
||||
{d.kind === "spider91" && (
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">上次抓取时间</span>
|
||||
<span className="admin-detail-value">
|
||||
{d.lastCrawlAt ? new Date(d.lastCrawlAt * 1000).toLocaleString() : "尚未抓取"}
|
||||
</span>
|
||||
<span className="admin-detail-label">配置状态</span>
|
||||
<span className="admin-detail-value">已废弃,请到爬虫管理添加</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
@@ -515,17 +605,27 @@ export function DrivesPage() {
|
||||
type="button"
|
||||
className="admin-btn is-primary"
|
||||
onClick={() => handleRescan(d)}
|
||||
disabled={!!scanningDriveId}
|
||||
disabled={d.kind === "spider91"}
|
||||
aria-disabled={d.kind === "spider91" || nightlyBusy || isDriveBusy(d) || !!scanningDriveIds[d.id]}
|
||||
title={
|
||||
d.kind === "spider91"
|
||||
? "91Spider 不再支持通过网盘运行,请到爬虫管理添加爬虫脚本"
|
||||
: nightlyBusy
|
||||
? nightlyBusyText(nightlyStatus) || NIGHTLY_BUSY_MESSAGE
|
||||
: isDriveBusy(d) || scanningDriveIds[d.id]
|
||||
? DRIVE_BUSY_MESSAGE
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
{d.kind === "spider91" ? (
|
||||
<>
|
||||
<Download size={13} className={scanningDriveId === d.id ? "admin-spin" : undefined} />
|
||||
{scanningDriveId === d.id ? "触发中..." : "立即抓取"}
|
||||
<Download size={13} className={scanningDriveIds[d.id] ? "admin-spin" : undefined} />
|
||||
已废弃
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<RefreshCw size={13} className={scanningDriveId === d.id ? "admin-spin" : undefined} />
|
||||
{scanningDriveId === d.id ? "触发中..." : "立即重扫"}
|
||||
<RefreshCw size={13} className={scanningDriveIds[d.id] ? "admin-spin" : undefined} />
|
||||
{scanningDriveIds[d.id] ? "触发中..." : "立即重扫"}
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
@@ -540,9 +640,11 @@ export function DrivesPage() {
|
||||
{stoppingDriveId === d.id ? "停止中..." : "停止所有任务"}
|
||||
</button>
|
||||
</div>
|
||||
<button type="button" className="admin-btn" onClick={() => openEdit(d)}>
|
||||
{d.kind === "spider91" ? "编辑配置" : "编辑配置凭证"}
|
||||
</button>
|
||||
{d.kind !== "spider91" && (
|
||||
<button type="button" className="admin-btn" onClick={() => openEdit(d)}>
|
||||
编辑配置凭证
|
||||
</button>
|
||||
)}
|
||||
<button type="button" className="admin-btn is-danger admin-detail-actions__danger" onClick={() => setDeleteTarget(d)}>
|
||||
<Trash2 size={13} /> 删除网盘
|
||||
</button>
|
||||
@@ -571,10 +673,13 @@ export function DrivesPage() {
|
||||
regenFailedThumbId={regenFailedThumbId}
|
||||
regenFailedFingerprintId={regenFailedFingerprintId}
|
||||
togglingTeaserId={togglingTeaserId}
|
||||
togglingTranscodeId={togglingTranscodeId}
|
||||
onToggleTeaser={() => handleToggleTeaser(d)}
|
||||
onRegenFailed={() => handleRegenFailed(d)}
|
||||
onRegenFailedThumbnails={() => handleRegenFailedThumbnails(d)}
|
||||
onRegenFailedFingerprints={() => handleRegenFailedFingerprints(d)}
|
||||
onStartTranscode={() => handleStartTranscode(d)}
|
||||
onStopTranscode={() => handleStopTranscode(d)}
|
||||
/>
|
||||
|
||||
<div className="admin-detail-card">
|
||||
@@ -720,7 +825,7 @@ export function DrivesPage() {
|
||||
<div className="admin-drive-card__header">
|
||||
<div className="admin-drive-card__title">
|
||||
<span className="admin-drive-card__brand-icon" data-kind={d.kind}>
|
||||
{d.kind.substring(0, 2)}
|
||||
{driveKindAbbr(d.kind)}
|
||||
</span>
|
||||
<span>{d.name || d.id}</span>
|
||||
</div>
|
||||
|
||||
+1
-6
@@ -75,12 +75,7 @@ export function Modal({ open, title, onClose, children, footer, className = "" }
|
||||
|
||||
if (!open) return null;
|
||||
return (
|
||||
<div
|
||||
className="admin-modal-backdrop"
|
||||
onMouseDown={(e) => {
|
||||
if (e.target === e.currentTarget) onClose();
|
||||
}}
|
||||
>
|
||||
<div className="admin-modal-backdrop">
|
||||
<div
|
||||
ref={dialogRef}
|
||||
className={`admin-modal${className ? ` ${className}` : ""}`}
|
||||
|
||||
+520
-158
@@ -1,5 +1,17 @@
|
||||
import { useEffect, useId, useState } from "react";
|
||||
import { ChevronDown, Edit, RefreshCw, Search, CheckSquare, Square, Image, Trash2 } from "lucide-react";
|
||||
import { useSearchParams } from "react-router-dom";
|
||||
import {
|
||||
ChevronDown,
|
||||
Edit,
|
||||
RefreshCw,
|
||||
Search,
|
||||
CheckSquare,
|
||||
Square,
|
||||
Image,
|
||||
Trash2,
|
||||
Ban,
|
||||
RotateCcw,
|
||||
} from "lucide-react";
|
||||
import * as api from "./api";
|
||||
import { useToast } from "./ToastContext";
|
||||
import { Modal } from "./Modal";
|
||||
@@ -10,7 +22,85 @@ const DESKTOP_VIDEOS_PAGE_SIZE = 50;
|
||||
const MOBILE_VIDEOS_PAGE_SIZE = 20;
|
||||
const VIDEOS_MOBILE_QUERY = "(max-width: 640px)";
|
||||
|
||||
type TabKey = "current" | "blacklist";
|
||||
|
||||
const TABS: { key: TabKey; label: string }[] = [
|
||||
{ key: "current", label: "当前视频" },
|
||||
{ key: "blacklist", label: "拉黑视频" },
|
||||
];
|
||||
|
||||
/**
|
||||
* 视频管理容器:顶部分段标签在「当前 / 隐藏 / 拉黑」三个视图间切换,
|
||||
* 激活标签同步到 URL ?tab=,标签上的计数来自 /videos/stats。
|
||||
*/
|
||||
export function VideosPage() {
|
||||
const [searchParams, setSearchParams] = useSearchParams();
|
||||
const rawTab = searchParams.get("tab");
|
||||
const activeTab: TabKey = rawTab === "blacklist" ? "blacklist" : "current";
|
||||
const [stats, setStats] = useState<api.VideoStats | null>(null);
|
||||
|
||||
async function refreshStats() {
|
||||
try {
|
||||
setStats(await api.getVideoStats());
|
||||
} catch {
|
||||
// 计数仅用于标签徽标,失败不阻塞主流程。
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
refreshStats();
|
||||
}, []);
|
||||
|
||||
function selectTab(key: TabKey) {
|
||||
setSearchParams(
|
||||
(prev) => {
|
||||
const next = new URLSearchParams(prev);
|
||||
if (key === "current") next.delete("tab");
|
||||
else next.set("tab", key);
|
||||
return next;
|
||||
},
|
||||
{ replace: true }
|
||||
);
|
||||
}
|
||||
|
||||
const counts: Record<TabKey, number | undefined> = {
|
||||
current: stats?.current,
|
||||
blacklist: stats?.blacklisted,
|
||||
};
|
||||
|
||||
return (
|
||||
<section>
|
||||
<header className="admin-page__header">
|
||||
<h1 className="admin-page__title">视频管理</h1>
|
||||
</header>
|
||||
|
||||
<div className="admin-video-tabs" role="tablist" aria-label="视频管理分类">
|
||||
{TABS.map((t) => (
|
||||
<button
|
||||
key={t.key}
|
||||
type="button"
|
||||
role="tab"
|
||||
aria-selected={activeTab === t.key}
|
||||
className={`admin-video-tab ${activeTab === t.key ? "is-active" : ""}`}
|
||||
onClick={() => selectTab(t.key)}
|
||||
>
|
||||
<span>{t.label}</span>
|
||||
{counts[t.key] !== undefined && (
|
||||
<span className="admin-video-tab__count">{counts[t.key]}</span>
|
||||
)}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{activeTab === "current" && <CurrentVideosTab onStatsChanged={refreshStats} />}
|
||||
{activeTab === "blacklist" && <BlacklistTab onStatsChanged={refreshStats} />}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- 当前视频 ----------
|
||||
|
||||
function CurrentVideosTab({ onStatsChanged }: { onStatsChanged: () => void }) {
|
||||
const [list, setList] = useState<api.AdminVideo[]>([]);
|
||||
const [drives, setDrives] = useState<api.AdminDrive[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
@@ -27,8 +117,10 @@ export function VideosPage() {
|
||||
const [batchRegening, setBatchRegening] = useState(false);
|
||||
const [batchDeleteOpen, setBatchDeleteOpen] = useState(false);
|
||||
const [batchDeleting, setBatchDeleting] = useState(false);
|
||||
const [batchDeleteSource, setBatchDeleteSource] = useState(false);
|
||||
const [deleteTarget, setDeleteTarget] = useState<api.AdminVideo | null>(null);
|
||||
const [deleting, setDeleting] = useState(false);
|
||||
const [deleteSource, setDeleteSource] = useState(false);
|
||||
const pageSize = useVideosPageSize();
|
||||
const { show } = useToast();
|
||||
|
||||
@@ -72,9 +164,7 @@ export function VideosPage() {
|
||||
return () => window.clearTimeout(timer);
|
||||
}, [keyword]);
|
||||
|
||||
const driveNameMap = new Map(
|
||||
drives.map((d) => [d.id, d.name || d.id])
|
||||
);
|
||||
const driveNameMap = new Map(drives.map((d) => [d.id, d.name || d.id]));
|
||||
|
||||
const listItems = list;
|
||||
const totalPages = Math.max(1, Math.ceil(total / pageSize));
|
||||
@@ -100,6 +190,7 @@ export function VideosPage() {
|
||||
|
||||
async function handleBatchDelete() {
|
||||
if (selectedIds.size === 0) return;
|
||||
setBatchDeleteSource(false);
|
||||
setBatchDeleteOpen(true);
|
||||
}
|
||||
|
||||
@@ -108,13 +199,14 @@ export function VideosPage() {
|
||||
setBatchRegening(true);
|
||||
let success = 0;
|
||||
try {
|
||||
const results = await Promise.allSettled(
|
||||
ids.map((id) => api.regenPreview(id))
|
||||
);
|
||||
const results = await Promise.allSettled(ids.map((id) => api.regenPreview(id)));
|
||||
for (const r of results) {
|
||||
if (r.status === "fulfilled") success++;
|
||||
}
|
||||
show(`批量触发完成,成功 ${success} / ${ids.length} 个`, success === ids.length ? "success" : "info");
|
||||
show(
|
||||
`批量触发完成,成功 ${success} / ${ids.length} 个`,
|
||||
success === ids.length ? "success" : "info"
|
||||
);
|
||||
setSelectedIds(new Set());
|
||||
setBatchRegenOpen(false);
|
||||
} finally {
|
||||
@@ -127,14 +219,16 @@ export function VideosPage() {
|
||||
const target = deleteTarget;
|
||||
setDeleting(true);
|
||||
try {
|
||||
const result = await api.deleteVideo(target.id);
|
||||
const result = await api.deleteVideo(target.id, { deleteSource });
|
||||
setDeleteTarget(null);
|
||||
setDeleteSource(false);
|
||||
setSelectedIds((ids) => {
|
||||
const next = new Set(ids);
|
||||
next.delete(target.id);
|
||||
return next;
|
||||
});
|
||||
show(result.deletedSource ? "已删除视频,并清理 91Spider 源文件" : "已删除视频", "success");
|
||||
show(result.deletedSource ? "已删除视频,并清理源文件" : "已删除视频", "success");
|
||||
onStatsChanged();
|
||||
if (listItems.length === 1 && page > 1) {
|
||||
setPage((p) => Math.max(1, p - 1));
|
||||
} else {
|
||||
@@ -156,7 +250,7 @@ export function VideosPage() {
|
||||
let deletedSources = 0;
|
||||
for (const id of ids) {
|
||||
try {
|
||||
const result = await api.deleteVideo(id);
|
||||
const result = await api.deleteVideo(id, { deleteSource: batchDeleteSource });
|
||||
success++;
|
||||
if (result.deletedSource) deletedSources++;
|
||||
} catch {
|
||||
@@ -165,13 +259,18 @@ export function VideosPage() {
|
||||
}
|
||||
const failed = ids.length - success;
|
||||
if (failed === 0) {
|
||||
const extra = deletedSources > 0 ? `,其中 ${deletedSources} 个清理了 91Spider 源文件` : "";
|
||||
const extra = deletedSources > 0 ? `,其中 ${deletedSources} 个清理了源文件` : "";
|
||||
show(`批量删除完成,成功 ${success} 个${extra}`, "success");
|
||||
} else {
|
||||
show(`批量删除完成,成功 ${success} / ${ids.length} 个,失败 ${failed} 个`, success > 0 ? "info" : "error");
|
||||
show(
|
||||
`批量删除完成,成功 ${success} / ${ids.length} 个,失败 ${failed} 个`,
|
||||
success > 0 ? "info" : "error"
|
||||
);
|
||||
}
|
||||
setSelectedIds(new Set());
|
||||
setBatchDeleteOpen(false);
|
||||
setBatchDeleteSource(false);
|
||||
onStatsChanged();
|
||||
if (success >= listItems.length && page > 1) {
|
||||
setPage((p) => Math.max(1, p - 1));
|
||||
} else {
|
||||
@@ -186,7 +285,7 @@ export function VideosPage() {
|
||||
if (selectedIds.size === listItems.length && listItems.length > 0) {
|
||||
setSelectedIds(new Set());
|
||||
} else {
|
||||
setSelectedIds(new Set(listItems.map(v => v.id)));
|
||||
setSelectedIds(new Set(listItems.map((v) => v.id)));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -204,52 +303,21 @@ export function VideosPage() {
|
||||
}
|
||||
|
||||
return (
|
||||
<section>
|
||||
<header className="admin-page__header">
|
||||
<h1 className="admin-page__title">视频管理</h1>
|
||||
<div className="admin-page__actions admin-videos-filter">
|
||||
<div className="admin-videos-filter__select-wrap">
|
||||
<select
|
||||
className="admin-videos-filter__select"
|
||||
value={driveId}
|
||||
onChange={(e) => {
|
||||
setDriveId(e.target.value);
|
||||
setPage(1);
|
||||
}}
|
||||
>
|
||||
<option value="">全部网盘</option>
|
||||
{drives.map((d) => (
|
||||
<option key={d.id} value={d.id}>
|
||||
{d.name || d.id}(已生成 {d.teaserReadyCount ?? 0},待生成{" "}
|
||||
{d.teaserPendingCount ?? 0})
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<ChevronDown size={15} className="admin-videos-filter__select-icon" aria-hidden="true" />
|
||||
</div>
|
||||
<form className="admin-videos-filter__search" onSubmit={handleSearchSubmit}>
|
||||
<Search size={14} className="admin-videos-filter__search-icon" />
|
||||
<input
|
||||
aria-label="搜索标题或作者"
|
||||
value={keyword}
|
||||
onChange={(e) => setKeyword(e.target.value)}
|
||||
placeholder="搜索标题 / 作者"
|
||||
/>
|
||||
</form>
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 刷新
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
<>
|
||||
<div className="admin-page__actions admin-videos-filter">
|
||||
<DriveFilter drives={drives} driveId={driveId} onChange={(id) => { setDriveId(id); setPage(1); }} withCounts />
|
||||
<SearchBox keyword={keyword} onChange={setKeyword} onSubmit={handleSearchSubmit} />
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 刷新
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{!loading && (
|
||||
<div className="admin-videos-list-toolbar">
|
||||
<div className="admin-videos-summary">{listSummary}</div>
|
||||
{selectedIds.size > 0 && (
|
||||
<div className="admin-videos-bulk-actions">
|
||||
<span className="admin-videos-bulk-actions__count">
|
||||
已选择 {selectedIds.size} 项
|
||||
</span>
|
||||
<span className="admin-videos-bulk-actions__count">已选择 {selectedIds.size} 项</span>
|
||||
<button type="button" className="admin-btn is-primary admin-videos-bulk-actions__btn" onClick={handleBatchRegen}>
|
||||
<RefreshCw size={13} /> 批量重生预览视频
|
||||
</button>
|
||||
@@ -262,18 +330,9 @@ export function VideosPage() {
|
||||
)}
|
||||
|
||||
{loading ? (
|
||||
<div className="admin-loading-state">
|
||||
<RefreshCw size={20} className="admin-spin" />
|
||||
<span>加载中...</span>
|
||||
</div>
|
||||
<LoadingState />
|
||||
) : loadError ? (
|
||||
<div className="admin-error-state">
|
||||
<strong>视频加载失败</strong>
|
||||
<span>{loadError}</span>
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 重试
|
||||
</button>
|
||||
</div>
|
||||
<ErrorState message={loadError} onRetry={refresh} />
|
||||
) : listItems.length === 0 ? (
|
||||
<div className="admin-empty-state">
|
||||
<div className="admin-empty-state__icon">
|
||||
@@ -290,14 +349,22 @@ export function VideosPage() {
|
||||
<table className="admin-table is-selectable admin-videos-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th className="is-checkbox" style={{ width: '40px' }}>
|
||||
<th className="is-checkbox" style={{ width: "40px" }}>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-table-checkbox-btn"
|
||||
onClick={toggleSelectAll}
|
||||
aria-label={selectedIds.size > 0 && selectedIds.size === listItems.length ? "清空当前页选择" : "选择当前页视频"}
|
||||
aria-label={
|
||||
selectedIds.size > 0 && selectedIds.size === listItems.length
|
||||
? "清空当前页选择"
|
||||
: "选择当前页视频"
|
||||
}
|
||||
>
|
||||
{selectedIds.size > 0 && selectedIds.size === listItems.length ? <CheckSquare size={16} /> : <Square size={16} />}
|
||||
{selectedIds.size > 0 && selectedIds.size === listItems.length ? (
|
||||
<CheckSquare size={16} />
|
||||
) : (
|
||||
<Square size={16} />
|
||||
)}
|
||||
</button>
|
||||
</th>
|
||||
<th>标题</th>
|
||||
@@ -318,35 +385,15 @@ export function VideosPage() {
|
||||
onClick={() => toggleSelect(v.id)}
|
||||
aria-label={`${selectedIds.has(v.id) ? "取消选择" : "选择"}视频 ${v.title}`}
|
||||
>
|
||||
{selectedIds.has(v.id) ? <CheckSquare size={16} color="var(--accent)" /> : <Square size={16} color="var(--border-strong)" />}
|
||||
{selectedIds.has(v.id) ? (
|
||||
<CheckSquare size={16} color="var(--accent)" />
|
||||
) : (
|
||||
<Square size={16} color="var(--border-strong)" />
|
||||
)}
|
||||
</button>
|
||||
</td>
|
||||
<td data-label="标题">
|
||||
<div className="admin-video-title-cell">
|
||||
<div className="admin-video-thumb-wrap" aria-hidden="true">
|
||||
{v.thumbnailUrl ? (
|
||||
<img className="admin-video-thumb" src={v.thumbnailUrl} alt="" />
|
||||
) : (
|
||||
<div className="admin-video-thumb-placeholder">
|
||||
<Image size={14} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="admin-video-title-body">
|
||||
<div className="admin-video-title">{v.title}</div>
|
||||
{fileMeta(v) && (
|
||||
<div className="admin-video-filemeta">{fileMeta(v)}</div>
|
||||
)}
|
||||
{(v.tags ?? []).length > 0 && (
|
||||
<div className="admin-pills admin-video-title-tags">
|
||||
{(v.tags ?? []).map((t) => (
|
||||
<span key={t} className="admin-pill">{t}</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<VideoFileMetaPills video={v} />
|
||||
</div>
|
||||
</div>
|
||||
<VideoTitleCell video={v} />
|
||||
</td>
|
||||
<td data-label="作者">{v.author || <span className="admin-text-faint">—</span>}</td>
|
||||
<td data-label="时长">{formatDur(v.durationSeconds)}</td>
|
||||
@@ -363,7 +410,15 @@ export function VideosPage() {
|
||||
<button type="button" className="admin-btn" onClick={() => handleRegen(v)} title="重生预览视频">
|
||||
<RefreshCw size={13} />
|
||||
</button>{" "}
|
||||
<button type="button" className="admin-btn is-danger" onClick={() => setDeleteTarget(v)} title="删除视频">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn is-danger"
|
||||
onClick={() => {
|
||||
setDeleteSource(false);
|
||||
setDeleteTarget(v);
|
||||
}}
|
||||
title="删除视频"
|
||||
>
|
||||
<Trash2 size={13} />
|
||||
</button>
|
||||
</td>
|
||||
@@ -371,43 +426,7 @@ export function VideosPage() {
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
<div className="admin-table-pagination">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage(1)}
|
||||
disabled={page <= 1}
|
||||
>
|
||||
首页
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage((p) => Math.max(1, p - 1))}
|
||||
disabled={page <= 1}
|
||||
>
|
||||
上一页
|
||||
</button>
|
||||
<span className="admin-table-pagination__info">
|
||||
第 {page} / {totalPages} 页,每页 {pageSize} 个
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage((p) => Math.min(totalPages, p + 1))}
|
||||
disabled={page >= totalPages}
|
||||
>
|
||||
下一页
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage(totalPages)}
|
||||
disabled={page >= totalPages}
|
||||
>
|
||||
末页
|
||||
</button>
|
||||
</div>
|
||||
<Pagination page={page} totalPages={totalPages} pageSize={pageSize} onPage={setPage} />
|
||||
</>
|
||||
)}
|
||||
|
||||
@@ -443,10 +462,15 @@ export function VideosPage() {
|
||||
modalClassName="admin-modal--delete-confirm"
|
||||
loading={deleting}
|
||||
onCancel={() => {
|
||||
if (!deleting) setDeleteTarget(null);
|
||||
if (!deleting) {
|
||||
setDeleteTarget(null);
|
||||
setDeleteSource(false);
|
||||
}
|
||||
}}
|
||||
onConfirm={confirmDeleteVideo}
|
||||
/>
|
||||
>
|
||||
<DeleteSourceOption checked={deleteSource} disabled={deleting} onChange={setDeleteSource} note="开启后会先删除源文件,失败则不会删除管理库记录。" />
|
||||
</ConfirmModal>
|
||||
<ConfirmModal
|
||||
open={batchDeleteOpen}
|
||||
title="批量删除视频"
|
||||
@@ -457,11 +481,353 @@ export function VideosPage() {
|
||||
modalClassName="admin-modal--delete-confirm"
|
||||
loading={batchDeleting}
|
||||
onCancel={() => {
|
||||
if (!batchDeleting) setBatchDeleteOpen(false);
|
||||
if (!batchDeleting) {
|
||||
setBatchDeleteOpen(false);
|
||||
setBatchDeleteSource(false);
|
||||
}
|
||||
}}
|
||||
onConfirm={confirmBatchDelete}
|
||||
>
|
||||
<DeleteSourceOption checked={batchDeleteSource} disabled={batchDeleting} onChange={setBatchDeleteSource} note="开启后会先删除源文件,失败的视频会保留管理库记录。" />
|
||||
</ConfirmModal>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- 拉黑视频 ----------
|
||||
|
||||
function BlacklistTab({ onStatsChanged }: { onStatsChanged: () => void }) {
|
||||
const [list, setList] = useState<api.AdminDeletedVideo[]>([]);
|
||||
const [drives, setDrives] = useState<api.AdminDrive[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [loadError, setLoadError] = useState("");
|
||||
const [keyword, setKeyword] = useState("");
|
||||
const [searchKeyword, setSearchKeyword] = useState("");
|
||||
const [page, setPage] = useState(1);
|
||||
const [total, setTotal] = useState(0);
|
||||
const [removeTarget, setRemoveTarget] = useState<api.AdminDeletedVideo | null>(null);
|
||||
const [removing, setRemoving] = useState(false);
|
||||
const pageSize = useVideosPageSize();
|
||||
const { show } = useToast();
|
||||
|
||||
async function refresh() {
|
||||
setLoading(true);
|
||||
setLoadError("");
|
||||
try {
|
||||
const [r, driveList] = await Promise.all([
|
||||
api.listBlacklist({ page, size: pageSize, keyword: searchKeyword }),
|
||||
api.listDrives(),
|
||||
]);
|
||||
setList(r.items ?? []);
|
||||
setTotal(r.total ?? 0);
|
||||
setDrives(driveList ?? []);
|
||||
} catch (e) {
|
||||
const message = e instanceof Error ? e.message : "加载失败";
|
||||
setLoadError(message);
|
||||
show(message, "error");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
refresh();
|
||||
}, [page, searchKeyword, pageSize]);
|
||||
|
||||
useEffect(() => {
|
||||
setPage(1);
|
||||
}, [pageSize]);
|
||||
|
||||
useEffect(() => {
|
||||
if (keyword === searchKeyword) return;
|
||||
const timer = window.setTimeout(() => {
|
||||
setSearchKeyword(keyword);
|
||||
setPage(1);
|
||||
}, 300);
|
||||
return () => window.clearTimeout(timer);
|
||||
}, [keyword]);
|
||||
|
||||
const driveNameMap = new Map(drives.map((d) => [d.id, d.name || d.id]));
|
||||
const totalPages = Math.max(1, Math.ceil(total / pageSize));
|
||||
|
||||
async function confirmRemove() {
|
||||
if (!removeTarget) return;
|
||||
const target = removeTarget;
|
||||
setRemoving(true);
|
||||
try {
|
||||
await api.removeBlacklist(target.id);
|
||||
setRemoveTarget(null);
|
||||
show("已移出黑名单,下次扫盘会重新入库", "success");
|
||||
onStatsChanged();
|
||||
if (list.length === 1 && page > 1) {
|
||||
setPage((p) => Math.max(1, p - 1));
|
||||
} else {
|
||||
refresh();
|
||||
}
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "操作失败", "error");
|
||||
} finally {
|
||||
setRemoving(false);
|
||||
}
|
||||
}
|
||||
|
||||
function handleSearchSubmit(e: React.FormEvent) {
|
||||
e.preventDefault();
|
||||
setSearchKeyword(keyword);
|
||||
setPage(1);
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="admin-tab-intro">
|
||||
被删除和被隐藏的视频会进入黑名单,扫盘时不再重新入库。这里只保留文件名等基本信息(原始记录、封面、预览已删除)。移出黑名单后,视频会在下次扫盘时被重新发现并入库
|
||||
</div>
|
||||
<div className="admin-page__actions admin-videos-filter">
|
||||
<SearchBox keyword={keyword} onChange={setKeyword} onSubmit={handleSearchSubmit} placeholder="搜索文件名" />
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 刷新
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{loading ? (
|
||||
<LoadingState />
|
||||
) : loadError ? (
|
||||
<ErrorState message={loadError} onRetry={refresh} />
|
||||
) : list.length === 0 ? (
|
||||
<div className="admin-empty-state">
|
||||
<div className="admin-empty-state__icon">
|
||||
<Ban size={48} />
|
||||
</div>
|
||||
<div className="admin-empty-state__text">黑名单为空。</div>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<div className="admin-videos-list-toolbar">
|
||||
<div className="admin-videos-summary">共 {total} 个拉黑视频</div>
|
||||
</div>
|
||||
<table className="admin-table admin-blacklist-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>文件名</th>
|
||||
<th>来源</th>
|
||||
<th>大小</th>
|
||||
<th>拉黑时间</th>
|
||||
<th className="is-actions">操作</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{list.map((v) => (
|
||||
<tr key={v.id}>
|
||||
<td data-label="文件名">
|
||||
<span className="admin-blacklist-filename">{v.fileName || <span className="admin-text-faint">(无文件名)</span>}</span>
|
||||
</td>
|
||||
<td data-label="来源" className="admin-mono-cell">
|
||||
{driveNameMap.get(v.driveId) ?? v.driveId}
|
||||
</td>
|
||||
<td data-label="大小">{v.size > 0 ? formatBytes(v.size) : <span className="admin-text-faint">—</span>}</td>
|
||||
<td data-label="拉黑时间">{formatDateTime(v.deletedAt)}</td>
|
||||
<td className="is-actions" data-label="操作">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn admin-blacklist-restore-btn"
|
||||
onClick={() => setRemoveTarget(v)}
|
||||
title="移出黑名单"
|
||||
>
|
||||
<RotateCcw size={13} /> 移出黑名单
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
<Pagination page={page} totalPages={totalPages} pageSize={pageSize} onPage={setPage} />
|
||||
</>
|
||||
)}
|
||||
|
||||
<ConfirmModal
|
||||
open={removeTarget !== null}
|
||||
title="移出黑名单"
|
||||
message={
|
||||
removeTarget
|
||||
? `确定把「${removeTarget.fileName || removeTarget.id}」移出黑名单吗?移出后它会在下次扫盘时被重新发现并入库。`
|
||||
: ""
|
||||
}
|
||||
confirmText="移出黑名单"
|
||||
centerMessage
|
||||
loading={removing}
|
||||
onCancel={() => {
|
||||
if (!removing) setRemoveTarget(null);
|
||||
}}
|
||||
onConfirm={confirmRemove}
|
||||
/>
|
||||
</section>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- 共享小组件 ----------
|
||||
|
||||
function DriveFilter({
|
||||
drives,
|
||||
driveId,
|
||||
onChange,
|
||||
withCounts = false,
|
||||
}: {
|
||||
drives: api.AdminDrive[];
|
||||
driveId: string;
|
||||
onChange: (id: string) => void;
|
||||
withCounts?: boolean;
|
||||
}) {
|
||||
return (
|
||||
<div className="admin-videos-filter__select-wrap">
|
||||
<select
|
||||
className="admin-videos-filter__select"
|
||||
value={driveId}
|
||||
onChange={(e) => onChange(e.target.value)}
|
||||
>
|
||||
<option value="">全部网盘</option>
|
||||
{drives.map((d) => (
|
||||
<option key={d.id} value={d.id}>
|
||||
{d.name || d.id}
|
||||
{withCounts ? `(已生成 ${d.teaserReadyCount ?? 0},待生成 ${d.teaserPendingCount ?? 0})` : ""}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<ChevronDown size={15} className="admin-videos-filter__select-icon" aria-hidden="true" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function SearchBox({
|
||||
keyword,
|
||||
onChange,
|
||||
onSubmit,
|
||||
placeholder = "搜索标题 / 作者",
|
||||
}: {
|
||||
keyword: string;
|
||||
onChange: (v: string) => void;
|
||||
onSubmit: (e: React.FormEvent) => void;
|
||||
placeholder?: string;
|
||||
}) {
|
||||
return (
|
||||
<form className="admin-videos-filter__search" onSubmit={onSubmit}>
|
||||
<Search size={14} className="admin-videos-filter__search-icon" />
|
||||
<input
|
||||
aria-label={placeholder}
|
||||
value={keyword}
|
||||
onChange={(e) => onChange(e.target.value)}
|
||||
placeholder={placeholder}
|
||||
/>
|
||||
</form>
|
||||
);
|
||||
}
|
||||
|
||||
function Pagination({
|
||||
page,
|
||||
totalPages,
|
||||
pageSize,
|
||||
onPage,
|
||||
}: {
|
||||
page: number;
|
||||
totalPages: number;
|
||||
pageSize: number;
|
||||
onPage: React.Dispatch<React.SetStateAction<number>>;
|
||||
}) {
|
||||
return (
|
||||
<div className="admin-table-pagination">
|
||||
<button type="button" className="admin-btn" onClick={() => onPage(() => 1)} disabled={page <= 1}>
|
||||
首页
|
||||
</button>
|
||||
<button type="button" className="admin-btn" onClick={() => onPage((p) => Math.max(1, p - 1))} disabled={page <= 1}>
|
||||
上一页
|
||||
</button>
|
||||
<span className="admin-table-pagination__info">
|
||||
第 {page} / {totalPages} 页,每页 {pageSize} 个
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => onPage((p) => Math.min(totalPages, p + 1))}
|
||||
disabled={page >= totalPages}
|
||||
>
|
||||
下一页
|
||||
</button>
|
||||
<button type="button" className="admin-btn" onClick={() => onPage(() => totalPages)} disabled={page >= totalPages}>
|
||||
末页
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function LoadingState() {
|
||||
return (
|
||||
<div className="admin-loading-state">
|
||||
<RefreshCw size={20} className="admin-spin" />
|
||||
<span>加载中...</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ErrorState({ message, onRetry }: { message: string; onRetry: () => void }) {
|
||||
return (
|
||||
<div className="admin-error-state">
|
||||
<strong>加载失败</strong>
|
||||
<span>{message}</span>
|
||||
<button type="button" className="admin-btn" onClick={onRetry}>
|
||||
<RefreshCw size={13} /> 重试
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function DeleteSourceOption({
|
||||
checked,
|
||||
disabled,
|
||||
onChange,
|
||||
note,
|
||||
}: {
|
||||
checked: boolean;
|
||||
disabled: boolean;
|
||||
onChange: (v: boolean) => void;
|
||||
note: string;
|
||||
}) {
|
||||
return (
|
||||
<label className="admin-delete-source-option">
|
||||
<input type="checkbox" checked={checked} disabled={disabled} onChange={(e) => onChange(e.target.checked)} />
|
||||
<span>
|
||||
<strong>同时删除网盘中的源文件</strong>
|
||||
<small>{note}</small>
|
||||
</span>
|
||||
</label>
|
||||
);
|
||||
}
|
||||
|
||||
function VideoTitleCell({ video: v }: { video: api.AdminVideo }) {
|
||||
return (
|
||||
<div className="admin-video-title-cell">
|
||||
<div className="admin-video-thumb-wrap" aria-hidden="true">
|
||||
{v.thumbnailUrl ? (
|
||||
<img className="admin-video-thumb" src={v.thumbnailUrl} alt="" />
|
||||
) : (
|
||||
<div className="admin-video-thumb-placeholder">
|
||||
<Image size={14} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="admin-video-title-body">
|
||||
<div className="admin-video-title">{v.title}</div>
|
||||
{fileMeta(v) && <div className="admin-video-filemeta">{fileMeta(v)}</div>}
|
||||
{(v.tags ?? []).length > 0 && (
|
||||
<div className="admin-pills admin-video-title-tags">
|
||||
{(v.tags ?? []).map((t) => (
|
||||
<span key={t} className="admin-pill">
|
||||
{t}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<VideoFileMetaPills video={v} />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -484,11 +850,7 @@ function VideoFileMetaPills({ video }: { video: api.AdminVideo }) {
|
||||
{part}
|
||||
</span>
|
||||
))}
|
||||
{category && (
|
||||
<span className="admin-video-filemeta-pill is-category">
|
||||
{category}
|
||||
</span>
|
||||
)}
|
||||
{category && <span className="admin-video-filemeta-pill is-category">{category}</span>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -500,11 +862,17 @@ function formatDur(sec: number): string {
|
||||
return `${m.toString().padStart(2, "0")}:${s.toString().padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
function formatDateTime(ms: number): string {
|
||||
if (!ms) return "—";
|
||||
const d = new Date(ms);
|
||||
if (Number.isNaN(d.getTime())) return "—";
|
||||
const pad = (n: number) => n.toString().padStart(2, "0");
|
||||
return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}`;
|
||||
}
|
||||
|
||||
function useVideosPageSize() {
|
||||
const [pageSize, setPageSize] = useState(() =>
|
||||
window.matchMedia(VIDEOS_MOBILE_QUERY).matches
|
||||
? MOBILE_VIDEOS_PAGE_SIZE
|
||||
: DESKTOP_VIDEOS_PAGE_SIZE
|
||||
window.matchMedia(VIDEOS_MOBILE_QUERY).matches ? MOBILE_VIDEOS_PAGE_SIZE : DESKTOP_VIDEOS_PAGE_SIZE
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -638,12 +1006,12 @@ function EditVideoModal({
|
||||
<div className="admin-thumbnail-preview">
|
||||
<input id={`${idPrefix}-video-thumbnail`} value={thumbnail} onChange={(e) => setThumbnail(e.target.value)} />
|
||||
{thumbnail && (
|
||||
<img
|
||||
src={thumbnail}
|
||||
alt="封面预览"
|
||||
className="admin-thumbnail-img"
|
||||
onError={(e) => (e.currentTarget.style.display = 'none')}
|
||||
onLoad={(e) => (e.currentTarget.style.display = 'block')}
|
||||
<img
|
||||
src={thumbnail}
|
||||
alt="封面预览"
|
||||
className="admin-thumbnail-img"
|
||||
onError={(e) => (e.currentTarget.style.display = "none")}
|
||||
onLoad={(e) => (e.currentTarget.style.display = "block")}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
@@ -685,11 +1053,7 @@ function fileMeta(v: api.AdminVideo): string {
|
||||
}
|
||||
|
||||
function fileMetaParts(v: api.AdminVideo): string[] {
|
||||
return [
|
||||
normalizeExt(v.ext),
|
||||
v.quality,
|
||||
v.size > 0 ? formatBytes(v.size) : "",
|
||||
].filter(Boolean);
|
||||
return [normalizeExt(v.ext), v.quality, v.size > 0 ? formatBytes(v.size) : ""].filter(Boolean);
|
||||
}
|
||||
|
||||
function normalizeExt(ext: string): string {
|
||||
@@ -705,7 +1069,5 @@ function splitList(s: string): string[] {
|
||||
}
|
||||
|
||||
function toggleTag(tags: string[], label: string): string[] {
|
||||
return tags.includes(label)
|
||||
? tags.filter((tag) => tag !== label)
|
||||
: [...tags, label];
|
||||
return tags.includes(label) ? tags.filter((tag) => tag !== label) : [...tags, label];
|
||||
}
|
||||
|
||||
+246
-11
@@ -12,13 +12,14 @@ async function request<T>(
|
||||
path: string,
|
||||
init: RequestInit = {}
|
||||
): Promise<T> {
|
||||
const headers = new Headers(init.headers ?? {});
|
||||
if (!(init.body instanceof FormData) && !headers.has("Content-Type")) {
|
||||
headers.set("Content-Type", "application/json");
|
||||
}
|
||||
const res = await fetch(BASE + path, {
|
||||
credentials: "include",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...(init.headers ?? {}),
|
||||
},
|
||||
...init,
|
||||
headers,
|
||||
});
|
||||
if (res.status === 401) {
|
||||
throw new UnauthorizedError();
|
||||
@@ -95,6 +96,11 @@ export type AdminDrive = {
|
||||
lastCrawlAt?: number;
|
||||
// spider91 专用代理地址;仅后台管理接口返回,用于编辑表单回显。
|
||||
spider91Proxy?: string;
|
||||
// Google Drive 是否使用 OpenList 在线续期 API;未配置时后端按 true 返回。
|
||||
googleDriveUseOnlineAPI?: boolean;
|
||||
// localstorage 的 .strm 是否允许指向存储根目录之外;未配置时后端按 false 返回。
|
||||
strmAllowOutsideRoot?: boolean;
|
||||
scanGenerationStatus?: DriveGenerationStatus;
|
||||
thumbnailGenerationStatus?: DriveGenerationStatus;
|
||||
previewGenerationStatus?: DriveGenerationStatus;
|
||||
fingerprintGenerationStatus?: DriveGenerationStatus;
|
||||
@@ -108,6 +114,12 @@ export type AdminDrive = {
|
||||
fingerprintReadyCount: number;
|
||||
fingerprintPendingCount: number;
|
||||
fingerprintFailedCount: number;
|
||||
// 浏览器兼容性转码:候选(待处理)/已转码/失败/检测后无需转码 计数与任务状态。
|
||||
transcodeGenerationStatus?: DriveGenerationStatus;
|
||||
transcodePendingCount: number;
|
||||
transcodeReadyCount: number;
|
||||
transcodeFailedCount: number;
|
||||
transcodeSkippedCount: number;
|
||||
};
|
||||
|
||||
export type DriveGenerationStatus = {
|
||||
@@ -115,6 +127,10 @@ export type DriveGenerationStatus = {
|
||||
currentTitle?: string;
|
||||
queueLength: number;
|
||||
cooldownUntil?: string;
|
||||
scannedCount: number;
|
||||
addedCount: number;
|
||||
doneCount: number;
|
||||
totalCount: number;
|
||||
};
|
||||
|
||||
export function listDrives() {
|
||||
@@ -170,7 +186,7 @@ export function deleteDrive(id: string, body: DeleteDriveInput) {
|
||||
}
|
||||
|
||||
export function rescan(id: string) {
|
||||
return request<{ ok: boolean }>(
|
||||
return request<{ ok: boolean; accepted: boolean; message?: string; status?: NightlyJobStatus }>(
|
||||
`/drives/${encodeURIComponent(id)}/rescan`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
@@ -183,6 +199,134 @@ export function stopDriveTasks(id: string) {
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- Crawlers ----------
|
||||
|
||||
export type AdminCrawler = {
|
||||
id: string;
|
||||
name: string;
|
||||
kind: "scriptcrawler" | "spider91";
|
||||
status: string;
|
||||
lastError?: string;
|
||||
scriptPath: string;
|
||||
scriptSourceUrl?: string;
|
||||
proxy?: string;
|
||||
targetNew?: string;
|
||||
uploadDriveId?: string;
|
||||
lastCrawlAt?: number;
|
||||
scanGenerationStatus?: DriveGenerationStatus;
|
||||
thumbnailGenerationStatus?: DriveGenerationStatus;
|
||||
previewGenerationStatus?: DriveGenerationStatus;
|
||||
fingerprintGenerationStatus?: DriveGenerationStatus;
|
||||
uploadGenerationStatus?: DriveGenerationStatus;
|
||||
thumbnailReadyCount: number;
|
||||
thumbnailPendingCount: number;
|
||||
thumbnailFailedCount: number;
|
||||
teaserReadyCount: number;
|
||||
teaserPendingCount: number;
|
||||
teaserFailedCount: number;
|
||||
fingerprintReadyCount: number;
|
||||
fingerprintPendingCount: number;
|
||||
fingerprintFailedCount: number;
|
||||
totalCrawledCount: number;
|
||||
localVideoCount: number;
|
||||
migratedVideoCount: number;
|
||||
};
|
||||
|
||||
export type UpsertCrawlerInput = {
|
||||
id?: string;
|
||||
scriptPath: string;
|
||||
scriptSourceUrl?: string;
|
||||
proxy?: string;
|
||||
targetNew?: string;
|
||||
uploadDriveId?: string;
|
||||
};
|
||||
|
||||
export type ImportCrawlerScriptResult = {
|
||||
scriptPath: string;
|
||||
name: string;
|
||||
sourceUrl?: string;
|
||||
};
|
||||
|
||||
export type CrawlerDryRunItem = {
|
||||
title: string;
|
||||
sourceId?: string;
|
||||
mediaUrl?: string;
|
||||
mediaLocalFile?: string;
|
||||
thumbnailUrl?: string;
|
||||
detailUrl?: string;
|
||||
};
|
||||
|
||||
export type CrawlerDryRunMediaCheck = {
|
||||
ok: boolean;
|
||||
status?: number;
|
||||
contentType?: string;
|
||||
contentLengthBytes?: number;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
export type CrawlerDryRunResult = {
|
||||
ok: boolean;
|
||||
items: CrawlerDryRunItem[];
|
||||
mediaCheck?: CrawlerDryRunMediaCheck;
|
||||
error?: string;
|
||||
log?: string[];
|
||||
durationMs: number;
|
||||
};
|
||||
|
||||
export function listCrawlers() {
|
||||
return request<AdminCrawler[]>("/crawlers");
|
||||
}
|
||||
|
||||
export function upsertCrawler(body: UpsertCrawlerInput) {
|
||||
return request<{ ok: boolean; id: string; warning?: string }>("/crawlers", {
|
||||
method: "POST",
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
export function importCrawlerScriptFile(file: File) {
|
||||
const form = new FormData();
|
||||
form.append("file", file);
|
||||
return request<ImportCrawlerScriptResult>("/crawlers/import-file", {
|
||||
method: "POST",
|
||||
body: form,
|
||||
});
|
||||
}
|
||||
|
||||
export function importCrawlerScriptURL(url: string) {
|
||||
return request<ImportCrawlerScriptResult>("/crawlers/import-url", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ url }),
|
||||
});
|
||||
}
|
||||
|
||||
export function testCrawlerScript(body: { scriptPath: string; proxy?: string }) {
|
||||
return request<CrawlerDryRunResult>("/crawlers/test-script", {
|
||||
method: "POST",
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
export function runCrawler(id: string) {
|
||||
return request<{ ok: boolean; accepted: boolean; message?: string; status?: NightlyJobStatus }>(
|
||||
`/crawlers/${encodeURIComponent(id)}/run`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
export function stopCrawlerTasks(id: string) {
|
||||
return request<{ ok: boolean; stopped: boolean }>(
|
||||
`/crawlers/${encodeURIComponent(id)}/tasks/stop`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
export function deleteCrawler(id: string) {
|
||||
return request<{ ok: boolean; deletedVideos: number; deletedScript?: boolean; warning?: string }>(`/crawlers/${encodeURIComponent(id)}`, {
|
||||
method: "DELETE",
|
||||
});
|
||||
}
|
||||
|
||||
export type P123QRSession = {
|
||||
loginUuid: string;
|
||||
uniID: string;
|
||||
@@ -210,6 +354,28 @@ export function getP123QRStatus(uniID: string, loginUuid: string) {
|
||||
);
|
||||
}
|
||||
|
||||
export type WopanQRSession = {
|
||||
uuid: string;
|
||||
qrImageDataUrl: string;
|
||||
expiresAt?: string;
|
||||
};
|
||||
|
||||
export type WopanQRStatus = {
|
||||
state: number;
|
||||
statusText: string;
|
||||
accessToken?: string;
|
||||
refreshToken?: string;
|
||||
familyID?: string;
|
||||
};
|
||||
|
||||
export function startWopanQRLogin() {
|
||||
return request<WopanQRSession>("/drives/wopan/qr", { method: "POST" });
|
||||
}
|
||||
|
||||
export function getWopanQRStatus(uuid: string) {
|
||||
return request<WopanQRStatus>(`/drives/wopan/qr/${encodeURIComponent(uuid)}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* 切换某个云盘的预览视频生成开关。点击网盘列表里行内的 toggle 按钮时调用。
|
||||
*
|
||||
@@ -291,6 +457,26 @@ export function regenFailedFingerprints(id: string) {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 手动开启某存储的浏览器兼容性转码(AVI/WMV 等浏览器播不动的视频转 H.264 MP4,
|
||||
* 产物上传回同一存储)。转码默认关闭、从不自动运行,这是唯一入口;
|
||||
* 任务处理完候选列表后自然结束。
|
||||
*/
|
||||
export function startDriveTranscode(id: string) {
|
||||
return request<{ ok: boolean; accepted: boolean; message?: string }>(
|
||||
`/drives/${encodeURIComponent(id)}/transcode/start`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
/** 手动停止某存储正在进行的转码任务。 */
|
||||
export function stopDriveTranscode(id: string) {
|
||||
return request<{ ok: boolean; stopped: boolean }>(
|
||||
`/drives/${encodeURIComponent(id)}/transcode/stop`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- Videos ----------
|
||||
|
||||
export type AdminVideo = {
|
||||
@@ -324,7 +510,9 @@ export type AdminVideoList = {
|
||||
size: number;
|
||||
};
|
||||
|
||||
export function listVideos(params: { driveId?: string; page?: number; size?: number; keyword?: string } = {}) {
|
||||
export function listVideos(
|
||||
params: { driveId?: string; page?: number; size?: number; keyword?: string } = {}
|
||||
) {
|
||||
const qs = new URLSearchParams();
|
||||
if (params.driveId) qs.set("driveId", params.driveId);
|
||||
if (params.page) qs.set("page", String(params.page));
|
||||
@@ -334,6 +522,50 @@ export function listVideos(params: { driveId?: string; page?: number; size?: num
|
||||
return request<AdminVideoList>(`/videos${suffix}`);
|
||||
}
|
||||
|
||||
// 后台视频管理两个标签页的计数。
|
||||
export type VideoStats = {
|
||||
current: number;
|
||||
blacklisted: number;
|
||||
};
|
||||
|
||||
export function getVideoStats() {
|
||||
return request<VideoStats>("/videos/stats");
|
||||
}
|
||||
|
||||
// 黑名单(被拉黑/手动删除、扫盘不再入库的视频)。原始记录已删除,
|
||||
// 只剩文件名/来源盘/大小/拉黑时间。
|
||||
export type AdminDeletedVideo = {
|
||||
id: string;
|
||||
driveId: string;
|
||||
fileId: string;
|
||||
fileName: string;
|
||||
size: number;
|
||||
deletedAt: number;
|
||||
};
|
||||
|
||||
export type AdminBlacklistList = {
|
||||
items: AdminDeletedVideo[];
|
||||
total: number;
|
||||
page: number;
|
||||
size: number;
|
||||
};
|
||||
|
||||
export function listBlacklist(params: { page?: number; size?: number; keyword?: string } = {}) {
|
||||
const qs = new URLSearchParams();
|
||||
if (params.page) qs.set("page", String(params.page));
|
||||
if (params.size) qs.set("size", String(params.size));
|
||||
if (params.keyword) qs.set("keyword", params.keyword);
|
||||
const suffix = qs.toString() ? `?${qs.toString()}` : "";
|
||||
return request<AdminBlacklistList>(`/blacklist${suffix}`);
|
||||
}
|
||||
|
||||
// 把视频移出黑名单(删除墓碑),下次扫盘会重新入库。
|
||||
export function removeBlacklist(id: string) {
|
||||
return request<{ ok: boolean }>(`/blacklist/${encodeURIComponent(id)}`, {
|
||||
method: "DELETE",
|
||||
});
|
||||
}
|
||||
|
||||
export type UpdateVideoInput = Partial<{
|
||||
title: string;
|
||||
author: string;
|
||||
@@ -353,10 +585,13 @@ export function updateVideo(id: string, body: UpdateVideoInput) {
|
||||
});
|
||||
}
|
||||
|
||||
export function deleteVideo(id: string) {
|
||||
export function deleteVideo(id: string, options: { deleteSource?: boolean } = {}) {
|
||||
return request<{ ok: boolean; deletedSource: boolean }>(
|
||||
`/videos/${encodeURIComponent(id)}`,
|
||||
{ method: "DELETE" }
|
||||
{
|
||||
method: "DELETE",
|
||||
body: JSON.stringify({ deleteSource: !!options.deleteSource }),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@@ -402,9 +637,9 @@ export type Theme = "dark" | "pink";
|
||||
export type Settings = {
|
||||
theme: Theme;
|
||||
/**
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115、p123 或 onedrive drive)。
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115、p123、onedrive、googledrive 或 wopan drive)。
|
||||
* - 空字符串:本地保存,不上传到云盘。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, p123, onedrive}。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, p123, onedrive, googledrive, wopan}。
|
||||
*/
|
||||
spider91UploadDriveId: string;
|
||||
};
|
||||
@@ -448,7 +683,7 @@ export function getNightlyJobStatus() {
|
||||
}
|
||||
|
||||
export function runNightlyJob() {
|
||||
return request<{ ok: boolean; accepted: boolean; status: NightlyJobStatus }>(
|
||||
return request<{ ok: boolean; accepted: boolean; status: NightlyJobStatus; message?: string }>(
|
||||
"/jobs/nightly/run",
|
||||
{ method: "POST" }
|
||||
);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { PlayCircle, Power, PowerOff, RotateCcw } from "lucide-react";
|
||||
import { CircleStop, PlayCircle, Power, PowerOff, RotateCcw, Wand2 } from "lucide-react";
|
||||
import * as api from "../api";
|
||||
import { formatBytes } from "../storageFormat";
|
||||
import {
|
||||
@@ -101,13 +101,17 @@ export function StatusTag({
|
||||
error?: string;
|
||||
hasCred: boolean;
|
||||
}) {
|
||||
if (kind === "spider91") {
|
||||
return (
|
||||
<span className="admin-status is-error" title={error || "请到爬虫管理添加爬虫脚本"}>
|
||||
已废弃
|
||||
</span>
|
||||
);
|
||||
}
|
||||
if (kind !== "spider91" && !hasCred) {
|
||||
return <span className="admin-status is-pending">未配置凭证</span>;
|
||||
}
|
||||
if (status === "ok") {
|
||||
if (kind === "spider91") {
|
||||
return <span className="admin-status is-ok">已就绪</span>;
|
||||
}
|
||||
return <span className="admin-status is-ok">已连接</span>;
|
||||
}
|
||||
if (status === "error")
|
||||
@@ -159,20 +163,26 @@ export function DriveGenerationPanel({
|
||||
regenFailedThumbId,
|
||||
regenFailedFingerprintId,
|
||||
togglingTeaserId,
|
||||
togglingTranscodeId,
|
||||
onToggleTeaser,
|
||||
onRegenFailed,
|
||||
onRegenFailedThumbnails,
|
||||
onRegenFailedFingerprints,
|
||||
onStartTranscode,
|
||||
onStopTranscode,
|
||||
}: {
|
||||
d: api.AdminDrive;
|
||||
regenFailedId: string;
|
||||
regenFailedThumbId: string;
|
||||
regenFailedFingerprintId: string;
|
||||
togglingTeaserId: string;
|
||||
togglingTranscodeId: string;
|
||||
onToggleTeaser: () => void;
|
||||
onRegenFailed: () => void;
|
||||
onRegenFailedThumbnails: () => void;
|
||||
onRegenFailedFingerprints: () => void;
|
||||
onStartTranscode: () => void;
|
||||
onStopTranscode: () => void;
|
||||
}) {
|
||||
const canQueueThumbnails =
|
||||
(d.thumbnailFailedCount ?? 0) > 0 ||
|
||||
@@ -182,6 +192,12 @@ export function DriveGenerationPanel({
|
||||
(d.teaserFailedCount ?? 0) > 0 || (d.teaserPendingCount ?? 0) > 0;
|
||||
const canQueueFingerprints =
|
||||
(d.fingerprintFailedCount ?? 0) > 0 || (d.fingerprintPendingCount ?? 0) > 0;
|
||||
// 转码默认不运行,只能在这里手动开启/停止。
|
||||
// 候选 = 还没出结果的不兼容格式视频 + 上次失败的(重新开始会自动重试)。
|
||||
const transcodeRunning =
|
||||
(d.transcodeGenerationStatus?.state || "idle") !== "idle";
|
||||
const canStartTranscode =
|
||||
(d.transcodePendingCount ?? 0) > 0 || (d.transcodeFailedCount ?? 0) > 0;
|
||||
|
||||
return (
|
||||
<div className="admin-detail-card">
|
||||
@@ -204,6 +220,11 @@ export function DriveGenerationPanel({
|
||||
</header>
|
||||
|
||||
<div className="admin-gen-columns">
|
||||
<DriveGenCol
|
||||
label={d.kind === "spider91" ? "已废弃" : "扫盘"}
|
||||
status={d.scanGenerationStatus}
|
||||
showCounts={false}
|
||||
/>
|
||||
<DriveGenCol
|
||||
label="封面"
|
||||
status={d.thumbnailGenerationStatus}
|
||||
@@ -226,6 +247,13 @@ export function DriveGenerationPanel({
|
||||
pending={d.fingerprintPendingCount}
|
||||
failed={d.fingerprintFailedCount}
|
||||
/>
|
||||
<DriveGenCol
|
||||
label="转码"
|
||||
status={d.transcodeGenerationStatus}
|
||||
ready={d.transcodeReadyCount}
|
||||
pending={d.transcodePendingCount}
|
||||
failed={d.transcodeFailedCount}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="admin-detail-actions">
|
||||
@@ -253,6 +281,33 @@ export function DriveGenerationPanel({
|
||||
<RotateCcw size={13} />
|
||||
<span>{(d.fingerprintFailedCount ?? 0) > 0 ? "重试失败指纹" : "继续生成指纹"}</span>
|
||||
</button>
|
||||
{transcodeRunning ? (
|
||||
<button
|
||||
className="admin-btn is-stop"
|
||||
disabled={togglingTranscodeId === d.id}
|
||||
onClick={onStopTranscode}
|
||||
title="停止当前的转码任务。未处理的视频保持原状态,下次开始时继续。"
|
||||
>
|
||||
<CircleStop size={13} />
|
||||
<span>{togglingTranscodeId === d.id ? "停止中..." : "停止转码"}</span>
|
||||
</button>
|
||||
) : (
|
||||
<button
|
||||
className="admin-btn"
|
||||
disabled={!canStartTranscode || togglingTranscodeId === d.id}
|
||||
onClick={onStartTranscode}
|
||||
title="把浏览器播放不了的视频(AVI/WMV/RMVB、MPEG-4 等老格式)转码成 H.264 MP4 并上传回本存储。转码不会自动运行,只能在这里手动开启。"
|
||||
>
|
||||
<Wand2 size={13} />
|
||||
<span>
|
||||
{togglingTranscodeId === d.id
|
||||
? "开启中..."
|
||||
: (d.transcodeFailedCount ?? 0) > 0 && (d.transcodePendingCount ?? 0) === 0
|
||||
? "重试失败转码"
|
||||
: "开始转码"}
|
||||
</span>
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
@@ -265,6 +320,7 @@ function DriveGenCol({
|
||||
pending,
|
||||
failed,
|
||||
extra,
|
||||
showCounts = true,
|
||||
}: {
|
||||
label: string;
|
||||
status?: api.DriveGenerationStatus;
|
||||
@@ -272,10 +328,14 @@ function DriveGenCol({
|
||||
pending?: number;
|
||||
failed?: number;
|
||||
extra?: number;
|
||||
showCounts?: boolean;
|
||||
}) {
|
||||
const state = status?.state || "idle";
|
||||
const detail = generationDetail(status);
|
||||
const title = generationTitle(status, detail);
|
||||
const stateLabel = label === "抓取" && state === "scanning" ? "抓取中" : generationStateLabel(state);
|
||||
const showScanProgress = !showCounts && (state === "scanning" || (status?.scannedCount ?? 0) > 0 || (status?.addedCount ?? 0) > 0);
|
||||
const scannedLabel = label === "抓取" ? "已抓取" : "已扫描";
|
||||
return (
|
||||
<div className="admin-gen-col">
|
||||
<div className="admin-gen-col__head">
|
||||
@@ -284,18 +344,26 @@ function DriveGenCol({
|
||||
className={`admin-status admin-generation-state is-${generationStateClass(state)}`}
|
||||
title={title || undefined}
|
||||
>
|
||||
{generationStateLabel(state)}
|
||||
{stateLabel}
|
||||
</span>
|
||||
</div>
|
||||
{detail && <div className="admin-gen-col__detail">{detail}</div>}
|
||||
<div className="admin-gen-col__counts">
|
||||
<div className="admin-gen-col__count"><span>就绪</span><strong>{ready ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>待生成</span><strong>{pending ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>失败</span><strong>{failed ?? 0}</strong></div>
|
||||
{(extra ?? 0) > 0 && (
|
||||
<div className="admin-gen-col__count"><span>待补时长</span><strong>{extra}</strong></div>
|
||||
)}
|
||||
</div>
|
||||
{showScanProgress && (
|
||||
<div className="admin-gen-col__counts admin-gen-col__counts--scan">
|
||||
<div className="admin-gen-col__count"><span>{scannedLabel}</span><strong>{status?.scannedCount ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>预计新增</span><strong>{status?.addedCount ?? 0}</strong></div>
|
||||
</div>
|
||||
)}
|
||||
{showCounts && (
|
||||
<div className="admin-gen-col__counts">
|
||||
<div className="admin-gen-col__count"><span>就绪</span><strong>{ready ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>待生成</span><strong>{pending ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>失败</span><strong>{failed ?? 0}</strong></div>
|
||||
{(extra ?? 0) > 0 && (
|
||||
<div className="admin-gen-col__count"><span>待补时长</span><strong>{extra}</strong></div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { useId, useMemo, useState } from "react";
|
||||
import { ArrowLeft } from "lucide-react";
|
||||
import { ArrowLeft, ChevronDown } from "lucide-react";
|
||||
import { P123QRCodeLogin } from "./P123QRCodeLogin";
|
||||
import { WopanQRCodeLogin } from "./WopanQRCodeLogin";
|
||||
import { Spider91UploadTargetField } from "./Spider91UploadTargetField";
|
||||
import {
|
||||
FormState,
|
||||
@@ -21,14 +22,13 @@ type DriveOption = {
|
||||
|
||||
const DRIVE_OPTIONS: DriveOption[] = [
|
||||
{ kind: "p115", label: "115 网盘", abbr: "115", desc: "302直链,不占带宽" },
|
||||
{ kind: "p123", label: "123 云盘", abbr: "123", desc: "扫码登录,302直链" },
|
||||
{ kind: "p123", label: "123网盘", abbr: "123", desc: "扫码登录,302直链" },
|
||||
{ kind: "pikpak", label: "PikPak", abbr: "Pk", desc: "302直链,稳定快速" },
|
||||
{ kind: "onedrive", label: "OneDrive", abbr: "OD", desc: "302直链,微软网盘" },
|
||||
{ kind: "googledrive", label: "Google Drive", abbr: "GD", desc: "服务器中转模式" },
|
||||
{ kind: "localstorage", label: "本地存储", abbr: "Lo", desc: "本机文件目录" },
|
||||
{ kind: "spider91", label: "91 爬虫", abbr: "91", desc: "自动抓取热门视频" },
|
||||
{ kind: "quark", label: "夸克网盘", abbr: "Qk", desc: "302直链" },
|
||||
{ kind: "wopan", label: "联通沃盘", abbr: "Wo", desc: "302直链" },
|
||||
{ kind: "wopan", label: "联通网盘", abbr: "Wo", desc: "302直链" },
|
||||
];
|
||||
|
||||
export function DriveForm({
|
||||
@@ -49,7 +49,7 @@ export function DriveForm({
|
||||
onBack?: () => void;
|
||||
}) {
|
||||
const idPrefix = useId();
|
||||
const fields = useMemo(() => credentialFields(form.kind), [form.kind]);
|
||||
const fields = useMemo(() => credentialFields(form.kind, form.creds), [form.kind, form.creds]);
|
||||
const help = credentialHelp(form.kind, isEdit);
|
||||
const [step, setStep] = useState<"type" | "form">(isEdit ? "form" : "type");
|
||||
const nameId = `${idPrefix}-drive-name`;
|
||||
@@ -178,27 +178,71 @@ export function DriveForm({
|
||||
/>
|
||||
)}
|
||||
|
||||
{form.kind === "wopan" && (
|
||||
<WopanQRCodeLogin
|
||||
onCredentials={(credentials) =>
|
||||
onChange({
|
||||
...form,
|
||||
creds: {
|
||||
...form.creds,
|
||||
access_token: credentials.accessToken,
|
||||
refresh_token: credentials.refreshToken,
|
||||
...(credentials.familyID ? { family_id: credentials.familyID } : {}),
|
||||
},
|
||||
})
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{fields.map((f) => (
|
||||
<div key={f.key} className="admin-form__row">
|
||||
<label htmlFor={`${idPrefix}-credential-${f.key}`}>
|
||||
{f.label}
|
||||
{f.required && " *"}
|
||||
</label>
|
||||
{f.multiline ? (
|
||||
<textarea
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
value={form.creds[f.key] ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
placeholder={f.placeholder}
|
||||
/>
|
||||
{f.type === "select" ? (
|
||||
<>
|
||||
<label htmlFor={`${idPrefix}-credential-${f.key}`}>
|
||||
{f.label}
|
||||
{f.required && " *"}
|
||||
</label>
|
||||
<div className="admin-form-select-wrap">
|
||||
<select
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
className="admin-form-select"
|
||||
value={form.creds[f.key] ?? f.defaultValue ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
>
|
||||
{(f.options ?? []).map((option) => (
|
||||
<option key={option.value} value={option.value}>
|
||||
{option.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<ChevronDown size={15} className="admin-form-select__icon" aria-hidden="true" />
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<input
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
type={credentialInputType(f.key)}
|
||||
value={form.creds[f.key] ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
placeholder={f.placeholder}
|
||||
/>
|
||||
<>
|
||||
<label htmlFor={`${idPrefix}-credential-${f.key}`}>
|
||||
{f.label}
|
||||
{f.required && " *"}
|
||||
</label>
|
||||
{f.multiline ? (
|
||||
<textarea
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
value={form.creds[f.key] ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
placeholder={f.placeholder}
|
||||
required={f.required && !isEdit}
|
||||
/>
|
||||
) : (
|
||||
<input
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
type={credentialInputType(f.key)}
|
||||
value={form.creds[f.key] ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
placeholder={f.placeholder}
|
||||
required={f.required && !isEdit}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
{f.help && <div className="admin-form__help">{f.help}</div>}
|
||||
</div>
|
||||
|
||||
@@ -113,11 +113,11 @@ export function P123QRCodeLogin({ onToken }: { onToken: (token: string) => void
|
||||
<img
|
||||
className="admin-p123-qr__image"
|
||||
src={session.qrImageDataUrl}
|
||||
alt="123 云盘扫码登录二维码"
|
||||
alt="123网盘扫码登录二维码"
|
||||
/>
|
||||
<div className="admin-p123-qr__meta">
|
||||
<div className="admin-form__help">
|
||||
使用微信或 123 云盘 App 扫码并确认登录;确认后系统会自动填入 access_token。
|
||||
使用微信或 123网盘 App 扫码并确认登录;确认后系统会自动填入 access_token。
|
||||
</div>
|
||||
{session.expiresAt && (
|
||||
<div className="admin-form__help">
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { QrCode } from "lucide-react";
|
||||
import * as api from "../api";
|
||||
import { useToast } from "../ToastContext";
|
||||
|
||||
function wopanQRStatusClass(
|
||||
status: api.WopanQRStatus | null,
|
||||
completed: boolean,
|
||||
error: string
|
||||
): string {
|
||||
if (completed || status?.state === 3) return "is-ok";
|
||||
if (error || status?.state === 4) return "is-error";
|
||||
return "is-pending";
|
||||
}
|
||||
|
||||
export function WopanQRCodeLogin({
|
||||
onCredentials,
|
||||
}: {
|
||||
onCredentials: (credentials: {
|
||||
accessToken: string;
|
||||
refreshToken: string;
|
||||
familyID?: string;
|
||||
}) => void;
|
||||
}) {
|
||||
const { show } = useToast();
|
||||
const [session, setSession] = useState<api.WopanQRSession | null>(null);
|
||||
const [status, setStatus] = useState<api.WopanQRStatus | null>(null);
|
||||
const [starting, setStarting] = useState(false);
|
||||
const [pollingError, setPollingError] = useState("");
|
||||
const [completed, setCompleted] = useState(false);
|
||||
|
||||
async function start() {
|
||||
setStarting(true);
|
||||
setPollingError("");
|
||||
setCompleted(false);
|
||||
setStatus(null);
|
||||
try {
|
||||
const next = await api.startWopanQRLogin();
|
||||
setSession(next);
|
||||
} catch (e) {
|
||||
setSession(null);
|
||||
show(e instanceof Error ? e.message : "生成二维码失败", "error");
|
||||
} finally {
|
||||
setStarting(false);
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (!session || completed) return;
|
||||
const activeSession = session;
|
||||
let stopped = false;
|
||||
let inFlight = false;
|
||||
let timer: number | undefined;
|
||||
|
||||
async function poll() {
|
||||
if (stopped || inFlight) return;
|
||||
inFlight = true;
|
||||
try {
|
||||
const next = await api.getWopanQRStatus(activeSession.uuid);
|
||||
if (stopped) return;
|
||||
setStatus(next);
|
||||
setPollingError("");
|
||||
if (next.accessToken && next.refreshToken) {
|
||||
stopped = true;
|
||||
if (timer) window.clearInterval(timer);
|
||||
setCompleted(true);
|
||||
onCredentials({
|
||||
accessToken: next.accessToken,
|
||||
refreshToken: next.refreshToken,
|
||||
familyID: next.familyID,
|
||||
});
|
||||
show("扫码成功,已填入 access_token 和 refresh_token,保存后生效", "success");
|
||||
return;
|
||||
}
|
||||
if (next.state === 4) {
|
||||
stopped = true;
|
||||
if (timer) window.clearInterval(timer);
|
||||
}
|
||||
} catch (e) {
|
||||
if (stopped) return;
|
||||
setPollingError(e instanceof Error ? e.message : "查询扫码状态失败");
|
||||
} finally {
|
||||
inFlight = false;
|
||||
}
|
||||
}
|
||||
|
||||
poll();
|
||||
timer = window.setInterval(poll, 1200);
|
||||
return () => {
|
||||
stopped = true;
|
||||
if (timer) window.clearInterval(timer);
|
||||
};
|
||||
}, [session, completed, onCredentials, show]);
|
||||
|
||||
const statusText = completed
|
||||
? "已获取凭证"
|
||||
: pollingError || status?.statusText || (session ? "等待扫码" : "未生成二维码");
|
||||
const statusClass = wopanQRStatusClass(status, completed, pollingError);
|
||||
|
||||
return (
|
||||
<div className="admin-form__row">
|
||||
<label>扫码登录</label>
|
||||
<div className="admin-p123-qr">
|
||||
<div className="admin-p123-qr__actions">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={start}
|
||||
disabled={starting}
|
||||
>
|
||||
<QrCode size={14} />
|
||||
{starting ? "生成中..." : session ? "重新生成二维码" : "生成二维码"}
|
||||
</button>
|
||||
<span className={`admin-status ${statusClass}`}>{statusText}</span>
|
||||
</div>
|
||||
|
||||
{session && (
|
||||
<div className="admin-p123-qr__body">
|
||||
<img
|
||||
className="admin-p123-qr__image"
|
||||
src={session.qrImageDataUrl}
|
||||
alt="联通网盘扫码登录二维码"
|
||||
/>
|
||||
<div className="admin-p123-qr__meta">
|
||||
<div className="admin-form__help">
|
||||
使用联通网盘 App 扫码并确认登录;确认后系统会自动填入 access_token 和 refresh_token。
|
||||
</div>
|
||||
{session.expiresAt && (
|
||||
<div className="admin-form__help">
|
||||
过期时间:{new Date(session.expiresAt).toLocaleTimeString("zh-CN", {
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
{status?.state === 4 && (
|
||||
<div className="admin-form__help">
|
||||
当前二维码已过期,请重新生成。
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -12,12 +12,22 @@ export const kindAbbr: Record<string, string> = {
|
||||
spider91: "91",
|
||||
};
|
||||
|
||||
export function driveKindAbbr(kind: string): string {
|
||||
const explicit = kindAbbr[kind];
|
||||
if (explicit) return explicit;
|
||||
|
||||
const trimmed = kind.trim();
|
||||
if (!trimmed) return "??";
|
||||
const compact = trimmed.replace(/[^a-zA-Z0-9]+/g, "");
|
||||
return (compact || trimmed).slice(0, 2).toUpperCase();
|
||||
}
|
||||
|
||||
export const kindLabel: Record<string, string> = {
|
||||
quark: "夸克网盘",
|
||||
p115: "115 网盘",
|
||||
p123: "123 云盘",
|
||||
p123: "123网盘",
|
||||
pikpak: "PikPak",
|
||||
wopan: "联通沃盘",
|
||||
wopan: "联通网盘",
|
||||
onedrive: "OneDrive",
|
||||
googledrive: "Google Drive",
|
||||
localstorage: "本地存储",
|
||||
@@ -56,12 +66,13 @@ export function nightlyButtonText(status: { running: boolean; queued: boolean },
|
||||
}
|
||||
|
||||
export function nightlyBusyText(status: { running: boolean; queued: boolean }) {
|
||||
if (status.running) return "扫描任务正在运行";
|
||||
if (status.queued) return "扫描任务已排队";
|
||||
if (status.running || status.queued) return "当前有全量扫描任务正在进行,请稍后重试";
|
||||
return "";
|
||||
}
|
||||
|
||||
export function generationStateLabel(state: string): string {
|
||||
if (state === "scanning") return "扫盘中";
|
||||
if (state === "uploading") return "上传中";
|
||||
if (state === "generating") return "生成中";
|
||||
if (state === "cooling") return "冷却中";
|
||||
if (state === "queued") return "排队中";
|
||||
@@ -69,7 +80,8 @@ export function generationStateLabel(state: string): string {
|
||||
}
|
||||
|
||||
export function generationStateClass(state: string): string {
|
||||
if (state === "generating" || state === "cooling" || state === "queued") {
|
||||
if (state === "scanning" || state === "uploading" || state === "generating" || state === "cooling" || state === "queued") {
|
||||
if (state === "scanning" || state === "uploading") return "generating";
|
||||
return state;
|
||||
}
|
||||
return "idle";
|
||||
@@ -138,32 +150,51 @@ export function credentialHelp(kind: Kind, isEdit: boolean): string {
|
||||
case "p115":
|
||||
return `登录 115.com 后复制 Cookie,形如 "UID=...; CID=...; SEID=...; KID=..."。${note}`;
|
||||
case "p123":
|
||||
return `推荐使用扫码登录自动获取 access_token;账号密码登录被 123 云盘风控拦截时,也可以只填写 access_token。播放走 302 跳转到 123 云盘返回的短期 CDN 地址。${note}`;
|
||||
return `推荐使用扫码登录自动获取 access_token;账号密码登录被 123网盘风控拦截时,也可以只填写 access_token。播放走 302 跳转到 123网盘返回的短期 CDN 地址。${note}`;
|
||||
case "pikpak":
|
||||
return `填写 PikPak 账号和密码即可。平台、设备 ID、验证码 token 和 refresh token 会由服务端自动处理并保存。${note}`;
|
||||
case "wopan":
|
||||
return `需要 access_token 和 refresh_token。后续会加扫码/短信登录入口,第一版只能手工粘贴。${note}`;
|
||||
return `推荐使用扫码登录自动获取 access_token 和 refresh_token;也可以手工粘贴已有凭证。${note}`;
|
||||
case "onedrive":
|
||||
return `按 OpenList 默认应用在线挂载,只需要 refresh_token;保存时会自动刷新并保存 token。${note}`;
|
||||
case "googledrive":
|
||||
return `按 OpenList 在线 API 挂载,只需要 Google Drive refresh_token;保存时会自动刷新并保存 token。播放不走 302,会由后端带 Authorization 代理转发。${note}`;
|
||||
return isEdit
|
||||
? "请参考OpenList文档中关于谷歌云盘的配置方法;如不修改凭证,留空即可,保存时会沿用旧值"
|
||||
: "请参考OpenList文档中关于谷歌云盘的配置方法";
|
||||
case "localstorage":
|
||||
return `填写服务器可访问的本地目录绝对路径,例如 /mnt/videos。系统会扫描该目录及子目录中的视频文件和 .strm 文件;.strm 可指向 HTTP/HTTPS 直链,或指向本地存储根目录内的真实视频路径。Docker 部署时请填写容器内路径。${note}`;
|
||||
return `填写服务器可访问的本地目录绝对路径,例如 /mnt/videos。系统会扫描该目录及子目录中的视频文件和 .strm 文件;.strm 可指向 HTTP/HTTPS 直链或本地视频路径(指向目录外需开启下方开关)。Docker 部署时请填写容器内路径。${note}`;
|
||||
case "spider91":
|
||||
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;可按服务器网络情况单独配置代理。后续流水线会把较早的视频上传到你选择的 115 / PikPak / OneDrive 目标盘。";
|
||||
return "91Spider 不再支持通过网盘添加或编辑。请到后台爬虫管理页面添加爬虫脚本。";
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
export function credentialFields(kind: Kind): Array<{
|
||||
export type CredentialField = {
|
||||
key: string;
|
||||
label: string;
|
||||
placeholder: string;
|
||||
type?: "text" | "select";
|
||||
options?: Array<{ value: string; label: string }>;
|
||||
multiline?: boolean;
|
||||
required?: boolean;
|
||||
defaultValue?: string;
|
||||
help?: string;
|
||||
}> {
|
||||
};
|
||||
|
||||
export function credentialBoolValue(value: string | undefined, defaultValue: boolean): boolean {
|
||||
const normalized = (value ?? "").trim().toLowerCase();
|
||||
if (normalized === "") return defaultValue;
|
||||
if (normalized === "true" || normalized === "1" || normalized === "yes" || normalized === "on") return true;
|
||||
if (normalized === "false" || normalized === "0" || normalized === "no" || normalized === "off") return false;
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
export function googleDriveUsesOnlineAPI(creds: Record<string, string> = {}): boolean {
|
||||
return credentialBoolValue(creds.use_online_api, true);
|
||||
}
|
||||
|
||||
export function credentialFields(kind: Kind, creds: Record<string, string> = {}): CredentialField[] {
|
||||
switch (kind) {
|
||||
case "quark":
|
||||
return [
|
||||
@@ -195,7 +226,7 @@ export function credentialFields(kind: Kind): Array<{
|
||||
{
|
||||
key: "password",
|
||||
label: "密码(可选)",
|
||||
placeholder: "123 云盘密码",
|
||||
placeholder: "123网盘密码",
|
||||
},
|
||||
{
|
||||
key: "access_token",
|
||||
@@ -227,6 +258,7 @@ export function credentialFields(kind: Kind): Array<{
|
||||
label: "access_token",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
help: "扫码成功后会自动填入该字段;如果 token 过期,重新扫码后保存即可。",
|
||||
},
|
||||
{
|
||||
key: "refresh_token",
|
||||
@@ -252,6 +284,17 @@ export function credentialFields(kind: Kind): Array<{
|
||||
];
|
||||
case "googledrive":
|
||||
return [
|
||||
{
|
||||
key: "use_online_api",
|
||||
label: "认证方式",
|
||||
placeholder: "",
|
||||
type: "select",
|
||||
defaultValue: "true",
|
||||
options: [
|
||||
{ value: "true", label: "OpenList 在线 API" },
|
||||
{ value: "false", label: "自建 Google OAuth 客户端" },
|
||||
],
|
||||
},
|
||||
{
|
||||
key: "refresh_token",
|
||||
label: "refresh_token",
|
||||
@@ -259,6 +302,24 @@ export function credentialFields(kind: Kind): Array<{
|
||||
multiline: true,
|
||||
required: true,
|
||||
},
|
||||
...(googleDriveUsesOnlineAPI(creds)
|
||||
? []
|
||||
: [
|
||||
{
|
||||
key: "client_id",
|
||||
label: "客户端 ID",
|
||||
placeholder: "xxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.apps.googleusercontent.com",
|
||||
required: true,
|
||||
help: "Google Cloud Console 中 OAuth 2.0 客户端的 Client ID",
|
||||
},
|
||||
{
|
||||
key: "client_secret",
|
||||
label: "客户端密钥",
|
||||
placeholder: "Google OAuth client secret",
|
||||
required: true,
|
||||
help: "Google Cloud Console 中同一个 OAuth 客户端的 Client Secret",
|
||||
},
|
||||
]),
|
||||
];
|
||||
case "localstorage":
|
||||
return [
|
||||
@@ -269,6 +330,18 @@ export function credentialFields(kind: Kind): Array<{
|
||||
required: true,
|
||||
help: "路径必须是后端服务器上的已有目录;保存后可手动重扫,系统会递归扫描支持的视频格式。",
|
||||
},
|
||||
{
|
||||
key: "strm_allow_outside_root",
|
||||
label: ".strm 允许指向目录外",
|
||||
placeholder: "",
|
||||
type: "select",
|
||||
defaultValue: "false",
|
||||
options: [
|
||||
{ value: "false", label: "关闭(默认,仅允许目录内路径)" },
|
||||
{ value: "true", label: "开启(允许任意本地路径)" },
|
||||
],
|
||||
help: "开启后 .strm 可指向本目录之外的本地文件(如 rclone 挂载点)。注意:等于允许通过 .strm 读取服务器上任意文件,请只在自己完全掌控媒体目录时开启。Docker 部署时路径必须是容器内路径。",
|
||||
},
|
||||
];
|
||||
case "spider91":
|
||||
return [
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
type SpiderIconProps = {
|
||||
size?: number;
|
||||
className?: string;
|
||||
};
|
||||
|
||||
export function SpiderIcon({ size = 16, className }: SpiderIconProps) {
|
||||
return (
|
||||
<svg
|
||||
className={className}
|
||||
width={size}
|
||||
height={size}
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
aria-hidden="true"
|
||||
focusable="false"
|
||||
>
|
||||
<path d="M12 7v3" />
|
||||
<path d="M9 5.5 7.5 3" />
|
||||
<path d="M15 5.5 16.5 3" />
|
||||
<path d="M9 10.5 4.5 8" />
|
||||
<path d="M15 10.5 19.5 8" />
|
||||
<path d="M8.5 13.5 3 13" />
|
||||
<path d="M15.5 13.5 21 13" />
|
||||
<path d="M9 16 5 20" />
|
||||
<path d="M15 16 19 20" />
|
||||
<ellipse cx="12" cy="14" rx="4" ry="5" />
|
||||
<circle cx="12" cy="8" r="2.5" />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { ReactNode } from "react";
|
||||
import { ReactNode, useEffect, useState } from "react";
|
||||
import { TopBar } from "./TopBar";
|
||||
import { MainNav } from "./MainNav";
|
||||
import { SubNav } from "./SubNav";
|
||||
@@ -7,14 +7,84 @@ import { BackToTop } from "./BackToTop";
|
||||
|
||||
type Props = {
|
||||
children: ReactNode;
|
||||
mobileAutoHideNav?: boolean;
|
||||
};
|
||||
|
||||
export function AppShell({ children }: Props) {
|
||||
const MOBILE_NAV_QUERY = "(max-width: 768px)";
|
||||
const SCROLL_DELTA_THRESHOLD = 6;
|
||||
const HIDE_AFTER_SCROLL_Y = 56;
|
||||
|
||||
export function AppShell({ children, mobileAutoHideNav = false }: Props) {
|
||||
const [mobileNavHidden, setMobileNavHidden] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!mobileAutoHideNav) {
|
||||
setMobileNavHidden(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const mediaQuery = window.matchMedia(MOBILE_NAV_QUERY);
|
||||
let lastScrollY = Math.max(window.scrollY, 0);
|
||||
let ticking = false;
|
||||
|
||||
const showNav = () => setMobileNavHidden(false);
|
||||
|
||||
const updateNavVisibility = () => {
|
||||
ticking = false;
|
||||
const currentScrollY = Math.max(window.scrollY, 0);
|
||||
|
||||
if (!mediaQuery.matches || currentScrollY <= 0) {
|
||||
showNav();
|
||||
lastScrollY = currentScrollY;
|
||||
return;
|
||||
}
|
||||
|
||||
const delta = currentScrollY - lastScrollY;
|
||||
if (Math.abs(delta) < SCROLL_DELTA_THRESHOLD) return;
|
||||
|
||||
if (delta > 0 && currentScrollY > HIDE_AFTER_SCROLL_Y) {
|
||||
setMobileNavHidden(true);
|
||||
} else if (delta < 0) {
|
||||
showNav();
|
||||
}
|
||||
|
||||
lastScrollY = currentScrollY;
|
||||
};
|
||||
|
||||
const handleScroll = () => {
|
||||
if (ticking) return;
|
||||
ticking = true;
|
||||
window.requestAnimationFrame(updateNavVisibility);
|
||||
};
|
||||
|
||||
const handleMediaChange = () => {
|
||||
lastScrollY = Math.max(window.scrollY, 0);
|
||||
showNav();
|
||||
};
|
||||
|
||||
handleMediaChange();
|
||||
window.addEventListener("scroll", handleScroll, { passive: true });
|
||||
mediaQuery.addEventListener("change", handleMediaChange);
|
||||
|
||||
return () => {
|
||||
window.removeEventListener("scroll", handleScroll);
|
||||
mediaQuery.removeEventListener("change", handleMediaChange);
|
||||
};
|
||||
}, [mobileAutoHideNav]);
|
||||
|
||||
const className = [
|
||||
"app-shell",
|
||||
mobileAutoHideNav ? "app-shell--mobile-auto-hide-nav" : "",
|
||||
mobileNavHidden ? "is-mobile-nav-hidden" : "",
|
||||
].filter(Boolean).join(" ");
|
||||
|
||||
return (
|
||||
<div className="app-shell">
|
||||
<TopBar />
|
||||
<MainNav />
|
||||
<SubNav />
|
||||
<div className={className}>
|
||||
<div className="app-shell__nav-stack">
|
||||
<TopBar />
|
||||
<MainNav />
|
||||
<SubNav />
|
||||
</div>
|
||||
<main className="app-shell__main">{children}</main>
|
||||
<Footer />
|
||||
<BackToTop />
|
||||
|
||||
@@ -38,13 +38,11 @@ export function RecommendedRail({ videos }: Props) {
|
||||
return (
|
||||
<aside className="vd-rail" aria-label="推荐视频">
|
||||
<header className="vd-rail__head">
|
||||
<span className="vd-rail__head-bar" aria-hidden="true" />
|
||||
<div className="vd-rail__head-text">
|
||||
<h2 className="vd-rail__head-title">推荐视频</h2>
|
||||
<span className="vd-rail__head-sub">
|
||||
根据当前视频 · {videos.length} 条
|
||||
</span>
|
||||
</div>
|
||||
<span className="vd-rail__head-icon" aria-hidden="true">
|
||||
<span />
|
||||
<span />
|
||||
</span>
|
||||
<h2 className="vd-rail__head-title">推荐视频</h2>
|
||||
</header>
|
||||
<ul className="vd-rail__list">
|
||||
{videos.map((v) => (
|
||||
|
||||
@@ -1,35 +1,48 @@
|
||||
import { useState } from "react";
|
||||
import { EyeOff, ThumbsDown, ThumbsUp } from "lucide-react";
|
||||
import { useEffect, useState } from "react";
|
||||
import { ThumbsDown, ThumbsUp, Trash2 } from "lucide-react";
|
||||
import type { VideoDetail } from "@/types";
|
||||
import { formatCount } from "@/lib/format";
|
||||
|
||||
type Props = {
|
||||
video: VideoDetail;
|
||||
onHideVideo: () => void;
|
||||
hideSaving?: boolean;
|
||||
onDeleteVideo: () => void;
|
||||
deleteSaving?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* 视频操作工具条。
|
||||
* - 整体是一张浮起的圆角玻璃卡,比上一版的横线分隔更"成体"。
|
||||
* - 点赞 + 点踩组成一个胶囊(中间一道竖线分隔),两侧分别带计数。
|
||||
* - "不再显示" 单独成一个次要按钮,hover 时露出 danger 色。
|
||||
* - 点赞 + 点踩是两个独立按钮。
|
||||
* - 删除是唯一的管理操作,hover 时露出 danger 色。
|
||||
*
|
||||
* 功能没变:
|
||||
* - 后端只有点赞接口(POST /api/video/:id/like),点踩仅本地 state。
|
||||
* - 后端只有点赞计数接口,点踩仅本地 state。
|
||||
* - 失败回滚已经处理。
|
||||
*/
|
||||
export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
export function VideoActions({
|
||||
video,
|
||||
onDeleteVideo,
|
||||
deleteSaving,
|
||||
}: Props) {
|
||||
const [likes, setLikes] = useState(video.likes ?? 0);
|
||||
const [dislikes, setDislikes] = useState(video.dislikes ?? 0);
|
||||
const [bursting, setBursting] = useState(false);
|
||||
const [liked, setLiked] = useState(false);
|
||||
const [disliked, setDisliked] = useState(false);
|
||||
const [likeSubmitted, setLikeSubmitted] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
setLikes(video.likes ?? 0);
|
||||
setDislikes(video.dislikes ?? 0);
|
||||
setBursting(false);
|
||||
setLiked(false);
|
||||
setDisliked(false);
|
||||
setLikeSubmitted(false);
|
||||
}, [video.id, video.likes, video.dislikes]);
|
||||
|
||||
async function handleLike() {
|
||||
if (liked) return;
|
||||
setLiked(true);
|
||||
setLikes((n) => n + 1);
|
||||
setBursting(true);
|
||||
window.setTimeout(() => setBursting(false), 320);
|
||||
|
||||
@@ -38,6 +51,11 @@ export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
setDislikes((n) => Math.max(0, n - 1));
|
||||
}
|
||||
|
||||
if (likeSubmitted) return;
|
||||
|
||||
setLikeSubmitted(true);
|
||||
setLikes((n) => n + 1);
|
||||
|
||||
try {
|
||||
const res = await fetch(
|
||||
`/api/video/${encodeURIComponent(video.id)}/like`,
|
||||
@@ -51,6 +69,7 @@ export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
} catch {
|
||||
setLikes((n) => Math.max(0, n - 1));
|
||||
setLiked(false);
|
||||
setLikeSubmitted(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,7 +83,6 @@ export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
setDislikes((n) => n + 1);
|
||||
if (liked) {
|
||||
setLiked(false);
|
||||
setLikes((n) => Math.max(0, n - 1));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,7 +101,6 @@ export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
<ThumbsUp size={18} fill={liked ? "currentColor" : "none"} />
|
||||
<span className="vd-actions__count">{formatCount(likes)}</span>
|
||||
</button>
|
||||
<span className="vd-actions__divider" aria-hidden="true" />
|
||||
<button
|
||||
type="button"
|
||||
className={`vd-actions__pill vd-actions__dislike${
|
||||
@@ -100,13 +117,13 @@ export function VideoActions({ video, onHideVideo, hideSaving }: Props) {
|
||||
|
||||
<button
|
||||
type="button"
|
||||
className="vd-actions__btn vd-actions__hide"
|
||||
onClick={onHideVideo}
|
||||
disabled={hideSaving}
|
||||
aria-label="不再显示这个视频"
|
||||
className="vd-actions__btn vd-actions__delete"
|
||||
onClick={onDeleteVideo}
|
||||
disabled={deleteSaving}
|
||||
aria-label="删除这个视频"
|
||||
>
|
||||
<EyeOff size={16} />
|
||||
<span>{hideSaving ? "处理中" : "不再显示"}</span>
|
||||
<Trash2 size={16} />
|
||||
<span>{deleteSaving ? "删除中" : "删除"}</span>
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -286,7 +286,7 @@ function withRetryParam(src: string, retry: number): string {
|
||||
}
|
||||
|
||||
// 从后端返回的 sourceLabel 推断网盘类型(用于颜色标识)。
|
||||
// 后端目前会下发中文名("夸克网盘" / "115 网盘" / "PikPak" / "联通沃盘" / "OneDrive")
|
||||
// 后端目前会下发中文名("夸克网盘" / "115 网盘" / "PikPak" / "联通网盘" / "OneDrive")
|
||||
// 或英文 kind。两边都尝试匹配;都没匹配上时返回空字符串,CSS 会回落到默认色。
|
||||
function sourceKindFromLabel(label: string): string {
|
||||
const value = label.toLowerCase();
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useMemo, useState } from "react";
|
||||
import { Hash, Pencil, X } from "lucide-react";
|
||||
import { Pencil, Tag, X } from "lucide-react";
|
||||
import type { TagItem, VideoDetail } from "@/types";
|
||||
|
||||
type Props = {
|
||||
@@ -17,7 +17,7 @@ type Props = {
|
||||
* 视觉上和上一版的"两张分离卡"相比,整体感更强:
|
||||
* - 一张大卡内分两个小区块,区块之间用细分隔线
|
||||
* - 简介区块加 "简介" 标题前缀
|
||||
* - 标签区块加 # 图标暗示
|
||||
* - 标签区块加标签轮廓图标暗示
|
||||
*/
|
||||
export function VideoInfoPanel({
|
||||
video,
|
||||
@@ -99,7 +99,7 @@ export function VideoInfoPanel({
|
||||
<div className="vd-info__tags">
|
||||
<div className="vd-info__section-head">
|
||||
<span className="vd-info__section-title">
|
||||
<Hash size={14} aria-hidden="true" />
|
||||
<Tag size={15} strokeWidth={2} aria-hidden="true" />
|
||||
标签
|
||||
</span>
|
||||
{onTagsChange && (
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { CalendarDays, Clock3, Eye } from "lucide-react";
|
||||
import type { VideoDetail } from "@/types";
|
||||
import { formatCount } from "@/lib/format";
|
||||
|
||||
@@ -9,12 +10,11 @@ type Props = {
|
||||
* 详情页标题块。
|
||||
*
|
||||
* 视觉:
|
||||
* - 标题:大、粗、最高两行
|
||||
* - meta:作者首字头像 + 名字 + 一组小胶囊(来源、画质、时长、观看数、发布时间)
|
||||
* - meta:一组小胶囊(来源、画质、时长、观看数、发布时间)
|
||||
* 每个胶囊有自己的语义色彩,避免传统 "·" 分隔列表的列表感。
|
||||
* - 标题:大、粗、最高两行,位于 meta 下方
|
||||
*/
|
||||
export function VideoMetaHeader({ video }: Props) {
|
||||
const author = (video.author ?? "").trim();
|
||||
const source = (video.sourceLabel ?? "").trim();
|
||||
const quality = (video.quality ?? "").trim();
|
||||
const duration = (video.duration ?? "").trim();
|
||||
@@ -23,20 +23,7 @@ export function VideoMetaHeader({ video }: Props) {
|
||||
|
||||
return (
|
||||
<header className="vd-header">
|
||||
<h1 className="vd-header__title" title={video.title}>
|
||||
{video.title}
|
||||
</h1>
|
||||
|
||||
<div className="vd-header__row">
|
||||
{author && (
|
||||
<div className="vd-author" aria-label={`作者 ${author}`}>
|
||||
<span className="vd-author__avatar" aria-hidden="true">
|
||||
{author.slice(0, 1)}
|
||||
</span>
|
||||
<span className="vd-author__name">{author}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<ul className="vd-meta" aria-label="视频信息">
|
||||
{source && (
|
||||
<li className="vd-meta__chip" data-tone={sourceKind || "neutral"}>
|
||||
@@ -52,13 +39,28 @@ export function VideoMetaHeader({ video }: Props) {
|
||||
{quality}
|
||||
</li>
|
||||
)}
|
||||
{duration && <li className="vd-meta__chip">{duration}</li>}
|
||||
<li className="vd-meta__chip">
|
||||
{duration && (
|
||||
<li className="vd-meta__chip vd-meta__chip--plain">
|
||||
<Clock3 size={14} aria-hidden="true" />
|
||||
{duration}
|
||||
</li>
|
||||
)}
|
||||
<li className="vd-meta__chip vd-meta__chip--plain">
|
||||
<Eye size={14} aria-hidden="true" />
|
||||
<strong>{formatCount(video.views)}</strong> 次观看
|
||||
</li>
|
||||
{published && <li className="vd-meta__chip">{published}</li>}
|
||||
{published && (
|
||||
<li className="vd-meta__chip vd-meta__chip--plain">
|
||||
<CalendarDays size={14} aria-hidden="true" />
|
||||
{published}
|
||||
</li>
|
||||
)}
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<h1 className="vd-header__title" title={video.title}>
|
||||
{video.title}
|
||||
</h1>
|
||||
</header>
|
||||
);
|
||||
}
|
||||
|
||||
+1443
-133
File diff suppressed because it is too large
Load Diff
+16
-5
@@ -52,6 +52,19 @@ export function hideVideo(id: string): Promise<{ ok: boolean }> {
|
||||
);
|
||||
}
|
||||
|
||||
export function deleteVideo(
|
||||
id: string,
|
||||
options: { deleteSource?: boolean } = {}
|
||||
): Promise<{ ok: boolean; deletedSource: boolean }> {
|
||||
return apiJSON<{ ok: boolean; deletedSource: boolean }>(
|
||||
`/admin/api/videos/${encodeURIComponent(id)}`,
|
||||
{
|
||||
method: "DELETE",
|
||||
body: JSON.stringify({ deleteSource: !!options.deleteSource }),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
export function recordView(id: string): Promise<{ views: number }> {
|
||||
return apiJSON<{ views: number }>(
|
||||
`/api/video/${encodeURIComponent(id)}/view`,
|
||||
@@ -119,19 +132,17 @@ export type ShortsNextResponse = {
|
||||
|
||||
/**
|
||||
* 拉取短视频流的下一批候选。把当前轮已看过的 video id 列表传给后端,
|
||||
* 服务器从未在列表中的视频里随机抽 count 条返回。preferredFromVideoId
|
||||
* 来自用户最近一次点赞成功的视频,用于按相似标签优先推荐。
|
||||
* 服务器从未在列表中的视频里随机抽 count 条返回。
|
||||
*
|
||||
* 失败时返回空批 + roundComplete=false,由调用方决定是否重试。
|
||||
*/
|
||||
export function fetchShortsNext(
|
||||
seenIds: string[],
|
||||
count: number,
|
||||
preferredFromVideoId?: string
|
||||
count: number
|
||||
): Promise<ShortsNextResponse> {
|
||||
return apiJSON<ShortsNextResponse>("/api/shorts/next", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ seenIds, count, preferredFromVideoId }),
|
||||
body: JSON.stringify({ seenIds, count }),
|
||||
}).catch(() => ({ items: [], total: 0, roundComplete: false }));
|
||||
}
|
||||
|
||||
|
||||
@@ -105,7 +105,7 @@ export default function HomePage() {
|
||||
const latest = latestVideos.slice(0, displayCount);
|
||||
|
||||
return (
|
||||
<AppShell>
|
||||
<AppShell mobileAutoHideNav>
|
||||
<div className="container page-section">
|
||||
<PromoStrip />
|
||||
<SearchPanel />
|
||||
|
||||
+214
-54
@@ -32,9 +32,21 @@ const BATCH_SIZE = 5;
|
||||
// 当队列里"还没看过的视频"少于这个数时,提前请求下一批。
|
||||
const PREFETCH_THRESHOLD = 2;
|
||||
|
||||
// 距离 activeIndex 多少屏内的视频会被 mount 真实 <video>。
|
||||
// =1 表示上一屏 / 当前 / 下一屏 都加载,这样切换时几乎无空白。
|
||||
const MOUNT_RADIUS = 1;
|
||||
// 当前视频至少有这么多秒的前向缓冲后,才允许后续视频开始预加载。
|
||||
const ACTIVE_PRELOAD_BUFFER_SECONDS = 12;
|
||||
|
||||
// 当前视频流畅播放后,向后预加载多少条视频。
|
||||
const PRELOAD_AHEAD_COUNT = 2;
|
||||
|
||||
// 预加载授权一旦发出,只有当前视频前向缓冲跌破这个秒数(或发生 stall)
|
||||
// 才收回。高低水位之间不动作,避免缓冲量在 12s 附近波动时
|
||||
// 反复绑定/剥离后续视频的 src、丢弃已预加载的数据。
|
||||
const ACTIVE_PRELOAD_KEEP_SECONDS = 4;
|
||||
|
||||
// 维护一个固定大小的视频窗口:窗口内才 mount 真实 <video> 壳。
|
||||
// 当前屏先绑定 src;后续预加载要等当前屏缓冲健康后才开始。
|
||||
// 窗口内只要已经产生过可复用缓冲,就保留 src 复用浏览器缓存。
|
||||
const VIDEO_WINDOW_SIZE = 6;
|
||||
|
||||
function loadSeenIds(): string[] {
|
||||
try {
|
||||
@@ -120,7 +132,6 @@ export default function ShortsPage() {
|
||||
|
||||
// seenIds 用 ref 维护,方便在异步 callback 里读到最新值
|
||||
const seenIdsRef = useRef<string[]>(loadSeenIds());
|
||||
const preferredFromVideoIdRef = useRef<string | null>(null);
|
||||
|
||||
const containerRef = useRef<HTMLDivElement | null>(null);
|
||||
// 整个页面根元素,用于 requestFullscreen
|
||||
@@ -130,6 +141,11 @@ export default function ShortsPage() {
|
||||
const activeIndexRef = useRef(0);
|
||||
const ignoreIntersectionUntilRef = useRef(0);
|
||||
const fullscreenRestoreTimersRef = useRef<number[]>([]);
|
||||
const [activeReadyForPreload, setActiveReadyForPreload] = useState(false);
|
||||
const [cacheableSourceIds, setCacheableSourceIds] = useState<Set<string>>(
|
||||
() => new Set()
|
||||
);
|
||||
const [cacheWindowHighIndex, setCacheWindowHighIndex] = useState(-1);
|
||||
|
||||
// 当前是否处在浏览器全屏(Fullscreen API)状态。
|
||||
// iOS Safari 不支持元素级 Fullscreen API,这里会一直保持 false,
|
||||
@@ -147,6 +163,29 @@ export default function ShortsPage() {
|
||||
activeIndexRef.current = activeIndex;
|
||||
}, [activeIndex]);
|
||||
|
||||
const handleActiveReadyForPreload = useCallback((index: number) => {
|
||||
if (index === activeIndexRef.current) {
|
||||
setActiveReadyForPreload(true);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const handleActiveNeedsPriority = useCallback((index: number) => {
|
||||
if (index === activeIndexRef.current) {
|
||||
setActiveReadyForPreload(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// 标记某条视频"浏览器里已有可复用的缓冲"。之后只要它还在缓存窗口内,
|
||||
// 就保留 src 不剥离,回滑/再前滑时直接续用已缓冲数据,秒开不卡顿。
|
||||
const handleSourceCached = useCallback((videoId: string) => {
|
||||
setCacheableSourceIds((prev) => {
|
||||
if (prev.has(videoId)) return prev;
|
||||
const next = new Set(prev);
|
||||
next.add(videoId);
|
||||
return next;
|
||||
});
|
||||
}, []);
|
||||
|
||||
/**
|
||||
* 切换点赞状态。
|
||||
* - liked=true:发 POST /api/video/:id/like
|
||||
@@ -171,11 +210,6 @@ export default function ShortsPage() {
|
||||
);
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const data = (await res.json()) as { likes?: number };
|
||||
if (liked) {
|
||||
preferredFromVideoIdRef.current = videoId;
|
||||
} else if (preferredFromVideoIdRef.current === videoId) {
|
||||
preferredFromVideoIdRef.current = null;
|
||||
}
|
||||
return typeof data.likes === "number" ? data.likes : null;
|
||||
} catch {
|
||||
// 请求失败:回滚集合,让 Slide 自己回滚 UI
|
||||
@@ -204,11 +238,7 @@ export default function ShortsPage() {
|
||||
setLoading(true);
|
||||
try {
|
||||
const seen = seenIdsRef.current;
|
||||
const resp = await fetchShortsNext(
|
||||
seen,
|
||||
BATCH_SIZE,
|
||||
preferredFromVideoIdRef.current ?? undefined
|
||||
);
|
||||
const resp = await fetchShortsNext(seen, BATCH_SIZE);
|
||||
if (resp.items.length === 0) {
|
||||
setEmpty((prev) => prev || true /* 维持 true 即可 */);
|
||||
setRoundComplete(true);
|
||||
@@ -242,6 +272,8 @@ export default function ShortsPage() {
|
||||
const active = items[activeIndex];
|
||||
if (!active) return;
|
||||
|
||||
setCacheWindowHighIndex((prev) => Math.max(prev, activeIndex));
|
||||
|
||||
if (!seenIdsRef.current.includes(active.id)) {
|
||||
seenIdsRef.current = [...seenIdsRef.current, active.id];
|
||||
saveSeenIds(seenIdsRef.current);
|
||||
@@ -250,8 +282,10 @@ export default function ShortsPage() {
|
||||
const remaining = items.length - 1 - activeIndex;
|
||||
if (remaining < PREFETCH_THRESHOLD && !loading) {
|
||||
if (roundComplete) {
|
||||
// 上一次后端说"本轮已耗尽",且当前已经看到队列接近末尾。
|
||||
// 清空 localStorage 后再请求即可开新一轮。
|
||||
// 上一次后端说"本轮已耗尽"时,必须等用户真正滑到当前队列最后一条
|
||||
// 再清空已看记录开新一轮。否则退出后重新进入会把未完成轮次提前重置,
|
||||
// 导致刚刷过的视频再次出现在下一次会话里。
|
||||
if (remaining > 0) return;
|
||||
seenIdsRef.current = [];
|
||||
saveSeenIds([]);
|
||||
setRoundComplete(false);
|
||||
@@ -280,7 +314,11 @@ export default function ShortsPage() {
|
||||
if (!Number.isNaN(idx)) bestIndex = idx;
|
||||
}
|
||||
}
|
||||
if (bestIndex >= 0) setActiveIndex(bestIndex);
|
||||
if (bestIndex >= 0 && bestIndex !== activeIndexRef.current) {
|
||||
activeIndexRef.current = bestIndex;
|
||||
setActiveReadyForPreload(false);
|
||||
setActiveIndex(bestIndex);
|
||||
}
|
||||
},
|
||||
{
|
||||
root,
|
||||
@@ -300,21 +338,10 @@ export default function ShortsPage() {
|
||||
video.muted = muted;
|
||||
video.volume = volume;
|
||||
if (video.paused) {
|
||||
// 切到这个视频时从头开始播
|
||||
try {
|
||||
video.currentTime = 0;
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
video.play().catch(() => undefined);
|
||||
}
|
||||
} else {
|
||||
if (!video.paused) video.pause();
|
||||
try {
|
||||
video.currentTime = 0;
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
});
|
||||
}, [activeIndex, muted, volume, items.length]);
|
||||
@@ -594,6 +621,8 @@ export default function ShortsPage() {
|
||||
}
|
||||
}, [items.length, showHud]);
|
||||
|
||||
const videoWindow = getVideoWindowBounds(cacheWindowHighIndex, items.length);
|
||||
|
||||
return (
|
||||
<div className="shorts-page" ref={pageRef}>
|
||||
<header className="shorts-header">
|
||||
@@ -652,26 +681,49 @@ export default function ShortsPage() {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{items.map((item, index) => (
|
||||
<ShortsSlide
|
||||
key={item.id}
|
||||
item={item}
|
||||
index={index}
|
||||
isActive={index === activeIndex}
|
||||
// 距离 active 在 MOUNT_RADIUS 之内才挂载真正的 <video>,
|
||||
// 其它槽位用海报占位以节省内存和带宽
|
||||
shouldMount={Math.abs(index - activeIndex) <= MOUNT_RADIUS}
|
||||
muted={muted}
|
||||
volume={volume}
|
||||
setMuted={setMuted}
|
||||
setVolume={setVolume}
|
||||
videoRef={setVideoRef(index)}
|
||||
onLikeToggle={handleLikeToggle}
|
||||
hasLiked={hasLiked}
|
||||
onHideSuccess={handleHideSuccess}
|
||||
showHud={showHud}
|
||||
/>
|
||||
))}
|
||||
{items.map((item, index) => {
|
||||
const isActiveSlide = index === activeIndex;
|
||||
const isInCacheWindow =
|
||||
index >= videoWindow.start && index <= videoWindow.end;
|
||||
const preloadOffset = index - activeIndex;
|
||||
const shouldPreload =
|
||||
activeReadyForPreload &&
|
||||
preloadOffset > 0 &&
|
||||
preloadOffset <= PRELOAD_AHEAD_COUNT;
|
||||
const shouldMount = isActiveSlide || isInCacheWindow || shouldPreload;
|
||||
// 视频窗口内已经缓冲过的视频保留 src:
|
||||
// 在窗口内来回切换时,直接复用浏览器已缓冲数据。
|
||||
const shouldRetainCached =
|
||||
isInCacheWindow && !isActiveSlide && cacheableSourceIds.has(item.id);
|
||||
const shouldLoad = isActiveSlide || shouldPreload || shouldRetainCached;
|
||||
const shouldEagerLoad = isActiveSlide || shouldPreload;
|
||||
return (
|
||||
<ShortsSlide
|
||||
key={item.id}
|
||||
item={item}
|
||||
index={index}
|
||||
isActive={isActiveSlide}
|
||||
// 固定 6 条视频窗口内才挂载 <video> 壳;
|
||||
// 当前屏先绑定 src;后两个视频等当前屏缓冲健康后再预加载;
|
||||
// 已缓冲过的窗口内视频保留 src,便于来回切换复用缓存。
|
||||
shouldMount={shouldMount}
|
||||
shouldLoad={shouldLoad}
|
||||
shouldEagerLoad={shouldEagerLoad}
|
||||
muted={muted}
|
||||
volume={volume}
|
||||
setMuted={setMuted}
|
||||
setVolume={setVolume}
|
||||
videoRef={setVideoRef(index)}
|
||||
onLikeToggle={handleLikeToggle}
|
||||
hasLiked={hasLiked}
|
||||
onHideSuccess={handleHideSuccess}
|
||||
onActiveReadyForPreload={handleActiveReadyForPreload}
|
||||
onActiveNeedsPriority={handleActiveNeedsPriority}
|
||||
onSourceCached={handleSourceCached}
|
||||
showHud={showHud}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
|
||||
{!empty && items.length > 0 && loading && (
|
||||
<div className="shorts-loading">
|
||||
@@ -689,6 +741,8 @@ type SlideProps = {
|
||||
index: number;
|
||||
isActive: boolean;
|
||||
shouldMount: boolean;
|
||||
shouldLoad: boolean;
|
||||
shouldEagerLoad: boolean;
|
||||
muted: boolean;
|
||||
volume: number;
|
||||
setMuted: (muted: boolean) => void;
|
||||
@@ -702,6 +756,10 @@ type SlideProps = {
|
||||
/** 父组件查询某 id 是否已经在本次会话内点过赞 */
|
||||
hasLiked: (videoId: string) => boolean;
|
||||
onHideSuccess: (index: number) => void;
|
||||
onActiveReadyForPreload: (index: number) => void;
|
||||
onActiveNeedsPriority: (index: number) => void;
|
||||
/** 本条视频在浏览器里已有可复用缓冲,之后在视频窗口内保留 src */
|
||||
onSourceCached: (videoId: string) => void;
|
||||
showHud: (text: string, icon?: React.ReactNode) => void;
|
||||
};
|
||||
|
||||
@@ -717,6 +775,8 @@ function ShortsSlide({
|
||||
index,
|
||||
isActive,
|
||||
shouldMount,
|
||||
shouldLoad,
|
||||
shouldEagerLoad,
|
||||
muted,
|
||||
volume,
|
||||
setMuted,
|
||||
@@ -725,6 +785,9 @@ function ShortsSlide({
|
||||
onLikeToggle,
|
||||
hasLiked,
|
||||
onHideSuccess,
|
||||
onActiveReadyForPreload,
|
||||
onActiveNeedsPriority,
|
||||
onSourceCached,
|
||||
showHud,
|
||||
}: SlideProps) {
|
||||
const localRef = useRef<HTMLVideoElement | null>(null);
|
||||
@@ -778,6 +841,23 @@ function ShortsSlide({
|
||||
[videoRef]
|
||||
);
|
||||
|
||||
// 非当前屏/后续预加载/视频窗口内缓存视频不保留媒体源,确保离开窗口后浏览器中止原始网盘流。
|
||||
useEffect(() => {
|
||||
if (shouldLoad) return;
|
||||
const video = localRef.current;
|
||||
if (!video) return;
|
||||
try {
|
||||
video.pause();
|
||||
video.removeAttribute("src");
|
||||
video.load();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
setDuration(0);
|
||||
setCurrentTime(0);
|
||||
setIsBuffering(false);
|
||||
}, [shouldLoad, item.id]);
|
||||
|
||||
// 离开活跃后清掉本地的暂停状态,避免回来时 UI 还显示着 paused
|
||||
useEffect(() => {
|
||||
if (!isActive) {
|
||||
@@ -810,7 +890,8 @@ function ShortsSlide({
|
||||
}, [isMarkedHidden]);
|
||||
|
||||
// 监听 video 的时长 / 进度 / 缓冲状态 / 音量物理键变化。
|
||||
// MOUNT_RADIUS 会让第三屏以后的 slide 先以海报占位,之后才挂载 video;
|
||||
// VIDEO_WINDOW_SIZE 会让窗口外的 slide 先以海报占位,之后才挂载 video 壳;
|
||||
// 只有 shouldLoad=true 的当前屏/后续预加载/缓存窗口视频会绑定 src,因此不会一次拉完整队列。
|
||||
// 因此这里必须跟随 shouldMount 重新绑定,否则后续视频没有 timeupdate 事件。
|
||||
useEffect(() => {
|
||||
if (!shouldMount) {
|
||||
@@ -832,14 +913,28 @@ function ShortsSlide({
|
||||
const handleTime = () => {
|
||||
// 拖动期间不要被 timeupdate 覆盖 UI
|
||||
if (!scrubbingRef.current) setCurrentTime(video.currentTime);
|
||||
syncActivePreloadReadiness(video);
|
||||
};
|
||||
const handleWaiting = () => {
|
||||
setIsBuffering(true);
|
||||
if (isActive) onActiveNeedsPriority(index);
|
||||
};
|
||||
const handlePlayingOrCanPlay = () => {
|
||||
setIsBuffering(false);
|
||||
// 已经能解码播放,说明浏览器里有了值得复用的数据。
|
||||
if (shouldLoad) onSourceCached(item.id);
|
||||
syncActivePreloadReadiness(video);
|
||||
};
|
||||
const handleProgress = () => {
|
||||
syncActivePreloadReadiness(video);
|
||||
// 窗口内视频只要已经产生缓冲,就标记为可复用;
|
||||
// 之后预加载授权被收回时不再丢弃它的 src 和已缓冲数据。
|
||||
if (shouldLoad && videoHasBufferedData(video)) {
|
||||
onSourceCached(item.id);
|
||||
}
|
||||
};
|
||||
const handleVolumeChange = () => {
|
||||
if (!isActive) return;
|
||||
// 当检测到 video 自身的 mute 状态或 volume 改变时,同步更新 React 状态。
|
||||
// 这可以在移动端浏览器支持物理音量键调整时,自动反向取消静音并展示音量 HUD。
|
||||
if (video.muted !== muted) {
|
||||
@@ -850,6 +945,17 @@ function ShortsSlide({
|
||||
}
|
||||
};
|
||||
|
||||
function syncActivePreloadReadiness(currentVideo: HTMLVideoElement) {
|
||||
if (!isActive) return;
|
||||
if (videoHasComfortableBuffer(currentVideo)) {
|
||||
onActiveReadyForPreload(index);
|
||||
} else if (videoBufferIsCritical(currentVideo)) {
|
||||
// 高低水位滞回:只有缓冲真正告急才收回预加载授权,
|
||||
// 在两个水位之间维持现状,避免阈值附近来回抖动。
|
||||
onActiveNeedsPriority(index);
|
||||
}
|
||||
}
|
||||
|
||||
handleLoaded();
|
||||
handleTime();
|
||||
video.addEventListener("loadedmetadata", handleLoaded);
|
||||
@@ -858,6 +964,7 @@ function ShortsSlide({
|
||||
video.addEventListener("waiting", handleWaiting);
|
||||
video.addEventListener("playing", handlePlayingOrCanPlay);
|
||||
video.addEventListener("canplay", handlePlayingOrCanPlay);
|
||||
video.addEventListener("progress", handleProgress);
|
||||
video.addEventListener("volumechange", handleVolumeChange);
|
||||
|
||||
// 挂载时如果已经在播放但是状态不到 ready 则置 buffering
|
||||
@@ -872,9 +979,10 @@ function ShortsSlide({
|
||||
video.removeEventListener("waiting", handleWaiting);
|
||||
video.removeEventListener("playing", handlePlayingOrCanPlay);
|
||||
video.removeEventListener("canplay", handlePlayingOrCanPlay);
|
||||
video.removeEventListener("progress", handleProgress);
|
||||
video.removeEventListener("volumechange", handleVolumeChange);
|
||||
};
|
||||
}, [shouldMount, item.id, muted, volume, setMuted, setVolume]);
|
||||
}, [shouldMount, shouldLoad, item.id, index, isActive, muted, volume, setMuted, setVolume, onActiveReadyForPreload, onActiveNeedsPriority, onSourceCached]);
|
||||
|
||||
// 长按 2 倍速:直接绑原生事件
|
||||
useEffect(() => {
|
||||
@@ -1175,9 +1283,9 @@ function ShortsSlide({
|
||||
<video
|
||||
ref={setRef}
|
||||
className="shorts-slide__video"
|
||||
src={item.videoSrc}
|
||||
src={shouldLoad ? item.videoSrc : undefined}
|
||||
poster={item.poster}
|
||||
preload="auto"
|
||||
preload={shouldLoad ? (shouldEagerLoad ? "auto" : "metadata") : "none"}
|
||||
playsInline
|
||||
loop
|
||||
muted={muted}
|
||||
@@ -1210,7 +1318,7 @@ function ShortsSlide({
|
||||
)}
|
||||
|
||||
{/* 视频加载/缓冲旋转器 */}
|
||||
{isBuffering && isActive && shouldMount && !isMarkedHidden && (
|
||||
{isBuffering && isActive && shouldLoad && !isMarkedHidden && (
|
||||
<div className="shorts-slide__buffering" aria-hidden="true">
|
||||
<Loader2 size={30} className="shorts-slide__buffering-icon" />
|
||||
</div>
|
||||
@@ -1309,7 +1417,7 @@ function ShortsSlide({
|
||||
)}
|
||||
|
||||
{/* 进度条 */}
|
||||
{shouldMount && !isMarkedHidden && (
|
||||
{isActive && shouldLoad && !isMarkedHidden && (
|
||||
<div
|
||||
className={`shorts-slide__progress ${
|
||||
scrubbing ? "is-scrubbing" : ""
|
||||
@@ -1347,6 +1455,58 @@ function clamp(n: number, min: number, max: number) {
|
||||
return n < min ? min : n > max ? max : n;
|
||||
}
|
||||
|
||||
function getVideoWindowBounds(highestViewedIndex: number, itemCount: number) {
|
||||
const size = Math.min(VIDEO_WINDOW_SIZE, itemCount);
|
||||
if (size <= 0 || highestViewedIndex < 0) return { start: 0, end: -1 };
|
||||
|
||||
const end = clamp(highestViewedIndex, 0, itemCount - 1);
|
||||
const start = Math.max(0, end - size + 1);
|
||||
return { start, end };
|
||||
}
|
||||
|
||||
/** 已经缓冲到片尾(含误差余量),不会再因网络卡顿 */
|
||||
function videoBufferedToEnd(video: HTMLVideoElement) {
|
||||
const duration = Number.isFinite(video.duration) ? video.duration : 0;
|
||||
if (duration <= 0) return false;
|
||||
const remaining = Math.max(0, duration - (video.currentTime || 0));
|
||||
return bufferedAheadSeconds(video) >= remaining - 0.25;
|
||||
}
|
||||
|
||||
function videoHasBufferedData(video: HTMLVideoElement) {
|
||||
for (let i = 0; i < video.buffered.length; i += 1) {
|
||||
if (video.buffered.end(i) > video.buffered.start(i)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** 前向缓冲健康(达到高水位或已缓冲到结尾),可以放心预加载后续视频 */
|
||||
function videoHasComfortableBuffer(video: HTMLVideoElement) {
|
||||
if (video.readyState < 3) return false;
|
||||
if (videoBufferedToEnd(video)) return true;
|
||||
return bufferedAheadSeconds(video) >= ACTIVE_PRELOAD_BUFFER_SECONDS;
|
||||
}
|
||||
|
||||
/** 前向缓冲告急(跌破低水位且没缓冲到结尾),应收回预加载授权 */
|
||||
function videoBufferIsCritical(video: HTMLVideoElement) {
|
||||
if (video.readyState < 3) return true;
|
||||
if (videoBufferedToEnd(video)) return false;
|
||||
return bufferedAheadSeconds(video) < ACTIVE_PRELOAD_KEEP_SECONDS;
|
||||
}
|
||||
|
||||
function bufferedAheadSeconds(video: HTMLVideoElement) {
|
||||
const current = video.currentTime || 0;
|
||||
for (let i = 0; i < video.buffered.length; i += 1) {
|
||||
const start = video.buffered.start(i);
|
||||
const end = video.buffered.end(i);
|
||||
if (start <= current + 0.25 && end > current) {
|
||||
return Math.max(0, end - current);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function formatClock(seconds: number) {
|
||||
if (!Number.isFinite(seconds) || seconds < 0) return "00:00";
|
||||
const total = Math.floor(seconds);
|
||||
|
||||
+151
-22
@@ -7,9 +7,9 @@ import { VideoMetaHeader } from "@/components/VideoMetaHeader";
|
||||
import { VideoInfoPanel } from "@/components/VideoInfoPanel";
|
||||
import { RecommendedRail } from "@/components/RecommendedRail";
|
||||
import {
|
||||
deleteVideo,
|
||||
fetchTags,
|
||||
fetchVideoDetail,
|
||||
hideVideo,
|
||||
recordView,
|
||||
updateVideoTags,
|
||||
} from "@/data/videos";
|
||||
@@ -22,7 +22,10 @@ export default function VideoDetailPage() {
|
||||
const [tags, setTags] = useState<TagItem[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [tagSaving, setTagSaving] = useState(false);
|
||||
const [hideSaving, setHideSaving] = useState(false);
|
||||
const [deleteOpen, setDeleteOpen] = useState(false);
|
||||
const [deleteSource, setDeleteSource] = useState(false);
|
||||
const [deleteSaving, setDeleteSaving] = useState(false);
|
||||
const [deleteError, setDeleteError] = useState("");
|
||||
const detailTopRef = useRef<HTMLDivElement | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -63,16 +66,33 @@ export default function VideoDetailPage() {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleHideVideo() {
|
||||
if (!detail || hideSaving) return;
|
||||
if (!window.confirm("确定以后不再展示这个视频吗?")) return;
|
||||
setHideSaving(true);
|
||||
function handleOpenDelete() {
|
||||
if (!detail || deleteSaving) return;
|
||||
setDeleteSource(false);
|
||||
setDeleteError("");
|
||||
setDeleteOpen(true);
|
||||
}
|
||||
|
||||
function handleCloseDelete() {
|
||||
if (deleteSaving) return;
|
||||
setDeleteOpen(false);
|
||||
setDeleteError("");
|
||||
}
|
||||
|
||||
async function handleConfirmDelete() {
|
||||
if (!detail || deleteSaving) return;
|
||||
setDeleteSaving(true);
|
||||
setDeleteError("");
|
||||
try {
|
||||
await hideVideo(detail.id);
|
||||
await deleteVideo(detail.id, { deleteSource });
|
||||
navigate("/list", { replace: true });
|
||||
} catch {
|
||||
setHideSaving(false);
|
||||
window.alert("隐藏失败,请稍后重试");
|
||||
setDeleteError(
|
||||
deleteSource
|
||||
? "删除失败。源文件未能删除时,管理库记录会保留。"
|
||||
: "删除失败,请稍后重试。"
|
||||
);
|
||||
setDeleteSaving(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,14 +104,66 @@ export default function VideoDetailPage() {
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<AppShell>
|
||||
<AppShell mobileAutoHideNav>
|
||||
<div className="vd-page">
|
||||
<div className="vd-ambient" aria-hidden="true" />
|
||||
<div className="container vd-page__inner">
|
||||
<div className="vd-skeleton">
|
||||
<div className="vd-skeleton__player" />
|
||||
<div className="vd-skeleton__title" />
|
||||
<div className="vd-skeleton__meta" />
|
||||
<div
|
||||
className="vd-layout vd-skeleton"
|
||||
aria-busy="true"
|
||||
aria-label="视频详情加载中"
|
||||
>
|
||||
<div className="vd-main">
|
||||
<div className="vd-skeleton__player" />
|
||||
|
||||
<div className="vd-skeleton__summary">
|
||||
<div className="vd-skeleton__chips">
|
||||
<span className="vd-skeleton__chip vd-skeleton__chip--source" />
|
||||
<span className="vd-skeleton__chip" />
|
||||
<span className="vd-skeleton__chip vd-skeleton__chip--plain" />
|
||||
<span className="vd-skeleton__chip vd-skeleton__chip--plain" />
|
||||
</div>
|
||||
<div className="vd-skeleton__title" />
|
||||
<div className="vd-skeleton__actions">
|
||||
<span />
|
||||
<span />
|
||||
<span />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="vd-skeleton__info">
|
||||
<span className="vd-skeleton__section-head" />
|
||||
<span className="vd-skeleton__line" />
|
||||
<span className="vd-skeleton__line vd-skeleton__line--short" />
|
||||
<div className="vd-skeleton__tag-row">
|
||||
<span />
|
||||
<span />
|
||||
<span />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<aside className="vd-rail vd-skeleton__rail">
|
||||
<div className="vd-rail__head">
|
||||
<span className="vd-rail__head-icon" aria-hidden="true">
|
||||
<span />
|
||||
<span />
|
||||
</span>
|
||||
<span className="vd-skeleton__rail-head" />
|
||||
</div>
|
||||
<ul className="vd-rail__list vd-skeleton__rail-list">
|
||||
{Array.from({ length: 6 }).map((_, index) => (
|
||||
<li key={index} className="vd-skeleton__rail-item">
|
||||
<span className="vd-skeleton__rail-thumb" />
|
||||
<span className="vd-skeleton__rail-body">
|
||||
<span className="vd-skeleton__rail-title" />
|
||||
<span className="vd-skeleton__rail-title vd-skeleton__rail-title--short" />
|
||||
<span className="vd-skeleton__rail-meta" />
|
||||
</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</aside>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -101,7 +173,7 @@ export default function VideoDetailPage() {
|
||||
|
||||
if (!detail) {
|
||||
return (
|
||||
<AppShell>
|
||||
<AppShell mobileAutoHideNav>
|
||||
<div className="vd-page">
|
||||
<div className="container vd-page__inner">
|
||||
<div className="vd-empty">视频不存在或已被移除</div>
|
||||
@@ -112,7 +184,7 @@ export default function VideoDetailPage() {
|
||||
}
|
||||
|
||||
return (
|
||||
<AppShell>
|
||||
<AppShell mobileAutoHideNav>
|
||||
<div className="vd-page">
|
||||
{/* Ambient 背景层:用海报作模糊底色,叠加渐变过渡到页面背景 */}
|
||||
<div
|
||||
@@ -131,21 +203,25 @@ export default function VideoDetailPage() {
|
||||
<div className="vd-player-wrap">
|
||||
<div className="vd-player">
|
||||
<VideoPlayer
|
||||
id={detail.id}
|
||||
src={detail.videoSrc}
|
||||
poster={detail.poster}
|
||||
previewSrc={detail.previewSrc}
|
||||
title={detail.title}
|
||||
onFirstPlay={handleFirstPlay}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<VideoMetaHeader video={detail} />
|
||||
<section className="vd-summary" aria-label="当前视频">
|
||||
<VideoMetaHeader video={detail} />
|
||||
|
||||
<VideoActions
|
||||
video={detail}
|
||||
onHideVideo={handleHideVideo}
|
||||
hideSaving={hideSaving}
|
||||
/>
|
||||
<VideoActions
|
||||
video={detail}
|
||||
onDeleteVideo={handleOpenDelete}
|
||||
deleteSaving={deleteSaving}
|
||||
/>
|
||||
</section>
|
||||
|
||||
<VideoInfoPanel
|
||||
video={detail}
|
||||
@@ -159,6 +235,59 @@ export default function VideoDetailPage() {
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{deleteOpen && (
|
||||
<div className="vd-delete-modal" role="presentation">
|
||||
<div
|
||||
className="vd-delete-dialog"
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
aria-labelledby="vd-delete-title"
|
||||
>
|
||||
<div className="vd-delete-head">
|
||||
<h2 id="vd-delete-title" className="vd-delete-title">
|
||||
删除视频
|
||||
</h2>
|
||||
<p className="vd-delete-text">
|
||||
确定删除「{detail.title}」吗?此操作会从管理库移除该视频。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<label className="vd-delete-option">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={deleteSource}
|
||||
disabled={deleteSaving}
|
||||
onChange={(e) => setDeleteSource(e.target.checked)}
|
||||
/>
|
||||
<span>
|
||||
<strong>同时删除网盘中的源文件</strong>
|
||||
</span>
|
||||
</label>
|
||||
|
||||
{deleteError && <div className="vd-delete-error">{deleteError}</div>}
|
||||
|
||||
<div className="vd-delete-actions">
|
||||
<button
|
||||
type="button"
|
||||
className="vd-delete-action vd-delete-cancel"
|
||||
onClick={handleCloseDelete}
|
||||
disabled={deleteSaving}
|
||||
>
|
||||
取消
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="vd-delete-action vd-delete-confirm"
|
||||
onClick={handleConfirmDelete}
|
||||
disabled={deleteSaving}
|
||||
>
|
||||
{deleteSaving ? "删除中..." : "删除"}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</AppShell>
|
||||
);
|
||||
}
|
||||
|
||||
+1366
-48
File diff suppressed because it is too large
Load Diff
@@ -10,6 +10,10 @@
|
||||
font-size: var(--font-base);
|
||||
}
|
||||
|
||||
.app-shell__nav-stack {
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
.app-shell__main {
|
||||
flex: 1;
|
||||
width: 100%;
|
||||
|
||||
@@ -179,6 +179,24 @@
|
||||
|
||||
/* ----- 响应式 ----- */
|
||||
@media (max-width: 768px) {
|
||||
.app-shell--mobile-auto-hide-nav .app-shell__nav-stack {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: var(--z-nav);
|
||||
transform: translateY(0);
|
||||
transition: transform 220ms var(--ease-out);
|
||||
will-change: transform;
|
||||
}
|
||||
|
||||
.app-shell--mobile-auto-hide-nav.is-mobile-nav-hidden .app-shell__nav-stack {
|
||||
transform: translateY(-100%);
|
||||
}
|
||||
|
||||
.app-shell--mobile-auto-hide-nav .main-nav {
|
||||
position: relative;
|
||||
z-index: auto;
|
||||
}
|
||||
|
||||
.main-nav__inner {
|
||||
height: 56px;
|
||||
gap: var(--space-3);
|
||||
|
||||
+1016
-297
File diff suppressed because it is too large
Load Diff
+143
-16
@@ -10,6 +10,18 @@ const driveComponentsSource = readFileSync(
|
||||
new URL("../src/admin/drive/DriveComponents.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
const crawlerPageSource = readFileSync(
|
||||
new URL("../src/admin/CrawlersPage.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
const adminLayoutSource = readFileSync(
|
||||
new URL("../src/admin/AdminLayout.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
const appSource = readFileSync(
|
||||
new URL("../src/App.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
const spider91UploadTargetSource = readFileSync(
|
||||
new URL("../src/admin/drive/Spider91UploadTargetField.tsx", import.meta.url),
|
||||
"utf8"
|
||||
@@ -51,22 +63,24 @@ function assertDriveTypeOption(value: string, label: string) {
|
||||
);
|
||||
}
|
||||
|
||||
test("spider91 drive form does not expose advanced crawler credentials", () => {
|
||||
assert.match(combinedSource, /key: "proxy"/);
|
||||
assert.match(combinedSource, /label: "代理地址(可选)"/);
|
||||
assert.match(combinedSource, /支持 http:\/\/、https:\/\/、socks5:\/\/、socks5h:\/\/代理/);
|
||||
assert.doesNotMatch(combinedSource, /target_new/);
|
||||
assert.doesNotMatch(combinedSource, /crawl_hour/);
|
||||
assert.doesNotMatch(combinedSource, /python_path/);
|
||||
assert.doesNotMatch(combinedSource, /script_path/);
|
||||
test("crawler sources are not selectable as storage drives", () => {
|
||||
assert.ok(
|
||||
!driveTypeOptions().some((option) => option.value === "spider91"),
|
||||
"spider91 should not be a storage drive option"
|
||||
);
|
||||
assert.ok(
|
||||
!driveTypeOptions().some((option) => option.value === "scriptcrawler"),
|
||||
"scriptcrawler should not be a storage drive option"
|
||||
);
|
||||
});
|
||||
|
||||
test("spider91 upload target uses explicit local-save option instead of auto target", () => {
|
||||
assert.match(combinedSource, /本地保存,不上传/);
|
||||
assert.match(
|
||||
combinedSource,
|
||||
/d\.kind === "pikpak" \|\| d\.kind === "p115" \|\| d\.kind === "p123" \|\| d\.kind === "onedrive"/
|
||||
/d\.kind === "pikpak"[\s\S]*d\.kind === "p115"[\s\S]*d\.kind === "p123"[\s\S]*d\.kind === "onedrive"[\s\S]*d\.kind === "googledrive"[\s\S]*d\.kind === "wopan"/
|
||||
);
|
||||
assert.match(crawlerPageSource, /UPLOAD_TARGET_KINDS[\s\S]*"wopan"/);
|
||||
assert.doesNotMatch(combinedSource, /自动:唯一/);
|
||||
assert.doesNotMatch(combinedSource, /自动模式/);
|
||||
assert.doesNotMatch(combinedSource, /较早的视频会上传到该云盘根目录下的 91 Spider 文件夹/);
|
||||
@@ -112,7 +126,7 @@ test("onedrive drive form only exposes required default-app fields", () => {
|
||||
assert.doesNotMatch(fields, /key: "site_id"/);
|
||||
});
|
||||
|
||||
test("googledrive drive form only exposes refresh token", () => {
|
||||
test("googledrive drive form supports online API and custom OAuth client modes", () => {
|
||||
assertDriveTypeOption("googledrive", "Google Drive");
|
||||
|
||||
const match =
|
||||
@@ -123,10 +137,25 @@ test("googledrive drive form only exposes refresh token", () => {
|
||||
const fields = match[1];
|
||||
|
||||
assert.match(fields, /key: "refresh_token"/);
|
||||
assert.doesNotMatch(fields, /key: "access_token"/);
|
||||
assert.match(fields, /key: "use_online_api"/);
|
||||
assert.match(fields, /type: "select"/);
|
||||
assert.match(fields, /defaultValue: "true"/);
|
||||
assert.match(fields, /OpenList 在线 API/);
|
||||
assert.match(fields, /自建 Google OAuth 客户端/);
|
||||
assert.match(fields, /key: "client_id"/);
|
||||
assert.match(fields, /key: "client_secret"/);
|
||||
assert.match(fields, /googleDriveUsesOnlineAPI\(creds\)/);
|
||||
assert.doesNotMatch(fields, /key: "api_url_address"/);
|
||||
assert.doesNotMatch(fields, /key: "client_id"/);
|
||||
assert.doesNotMatch(fields, /key: "client_secret"/);
|
||||
assert.doesNotMatch(fields, /在线 API 模式填写 OpenList 获取的 refresh_token/);
|
||||
assert.doesNotMatch(constantsSource, /请参考OpenList文档中关于谷歌云盘的配置方法。/);
|
||||
assert.doesNotMatch(constantsSource, /选择自建 Google OAuth 客户端后,服务端会直接请求 Google OAuth token 接口续期。/);
|
||||
assert.match(driveFormSource, /<select/);
|
||||
assert.match(driveFormSource, /value=\{form\.creds\[f\.key\] \?\? f\.defaultValue \?\? ""\}/);
|
||||
assert.match(driveFormSource, /className="admin-form-select"/);
|
||||
assert.match(driveFormSource, /ChevronDown/);
|
||||
assert.match(drivesPageSource, /googleDriveUseOnlineAPI/);
|
||||
assert.match(apiSource, /googleDriveUseOnlineAPI\?: boolean/);
|
||||
assert.doesNotMatch(fields, /key: "access_token"/);
|
||||
});
|
||||
|
||||
test("pikpak drive form only exposes account login fields", () => {
|
||||
@@ -165,17 +194,78 @@ test("localstorage drive form asks for a server directory path", () => {
|
||||
test("drive type selector keeps primary source order", () => {
|
||||
assert.deepEqual(driveTypeOptions(), [
|
||||
{ value: "p115", label: "115 网盘" },
|
||||
{ value: "p123", label: "123 云盘" },
|
||||
{ value: "p123", label: "123网盘" },
|
||||
{ value: "pikpak", label: "PikPak" },
|
||||
{ value: "onedrive", label: "OneDrive" },
|
||||
{ value: "googledrive", label: "Google Drive" },
|
||||
{ value: "localstorage", label: "本地存储" },
|
||||
{ value: "spider91", label: "91 爬虫" },
|
||||
{ value: "quark", label: "夸克网盘" },
|
||||
{ value: "wopan", label: "联通沃盘" },
|
||||
{ value: "wopan", label: "联通网盘" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("crawler management is a separate admin section", () => {
|
||||
assert.match(adminLayoutSource, /to="\/admin\/crawlers"/);
|
||||
assert.match(adminLayoutSource, /admin-nav__title">爬虫管理/);
|
||||
assert.match(adminLayoutSource, /admin-nav__icon"><SpiderIcon size=\{16\} \/>/);
|
||||
assert.match(appSource, /path="crawlers" element=\{<CrawlersPage \/>/);
|
||||
assert.match(crawlerPageSource, /export function CrawlersPage/);
|
||||
assert.match(crawlerPageSource, /SpiderIcon/);
|
||||
assert.match(crawlerPageSource, /添加爬虫/);
|
||||
// 新设计:列表 + Modal 三步编辑器,删除确认走 ConfirmModal,任务进行中自动轮询
|
||||
assert.match(crawlerPageSource, /CrawlerEditorModal/);
|
||||
assert.match(crawlerPageSource, /ConfirmModal/);
|
||||
assert.doesNotMatch(crawlerPageSource, /window\.confirm/);
|
||||
assert.match(crawlerPageSource, /POLL_INTERVAL_MS/);
|
||||
assert.match(crawlerPageSource, /api\.listCrawlers/);
|
||||
assert.match(crawlerPageSource, /api\.listDrives/);
|
||||
assert.match(crawlerPageSource, /api\.upsertCrawler/);
|
||||
assert.match(crawlerPageSource, /api\.runCrawler/);
|
||||
assert.match(crawlerPageSource, /api\.stopCrawlerTasks/);
|
||||
assert.match(crawlerPageSource, /api\.deleteCrawler/);
|
||||
assert.match(crawlerPageSource, /api\.importCrawlerScriptFile/);
|
||||
assert.match(crawlerPageSource, /api\.importCrawlerScriptURL/);
|
||||
assert.match(crawlerPageSource, /api\.testCrawlerScript/);
|
||||
assert.match(crawlerPageSource, /type="file"/);
|
||||
assert.match(crawlerPageSource, /链接导入/);
|
||||
assert.match(crawlerPageSource, /测试脚本/);
|
||||
assert.match(crawlerPageSource, /测试通过/);
|
||||
assert.match(crawlerPageSource, /Spider91UploadTargetField/);
|
||||
assert.match(crawlerPageSource, /uploadDriveId/);
|
||||
assert.match(crawlerPageSource, /UPLOAD_TARGET_KINDS/);
|
||||
assert.doesNotMatch(crawlerPageSource, /新建脚本/);
|
||||
assert.doesNotMatch(crawlerPageSource, /爬虫 ID/);
|
||||
assert.doesNotMatch(crawlerPageSource, /crawler-id/);
|
||||
assert.doesNotMatch(crawlerPageSource, /crawler-name/);
|
||||
// 脚本路径只读展示,不允许手动填写
|
||||
assert.doesNotMatch(crawlerPageSource, /crawler-script-path/);
|
||||
assert.doesNotMatch(crawlerPageSource, /Python 解释器/);
|
||||
assert.doesNotMatch(crawlerPageSource, /自定义配置 JSON/);
|
||||
assert.doesNotMatch(crawlerPageSource, /Bot/);
|
||||
// 项目不再内置任何爬虫:不允许出现内置 91 预设
|
||||
assert.doesNotMatch(crawlerPageSource, /builtin/);
|
||||
assert.doesNotMatch(crawlerPageSource, /内置 91/);
|
||||
assert.match(apiSource, /type AdminCrawler/);
|
||||
assert.match(apiSource, /uploadDriveId\?: string/);
|
||||
assert.match(apiSource, /"\/crawlers"/);
|
||||
assert.match(apiSource, /"\/crawlers\/import-file"/);
|
||||
assert.match(apiSource, /"\/crawlers\/import-url"/);
|
||||
assert.match(apiSource, /"\/crawlers\/test-script"/);
|
||||
assert.match(apiSource, /type CrawlerDryRunResult/);
|
||||
assert.match(apiSource, /id\?: string/);
|
||||
assert.match(apiSource, /new FormData\(\)/);
|
||||
assert.doesNotMatch(driveFormSource, /scriptcrawler/);
|
||||
});
|
||||
|
||||
test("drive cards use configured abbreviations and visible fallback icon colors", () => {
|
||||
assert.match(constantsSource, /googledrive:\s*"GD"/);
|
||||
assert.match(constantsSource, /function driveKindAbbr\(kind: string\)/);
|
||||
assert.match(constantsSource, /\.slice\(0, 2\)\.toUpperCase\(\)/);
|
||||
assert.match(drivesPageSource, /driveKindAbbr\(d\.kind\)/);
|
||||
assert.match(adminCss, /\.admin-drive-card__brand-icon\s*\{[^}]*background:\s*var\(--accent\);/s);
|
||||
assert.match(adminCss, /\.admin-drive-card__brand-icon\[data-kind="googledrive"\]\s*\{\s*background:\s*#4285f4;\s*\}/);
|
||||
});
|
||||
|
||||
test("drive management exposes stop task controls", () => {
|
||||
assert.match(apiSource, /stopDriveTasks/);
|
||||
assert.match(apiSource, /\/drives\/\$\{encodeURIComponent\(id\)\}\/tasks\/stop/);
|
||||
@@ -186,6 +276,43 @@ test("drive management exposes stop task controls", () => {
|
||||
assert.match(drivesPageSource, /停止所有网盘任务/);
|
||||
});
|
||||
|
||||
test("drive rescan reports busy storage tasks instead of queueing duplicates", () => {
|
||||
assert.match(apiSource, /accepted:\s*boolean;\s*message\?:\s*string/);
|
||||
assert.match(apiSource, /scanGenerationStatus\?: DriveGenerationStatus/);
|
||||
assert.match(drivesPageSource, /当前存储有正在进行的任务,请稍后重试/);
|
||||
assert.match(drivesPageSource, /function isDriveBusy\(d: api\.AdminDrive\)/);
|
||||
assert.match(drivesPageSource, /d\.scanGenerationStatus/);
|
||||
assert.match(drivesPageSource, /status\?\.state \|\| "idle"/);
|
||||
assert.match(drivesPageSource, /scanningDriveIdsRef\.current\.has\(d\.id\)/);
|
||||
assert.match(drivesPageSource, /if \(!resp\.accepted\)/);
|
||||
assert.doesNotMatch(drivesPageSource, /disabled=\{!!scanningDriveId\}/);
|
||||
});
|
||||
|
||||
test("nightly scan duplicate trigger uses full-scan busy message", () => {
|
||||
assert.match(apiSource, /status:\s*NightlyJobStatus;\s*message\?:\s*string/);
|
||||
assert.match(drivesPageSource, /当前有全量扫描任务正在进行,请稍后重试/);
|
||||
assert.match(drivesPageSource, /resp\.message \|\| NIGHTLY_BUSY_MESSAGE/);
|
||||
assert.match(constantsSource, /当前有全量扫描任务正在进行,请稍后重试/);
|
||||
});
|
||||
|
||||
test("drive generation panel shows scan or crawler status first", () => {
|
||||
assert.match(driveComponentsSource, /label=\{d\.kind === "spider91" \? "已废弃" : "扫盘"\}/);
|
||||
assert.match(driveComponentsSource, /status=\{d\.scanGenerationStatus\}/);
|
||||
assert.match(driveComponentsSource, /showCounts=\{false\}/);
|
||||
assert.match(driveComponentsSource, /status\?\.scannedCount/);
|
||||
assert.match(driveComponentsSource, /预计新增/);
|
||||
assert.match(apiSource, /scannedCount:\s*number/);
|
||||
assert.match(apiSource, /addedCount:\s*number/);
|
||||
assert.match(constantsSource, /if \(state === "scanning"\) return "扫盘中"/);
|
||||
});
|
||||
|
||||
test("legacy spider91 storage is disabled in drive management", () => {
|
||||
assert.match(drivesPageSource, /91Spider 不再支持通过网盘运行,请到爬虫管理添加爬虫脚本/);
|
||||
assert.match(drivesPageSource, /disabled=\{d\.kind === "spider91"\}/);
|
||||
assert.match(drivesPageSource, /已废弃,请到爬虫管理添加/);
|
||||
assert.match(constantsSource, /91Spider 不再支持通过网盘添加或编辑/);
|
||||
});
|
||||
|
||||
test("drive detail selection is stored in the URL history", () => {
|
||||
assert.match(drivesPageSource, /useSearchParams/);
|
||||
assert.match(drivesPageSource, /searchParams\.get\("drive"\)/);
|
||||
|
||||
@@ -15,3 +15,9 @@ test("admin modal does not reset focus when close handler identity changes", ()
|
||||
assert.match(modalSource, /\}, \[open\]\);/);
|
||||
assert.doesNotMatch(modalSource, /\}, \[open, onClose\]\);/);
|
||||
});
|
||||
|
||||
test("admin modal backdrop clicks do not close dialogs", () => {
|
||||
assert.match(modalSource, /className="admin-modal-backdrop"/);
|
||||
assert.doesNotMatch(modalSource, /onMouseDown=\{\(e\) =>/);
|
||||
assert.doesNotMatch(modalSource, /e\.target === e\.currentTarget/);
|
||||
});
|
||||
|
||||
@@ -100,6 +100,69 @@ test("admin video bulk actions use semantic theme colors", () => {
|
||||
assert.doesNotMatch(bulkBodies, /#ff5b8a|#fff6f9|rgba\(255,\s*91,\s*138/);
|
||||
});
|
||||
|
||||
test("admin video list summary stays below filter controls", () => {
|
||||
const toolbar = ruleBody(adminCss, ".admin-videos-list-toolbar");
|
||||
|
||||
assert.match(toolbar, /margin\s*:\s*var\(--space-2\)\s+0\s+var\(--space-4\)/);
|
||||
assert.doesNotMatch(toolbar, /margin\s*:\s*-/);
|
||||
});
|
||||
|
||||
test("admin table action headers center-align with action buttons", () => {
|
||||
const actionHeader = ruleBody(adminCss, ".admin-table th.is-actions");
|
||||
const actionCell = ruleBody(adminCss, ".admin-table td.is-actions");
|
||||
|
||||
assert.match(actionHeader, /text-align\s*:\s*center/);
|
||||
assert.match(actionCell, /text-align\s*:\s*center/);
|
||||
});
|
||||
|
||||
test("blacklist restore action uses a light button style", () => {
|
||||
const restoreButton = ruleBody(adminCss, ".admin-blacklist-restore-btn");
|
||||
|
||||
assert.match(videosPageSource, /className="admin-btn admin-blacklist-restore-btn"/);
|
||||
assert.match(restoreButton, /background\s*:\s*var\(--accent-softer\)/);
|
||||
assert.match(restoreButton, /color\s*:\s*var\(--accent\)/);
|
||||
assert.doesNotMatch(restoreButton, /background\s*:\s*var\(--accent\)/);
|
||||
});
|
||||
|
||||
test("admin video management controls wrap instead of covering text on mobile", () => {
|
||||
const css = mobileCss();
|
||||
const paginationInfo = allRuleBodies(css, ".admin-table-pagination__info");
|
||||
const bulkActions = allRuleBodies(css, ".admin-videos-bulk-actions");
|
||||
const bulkCount = allRuleBodies(css, ".admin-videos-bulk-actions__count");
|
||||
const bulkButton = allRuleBodies(css, ".admin-videos-bulk-actions__btn");
|
||||
const blacklistName = ruleBody(
|
||||
css,
|
||||
'.admin-blacklist-table:not(.admin-drives-table) td[data-label="文件名"]'
|
||||
);
|
||||
const blacklistTime = ruleBody(
|
||||
css,
|
||||
'.admin-blacklist-table:not(.admin-drives-table) td[data-label="拉黑时间"]'
|
||||
);
|
||||
const blacklistActions = ruleBody(
|
||||
css,
|
||||
".admin-blacklist-table:not(.admin-drives-table) td.is-actions"
|
||||
);
|
||||
const blacklistActionsLabel = ruleBody(
|
||||
css,
|
||||
".admin-blacklist-table:not(.admin-drives-table) td.is-actions::before"
|
||||
);
|
||||
const blacklistActionButton = ruleBody(
|
||||
css,
|
||||
".admin-blacklist-table:not(.admin-drives-table) td.is-actions .admin-btn"
|
||||
);
|
||||
|
||||
assert.match(paginationInfo, /flex\s*:\s*1\s+0\s+100%/);
|
||||
assert.match(bulkActions, /flex-wrap\s*:\s*wrap/);
|
||||
assert.match(bulkCount, /flex\s*:\s*1\s+0\s+100%/);
|
||||
assert.match(bulkButton, /min-width\s*:\s*0/);
|
||||
assert.match(blacklistName, /grid-column\s*:\s*1\s*\/\s*-1/);
|
||||
assert.match(blacklistTime, /grid-column\s*:\s*1/);
|
||||
assert.match(blacklistActions, /grid-column\s*:\s*2/);
|
||||
assert.match(blacklistActions, /justify-content\s*:\s*flex-end/);
|
||||
assert.match(blacklistActionsLabel, /content\s*:\s*none/);
|
||||
assert.match(blacklistActionButton, /white-space\s*:\s*normal/);
|
||||
});
|
||||
|
||||
test("admin loading spinner rotates around icon center", () => {
|
||||
const spinner = ruleBody(adminCss, ".admin-spin");
|
||||
const reducedMotion = ruleBodyByContains(adminCss, ".admin-sidebar__check-update:disabled svg");
|
||||
@@ -144,6 +207,7 @@ test("admin modals and action footers adapt on mobile", () => {
|
||||
// .admin-modal 桌面段已用 `width: min(620px, 100%)`,窄屏自然 100%;mobile 段
|
||||
// 只重写 max-height,所以这里断桌面规则即可。
|
||||
assert.match(ruleBody(adminCss, ".admin-modal"), /width\s*:\s*min\(\d+px,\s*100%\)/);
|
||||
assert.match(ruleBody(adminCss, ".admin-modal.admin-modal--crawler"), /width\s*:\s*min\(1080px,\s*100%\)/);
|
||||
// 多按钮 footer 在 mobile 下要换行避免溢出。
|
||||
assert.match(allRuleBodies(css, ".admin-modal__footer"), /flex-wrap\s*:\s*wrap/);
|
||||
// 删除/放弃类确认弹窗在 mobile 下不能跟随通用 modal stretch 到顶部。
|
||||
|
||||
@@ -14,9 +14,9 @@ test("admin videos page uses responsive page size", () => {
|
||||
|
||||
test("admin videos batch delete runs deletions sequentially", () => {
|
||||
assert.match(videosPageSource, /for \(const id of ids\) \{/);
|
||||
assert.match(videosPageSource, /const result = await api\.deleteVideo\(id\);/);
|
||||
assert.match(videosPageSource, /const result = await api\.deleteVideo\(id, \{ deleteSource: batchDeleteSource \}\);/);
|
||||
assert.doesNotMatch(
|
||||
videosPageSource,
|
||||
/Promise\.allSettled\(\s*ids\.map\(\(id\) => api\.deleteVideo\(id\)\)\s*\)/
|
||||
/Promise\.allSettled\(\s*ids\.map\(\(id\) => api\.deleteVideo\(id(?:, [^)]+)?\)\)\s*\)/
|
||||
);
|
||||
});
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user