mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-16 01:05:42 +08:00
Compare commits
42 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2437fbd779 | |||
| 4dd66b8120 | |||
| 30b736cf36 | |||
| 57391e0e98 | |||
| 052e142520 | |||
| f9351324c6 | |||
| bb83277d62 | |||
| aa856db1f6 | |||
| 7e5e67697e | |||
| 9cc8e02bec | |||
| 139e63eef2 | |||
| b8388eba59 | |||
| 76782f3801 | |||
| 1ae1408fb6 | |||
| 738406162a | |||
| 0f111b846d | |||
| 4dd9015bd7 | |||
| 84fbb6f51c | |||
| 992b20da93 | |||
| 1770693666 | |||
| 177041633a | |||
| ae324d3752 | |||
| 7f1e4eaa29 | |||
| 811d87cc27 | |||
| e4408f5655 | |||
| e93c906921 | |||
| 96e423b952 | |||
| a8ccc19e9e | |||
| 7ddf33d726 | |||
| c1355385e1 | |||
| ec5a01b6aa | |||
| 71d4a16db1 | |||
| 940e5dd76d | |||
| e826c05d5c | |||
| 3465b9e837 | |||
| d33c1b1b20 | |||
| 5fc8e9ebb7 | |||
| dc7d2a5de3 | |||
| 2f2bfbfcdc | |||
| 9def08b0c5 | |||
| c87208117e | |||
| a770b3af6b |
+15
@@ -35,3 +35,18 @@ tmp/
|
||||
91VideoSpider/__pycache__/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Local scratch images
|
||||
/*.png
|
||||
/*.jpg
|
||||
/*.jpeg
|
||||
/*.gif
|
||||
/*.webp
|
||||
/*.bmp
|
||||
/*.ico
|
||||
/image.jpg
|
||||
/image003.jpg
|
||||
/image004.jpg
|
||||
/image005.png
|
||||
/image006.png
|
||||
/image02.png
|
||||
|
||||
@@ -12,6 +12,9 @@
|
||||
pip install requests beautifulsoup4 lxml PySocks
|
||||
|
||||
使用方法:
|
||||
# 作为 video-site-91 通用爬虫脚本运行(后台会自动这样调用)
|
||||
python spider_91porn.py --job /path/to/job.json
|
||||
|
||||
# 全量爬取(默认行为,从 page=1 一直爬到末尾,写到 OUTPUT_FILE)
|
||||
python spider_91porn.py
|
||||
|
||||
@@ -22,6 +25,7 @@
|
||||
python spider_91porn.py --target-new 15 --seen-viewkeys-file /tmp/seen.txt --output /tmp/new.json
|
||||
|
||||
CLI 参数:
|
||||
--job FILE crawler.v1 job JSON 路径;后台爬虫管理会使用此模式
|
||||
--page N 只爬第 N 页,配合 --output 用于手动调试
|
||||
--target-new N 从 page 1 起翻页直到凑够 N 个新视频(不在 seen 列表里的)
|
||||
--seen-viewkeys-file FILE 每行一个已知 viewkey 或 mp4 源 ID,命中即跳过;与 --target-new 配合使用
|
||||
@@ -37,6 +41,8 @@ CLI 参数:
|
||||
- OUTPUT_FILE : 输出文件名
|
||||
|
||||
输出格式 (JSON):
|
||||
--job 模式下 stdout 输出 crawler.v1 JSON Lines,日志全部写到 stderr。
|
||||
手动运行模式仍会写传统 JSON 文件:
|
||||
{
|
||||
"videos": [
|
||||
{
|
||||
@@ -77,8 +83,8 @@ from datetime import datetime
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except ImportError:
|
||||
print("错误: 缺少依赖库 beautifulsoup4")
|
||||
print("请运行: pip install beautifulsoup4 lxml")
|
||||
print("错误: 缺少依赖库 beautifulsoup4", file=sys.stderr)
|
||||
print("请运行: pip install beautifulsoup4 lxml", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@@ -148,9 +154,24 @@ OUTPUT_FILE = "91porn_videos.json"
|
||||
MAX_PAGES = None # 设置为 None 爬取所有页,或设置整数如 5 只爬前5页
|
||||
RESUME = True # 是否跳过输出文件中已存在的 viewkey (断点续爬)
|
||||
MAX_EMPTY_PAGES = 2 # 连续空页数达到此值时停止爬取
|
||||
CRAWLER_NAME = "91Porn"
|
||||
CRAWLER_PROTOCOL = "crawler.v1"
|
||||
# ===================================================
|
||||
|
||||
|
||||
def crawler_source_id(raw: str) -> str:
|
||||
"""Return a backend-safe source_id, preserving existing numeric 91 IDs."""
|
||||
value = str(raw or "").strip()
|
||||
if not value:
|
||||
return ""
|
||||
safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", value).strip("._-")
|
||||
return safe[:160]
|
||||
|
||||
|
||||
def write_jsonl(event: dict):
|
||||
print(json.dumps(event, ensure_ascii=False), flush=True)
|
||||
|
||||
|
||||
class Porn91Spider:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -163,6 +184,7 @@ class Porn91Spider:
|
||||
target_new: int = None,
|
||||
seen_viewkeys: list = None,
|
||||
stream_output: bool = False,
|
||||
stream_protocol: str = "legacy",
|
||||
):
|
||||
"""
|
||||
构造函数。所有参数都有默认值,等同于使用脚本顶部的全局配置。
|
||||
@@ -198,6 +220,7 @@ class Porn91Spider:
|
||||
# (配合 backend Go 端 bufio.Scanner 实时消费,下载一个就开始下一个)。
|
||||
# 开启后所有 log 都走 stderr。
|
||||
self.stream_output = bool(stream_output)
|
||||
self.stream_protocol = stream_protocol or "legacy"
|
||||
|
||||
# 添加重试适配器
|
||||
try:
|
||||
@@ -263,7 +286,28 @@ class Porn91Spider:
|
||||
if not self.stream_output:
|
||||
return
|
||||
try:
|
||||
print(json.dumps(video, ensure_ascii=False), flush=True)
|
||||
if self.stream_protocol == "crawler.v1":
|
||||
source_id = crawler_source_id(video.get("source_id") or video.get("viewkey") or "")
|
||||
item = {
|
||||
"title": video.get("title") or "",
|
||||
"detail_url": video.get("detail_url") or "",
|
||||
"author": "91porn",
|
||||
"tags": ["91porn"],
|
||||
"media_url": video.get("video_url") or "",
|
||||
"thumbnail_url": video.get("thumb_url") or "",
|
||||
"headers": {
|
||||
"Referer": video.get("detail_url") or BASE_URL,
|
||||
},
|
||||
}
|
||||
if source_id:
|
||||
item["source_id"] = source_id
|
||||
event = {
|
||||
"type": "item",
|
||||
"item": item,
|
||||
}
|
||||
write_jsonl(event)
|
||||
else:
|
||||
print(json.dumps(video, ensure_ascii=False), flush=True)
|
||||
except Exception as e:
|
||||
# stdout 异常基本只在管道断开时发生(消费方进程死了);
|
||||
# 写到 stderr 让 backend 看到,然后让 crawl 循环自己 break。
|
||||
@@ -697,8 +741,9 @@ class Porn91Spider:
|
||||
except Exception as e:
|
||||
self.log(f"保存文件失败: {e}")
|
||||
# 尝试输出到控制台作为备份
|
||||
print("\n--- 备份输出 ---")
|
||||
print(json.dumps(output_data, ensure_ascii=False, indent=2))
|
||||
backup_out = sys.stderr if self.stream_output else sys.stdout
|
||||
print("\n--- 备份输出 ---", file=backup_out, flush=True)
|
||||
print(json.dumps(output_data, ensure_ascii=False, indent=2), file=backup_out, flush=True)
|
||||
|
||||
def _print_summary(self):
|
||||
"""
|
||||
@@ -751,6 +796,84 @@ def print_help():
|
||||
""")
|
||||
|
||||
|
||||
def run_job(job_path: str):
|
||||
"""Run as a crawler.v1 script plugin.
|
||||
|
||||
The Go host passes a job JSON file and expects stdout JSONL events. Logs go
|
||||
to stderr so stdout stays machine-readable.
|
||||
"""
|
||||
with open(job_path, "r", encoding="utf-8") as f:
|
||||
job = json.load(f)
|
||||
|
||||
if job.get("protocol") != CRAWLER_PROTOCOL:
|
||||
raise ValueError(f"unsupported crawler protocol: {job.get('protocol')!r}")
|
||||
if job.get("mode") not in ("", None, "crawl"):
|
||||
raise ValueError(f"unsupported crawler mode: {job.get('mode')!r}")
|
||||
|
||||
try:
|
||||
target_new = int(job.get("target_new") or 15)
|
||||
except (TypeError, ValueError):
|
||||
target_new = 15
|
||||
if target_new <= 0:
|
||||
target_new = 15
|
||||
seen_file = job.get("seen_source_ids_file") or ""
|
||||
output_dir = job.get("output_dir") or os.getcwd()
|
||||
run_id = job.get("run_id") or datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
output_file = os.path.join(output_dir, f"spider91-{run_id}.json")
|
||||
|
||||
network = job.get("network") if isinstance(job.get("network"), dict) else {}
|
||||
proxy_url = str(network.get("proxy_url") or "").strip()
|
||||
if proxy_url:
|
||||
os.environ["HTTP_PROXY"] = proxy_url
|
||||
os.environ["HTTPS_PROXY"] = proxy_url
|
||||
os.environ["http_proxy"] = proxy_url
|
||||
os.environ["https_proxy"] = proxy_url
|
||||
os.environ["NO_PROXY"] = ""
|
||||
os.environ["no_proxy"] = ""
|
||||
|
||||
seen_viewkeys = []
|
||||
if seen_file:
|
||||
try:
|
||||
with open(seen_file, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
seen_viewkeys.append(line)
|
||||
except FileNotFoundError:
|
||||
print(f"警告: seen_source_ids_file 不存在: {seen_file}", file=sys.stderr, flush=True)
|
||||
except Exception as e:
|
||||
print(f"警告: 读取 seen_source_ids_file 失败: {e}", file=sys.stderr, flush=True)
|
||||
|
||||
prefer_ipv4_for_plain_socks5_proxy()
|
||||
spider = Porn91Spider(
|
||||
output_file=output_file,
|
||||
start_page=1,
|
||||
max_pages=None,
|
||||
resume=False,
|
||||
quiet=True,
|
||||
target_new=target_new,
|
||||
seen_viewkeys=seen_viewkeys,
|
||||
stream_output=True,
|
||||
stream_protocol="crawler.v1",
|
||||
)
|
||||
try:
|
||||
spider.crawl()
|
||||
done = {
|
||||
"type": "done",
|
||||
"stats": {
|
||||
"emitted": spider.processed_videos,
|
||||
"failed": spider.failed_videos,
|
||||
"skipped": spider.skipped_videos,
|
||||
},
|
||||
}
|
||||
write_jsonl(done)
|
||||
except KeyboardInterrupt:
|
||||
spider.log("\n用户中断,正在保存已爬取的数据...")
|
||||
spider._save_results()
|
||||
raise
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) > 1 and sys.argv[1] in ('-h', '--help', 'help'):
|
||||
print_help()
|
||||
@@ -778,8 +901,14 @@ def main():
|
||||
parser.add_argument("--stream-output", action="store_true",
|
||||
help="流式模式:每解析一条视频直链就立即把它作为一行 JSON 写到 stdout 并 flush;"
|
||||
"日志改走 stderr。配合 backend 边读边下载使用。")
|
||||
parser.add_argument("--job", type=str, default=None,
|
||||
help="crawler.v1 job JSON 路径;作为通用脚本爬虫运行。")
|
||||
|
||||
args, _ = parser.parse_known_args()
|
||||
if args.job:
|
||||
run_job(args.job)
|
||||
return
|
||||
|
||||
cli_out = sys.stderr if args.stream_output else sys.stdout
|
||||
prefer_ipv4_for_plain_socks5_proxy()
|
||||
|
||||
|
||||
@@ -48,7 +48,6 @@ WORKDIR /opt/video-site-91
|
||||
COPY --from=backend /out/server ./server
|
||||
COPY --from=frontend /app/dist ./dist
|
||||
COPY backend/config.example.yaml ./config.example.yaml
|
||||
COPY 91VideoSpider/ ./91VideoSpider/
|
||||
COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
@@ -20,14 +20,11 @@
|
||||
|
||||
## 功能特性
|
||||
|
||||
- **多后端支持** — 兼容 115 云盘、PikPak 云盘、123云盘、OneDrive、Google Drive 和本地存储
|
||||
- **低带宽播放** — 115 云盘、PikPak 云盘、123云盘、OneDrive 都支持302模式,在线播放视频时,不占用服务器带宽,播放体验不受服务器带宽影响;Google Drive 不支持302模式,走服务器中转,观看体验会受服务器带宽影响
|
||||
- **多后端支持** — 兼容 115 云盘、PikPak 云盘、123网盘、联通网盘、光鸭网盘、OneDrive、Google Drive 和本地存储
|
||||
- **低带宽播放** — 115 云盘、PikPak 云盘、123网盘、联通网盘、光鸭网盘、OneDrive 支持302模式,在线播放视频时,不占用服务器带宽,播放体验不受服务器带宽影响;Google Drive 不支持302模式,走服务器中转,观看体验会受服务器带宽影响
|
||||
- **封面 & 预览片段** — 自动为每个视频生成封面图和预览片段,首页快速选片
|
||||
- **91 爬虫** — 内置爬虫,支持抓取 91 本月最热视频
|
||||
- **双主题** — 黑黄经典主题 / 粉白清新主题,随时切换
|
||||
- **爬虫脚本** — 项目支持导入自定义脚本,但是有一些规范,具体可以参考 [SpiderFor91](https://github.com/Just-Spider/SpiderFor91),项目不再内置任何爬虫脚本
|
||||
- **短视频模式** — 一键切换抖音风格,沉浸刷片
|
||||
- **低资源占用** — 2C2G 服务器稳定运行,主要性能消耗就是封面图和预览视频的生成
|
||||
|
||||
---
|
||||
|
||||
## 预览图
|
||||
@@ -84,6 +81,14 @@ sudo bash install.sh
|
||||
|
||||
> `video-site-91` 为等效别名,两者可互换使用。
|
||||
|
||||
**已部署用户升级:**
|
||||
|
||||
```bash
|
||||
91 update
|
||||
```
|
||||
|
||||
升级会保留现有 `config.yaml`、数据库、封面、预览、上传文件和爬虫数据。脚本会自动安装或检查 `ffmpeg` / `ffprobe` 等运行依赖,并在新版本启动失败时回滚到升级前文件。
|
||||
|
||||
**自定义端口:**
|
||||
|
||||
```bash
|
||||
@@ -155,6 +160,7 @@ docker compose up -d # 更新并重启
|
||||
```
|
||||
|
||||
> 所有配置、数据库、封面、预览及上传文件均保存在 `./data/` 目录下。
|
||||
> 从旧版本升级 Docker 部署时,执行 `docker compose pull && docker compose up -d` 即可;`./data/` 不会被镜像更新覆盖。
|
||||
|
||||
---
|
||||
|
||||
@@ -180,14 +186,6 @@ docker compose up -d # 更新并重启
|
||||
|
||||
---
|
||||
|
||||
## 更多文档
|
||||
|
||||
| 文档 | 内容 |
|
||||
|------|------|
|
||||
| [backend/README.md](backend/README.md) | 后端实现、接口说明、网盘字段 |
|
||||
|
||||
---
|
||||
|
||||
## 使用须知
|
||||
|
||||
本项目面向**个人私有部署**,请仅接入你有权访问和管理的内容,并遵守对应网盘、站点的服务条款及所在地法律法规。
|
||||
@@ -196,6 +194,14 @@ docker compose up -d # 更新并重启
|
||||
|
||||
---
|
||||
|
||||
## PR提交规范
|
||||
欢迎大家提交PR,一起来完善这个项目,但是这里要说明一下PR提交的规范
|
||||
1. 一个PR的功能改动要单一,不建议一个PR修改了大量功能。单个PR单个功能修改,这样也更容易Merge
|
||||
2. 完善项目的PR比新增功能的PR更容易Merge(例如:例如你发现开发者没有实现爬取的视频上传到某个网盘,并且你有这个需求,此时你可以实现一下这个功能然后提交PR,也感谢你为开发者分担工作量)
|
||||
3. 新增功能的PR不容易Merge,因为某些功能的需求可能不是所有人都需要的,如果一味的不断增加功能,会让项目变得过于庞大。当然如果你肯定你的新功能和想法很好,并且相信将会对于项目有很大的改善,那么热烈欢迎你的PR
|
||||
|
||||
---
|
||||
|
||||
## 许可证
|
||||
|
||||
本项目基于 [MIT License](LICENSE) 开源。
|
||||
|
||||
+11
-9
@@ -2,7 +2,7 @@
|
||||
|
||||
视频聚合站的 Go 后端。提供三件事:
|
||||
|
||||
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive / Google Drive / 本地存储)
|
||||
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通网盘 / 光鸭网盘 / OneDrive / Google Drive / 本地存储)
|
||||
2. 视频元数据目录(SQLite)+ 扫描 + 预览视频预生成
|
||||
3. REST API(前台)+ 管理后台 + 直链代理
|
||||
4. 标签池、视频隐藏、按网盘统计和详情页来源网盘类型展示能力
|
||||
@@ -19,7 +19,8 @@ internal/
|
||||
quark/ 夸克(自己实现,参考 OpenList quark_uc)
|
||||
p115/ 115(壳子 + SheltonZhu/115driver)
|
||||
pikpak/ PikPak(自己实现,参考 OpenList pikpak)
|
||||
wopan/ 联通沃盘(壳子 + OpenListTeam/wopan-sdk-go)
|
||||
wopan/ 联通网盘(壳子 + OpenListTeam/wopan-sdk-go)
|
||||
guangyapan/ 光鸭网盘(参考 AList GuangYaPan)
|
||||
onedrive/ OneDrive(OpenList 在线续期 + Microsoft Graph 文件接口)
|
||||
googledrive/ Google Drive(OpenList 在线续期 + Google Drive API;播放走后端代理)
|
||||
localstorage/ 本地目录扫描(服务器已有视频目录)
|
||||
@@ -108,8 +109,9 @@ go run ./cmd/server 后端 9192
|
||||
| p115 | `cookie`(形如 `UID=...; CID=...; SEID=...; KID=...`) |
|
||||
| pikpak | `username`、`password`(token、验证码和设备 ID 由服务端自动处理并保存) |
|
||||
| wopan | `access_token`、`refresh_token`,可选 `family_id` |
|
||||
| guangyapan | 推荐后台扫码登录自动写入 `access_token`、`refresh_token`;也可手工填写 token;可选 `root_path` |
|
||||
| onedrive | `refresh_token` |
|
||||
| googledrive | `refresh_token` |
|
||||
| googledrive | 默认只需 `refresh_token`;自建 OAuth 客户端模式还需 `use_online_api=false`、`client_id`、`client_secret` |
|
||||
| localstorage | `path`(服务器上的已有视频目录,如 `/mnt/videos`) |
|
||||
|
||||
### PikPak 速度说明
|
||||
@@ -120,7 +122,7 @@ go run ./cmd/server 后端 9192
|
||||
|
||||
OneDrive 按 OpenList 默认应用方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。后台新建 OneDrive 时只需要填 OpenList 代刷得到的 `refresh_token`;服务端会默认挂载根目录并自动回写新 token。
|
||||
|
||||
Google Drive 按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/renewapi` 刷新 token。后台新建 Google Drive 时只需要填 OpenList Google Drive 获取到的 `refresh_token`。Google Drive 下载地址必须携带 `Authorization` 头,浏览器不能直接 302 使用,所以本站会由后端代理 `/p/stream` 播放,不加入零带宽 302 白名单。
|
||||
Google Drive 默认按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/renewapi` 刷新 token。后台新建 Google Drive 时只需要填 OpenList Google Drive 获取到的 `refresh_token`。如果不想依赖 OpenList 在线 API,可以关闭“使用 OpenList 在线续期 API”,并填写同一个 Google OAuth 客户端授权得到的 `refresh_token`、`client_id`、`client_secret`,服务端会直接请求 Google OAuth token 接口续期。Google Drive 下载地址必须携带 `Authorization` 头,浏览器不能直接 302 使用,所以本站会由后端代理 `/p/stream` 播放,不加入零带宽 302 白名单。
|
||||
|
||||
## 文件名约定
|
||||
|
||||
@@ -145,18 +147,18 @@ Google Drive 按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/ren
|
||||
|
||||
1. 同一网盘同一文件按 `(drive_id, file_id)` 形成稳定视频 ID,重复扫描只更新同一行。
|
||||
2. 扫描时优先按网盘侧 `content_hash` 去重;没有 hash 时退化为 `file_name + size_bytes`。
|
||||
3. 扫描、爬虫、本地上传或服务启动挂载网盘后,后台指纹 worker 会异步读取视频的少量 Range 片段,生成 `sampled_sha256`。前台列表、首页、搜索、推荐会按 `size_bytes + sampled_sha256` 只展示最早入库的 canonical 视频。
|
||||
3. 扫描、本地上传或服务启动挂载网盘后,后台指纹 worker 会异步读取视频的少量 Range 片段,生成 `sampled_sha256`。前台列表、首页、搜索、推荐会按 `size_bytes + sampled_sha256` 只展示最早入库的 canonical 视频。
|
||||
|
||||
`sampled_sha256` 是文件级去重:适合识别同一个视频文件被复制到 115 / PikPak / OneDrive 等不同网盘的情况。它不会删除任何网盘文件,也不用于识别转码、裁剪、加水印后的同源视频。
|
||||
`sampled_sha256` 是文件级去重:适合识别同一个视频文件被复制到 115 / PikPak / OneDrive / Google Drive 等不同网盘的情况。它不会删除任何网盘文件,也不用于识别转码、裁剪、加水印后的同源视频。
|
||||
|
||||
封面和预览视频仍然优先生成,不等待指纹完成。夜间流水线最后会做一次重复资产清理:对 `size_bytes + sampled_sha256` 命中的非 canonical 视频,只删除本机生成的重复封面和预览视频,并把对应字段重置为 `pending`。网盘原文件和视频元数据记录不会被删除;如果 canonical 视频以后被移除,这些重复项会重新进入生成队列。
|
||||
|
||||
## 管理能力
|
||||
|
||||
- `/admin/drives`:新增、编辑、删除网盘,触发扫描。
|
||||
- `/admin/videos`:按网盘筛选视频,每页 100 条分页,查看各网盘预览视频统计,编辑标题/作者/分类/标签,单条或全量重生预览视频。
|
||||
- `/admin/videos`:按网盘筛选视频,每页 100 条分页,查看各网盘预览视频统计,编辑标题/作者/分类/标签,单条或全量重生预览视频;拉黑视频页可查看被删除或被隐藏的视频,并支持移出黑名单后在下次扫盘重新入库。
|
||||
- `/admin/tags`:新增标签并用内置规则自动匹配已有视频;删除非系统标签时会从所有视频上同步移除该标签。
|
||||
- 播放页视频信息会展示来源网盘类型;同时提供“不再展示”,点击后会把视频标记为全局隐藏。隐藏视频不会再出现在首页、列表、搜索、相关推荐和详情接口中。目前没有管理后台恢复入口,如需恢复可把数据库里对应视频的 `hidden` 字段改回 `0`。
|
||||
- 播放页视频信息会展示来源网盘类型,并提供删除入口。被删除或被隐藏的视频会进入黑名单,不会再出现在首页、列表、搜索和详情接口中;在后台移出黑名单后,会在下次扫盘时重新发现并入库。
|
||||
|
||||
## 预览视频生成
|
||||
|
||||
@@ -170,7 +172,7 @@ ffmpeg -ss <起点> -headers "UA/Cookie/Referer" -i <直链> \
|
||||
|
||||
当前策略是每段固定 3 秒;30 秒以下最多 3 段,30 秒及以上固定 4 段;长视频在 20% 到 80% 区间均匀取段。生成的预览视频和封面都只保存在本地 `data/previews/`,不会回写到网盘;旧数据中的 `preview_file_id` 会被忽略。
|
||||
|
||||
服务启动或网盘重新挂载时,如果预览视频开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 扫盘和直链生成预览视频 / 封面时可能触发 Microsoft Graph 429、`TooManyRequests`、`activityLimitReached` 或 throttled 文本;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。扫盘阶段会按 `Retry-After` 或默认冷却时间等待后继续当前目录。
|
||||
服务启动或网盘重新挂载时,如果预览视频开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 扫盘和直链生成预览视频 / 封面时可能触发 Microsoft Graph 429、`TooManyRequests`、`activityLimitReached` 或 throttled 文本;Google Drive 可能返回 429、`usageLimits`、`userRateLimitExceeded`、`downloadQuotaExceeded` 等限制标识。后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。扫盘阶段会按 `Retry-After` 或默认冷却时间等待后继续当前目录。
|
||||
|
||||
前端卡片的 `previewSrc` 统一指向 `/p/preview/<videoID>`,后端只从本地 `preview_local` 文件读取。
|
||||
|
||||
|
||||
+1128
-189
File diff suppressed because it is too large
Load Diff
@@ -40,6 +40,8 @@ func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
|
||||
reg.Set("p115-one", &spider91UploadTargetFakeDrive{id: "p115-one", kind: "p115"})
|
||||
reg.Set("p123-one", &spider91UploadTargetFakeDrive{id: "p123-one", kind: "p123"})
|
||||
reg.Set("onedrive-one", &spider91UploadTargetFakeDrive{id: "onedrive-one", kind: "onedrive"})
|
||||
reg.Set("wopan-one", &spider91UploadTargetFakeDrive{id: "wopan-one", kind: "wopan"})
|
||||
reg.Set("guangyapan-one", &spider91UploadTargetFakeDrive{id: "guangyapan-one", kind: "guangyapan"})
|
||||
|
||||
app := &App{registry: reg}
|
||||
if got := app.Spider91UploadDriveID(); got != "" {
|
||||
@@ -61,6 +63,16 @@ func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
|
||||
t.Fatalf("explicit onedrive upload target = %q, want onedrive-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "wopan-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "wopan-one" {
|
||||
t.Fatalf("explicit wopan upload target = %q, want wopan-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "guangyapan-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "guangyapan-one" {
|
||||
t.Fatalf("explicit guangyapan upload target = %q, want guangyapan-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "missing"
|
||||
if got := app.Spider91UploadDriveID(); got != "" {
|
||||
t.Fatalf("missing upload target = %q, want empty", got)
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -13,6 +14,7 @@ import (
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/config"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/scriptcrawler"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/fingerprint"
|
||||
"github.com/video-site/backend/internal/preview"
|
||||
@@ -225,6 +227,53 @@ func TestRegisterPreviewWorkersBackfillsHistoricalFingerprints(t *testing.T) {
|
||||
t.Fatalf("fingerprint status=%q sampled=%q, want ready with hash", got.FingerprintStatus, got.SampledSHA256)
|
||||
}
|
||||
|
||||
func TestUpdateScriptCrawlerRunStatePreservesCurrentTeaserSwitch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "crawler-id",
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Crawler",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"script_path": "/tmp/crawler.py",
|
||||
"target_new": "10",
|
||||
},
|
||||
TeaserEnabled: false,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed crawler drive: %v", err)
|
||||
}
|
||||
if err := cat.SetDriveTeaserEnabled(ctx, "crawler-id", true); err != nil {
|
||||
t.Fatalf("toggle teaser: %v", err)
|
||||
}
|
||||
|
||||
app := &App{cat: cat}
|
||||
if err := app.updateScriptCrawlerRunState(ctx, "crawler-id", nil); err != nil {
|
||||
t.Fatalf("update run state: %v", err)
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, "crawler-id")
|
||||
if err != nil {
|
||||
t.Fatalf("get crawler drive: %v", err)
|
||||
}
|
||||
if !got.TeaserEnabled {
|
||||
t.Fatal("teaserEnabled = false after run state update, want preserved true")
|
||||
}
|
||||
if got.Status != "ok" || got.LastError != "" {
|
||||
t.Fatalf("status=%q lastError=%q, want ok with no error", got.Status, got.LastError)
|
||||
}
|
||||
if got.Credentials["last_crawl_at"] == "" || got.Credentials["target_new"] != "10" {
|
||||
t.Fatalf("credentials after run state update = %#v", got.Credentials)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopDriveTasksCancelsQueuedTasksAndReplacesWorkers(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
@@ -260,6 +309,7 @@ func TestStopDriveTasksCancelsQueuedTasksAndReplacesWorkers(t *testing.T) {
|
||||
"drive-id": func() { close(oldCanceled) },
|
||||
},
|
||||
scanQueued: map[string]bool{"drive-id": true},
|
||||
scanProgress: map[string]driveScanProgress{"drive-id": {Scanned: 8, Added: 2}},
|
||||
fingerprintQueueing: map[string]bool{"drive-id": true},
|
||||
}
|
||||
taskCtx, done := app.registerDriveTaskContext(ctx, "drive-id")
|
||||
@@ -279,6 +329,9 @@ func TestStopDriveTasksCancelsQueuedTasksAndReplacesWorkers(t *testing.T) {
|
||||
if app.scanQueued["drive-id"] {
|
||||
t.Fatal("scan queue marker was not cleared")
|
||||
}
|
||||
if _, ok := app.scanProgress["drive-id"]; ok {
|
||||
t.Fatal("scan progress marker was not cleared")
|
||||
}
|
||||
if app.fingerprintQueueing["drive-id"] {
|
||||
t.Fatal("fingerprint queue marker was not cleared")
|
||||
}
|
||||
@@ -304,6 +357,227 @@ func TestStopDriveTasksCancelsQueuedTasksAndReplacesWorkers(t *testing.T) {
|
||||
newCancel()
|
||||
}
|
||||
|
||||
func TestScheduleScanRejectsDriveWithActiveGenerationWork(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
thumbWorker := preview.NewThumbWorker(&serverFakeTeaserGenerator{}, nil, &serverFakeDrive{})
|
||||
if !thumbWorker.Enqueue(&catalog.Video{ID: "busy-video", DriveID: "drive-id", Title: "Busy Video"}) {
|
||||
t.Fatal("failed to enqueue busy thumbnail task")
|
||||
}
|
||||
app := &App{
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{"drive-id": thumbWorker},
|
||||
}
|
||||
|
||||
if app.scheduleScan(ctx, "drive-id") {
|
||||
t.Fatal("scheduleScan accepted a drive with active generation work")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleScanRunsDifferentDrivesConcurrently(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
seedDriveWithTeaser(t, cat, "drive-a", true)
|
||||
seedDriveWithTeaser(t, cat, "drive-b", true)
|
||||
|
||||
started := make(chan string, 2)
|
||||
release := make(chan struct{})
|
||||
registry := proxy.NewRegistry()
|
||||
registry.Set("drive-a", &serverBlockingListDrive{id: "drive-a", started: started, release: release})
|
||||
registry.Set("drive-b", &serverBlockingListDrive{id: "drive-b", started: started, release: release})
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{
|
||||
Scanner: config.Scanner{VideoExtensions: []string{".mp4"}},
|
||||
},
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
}
|
||||
|
||||
if !app.scheduleScan(ctx, "drive-a") {
|
||||
t.Fatal("scheduleScan drive-a was rejected")
|
||||
}
|
||||
if !app.scheduleScan(ctx, "drive-b") {
|
||||
t.Fatal("scheduleScan drive-b was rejected")
|
||||
}
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
deadline := time.After(time.Second)
|
||||
for len(seen) < 2 {
|
||||
select {
|
||||
case id := <-started:
|
||||
seen[id] = struct{}{}
|
||||
case <-deadline:
|
||||
close(release)
|
||||
t.Fatalf("started drives = %#v, want both drives before releasing List", seen)
|
||||
}
|
||||
}
|
||||
close(release)
|
||||
}
|
||||
|
||||
func TestDriveGenerationStatusIncludesScanState(t *testing.T) {
|
||||
app := &App{
|
||||
scanQueued: map[string]bool{"drive-id": true},
|
||||
scanProgress: map[string]driveScanProgress{"drive-id": {Scanned: 12, Added: 3}},
|
||||
}
|
||||
|
||||
status := app.driveGenerationStatuses()["drive-id"].Scan
|
||||
if status.State != "scanning" {
|
||||
t.Fatalf("scan status = %#v, want scanning", status)
|
||||
}
|
||||
if status.ScannedCount != 12 || status.AddedCount != 3 {
|
||||
t.Fatalf("scan counts = scanned %d added %d, want 12 and 3", status.ScannedCount, status.AddedCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriveGenerationStatusIncludesScanCooldown(t *testing.T) {
|
||||
until := time.Now().Add(time.Hour).Round(time.Second)
|
||||
app := &App{
|
||||
scanQueued: map[string]bool{"drive-id": true},
|
||||
scanProgress: map[string]driveScanProgress{
|
||||
"drive-id": {Scanned: 12, Added: 3, CooldownUntil: until},
|
||||
},
|
||||
}
|
||||
|
||||
status := app.driveGenerationStatuses()["drive-id"].Scan
|
||||
if status.State != "cooling" {
|
||||
t.Fatalf("scan status = %#v, want cooling", status)
|
||||
}
|
||||
if status.CooldownUntil != until.Format(time.RFC3339) {
|
||||
t.Fatalf("cooldown until = %q, want %q", status.CooldownUntil, until.Format(time.RFC3339))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuangYaPanGenerationCooldowns(t *testing.T) {
|
||||
drv := &serverFakeKindDrive{id: "gy", kind: "guangyapan"}
|
||||
if got := generationCooldownForDrive(drv); got != 10*time.Minute {
|
||||
t.Fatalf("generation cooldown = %s, want 10m", got)
|
||||
}
|
||||
if got := fingerprintConfigForDrive(drv).RateLimitCooldown; got != 10*time.Minute {
|
||||
t.Fatalf("fingerprint cooldown = %s, want 10m", got)
|
||||
}
|
||||
if got := scanCooldownForDrive(drv); got != 10*time.Minute {
|
||||
t.Fatalf("scan cooldown = %s, want 10m", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSpider91MigrationAfterManualCrawlRequiresConfiguredUploadTarget(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
registry := proxy.NewRegistry()
|
||||
migrator := &serverFakeSpider91MigrationRunner{}
|
||||
app := &App{
|
||||
registry: registry,
|
||||
spider91Migrator: migrator,
|
||||
workers: map[string]*preview.Worker{},
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{},
|
||||
fingerprintWorkers: map[string]*fingerprint.Worker{},
|
||||
}
|
||||
|
||||
app.runSpider91MigrationAfterManualCrawl(ctx, "91spider")
|
||||
if migrator.called != 0 {
|
||||
t.Fatalf("migration called without upload target")
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "pikpak"
|
||||
app.runSpider91MigrationAfterManualCrawl(ctx, "91spider")
|
||||
if migrator.called != 0 {
|
||||
t.Fatalf("migration called when upload target is not attached")
|
||||
}
|
||||
|
||||
registry.Set("pikpak", &serverFakeKindDrive{id: "pikpak", kind: "pikpak"})
|
||||
app.runSpider91MigrationAfterManualCrawl(ctx, "91spider")
|
||||
if migrator.called != 1 {
|
||||
t.Fatalf("migration calls = %d, want 1", migrator.called)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleCrawlerUploadMigrationRunsForConfiguredCrawler(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "crawler-truvaze",
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Truvaze",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"script_path": "/tmp/Truvaze.py",
|
||||
"upload_drive_id": "pikpak",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed crawler: %v", err)
|
||||
}
|
||||
registry := proxy.NewRegistry()
|
||||
registry.Set("crawler-truvaze", &serverFakeKindDrive{id: "crawler-truvaze", kind: scriptcrawler.Kind})
|
||||
migrator := &serverFakeSpider91MigrationRunner{}
|
||||
app := &App{
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
spider91Migrator: migrator,
|
||||
workers: map[string]*preview.Worker{},
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{},
|
||||
fingerprintWorkers: map[string]*fingerprint.Worker{},
|
||||
}
|
||||
|
||||
if !app.scheduleCrawlerUploadMigration(ctx, "crawler-truvaze") {
|
||||
t.Fatal("scheduleCrawlerUploadMigration returned false, want true")
|
||||
}
|
||||
deadline := time.After(time.Second)
|
||||
for migrator.called == 0 {
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatalf("migration calls = %d, want 1", migrator.called)
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleCrawlerUploadMigrationSkipsWithoutUploadTarget(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "crawler-local",
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Local Only",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{"script_path": "/tmp/local.py"},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed crawler: %v", err)
|
||||
}
|
||||
migrator := &serverFakeSpider91MigrationRunner{}
|
||||
app := &App{cat: cat, registry: proxy.NewRegistry(), spider91Migrator: migrator}
|
||||
|
||||
if app.scheduleCrawlerUploadMigration(ctx, "crawler-local") {
|
||||
t.Fatal("scheduleCrawlerUploadMigration returned true without upload target")
|
||||
}
|
||||
if migrator.called != 0 {
|
||||
t.Fatalf("migration calls = %d, want 0", migrator.called)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriveGenerationStatusUsesWorkerQueueNotPendingCatalogRows(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -491,7 +765,9 @@ func TestNightlyTargetsComeFromCatalogBeforeDriveAttach(t *testing.T) {
|
||||
for _, d := range []*catalog.Drive{
|
||||
{ID: "115", Kind: "p115", Name: "115", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "pikpak", Kind: "pikpak", Name: "PikPak", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "91-spider", Kind: "spider91", Name: "91 Spider", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "91-legacy", Kind: "spider91", Name: "91 Legacy", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "91-crawler", Kind: scriptcrawler.Kind, Name: "91 Spider", RootID: "/", Credentials: map[string]string{"script_path": "/tmp/crawler.py"}, TeaserEnabled: true},
|
||||
{ID: "91-crawler-deleted", Kind: scriptcrawler.Kind, Name: "Deleted Spider", RootID: "/", Credentials: map[string]string{}, TeaserEnabled: true},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
t.Fatalf("seed drive %s: %v", d.ID, err)
|
||||
@@ -504,8 +780,47 @@ func TestNightlyTargetsComeFromCatalogBeforeDriveAttach(t *testing.T) {
|
||||
t.Fatalf("scan target ids = %#v, want 115 and pikpak from catalog", scanIDs)
|
||||
}
|
||||
spiderIDs := app.listSpider91DriveIDs(ctx)
|
||||
if len(spiderIDs) != 1 || spiderIDs[0] != "91-spider" {
|
||||
t.Fatalf("spider91 ids = %#v, want catalog spider drive", spiderIDs)
|
||||
if len(spiderIDs) != 1 || spiderIDs[0] != "91-crawler" {
|
||||
t.Fatalf("spider91 ids = %#v, want crawler-page script drive", spiderIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAttachDriveRejectsLegacySpider91Storage(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
d := &catalog.Drive{
|
||||
ID: "91-legacy",
|
||||
Kind: spider91.Kind,
|
||||
Name: "91 Legacy",
|
||||
RootID: "/",
|
||||
TeaserEnabled: true,
|
||||
}
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
app := &App{cat: cat, registry: proxy.NewRegistry()}
|
||||
err = app.attachDrive(ctx, d)
|
||||
if err == nil || !strings.Contains(err.Error(), "爬虫管理") {
|
||||
t.Fatalf("attach err = %v, want crawler management guidance", err)
|
||||
}
|
||||
if _, ok := app.registry.Get(d.ID); ok {
|
||||
t.Fatal("legacy spider91 drive should not be registered")
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, d.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.Status != "error" || !strings.Contains(got.LastError, "爬虫管理") {
|
||||
t.Fatalf("status/error = %q/%q, want deprecated error", got.Status, got.LastError)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -918,7 +1233,6 @@ func TestCleanupDriveVideosForDeleteRemovesRowsAndGeneratedAssetsOnly(t *testing
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
fingerprintWorkers: make(map[string]*fingerprint.Worker),
|
||||
spider91Crawlers: make(map[string]*spider91.Crawler),
|
||||
}
|
||||
removed, err := app.cleanupDriveVideosForDelete(ctx, "local-main")
|
||||
if err != nil {
|
||||
@@ -1007,7 +1321,7 @@ func TestDeleteVideoRemovesGeneratedAssetsKeepsLocalOriginalAndTombstones(t *tes
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
}
|
||||
result, err := app.deleteVideo(ctx, "localstorage-local-main-file")
|
||||
result, err := app.deleteVideo(ctx, "localstorage-local-main-file", false)
|
||||
if err != nil {
|
||||
t.Fatalf("delete video: %v", err)
|
||||
}
|
||||
@@ -1034,6 +1348,126 @@ func TestDeleteVideoRemovesGeneratedAssetsKeepsLocalOriginalAndTombstones(t *tes
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteVideoRemovesSourceFileWhenRequested(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
localDir := filepath.Join(root, "previews")
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
previewPath := filepath.Join(localDir, "video-with-source.mp4")
|
||||
thumbPath := filepath.Join(localDir, "thumbs", "video-with-source.jpg")
|
||||
for _, path := range []string{previewPath, thumbPath} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("file"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "video-with-source",
|
||||
DriveID: "source-drive",
|
||||
FileID: "source-file",
|
||||
FileName: "clip.mp4",
|
||||
Title: "Source File",
|
||||
PreviewLocal: previewPath,
|
||||
PreviewStatus: "ready",
|
||||
ThumbnailURL: "/p/thumb/video-with-source",
|
||||
Size: 123,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
registry := proxy.NewRegistry()
|
||||
drv := &serverRemovableFakeDrive{id: "source-drive"}
|
||||
registry.Set(drv.ID(), drv)
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
}
|
||||
result, err := app.deleteVideo(ctx, "video-with-source", true)
|
||||
if err != nil {
|
||||
t.Fatalf("delete video: %v", err)
|
||||
}
|
||||
if !result.OK || !result.DeletedSource {
|
||||
t.Fatalf("delete result = %#v, want source deleted", result)
|
||||
}
|
||||
if got, want := drv.removedFileID, "source-file"; got != want {
|
||||
t.Fatalf("removed source fileID = %q, want %q", got, want)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "video-with-source"); err != sql.ErrNoRows {
|
||||
t.Fatalf("deleted video lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
for _, path := range []string{previewPath, thumbPath} {
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("generated asset %s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteVideoUsesSourceRemoverWithCatalogMetadata(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "video-with-rich-source",
|
||||
DriveID: "source-drive",
|
||||
FileID: "source-fid",
|
||||
ParentID: "parent-dir",
|
||||
FileName: "clip.mp4",
|
||||
Title: "Source File",
|
||||
Size: 123,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
registry := proxy.NewRegistry()
|
||||
drv := &serverSourceRemovableFakeDrive{id: "source-drive"}
|
||||
registry.Set(drv.ID(), drv)
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: filepath.Join(t.TempDir(), "previews")}},
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
}
|
||||
result, err := app.deleteVideo(ctx, "video-with-rich-source", true)
|
||||
if err != nil {
|
||||
t.Fatalf("delete video: %v", err)
|
||||
}
|
||||
if !result.OK || !result.DeletedSource {
|
||||
t.Fatalf("delete result = %#v, want source deleted", result)
|
||||
}
|
||||
if drv.fallbackRemoveCalled {
|
||||
t.Fatal("fallback Remove was called, want SourceRemover")
|
||||
}
|
||||
want := drives.SourceFile{
|
||||
FileID: "source-fid",
|
||||
ParentID: "parent-dir",
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
}
|
||||
if drv.removedSource != want {
|
||||
t.Fatalf("removed source = %#v, want %#v", drv.removedSource, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteVideoRemovesSpider91SourceFile(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
@@ -1090,7 +1524,7 @@ func TestDeleteVideoRemovesSpider91SourceFile(t *testing.T) {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
result, err := app.deleteVideo(ctx, "spider91-spider-main-source")
|
||||
result, err := app.deleteVideo(ctx, "spider91-spider-main-source", true)
|
||||
if err != nil {
|
||||
t.Fatalf("delete spider video: %v", err)
|
||||
}
|
||||
@@ -1198,7 +1632,6 @@ func TestCleanupDriveVideosForDeleteSpider91RemovesCrawledDirAndOriginRecords(t
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
fingerprintWorkers: make(map[string]*fingerprint.Worker),
|
||||
spider91Crawlers: make(map[string]*spider91.Crawler),
|
||||
}
|
||||
removed, err := app.cleanupDriveVideosForDelete(ctx, driveID)
|
||||
if err != nil {
|
||||
@@ -1496,6 +1929,103 @@ func (d *serverFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
}
|
||||
func (d *serverFakeDrive) RootID() string { return "root" }
|
||||
|
||||
type serverFakeKindDrive struct {
|
||||
serverFakeDrive
|
||||
id string
|
||||
kind string
|
||||
}
|
||||
|
||||
func (d *serverFakeKindDrive) Kind() string { return d.kind }
|
||||
func (d *serverFakeKindDrive) ID() string { return d.id }
|
||||
|
||||
type serverRemovableFakeDrive struct {
|
||||
serverFakeDrive
|
||||
id string
|
||||
removedFileID string
|
||||
}
|
||||
|
||||
func (d *serverRemovableFakeDrive) Kind() string { return "fake-removable" }
|
||||
func (d *serverRemovableFakeDrive) ID() string { return d.id }
|
||||
func (d *serverRemovableFakeDrive) Remove(ctx context.Context, fileID string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
d.removedFileID = fileID
|
||||
return nil
|
||||
}
|
||||
|
||||
type serverSourceRemovableFakeDrive struct {
|
||||
serverFakeDrive
|
||||
id string
|
||||
removedSource drives.SourceFile
|
||||
fallbackRemoveCalled bool
|
||||
}
|
||||
|
||||
func (d *serverSourceRemovableFakeDrive) Kind() string { return "fake-source-removable" }
|
||||
func (d *serverSourceRemovableFakeDrive) ID() string { return d.id }
|
||||
func (d *serverSourceRemovableFakeDrive) RemoveSource(ctx context.Context, source drives.SourceFile) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
d.removedSource = source
|
||||
return nil
|
||||
}
|
||||
func (d *serverSourceRemovableFakeDrive) Remove(ctx context.Context, fileID string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
d.fallbackRemoveCalled = true
|
||||
return nil
|
||||
}
|
||||
|
||||
type serverFakeSpider91MigrationRunner struct {
|
||||
called int
|
||||
}
|
||||
|
||||
func (r *serverFakeSpider91MigrationRunner) RunOnce(context.Context) error {
|
||||
r.called++
|
||||
return nil
|
||||
}
|
||||
|
||||
type serverBlockingListDrive struct {
|
||||
id string
|
||||
started chan string
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (d *serverBlockingListDrive) Kind() string { return "fake" }
|
||||
func (d *serverBlockingListDrive) ID() string { return d.id }
|
||||
func (d *serverBlockingListDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *serverBlockingListDrive) List(ctx context.Context, _ string) ([]drives.Entry, error) {
|
||||
if d.started != nil {
|
||||
select {
|
||||
case d.started <- d.id:
|
||||
default:
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-d.release:
|
||||
return nil, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
func (d *serverBlockingListDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *serverBlockingListDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
|
||||
return &drives.StreamLink{URL: "https://video.example/clip.mp4"}, nil
|
||||
}
|
||||
func (d *serverBlockingListDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *serverBlockingListDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *serverBlockingListDrive) RootID() string { return "root" }
|
||||
|
||||
type serverFingerprintFakeDrive struct {
|
||||
serverFakeDrive
|
||||
path string
|
||||
|
||||
@@ -33,14 +33,11 @@ scanner:
|
||||
# 单次扫描每家网盘目录递归层数上限
|
||||
max_depth: 5
|
||||
# 被扫描的扩展名
|
||||
video_extensions: [".mp4", ".mkv", ".mov", ".webm", ".avi"]
|
||||
video_extensions: [".mp4", ".mkv", ".mov", ".webm", ".avi", ".strm"]
|
||||
|
||||
nightly:
|
||||
# 凌晨流水线触发整点(0-23),默认 1 即每天 01:00。流程:
|
||||
# Phase 1 扫所有非 spider91 / 非 localupload 网盘 → 检测新增 / 删除
|
||||
# → 入队封面和预览视频 → 等所有队列 idle
|
||||
# Phase 2 spider91 爬虫(如配置)→ 入队预览视频 → 等队列 idle
|
||||
# Phase 3 spider91 → 云盘迁移(一次性 sweep)
|
||||
# 凌晨流水线触发整点(0-23),默认 1 即每天 01:00。
|
||||
# 运行时会统一编排扫描、媒体资产生成和后续清理任务。
|
||||
cron_hour: 1
|
||||
# 单次流水线总耗时上限(软超时);超过后当前 phase 跑完不启动后续 phase。
|
||||
max_duration: 6h
|
||||
@@ -59,7 +56,7 @@ preview:
|
||||
width: 480
|
||||
|
||||
# 盘列表。上线后请通过管理后台添加,本文件可留空。
|
||||
# kind 支持 quark / p115 / p123 / pikpak / wopan / onedrive / googledrive / localstorage。
|
||||
# kind 支持 quark / p115 / p123 / pikpak / wopan / guangyapan / onedrive / googledrive / localstorage。
|
||||
# OneDrive 示例:
|
||||
# - id: "my-onedrive"
|
||||
# kind: "onedrive"
|
||||
@@ -74,11 +71,29 @@ preview:
|
||||
# root_id: "root"
|
||||
# params:
|
||||
# refresh_token: "..."
|
||||
# # 默认 use_online_api=true,会使用 OpenList 在线续期 API。
|
||||
# # 如需使用自己创建的 Google OAuth 客户端,取消下面三行注释:
|
||||
# # use_online_api: "false"
|
||||
# # client_id: "..."
|
||||
# # client_secret: "..."
|
||||
# 光鸭网盘示例:
|
||||
# - id: "my-guangyapan"
|
||||
# kind: "guangyapan"
|
||||
# name: "我的光鸭网盘"
|
||||
# # 留空表示光鸭网盘根目录;也可以填写光鸭目录 fileId
|
||||
# root_id: ""
|
||||
# params:
|
||||
# # 推荐在后台使用扫码登录自动写入 access_token / refresh_token。
|
||||
# refresh_token: "..."
|
||||
# # 可选:按路径解析扫描根目录,优先于 root_id
|
||||
# # root_path: "影视/电影"
|
||||
# 本地存储示例:
|
||||
# - id: "local-media"
|
||||
# kind: "localstorage"
|
||||
# name: "本地视频目录"
|
||||
# root_id: "/"
|
||||
# params:
|
||||
# # Docker 部署时这里和 .strm 里的绝对路径都必须使用容器内路径。
|
||||
# # 例如宿主机 /mnt/videos 挂载为 /media,就填写 /media。
|
||||
# path: "/mnt/videos"
|
||||
drives: []
|
||||
|
||||
+1121
-29
File diff suppressed because it is too large
Load Diff
+1348
-88
File diff suppressed because it is too large
Load Diff
+118
-47
@@ -11,6 +11,7 @@ import (
|
||||
"io"
|
||||
"math/rand/v2"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
@@ -54,12 +55,16 @@ type Server struct {
|
||||
LocalDir string
|
||||
UploadDir string
|
||||
OnVideoUploaded func(*catalog.Video)
|
||||
// OnHideVideo 处理前台「不再展示」。隐藏机制已废弃,改走拉黑逻辑:
|
||||
// 删除库中记录 + 本地封面/预览,保留网盘源文件,并写黑名单墓碑
|
||||
// (扫盘不再入库)。未注入时回退为旧的 hidden 标记。
|
||||
OnHideVideo func(ctx context.Context, videoID string) error
|
||||
|
||||
tagCacheMu sync.Mutex
|
||||
tagCacheUntil time.Time
|
||||
tagCache []TagDTO
|
||||
|
||||
// GetTheme 返回当前生效的主题("dark" | "pink")。前台 /api/settings/theme 用,
|
||||
// GetTheme 返回当前生效的主题("dark" | "pink" | "sky")。前台 /api/settings/theme 用,
|
||||
// 不需要登录。无注入时返回 "dark"。
|
||||
GetTheme func() string
|
||||
}
|
||||
@@ -146,7 +151,7 @@ func (s *Server) RegisterRoutes(r chi.Router, a *auth.Authenticator) {
|
||||
r.Post("/api/shorts/next", s.handleShortsNext)
|
||||
|
||||
// 代理路由同样需要鉴权,防止绕过
|
||||
r.Get("/p/stream/{driveID}/{fileID}", s.handleStream)
|
||||
r.Get("/p/stream/{driveID}/*", s.handleStream)
|
||||
r.Get("/p/upload/{videoID}", s.handleUploadedVideo)
|
||||
r.Get("/p/spider91/{videoID}", s.handleSpider91Video)
|
||||
r.Get("/p/preview/{videoID}", s.handlePreview)
|
||||
@@ -155,11 +160,11 @@ func (s *Server) RegisterRoutes(r chi.Router, a *auth.Authenticator) {
|
||||
}
|
||||
|
||||
// handleGetTheme 返回当前生效的主题。无需登录。响应永远是
|
||||
// {"theme": "dark"} 或 {"theme": "pink"},便于前端无脑解析。
|
||||
// {"theme": "dark" | "pink" | "sky"},便于前端无脑解析。
|
||||
func (s *Server) handleGetTheme(w http.ResponseWriter, r *http.Request) {
|
||||
theme := "dark"
|
||||
if s.GetTheme != nil {
|
||||
if v := s.GetTheme(); v == "pink" || v == "dark" {
|
||||
if v := s.GetTheme(); v == "pink" || v == "dark" || v == "sky" {
|
||||
theme = v
|
||||
}
|
||||
}
|
||||
@@ -313,7 +318,7 @@ func (s *Server) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleVideoDetail(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
id := routeParam(r, "id")
|
||||
v, err := s.Catalog.GetVideo(r.Context(), id)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusNotFound, err)
|
||||
@@ -343,7 +348,7 @@ func (s *Server) handleVideoDetail(w http.ResponseWriter, r *http.Request) {
|
||||
VideoSrc: s.videoSource(v),
|
||||
Poster: thumbnailURL(v),
|
||||
Description: v.Description,
|
||||
EmbedURL: fmt.Sprintf(`<iframe src="/embed/%s" width="640" height="360" frameborder="0" allowfullscreen></iframe>`, v.ID),
|
||||
EmbedURL: fmt.Sprintf(`<iframe src="/embed/%s" width="640" height="360" frameborder="0" allowfullscreen></iframe>`, pathSegment(v.ID)),
|
||||
AuthorProfile: AuthorProfile{
|
||||
ID: "author-" + v.Author,
|
||||
Name: v.Author,
|
||||
@@ -525,11 +530,9 @@ func (s *Server) handleTags(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// shortsNextReq 客户端把当前轮已看过的 video id 列表传上来。
|
||||
// PreferredFromVideoID 来自短视频页最近一次点赞成功的视频,用于优先推荐相似标签。
|
||||
type shortsNextReq struct {
|
||||
SeenIDs []string `json:"seenIds"`
|
||||
Count int `json:"count"`
|
||||
PreferredFromVideoID string `json:"preferredFromVideoId"`
|
||||
SeenIDs []string `json:"seenIds"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// ShortsItemDTO 是短视频流单条的精简结构。比 VideoDTO 多 videoSrc / poster,
|
||||
@@ -547,8 +550,8 @@ type ShortsItemDTO struct {
|
||||
// - 服务器从未在 seenIds 中的可见视频里随机抽至多 count 条返回
|
||||
// - 当返回数量 < count 且小于全库可见总数时,说明本轮即将结束,
|
||||
// 返回 roundComplete=true,前端应在用户看完返回的这些后清空本地已看记录开新一轮
|
||||
// - 当 seenIds 已经覆盖全库时,本接口直接返回新一轮的随机一批
|
||||
// (传 seenIds=[] 即可让客户端在轮次完成后重新开始)
|
||||
// - 当 seenIds 真实覆盖当前全部可见视频时,本接口直接返回新一轮的随机一批
|
||||
// (不能仅看 seenIds 长度,里面可能有隐藏、删除或历史脏 ID)
|
||||
func (s *Server) handleShortsNext(w http.ResponseWriter, r *http.Request) {
|
||||
var body shortsNextReq
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
|
||||
@@ -569,22 +572,18 @@ func (s *Server) handleShortsNext(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// 如果客户端已看记录已经 ≥ 全库,则视为新一轮,直接忽略 seenIds
|
||||
exclude := body.SeenIDs
|
||||
if total > 0 && len(exclude) >= total {
|
||||
exclude = nil
|
||||
}
|
||||
|
||||
var items []*catalog.Video
|
||||
if strings.TrimSpace(body.PreferredFromVideoID) != "" {
|
||||
items, err = s.Catalog.RandomVideosForPreferredVideoExcluding(r.Context(), body.PreferredFromVideoID, exclude, count)
|
||||
} else {
|
||||
items, err = s.Catalog.RandomVideosExcluding(r.Context(), exclude, count)
|
||||
}
|
||||
items, err := s.Catalog.RandomVideosExcluding(r.Context(), body.SeenIDs, count)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
if total > 0 && len(items) == 0 && len(body.SeenIDs) > 0 {
|
||||
items, err = s.Catalog.RandomVideosExcluding(r.Context(), nil, count)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 注入 sourceLabel 以便前端展示来源网盘
|
||||
driveLabels := make(map[string]string)
|
||||
@@ -622,7 +621,7 @@ type updateVideoTagsReq struct {
|
||||
}
|
||||
|
||||
func (s *Server) handleUpdateVideoTags(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
id := routeParam(r, "id")
|
||||
var body updateVideoTagsReq
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
writeErr(w, http.StatusBadRequest, err)
|
||||
@@ -645,7 +644,7 @@ func (s *Server) handleUpdateVideoTags(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleLike(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
id := routeParam(r, "id")
|
||||
likes, err := s.Catalog.IncrementLike(r.Context(), id)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
@@ -657,7 +656,7 @@ func (s *Server) handleLike(w http.ResponseWriter, r *http.Request) {
|
||||
// handleUnlike 取消点赞:likes - 1(保底 0)。
|
||||
// 短视频模式中爱心按钮点击切换状态时使用。
|
||||
func (s *Server) handleUnlike(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
id := routeParam(r, "id")
|
||||
likes, err := s.Catalog.DecrementLike(r.Context(), id)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
@@ -671,7 +670,7 @@ func (s *Server) handleUnlike(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleView(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
id := routeParam(r, "id")
|
||||
views, err := s.Catalog.IncrementView(r.Context(), id)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
@@ -685,8 +684,15 @@ func (s *Server) handleView(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleHideVideo(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
if err := s.Catalog.HideVideo(r.Context(), id); err != nil {
|
||||
id := routeParam(r, "id")
|
||||
var err error
|
||||
if s.OnHideVideo != nil {
|
||||
// 走拉黑逻辑:删记录 + 删本地封面/预览 + 写墓碑,保留网盘源文件。
|
||||
err = s.OnHideVideo(r.Context(), id)
|
||||
} else {
|
||||
err = s.Catalog.HideVideo(r.Context(), id)
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
writeErr(w, http.StatusNotFound, err)
|
||||
return
|
||||
@@ -802,12 +808,12 @@ func (s *Server) handleUploadVideo(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleStream(w http.ResponseWriter, r *http.Request) {
|
||||
driveID := chi.URLParam(r, "driveID")
|
||||
fileID := chi.URLParam(r, "fileID")
|
||||
driveID := routeParam(r, "driveID")
|
||||
fileID := routeWildcardParam(r, "*")
|
||||
s.Proxy.ServeStream(w, r, driveID, fileID)
|
||||
}
|
||||
func (s *Server) handleUploadedVideo(w http.ResponseWriter, r *http.Request) {
|
||||
videoID := chi.URLParam(r, "videoID")
|
||||
videoID := routeParam(r, "videoID")
|
||||
v, err := s.Catalog.GetVideo(r.Context(), videoID)
|
||||
if err != nil || v.Hidden || v.DriveID != localUploadDriveID {
|
||||
http.NotFound(w, r)
|
||||
@@ -831,7 +837,7 @@ func (s *Server) handleUploadedVideo(w http.ResponseWriter, r *http.Request) {
|
||||
// 路径形如 /p/spider91/<videoID>,videoID = "spider91-<driveID>-<sourceID>"。
|
||||
// 通过 catalog 拿到 file_id("<sourceID>.mp4"),再让 driver 解析到绝对路径并 ServeFile。
|
||||
func (s *Server) handleSpider91Video(w http.ResponseWriter, r *http.Request) {
|
||||
videoID := chi.URLParam(r, "videoID")
|
||||
videoID := routeParam(r, "videoID")
|
||||
v, err := s.Catalog.GetVideo(r.Context(), videoID)
|
||||
if err != nil || v.Hidden {
|
||||
http.NotFound(w, r)
|
||||
@@ -866,7 +872,7 @@ func (s *Server) handleSpider91Video(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) {
|
||||
videoID := chi.URLParam(r, "videoID")
|
||||
videoID := routeParam(r, "videoID")
|
||||
v, err := s.Catalog.GetVideo(r.Context(), videoID)
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
@@ -891,7 +897,7 @@ func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleThumb(w http.ResponseWriter, r *http.Request) {
|
||||
videoID := chi.URLParam(r, "videoID")
|
||||
videoID := routeParam(r, "videoID")
|
||||
var clean string
|
||||
for _, path := range mediaasset.ThumbnailPathCandidates(s.LocalDir, videoID) {
|
||||
candidate := filepath.Clean(path)
|
||||
@@ -926,7 +932,7 @@ func mapVideo(v *catalog.Video) VideoDTO {
|
||||
}
|
||||
return VideoDTO{
|
||||
ID: v.ID,
|
||||
Href: "/video/" + v.ID,
|
||||
Href: "/video/" + pathSegment(v.ID),
|
||||
Title: v.Title,
|
||||
Thumbnail: thumbnailURL(v),
|
||||
PreviewSrc: previewURL(v),
|
||||
@@ -948,7 +954,7 @@ func mapVideo(v *catalog.Video) VideoDTO {
|
||||
}
|
||||
|
||||
func previewURL(v *catalog.Video) string {
|
||||
base := "/p/preview/" + v.ID
|
||||
base := "/p/preview/" + pathSegment(v.ID)
|
||||
if v.UpdatedAt.IsZero() {
|
||||
return base
|
||||
}
|
||||
@@ -956,9 +962,12 @@ func previewURL(v *catalog.Video) string {
|
||||
}
|
||||
|
||||
func thumbnailURL(v *catalog.Video) string {
|
||||
base := "/p/thumb/" + v.ID
|
||||
base := "/p/thumb/" + pathSegment(v.ID)
|
||||
if v.ThumbnailURL != "" {
|
||||
base = v.ThumbnailURL
|
||||
if thumbnailURLMatchesVideoID(base, v.ID) {
|
||||
base = "/p/thumb/" + pathSegment(v.ID)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(base, "/p/thumb/") || v.UpdatedAt.IsZero() {
|
||||
return base
|
||||
@@ -966,25 +975,85 @@ func thumbnailURL(v *catalog.Video) string {
|
||||
return base + "?v=" + strconv.FormatInt(v.UpdatedAt.UnixMilli(), 10)
|
||||
}
|
||||
|
||||
// transcodedSource 在视频有就绪的浏览器兼容性转码产物时返回产物的播放地址。
|
||||
// 产物和原始文件在同一个 drive 上,走同一条 /p/stream 代理/302 链路。
|
||||
func transcodedSource(v *catalog.Video) (string, bool) {
|
||||
if v.TranscodeStatus == "ready" && v.TranscodedFileID != "" && v.DriveID != localUploadDriveID {
|
||||
return fmt.Sprintf("/p/stream/%s/%s", pathSegment(v.DriveID), pathSegment(v.TranscodedFileID)), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func (s *Server) videoSource(v *catalog.Video) string {
|
||||
if v.DriveID == localUploadDriveID {
|
||||
return "/p/upload/" + v.ID
|
||||
return "/p/upload/" + pathSegment(v.ID)
|
||||
}
|
||||
if s.Proxy != nil && s.Proxy.Registry != nil {
|
||||
if d, ok := s.Proxy.Registry.Get(v.DriveID); ok && d.Kind() == spider91.Kind {
|
||||
return "/p/spider91/" + v.ID
|
||||
if d, ok := s.Proxy.Registry.Get(v.DriveID); ok {
|
||||
switch d.Kind() {
|
||||
case spider91.Kind:
|
||||
return "/p/spider91/" + pathSegment(v.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("/p/stream/%s/%s", v.DriveID, v.FileID)
|
||||
if src, ok := transcodedSource(v); ok {
|
||||
return src
|
||||
}
|
||||
return fmt.Sprintf("/p/stream/%s/%s", pathSegment(v.DriveID), pathSegment(v.FileID))
|
||||
}
|
||||
|
||||
// videoSource 兼容旧调用点,没有 server context 时按之前逻辑回退到 /p/stream。
|
||||
// 内部新增的代码请使用 (*Server).videoSource。
|
||||
func videoSource(v *catalog.Video) string {
|
||||
if v.DriveID == localUploadDriveID {
|
||||
return "/p/upload/" + v.ID
|
||||
return "/p/upload/" + pathSegment(v.ID)
|
||||
}
|
||||
return fmt.Sprintf("/p/stream/%s/%s", v.DriveID, v.FileID)
|
||||
if src, ok := transcodedSource(v); ok {
|
||||
return src
|
||||
}
|
||||
return fmt.Sprintf("/p/stream/%s/%s", pathSegment(v.DriveID), pathSegment(v.FileID))
|
||||
}
|
||||
|
||||
func pathSegment(value string) string {
|
||||
return url.PathEscape(value)
|
||||
}
|
||||
|
||||
func routeParam(r *http.Request, key string) string {
|
||||
value := chi.URLParam(r, key)
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
if decoded, err := url.PathUnescape(value); err == nil {
|
||||
return decoded
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func routeWildcardParam(r *http.Request, key string) string {
|
||||
value := chi.URLParam(r, key)
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
value = strings.TrimPrefix(value, "/")
|
||||
if decoded, err := url.PathUnescape(value); err == nil {
|
||||
return decoded
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func thumbnailURLMatchesVideoID(value, videoID string) bool {
|
||||
if !strings.HasPrefix(value, "/p/thumb/") {
|
||||
return false
|
||||
}
|
||||
tail := strings.TrimPrefix(value, "/p/thumb/")
|
||||
if idx := strings.IndexByte(tail, '?'); idx >= 0 {
|
||||
tail = tail[:idx]
|
||||
}
|
||||
if tail == videoID {
|
||||
return true
|
||||
}
|
||||
decoded, err := url.PathUnescape(tail)
|
||||
return err == nil && decoded == videoID
|
||||
}
|
||||
|
||||
func driveKindLabel(kind string) string {
|
||||
@@ -994,11 +1063,13 @@ func driveKindLabel(kind string) string {
|
||||
case "p115":
|
||||
return "115 网盘"
|
||||
case "p123":
|
||||
return "123 云盘"
|
||||
return "123网盘"
|
||||
case "pikpak":
|
||||
return "PikPak"
|
||||
case "wopan":
|
||||
return "联通沃盘"
|
||||
return "联通网盘"
|
||||
case "guangyapan":
|
||||
return "光鸭网盘"
|
||||
case "onedrive":
|
||||
return "OneDrive"
|
||||
case "googledrive":
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
)
|
||||
@@ -66,6 +68,68 @@ func TestVideoSourceKeepsDirectStreamForMp4(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoURLsEscapePathSegments(t *testing.T) {
|
||||
updated := time.UnixMilli(1778863000123)
|
||||
v := &catalog.Video{
|
||||
ID: "wopan-drive-fid/with space",
|
||||
DriveID: "drive-1",
|
||||
FileID: "fid/with space",
|
||||
Title: "Video",
|
||||
UpdatedAt: updated,
|
||||
}
|
||||
|
||||
dto := mapVideo(v)
|
||||
if dto.Href != "/video/wopan-drive-fid%2Fwith%20space" {
|
||||
t.Fatalf("href = %q, want escaped video id", dto.Href)
|
||||
}
|
||||
if dto.PreviewSrc != "/p/preview/wopan-drive-fid%2Fwith%20space?v=1778863000123" {
|
||||
t.Fatalf("preview = %q, want escaped video id", dto.PreviewSrc)
|
||||
}
|
||||
if dto.Thumbnail != "/p/thumb/wopan-drive-fid%2Fwith%20space?v=1778863000123" {
|
||||
t.Fatalf("thumbnail = %q, want escaped video id", dto.Thumbnail)
|
||||
}
|
||||
if got := videoSource(v); got != "/p/stream/drive-1/fid%2Fwith%20space" {
|
||||
t.Fatalf("video source = %q, want escaped file id", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailURLRewritesStoredLocalURLForUnsafeVideoID(t *testing.T) {
|
||||
got := thumbnailURL(&catalog.Video{
|
||||
ID: "wopan-drive-fid/with space",
|
||||
ThumbnailURL: "/p/thumb/wopan-drive-fid/with space",
|
||||
UpdatedAt: time.UnixMilli(1778863000123),
|
||||
})
|
||||
|
||||
if got != "/p/thumb/wopan-drive-fid%2Fwith%20space?v=1778863000123" {
|
||||
t.Fatalf("thumbnail URL = %q, want escaped local URL", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStreamDecodesEscapedWildcardFileID(t *testing.T) {
|
||||
local := filepath.Join(t.TempDir(), "video.mp4")
|
||||
if err := os.WriteFile(local, []byte("ok"), 0o644); err != nil {
|
||||
t.Fatalf("write local video: %v", err)
|
||||
}
|
||||
drv := &apiStreamFakeDrive{localPath: local}
|
||||
reg := proxy.NewRegistry()
|
||||
reg.Set("drive-1", drv)
|
||||
srv := &Server{Proxy: proxy.New(reg)}
|
||||
|
||||
router := chi.NewRouter()
|
||||
router.Get("/p/stream/{driveID}/*", srv.handleStream)
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/drive-1/fid%2Fwith%20space", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
router.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if drv.fileID != "fid/with space" {
|
||||
t.Fatalf("fileID = %q, want decoded original", drv.fileID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoSourceUsesLocalUploadRoute(t *testing.T) {
|
||||
v := &catalog.Video{
|
||||
ID: "video-1",
|
||||
@@ -100,6 +164,49 @@ func TestPreviewURLFallsBackWithoutUpdatedAt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleVideoDetailDecodesEscapedVideoID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "wopan-drive-fid/with space",
|
||||
DriveID: "drive-1",
|
||||
FileID: "fid/with space",
|
||||
Title: "Video",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
router := chi.NewRouter()
|
||||
router.Get("/api/video/{id}", (&Server{Catalog: cat}).handleVideoDetail)
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/video/wopan-drive-fid%2Fwith%20space", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
router.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got VideoDetailDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.ID != "wopan-drive-fid/with space" {
|
||||
t.Fatalf("id = %q, want original video id", got.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailURLVersionsLocalGeneratedThumbnails(t *testing.T) {
|
||||
got := thumbnailURL(&catalog.Video{
|
||||
ID: "video-1",
|
||||
@@ -703,7 +810,7 @@ func TestHandleTagsReturnsUnifiedTagPool(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleShortsNextUsesPreferredVideoLeastPopulatedTag(t *testing.T) {
|
||||
func TestHandleShortsNextReturnsRandomBatchExcludingSeen(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -727,7 +834,7 @@ func TestHandleShortsNextUsesPreferredVideoLeastPopulatedTag(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/shorts/next", strings.NewReader(`{"seenIds":["current"],"count":3,"preferredFromVideoId":"current"}`))
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/shorts/next", strings.NewReader(`{"seenIds":["current"],"count":3}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&Server{Catalog: cat}).handleShortsNext(rr, req)
|
||||
|
||||
@@ -750,10 +857,7 @@ func TestHandleShortsNextUsesPreferredVideoLeastPopulatedTag(t *testing.T) {
|
||||
t.Fatalf("total = %d, want 4", got.Total)
|
||||
}
|
||||
if got.RoundComplete {
|
||||
t.Fatalf("roundComplete = true, want false with fallback-filled batch")
|
||||
}
|
||||
if !containsString(ids, "rare-1") {
|
||||
t.Fatalf("ids = %#v, want rare-1 from least populated tag", ids)
|
||||
t.Fatalf("roundComplete = true, want false with a full remaining batch")
|
||||
}
|
||||
if containsString(ids, "current") {
|
||||
t.Fatalf("ids = %#v, should exclude current", ids)
|
||||
@@ -761,6 +865,76 @@ func TestHandleShortsNextUsesPreferredVideoLeastPopulatedTag(t *testing.T) {
|
||||
if len(ids) != 3 {
|
||||
t.Fatalf("ids = %#v, want 3 items", ids)
|
||||
}
|
||||
for _, want := range []string{"common-1", "common-2", "rare-1"} {
|
||||
if !containsString(ids, want) {
|
||||
t.Fatalf("ids = %#v, want remaining id %s", ids, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleShortsNextDoesNotResetForStaleSeenIDs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*catalog.Video{
|
||||
{ID: "seen-1", DriveID: "drive", FileID: "f-seen-1", Title: "seen 1", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "fresh-1", DriveID: "drive", FileID: "f-fresh-1", Title: "fresh 1", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "fresh-2", DriveID: "drive", FileID: "f-fresh-2", Title: "fresh 2", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "hidden-1", DriveID: "drive", FileID: "f-hidden-1", Title: "hidden 1", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
if err := cat.HideVideo(ctx, "hidden-1"); err != nil {
|
||||
t.Fatalf("hide hidden-1: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/shorts/next", strings.NewReader(`{"seenIds":["seen-1","hidden-1","deleted-stale"],"count":3}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&Server{Catalog: cat}).handleShortsNext(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
Items []ShortsItemDTO `json:"items"`
|
||||
Total int `json:"total"`
|
||||
RoundComplete bool `json:"roundComplete"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
ids := make([]string, 0, len(got.Items))
|
||||
for _, item := range got.Items {
|
||||
ids = append(ids, item.ID)
|
||||
}
|
||||
if got.Total != 3 {
|
||||
t.Fatalf("total = %d, want 3", got.Total)
|
||||
}
|
||||
if !got.RoundComplete {
|
||||
t.Fatalf("roundComplete = false, want true after returning all unviewed visible videos")
|
||||
}
|
||||
if containsString(ids, "seen-1") || containsString(ids, "hidden-1") {
|
||||
t.Fatalf("ids = %#v, should not reset and return seen or hidden videos", ids)
|
||||
}
|
||||
for _, want := range []string{"fresh-1", "fresh-2"} {
|
||||
if !containsString(ids, want) {
|
||||
t.Fatalf("ids = %#v, want %s", ids, want)
|
||||
}
|
||||
}
|
||||
if len(ids) != 2 {
|
||||
t.Fatalf("ids = %#v, want exactly the two unviewed visible videos", ids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpdateVideoTagsRejectsUnknownTags(t *testing.T) {
|
||||
@@ -1084,6 +1258,37 @@ func sameStringSet(a, b []string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
type apiStreamFakeDrive struct {
|
||||
localPath string
|
||||
fileID string
|
||||
}
|
||||
|
||||
func (d *apiStreamFakeDrive) Kind() string { return "fake" }
|
||||
func (d *apiStreamFakeDrive) ID() string { return "drive-1" }
|
||||
func (d *apiStreamFakeDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *apiStreamFakeDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *apiStreamFakeDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *apiStreamFakeDrive) StreamURL(_ context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
d.fileID = fileID
|
||||
return &drives.StreamLink{
|
||||
URL: d.localPath,
|
||||
Expires: time.Now().Add(time.Minute),
|
||||
}, nil
|
||||
}
|
||||
func (d *apiStreamFakeDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *apiStreamFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *apiStreamFakeDrive) RootID() string { return "root" }
|
||||
|
||||
func requestWithVideoID(method, target, videoID string, body *strings.Reader) *http.Request {
|
||||
return requestWithRouteParam(method, target, "id", videoID, body)
|
||||
}
|
||||
|
||||
+522
-198
@@ -20,6 +20,15 @@ type Catalog struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
type CrawlerAssetCounts struct {
|
||||
Total int
|
||||
Local int
|
||||
Migrated int
|
||||
Thumbnail DriveThumbnailCounts
|
||||
Teaser DriveTeaserCounts
|
||||
Fingerprint DriveFingerprintCounts
|
||||
}
|
||||
|
||||
func Open(path string) (*Catalog, error) {
|
||||
db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
@@ -42,43 +51,54 @@ func (c *Catalog) Close() error { return c.db.Close() }
|
||||
// ---------- Video ----------
|
||||
|
||||
type Video struct {
|
||||
ID string `json:"id"`
|
||||
DriveID string `json:"driveId"`
|
||||
FileID string `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
ContentHash string `json:"contentHash"`
|
||||
SampledSHA256 string `json:"sampledSha256"`
|
||||
FingerprintStatus string `json:"fingerprintStatus"`
|
||||
FingerprintError string `json:"fingerprintError"`
|
||||
ParentID string `json:"parentId"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Tags []string `json:"tags"`
|
||||
DurationSeconds int `json:"durationSeconds"`
|
||||
Size int64 `json:"size"`
|
||||
Ext string `json:"ext"`
|
||||
Quality string `json:"quality"`
|
||||
ThumbnailURL string `json:"thumbnailUrl"`
|
||||
PreviewFileID string `json:"previewFileId"`
|
||||
PreviewLocal string `json:"previewLocal"`
|
||||
PreviewStatus string `json:"previewStatus"`
|
||||
Views int `json:"views"`
|
||||
Favorites int `json:"favorites"`
|
||||
Comments int `json:"comments"`
|
||||
Likes int `json:"likes"`
|
||||
Dislikes int `json:"dislikes"`
|
||||
Category string `json:"category"`
|
||||
Hidden bool `json:"hidden"`
|
||||
Badges []string `json:"badges"`
|
||||
Description string `json:"description"`
|
||||
PublishedAt time.Time `json:"publishedAt"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
ID string `json:"id"`
|
||||
DriveID string `json:"driveId"`
|
||||
FileID string `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
ContentHash string `json:"contentHash"`
|
||||
SampledSHA256 string `json:"sampledSha256"`
|
||||
FingerprintStatus string `json:"fingerprintStatus"`
|
||||
FingerprintError string `json:"fingerprintError"`
|
||||
ParentID string `json:"parentId"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Tags []string `json:"tags"`
|
||||
DurationSeconds int `json:"durationSeconds"`
|
||||
Size int64 `json:"size"`
|
||||
Ext string `json:"ext"`
|
||||
Quality string `json:"quality"`
|
||||
ThumbnailURL string `json:"thumbnailUrl"`
|
||||
PreviewFileID string `json:"previewFileId"`
|
||||
PreviewLocal string `json:"previewLocal"`
|
||||
PreviewStatus string `json:"previewStatus"`
|
||||
// TranscodeStatus:浏览器兼容性转码状态。
|
||||
// ''=未检测 / pending=已入队 / ready=已转码 / skipped=无需转码 / failed=失败。
|
||||
TranscodeStatus string `json:"transcodeStatus"`
|
||||
TranscodeError string `json:"transcodeError"`
|
||||
TranscodedFileID string `json:"transcodedFileId"`
|
||||
TranscodedSize int64 `json:"transcodedSize"`
|
||||
Views int `json:"views"`
|
||||
Favorites int `json:"favorites"`
|
||||
Comments int `json:"comments"`
|
||||
Likes int `json:"likes"`
|
||||
Dislikes int `json:"dislikes"`
|
||||
Category string `json:"category"`
|
||||
Hidden bool `json:"hidden"`
|
||||
Badges []string `json:"badges"`
|
||||
Description string `json:"description"`
|
||||
PublishedAt time.Time `json:"publishedAt"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
func (c *Catalog) UpsertVideo(ctx context.Context, v *Video) error {
|
||||
existed := c.videoExists(ctx, v.ID)
|
||||
v.ContentHash = normalizeContentHash(v.ContentHash)
|
||||
v.SampledSHA256 = normalizeContentHash(v.SampledSHA256)
|
||||
fingerprintStatus := nullableStatus(v.FingerprintStatus)
|
||||
if v.SampledSHA256 != "" && (v.FingerprintStatus == "" || v.FingerprintStatus == "pending") {
|
||||
fingerprintStatus = "ready"
|
||||
}
|
||||
tagsJSON, _ := json.Marshal(v.Tags)
|
||||
badgesJSON, _ := json.Marshal(v.Badges)
|
||||
now := time.Now().UnixMilli()
|
||||
@@ -89,13 +109,13 @@ func (c *Catalog) UpsertVideo(ctx context.Context, v *Video) error {
|
||||
|
||||
_, err := c.db.ExecContext(ctx, `
|
||||
INSERT INTO videos (
|
||||
id, drive_id, file_id, file_name, content_hash, parent_id, title, author, tags,
|
||||
id, drive_id, file_id, file_name, content_hash, sampled_sha256, fingerprint_status, fingerprint_error, parent_id, title, author, tags,
|
||||
duration_seconds, size_bytes, ext, quality, thumbnail_url, thumbnail_status,
|
||||
preview_file_id, preview_local, preview_status,
|
||||
views, favorites, comments, likes, dislikes,
|
||||
category, hidden, badges, description, published_at, created_at, updated_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, CASE WHEN COALESCE(?, '') != '' THEN 'ready' ELSE 'pending' END,
|
||||
?, ?, ?,
|
||||
?, ?, ?, ?, ?,
|
||||
@@ -114,15 +134,18 @@ ON CONFLICT(id) DO UPDATE SET
|
||||
ELSE videos.content_hash
|
||||
END,
|
||||
sampled_sha256 = CASE
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN ''
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN excluded.sampled_sha256
|
||||
WHEN excluded.sampled_sha256 != '' THEN excluded.sampled_sha256
|
||||
ELSE videos.sampled_sha256
|
||||
END,
|
||||
fingerprint_status = CASE
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN 'pending'
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN COALESCE(excluded.fingerprint_status, 'pending')
|
||||
WHEN excluded.sampled_sha256 != '' THEN COALESCE(excluded.fingerprint_status, 'ready')
|
||||
ELSE COALESCE(videos.fingerprint_status, 'pending')
|
||||
END,
|
||||
fingerprint_error = CASE
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN ''
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN COALESCE(excluded.fingerprint_error, '')
|
||||
WHEN excluded.sampled_sha256 != '' THEN COALESCE(excluded.fingerprint_error, '')
|
||||
ELSE COALESCE(videos.fingerprint_error, '')
|
||||
END,
|
||||
duration_seconds= excluded.duration_seconds,
|
||||
@@ -143,7 +166,7 @@ ON CONFLICT(id) DO UPDATE SET
|
||||
description = excluded.description,
|
||||
updated_at = excluded.updated_at
|
||||
`,
|
||||
v.ID, v.DriveID, v.FileID, v.FileName, v.ContentHash, v.ParentID, v.Title, v.Author, string(tagsJSON),
|
||||
v.ID, v.DriveID, v.FileID, v.FileName, v.ContentHash, v.SampledSHA256, fingerprintStatus, v.FingerprintError, v.ParentID, v.Title, v.Author, string(tagsJSON),
|
||||
v.DurationSeconds, v.Size, v.Ext, v.Quality, v.ThumbnailURL, v.ThumbnailURL,
|
||||
v.PreviewFileID, v.PreviewLocal, nullableStatus(v.PreviewStatus),
|
||||
v.Views, v.Favorites, v.Comments, v.Likes, v.Dislikes,
|
||||
@@ -173,6 +196,84 @@ func (c *Catalog) UpdatePreview(ctx context.Context, id, previewLocal, status st
|
||||
return err
|
||||
}
|
||||
|
||||
// transcodeCandidateWhereSQL 圈定"可能需要浏览器兼容性转码"的视频:
|
||||
// mp4/webm/m4v 默认浏览器可播不进候选;strm 是远程引用没有本体。
|
||||
// 其余扩展名都先入候选,由转码 worker probe 实际编码后决定转码还是跳过
|
||||
// (skipped)。failed 也保留在候选里,重新点开始转码时会自动重试。
|
||||
const transcodeCandidateWhereSQL = `COALESCE(ext, '') NOT IN ('mp4', 'webm', 'm4v', 'strm')
|
||||
AND COALESCE(transcode_status, '') IN ('', 'pending', 'failed')`
|
||||
|
||||
// ListTranscodeCandidates 列出某盘所有转码候选视频。limit<=0 表示不限制。
|
||||
func (c *Catalog) ListTranscodeCandidates(ctx context.Context, driveID string, limit int) ([]*Video, error) {
|
||||
query := `SELECT ` + allVideoCols + ` FROM videos
|
||||
WHERE drive_id = ? AND ` + transcodeCandidateWhereSQL + `
|
||||
ORDER BY created_at ASC, id ASC`
|
||||
args := []any{driveID}
|
||||
if limit > 0 {
|
||||
query += ` LIMIT ?`
|
||||
args = append(args, limit)
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// UpdateVideoTranscode 写回单条视频的转码结果。
|
||||
// status=ready 时 transcodedFileID/transcodedSize 指向转码产物;
|
||||
// 其它 status 调用方应传空值,本函数会按传入值原样覆盖。
|
||||
func (c *Catalog) UpdateVideoTranscode(ctx context.Context, id, status, errMsg, transcodedFileID string, transcodedSize int64) error {
|
||||
_, err := c.db.ExecContext(ctx,
|
||||
`UPDATE videos SET transcode_status = ?, transcode_error = ?, transcoded_file_id = ?, transcoded_size = ?, updated_at = ? WHERE id = ?`,
|
||||
status, errMsg, transcodedFileID, transcodedSize, time.Now().UnixMilli(), id)
|
||||
return err
|
||||
}
|
||||
|
||||
// DriveTranscodeCounts 是单盘的转码进度统计。
|
||||
type DriveTranscodeCounts struct {
|
||||
// Pending 是仍在候选集合里、还没有出结果的数量(含从未检测过的)。
|
||||
Pending int
|
||||
Ready int
|
||||
Failed int
|
||||
Skipped int
|
||||
}
|
||||
|
||||
func (c *Catalog) CountTranscodesByDrive(ctx context.Context) (map[string]DriveTranscodeCounts, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT drive_id,
|
||||
COUNT(CASE WHEN COALESCE(ext, '') NOT IN ('mp4', 'webm', 'm4v', 'strm')
|
||||
AND COALESCE(transcode_status, '') IN ('', 'pending') THEN 1 END) AS pending_count,
|
||||
COUNT(CASE WHEN COALESCE(transcode_status, '') = 'ready' THEN 1 END) AS ready_count,
|
||||
COUNT(CASE WHEN COALESCE(transcode_status, '') = 'failed' THEN 1 END) AS failed_count,
|
||||
COUNT(CASE WHEN COALESCE(transcode_status, '') = 'skipped' THEN 1 END) AS skipped_count
|
||||
FROM videos
|
||||
GROUP BY drive_id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make(map[string]DriveTranscodeCounts)
|
||||
for rows.Next() {
|
||||
var driveID string
|
||||
var counts DriveTranscodeCounts
|
||||
if err := rows.Scan(&driveID, &counts.Pending, &counts.Ready, &counts.Failed, &counts.Skipped); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[driveID] = counts
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) HideVideo(ctx context.Context, id string) error {
|
||||
res, err := c.db.ExecContext(ctx,
|
||||
`UPDATE videos SET hidden = 1, updated_at = ? WHERE id = ?`,
|
||||
@@ -186,6 +287,27 @@ func (c *Catalog) HideVideo(ctx context.Context, id string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListHiddenVideos 返回所有被标记隐藏(hidden=1)的视频。
|
||||
// 仅用于一次性把历史「隐藏」视频迁移为黑名单墓碑——隐藏机制已废弃,
|
||||
// 前台「不再展示」改走拉黑逻辑。
|
||||
func (c *Catalog) ListHiddenVideos(ctx context.Context) ([]*Video, error) {
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos WHERE COALESCE(hidden, 0) = 1`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// MigrateVideoToDrive 把 catalog 里 id=videoID 这条视频迁移到另一个 drive。
|
||||
// 用于 spider91 → PikPak 的迁移:上传成功后改写 drive_id / file_id /
|
||||
// content_hash,保留视频自身的 id(spider91-<driveID>-<sourceID>),这样
|
||||
@@ -706,12 +828,27 @@ func (c *Catalog) ListVideoFileIDsByDrive(ctx context.Context, driveID string) (
|
||||
// 用途:crawler 把这个集合写到 seen 文件,让 Python/Go 跳过已爬过的视频,
|
||||
// 配合 --target-new 真正凑出 N 个未爬过的视频。
|
||||
func (c *Catalog) ListSpider91Viewkeys(ctx context.Context, driveID string) ([]string, error) {
|
||||
prefix := "spider91-" + driveID + "-"
|
||||
return c.ListCrawlerSourceIDs(ctx, "spider91", driveID)
|
||||
}
|
||||
|
||||
// ListCrawlerSourceIDs lists source IDs that were already imported by a
|
||||
// crawler-like drive. It reads both videos and deleted_videos so explicit admin
|
||||
// deletions remain tombstoned for future crawler runs.
|
||||
func (c *Catalog) ListCrawlerSourceIDs(ctx context.Context, kind, driveID string) ([]string, error) {
|
||||
kind = strings.TrimSpace(kind)
|
||||
driveID = strings.TrimSpace(driveID)
|
||||
if kind == "" || driveID == "" {
|
||||
return nil, nil
|
||||
}
|
||||
prefix := kind + "-" + driveID + "-"
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT SUBSTR(id, ?) FROM videos WHERE id LIKE ? || '%'
|
||||
UNION
|
||||
SELECT SUBSTR(id, ?) FROM deleted_videos WHERE id LIKE ? || '%'`,
|
||||
len(prefix)+1, prefix, len(prefix)+1, prefix)
|
||||
SELECT SUBSTR(id, ?) FROM deleted_videos WHERE id LIKE ? || '%'
|
||||
UNION
|
||||
SELECT source_id FROM crawler_seen_sources
|
||||
WHERE kind = ? AND drive_id = ? AND status IN ('imported', 'duplicate')`,
|
||||
len(prefix)+1, prefix, len(prefix)+1, prefix, kind, driveID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -729,6 +866,47 @@ func (c *Catalog) ListSpider91Viewkeys(ctx context.Context, driveID string) ([]s
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// MarkCrawlerSourceSeen records the outcome for a crawler source item. Duplicate
|
||||
// source IDs are included in future seen files so scripts can skip them before
|
||||
// the backend downloads the same duplicate content again.
|
||||
func (c *Catalog) MarkCrawlerSourceSeen(ctx context.Context, kind, driveID, sourceID, status, canonicalVideoID, sampledSHA256 string, size int64) error {
|
||||
kind = strings.TrimSpace(kind)
|
||||
driveID = strings.TrimSpace(driveID)
|
||||
sourceID = strings.TrimSpace(sourceID)
|
||||
status = strings.TrimSpace(status)
|
||||
if kind == "" || driveID == "" || sourceID == "" {
|
||||
return nil
|
||||
}
|
||||
switch status {
|
||||
case "imported", "duplicate":
|
||||
default:
|
||||
return fmt.Errorf("catalog: unsupported crawler source status %q", status)
|
||||
}
|
||||
sampledSHA256 = normalizeContentHash(sampledSHA256)
|
||||
if size < 0 {
|
||||
size = 0
|
||||
}
|
||||
now := time.Now().UnixMilli()
|
||||
_, err := c.db.ExecContext(ctx, `
|
||||
INSERT INTO crawler_seen_sources (
|
||||
kind, drive_id, source_id, status, canonical_video_id, sampled_sha256, size_bytes, first_seen_at, last_seen_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(kind, drive_id, source_id) DO UPDATE SET
|
||||
status = excluded.status,
|
||||
canonical_video_id = excluded.canonical_video_id,
|
||||
sampled_sha256 = CASE
|
||||
WHEN excluded.sampled_sha256 != '' THEN excluded.sampled_sha256
|
||||
ELSE crawler_seen_sources.sampled_sha256
|
||||
END,
|
||||
size_bytes = CASE
|
||||
WHEN excluded.size_bytes > 0 THEN excluded.size_bytes
|
||||
ELSE crawler_seen_sources.size_bytes
|
||||
END,
|
||||
last_seen_at = excluded.last_seen_at`,
|
||||
kind, driveID, sourceID, status, strings.TrimSpace(canonicalVideoID), sampledSHA256, size, now, now)
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteVideoWithTombstone records that an administrator explicitly deleted a
|
||||
// video, then removes the visible catalog row. The tombstone is used by
|
||||
// scanners/crawlers to avoid importing the same source file again.
|
||||
@@ -825,6 +1003,92 @@ func (c *Catalog) DeleteVideo(ctx context.Context, id string) error {
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeletedVideo 是黑名单(墓碑)表里的一条记录。原始视频行已删除,
|
||||
// 这里只保留扫盘去重和后台展示需要的最小字段;没有 title/封面/作者。
|
||||
type DeletedVideo struct {
|
||||
ID string `json:"id"`
|
||||
DriveID string `json:"driveId"`
|
||||
FileID string `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
Size int64 `json:"size"`
|
||||
DeletedAt int64 `json:"deletedAt"` // unix 毫秒
|
||||
}
|
||||
|
||||
// ListDeletedVideos 分页列出黑名单视频,按拉黑时间倒序。
|
||||
// keyword 非空时按文件名模糊匹配。
|
||||
func (c *Catalog) ListDeletedVideos(ctx context.Context, keyword string, page, size int) ([]*DeletedVideo, int, error) {
|
||||
if size <= 0 {
|
||||
size = 50
|
||||
}
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
var where []string
|
||||
var args []any
|
||||
if kw := strings.TrimSpace(keyword); kw != "" {
|
||||
where = append(where, "file_name LIKE ?")
|
||||
args = append(args, "%"+kw+"%")
|
||||
}
|
||||
whereSQL := ""
|
||||
if len(where) > 0 {
|
||||
whereSQL = " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
|
||||
var total int
|
||||
if err := c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM deleted_videos`+whereSQL, args...).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
offset := (page - 1) * size
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT id, COALESCE(drive_id, ''), COALESCE(file_id, ''), COALESCE(file_name, ''), COALESCE(size_bytes, 0), deleted_at
|
||||
FROM deleted_videos`+whereSQL+`
|
||||
ORDER BY deleted_at DESC
|
||||
LIMIT ? OFFSET ?`,
|
||||
append(args, size, offset)...)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []*DeletedVideo
|
||||
for rows.Next() {
|
||||
v := &DeletedVideo{}
|
||||
if err := rows.Scan(&v.ID, &v.DriveID, &v.FileID, &v.FileName, &v.Size, &v.DeletedAt); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, total, rows.Err()
|
||||
}
|
||||
|
||||
// RemoveDeletedVideo 把视频移出黑名单(删除墓碑)。移除后该视频会在
|
||||
// 下次扫盘/凌晨流水线时被重新发现并入库,本函数不主动触发扫描。
|
||||
func (c *Catalog) RemoveDeletedVideo(ctx context.Context, id string) error {
|
||||
res, err := c.db.ExecContext(ctx, `DELETE FROM deleted_videos WHERE id = ?`, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rows, err := res.RowsAffected(); err == nil && rows == 0 {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// VideoManagementCounts 返回后台视频管理两个标签的计数:
|
||||
// current=当前可见(与「当前视频」页一致的去重+在线盘+hidden=0 口径),
|
||||
// blacklisted=黑名单墓碑总数。
|
||||
func (c *Catalog) VideoManagementCounts(ctx context.Context) (current, blacklisted int, err error) {
|
||||
currentSQL := `SELECT COUNT(*) FROM videos WHERE COALESCE(hidden, 0) = 0 AND ` + activeDriveWhereSQL + ` AND ` + uniqueVideoWhereSQL
|
||||
if err = c.db.QueryRowContext(ctx, currentSQL).Scan(¤t); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if err = c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM deleted_videos`).Scan(&blacklisted); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return current, blacklisted, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) IsVideoDeleted(ctx context.Context, id string) (bool, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
@@ -901,6 +1165,103 @@ func (c *Catalog) FindVideoByFileSignature(ctx context.Context, fileName string,
|
||||
return scanVideo(row)
|
||||
}
|
||||
|
||||
// FindEquivalentVideo returns the earliest visible video that represents the
|
||||
// same content as source by strong hash or sampled fingerprint, regardless of
|
||||
// which drive currently owns it.
|
||||
func (c *Catalog) FindEquivalentVideo(ctx context.Context, source *Video) (*Video, error) {
|
||||
if source == nil {
|
||||
return nil, sql.ErrNoRows
|
||||
}
|
||||
where, args, ok := equivalentVideoLookupWhere(source)
|
||||
if !ok {
|
||||
return nil, sql.ErrNoRows
|
||||
}
|
||||
args = append([]any{source.ID}, args...)
|
||||
row := c.db.QueryRowContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE id != ?
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
AND COALESCE(file_id, '') != ''
|
||||
AND (`+where+`)
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT 1`, args...)
|
||||
return scanVideo(row)
|
||||
}
|
||||
|
||||
// FindEquivalentVideoOnDrive returns a visible video on driveID that represents
|
||||
// the same content as source by strong hash or sampled fingerprint.
|
||||
func (c *Catalog) FindEquivalentVideoOnDrive(ctx context.Context, source *Video, driveID string) (*Video, error) {
|
||||
driveID = strings.TrimSpace(driveID)
|
||||
if source == nil || driveID == "" {
|
||||
return nil, sql.ErrNoRows
|
||||
}
|
||||
where, args, ok := equivalentVideoLookupWhere(source)
|
||||
if !ok {
|
||||
return nil, sql.ErrNoRows
|
||||
}
|
||||
args = append([]any{driveID, source.ID}, args...)
|
||||
row := c.db.QueryRowContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND id != ?
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
AND COALESCE(file_id, '') != ''
|
||||
AND (`+where+`)
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT 1`, args...)
|
||||
return scanVideo(row)
|
||||
}
|
||||
|
||||
// HasReadyEquivalentPreview reports whether another visible row for the same
|
||||
// content already has a ready preview video.
|
||||
func (c *Catalog) HasReadyEquivalentPreview(ctx context.Context, source *Video) (bool, error) {
|
||||
if source == nil {
|
||||
return false, nil
|
||||
}
|
||||
where, args, ok := equivalentVideoLookupWhere(source)
|
||||
if !ok {
|
||||
return false, nil
|
||||
}
|
||||
args = append([]any{source.ID}, args...)
|
||||
var found int
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
`SELECT 1 FROM videos
|
||||
WHERE id != ?
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
AND COALESCE(preview_status, 'pending') = 'ready'
|
||||
AND (`+where+`)
|
||||
LIMIT 1`, args...).Scan(&found)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func equivalentVideoLookupWhere(source *Video) (string, []any, bool) {
|
||||
if source == nil {
|
||||
return "", nil, false
|
||||
}
|
||||
var parts []string
|
||||
var args []any
|
||||
if hash := normalizeContentHash(source.ContentHash); hash != "" {
|
||||
parts = append(parts, "(COALESCE(content_hash, '') != '' AND content_hash = ?)")
|
||||
args = append(args, hash)
|
||||
}
|
||||
if source.Size > 0 {
|
||||
if sampled := normalizeContentHash(source.SampledSHA256); sampled != "" {
|
||||
parts = append(parts, "(size_bytes = ? AND COALESCE(sampled_sha256, '') != '' AND sampled_sha256 = ?)")
|
||||
args = append(args, source.Size, sampled)
|
||||
}
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "", nil, false
|
||||
}
|
||||
return strings.Join(parts, " OR "), args, true
|
||||
}
|
||||
|
||||
func (c *Catalog) ListVideosNeedingFingerprint(ctx context.Context, driveID string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10000
|
||||
@@ -1172,160 +1533,6 @@ func cleanVideoIDs(ids []string) []string {
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func cleanTagLabels(labels []string) []string {
|
||||
seen := make(map[string]struct{}, len(labels))
|
||||
cleaned := make([]string, 0, len(labels))
|
||||
for _, label := range labels {
|
||||
label = strings.TrimSpace(label)
|
||||
if label == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(label)
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
cleaned = append(cleaned, label)
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func (c *Catalog) LeastPopulatedVisibleUniqueTag(ctx context.Context, labels []string) (string, error) {
|
||||
cleaned := cleanTagLabels(labels)
|
||||
bestLabel := ""
|
||||
bestCount := 0
|
||||
for _, label := range cleaned {
|
||||
var count int
|
||||
if err := c.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*)
|
||||
FROM videos
|
||||
WHERE COALESCE(hidden, 0) = 0
|
||||
AND `+activeDriveWhereSQL+`
|
||||
AND `+uniqueVideoWhereSQL+`
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
JOIN tags t ON t.id = vt.tag_id
|
||||
WHERE vt.video_id = videos.id
|
||||
AND t.label = ? COLLATE NOCASE
|
||||
)`,
|
||||
label,
|
||||
).Scan(&count); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if count == 0 {
|
||||
continue
|
||||
}
|
||||
if bestLabel == "" || count < bestCount {
|
||||
bestLabel = label
|
||||
bestCount = count
|
||||
}
|
||||
}
|
||||
return bestLabel, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) RandomVideosByTagExcluding(ctx context.Context, tag string, excludeIDs []string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
tag = strings.TrimSpace(tag)
|
||||
if tag == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
cleaned := cleanVideoIDs(excludeIDs)
|
||||
args := make([]any, 0, len(cleaned)+2)
|
||||
args = append(args, tag)
|
||||
whereSQL := `WHERE COALESCE(hidden, 0) = 0
|
||||
AND ` + activeDriveWhereSQL + `
|
||||
AND ` + uniqueVideoWhereSQL + `
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
JOIN tags t ON t.id = vt.tag_id
|
||||
WHERE vt.video_id = videos.id
|
||||
AND t.label = ? COLLATE NOCASE
|
||||
)`
|
||||
if len(cleaned) > 0 {
|
||||
placeholders := strings.Repeat("?,", len(cleaned))
|
||||
placeholders = placeholders[:len(placeholders)-1]
|
||||
whereSQL += " AND id NOT IN (" + placeholders + ")"
|
||||
for _, id := range cleaned {
|
||||
args = append(args, id)
|
||||
}
|
||||
}
|
||||
args = append(args, limit)
|
||||
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos `+whereSQL+`
|
||||
ORDER BY RANDOM() LIMIT ?`,
|
||||
args...,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) RandomVideosForPreferredVideoExcluding(ctx context.Context, preferredVideoID string, excludeIDs []string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
preferredVideoID = strings.TrimSpace(preferredVideoID)
|
||||
if preferredVideoID == "" {
|
||||
return c.RandomVideosExcluding(ctx, excludeIDs, limit)
|
||||
}
|
||||
|
||||
preferredExclude := append([]string{}, excludeIDs...)
|
||||
preferredExclude = append(preferredExclude, preferredVideoID)
|
||||
|
||||
preferred, err := c.GetVideo(ctx, preferredVideoID)
|
||||
if err != nil || preferred == nil || preferred.Hidden || len(preferred.Tags) == 0 {
|
||||
return c.RandomVideosExcluding(ctx, preferredExclude, limit)
|
||||
}
|
||||
tag, err := c.LeastPopulatedVisibleUniqueTag(ctx, preferred.Tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tag == "" {
|
||||
return c.RandomVideosExcluding(ctx, preferredExclude, limit)
|
||||
}
|
||||
|
||||
items, err := c.RandomVideosByTagExcluding(ctx, tag, preferredExclude, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(items) >= limit {
|
||||
return items, nil
|
||||
}
|
||||
|
||||
mergedExclude := make([]string, 0, len(preferredExclude)+len(items))
|
||||
mergedExclude = append(mergedExclude, preferredExclude...)
|
||||
for _, item := range items {
|
||||
if item != nil {
|
||||
mergedExclude = append(mergedExclude, item.ID)
|
||||
}
|
||||
}
|
||||
fallback, err := c.RandomVideosExcluding(ctx, mergedExclude, limit-len(items))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(items, fallback...), nil
|
||||
}
|
||||
|
||||
type DriveTeaserCounts struct {
|
||||
Ready int
|
||||
Pending int
|
||||
@@ -1443,6 +1650,121 @@ func (c *Catalog) CountFingerprintsByDrive(ctx context.Context) (map[string]Driv
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) CountCrawlerAssets(ctx context.Context, crawlerID string, prefixes []string) (CrawlerAssetCounts, error) {
|
||||
var out CrawlerAssetCounts
|
||||
crawlerID = strings.TrimSpace(crawlerID)
|
||||
prefixes = cleanCrawlerIDPrefixes(prefixes)
|
||||
if crawlerID == "" || len(prefixes) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
where := make([]string, 0, len(prefixes))
|
||||
args := make([]any, 0, 2+len(prefixes))
|
||||
args = append(args, crawlerID, crawlerID)
|
||||
for range prefixes {
|
||||
where = append(where, "id LIKE ? ESCAPE '\\'")
|
||||
}
|
||||
for _, prefix := range prefixes {
|
||||
args = append(args, escapeSQLLike(prefix)+"%")
|
||||
}
|
||||
query := `SELECT
|
||||
COUNT(*) AS total_count,
|
||||
COUNT(CASE WHEN drive_id = ? THEN 1 END) AS local_count,
|
||||
COUNT(CASE WHEN drive_id != ? THEN 1 END) AS migrated_count,
|
||||
COUNT(CASE WHEN EXISTS (
|
||||
SELECT 1 FROM videos AS asset_dup
|
||||
WHERE ` + crawlerAssetEquivalentSQL("asset_dup", "videos") + `
|
||||
AND COALESCE(asset_dup.thumbnail_url, '') != ''
|
||||
) THEN 1 END) AS thumbnail_ready_count,
|
||||
COUNT(CASE WHEN NOT EXISTS (
|
||||
SELECT 1 FROM videos AS asset_dup
|
||||
WHERE ` + crawlerAssetEquivalentSQL("asset_dup", "videos") + `
|
||||
AND COALESCE(asset_dup.thumbnail_url, '') != ''
|
||||
)
|
||||
AND COALESCE(thumbnail_url, '') = ''
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('failed', 'skipped') THEN 1 END) AS thumbnail_pending_count,
|
||||
COUNT(CASE WHEN NOT EXISTS (
|
||||
SELECT 1 FROM videos AS asset_dup
|
||||
WHERE ` + crawlerAssetEquivalentSQL("asset_dup", "videos") + `
|
||||
AND COALESCE(asset_dup.thumbnail_url, '') != ''
|
||||
)
|
||||
AND COALESCE(thumbnail_url, '') = ''
|
||||
AND COALESCE(thumbnail_status, 'pending') = 'failed' THEN 1 END) AS thumbnail_failed_count,
|
||||
COUNT(CASE WHEN EXISTS (
|
||||
SELECT 1 FROM videos AS asset_dup
|
||||
WHERE ` + crawlerAssetEquivalentSQL("asset_dup", "videos") + `
|
||||
AND COALESCE(asset_dup.preview_status, 'pending') = 'ready'
|
||||
) THEN 1 END) AS teaser_ready_count,
|
||||
COUNT(CASE WHEN NOT EXISTS (
|
||||
SELECT 1 FROM videos AS asset_dup
|
||||
WHERE ` + crawlerAssetEquivalentSQL("asset_dup", "videos") + `
|
||||
AND COALESCE(asset_dup.preview_status, 'pending') = 'ready'
|
||||
)
|
||||
AND COALESCE(preview_status, 'pending') = 'pending' THEN 1 END) AS teaser_pending_count,
|
||||
COUNT(CASE WHEN NOT EXISTS (
|
||||
SELECT 1 FROM videos AS asset_dup
|
||||
WHERE ` + crawlerAssetEquivalentSQL("asset_dup", "videos") + `
|
||||
AND COALESCE(asset_dup.preview_status, 'pending') = 'ready'
|
||||
)
|
||||
AND COALESCE(preview_status, 'pending') = 'failed' THEN 1 END) AS teaser_failed_count,
|
||||
COUNT(CASE WHEN COALESCE(sampled_sha256, '') != ''
|
||||
OR COALESCE(fingerprint_status, 'pending') = 'ready' THEN 1 END) AS fingerprint_ready_count,
|
||||
COUNT(CASE WHEN size_bytes > 0
|
||||
AND COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = 'pending' THEN 1 END) AS fingerprint_pending_count,
|
||||
COUNT(CASE WHEN COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = 'failed' THEN 1 END) AS fingerprint_failed_count
|
||||
FROM videos
|
||||
WHERE COALESCE(hidden, 0) = 0
|
||||
AND (` + strings.Join(where, " OR ") + `)`
|
||||
err := c.db.QueryRowContext(ctx, query, args...).Scan(
|
||||
&out.Total,
|
||||
&out.Local,
|
||||
&out.Migrated,
|
||||
&out.Thumbnail.Ready,
|
||||
&out.Thumbnail.Pending,
|
||||
&out.Thumbnail.Failed,
|
||||
&out.Teaser.Ready,
|
||||
&out.Teaser.Pending,
|
||||
&out.Teaser.Failed,
|
||||
&out.Fingerprint.Ready,
|
||||
&out.Fingerprint.Pending,
|
||||
&out.Fingerprint.Failed,
|
||||
)
|
||||
return out, err
|
||||
}
|
||||
|
||||
func crawlerAssetEquivalentSQL(candidateAlias, sourceAlias string) string {
|
||||
return fmt.Sprintf(`(%[1]s.id = %[2]s.id
|
||||
OR (COALESCE(%[2]s.content_hash, '') != ''
|
||||
AND %[1]s.content_hash = %[2]s.content_hash)
|
||||
OR (%[2]s.size_bytes > 0
|
||||
AND COALESCE(%[2]s.sampled_sha256, '') != ''
|
||||
AND %[1]s.size_bytes = %[2]s.size_bytes
|
||||
AND %[1]s.sampled_sha256 = %[2]s.sampled_sha256))`, candidateAlias, sourceAlias)
|
||||
}
|
||||
|
||||
func cleanCrawlerIDPrefixes(prefixes []string) []string {
|
||||
out := make([]string, 0, len(prefixes))
|
||||
seen := map[string]bool{}
|
||||
for _, prefix := range prefixes {
|
||||
prefix = strings.TrimSpace(prefix)
|
||||
if prefix == "" || seen[prefix] {
|
||||
continue
|
||||
}
|
||||
seen[prefix] = true
|
||||
out = append(out, prefix)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func escapeSQLLike(raw string) string {
|
||||
raw = strings.ReplaceAll(raw, `\`, `\\`)
|
||||
raw = strings.ReplaceAll(raw, `%`, `\%`)
|
||||
raw = strings.ReplaceAll(raw, `_`, `\_`)
|
||||
return raw
|
||||
}
|
||||
|
||||
func (c *Catalog) CountVideosNeedingFingerprint(ctx context.Context, driveID string) (int, error) {
|
||||
var count int
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
@@ -1615,7 +1937,7 @@ type Drive struct {
|
||||
Credentials map[string]string `json:"credentials,omitempty"`
|
||||
Status string `json:"status"`
|
||||
LastError string `json:"lastError,omitempty"`
|
||||
// TeaserEnabled 控制是否给本盘生成预览视频/封面。
|
||||
// TeaserEnabled 控制是否给本盘生成预览视频;封面生成不受影响。
|
||||
// 替代早期的全局 preview.enabled 开关;新建 drive 时 UpsertDrive 默认置 true。
|
||||
TeaserEnabled bool `json:"teaserEnabled"`
|
||||
// SkipDirIDs 是用户在管理后台为该盘选定的"扫描跳过目录"集合(网盘侧的目录 fileID)。
|
||||
@@ -1670,7 +1992,7 @@ func normalizeDriveRootFields(d *Drive) {
|
||||
func normalizeDriveRootID(kind, rootID string) string {
|
||||
rootID = strings.TrimSpace(rootID)
|
||||
switch kind {
|
||||
case "pikpak":
|
||||
case "pikpak", "guangyapan":
|
||||
if rootID == "0" {
|
||||
return ""
|
||||
}
|
||||
@@ -1748,7 +2070,7 @@ func (c *Catalog) DeleteDrive(ctx context.Context, id string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// SetDriveTeaserEnabled 切换某盘的预览视频/封面生成开关。
|
||||
// SetDriveTeaserEnabled 切换某盘的预览视频生成开关。
|
||||
//
|
||||
// 与 UpsertDrive 的区别:只动 teaser_enabled + updated_at 一列,不要求调用方
|
||||
// 重传 kind / name / credentials 等容易踩坑的字段。
|
||||
@@ -1880,6 +2202,7 @@ COALESCE(sampled_sha256, ''), COALESCE(fingerprint_status, 'pending'), COALESCE(
|
||||
COALESCE(parent_id, ''), title, COALESCE(author, ''), COALESCE(tags, '[]'),
|
||||
duration_seconds, size_bytes, COALESCE(ext, ''), COALESCE(quality, ''), COALESCE(thumbnail_url, ''),
|
||||
COALESCE(preview_file_id, ''), COALESCE(preview_local, ''), COALESCE(preview_status, 'pending'),
|
||||
COALESCE(transcode_status, ''), COALESCE(transcode_error, ''), COALESCE(transcoded_file_id, ''), COALESCE(transcoded_size, 0),
|
||||
views, favorites, comments, likes, dislikes,
|
||||
COALESCE(category, ''), COALESCE(hidden, 0), COALESCE(badges, '[]'), COALESCE(description, ''),
|
||||
published_at, created_at, updated_at
|
||||
@@ -1951,6 +2274,7 @@ func scanVideo(row rowScanner) (*Video, error) {
|
||||
&v.ParentID, &v.Title, &v.Author, &tagsJSON,
|
||||
&v.DurationSeconds, &v.Size, &v.Ext, &v.Quality, &v.ThumbnailURL,
|
||||
&v.PreviewFileID, &v.PreviewLocal, &v.PreviewStatus,
|
||||
&v.TranscodeStatus, &v.TranscodeError, &v.TranscodedFileID, &v.TranscodedSize,
|
||||
&v.Views, &v.Favorites, &v.Comments, &v.Likes, &v.Dislikes,
|
||||
&v.Category, &hidden, &badgesJSON, &v.Description,
|
||||
&publishedAt, &createdAt, &updatedAt,
|
||||
|
||||
@@ -58,6 +58,7 @@ func TestUpsertDriveDefaultsRootIDByKind(t *testing.T) {
|
||||
}{
|
||||
{id: "p115", kind: "p115", want: "0"},
|
||||
{id: "pikpak", kind: "pikpak", want: ""},
|
||||
{id: "guangyapan", kind: "guangyapan", want: ""},
|
||||
{id: "onedrive", kind: "onedrive", want: "root"},
|
||||
{id: "googledrive", kind: "googledrive", want: "root"},
|
||||
{id: "localstorage", kind: "localstorage", want: "/"},
|
||||
|
||||
@@ -21,7 +21,11 @@ CREATE TABLE IF NOT EXISTS videos (
|
||||
thumbnail_failures INTEGER DEFAULT 0, -- consecutive transient thumbnail generation failures
|
||||
preview_file_id TEXT, -- deprecated: 旧版回写网盘后的预览视频 file id
|
||||
preview_local TEXT, -- 本地预览视频路径(兜底)
|
||||
preview_status TEXT DEFAULT 'pending', -- pending / ready / failed
|
||||
preview_status TEXT DEFAULT 'pending', -- pending / ready / failed / disabled
|
||||
transcode_status TEXT DEFAULT '', -- '' / pending / ready / skipped / failed(浏览器兼容性转码)
|
||||
transcode_error TEXT DEFAULT '',
|
||||
transcoded_file_id TEXT DEFAULT '', -- 转码产物在同一 drive 上的 fileID,播放源优先用它
|
||||
transcoded_size INTEGER DEFAULT 0,
|
||||
views INTEGER DEFAULT 0,
|
||||
favorites INTEGER DEFAULT 0,
|
||||
comments INTEGER DEFAULT 0,
|
||||
@@ -89,17 +93,35 @@ CREATE INDEX IF NOT EXISTS idx_deleted_videos_drive_hash
|
||||
CREATE INDEX IF NOT EXISTS idx_deleted_videos_drive_signature
|
||||
ON deleted_videos(drive_id, file_name, size_bytes);
|
||||
|
||||
-- 爬虫来源记录。用于把已确认重复的 source_id 写回 seen 列表,
|
||||
-- 避免后续爬虫反复下载同一个候选视频。
|
||||
CREATE TABLE IF NOT EXISTS crawler_seen_sources (
|
||||
kind TEXT NOT NULL,
|
||||
drive_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'imported', -- imported / duplicate
|
||||
canonical_video_id TEXT NOT NULL DEFAULT '',
|
||||
sampled_sha256 TEXT NOT NULL DEFAULT '',
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
first_seen_at INTEGER NOT NULL,
|
||||
last_seen_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (kind, drive_id, source_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_crawler_seen_sources_drive
|
||||
ON crawler_seen_sources(kind, drive_id, status);
|
||||
|
||||
-- 网盘账户
|
||||
CREATE TABLE IF NOT EXISTS drives (
|
||||
id TEXT PRIMARY KEY,
|
||||
kind TEXT NOT NULL, -- quark / p115 / p123 / pikpak / wopan / onedrive / googledrive / localstorage / spider91
|
||||
kind TEXT NOT NULL, -- quark / p115 / p123 / pikpak / wopan / guangyapan / onedrive / googledrive / localstorage / spider91
|
||||
name TEXT NOT NULL,
|
||||
root_id TEXT NOT NULL DEFAULT '0',
|
||||
scan_root_id TEXT, -- deprecated: 扫描起点固定等于 root_id
|
||||
credentials TEXT, -- JSON: cookie / refresh_token 等
|
||||
status TEXT DEFAULT 'disconnected', -- disconnected / ok / error
|
||||
last_error TEXT,
|
||||
-- 是否给该盘生成预览视频/封面:1 开 / 0 关。
|
||||
-- 是否给该盘生成预览视频:1 开 / 0 关。封面生成不受影响。
|
||||
-- 替代了早期的全局 preview.enabled 设置(保留旧 setting 行不再读)。
|
||||
teaser_enabled INTEGER NOT NULL DEFAULT 1,
|
||||
-- 扫描时要跳过的目录 ID 集合(JSON array of string)。命中其中任意一个的目录及其
|
||||
|
||||
@@ -165,171 +165,3 @@ func TestRandomVideosWithReadyThumbnailsExcluding(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoChoosesLeastPopulatedTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "current", DriveID: "drive", FileID: "f-current", Title: "current", Tags: []string{"common", "rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-1", DriveID: "drive", FileID: "f-common-1", Title: "common 1", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-2", DriveID: "drive", FileID: "f-common-2", Title: "common 2", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "rare-1", DriveID: "drive", FileID: "f-rare-1", Title: "rare 1", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
tag, err := cat.LeastPopulatedVisibleUniqueTag(ctx, []string{"common", "rare"})
|
||||
if err != nil {
|
||||
t.Fatalf("least populated tag: %v", err)
|
||||
}
|
||||
if tag != "rare" {
|
||||
t.Fatalf("least populated tag = %q, want rare", tag)
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosForPreferredVideoExcluding(ctx, "current", []string{"current"}, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("random preferred: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID != "rare-1" {
|
||||
t.Fatalf("preferred result = %#v, want rare-1", videoIDs(got))
|
||||
}
|
||||
|
||||
got, err = cat.RandomVideosForPreferredVideoExcluding(ctx, "current", nil, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("random preferred without explicit exclude: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID == "current" {
|
||||
t.Fatalf("preferred result without explicit exclude = %#v, should not return current", videoIDs(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoFallsBackToFillBatch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "current", DriveID: "drive", FileID: "f-current", Title: "current", Tags: []string{"common", "rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-1", DriveID: "drive", FileID: "f-common-1", Title: "common 1", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-2", DriveID: "drive", FileID: "f-common-2", Title: "common 2", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "rare-1", DriveID: "drive", FileID: "f-rare-1", Title: "rare 1", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "hidden-rare", DriveID: "drive", FileID: "f-hidden-rare", Title: "hidden rare", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
if err := cat.HideVideo(ctx, "hidden-rare"); err != nil {
|
||||
t.Fatalf("hide hidden-rare: %v", err)
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosForPreferredVideoExcluding(ctx, "current", []string{"current"}, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("random preferred: %v", err)
|
||||
}
|
||||
ids := videoIDs(got)
|
||||
if len(ids) != 3 {
|
||||
t.Fatalf("result ids = %#v, want 3 items", ids)
|
||||
}
|
||||
for _, excluded := range []string{"current", "hidden-rare"} {
|
||||
if hasVideoID(ids, excluded) {
|
||||
t.Fatalf("result ids = %#v, should not include %s", ids, excluded)
|
||||
}
|
||||
}
|
||||
if !hasVideoID(ids, "rare-1") {
|
||||
t.Fatalf("result ids = %#v, want rare-1 from least populated tag", ids)
|
||||
}
|
||||
if len(uniqueVideoIDs(ids)) != len(ids) {
|
||||
t.Fatalf("result ids = %#v, want no duplicates", ids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoFallbacksWhenPreferenceUnavailable(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "untagged", DriveID: "drive", FileID: "f-untagged", Title: "untagged", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "visible-1", DriveID: "drive", FileID: "f-visible-1", Title: "visible 1", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "visible-2", DriveID: "drive", FileID: "f-visible-2", Title: "visible 2", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosForPreferredVideoExcluding(ctx, "missing", []string{"untagged"}, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("random missing preferred: %v", err)
|
||||
}
|
||||
if !sameVideoIDSet(videoIDs(got), []string{"visible-1", "visible-2"}) {
|
||||
t.Fatalf("missing preferred ids = %#v, want visible fallback videos", videoIDs(got))
|
||||
}
|
||||
|
||||
got, err = cat.RandomVideosForPreferredVideoExcluding(ctx, "untagged", []string{"untagged"}, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("random untagged preferred: %v", err)
|
||||
}
|
||||
if !sameVideoIDSet(videoIDs(got), []string{"visible-1", "visible-2"}) {
|
||||
t.Fatalf("untagged preferred ids = %#v, want visible fallback videos", videoIDs(got))
|
||||
}
|
||||
}
|
||||
|
||||
func videoIDs(videos []*Video) []string {
|
||||
ids := make([]string, 0, len(videos))
|
||||
for _, v := range videos {
|
||||
ids = append(ids, v.ID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func hasVideoID(ids []string, want string) bool {
|
||||
for _, id := range ids {
|
||||
if id == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func uniqueVideoIDs(ids []string) map[string]struct{} {
|
||||
seen := make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
seen[id] = struct{}{}
|
||||
}
|
||||
return seen
|
||||
}
|
||||
|
||||
func sameVideoIDSet(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
seen := make(map[string]int, len(a))
|
||||
for _, value := range a {
|
||||
seen[value]++
|
||||
}
|
||||
for _, value := range b {
|
||||
if seen[value] == 0 {
|
||||
return false
|
||||
}
|
||||
seen[value]--
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -66,6 +66,21 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "thumbnail_failures", "INTEGER DEFAULT 0"); err != nil {
|
||||
return err
|
||||
}
|
||||
// videos.transcode_*:浏览器兼容性转码状态。
|
||||
// status:''=未检测 / pending=已入队 / ready=已转码 / skipped=检测后无需转码 / failed=失败。
|
||||
// transcoded_file_id 指向转码产物在同一 drive 上的 fileID,播放源优先使用它。
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "transcode_status", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "transcode_error", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "transcoded_file_id", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "transcoded_size", "INTEGER DEFAULT 0"); err != nil {
|
||||
return err
|
||||
}
|
||||
// drives.teaser_enabled:每盘预览视频开关,替代旧的全局 preview.enabled。
|
||||
// 升级路径:直接让 ALTER TABLE 的 DEFAULT 1 兜底 —— 每个现存 drive 都默认开启,
|
||||
// 不读旧的 settings.preview.enabled 字段。这样老用户即便之前关过全局开关,
|
||||
@@ -109,6 +124,9 @@ CREATE TABLE IF NOT EXISTS deleted_videos (
|
||||
if err := c.reconcileThumbnailStatusOnce(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.requeueSkippedPreviews(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_content_hash ON videos(content_hash)`); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -281,6 +299,24 @@ UPDATE videos
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Catalog) requeueSkippedPreviews(ctx context.Context) error {
|
||||
res, err := c.db.ExecContext(ctx, `
|
||||
UPDATE videos
|
||||
SET preview_file_id = '',
|
||||
preview_local = '',
|
||||
preview_status = 'pending',
|
||||
updated_at = ?
|
||||
WHERE COALESCE(preview_status, 'pending') = 'skipped'
|
||||
`, time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
return fmt.Errorf("requeue skipped previews: %w", err)
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected > 0 {
|
||||
log.Printf("[catalog] requeued %d skipped preview(s) for generation", affected)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Catalog) clearVolatileOneDriveThumbnails(ctx context.Context) error {
|
||||
// 把 OneDrive 过期的 mediap.svc.ms thumb URL 清空,让 worker 重新抽帧生成本地封面。
|
||||
// 同步把 thumbnail_status 重置为 'pending':清空后 url 是空的,本应进 worker 重做,
|
||||
@@ -297,7 +333,7 @@ UPDATE videos
|
||||
}
|
||||
|
||||
func (c *Catalog) clearRemoteP123ThumbnailsOnce(ctx context.Context) error {
|
||||
// 123 云盘列表返回的缩略图尺寸和稳定性都不适合作为站内封面;清空历史写入的
|
||||
// 123网盘列表返回的缩略图尺寸和稳定性都不适合作为站内封面;清空历史写入的
|
||||
// 远程 URL,让封面 worker 统一从视频直链抽帧生成本地 /p/thumb/<id>。
|
||||
const markerKey = "videos.p123.remote_thumbnails_cleared"
|
||||
marker, err := c.GetSetting(ctx, markerKey, "")
|
||||
|
||||
@@ -1539,6 +1539,70 @@ func TestReconcileThumbnailStatusOnce(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequeueSkippedPreviews(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
cases := []struct {
|
||||
id string
|
||||
status string
|
||||
local string
|
||||
fileID string
|
||||
wantStatus string
|
||||
wantLocal string
|
||||
wantFileID string
|
||||
}{
|
||||
{"preview-skipped", "skipped", "/tmp/old-preview.mp4", "old-preview-file", "pending", "", ""},
|
||||
{"preview-ready", "ready", "/tmp/ready-preview.mp4", "ready-preview-file", "ready", "/tmp/ready-preview.mp4", "ready-preview-file"},
|
||||
{"preview-failed", "failed", "/tmp/failed-preview.mp4", "failed-preview-file", "failed", "/tmp/failed-preview.mp4", "failed-preview-file"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: c.id, DriveID: "d", FileID: "source-" + c.id, Title: c.id,
|
||||
PreviewStatus: c.status, PreviewLocal: c.local, PreviewFileID: c.fileID,
|
||||
PublishedAt: now, CreatedAt: now, UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", c.id, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := cat.requeueSkippedPreviews(ctx); err != nil {
|
||||
t.Fatalf("requeue skipped previews: %v", err)
|
||||
}
|
||||
if err := cat.requeueSkippedPreviews(ctx); err != nil {
|
||||
t.Fatalf("second requeue skipped previews: %v", err)
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
got, err := cat.GetVideo(ctx, c.id)
|
||||
if err != nil {
|
||||
t.Fatalf("get %s: %v", c.id, err)
|
||||
}
|
||||
if got.PreviewStatus != c.wantStatus {
|
||||
t.Errorf("%s: preview status = %q, want %q", c.id, got.PreviewStatus, c.wantStatus)
|
||||
}
|
||||
if got.PreviewLocal != c.wantLocal {
|
||||
t.Errorf("%s: preview local = %q, want %q", c.id, got.PreviewLocal, c.wantLocal)
|
||||
}
|
||||
if got.PreviewFileID != c.wantFileID {
|
||||
t.Errorf("%s: preview file id = %q, want %q", c.id, got.PreviewFileID, c.wantFileID)
|
||||
}
|
||||
}
|
||||
|
||||
pending, err := cat.ListVideosByPreviewStatus(ctx, "d", "pending", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list pending previews: %v", err)
|
||||
}
|
||||
if len(pending) != 1 || pending[0].ID != "preview-skipped" {
|
||||
t.Fatalf("pending previews = %#v, want only preview-skipped", pending)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpsertVideoSyncsThumbnailStatus 验证 scanner 创建/补回视频时
|
||||
// thumbnail_status 跟随 thumbnail_url 自动设。这是历史 bug 的修复回归测试 ——
|
||||
// 之前 UpsertVideo 的 SQL 不带 thumbnail_status 列,所有新视频都依赖
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestListHiddenVideosForMigration 验证:隐藏的视频不进可见列表,
|
||||
// 但能被 ListHiddenVideos 拿到(供一次性迁移为墓碑)。
|
||||
func TestListHiddenVideosForMigration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, id := range []string{"v1", "v2", "v3"} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id, DriveID: "drive", FileID: "f-" + id, Title: id,
|
||||
PublishedAt: now, CreatedAt: now, UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
if err := cat.HideVideo(ctx, "v2"); err != nil {
|
||||
t.Fatalf("hide v2: %v", err)
|
||||
}
|
||||
|
||||
visible, total, err := cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 50})
|
||||
if err != nil {
|
||||
t.Fatalf("list visible: %v", err)
|
||||
}
|
||||
if total != 2 || len(visible) != 2 {
|
||||
t.Fatalf("visible total/len = %d/%d, want 2/2", total, len(visible))
|
||||
}
|
||||
for _, v := range visible {
|
||||
if v.ID == "v2" {
|
||||
t.Fatalf("hidden v2 leaked into visible list")
|
||||
}
|
||||
}
|
||||
|
||||
hidden, err := cat.ListHiddenVideos(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list hidden: %v", err)
|
||||
}
|
||||
if len(hidden) != 1 || hidden[0].ID != "v2" {
|
||||
t.Fatalf("ListHiddenVideos = %v, want only v2", hidden)
|
||||
}
|
||||
|
||||
current, blacklisted, err := cat.VideoManagementCounts(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("counts: %v", err)
|
||||
}
|
||||
if current != 2 || blacklisted != 0 {
|
||||
t.Fatalf("counts = current %d blacklisted %d, want 2/0", current, blacklisted)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBlacklistListAndRemove 验证墓碑表的列出、关键字过滤和移除。
|
||||
func TestBlacklistListAndRemove(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
seed := []struct{ id, file string }{
|
||||
{"d1", "movie-alpha.avi"},
|
||||
{"d2", "movie-beta.mp4"},
|
||||
{"d3", "clip-gamma.wmv"},
|
||||
}
|
||||
for _, s := range seed {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: s.id, DriveID: "drive", FileID: "f-" + s.id, FileName: s.file,
|
||||
Title: s.id, PublishedAt: now, CreatedAt: now, UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", s.id, err)
|
||||
}
|
||||
if err := cat.DeleteVideoWithTombstone(ctx, s.id); err != nil {
|
||||
t.Fatalf("tombstone %s: %v", s.id, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, total, err := cat.ListDeletedVideos(ctx, "", 1, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list deleted: %v", err)
|
||||
}
|
||||
if total != 3 || len(items) != 3 {
|
||||
t.Fatalf("deleted total/len = %d/%d, want 3/3", total, len(items))
|
||||
}
|
||||
|
||||
// 关键字过滤
|
||||
filtered, ftotal, err := cat.ListDeletedVideos(ctx, "movie", 1, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list deleted filtered: %v", err)
|
||||
}
|
||||
if ftotal != 2 || len(filtered) != 2 {
|
||||
t.Fatalf("filtered total/len = %d/%d, want 2/2", ftotal, len(filtered))
|
||||
}
|
||||
|
||||
// 移出黑名单
|
||||
if err := cat.RemoveDeletedVideo(ctx, "d1"); err != nil {
|
||||
t.Fatalf("remove d1: %v", err)
|
||||
}
|
||||
if deleted, err := cat.IsVideoDeleted(ctx, "d1"); err != nil || deleted {
|
||||
t.Fatalf("d1 should no longer be blacklisted (deleted=%v err=%v)", deleted, err)
|
||||
}
|
||||
_, total, err = cat.ListDeletedVideos(ctx, "", 1, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list deleted after remove: %v", err)
|
||||
}
|
||||
if total != 2 {
|
||||
t.Fatalf("deleted total after remove = %d, want 2", total)
|
||||
}
|
||||
|
||||
if err := cat.RemoveDeletedVideo(ctx, "does-not-exist"); err == nil {
|
||||
t.Fatalf("remove missing id should return error")
|
||||
}
|
||||
|
||||
// counts: 删完一个还剩 2 个黑名单;可见视频已全部被墓碑删除
|
||||
current, blacklisted, err := cat.VideoManagementCounts(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("counts: %v", err)
|
||||
}
|
||||
if current != 0 || blacklisted != 2 {
|
||||
t.Fatalf("counts = current %d blacklisted %d, want 0/2", current, blacklisted)
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,11 @@ const (
|
||||
DefaultAdminPassword = "admin123"
|
||||
)
|
||||
|
||||
var (
|
||||
legacyDefaultVideoExtensions = []string{".mp4", ".mkv", ".mov", ".webm", ".avi"}
|
||||
defaultVideoExtensions = []string{".mp4", ".mkv", ".mov", ".webm", ".avi", ".strm"}
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Server Server `yaml:"server"`
|
||||
Storage Storage `yaml:"storage"`
|
||||
@@ -202,7 +207,7 @@ type Nightly struct {
|
||||
// 这里保留 yaml 中的静态定义,用于启动时预置盘。生产建议只在 DB 里维护。
|
||||
type Drive struct {
|
||||
ID string `yaml:"id"`
|
||||
Kind string `yaml:"kind"` // quark / p115 / p123 / pikpak / wopan / onedrive / googledrive / localstorage
|
||||
Kind string `yaml:"kind"` // quark / p115 / p123 / pikpak / wopan / guangyapan / onedrive / googledrive / localstorage
|
||||
Name string `yaml:"name"`
|
||||
RootID string `yaml:"root_id"`
|
||||
Params map[string]string `yaml:"params,omitempty"`
|
||||
@@ -247,7 +252,9 @@ func (c *Config) applyDefaults() {
|
||||
c.Scanner.MaxDepth = 5
|
||||
}
|
||||
if len(c.Scanner.VideoExtensions) == 0 {
|
||||
c.Scanner.VideoExtensions = []string{".mp4", ".mkv", ".mov", ".webm", ".avi"}
|
||||
c.Scanner.VideoExtensions = append([]string{}, defaultVideoExtensions...)
|
||||
} else if isLegacyDefaultVideoExtensions(c.Scanner.VideoExtensions) {
|
||||
c.Scanner.VideoExtensions = append(c.Scanner.VideoExtensions, ".strm")
|
||||
}
|
||||
if c.Preview.FFmpegPath == "" {
|
||||
c.Preview.FFmpegPath = "ffmpeg"
|
||||
@@ -276,3 +283,19 @@ func (c *Config) applyDefaults() {
|
||||
c.Nightly.CronHour = 1
|
||||
}
|
||||
}
|
||||
|
||||
func isLegacyDefaultVideoExtensions(exts []string) bool {
|
||||
if len(exts) != len(legacyDefaultVideoExtensions) {
|
||||
return false
|
||||
}
|
||||
seen := make(map[string]struct{}, len(exts))
|
||||
for _, ext := range exts {
|
||||
seen[strings.ToLower(strings.TrimSpace(ext))] = struct{}{}
|
||||
}
|
||||
for _, ext := range legacyDefaultVideoExtensions {
|
||||
if _, ok := seen[ext]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package config
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -50,3 +51,64 @@ storage:
|
||||
t.Fatalf("db path = %q, want preserved value", cfg.Storage.DBPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadDefaultScannerVideoExtensionsIncludeSTRM(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(`{}`), 0o644); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
if !hasVideoExtension(cfg.Scanner.VideoExtensions, ".strm") {
|
||||
t.Fatalf("video extensions = %#v, want .strm", cfg.Scanner.VideoExtensions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadLegacyDefaultScannerVideoExtensionsIncludeSTRM(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(`
|
||||
scanner:
|
||||
video_extensions: [".mp4", ".mkv", ".mov", ".webm", ".avi"]
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
if !hasVideoExtension(cfg.Scanner.VideoExtensions, ".strm") {
|
||||
t.Fatalf("video extensions = %#v, want .strm appended for legacy default list", cfg.Scanner.VideoExtensions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCustomScannerVideoExtensionsArePreserved(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(`
|
||||
scanner:
|
||||
video_extensions: [".mp4"]
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
if len(cfg.Scanner.VideoExtensions) != 1 || cfg.Scanner.VideoExtensions[0] != ".mp4" {
|
||||
t.Fatalf("video extensions = %#v, want custom list preserved", cfg.Scanner.VideoExtensions)
|
||||
}
|
||||
}
|
||||
|
||||
func hasVideoExtension(exts []string, want string) bool {
|
||||
want = strings.ToLower(strings.TrimSpace(want))
|
||||
for _, ext := range exts {
|
||||
if strings.ToLower(strings.TrimSpace(ext)) == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
package googledrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -21,10 +28,13 @@ import (
|
||||
const (
|
||||
Kind = "googledrive"
|
||||
defaultAPIBaseURL = "https://www.googleapis.com/drive/v3"
|
||||
defaultUploadAPIURL = "https://www.googleapis.com/upload/drive/v3"
|
||||
defaultOAuthURL = "https://www.googleapis.com/oauth2/v4/token"
|
||||
defaultRenewAPIURL = "https://api.oplist.org/googleui/renewapi"
|
||||
defaultListInterval = 1 * time.Second
|
||||
defaultListCooldown = 5 * time.Minute
|
||||
defaultLinkCooldown = 5 * time.Minute
|
||||
uploadChunkSize = int64(8 * 1024 * 1024)
|
||||
|
||||
filesListFields = "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken"
|
||||
fileInfoFields = "id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum"
|
||||
@@ -41,13 +51,19 @@ type Driver struct {
|
||||
renewAPIURL string
|
||||
oauthURL string
|
||||
apiBaseURL string
|
||||
uploadBaseURL string
|
||||
client *resty.Client
|
||||
httpClient *http.Client
|
||||
onTokenUpdate func(access, refresh string)
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
listInterval time.Duration
|
||||
listCooldown time.Duration
|
||||
|
||||
linkCooldownMu sync.Mutex
|
||||
linkCooldownUntil time.Time
|
||||
linkCooldownDuration time.Duration
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -61,6 +77,7 @@ type Config struct {
|
||||
RenewAPIURL string
|
||||
OAuthURL string
|
||||
APIBaseURL string
|
||||
UploadAPIURL string
|
||||
|
||||
OnTokenUpdate func(access, refresh string)
|
||||
}
|
||||
@@ -82,6 +99,10 @@ func New(c Config) *Driver {
|
||||
if apiBaseURL == "" {
|
||||
apiBaseURL = defaultAPIBaseURL
|
||||
}
|
||||
uploadBaseURL := strings.TrimRight(strings.TrimSpace(c.UploadAPIURL), "/")
|
||||
if uploadBaseURL == "" {
|
||||
uploadBaseURL = deriveUploadBaseURL(apiBaseURL)
|
||||
}
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
rootID: rootID,
|
||||
@@ -93,15 +114,34 @@ func New(c Config) *Driver {
|
||||
renewAPIURL: renewAPIURL,
|
||||
oauthURL: oauthURL,
|
||||
apiBaseURL: apiBaseURL,
|
||||
uploadBaseURL: uploadBaseURL,
|
||||
onTokenUpdate: c.OnTokenUpdate,
|
||||
client: resty.New().
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
listInterval: defaultListInterval,
|
||||
listCooldown: defaultListCooldown,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 0,
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
},
|
||||
listInterval: defaultListInterval,
|
||||
listCooldown: defaultListCooldown,
|
||||
linkCooldownDuration: defaultLinkCooldown,
|
||||
}
|
||||
}
|
||||
|
||||
func deriveUploadBaseURL(apiBaseURL string) string {
|
||||
apiBaseURL = strings.TrimRight(strings.TrimSpace(apiBaseURL), "/")
|
||||
if apiBaseURL == "" || apiBaseURL == defaultAPIBaseURL {
|
||||
return defaultUploadAPIURL
|
||||
}
|
||||
if strings.HasSuffix(apiBaseURL, "/drive/v3") {
|
||||
return strings.TrimSuffix(apiBaseURL, "/drive/v3") + "/upload/drive/v3"
|
||||
}
|
||||
return apiBaseURL
|
||||
}
|
||||
|
||||
func (d *Driver) Kind() string { return Kind }
|
||||
func (d *Driver) ID() string { return d.id }
|
||||
func (d *Driver) RootID() string { return d.rootID }
|
||||
@@ -209,8 +249,19 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
if fileID == "" {
|
||||
return nil, errors.New("googledrive stream: empty file id")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := d.linkCooldownError(time.Now()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := d.Stat(ctx, fileID); err != nil {
|
||||
return nil, fmt.Errorf("googledrive stream: %w", err)
|
||||
err = fmt.Errorf("googledrive stream: %w", err)
|
||||
if wait, ok := drives.RateLimitRetryAfter(err); ok {
|
||||
until := d.pauseLinkCooldown(wait)
|
||||
log.Printf("[googledrive] stream link cooling down drive=%s until=%s err=%v", d.id, until.Format(time.RFC3339), err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
u := d.fileURL(fileID) + "?alt=media&acknowledgeAbuse=true&supportsAllDrives=true"
|
||||
return &drives.StreamLink{
|
||||
@@ -222,12 +273,396 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
func (d *Driver) linkCooldownError(now time.Time) error {
|
||||
d.linkCooldownMu.Lock()
|
||||
defer d.linkCooldownMu.Unlock()
|
||||
if d.linkCooldownUntil.IsZero() {
|
||||
return nil
|
||||
}
|
||||
if !now.Before(d.linkCooldownUntil) {
|
||||
d.linkCooldownUntil = time.Time{}
|
||||
return nil
|
||||
}
|
||||
wait := d.linkCooldownUntil.Sub(now)
|
||||
if wait <= 0 {
|
||||
return nil
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: wait,
|
||||
Err: fmt.Errorf("googledrive stream link cooling down until %s", d.linkCooldownUntil.Format(time.RFC3339)),
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
func (d *Driver) pauseLinkCooldown(wait time.Duration) time.Time {
|
||||
if wait <= 0 {
|
||||
wait = d.linkCooldownDuration
|
||||
}
|
||||
if wait <= 0 {
|
||||
wait = defaultLinkCooldown
|
||||
}
|
||||
until := time.Now().Add(wait)
|
||||
d.linkCooldownMu.Lock()
|
||||
if until.After(d.linkCooldownUntil) {
|
||||
d.linkCooldownUntil = until
|
||||
} else {
|
||||
until = d.linkCooldownUntil
|
||||
}
|
||||
d.linkCooldownMu.Unlock()
|
||||
return until
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
res, err := d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return res.FileID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
parentID, name, err := d.normalizeUploadArgs(parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
sessionURL, err := d.createUploadSession(ctx, parentID, name, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if strings.TrimSpace(sessionURL) == "" {
|
||||
return UploadResult{}, errors.New("googledrive upload session: empty upload url")
|
||||
}
|
||||
|
||||
hasher := md5.New()
|
||||
var item driveFile
|
||||
var copied int64
|
||||
if size == 0 {
|
||||
completed, err := d.putUploadSessionChunkWithRetry(ctx, sessionURL, 0, 0, nil, hasher)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if completed != nil {
|
||||
item = *completed
|
||||
}
|
||||
} else {
|
||||
chunkSize := uploadChunkSize
|
||||
if chunkSize <= 0 {
|
||||
chunkSize = 8 * 1024 * 1024
|
||||
}
|
||||
if chunkSize > int64(math.MaxInt32) {
|
||||
chunkSize = int64(math.MaxInt32)
|
||||
}
|
||||
buf := make([]byte, int(chunkSize))
|
||||
for copied < size {
|
||||
partSize := minInt64(chunkSize, size-copied)
|
||||
chunk := buf[:int(partSize)]
|
||||
n, err := io.ReadFull(r, chunk)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return UploadResult{}, fmt.Errorf("googledrive upload: size mismatch: declared %d, copied %d", size, copied+int64(n))
|
||||
}
|
||||
return UploadResult{}, fmt.Errorf("googledrive upload: read body: %w", err)
|
||||
}
|
||||
chunk = chunk[:n]
|
||||
completed, err := d.putUploadSessionChunkWithRetry(ctx, sessionURL, copied, size, chunk, hasher)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if completed != nil {
|
||||
item = *completed
|
||||
}
|
||||
copied += int64(n)
|
||||
}
|
||||
}
|
||||
|
||||
hashHex := hex.EncodeToString(hasher.Sum(nil))
|
||||
if item.ID == "" {
|
||||
fileID, err := d.findUploadedFileID(ctx, parentID, name, hashHex)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
item.ID = fileID
|
||||
}
|
||||
return UploadResult{FileID: item.ID, Hash: hashHex, Size: copied}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) normalizeUploadArgs(parentID, name string, r io.Reader, size int64) (string, string, error) {
|
||||
if r == nil {
|
||||
return "", "", errors.New("googledrive upload: body is required")
|
||||
}
|
||||
if size < 0 {
|
||||
return "", "", fmt.Errorf("googledrive upload: invalid size %d", size)
|
||||
}
|
||||
parentID = strings.TrimSpace(parentID)
|
||||
if parentID == "" || parentID == "/" {
|
||||
parentID = d.rootID
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return "", "", errors.New("googledrive upload: empty file name")
|
||||
}
|
||||
return parentID, name, nil
|
||||
}
|
||||
|
||||
func (d *Driver) createUploadSession(ctx context.Context, parentID, name string, size int64) (string, error) {
|
||||
return d.createUploadSessionOnce(ctx, parentID, name, size, true)
|
||||
}
|
||||
|
||||
func (d *Driver) createUploadSessionOnce(ctx context.Context, parentID, name string, size int64, retry bool) (string, error) {
|
||||
var apiErr apiErrorResp
|
||||
res, err := d.client.R().
|
||||
SetContext(ctx).
|
||||
SetHeader("Authorization", "Bearer "+d.accessToken).
|
||||
SetHeader("X-Upload-Content-Type", mimeType(driveFile{Name: name})).
|
||||
SetHeader("X-Upload-Content-Length", strconv.FormatInt(size, 10)).
|
||||
SetQueryParams(map[string]string{
|
||||
"uploadType": "resumable",
|
||||
"supportsAllDrives": "true",
|
||||
"fields": fileInfoFields,
|
||||
}).
|
||||
SetBody(map[string]any{
|
||||
"name": name,
|
||||
"parents": []string{parentID},
|
||||
}).
|
||||
SetError(&apiErr).
|
||||
Post(d.uploadFilesURL())
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("googledrive upload session: %w", err)
|
||||
}
|
||||
if isGoogleRateLimit(res, apiErr.Error) {
|
||||
return "", googleRateLimitError(res, apiErr.Error.Message)
|
||||
}
|
||||
if apiErr.Error.Code != 0 {
|
||||
if apiErr.Error.Code == http.StatusUnauthorized && retry {
|
||||
if err := d.refresh(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return d.createUploadSessionOnce(ctx, parentID, name, size, false)
|
||||
}
|
||||
return "", googleAPIError(apiErr.Error)
|
||||
}
|
||||
if res.IsError() {
|
||||
return "", fmt.Errorf("googledrive upload session: status=%d body=%s", res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return strings.TrimSpace(res.Header().Get("Location")), nil
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunkWithRetry(ctx context.Context, uploadURL string, start, total int64, data []byte, hasher hash.Hash) (*driveFile, error) {
|
||||
var last error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if attempt > 0 {
|
||||
if err := sleepContext(ctx, time.Duration(attempt)*time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
item, retryable, err := d.putUploadSessionChunk(ctx, uploadURL, start, total, data)
|
||||
if err == nil {
|
||||
if hasher != nil && len(data) > 0 {
|
||||
_, _ = hasher.Write(data)
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
last = err
|
||||
if !retryable {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if last == nil {
|
||||
last = errors.New("googledrive upload session: retry attempts exhausted")
|
||||
}
|
||||
return nil, last
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunk(ctx context.Context, uploadURL string, start, total int64, data []byte) (*driveFile, bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
req.ContentLength = int64(len(data))
|
||||
req.Header.Set("Authorization", "Bearer "+d.accessToken)
|
||||
req.Header.Set("Content-Length", strconv.Itoa(len(data)))
|
||||
if total == 0 {
|
||||
req.Header.Set("Content-Range", "bytes */0")
|
||||
} else {
|
||||
end := start + int64(len(data)) - 1
|
||||
req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, total))
|
||||
}
|
||||
hc := d.httpClient
|
||||
if hc == nil {
|
||||
hc = http.DefaultClient
|
||||
}
|
||||
res, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, true, fmt.Errorf("googledrive upload session: put chunk: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK, http.StatusCreated:
|
||||
var item driveFile
|
||||
if err := json.NewDecoder(res.Body).Decode(&item); err != nil {
|
||||
return nil, false, fmt.Errorf("googledrive upload session: decode completed file: %w", err)
|
||||
}
|
||||
return &item, false, nil
|
||||
case http.StatusPermanentRedirect:
|
||||
return nil, false, nil
|
||||
case http.StatusUnauthorized:
|
||||
if err := d.refresh(ctx); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return nil, true, fmt.Errorf("googledrive upload session: unauthorized")
|
||||
default:
|
||||
body, _ := io.ReadAll(io.LimitReader(res.Body, 64*1024))
|
||||
var apiErr apiErrorResp
|
||||
_ = json.Unmarshal(body, &apiErr)
|
||||
if isGoogleUploadHTTPRateLimit(res.StatusCode, res.Header, body, apiErr.Error) {
|
||||
return nil, false, googleUploadRateLimitError(res.StatusCode, res.Header, body, apiErr.Error.Message)
|
||||
}
|
||||
retryable := res.StatusCode == http.StatusTooManyRequests || (res.StatusCode >= 500 && res.StatusCode <= 504)
|
||||
return nil, retryable, fmt.Errorf("googledrive upload session: status=%d body=%s", res.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
currentID := d.rootID
|
||||
for _, name := range splitPath(pathFromRoot) {
|
||||
childID, err := d.findChildDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if childID == "" {
|
||||
childID, err = d.makeDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
currentID = childID
|
||||
}
|
||||
return currentID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) findChildDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
entries, err := d.List(ctx, parentID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir && e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
var item driveFile
|
||||
err := d.request(ctx, d.filesURL(), http.MethodPost, func(req *resty.Request) {
|
||||
req.SetQueryParam("fields", fileInfoFields)
|
||||
req.SetBody(map[string]any{
|
||||
"name": name,
|
||||
"parents": []string{parentID},
|
||||
"mimeType": "application/vnd.google-apps.folder",
|
||||
})
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("googledrive mkdir %s: %w", name, err)
|
||||
}
|
||||
if item.ID == "" {
|
||||
return "", fmt.Errorf("googledrive mkdir %s: empty file id", name)
|
||||
}
|
||||
return item.ID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("googledrive rename: empty file id")
|
||||
}
|
||||
newName = strings.TrimSpace(newName)
|
||||
if newName == "" {
|
||||
return errors.New("googledrive rename: empty new name")
|
||||
}
|
||||
var item driveFile
|
||||
err := d.request(ctx, d.fileURL(fileID), http.MethodPatch, func(req *resty.Request) {
|
||||
req.SetQueryParam("fields", fileInfoFields)
|
||||
req.SetBody(map[string]string{"name": newName})
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return fmt.Errorf("googledrive rename: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("googledrive remove: empty file id")
|
||||
}
|
||||
if err := d.request(ctx, d.fileURL(fileID), http.MethodDelete, nil, nil); err != nil {
|
||||
return fmt.Errorf("googledrive remove: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) findUploadedFileID(ctx context.Context, parentID, name, md5Hex string) (string, error) {
|
||||
entries, err := d.List(ctx, parentID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("googledrive upload verify: %w", err)
|
||||
}
|
||||
var hashHit string
|
||||
for _, e := range entries {
|
||||
if e.IsDir {
|
||||
continue
|
||||
}
|
||||
if !strings.EqualFold(e.Hash, md5Hex) {
|
||||
continue
|
||||
}
|
||||
if e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
if hashHit == "" {
|
||||
hashHit = e.ID
|
||||
}
|
||||
}
|
||||
if hashHit != "" {
|
||||
return hashHit, nil
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir && e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("googledrive upload: uploaded file %q not found in parent %q", name, parentID)
|
||||
}
|
||||
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
func isGoogleUploadHTTPRateLimit(status int, header http.Header, body []byte, apiErr apiErrorBody) bool {
|
||||
if status == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if status == http.StatusForbidden && strings.TrimSpace(header.Get("Retry-After")) != "" {
|
||||
return true
|
||||
}
|
||||
if isGoogleRateLimit(nil, apiErr) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func googleUploadRateLimitError(status int, header http.Header, body []byte, message string) error {
|
||||
if strings.TrimSpace(message) == "" {
|
||||
message = "google drive upload rate limited"
|
||||
}
|
||||
bodyText := strings.TrimSpace(string(body))
|
||||
if bodyText != "" {
|
||||
message = fmt.Sprintf("%s: status=%d body=%s", message, status, bodyText)
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: parseRetryAfterHeader(header.Get("Retry-After")),
|
||||
Err: errors.New(message),
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) refresh(ctx context.Context) error {
|
||||
@@ -288,6 +723,26 @@ func (d *Driver) applyToken(out tokenResp) {
|
||||
}
|
||||
|
||||
func tokenResponseError(prefix string, res *resty.Response, out tokenResp, requireRefresh bool) error {
|
||||
if isGoogleTokenRateLimit(res, out) {
|
||||
message := strings.TrimSpace(out.Text)
|
||||
if message == "" {
|
||||
message = strings.TrimSpace(out.ErrorDescription)
|
||||
}
|
||||
if message == "" {
|
||||
message = strings.TrimSpace(out.Error)
|
||||
}
|
||||
if message == "" {
|
||||
message = "google drive token refresh rate limited"
|
||||
}
|
||||
if res != nil && strings.TrimSpace(res.String()) != "" {
|
||||
message = fmt.Sprintf("%s: status=%d body=%s", message, res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: parseRetryAfter(res),
|
||||
Err: fmt.Errorf("%s: %s", prefix, message),
|
||||
}
|
||||
}
|
||||
if out.Text != "" {
|
||||
return fmt.Errorf("%s: %s", prefix, out.Text)
|
||||
}
|
||||
@@ -380,6 +835,10 @@ func (d *Driver) filesURL() string {
|
||||
return d.apiBaseURL + "/files"
|
||||
}
|
||||
|
||||
func (d *Driver) uploadFilesURL() string {
|
||||
return d.uploadBaseURL + "/files"
|
||||
}
|
||||
|
||||
func (d *Driver) fileURL(fileID string) string {
|
||||
return d.filesURL() + "/" + url.PathEscape(fileID)
|
||||
}
|
||||
@@ -444,18 +903,58 @@ func isGoogleRateLimit(res *resty.Response, body apiErrorBody) bool {
|
||||
if res != nil && res.StatusCode() == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if res != nil && res.StatusCode() == http.StatusForbidden && strings.TrimSpace(res.Header().Get("Retry-After")) != "" {
|
||||
return true
|
||||
}
|
||||
if body.Code == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
for _, e := range body.Errors {
|
||||
reason := strings.ToLower(strings.TrimSpace(e.Reason))
|
||||
switch reason {
|
||||
case "ratelimitexceeded", "userratelimitexceeded", "downloadquotaexceeded", "sharingratelimitexceeded":
|
||||
if googleLimitReason(e.Reason) {
|
||||
return true
|
||||
}
|
||||
domain := compactGoogleLimitText(e.Domain)
|
||||
if domain == "usagelimits" && (body.Code == http.StatusForbidden || body.Code == http.StatusTooManyRequests) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
msg := strings.ToLower(body.Message)
|
||||
return strings.Contains(msg, "rate limit") || strings.Contains(msg, "too many requests") || strings.Contains(msg, "quota exceeded")
|
||||
return false
|
||||
}
|
||||
|
||||
func isGoogleTokenRateLimit(res *resty.Response, out tokenResp) bool {
|
||||
if res != nil {
|
||||
if res.StatusCode() == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if res.StatusCode() == http.StatusForbidden && strings.TrimSpace(res.Header().Get("Retry-After")) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return googleLimitReason(out.Error)
|
||||
}
|
||||
|
||||
func googleLimitReason(reason string) bool {
|
||||
switch compactGoogleLimitText(reason) {
|
||||
case "ratelimitexceeded",
|
||||
"userratelimitexceeded",
|
||||
"dailylimitexceeded",
|
||||
"dailylimitexceededunreg",
|
||||
"downloadquotaexceeded",
|
||||
"sharingratelimitexceeded",
|
||||
"quotaexceeded",
|
||||
"uploadlimitexceeded",
|
||||
"storagelimitexceeded",
|
||||
"storagequotaexceeded":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func compactGoogleLimitText(text string) string {
|
||||
text = strings.ToLower(strings.TrimSpace(text))
|
||||
replacer := strings.NewReplacer("_", "", "-", "", " ", "", ".", "", ":", "")
|
||||
return replacer.Replace(text)
|
||||
}
|
||||
|
||||
func googleRateLimitError(res *resty.Response, message string) error {
|
||||
@@ -486,7 +985,11 @@ func parseRetryAfter(res *resty.Response) time.Duration {
|
||||
if res == nil {
|
||||
return 0
|
||||
}
|
||||
raw := strings.TrimSpace(res.Header().Get("Retry-After"))
|
||||
return parseRetryAfterHeader(res.Header().Get("Retry-After"))
|
||||
}
|
||||
|
||||
func parseRetryAfterHeader(raw string) time.Duration {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return 0
|
||||
}
|
||||
@@ -502,4 +1005,19 @@ func parseRetryAfter(res *resty.Response) time.Duration {
|
||||
return 0
|
||||
}
|
||||
|
||||
func splitPath(p string) []string {
|
||||
p = strings.Trim(p, "/")
|
||||
if p == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(p, "/")
|
||||
}
|
||||
|
||||
func minInt64(a, b int64) int64 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
|
||||
@@ -2,11 +2,18 @@ package googledrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestInitUsesOnlineRenewAPI(t *testing.T) {
|
||||
@@ -131,6 +138,134 @@ func TestStreamURLReturnsAuthenticatedMediaLinkWithoutRedirectRequirement(t *tes
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAndReportHashUsesResumableSession(t *testing.T) {
|
||||
body := "hello google drive"
|
||||
wantHash := md5.Sum([]byte(body))
|
||||
var sawSession bool
|
||||
var sawUpload bool
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/upload/drive/v3/files":
|
||||
sawSession = true
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("session Authorization = %q", got)
|
||||
}
|
||||
if got := r.URL.Query().Get("uploadType"); got != "resumable" {
|
||||
t.Fatalf("uploadType = %q", got)
|
||||
}
|
||||
if got := r.Header.Get("X-Upload-Content-Length"); got != "18" {
|
||||
t.Fatalf("X-Upload-Content-Length = %q", got)
|
||||
}
|
||||
var meta struct {
|
||||
Name string `json:"name"`
|
||||
Parents []string `json:"parents"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&meta); err != nil {
|
||||
t.Fatalf("decode session metadata: %v", err)
|
||||
}
|
||||
if meta.Name != "clip.mp4" || len(meta.Parents) != 1 || meta.Parents[0] != "parent-1" {
|
||||
t.Fatalf("metadata = %+v", meta)
|
||||
}
|
||||
w.Header().Set("Location", srv.URL+"/upload/session/1")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case "/upload/session/1":
|
||||
sawUpload = true
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("upload Authorization = %q", got)
|
||||
}
|
||||
if got := r.Header.Get("Content-Range"); got != "bytes 0-17/18" {
|
||||
t.Fatalf("Content-Range = %q", got)
|
||||
}
|
||||
gotBody, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read upload body: %v", err)
|
||||
}
|
||||
if string(gotBody) != body {
|
||||
t.Fatalf("upload body = %q", string(gotBody))
|
||||
}
|
||||
writeTestJSONStatus(w, http.StatusCreated, driveFile{
|
||||
ID: "file-uploaded",
|
||||
Name: "clip.mp4",
|
||||
Size: "18",
|
||||
MD5Checksum: hex.EncodeToString(wantHash[:]),
|
||||
})
|
||||
default:
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", APIBaseURL: srv.URL + "/drive/v3"})
|
||||
d.accessToken = "access"
|
||||
res, err := d.UploadAndReportHash(context.Background(), "parent-1", "clip.mp4", strings.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("UploadAndReportHash() error = %v", err)
|
||||
}
|
||||
if !sawSession || !sawUpload {
|
||||
t.Fatalf("saw session/upload = %v/%v, want both", sawSession, sawUpload)
|
||||
}
|
||||
if res.FileID != "file-uploaded" || res.Size != int64(len(body)) || res.Hash != hex.EncodeToString(wantHash[:]) {
|
||||
t.Fatalf("upload result = %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDirAndRenameUseGoogleDriveFileAPI(t *testing.T) {
|
||||
var madeDir bool
|
||||
var renamed bool
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/drive/v3/files":
|
||||
writeTestJSON(w, filesResp{})
|
||||
case r.Method == http.MethodPost && r.URL.Path == "/drive/v3/files":
|
||||
madeDir = true
|
||||
var meta struct {
|
||||
Name string `json:"name"`
|
||||
Parents []string `json:"parents"`
|
||||
MimeType string `json:"mimeType"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&meta); err != nil {
|
||||
t.Fatalf("decode mkdir body: %v", err)
|
||||
}
|
||||
if meta.Name != "91 Spider" || len(meta.Parents) != 1 || meta.Parents[0] != "root" || meta.MimeType != "application/vnd.google-apps.folder" {
|
||||
t.Fatalf("mkdir body = %+v", meta)
|
||||
}
|
||||
writeTestJSON(w, driveFile{ID: "folder-91", Name: "91 Spider", MimeType: "application/vnd.google-apps.folder"})
|
||||
case r.Method == http.MethodPatch && r.URL.Path == "/drive/v3/files/file-1":
|
||||
renamed = true
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode rename body: %v", err)
|
||||
}
|
||||
if body["name"] != "new-name.mp4" {
|
||||
t.Fatalf("rename body = %+v", body)
|
||||
}
|
||||
writeTestJSON(w, driveFile{ID: "file-1", Name: "new-name.mp4"})
|
||||
default:
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", RootID: "root", APIBaseURL: srv.URL + "/drive/v3"})
|
||||
d.accessToken = "access"
|
||||
d.listInterval = -1
|
||||
|
||||
dirID, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureDir() error = %v", err)
|
||||
}
|
||||
if dirID != "folder-91" || !madeDir {
|
||||
t.Fatalf("dirID/madeDir = %q/%v, want folder-91/true", dirID, madeDir)
|
||||
}
|
||||
if err := d.Rename(context.Background(), "file-1", "new-name.mp4"); err != nil {
|
||||
t.Fatalf("Rename() error = %v", err)
|
||||
}
|
||||
if !renamed {
|
||||
t.Fatal("rename endpoint was not called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestRefreshesOnUnauthorized(t *testing.T) {
|
||||
var fileCalls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -179,6 +314,88 @@ func TestRequestRefreshesOnUnauthorized(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateLimitReasonsFollowGoogleDriveErrorShape(t *testing.T) {
|
||||
reasons := []string{
|
||||
"rateLimitExceeded",
|
||||
"userRateLimitExceeded",
|
||||
"dailyLimitExceeded",
|
||||
"dailyLimitExceededUnreg",
|
||||
"downloadQuotaExceeded",
|
||||
"sharingRateLimitExceeded",
|
||||
"quotaExceeded",
|
||||
}
|
||||
for _, reason := range reasons {
|
||||
body := apiErrorBody{
|
||||
Code: http.StatusForbidden,
|
||||
Message: "google drive quota or rate limited",
|
||||
Errors: []struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
LocationType string `json:"location_type"`
|
||||
Location string `json:"location"`
|
||||
}{
|
||||
{Domain: "usageLimits", Reason: reason, Message: reason},
|
||||
},
|
||||
}
|
||||
if !isGoogleRateLimit(nil, body) {
|
||||
t.Fatalf("reason %q not treated as rate limit", reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRateLimitStartsSharedLinkCooldown(t *testing.T) {
|
||||
var calls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
w.Header().Set("Retry-After", "120")
|
||||
writeTestJSONStatus(w, http.StatusForbidden, apiErrorResp{Error: apiErrorBody{
|
||||
Code: http.StatusForbidden,
|
||||
Message: "User rate limit exceeded.",
|
||||
Errors: []struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
LocationType string `json:"location_type"`
|
||||
Location string `json:"location"`
|
||||
}{
|
||||
{Domain: "usageLimits", Reason: "userRateLimitExceeded", Message: "User rate limit exceeded."},
|
||||
},
|
||||
}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", APIBaseURL: srv.URL})
|
||||
d.accessToken = "access"
|
||||
d.linkCooldownDuration = time.Hour
|
||||
|
||||
_, err := d.StreamURL(context.Background(), "file-1")
|
||||
if err == nil {
|
||||
t.Fatal("first StreamURL succeeded, want rate limit")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("first error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != 2*time.Minute {
|
||||
t.Fatalf("retry after = %s, want 2m", rateLimit.RetryAfter)
|
||||
}
|
||||
|
||||
_, err = d.StreamURL(context.Background(), "file-1")
|
||||
if err == nil {
|
||||
t.Fatal("second StreamURL succeeded during cooldown")
|
||||
}
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("second error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("remote calls = %d, want 1; second call should use shared cooldown", calls)
|
||||
}
|
||||
if rateLimit.RetryAfter <= 0 || rateLimit.RetryAfter > 2*time.Minute {
|
||||
t.Fatalf("second retry after = %s, want remaining cooldown", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestJSON(w http.ResponseWriter, v any) {
|
||||
writeTestJSONStatus(w, http.StatusOK, v)
|
||||
}
|
||||
|
||||
@@ -42,8 +42,16 @@ type apiErrorBody struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Errors []struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
LocationType string `json:"location_type"`
|
||||
Location string `json:"location"`
|
||||
} `json:"errors"`
|
||||
}
|
||||
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
Hash string
|
||||
Size int64
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,300 @@
|
||||
package guangyapan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestDriverRefreshListAndStream(t *testing.T) {
|
||||
var refreshed bool
|
||||
var listedRoot bool
|
||||
updates := map[string]string{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/v1/auth/token":
|
||||
refreshed = true
|
||||
writeTestJSON(w, map[string]any{
|
||||
"access_token": "new-access",
|
||||
"refresh_token": "new-refresh",
|
||||
})
|
||||
case "/v1/user/me":
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer new-access" {
|
||||
t.Fatalf("auth header = %q, want new access token", got)
|
||||
}
|
||||
writeTestJSON(w, map[string]any{"sub": "user-1"})
|
||||
case "/userres/v1/file/get_file_list":
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer new-access" {
|
||||
t.Fatalf("api auth header = %q, want new access token", got)
|
||||
}
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode list body: %v", err)
|
||||
}
|
||||
if body["parentId"] != "" {
|
||||
t.Fatalf("parentId = %#v, want root empty string", body["parentId"])
|
||||
}
|
||||
listedRoot = true
|
||||
writeTestJSON(w, map[string]any{
|
||||
"code": 0,
|
||||
"msg": "success",
|
||||
"data": map[string]any{
|
||||
"total": 2,
|
||||
"list": []map[string]any{
|
||||
{"fileId": "dir-1", "parentId": "", "fileName": "Movies", "resType": 2},
|
||||
{"fileId": "file-1", "parentId": "", "fileName": "clip.mp4", "fileSize": 123, "resType": 1, "utime": 1700000000},
|
||||
},
|
||||
},
|
||||
})
|
||||
case "/nd.bizuserres.s/v1/get_res_download_url":
|
||||
writeTestJSON(w, map[string]any{
|
||||
"code": 0,
|
||||
"msg": "success",
|
||||
"data": map[string]any{"signedURL": "https://cdn.example.test/clip.mp4"},
|
||||
})
|
||||
default:
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "gy",
|
||||
RefreshToken: "old-refresh",
|
||||
AccountBaseURL: srv.URL,
|
||||
APIBaseURL: srv.URL,
|
||||
OnCredentialsUpdate: func(values map[string]string) {
|
||||
for k, v := range values {
|
||||
updates[k] = v
|
||||
}
|
||||
},
|
||||
})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
if !refreshed {
|
||||
t.Fatal("refresh token endpoint was not called")
|
||||
}
|
||||
if updates["access_token"] != "new-access" || updates["refresh_token"] != "new-refresh" {
|
||||
t.Fatalf("updates = %#v, want refreshed tokens", updates)
|
||||
}
|
||||
|
||||
entries, err := d.List(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if !listedRoot || len(entries) != 2 {
|
||||
t.Fatalf("listedRoot=%v entries=%#v", listedRoot, entries)
|
||||
}
|
||||
if !entries[0].IsDir || entries[1].ID != "file-1" || entries[1].Size != 123 {
|
||||
t.Fatalf("entries = %#v", entries)
|
||||
}
|
||||
|
||||
link, err := d.StreamURL(context.Background(), "file-1")
|
||||
if err != nil {
|
||||
t.Fatalf("stream url: %v", err)
|
||||
}
|
||||
if link.URL != "https://cdn.example.test/clip.mp4" {
|
||||
t.Fatalf("stream url = %q", link.URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriverResolvesRootPath(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/v1/user/me":
|
||||
writeTestJSON(w, map[string]any{"sub": "user-1"})
|
||||
case "/userres/v1/file/get_file_list":
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode list body: %v", err)
|
||||
}
|
||||
parent, _ := body["parentId"].(string)
|
||||
switch parent {
|
||||
case "":
|
||||
writeTestJSON(w, listTestResponse([]map[string]any{
|
||||
{"fileId": "folder-a", "parentId": "", "fileName": "影视", "resType": 2},
|
||||
}))
|
||||
case "folder-a":
|
||||
writeTestJSON(w, listTestResponse([]map[string]any{
|
||||
{"fileId": "folder-b", "parentId": "folder-a", "fileName": "电影", "resType": 2},
|
||||
}))
|
||||
case "folder-b":
|
||||
writeTestJSON(w, listTestResponse([]map[string]any{
|
||||
{"fileId": "file-1", "parentId": "folder-b", "fileName": "movie.mp4", "fileSize": 456, "resType": 1},
|
||||
}))
|
||||
default:
|
||||
t.Fatalf("unexpected parent %q", parent)
|
||||
}
|
||||
default:
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "gy",
|
||||
RootID: "configured-root",
|
||||
RootPath: "影视/电影",
|
||||
AccessToken: "access",
|
||||
AccountBaseURL: srv.URL,
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
if d.RootID() != "folder-b" {
|
||||
t.Fatalf("root id = %q, want folder-b", d.RootID())
|
||||
}
|
||||
entries, err := d.List(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("list resolved root: %v", err)
|
||||
}
|
||||
if len(entries) != 1 || entries[0].ID != "file-1" {
|
||||
t.Fatalf("entries = %#v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriverSendSMSCodeUpdatesVerificationState(t *testing.T) {
|
||||
updates := map[string]string{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/v1/shield/captcha/init":
|
||||
writeTestJSON(w, map[string]any{"captcha_token": "captcha-1"})
|
||||
case "/v1/auth/verification":
|
||||
writeTestJSON(w, map[string]any{"verification_id": "verify-1"})
|
||||
default:
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "gy",
|
||||
PhoneNumber: "13800000000",
|
||||
SendCode: true,
|
||||
AccountBaseURL: srv.URL,
|
||||
APIBaseURL: srv.URL,
|
||||
OnCredentialsUpdate: func(values map[string]string) {
|
||||
for k, v := range values {
|
||||
updates[k] = v
|
||||
}
|
||||
},
|
||||
})
|
||||
err := d.Init(context.Background())
|
||||
if err == nil || !strings.Contains(err.Error(), "验证码已发送") {
|
||||
t.Fatalf("init err = %v, want verification prompt", err)
|
||||
}
|
||||
if updates["captcha_token"] != "captcha-1" || updates["verification_id"] != "verify-1" || updates["send_code"] != "false" {
|
||||
t.Fatalf("updates = %#v, want sms state saved", updates)
|
||||
}
|
||||
if updates["device_id"] == "" {
|
||||
t.Fatalf("updates = %#v, want generated device id saved", updates)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListHTTP429ReturnsRateLimitError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/userres/v1/file/get_file_list" {
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
w.Header().Set("Retry-After", "120")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
writeTestJSON(w, map[string]any{"code": 429, "msg": "操作频繁,请稍后重试"})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "gy",
|
||||
AccessToken: "access",
|
||||
AccountBaseURL: srv.URL,
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
_, err := d.List(context.Background(), "")
|
||||
if err == nil {
|
||||
t.Fatal("list succeeded, want rate limit error")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != 2*time.Minute {
|
||||
t.Fatalf("retry after = %s, want 2m", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListCode429ReturnsRateLimitError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/userres/v1/file/get_file_list" {
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
writeTestJSON(w, map[string]any{"code": 429, "msg": "操作频繁,请稍后再试"})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "gy",
|
||||
AccessToken: "access",
|
||||
AccountBaseURL: srv.URL,
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
_, err := d.List(context.Background(), "")
|
||||
if err == nil {
|
||||
t.Fatal("list succeeded, want rate limit error")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListInvalidToken403DoesNotReturnRateLimitError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/userres/v1/file/get_file_list" {
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
writeTestJSON(w, map[string]any{"code": 401, "msg": "invalid access token"})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "gy",
|
||||
AccessToken: "access",
|
||||
AccountBaseURL: srv.URL,
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
_, err := d.List(context.Background(), "")
|
||||
if err == nil {
|
||||
t.Fatal("list succeeded, want auth error")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want non-rate-limit error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func listTestResponse(items []map[string]any) map[string]any {
|
||||
return map[string]any{
|
||||
"code": 0,
|
||||
"msg": "success",
|
||||
"data": map[string]any{
|
||||
"total": len(items),
|
||||
"list": items,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestJSON(w http.ResponseWriter, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(v); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
package guangyapan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-resty/resty/v2"
|
||||
"github.com/skip2/go-qrcode"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultQRScope = "user"
|
||||
deviceCodeGrantType = "urn:ietf:params:oauth:grant-type:device_code"
|
||||
defaultQRUserAgent = "GuangYaPan-Login/1.0"
|
||||
)
|
||||
|
||||
type QRConfig struct {
|
||||
AccountBaseURL string
|
||||
HTTPClient *http.Client
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type QRClient struct {
|
||||
accountBaseURL string
|
||||
client *resty.Client
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type QRCodeSession struct {
|
||||
DeviceCode string `json:"deviceCode"`
|
||||
QRCodeURL string `json:"qrCodeUrl"`
|
||||
QRImageDataURL string `json:"qrImageDataUrl"`
|
||||
IntervalSeconds int `json:"intervalSeconds"`
|
||||
ExpiresAt string `json:"expiresAt,omitempty"`
|
||||
}
|
||||
|
||||
type QRCodeStatus struct {
|
||||
State string `json:"state"`
|
||||
StatusText string `json:"statusText"`
|
||||
IntervalSeconds int `json:"intervalSeconds,omitempty"`
|
||||
AccessToken string `json:"accessToken,omitempty"`
|
||||
RefreshToken string `json:"refreshToken,omitempty"`
|
||||
TokenType string `json:"tokenType,omitempty"`
|
||||
ExpiresIn int64 `json:"expiresIn,omitempty"`
|
||||
}
|
||||
|
||||
type deviceCodeResp struct {
|
||||
DeviceCode string `json:"device_code"`
|
||||
VerificationURIComplete string `json:"verification_uri_complete"`
|
||||
ShortURIComplete string `json:"short_uri_complete"`
|
||||
Interval int `json:"interval"`
|
||||
ExpiresIn int `json:"expires_in"`
|
||||
Error string `json:"error"`
|
||||
ErrorCode int `json:"error_code"`
|
||||
ErrorDesc string `json:"error_description"`
|
||||
}
|
||||
|
||||
type deviceTokenResp struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
RefreshToken string `json:"refresh_token"`
|
||||
TokenType string `json:"token_type"`
|
||||
ExpiresIn int64 `json:"expires_in"`
|
||||
Scope string `json:"scope"`
|
||||
Error string `json:"error"`
|
||||
ErrorCode int `json:"error_code"`
|
||||
ErrorDesc string `json:"error_description"`
|
||||
}
|
||||
|
||||
func NewQRClient(c QRConfig) *QRClient {
|
||||
accountBaseURL := strings.TrimRight(strings.TrimSpace(c.AccountBaseURL), "/")
|
||||
if accountBaseURL == "" {
|
||||
accountBaseURL = defaultAccountBaseURL
|
||||
}
|
||||
httpClient := c.HTTPClient
|
||||
if httpClient == nil {
|
||||
httpClient = &http.Client{Timeout: 20 * time.Second}
|
||||
}
|
||||
now := c.Now
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
return &QRClient{
|
||||
accountBaseURL: accountBaseURL,
|
||||
client: resty.NewWithClient(httpClient).
|
||||
SetTimeout(20*time.Second).
|
||||
SetBaseURL(accountBaseURL).
|
||||
SetHeader("User-Agent", defaultQRUserAgent).
|
||||
SetHeader("Accept", "application/json").
|
||||
SetHeader("Content-Type", "application/json"),
|
||||
now: now,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *QRClient) Generate(ctx context.Context) (QRCodeSession, error) {
|
||||
var out deviceCodeResp
|
||||
var errOut deviceCodeResp
|
||||
resp, err := c.client.R().
|
||||
SetContext(ctx).
|
||||
SetBody(map[string]any{
|
||||
"client_id": defaultClientID,
|
||||
"scope": defaultQRScope,
|
||||
}).
|
||||
SetResult(&out).
|
||||
SetError(&errOut).
|
||||
Post("/v1/auth/device/code")
|
||||
if err != nil {
|
||||
return QRCodeSession{}, err
|
||||
}
|
||||
if resp.IsError() || out.Error != "" {
|
||||
if out.Error == "" {
|
||||
out = errOut
|
||||
}
|
||||
return QRCodeSession{}, fmt.Errorf("guangyapan qr: %s", deviceAPIError(out.ErrorDesc, out.Error, resp))
|
||||
}
|
||||
|
||||
deviceCode := strings.TrimSpace(out.DeviceCode)
|
||||
if deviceCode == "" {
|
||||
return QRCodeSession{}, errors.New("guangyapan qr: empty device_code")
|
||||
}
|
||||
qrURL := strings.TrimSpace(out.VerificationURIComplete)
|
||||
if qrURL == "" {
|
||||
qrURL = strings.TrimSpace(out.ShortURIComplete)
|
||||
}
|
||||
if qrURL == "" {
|
||||
return QRCodeSession{}, errors.New("guangyapan qr: empty verification uri")
|
||||
}
|
||||
interval := out.Interval
|
||||
if interval <= 0 {
|
||||
interval = 5
|
||||
}
|
||||
expiresIn := out.ExpiresIn
|
||||
if expiresIn <= 0 {
|
||||
expiresIn = 300
|
||||
}
|
||||
png, err := qrcode.Encode(qrURL, qrcode.Medium, 220)
|
||||
if err != nil {
|
||||
return QRCodeSession{}, err
|
||||
}
|
||||
return QRCodeSession{
|
||||
DeviceCode: deviceCode,
|
||||
QRCodeURL: qrURL,
|
||||
QRImageDataURL: "data:image/png;base64," + base64.StdEncoding.EncodeToString(png),
|
||||
IntervalSeconds: interval,
|
||||
ExpiresAt: c.now().Add(time.Duration(expiresIn) * time.Second).Format(time.RFC3339),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *QRClient) Poll(ctx context.Context, deviceCode string) (QRCodeStatus, error) {
|
||||
deviceCode = strings.TrimSpace(deviceCode)
|
||||
if deviceCode == "" {
|
||||
return QRCodeStatus{}, errors.New("deviceCode is required")
|
||||
}
|
||||
|
||||
var out deviceTokenResp
|
||||
var errOut deviceTokenResp
|
||||
resp, err := c.client.R().
|
||||
SetContext(ctx).
|
||||
SetBody(map[string]any{
|
||||
"client_id": defaultClientID,
|
||||
"grant_type": deviceCodeGrantType,
|
||||
"device_code": deviceCode,
|
||||
}).
|
||||
SetResult(&out).
|
||||
SetError(&errOut).
|
||||
Post("/v1/auth/token")
|
||||
if err != nil {
|
||||
return QRCodeStatus{}, err
|
||||
}
|
||||
if resp.IsError() && out.Error == "" {
|
||||
out = errOut
|
||||
}
|
||||
if resp.IsError() && out.Error == "" {
|
||||
_ = json.Unmarshal(resp.Body(), &out)
|
||||
}
|
||||
if out.Error != "" {
|
||||
return qrStatusForDeviceError(out), nil
|
||||
}
|
||||
if resp.IsError() {
|
||||
return QRCodeStatus{}, fmt.Errorf("guangyapan qr: status=%d body=%s", resp.StatusCode(), resp.String())
|
||||
}
|
||||
access := strings.TrimSpace(out.AccessToken)
|
||||
refresh := strings.TrimSpace(out.RefreshToken)
|
||||
if access == "" || refresh == "" {
|
||||
return QRCodeStatus{}, errors.New("guangyapan qr: login succeeded but token response is incomplete")
|
||||
}
|
||||
tokenType := strings.TrimSpace(out.TokenType)
|
||||
if tokenType == "" {
|
||||
tokenType = "Bearer"
|
||||
}
|
||||
return QRCodeStatus{
|
||||
State: "success",
|
||||
StatusText: "登录成功",
|
||||
AccessToken: access,
|
||||
RefreshToken: refresh,
|
||||
TokenType: tokenType,
|
||||
ExpiresIn: out.ExpiresIn,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func qrStatusForDeviceError(out deviceTokenResp) QRCodeStatus {
|
||||
errCode := strings.TrimSpace(out.Error)
|
||||
switch errCode {
|
||||
case "authorization_pending":
|
||||
return QRCodeStatus{State: "pending", StatusText: "等待扫码确认"}
|
||||
case "slow_down":
|
||||
return QRCodeStatus{State: "pending", StatusText: "等待扫码确认,已降低查询频率", IntervalSeconds: 10}
|
||||
case "expired_token":
|
||||
return QRCodeStatus{State: "expired", StatusText: "二维码已过期"}
|
||||
case "access_denied":
|
||||
return QRCodeStatus{State: "denied", StatusText: "用户拒绝了授权"}
|
||||
default:
|
||||
msg := strings.TrimSpace(out.ErrorDesc)
|
||||
if msg == "" {
|
||||
msg = errCode
|
||||
}
|
||||
if msg == "" {
|
||||
msg = "未知错误"
|
||||
}
|
||||
return QRCodeStatus{State: "error", StatusText: msg}
|
||||
}
|
||||
}
|
||||
|
||||
func deviceAPIError(desc, short string, resp *resty.Response) string {
|
||||
msg := strings.TrimSpace(desc)
|
||||
if msg == "" {
|
||||
msg = strings.TrimSpace(short)
|
||||
}
|
||||
if msg == "" && resp != nil {
|
||||
msg = strings.TrimSpace(resp.String())
|
||||
}
|
||||
if msg == "" && resp != nil {
|
||||
msg = fmt.Sprintf("status=%d", resp.StatusCode())
|
||||
}
|
||||
if msg == "" {
|
||||
msg = "unknown error"
|
||||
}
|
||||
return msg
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package guangyapan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestQRClientGenerate(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1/auth/device/code" {
|
||||
t.Fatalf("path = %s, want device code endpoint", r.URL.Path)
|
||||
}
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode body: %v", err)
|
||||
}
|
||||
if body["client_id"] != defaultClientID || body["scope"] != defaultQRScope {
|
||||
t.Fatalf("body = %#v", body)
|
||||
}
|
||||
writeTestJSON(w, map[string]any{
|
||||
"device_code": "device-1",
|
||||
"verification_uri_complete": "https://account.guangyapan.com/device?code=abc",
|
||||
"interval": 7,
|
||||
"expires_in": 180,
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client := NewQRClient(QRConfig{
|
||||
AccountBaseURL: srv.URL,
|
||||
Now: func() time.Time { return time.Unix(1700000000, 0) },
|
||||
})
|
||||
session, err := client.Generate(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("generate: %v", err)
|
||||
}
|
||||
if session.DeviceCode != "device-1" || session.QRCodeURL != "https://account.guangyapan.com/device?code=abc" {
|
||||
t.Fatalf("session = %#v", session)
|
||||
}
|
||||
if session.IntervalSeconds != 7 {
|
||||
t.Fatalf("interval = %d, want 7", session.IntervalSeconds)
|
||||
}
|
||||
if session.ExpiresAt != time.Unix(1700000180, 0).Format(time.RFC3339) {
|
||||
t.Fatalf("expiresAt = %q", session.ExpiresAt)
|
||||
}
|
||||
if !strings.HasPrefix(session.QRImageDataURL, "data:image/png;base64,") {
|
||||
t.Fatalf("qr image = %q", session.QRImageDataURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQRClientPollPendingAndSuccess(t *testing.T) {
|
||||
var calls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1/auth/token" {
|
||||
t.Fatalf("path = %s, want token endpoint", r.URL.Path)
|
||||
}
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode body: %v", err)
|
||||
}
|
||||
if body["client_id"] != defaultClientID ||
|
||||
body["grant_type"] != deviceCodeGrantType ||
|
||||
body["device_code"] != "device-1" {
|
||||
t.Fatalf("body = %#v", body)
|
||||
}
|
||||
calls++
|
||||
if calls == 1 {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
writeTestJSON(w, map[string]any{"error": "authorization_pending"})
|
||||
return
|
||||
}
|
||||
writeTestJSON(w, map[string]any{
|
||||
"access_token": "access-1",
|
||||
"refresh_token": "refresh-1",
|
||||
"token_type": "Bearer",
|
||||
"expires_in": 7200,
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client := NewQRClient(QRConfig{AccountBaseURL: srv.URL})
|
||||
pending, err := client.Poll(context.Background(), "device-1")
|
||||
if err != nil {
|
||||
t.Fatalf("poll pending: %v", err)
|
||||
}
|
||||
if pending.State != "pending" || pending.AccessToken != "" {
|
||||
t.Fatalf("pending = %#v", pending)
|
||||
}
|
||||
|
||||
success, err := client.Poll(context.Background(), "device-1")
|
||||
if err != nil {
|
||||
t.Fatalf("poll success: %v", err)
|
||||
}
|
||||
if success.State != "success" || success.AccessToken != "access-1" || success.RefreshToken != "refresh-1" {
|
||||
t.Fatalf("success = %#v", success)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package guangyapan
|
||||
|
||||
import "time"
|
||||
|
||||
type tokenResp struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
RefreshToken string `json:"refresh_token"`
|
||||
Error string `json:"error"`
|
||||
ErrorCode int `json:"error_code"`
|
||||
ErrorDesc string `json:"error_description"`
|
||||
}
|
||||
|
||||
type verificationResp struct {
|
||||
VerificationID string `json:"verification_id"`
|
||||
Error string `json:"error"`
|
||||
ErrorCode int `json:"error_code"`
|
||||
ErrorDesc string `json:"error_description"`
|
||||
}
|
||||
|
||||
type captchaInitResp struct {
|
||||
CaptchaToken string `json:"captcha_token"`
|
||||
Error string `json:"error"`
|
||||
ErrorCode int `json:"error_code"`
|
||||
ErrorDesc string `json:"error_description"`
|
||||
}
|
||||
|
||||
type verifyResp struct {
|
||||
VerificationToken string `json:"verification_token"`
|
||||
Error string `json:"error"`
|
||||
ErrorCode int `json:"error_code"`
|
||||
ErrorDesc string `json:"error_description"`
|
||||
}
|
||||
|
||||
type userMeResp struct {
|
||||
Sub string `json:"sub"`
|
||||
}
|
||||
|
||||
type listResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
Total int `json:"total"`
|
||||
List []fileItem `json:"list"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type fileItem struct {
|
||||
FileID string `json:"fileId"`
|
||||
ParentID string `json:"parentId"`
|
||||
FileName string `json:"fileName"`
|
||||
FileSize int64 `json:"fileSize"`
|
||||
ResType int `json:"resType"`
|
||||
CTime int64 `json:"ctime"`
|
||||
UTime int64 `json:"utime"`
|
||||
}
|
||||
|
||||
type downloadResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
SignedURL string `json:"signedURL"`
|
||||
DownloadURL string `json:"downloadUrl"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type createDirResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
FileID string `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
ResType int `json:"resType"`
|
||||
CTime int64 `json:"ctime"`
|
||||
UTime int64 `json:"utime"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type deleteResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
TaskID string `json:"taskId"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type taskStatusResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
Status int `json:"status"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type uploadTokenResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data uploadTokenData `json:"data"`
|
||||
}
|
||||
|
||||
type uploadTokenData struct {
|
||||
TaskID string `json:"taskId"`
|
||||
ObjectPath string `json:"objectPath"`
|
||||
BucketName string `json:"bucketName"`
|
||||
EndPoint string `json:"endPoint"`
|
||||
FullEndPoint string `json:"fullEndPoint"`
|
||||
AccessKeyID string `json:"accessKeyID"`
|
||||
SecretAccessKey string `json:"secretAccessKey"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
Creds struct {
|
||||
AccessKeyID string `json:"accessKeyID"`
|
||||
SecretAccessKey string `json:"secretAccessKey"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
} `json:"creds"`
|
||||
}
|
||||
|
||||
type taskInfoResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
FileID string `json:"fileId"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func unixOrZero(v int64) time.Time {
|
||||
if v <= 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Unix(v, 0)
|
||||
}
|
||||
@@ -5,12 +5,14 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Drive 是多家网盘统一抽象。上层不区分盘,只区分 Kind。
|
||||
type Drive interface {
|
||||
// Kind 返回驱动代号:"quark" / "p115" / "p123" / "pikpak" / "wopan" / "onedrive" / "googledrive" / "localstorage"
|
||||
// Kind 返回驱动代号:"quark" / "p115" / "p123" / "pikpak" / "wopan" / "guangyapan" / "onedrive" / "googledrive" / "localstorage"
|
||||
Kind() string
|
||||
|
||||
// ID 返回该盘在 catalog 中的唯一标识
|
||||
@@ -40,6 +42,27 @@ type Drive interface {
|
||||
RootID() string
|
||||
}
|
||||
|
||||
// Remover is an optional drive capability. It mirrors OpenList's optional
|
||||
// Remove interface: callers must type-assert before deleting a source file.
|
||||
type Remover interface {
|
||||
Remove(ctx context.Context, fileID string) error
|
||||
}
|
||||
|
||||
// SourceFile carries the catalog metadata available when an administrator
|
||||
// requests deletion of the original source file.
|
||||
type SourceFile struct {
|
||||
FileID string
|
||||
ParentID string
|
||||
Name string
|
||||
Size int64
|
||||
}
|
||||
|
||||
// SourceRemover is an optional, richer removal capability for providers whose
|
||||
// playback ID is not the same ID required by their delete API.
|
||||
type SourceRemover interface {
|
||||
RemoveSource(ctx context.Context, source SourceFile) error
|
||||
}
|
||||
|
||||
type Entry struct {
|
||||
ID string
|
||||
Name string
|
||||
@@ -98,3 +121,42 @@ func RateLimitRetryAfter(err error) (time.Duration, bool) {
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// TextMentionsHTTPStatus only looks for explicit numeric HTTP status contexts
|
||||
// in errors from tools that do not expose structured response metadata.
|
||||
func TextMentionsHTTPStatus(text string, statuses ...int) bool {
|
||||
text = strings.ToLower(strings.TrimSpace(text))
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
for _, status := range statuses {
|
||||
if status <= 0 {
|
||||
continue
|
||||
}
|
||||
code := strconv.Itoa(status)
|
||||
if strings.HasPrefix(text, code+" ") ||
|
||||
strings.Contains(text, "status="+code) ||
|
||||
strings.Contains(text, "status: "+code) ||
|
||||
strings.Contains(text, "status "+code) ||
|
||||
strings.Contains(text, "status code "+code) ||
|
||||
strings.Contains(text, "http "+code) ||
|
||||
strings.Contains(text, "http status="+code) ||
|
||||
strings.Contains(text, "http status: "+code) ||
|
||||
strings.Contains(text, "http status "+code) ||
|
||||
strings.Contains(text, "server returned "+code) ||
|
||||
strings.Contains(text, "code="+code) ||
|
||||
strings.Contains(text, "code: "+code) ||
|
||||
strings.Contains(text, "error_code="+code) ||
|
||||
strings.Contains(text, "error_code: "+code) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ErrorMentionsHTTPStatus(err error, statuses ...int) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
return TextMentionsHTTPStatus(err.Error(), statuses...)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package drives
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestTextMentionsHTTPStatus(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
text string
|
||||
want bool
|
||||
}{
|
||||
{name: "status context", text: "request failed with status: 429 Too Many Requests", want: true},
|
||||
{name: "http context", text: "http 503 service unavailable", want: true},
|
||||
{name: "server returned context", text: "Server returned 403 Forbidden", want: true},
|
||||
{name: "message only", text: "操作频繁,请稍后重试", want: false},
|
||||
{name: "unrelated number", text: "generated 429 bytes", want: false},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := TextMentionsHTTPStatus(tc.text, 403, 429, 503); got != tc.want {
|
||||
t.Fatalf("TextMentionsHTTPStatus(%q) = %v, want %v", tc.text, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -17,20 +18,29 @@ import (
|
||||
|
||||
const Kind = "localstorage"
|
||||
|
||||
const maxSTRMBytes = 64 * 1024
|
||||
|
||||
type Config struct {
|
||||
ID string
|
||||
RootPath string
|
||||
// STRMAllowOutsideRoot 允许 .strm 指向存储根目录之外的本地路径。
|
||||
// 默认关闭:strm 等于可以让 /p/stream 读到服务器上的任意文件,只有
|
||||
// 管理员明确知道自己在做什么(例如 strm 库与 rclone 挂载目录分离)
|
||||
// 时才应打开。
|
||||
STRMAllowOutsideRoot bool
|
||||
}
|
||||
|
||||
type Driver struct {
|
||||
id string
|
||||
rootPath string
|
||||
id string
|
||||
rootPath string
|
||||
strmAllowOutsideRoot bool
|
||||
}
|
||||
|
||||
func New(c Config) *Driver {
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
rootPath: c.RootPath,
|
||||
id: c.ID,
|
||||
rootPath: c.RootPath,
|
||||
strmAllowOutsideRoot: c.STRMAllowOutsideRoot,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -122,7 +132,13 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.IsDir() || !info.Mode().IsRegular() || info.Size() <= 0 {
|
||||
if info.IsDir() || !info.Mode().IsRegular() {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
if strings.EqualFold(filepath.Ext(p), ".strm") {
|
||||
return d.streamURLFromSTRM(ctx, p)
|
||||
}
|
||||
if info.Size() <= 0 {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return &drives.StreamLink{
|
||||
@@ -131,6 +147,115 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) streamURLFromSTRM(ctx context.Context, strmPath string) (*drives.StreamLink, error) {
|
||||
target, err := readSTRMTarget(strmPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if filepath.IsAbs(target) {
|
||||
return d.localSTRMLink(strmPath, target)
|
||||
}
|
||||
u, err := url.Parse(target)
|
||||
if err == nil {
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https":
|
||||
if u.Host == "" {
|
||||
return nil, fmt.Errorf("localstorage: invalid strm url %q", target)
|
||||
}
|
||||
return &drives.StreamLink{
|
||||
URL: target,
|
||||
Expires: time.Now().Add(24 * time.Hour),
|
||||
}, nil
|
||||
case "file":
|
||||
if u.Host != "" && !strings.EqualFold(u.Host, "localhost") {
|
||||
return nil, fmt.Errorf("localstorage: unsupported strm file url host %q", u.Host)
|
||||
}
|
||||
return d.localSTRMLink(strmPath, u.Path)
|
||||
case "":
|
||||
// Local path below.
|
||||
default:
|
||||
return nil, fmt.Errorf("localstorage: unsupported strm target scheme %q", u.Scheme)
|
||||
}
|
||||
} else if strings.Contains(target, "://") {
|
||||
return nil, fmt.Errorf("localstorage: invalid strm url %q: %w", target, err)
|
||||
}
|
||||
return d.localSTRMLink(strmPath, target)
|
||||
}
|
||||
|
||||
func readSTRMTarget(path string) (string, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(f, maxSTRMBytes+1))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(data) > maxSTRMBytes {
|
||||
return "", errors.New("localstorage: strm file is too large")
|
||||
}
|
||||
lines := strings.Split(string(data), "\n")
|
||||
for i, line := range lines {
|
||||
if i == 0 {
|
||||
line = strings.TrimPrefix(line, "\ufeff")
|
||||
}
|
||||
line = strings.TrimSpace(line)
|
||||
if line != "" {
|
||||
return line, nil
|
||||
}
|
||||
}
|
||||
return "", errors.New("localstorage: empty strm target")
|
||||
}
|
||||
|
||||
func (d *Driver) localSTRMLink(strmPath, target string) (*drives.StreamLink, error) {
|
||||
target = strings.TrimSpace(target)
|
||||
if target == "" {
|
||||
return nil, errors.New("localstorage: empty strm target")
|
||||
}
|
||||
|
||||
var p string
|
||||
if filepath.IsAbs(target) {
|
||||
p = filepath.Clean(target)
|
||||
} else {
|
||||
p = filepath.Join(filepath.Dir(strmPath), filepath.FromSlash(target))
|
||||
}
|
||||
p, err := filepath.Abs(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root, err := d.root()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
realPath, within, err := realPathWithinRoot(root, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !within && !d.strmAllowOutsideRoot {
|
||||
return nil, errors.New("localstorage: strm target escapes root (enable strm_allow_outside_root to allow)")
|
||||
}
|
||||
if strings.EqualFold(filepath.Ext(p), ".strm") || strings.EqualFold(filepath.Ext(realPath), ".strm") {
|
||||
return nil, errors.New("localstorage: nested strm target is not supported")
|
||||
}
|
||||
info, err := os.Stat(realPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.IsDir() || !info.Mode().IsRegular() || info.Size() <= 0 {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return &drives.StreamLink{
|
||||
URL: realPath,
|
||||
Expires: time.Now().Add(24 * time.Hour),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
@@ -139,6 +264,39 @@ func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
p, rel, err := d.pathForID(fileID)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if rel == "" {
|
||||
return errors.New("localstorage: refusing to remove root")
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return errors.New("localstorage: refusing to remove directory")
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return errors.New("localstorage: refusing to remove non-regular file")
|
||||
}
|
||||
if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) root() (string, error) {
|
||||
raw := strings.TrimSpace(d.rootPath)
|
||||
if raw == "" {
|
||||
@@ -158,6 +316,8 @@ func (d *Driver) root() (string, error) {
|
||||
return filepath.Abs(raw)
|
||||
}
|
||||
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
func (d *Driver) pathForID(id string) (string, string, error) {
|
||||
root, err := d.root()
|
||||
if err != nil {
|
||||
@@ -177,6 +337,11 @@ func (d *Driver) pathForID(id string) (string, string, error) {
|
||||
if !pathWithinRoot(root, p) {
|
||||
return "", "", errors.New("localstorage: path escapes root")
|
||||
}
|
||||
if _, within, err := realPathWithinRoot(root, p); err != nil {
|
||||
return "", "", err
|
||||
} else if !within {
|
||||
return "", "", errors.New("localstorage: path escapes root")
|
||||
}
|
||||
return p, rel, nil
|
||||
}
|
||||
|
||||
@@ -188,6 +353,26 @@ func pathWithinRoot(root, path string) bool {
|
||||
return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator)))
|
||||
}
|
||||
|
||||
func realPathWithinRoot(root, path string) (string, bool, error) {
|
||||
realRoot, err := filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
realRoot, err = filepath.Abs(realRoot)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
realPath, err := filepath.EvalSymlinks(path)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
realPath, err = filepath.Abs(realPath)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return realPath, pathWithinRoot(realRoot, realPath), nil
|
||||
}
|
||||
|
||||
func localStoragePathHint(configured string) string {
|
||||
cwd, _ := os.Getwd()
|
||||
parts := []string{}
|
||||
|
||||
@@ -58,6 +58,199 @@ func TestListEncodesRelativePathsAndStreamURLResolvesFile(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLResolvesHTTPSTRM(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
strmPath := filepath.Join(root, "movie.strm")
|
||||
target := "https://media.example/clip.mp4?token=abc"
|
||||
if err := os.WriteFile(strmPath, []byte("\ufeff\n "+target+"\n"), 0o644); err != nil {
|
||||
t.Fatalf("write strm: %v", err)
|
||||
}
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
|
||||
link, err := drv.StreamURL(context.Background(), encodeRel("movie.strm"))
|
||||
if err != nil {
|
||||
t.Fatalf("stream url: %v", err)
|
||||
}
|
||||
if link.URL != target {
|
||||
t.Fatalf("url = %q, want %q", link.URL, target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLResolvesRelativeLocalSTRM(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(root, "links"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir links: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(root, "media"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir media: %v", err)
|
||||
}
|
||||
videoPath := filepath.Join(root, "media", "clip.mp4")
|
||||
if err := os.WriteFile(videoPath, []byte("video"), 0o644); err != nil {
|
||||
t.Fatalf("write video: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "links", "movie.strm"), []byte("../media/clip.mp4\n"), 0o644); err != nil {
|
||||
t.Fatalf("write strm: %v", err)
|
||||
}
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
|
||||
link, err := drv.StreamURL(context.Background(), encodeRel("links/movie.strm"))
|
||||
if err != nil {
|
||||
t.Fatalf("stream url: %v", err)
|
||||
}
|
||||
if link.URL != videoPath {
|
||||
t.Fatalf("url = %q, want %q", link.URL, videoPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRejectsInvalidSTRMTargets(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(t *testing.T, root string) string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
setup: func(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "empty.strm"), []byte("\n \r\n"))
|
||||
return "empty.strm"
|
||||
},
|
||||
want: "empty strm target",
|
||||
},
|
||||
{
|
||||
name: "escapes root",
|
||||
setup: func(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
writeLocalStorageTestFile(t, filepath.Join(filepath.Dir(root), "outside.mp4"), []byte("video"))
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "escape.strm"), []byte("../outside.mp4\n"))
|
||||
return "escape.strm"
|
||||
},
|
||||
want: "escapes root",
|
||||
},
|
||||
{
|
||||
name: "nested",
|
||||
setup: func(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "nested.strm"), []byte("https://media.example/clip.mp4\n"))
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "outer.strm"), []byte("nested.strm\n"))
|
||||
return "outer.strm"
|
||||
},
|
||||
want: "nested strm target",
|
||||
},
|
||||
{
|
||||
name: "unsupported scheme",
|
||||
setup: func(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "ftp.strm"), []byte("ftp://media.example/clip.mp4\n"))
|
||||
return "ftp.strm"
|
||||
},
|
||||
want: "unsupported strm target scheme",
|
||||
},
|
||||
{
|
||||
name: "too large",
|
||||
setup: func(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "large.strm"), []byte(strings.Repeat("x", maxSTRMBytes+1)))
|
||||
return "large.strm"
|
||||
},
|
||||
want: "strm file is too large",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
rel := tt.setup(t, root)
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
|
||||
_, err := drv.StreamURL(context.Background(), encodeRel(rel))
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("error = %v, want contain %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRejectsSTRMTargetEscapingRootThroughSymlink(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
writeLocalStorageTestFile(t, filepath.Join(outside, "secret.mp4"), []byte("secret"))
|
||||
if err := os.MkdirAll(filepath.Join(root, "links"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir links: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(root, "real"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir real: %v", err)
|
||||
}
|
||||
if err := os.Symlink(outside, filepath.Join(root, "real", "outside")); err != nil {
|
||||
t.Fatalf("symlink: %v", err)
|
||||
}
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "links", "movie.strm"), []byte("../real/outside/secret.mp4\n"))
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
|
||||
_, err := drv.StreamURL(context.Background(), encodeRel("links/movie.strm"))
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "strm target escapes root") {
|
||||
t.Fatalf("error = %v, want strm target escapes root", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLAllowsSTRMTargetOutsideRootWhenEnabled(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
target := filepath.Join(outside, "movie.mp4")
|
||||
writeLocalStorageTestFile(t, target, []byte("movie-data"))
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "movie.strm"), []byte(target+"\n"))
|
||||
|
||||
// 默认关闭:根目录外的目标仍被拒绝
|
||||
strict := New(Config{ID: "local", RootPath: root})
|
||||
if _, err := strict.StreamURL(context.Background(), encodeRel("movie.strm")); err == nil || !strings.Contains(err.Error(), "strm target escapes root") {
|
||||
t.Fatalf("default error = %v, want strm target escapes root", err)
|
||||
}
|
||||
|
||||
// 开启 strm_allow_outside_root 后放行
|
||||
relaxed := New(Config{ID: "local", RootPath: root, STRMAllowOutsideRoot: true})
|
||||
link, err := relaxed.StreamURL(context.Background(), encodeRel("movie.strm"))
|
||||
if err != nil {
|
||||
t.Fatalf("StreamURL with allow-outside-root: %v", err)
|
||||
}
|
||||
resolved, err := filepath.EvalSymlinks(target)
|
||||
if err != nil {
|
||||
t.Fatalf("eval target: %v", err)
|
||||
}
|
||||
if link.URL != resolved {
|
||||
t.Fatalf("link url = %q, want %q", link.URL, resolved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLAllowOutsideRootStillRejectsNestedSTRM(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
writeLocalStorageTestFile(t, filepath.Join(outside, "inner.strm"), []byte("http://example.com/v.mp4\n"))
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "movie.strm"), []byte(filepath.Join(outside, "inner.strm")+"\n"))
|
||||
|
||||
drv := New(Config{ID: "local", RootPath: root, STRMAllowOutsideRoot: true})
|
||||
if _, err := drv.StreamURL(context.Background(), encodeRel("movie.strm")); err == nil || !strings.Contains(err.Error(), "nested strm") {
|
||||
t.Fatalf("error = %v, want nested strm rejection", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRejectsSymlinkFileIDEscapingRoot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
writeLocalStorageTestFile(t, filepath.Join(outside, "secret.mp4"), []byte("secret"))
|
||||
if err := os.Symlink(filepath.Join(outside, "secret.mp4"), filepath.Join(root, "link.mp4")); err != nil {
|
||||
t.Fatalf("symlink: %v", err)
|
||||
}
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
|
||||
_, err := drv.StreamURL(context.Background(), encodeRel("link.mp4"))
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "path escapes root") {
|
||||
t.Fatalf("error = %v, want path escapes root", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRejectsEscapingID(t *testing.T) {
|
||||
drv := New(Config{ID: "local", RootPath: t.TempDir()})
|
||||
escaped := base64.RawURLEncoding.EncodeToString([]byte("../secret.mp4"))
|
||||
@@ -100,6 +293,45 @@ func TestPathForIDAllowsRootPathSlash(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerPersistsLocalStorageSTRM(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(root, "collection"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir collection: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "collection", "clip.strm"), []byte("https://media.example/clip.mp4\n"), 0o644); err != nil {
|
||||
t.Fatalf("write strm: %v", err)
|
||||
}
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
sc := scanner.New(cat, drv, []string{".strm"}, nil, nil)
|
||||
stats, err := sc.Run(ctx, drv.RootID())
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Added != 1 {
|
||||
t.Fatalf("added = %d, want 1", stats.Added)
|
||||
}
|
||||
|
||||
fileID := encodeRel("collection/clip.strm")
|
||||
got, err := cat.GetVideo(ctx, Kind+"-local-"+fileID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.Ext != "strm" || got.FileID != fileID || got.Category != "collection" {
|
||||
t.Fatalf("video = %#v, want local strm video in collection", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerPersistsLocalStorageVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
@@ -138,3 +370,10 @@ func TestScannerPersistsLocalStorageVideo(t *testing.T) {
|
||||
t.Fatalf("video = %#v, want local drive video in collection", got)
|
||||
}
|
||||
}
|
||||
|
||||
func writeLocalStorageTestFile(t *testing.T, path string, data []byte) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,12 +78,38 @@ func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
path, err := d.uploadPath(fileID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return errors.New("localupload: refusing to remove directory")
|
||||
}
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) RootID() string { return d.uploadDir() }
|
||||
|
||||
func (d *Driver) uploadDir() string {
|
||||
return d.uploadDirPath
|
||||
}
|
||||
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
func (d *Driver) uploadPath(fileID string) (string, error) {
|
||||
if strings.TrimSpace(fileID) == "" || filepath.Base(fileID) != fileID {
|
||||
return "", errors.New("invalid upload file id")
|
||||
|
||||
@@ -501,6 +501,17 @@ func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("onedrive remove: empty file id")
|
||||
}
|
||||
if err := d.request(ctx, d.itemURL(fileID), http.MethodDelete, nil, nil); err != nil {
|
||||
return fmt.Errorf("onedrive remove: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) request(ctx context.Context, rawURL, method string, configure func(*resty.Request), out any) error {
|
||||
return d.requestOnce(ctx, rawURL, method, configure, out, true)
|
||||
}
|
||||
@@ -583,8 +594,8 @@ func (d *Driver) refresh(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func isRateLimitResponse(res *resty.Response, code, message string) bool {
|
||||
if isRateLimitCode(code) || isRateLimitMessage(message) {
|
||||
func isRateLimitResponse(res *resty.Response, code, _ string) bool {
|
||||
if isRateLimitCode(code) {
|
||||
return true
|
||||
}
|
||||
if res == nil {
|
||||
@@ -621,18 +632,6 @@ func isRateLimitCode(code string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
func isRateLimitMessage(message string) bool {
|
||||
text := strings.ToLower(strings.TrimSpace(message))
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "throttl") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "activity limit") ||
|
||||
strings.Contains(text, "temporarily blocked")
|
||||
}
|
||||
|
||||
func onedriveRateLimitError(res *resty.Response, message string) error {
|
||||
if strings.TrimSpace(message) == "" {
|
||||
message = "onedrive rate limited"
|
||||
@@ -741,3 +740,4 @@ func guessMime(name string) string {
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
@@ -214,7 +214,7 @@ func TestGraph429ReturnsRateLimitErrorWithRetryAfter(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphThrottleMessageReturnsRateLimitError(t *testing.T) {
|
||||
func TestGraphThrottleMessageDoesNotReturnRateLimitError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
@@ -238,11 +238,11 @@ func TestGraphThrottleMessageReturnsRateLimitError(t *testing.T) {
|
||||
|
||||
_, err := d.StreamURL(context.Background(), "file-id")
|
||||
if err == nil {
|
||||
t.Fatal("list succeeded, want rate limit error")
|
||||
t.Fatal("list succeeded, want graph error")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
if errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want non-rate-limit error", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -87,7 +87,7 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
|
||||
// p115ListCooldown 是列目录触发疑似风控错误时的冷却时长。
|
||||
//
|
||||
// 历史上是 [30min × 3],3 次都失败就放弃;新策略改为 10 分钟无限重试 ——
|
||||
// 只要错误仍属 transient(429 / 405 / WAF / blocked / 安全威胁 / unexpected),
|
||||
// 只要错误仍属明确 HTTP transient 状态(429 / 405),
|
||||
// 就持续等 10 分钟再发一次列目录请求,直到成功或 ctx 取消。这样即使 115
|
||||
// 风控持续较长时间,扫描会自然延后到风控结束,不再丢半棵子树。
|
||||
const p115ListCooldown = 10 * time.Minute
|
||||
@@ -156,17 +156,7 @@ func isTransient115UpstreamError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
text := strings.ToLower(err.Error())
|
||||
return strings.Contains(text, "405") ||
|
||||
strings.Contains(text, "429") ||
|
||||
strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "blocked") ||
|
||||
strings.Contains(text, "security") ||
|
||||
strings.Contains(text, "waf") ||
|
||||
strings.Contains(text, "unexpected error") ||
|
||||
strings.Contains(text, "访问被阻断") ||
|
||||
strings.Contains(text, "安全威胁")
|
||||
return drives.ErrorMentionsHTTPStatus(err, http.StatusMethodNotAllowed, http.StatusTooManyRequests)
|
||||
}
|
||||
|
||||
// ListDirsOnly 只列指定目录的直接**子目录**,不返回文件条目。专为 admin 后台
|
||||
@@ -461,6 +451,23 @@ func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
if d.client == nil {
|
||||
return errors.New("p115 remove: driver not initialized")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("p115 remove: empty fileID")
|
||||
}
|
||||
if err := d.client.Delete(fileID); err != nil {
|
||||
return fmt.Errorf("p115 remove: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// bufferAndHashSha1 把 r 全量复制到一个临时文件,同时计算 SHA1。
|
||||
// 返回临时文件(位置在末尾,需调用方 Seek 回 0)、SHA1 hex 大写、实际字节数。
|
||||
//
|
||||
@@ -563,3 +570,4 @@ func guessMime(name string) string {
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
@@ -22,8 +22,9 @@ func TestIsTransient115ListError(t *testing.T) {
|
||||
want bool
|
||||
}{
|
||||
{name: "nil", err: nil, want: false},
|
||||
{name: "blocked html", err: errors.New(`<!doctype html><title>405</title>Sorry, your request has been blocked as it may cause potential threats to the server's security.`), want: true},
|
||||
{name: "chinese waf", err: errors.New("很抱歉,由于您访问的URL有可能对网站造成安全威胁,您的访问被阻断。"), want: true},
|
||||
{name: "blocked html without status context", err: errors.New(`<!doctype html><title>405</title>Sorry, your request has been blocked as it may cause potential threats to the server's security.`), want: false},
|
||||
{name: "chinese waf", err: errors.New("很抱歉,由于您访问的URL有可能对网站造成安全威胁,您的访问被阻断。"), want: false},
|
||||
{name: "status 405", err: errors.New("request failed with status: 405"), want: true},
|
||||
{name: "rate limit", err: errors.New("429 too many requests"), want: true},
|
||||
{name: "regular auth error", err: errors.New("invalid credential"), want: false},
|
||||
}
|
||||
@@ -43,10 +44,10 @@ func TestWrap115StreamTransientError(t *testing.T) {
|
||||
err error
|
||||
wantRateLimit bool
|
||||
}{
|
||||
{name: "unexpected", err: errors.New("unexpected error"), wantRateLimit: true},
|
||||
{name: "unexpected", err: errors.New("unexpected error"), wantRateLimit: false},
|
||||
{name: "405 blocked", err: errors.New("405 request has been blocked"), wantRateLimit: true},
|
||||
{name: "429", err: errors.New("429 too many requests"), wantRateLimit: true},
|
||||
{name: "blocked", err: errors.New("blocked by waf"), wantRateLimit: true},
|
||||
{name: "blocked", err: errors.New("blocked by waf"), wantRateLimit: false},
|
||||
{name: "auth", err: errors.New("invalid credential"), wantRateLimit: false},
|
||||
}
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ const (
|
||||
endpointDownloadInfo = "/file/download_info"
|
||||
endpointMkdir = "/file/upload_request"
|
||||
endpointRename = "/file/rename"
|
||||
endpointTrash = "/file/trash"
|
||||
endpointUpload = "/file/upload_request"
|
||||
endpointS3Auth = "/file/s3_upload_object/auth"
|
||||
endpointS3Parts = "/file/s3_repare_upload_parts_batch"
|
||||
@@ -259,8 +260,8 @@ func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader,
|
||||
|
||||
// UploadResult 是 UploadAndReportHash 的返回值。
|
||||
//
|
||||
// FileID 是 123 云盘分配的新文件 ID;Hash 是本次上传的 MD5 HEX(小写),
|
||||
// 与 123 云盘列表返回的 Etag 一致;Size 是实际上传字节数。
|
||||
// FileID 是 123网盘分配的新文件 ID;Hash 是本次上传的 MD5 HEX(小写),
|
||||
// 与 123网盘列表返回的 Etag 一致;Size 是实际上传字节数。
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
Hash string
|
||||
@@ -269,7 +270,7 @@ type UploadResult struct {
|
||||
|
||||
// UploadAndReportHash 把 r 上传到 parentID 目录下的指定文件名,返回新文件元数据。
|
||||
//
|
||||
// 123 云盘 Web 上传协议需要先计算文件 MD5 作为 etag 申请 upload_request。
|
||||
// 123网盘 Web 上传协议需要先计算文件 MD5 作为 etag 申请 upload_request。
|
||||
// 命中 Reuse 时服务端已经秒传;否则用返回的 S3 预签名 URL 分片 PUT,最后
|
||||
// 调 upload_complete/v2 完成。
|
||||
func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
@@ -522,7 +523,7 @@ func (d *Driver) cacheUploadedFile(fileID, parentID, name, md5Hex string, size i
|
||||
}, parentID)
|
||||
}
|
||||
|
||||
// Rename 调用 123 云盘 Web API 把指定 fileID 重命名为 newName。
|
||||
// Rename 调用 123网盘 Web API 把指定 fileID 重命名为 newName。
|
||||
func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
@@ -545,6 +546,32 @@ func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("123pan remove: empty file id")
|
||||
}
|
||||
f, _, err := d.findFile(ctx, fileID)
|
||||
if err != nil {
|
||||
if strings.Contains(strings.ToLower(err.Error()), "not found") {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("123pan remove metadata: %w", err)
|
||||
}
|
||||
body := map[string]any{
|
||||
"driveId": 0,
|
||||
"operation": true,
|
||||
"fileTrashInfoList": []panFile{f},
|
||||
}
|
||||
if _, err := d.request(ctx, endpointTrash, http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(body)
|
||||
}, nil); err != nil {
|
||||
return fmt.Errorf("123pan remove: %w", err)
|
||||
}
|
||||
d.removeCachedFile(fileID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
parts := splitPath(pathFromRoot)
|
||||
currentID := d.rootID
|
||||
@@ -583,7 +610,7 @@ func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, er
|
||||
if resp.Data.FileID != 0 {
|
||||
return strconv.FormatInt(resp.Data.FileID, 10), nil
|
||||
}
|
||||
// 123 云盘创建目录的返回字段不稳定;创建成功但没回 fileId 时回读父目录确认。
|
||||
// 123网盘创建目录的返回字段不稳定;创建成功但没回 fileId 时回读父目录确认。
|
||||
childID, err := d.findChildDir(ctx, parentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -727,8 +754,8 @@ func (d *Driver) request(ctx context.Context, endpoint, method string, configure
|
||||
return nil, errors.New("123pan request: unauthorized")
|
||||
}
|
||||
|
||||
func isP123RateLimitResponse(res *resty.Response, code int, message string) bool {
|
||||
if code == http.StatusTooManyRequests || isP123RateLimitMessage(message) {
|
||||
func isP123RateLimitResponse(res *resty.Response, code int, _ string) bool {
|
||||
if code == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if res == nil {
|
||||
@@ -737,7 +764,7 @@ func isP123RateLimitResponse(res *resty.Response, code int, message string) bool
|
||||
return isP123RateLimitHTTPResponse(res.StatusCode(), res.Header().Get("Retry-After"), res.String())
|
||||
}
|
||||
|
||||
func isP123RateLimitHTTPResponse(status int, retryAfter, body string) bool {
|
||||
func isP123RateLimitHTTPResponse(status int, retryAfter, _ string) bool {
|
||||
if status == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
@@ -747,35 +774,9 @@ func isP123RateLimitHTTPResponse(status int, retryAfter, body string) bool {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if isP123RateLimitMessage(body) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isP123RateLimitMessage(message string) bool {
|
||||
text := strings.ToLower(strings.TrimSpace(message))
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(text, "请求太频繁") ||
|
||||
strings.Contains(text, "请求过于频繁") ||
|
||||
strings.Contains(text, "请求频繁") ||
|
||||
strings.Contains(text, "操作频繁") ||
|
||||
strings.Contains(text, "频率限制") ||
|
||||
strings.Contains(text, "请求次数过多") ||
|
||||
strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "rate-limit") ||
|
||||
strings.Contains(text, "ratelimit") ||
|
||||
strings.Contains(text, "throttl") ||
|
||||
strings.Contains(text, "temporarily blocked") ||
|
||||
strings.Contains(text, "request has been blocked") ||
|
||||
strings.Contains(text, "blocked") ||
|
||||
strings.Contains(text, "访问被阻断")
|
||||
}
|
||||
|
||||
func p123RateLimitError(res *resty.Response, code int, message string) error {
|
||||
if strings.TrimSpace(message) == "" {
|
||||
message = "123pan rate limited"
|
||||
@@ -942,6 +943,12 @@ func (d *Driver) renameCachedFile(fileID, newName string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) removeCachedFile(fileID string) {
|
||||
d.fileMu.Lock()
|
||||
delete(d.files, fileID)
|
||||
d.fileMu.Unlock()
|
||||
}
|
||||
|
||||
func (d *Driver) cachedFile(fileID string) (panFile, string, bool) {
|
||||
d.fileMu.RLock()
|
||||
defer d.fileMu.RUnlock()
|
||||
@@ -1008,7 +1015,7 @@ func loginError(message string) error {
|
||||
message = strings.TrimSpace(message)
|
||||
if strings.Contains(message, "境外登录风险") ||
|
||||
(strings.Contains(message, "短信验证码") && strings.Contains(message, "微信")) {
|
||||
return errors.New("123pan login: 账号密码登录被 123 云盘风控拦截,请在浏览器完成短信/微信验证后复制 access_token,并在后台编辑该 123 云盘时只填写 access_token")
|
||||
return errors.New("123pan login: 账号密码登录被 123网盘风控拦截,请在浏览器完成短信/微信验证后复制 access_token,并在后台编辑该 123网盘时只填写 access_token")
|
||||
}
|
||||
if message == "" {
|
||||
message = "login failed"
|
||||
@@ -1111,3 +1118,4 @@ func guessMime(name string) string {
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
@@ -278,7 +278,7 @@ func qrScanPlatformText(platform int) string {
|
||||
case 4:
|
||||
return "微信"
|
||||
case 7:
|
||||
return "123 云盘 App"
|
||||
return "123网盘 App"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -150,7 +150,7 @@ func TestQRCodePollUsesAppToken(t *testing.T) {
|
||||
if wxCodeRequested {
|
||||
t.Fatalf("wx_code should not be called when app token is already returned")
|
||||
}
|
||||
if got.AccessToken != "app-token" || got.PlatformText != "123 云盘 App" {
|
||||
if got.AccessToken != "app-token" || got.PlatformText != "123网盘 App" {
|
||||
t.Fatalf("status = %#v, want app token", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"path"
|
||||
@@ -43,8 +44,9 @@ type Driver struct {
|
||||
algorithms []string
|
||||
userAgent string
|
||||
|
||||
client *resty.Client
|
||||
onTokenUpdate func(access, refresh, captcha, deviceID string)
|
||||
client *resty.Client
|
||||
onTokenUpdate func(access, refresh, captcha, deviceID string)
|
||||
uploadToOSSFunc func(context.Context, *s3Params, io.Reader) error
|
||||
|
||||
// captchaMu serializes captcha-token refreshes triggered by 4002 / 9
|
||||
// recovery in requestOnce. Without it, N concurrent callers all hitting
|
||||
@@ -173,8 +175,8 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
|
||||
|
||||
// pikpakListCooldown 是列目录触发疑似限流错误时的冷却时长。
|
||||
//
|
||||
// 与 p115 driver 的 listCooldown 同语义:只要错误属 transient
|
||||
// (error_code=10 / HTTP 429 / 5xx / 通用 "rate limit" 文本),就持续
|
||||
// 与 p115 driver 的 listCooldown 同语义:只要错误属明确限流/临时状态
|
||||
// (结构化 error_code=10 / HTTP 429 / 5xx),就持续
|
||||
// 等 10 分钟再发一次列目录请求,直到成功或 ctx 取消。这样即使 PikPak
|
||||
// 风控持续较长时间,扫描会自然延后到风控结束,不再丢半棵子树。
|
||||
const pikpakListCooldown = 10 * time.Minute
|
||||
@@ -240,7 +242,6 @@ func pikpakSleepContext(ctx context.Context, d time.Duration) error {
|
||||
//
|
||||
// - PikPak 业务码 error_code=10 ("操作频繁",见 OpenList drivers/pikpak/util.go)
|
||||
// - HTTP 429 / 500 / 502 / 503 / 504 / 509(rclone 也把这些归为 retry)
|
||||
// - 通用文本:rate limit / too many requests / blocked / temporarily unavailable
|
||||
//
|
||||
// 不包含 4122/4121/16(access_token 过期)和 9/4002(captcha 过期)—— 这些
|
||||
// 由 requestOnce 内部已经做过一次自动恢复重试;如果恢复后仍然报这类错误,
|
||||
@@ -257,22 +258,14 @@ func isTransientPikPakListError(err error) bool {
|
||||
return true
|
||||
}
|
||||
}
|
||||
text := strings.ToLower(err.Error())
|
||||
return strings.Contains(text, "error_code=10") ||
|
||||
strings.Contains(text, "429") ||
|
||||
strings.Contains(text, "http 500") ||
|
||||
strings.Contains(text, "http 502") ||
|
||||
strings.Contains(text, "http 503") ||
|
||||
strings.Contains(text, "http 504") ||
|
||||
strings.Contains(text, "http 509") ||
|
||||
strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "operation frequent") ||
|
||||
strings.Contains(text, "操作频繁") ||
|
||||
strings.Contains(text, "blocked") ||
|
||||
strings.Contains(text, "temporarily unavailable") ||
|
||||
strings.Contains(text, "service unavailable")
|
||||
return drives.ErrorMentionsHTTPStatus(err,
|
||||
http.StatusTooManyRequests,
|
||||
http.StatusInternalServerError,
|
||||
http.StatusBadGateway,
|
||||
http.StatusServiceUnavailable,
|
||||
http.StatusGatewayTimeout,
|
||||
509,
|
||||
)
|
||||
}
|
||||
|
||||
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
|
||||
@@ -354,6 +347,19 @@ func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("pikpak remove: empty file id")
|
||||
}
|
||||
if err := d.request(ctx, filesURL+":batchTrash", http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(map[string]any{"ids": []string{fileID}})
|
||||
}, nil); err != nil {
|
||||
return fmt.Errorf("pikpak remove: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
currentID := d.rootID
|
||||
for _, name := range splitPath(pathFromRoot) {
|
||||
@@ -563,3 +569,4 @@ func ParseBoolDefault(raw string, def bool) bool {
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
@@ -6,7 +6,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -26,7 +29,7 @@ import (
|
||||
// - 未命中:resumable.params 含 S3 兼容凭证(access_key / secret /
|
||||
// bucket / endpoint / key / security_token)
|
||||
//
|
||||
// 3. 用 Aliyun OSS SDK PutObject 把字节传到 endpoint+bucket+key
|
||||
// 3. 用 Aliyun OSS SDK PutObject 把字节传到 PikPak 返回的临时 OSS endpoint
|
||||
//
|
||||
// 4. PikPak 服务端轮询 OSS,发现完成后把 resp.File.ID 标记为可用;
|
||||
// 所以 Upload 完成后直接返回 resp.File.ID 即可(一开始就有,
|
||||
@@ -39,6 +42,9 @@ const (
|
||||
// spider91 视频通常 ~100MiB,远低于该值。超过则需走 multipart,
|
||||
// 当前未实现,遇到会显式报错。
|
||||
maxSinglePutSize = 5*1024*1024*1024 - 1
|
||||
// 首次上传失败后最多再重试 3 次。每次重试都会重新申请 PikPak
|
||||
// upload session,以避开偶发不可解析/不可达的临时上传 endpoint。
|
||||
pikpakUploadMaxAttempts = 4
|
||||
)
|
||||
|
||||
// uploadTaskData 是 POST /drive/v1/files 的响应结构。
|
||||
@@ -129,13 +135,49 @@ func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string,
|
||||
_ = os.Remove(tmp.Name())
|
||||
}()
|
||||
|
||||
// 2) 申请上传会话。
|
||||
result := UploadResult{Hash: gcidHex, Size: actualSize}
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= pikpakUploadMaxAttempts; attempt++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
|
||||
resp, err := d.requestUploadSession(ctx, parentID, name, actualSize, gcidHex)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("pikpak upload: request session: %w", err)
|
||||
if !shouldRetryPikPakUploadAttempt(lastErr, attempt) {
|
||||
return UploadResult{}, lastErr
|
||||
}
|
||||
d.logUploadRetry(name, attempt, lastErr)
|
||||
if err := pikpakSleepContext(ctx, pikpakUploadRetryDelay(attempt)); err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
out, err := d.completeUploadAttempt(ctx, tmp, parentID, name, result, resp)
|
||||
if err == nil {
|
||||
return out, nil
|
||||
}
|
||||
lastErr = err
|
||||
if !shouldRetryPikPakUploadAttempt(lastErr, attempt) {
|
||||
return UploadResult{}, lastErr
|
||||
}
|
||||
d.logUploadRetry(name, attempt, lastErr)
|
||||
if err := pikpakSleepContext(ctx, pikpakUploadRetryDelay(attempt)); err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
}
|
||||
return UploadResult{}, lastErr
|
||||
}
|
||||
|
||||
func (d *Driver) requestUploadSession(ctx context.Context, parentID, name string, size int64, gcidHex string) (uploadTaskData, error) {
|
||||
var resp uploadTaskData
|
||||
if err := d.request(ctx, filesURL, http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(map[string]any{
|
||||
"kind": "drive#file",
|
||||
"name": name,
|
||||
"size": actualSize,
|
||||
"size": size,
|
||||
"hash": gcidHex,
|
||||
"upload_type": "UPLOAD_TYPE_RESUMABLE",
|
||||
"objProvider": map[string]any{"provider": "UPLOAD_TYPE_UNKNOWN"},
|
||||
@@ -143,12 +185,13 @@ func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string,
|
||||
"folder_type": "NORMAL",
|
||||
})
|
||||
}, &resp); err != nil {
|
||||
return UploadResult{}, fmt.Errorf("pikpak upload: request session: %w", err)
|
||||
return uploadTaskData{}, err
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
result := UploadResult{Hash: gcidHex, Size: actualSize}
|
||||
|
||||
// 3) 命中秒传:服务端已经知道这个 hash,直接返回新文件 ID。
|
||||
func (d *Driver) completeUploadAttempt(ctx context.Context, tmp *os.File, parentID, name string, result UploadResult, resp uploadTaskData) (UploadResult, error) {
|
||||
// 命中秒传:服务端已经知道这个 hash,直接返回新文件 ID。
|
||||
if resp.Resumable == nil {
|
||||
if resp.File.ID != "" {
|
||||
result.FileID = resp.File.ID
|
||||
@@ -163,7 +206,7 @@ func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string,
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// 4) 未命中秒传:把字节传到 S3 兼容存储。
|
||||
// 未命中秒传:把字节传到 S3 兼容存储。
|
||||
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
|
||||
return UploadResult{}, fmt.Errorf("pikpak upload: seek tmp: %w", err)
|
||||
}
|
||||
@@ -171,7 +214,7 @@ func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string,
|
||||
return UploadResult{}, fmt.Errorf("pikpak upload: oss put: %w", err)
|
||||
}
|
||||
|
||||
// 5) 拿到 fileID。优先走响应里的预分配 ID;为空就回查目录。
|
||||
// 拿到 fileID。优先走响应里的预分配 ID;为空就回查目录。
|
||||
if resp.File.ID != "" {
|
||||
result.FileID = resp.File.ID
|
||||
return result, nil
|
||||
@@ -184,6 +227,58 @@ func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string,
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func shouldRetryPikPakUploadAttempt(err error, attempt int) bool {
|
||||
return attempt < pikpakUploadMaxAttempts && isRetryablePikPakUploadError(err)
|
||||
}
|
||||
|
||||
func pikpakUploadRetryDelay(attempt int) time.Duration {
|
||||
if attempt <= 0 {
|
||||
return 0
|
||||
}
|
||||
return time.Duration(attempt) * time.Second
|
||||
}
|
||||
|
||||
func (d *Driver) logUploadRetry(name string, attempt int, err error) {
|
||||
log.Printf("[pikpak] upload retry drive=%s name=%q next_attempt=%d/%d err=%v",
|
||||
d.id, name, attempt+1, pikpakUploadMaxAttempts, err)
|
||||
}
|
||||
|
||||
func isRetryablePikPakUploadError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return false
|
||||
}
|
||||
var serviceErr oss.ServiceError
|
||||
if errors.As(err, &serviceErr) {
|
||||
return serviceErr.StatusCode == http.StatusTooManyRequests || serviceErr.StatusCode >= 500
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) {
|
||||
return true
|
||||
}
|
||||
text := strings.ToLower(err.Error())
|
||||
return strings.Contains(text, "no such host") ||
|
||||
strings.Contains(text, "temporary failure in name resolution") ||
|
||||
strings.Contains(text, "server misbehaving") ||
|
||||
strings.Contains(text, "connection reset") ||
|
||||
strings.Contains(text, "connection refused") ||
|
||||
strings.Contains(text, "broken pipe") ||
|
||||
strings.Contains(text, "eof") ||
|
||||
strings.Contains(text, "i/o timeout") ||
|
||||
strings.Contains(text, "tls handshake timeout") ||
|
||||
strings.Contains(text, "http 429") ||
|
||||
strings.Contains(text, "http 500") ||
|
||||
strings.Contains(text, "http 502") ||
|
||||
strings.Contains(text, "http 503") ||
|
||||
strings.Contains(text, "http 504") ||
|
||||
strings.Contains(text, "http 509") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "temporarily unavailable") ||
|
||||
strings.Contains(text, "service unavailable")
|
||||
}
|
||||
|
||||
// bufferAndHashGCID 把 r 复制到一个临时文件,同时计算 GCID。
|
||||
// 返回临时文件(位置在末尾,需要调用方 Seek 回 0)、GCID hex 大写、实际写入字节数。
|
||||
//
|
||||
@@ -215,10 +310,13 @@ func bufferAndHashGCID(r io.Reader, size int64) (*os.File, string, int64, error)
|
||||
//
|
||||
// 参数复用 PikPak 的临时凭证;必须带 Security Token 头部 + UserAgent,与 OpenList 一致。
|
||||
func (d *Driver) uploadToOSS(ctx context.Context, p *s3Params, body io.Reader) error {
|
||||
if d.uploadToOSSFunc != nil {
|
||||
return d.uploadToOSSFunc(ctx, p, body)
|
||||
}
|
||||
if p == nil {
|
||||
return errors.New("pikpak upload: nil s3 params")
|
||||
}
|
||||
client, err := oss.New(p.Endpoint, p.AccessKeyID, p.AccessKeySecret)
|
||||
client, err := newPikPakOSSClient(p)
|
||||
if err != nil {
|
||||
return fmt.Errorf("oss client: %w", err)
|
||||
}
|
||||
@@ -235,6 +333,44 @@ func (d *Driver) uploadToOSS(ctx context.Context, p *s3Params, body io.Reader) e
|
||||
)
|
||||
}
|
||||
|
||||
func newPikPakOSSClient(p *s3Params, options ...oss.ClientOption) (*oss.Client, error) {
|
||||
if p == nil {
|
||||
return nil, errors.New("pikpak upload: nil s3 params")
|
||||
}
|
||||
clientOptions := make([]oss.ClientOption, 0, len(options)+1)
|
||||
if isPikPakCNAMEEndpoint(p.Endpoint) {
|
||||
clientOptions = append(clientOptions, oss.UseCname(true))
|
||||
}
|
||||
clientOptions = append(clientOptions, options...)
|
||||
return oss.New(p.Endpoint, p.AccessKeyID, p.AccessKeySecret, clientOptions...)
|
||||
}
|
||||
|
||||
func isPikPakCNAMEEndpoint(endpoint string) bool {
|
||||
host := endpointHost(endpoint)
|
||||
if host == "" {
|
||||
return false
|
||||
}
|
||||
host = strings.TrimSuffix(strings.ToLower(host), ".")
|
||||
return host != "mypikpak.com" && host != "mypikpak.net" &&
|
||||
(strings.HasSuffix(host, ".mypikpak.com") || strings.HasSuffix(host, ".mypikpak.net"))
|
||||
}
|
||||
|
||||
func endpointHost(endpoint string) string {
|
||||
endpoint = strings.TrimSpace(endpoint)
|
||||
if endpoint == "" {
|
||||
return ""
|
||||
}
|
||||
if u, err := url.Parse(endpoint); err == nil && u.Host != "" {
|
||||
endpoint = u.Host
|
||||
} else if idx := strings.IndexByte(endpoint, '/'); idx >= 0 {
|
||||
endpoint = endpoint[:idx]
|
||||
}
|
||||
if host, _, err := net.SplitHostPort(endpoint); err == nil {
|
||||
endpoint = host
|
||||
}
|
||||
return strings.Trim(endpoint, "[]")
|
||||
}
|
||||
|
||||
type readerWithCtx struct {
|
||||
ctx context.Context
|
||||
r io.Reader
|
||||
|
||||
@@ -6,12 +6,15 @@ import (
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||||
"github.com/go-resty/resty/v2"
|
||||
)
|
||||
|
||||
@@ -181,6 +184,95 @@ func TestUploadInstantSuccessFallsBackToListWhenFileIDMissing(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadRetriesWithNewSessionWhenOSSEndpointDNSFails(t *testing.T) {
|
||||
sessionRequests := 0
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
t.Errorf("method = %q, want POST", r.Method)
|
||||
}
|
||||
sessionRequests++
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(fmt.Sprintf(`{
|
||||
"upload_type": "UPLOAD_TYPE_RESUMABLE",
|
||||
"resumable": {
|
||||
"kind": "drive#resumable",
|
||||
"provider": "UPLOAD_TYPE_UNKNOWN",
|
||||
"params": {
|
||||
"access_key_id": "ak",
|
||||
"access_key_secret": "sk",
|
||||
"bucket": "bucket",
|
||||
"endpoint": "https://vip-lixian-%02d.upload-a10b.mypikpak.com",
|
||||
"key": "object-key-%02d",
|
||||
"security_token": "token"
|
||||
}
|
||||
},
|
||||
"file": {"id": "retry-file-%02d", "name": "retry.mp4", "kind": "drive#file"}
|
||||
}`, sessionRequests, sessionRequests, sessionRequests)))
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
uploadAttempts := 0
|
||||
var uploaded []byte
|
||||
d.uploadToOSSFunc = func(_ context.Context, _ *s3Params, body io.Reader) error {
|
||||
uploadAttempts++
|
||||
if uploadAttempts == 1 {
|
||||
return &net.DNSError{Err: "no such host", Name: "vip-lixian-01.upload-a10b.mypikpak.com"}
|
||||
}
|
||||
var err error
|
||||
uploaded, err = io.ReadAll(body)
|
||||
return err
|
||||
}
|
||||
|
||||
payload := []byte("retry payload body")
|
||||
id, err := d.Upload(context.Background(), "parent-id", "retry.mp4", bytes.NewReader(payload), int64(len(payload)))
|
||||
if err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
if id != "retry-file-02" {
|
||||
t.Fatalf("file id = %q, want retry-file-02 from the second session", id)
|
||||
}
|
||||
if sessionRequests != 2 {
|
||||
t.Fatalf("session requests = %d, want 2", sessionRequests)
|
||||
}
|
||||
if uploadAttempts != 2 {
|
||||
t.Fatalf("upload attempts = %d, want 2", uploadAttempts)
|
||||
}
|
||||
if !bytes.Equal(uploaded, payload) {
|
||||
t.Fatalf("uploaded body = %q, want %q", string(uploaded), string(payload))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPikPakOSSClientUsesCNAMEForPikPakUploadEndpoint(t *testing.T) {
|
||||
params := &s3Params{
|
||||
AccessKeyID: "ak",
|
||||
AccessKeySecret: "sk",
|
||||
Bucket: "vip-lixian-07",
|
||||
Endpoint: "http://upload-a10b.mypikpak.com",
|
||||
Key: "upload_tmp/object-key",
|
||||
}
|
||||
client, err := newPikPakOSSClient(params)
|
||||
if err != nil {
|
||||
t.Fatalf("new oss client: %v", err)
|
||||
}
|
||||
bucket, err := client.Bucket(params.Bucket)
|
||||
if err != nil {
|
||||
t.Fatalf("bucket: %v", err)
|
||||
}
|
||||
signed, err := bucket.SignURL(params.Key, oss.HTTPPut, 60)
|
||||
if err != nil {
|
||||
t.Fatalf("sign url: %v", err)
|
||||
}
|
||||
if strings.Contains(signed, "vip-lixian-07.upload-a10b.mypikpak.com") {
|
||||
t.Fatalf("signed url uses invalid bucket-prefixed PikPak host: %s", signed)
|
||||
}
|
||||
if !strings.Contains(signed, "http://upload-a10b.mypikpak.com/upload_tmp%2Fobject-key") {
|
||||
t.Fatalf("signed url = %s, want PikPak endpoint host with object key path", signed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadRejectsInvalidArguments(t *testing.T) {
|
||||
d := New(Config{ID: "x", Username: "u", Password: "p", Platform: "web"})
|
||||
cases := []struct {
|
||||
|
||||
@@ -16,23 +16,23 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
defaultUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) quark-cloud-drive/2.5.20 Chrome/100.0.4896.160 Electron/18.3.5.4-b478491100 Safari/537.36 Channel/pckk_other_ch"
|
||||
defaultUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) quark-cloud-drive/2.5.20 Chrome/100.0.4896.160 Electron/18.3.5.4-b478491100 Safari/537.36 Channel/pckk_other_ch"
|
||||
defaultReferer = "https://pan.quark.cn"
|
||||
defaultAPI = "https://drive.quark.cn/1/clouddrive"
|
||||
defaultPR = "ucpro"
|
||||
)
|
||||
|
||||
type Driver struct {
|
||||
id string
|
||||
cookie string
|
||||
rootID string
|
||||
ua string
|
||||
referer string
|
||||
apiBase string
|
||||
pr string
|
||||
client *resty.Client
|
||||
onCookieUpdate func(string)
|
||||
useTranscodingAddress bool
|
||||
id string
|
||||
cookie string
|
||||
rootID string
|
||||
ua string
|
||||
referer string
|
||||
apiBase string
|
||||
pr string
|
||||
client *resty.Client
|
||||
onCookieUpdate func(string)
|
||||
useTranscodingAddress bool
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -60,7 +60,7 @@ func New(c Config) *Driver {
|
||||
onCookieUpdate: c.OnCookieUpdate,
|
||||
}
|
||||
d.client = resty.New().
|
||||
SetTimeout(30 * time.Second).
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*").
|
||||
SetHeader("Referer", d.referer).
|
||||
SetHeader("User-Agent", d.ua)
|
||||
@@ -269,6 +269,22 @@ func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader,
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("quark remove: empty file id")
|
||||
}
|
||||
body := map[string]any{
|
||||
"action_type": 1,
|
||||
"exclude_fids": []string{},
|
||||
"filelist": []string{fileID},
|
||||
}
|
||||
if err := d.request(ctx, "/file/delete", http.MethodPost, nil, body, nil); err != nil {
|
||||
return fmt.Errorf("quark remove: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------- helpers ----------
|
||||
|
||||
func fileToEntry(f *file, parentID string) drives.Entry {
|
||||
@@ -343,3 +359,4 @@ func setCookieValue(cookie, key, value string) string {
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,775 @@
|
||||
package scriptcrawler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/fingerprint"
|
||||
)
|
||||
|
||||
const (
|
||||
scriptCrawlerDuplicateBytes = "duplicate-video-bytes"
|
||||
scriptCrawlerUniqueBytes = "unique-video-bytes"
|
||||
)
|
||||
|
||||
func writeScriptCrawlerFFprobeStub(t *testing.T, dir string, ok bool) string {
|
||||
t.Helper()
|
||||
name := "ffprobe-ok.sh"
|
||||
body := "#!/bin/sh\necho video\nexit 0\n"
|
||||
if !ok {
|
||||
name = "ffprobe-fail.sh"
|
||||
body = "#!/bin/sh\necho 'moov atom not found' >&2\nexit 1\n"
|
||||
}
|
||||
path := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
|
||||
t.Fatalf("write ffprobe stub: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func writeScriptCrawlerFFmpegStub(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, "ffmpeg-hls.sh")
|
||||
body := "#!/bin/sh\nout=\"\"\nfor arg do out=\"$arg\"; done\nprintf 'hls-video-bytes' > \"$out\"\n"
|
||||
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
|
||||
t.Fatalf("write ffmpeg stub: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceImportsLocalFileAndSkipsExisting(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
dummyScript := filepath.Join(tmp, "helper-script")
|
||||
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
|
||||
t.Fatalf("write dummy script: %v", err)
|
||||
}
|
||||
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
|
||||
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
|
||||
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
|
||||
t.Fatalf("write helper wrapper: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
CrawlerName: "Demo Crawler",
|
||||
PythonPath: wrapper,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
ScriptPath: dummyScript,
|
||||
})
|
||||
res, err := c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 || res.Skipped != 0 || res.Failed != 0 {
|
||||
t.Fatalf("result = new:%d skipped:%d failed:%d, want 1/0/0", res.NewVideos, res.Skipped, res.Failed)
|
||||
}
|
||||
v, err := cat.GetVideo(ctx, BuildVideoID("demo", "abc-123"))
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if v.Title != "Imported From Helper" || v.FileID != "abc-123.mp4" || v.Size == 0 {
|
||||
t.Fatalf("video = title:%q file:%q size:%d", v.Title, v.FileID, v.Size)
|
||||
}
|
||||
if !hasString(v.Tags, "Demo Crawler") {
|
||||
t.Fatalf("video tags = %#v, want crawler name tag", v.Tags)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(drv.VideosDir(), "abc-123.mp4")); err != nil {
|
||||
t.Fatalf("video file not copied: %v", err)
|
||||
}
|
||||
|
||||
res, err = c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("second run: %v", err)
|
||||
}
|
||||
if res.NewVideos != 0 || res.Skipped != 1 {
|
||||
t.Fatalf("second result = new:%d skipped:%d, want 0/1", res.NewVideos, res.Skipped)
|
||||
}
|
||||
if res.SeenSnapshot != 1 {
|
||||
t.Fatalf("seen snapshot = %d, want 1", res.SeenSnapshot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceMarksPreviewDisabledWhenConfigured(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
dummyScript := filepath.Join(tmp, "helper-script")
|
||||
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
|
||||
t.Fatalf("write dummy script: %v", err)
|
||||
}
|
||||
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
|
||||
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
|
||||
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
|
||||
t.Fatalf("write helper wrapper: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
PythonPath: wrapper,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
ScriptPath: dummyScript,
|
||||
DisablePreview: true,
|
||||
})
|
||||
res, err := c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 || res.Failed != 0 {
|
||||
t.Fatalf("result = new:%d failed:%d, want 1/0", res.NewVideos, res.Failed)
|
||||
}
|
||||
v, err := cat.GetVideo(ctx, BuildVideoID("demo", "abc-123"))
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if v.PreviewStatus != "disabled" {
|
||||
t.Fatalf("preview status = %q, want disabled", v.PreviewStatus)
|
||||
}
|
||||
if v.FingerprintStatus != "ready" || v.SampledSHA256 == "" {
|
||||
t.Fatalf("fingerprint status=%q sampled=%q, want ready and sampled hash", v.FingerprintStatus, v.SampledSHA256)
|
||||
}
|
||||
pending, err := cat.ListVideosByPreviewStatus(ctx, "demo", "pending", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list pending previews: %v", err)
|
||||
}
|
||||
if len(pending) != 0 {
|
||||
t.Fatalf("pending previews = %d, want 0", len(pending))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceUsesCurrentDrivePreviewSwitch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: drv.ID(),
|
||||
Kind: Kind,
|
||||
Name: "Demo",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{"script_path": "/tmp/crawler.py"},
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
dummyScript := filepath.Join(tmp, "helper-script")
|
||||
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
|
||||
t.Fatalf("write dummy script: %v", err)
|
||||
}
|
||||
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
|
||||
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
|
||||
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
|
||||
t.Fatalf("write helper wrapper: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
PythonPath: wrapper,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
ScriptPath: dummyScript,
|
||||
DisablePreview: true,
|
||||
})
|
||||
res, err := c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 || res.Failed != 0 {
|
||||
t.Fatalf("result = new:%d failed:%d, want 1/0", res.NewVideos, res.Failed)
|
||||
}
|
||||
v, err := cat.GetVideo(ctx, BuildVideoID("demo", "abc-123"))
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if v.PreviewStatus != "pending" {
|
||||
t.Fatalf("preview status = %q, want pending from current drive switch", v.PreviewStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceUsesSourceKindNamespace(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
dummyScript := filepath.Join(tmp, "helper-script")
|
||||
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
|
||||
t.Fatalf("write dummy script: %v", err)
|
||||
}
|
||||
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
|
||||
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
|
||||
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
|
||||
t.Fatalf("write helper wrapper: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
SourceKind: "spider91",
|
||||
PythonPath: wrapper,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
ScriptPath: dummyScript,
|
||||
})
|
||||
res, err := c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 || res.SeenSnapshot != 0 {
|
||||
t.Fatalf("result = new:%d seen:%d, want 1/0", res.NewVideos, res.SeenSnapshot)
|
||||
}
|
||||
videoID := BuildVideoIDForKind("spider91", "demo", "abc-123")
|
||||
if _, err := cat.GetVideo(ctx, videoID); err != nil {
|
||||
t.Fatalf("get source-kind video: %v", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, BuildVideoID("demo", "abc-123")); err == nil {
|
||||
t.Fatalf("default namespace video unexpectedly exists")
|
||||
}
|
||||
|
||||
res, err = c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("second run: %v", err)
|
||||
}
|
||||
if res.NewVideos != 0 || res.Skipped != 1 || res.SeenSnapshot != 1 {
|
||||
t.Fatalf("second result = new:%d skipped:%d seen:%d, want 0/1/1", res.NewVideos, res.Skipped, res.SeenSnapshot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOncePassesAbsoluteJobPathsWhenWorkDirDiffers(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
t.Chdir(tmp)
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join("data", "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
scriptDir := filepath.Join(tmp, "scripts")
|
||||
if err := os.MkdirAll(scriptDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir script dir: %v", err)
|
||||
}
|
||||
dummyScript := filepath.Join(scriptDir, "helper-script")
|
||||
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
|
||||
t.Fatalf("write dummy script: %v", err)
|
||||
}
|
||||
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
|
||||
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
|
||||
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
|
||||
t.Fatalf("write helper wrapper: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_ASSERT_ABS", "1")
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
PythonPath: wrapper,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
ScriptPath: dummyScript,
|
||||
WorkDir: scriptDir,
|
||||
})
|
||||
res, err := c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 || res.Skipped != 0 || res.Failed != 0 {
|
||||
t.Fatalf("result = new:%d skipped:%d failed:%d, want 1/0/0", res.NewVideos, res.Skipped, res.Failed)
|
||||
}
|
||||
if !filepath.IsAbs(res.JobFile) || !filepath.IsAbs(res.SeenFile) {
|
||||
t.Fatalf("result paths should be absolute: job=%q seen=%q", res.JobFile, res.SeenFile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceImportsSimpleMediaURLWithoutSourceID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/video.mp4" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte("simple-video-bytes"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
dummyScript := filepath.Join(tmp, "helper-script")
|
||||
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
|
||||
t.Fatalf("write dummy script: %v", err)
|
||||
}
|
||||
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
|
||||
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
|
||||
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
|
||||
t.Fatalf("write helper wrapper: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_SIMPLE", "1")
|
||||
t.Setenv("GO_SCRIPTCRAWLER_MEDIA_URL", srv.URL+"/video.mp4?token=first")
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
PythonPath: wrapper,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
ScriptPath: dummyScript,
|
||||
HTTPClient: srv.Client(),
|
||||
})
|
||||
res, err := c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 || res.Skipped != 0 || res.Failed != 0 {
|
||||
t.Fatalf("result = new:%d skipped:%d failed:%d, want 1/0/0", res.NewVideos, res.Skipped, res.Failed)
|
||||
}
|
||||
videos, err := cat.ListVideosByDrive(ctx, "demo")
|
||||
if err != nil {
|
||||
t.Fatalf("list videos: %v", err)
|
||||
}
|
||||
if len(videos) != 1 {
|
||||
t.Fatalf("videos = %d, want 1", len(videos))
|
||||
}
|
||||
v := videos[0]
|
||||
if !strings.HasPrefix(v.ID, BuildVideoID("demo", "auto-")) {
|
||||
t.Fatalf("video id = %q, want generated auto source id", v.ID)
|
||||
}
|
||||
if v.Title != "Simple Protocol Video" || v.Ext != "mp4" || v.ThumbnailURL != "" || v.Size == 0 {
|
||||
t.Fatalf("video = title:%q ext:%q thumb:%q size:%d", v.Title, v.Ext, v.ThumbnailURL, v.Size)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(drv.VideosDir(), v.FileID)); err != nil {
|
||||
t.Fatalf("video file not downloaded: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_SCRIPTCRAWLER_MEDIA_URL", srv.URL+"/video.mp4?token=second")
|
||||
res, err = c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("second run: %v", err)
|
||||
}
|
||||
if res.NewVideos != 0 || res.Skipped != 1 {
|
||||
t.Fatalf("second result = new:%d skipped:%d, want 0/1", res.NewVideos, res.Skipped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceSkipsFingerprintDuplicateAndContinues(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
|
||||
seedFile := "seed-canonical.mp4"
|
||||
if err := os.WriteFile(filepath.Join(drv.VideosDir(), seedFile), []byte(scriptCrawlerDuplicateBytes), 0o644); err != nil {
|
||||
t.Fatalf("write seed video: %v", err)
|
||||
}
|
||||
seed := &catalog.Video{
|
||||
ID: "seed-for-hash",
|
||||
DriveID: drv.ID(),
|
||||
FileID: seedFile,
|
||||
Title: "Seed",
|
||||
Size: int64(len(scriptCrawlerDuplicateBytes)),
|
||||
PublishedAt: time.Now(),
|
||||
}
|
||||
sampled, err := fingerprint.Compute(ctx, drv, seed, fingerprint.Config{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("compute seed fingerprint: %v", err)
|
||||
}
|
||||
_ = os.Remove(filepath.Join(drv.VideosDir(), seedFile))
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "existing-canonical",
|
||||
DriveID: "other-drive",
|
||||
FileID: "existing.mp4",
|
||||
FileName: "existing.mp4",
|
||||
Title: "Existing Canonical",
|
||||
Size: int64(len(scriptCrawlerDuplicateBytes)),
|
||||
Ext: "mp4",
|
||||
SampledSHA256: sampled,
|
||||
FingerprintStatus: "ready",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed canonical video: %v", err)
|
||||
}
|
||||
|
||||
dummyScript := filepath.Join(tmp, "helper-script")
|
||||
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
|
||||
t.Fatalf("write dummy script: %v", err)
|
||||
}
|
||||
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
|
||||
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
|
||||
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
|
||||
t.Fatalf("write helper wrapper: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_DUP_UNIQUE", "1")
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
PythonPath: wrapper,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
ScriptPath: dummyScript,
|
||||
})
|
||||
res, err := c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 || res.Skipped != 1 || res.Failed != 0 || res.TotalEntries != 2 {
|
||||
t.Fatalf("result = total:%d new:%d skipped:%d failed:%d, want 2/1/1/0", res.TotalEntries, res.NewVideos, res.Skipped, res.Failed)
|
||||
}
|
||||
if res.CandidateBudget <= res.TargetNew {
|
||||
t.Fatalf("candidate budget = %d, target = %d; want expanded budget", res.CandidateBudget, res.TargetNew)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, BuildVideoID("demo", "dup-source")); err == nil {
|
||||
t.Fatal("duplicate candidate should not be imported")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(drv.VideosDir(), "dup-source.mp4")); !os.IsNotExist(err) {
|
||||
t.Fatalf("duplicate local file stat = %v, want removed", err)
|
||||
}
|
||||
v, err := cat.GetVideo(ctx, BuildVideoID("demo", "unique-source"))
|
||||
if err != nil {
|
||||
t.Fatalf("unique video should be imported: %v", err)
|
||||
}
|
||||
if v.SampledSHA256 == "" || v.FingerprintStatus != "ready" {
|
||||
t.Fatalf("unique fingerprint = %q status=%q, want ready sampled fingerprint", v.SampledSHA256, v.FingerprintStatus)
|
||||
}
|
||||
seen, err := cat.ListCrawlerSourceIDs(ctx, Kind, "demo")
|
||||
if err != nil {
|
||||
t.Fatalf("list seen source ids: %v", err)
|
||||
}
|
||||
seenSet := map[string]bool{}
|
||||
for _, id := range seen {
|
||||
seenSet[id] = true
|
||||
}
|
||||
if !seenSet["dup-source"] || !seenSet["unique-source"] {
|
||||
t.Fatalf("seen ids = %#v, want duplicate and imported source ids", seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceRejectsInvalidDownloadedVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
dummyScript := filepath.Join(tmp, "helper-script")
|
||||
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
|
||||
t.Fatalf("write dummy script: %v", err)
|
||||
}
|
||||
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
|
||||
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
|
||||
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
|
||||
t.Fatalf("write helper wrapper: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
CrawlerName: "Demo Crawler",
|
||||
PythonPath: wrapper,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, false),
|
||||
ScriptPath: dummyScript,
|
||||
})
|
||||
res, err := c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if res.NewVideos != 0 || res.Skipped != 0 || res.Failed != 1 || res.TotalEntries != 1 {
|
||||
t.Fatalf("result = total:%d new:%d skipped:%d failed:%d, want 1/0/0/1", res.TotalEntries, res.NewVideos, res.Skipped, res.Failed)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, BuildVideoID("demo", "abc-123")); err == nil {
|
||||
t.Fatal("invalid video should not be imported")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(drv.VideosDir(), "abc-123.mp4")); !os.IsNotExist(err) {
|
||||
t.Fatalf("invalid local video stat = %v, want removed", err)
|
||||
}
|
||||
seen, err := cat.ListCrawlerSourceIDs(ctx, Kind, "demo")
|
||||
if err != nil {
|
||||
t.Fatalf("list seen source ids: %v", err)
|
||||
}
|
||||
if len(seen) != 0 {
|
||||
t.Fatalf("seen ids = %#v, want none for invalid video", seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceDownloadsHLSMediaURL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
dummyScript := filepath.Join(tmp, "helper-script")
|
||||
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
|
||||
t.Fatalf("write dummy script: %v", err)
|
||||
}
|
||||
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
|
||||
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
|
||||
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
|
||||
t.Fatalf("write helper wrapper: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HLS", "1")
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
CrawlerName: "Demo Crawler",
|
||||
PythonPath: wrapper,
|
||||
FFmpegPath: writeScriptCrawlerFFmpegStub(t, tmp),
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
ScriptPath: dummyScript,
|
||||
})
|
||||
res, err := c.RunOnce(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 || res.Skipped != 0 || res.Failed != 0 {
|
||||
t.Fatalf("result = new:%d skipped:%d failed:%d, want 1/0/0", res.NewVideos, res.Skipped, res.Failed)
|
||||
}
|
||||
v, err := cat.GetVideo(ctx, BuildVideoID("demo", "hls-source"))
|
||||
if err != nil {
|
||||
t.Fatalf("get hls video: %v", err)
|
||||
}
|
||||
if v.FileID != "hls-source.mp4" || v.Size != int64(len("hls-video-bytes")) {
|
||||
t.Fatalf("video file=%q size=%d, want hls-source.mp4 size %d", v.FileID, v.Size, len("hls-video-bytes"))
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(drv.VideosDir(), "hls-source.mp4"))
|
||||
if err != nil {
|
||||
t.Fatalf("read hls output: %v", err)
|
||||
}
|
||||
if string(data) != "hls-video-bytes" {
|
||||
t.Fatalf("hls output = %q", string(data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptCrawlerHelperProcess(t *testing.T) {
|
||||
if os.Getenv("GO_WANT_SCRIPTCRAWLER_HELPER") != "1" {
|
||||
return
|
||||
}
|
||||
args := os.Args
|
||||
jobPath := ""
|
||||
for i := 0; i < len(args)-1; i++ {
|
||||
if args[i] == "--job" {
|
||||
jobPath = args[i+1]
|
||||
break
|
||||
}
|
||||
}
|
||||
if jobPath == "" {
|
||||
fmt.Fprintln(os.Stderr, "missing --job")
|
||||
os.Exit(2)
|
||||
}
|
||||
data, err := os.ReadFile(jobPath)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
var job Job
|
||||
if err := json.Unmarshal(data, &job); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
if os.Getenv("GO_WANT_SCRIPTCRAWLER_ASSERT_ABS") == "1" {
|
||||
if !filepath.IsAbs(jobPath) || !filepath.IsAbs(job.SeenSourceIDsFile) || !filepath.IsAbs(job.OutputDir) {
|
||||
fmt.Fprintf(os.Stderr, "expected absolute paths, got job=%q seen=%q output=%q\n", jobPath, job.SeenSourceIDsFile, job.OutputDir)
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
if os.Getenv("GO_WANT_SCRIPTCRAWLER_SIMPLE") == "1" {
|
||||
event := map[string]any{
|
||||
"title": "Simple Protocol Video",
|
||||
"media_url": os.Getenv("GO_SCRIPTCRAWLER_MEDIA_URL"),
|
||||
}
|
||||
_ = json.NewEncoder(os.Stdout).Encode(event)
|
||||
os.Exit(0)
|
||||
}
|
||||
if os.Getenv("GO_WANT_SCRIPTCRAWLER_HLS") == "1" {
|
||||
event := Event{
|
||||
Type: "item",
|
||||
Item: Item{
|
||||
SourceID: "hls-source",
|
||||
Title: "HLS Protocol Video",
|
||||
Author: "helper",
|
||||
Media: MediaRef{
|
||||
URL: "https://media.example.test/video.m3u8",
|
||||
Headers: map[string]string{
|
||||
"Referer": "https://example.test/",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_ = json.NewEncoder(os.Stdout).Encode(event)
|
||||
os.Exit(0)
|
||||
}
|
||||
if os.Getenv("GO_WANT_SCRIPTCRAWLER_DUP_UNIQUE") == "1" {
|
||||
duplicateFile := filepath.Join(job.OutputDir, "duplicate.mp4")
|
||||
if err := os.WriteFile(duplicateFile, []byte(scriptCrawlerDuplicateBytes), 0o644); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
uniqueFile := filepath.Join(job.OutputDir, "unique.mp4")
|
||||
if err := os.WriteFile(uniqueFile, []byte(scriptCrawlerUniqueBytes), 0o644); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
for _, event := range []Event{
|
||||
{
|
||||
Type: "item",
|
||||
Item: Item{
|
||||
SourceID: "dup-source",
|
||||
Title: "Duplicate Candidate",
|
||||
Author: "helper",
|
||||
Media: MediaRef{LocalFile: duplicateFile},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: "item",
|
||||
Item: Item{
|
||||
SourceID: "unique-source",
|
||||
Title: "Unique Candidate",
|
||||
Author: "helper",
|
||||
Media: MediaRef{LocalFile: uniqueFile},
|
||||
},
|
||||
},
|
||||
} {
|
||||
_ = json.NewEncoder(os.Stdout).Encode(event)
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
localFile := filepath.Join(job.OutputDir, "helper.mp4")
|
||||
if err := os.WriteFile(localFile, []byte("helper-video"), 0o644); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
event := Event{
|
||||
Type: "item",
|
||||
Item: Item{
|
||||
SourceID: "abc-123",
|
||||
Title: "Imported From Helper",
|
||||
Author: "helper",
|
||||
Media: MediaRef{LocalFile: localFile},
|
||||
},
|
||||
}
|
||||
_ = json.NewEncoder(os.Stdout).Encode(event)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
func hasString(values []string, want string) bool {
|
||||
for _, value := range values {
|
||||
if value == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
// Package scriptcrawler provides a generic local drive for script-based
|
||||
// crawlers. A crawler script discovers videos; the Go runner downloads them
|
||||
// into this drive and the existing preview/fingerprint workers consume them
|
||||
// through the normal drives.Drive interface.
|
||||
package scriptcrawler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
const Kind = "scriptcrawler"
|
||||
|
||||
type Config struct {
|
||||
ID string
|
||||
RootDir string
|
||||
}
|
||||
|
||||
type Driver struct {
|
||||
id string
|
||||
rootDir string
|
||||
}
|
||||
|
||||
func New(c Config) *Driver {
|
||||
return &Driver{id: c.ID, rootDir: c.RootDir}
|
||||
}
|
||||
|
||||
func (d *Driver) Kind() string { return Kind }
|
||||
|
||||
func (d *Driver) ID() string { return d.id }
|
||||
|
||||
func (d *Driver) RootID() string { return "/" }
|
||||
|
||||
func (d *Driver) Init(context.Context) error {
|
||||
if strings.TrimSpace(d.id) == "" {
|
||||
return errors.New("scriptcrawler: empty drive id")
|
||||
}
|
||||
if strings.TrimSpace(d.rootDir) == "" {
|
||||
return errors.New("scriptcrawler: empty root dir")
|
||||
}
|
||||
for _, sub := range []string{"videos", "thumbs", "output", ".crawl"} {
|
||||
if err := os.MkdirAll(filepath.Join(d.rootDir, sub), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) RootDir() string { return d.rootDir }
|
||||
|
||||
func (d *Driver) VideosDir() string { return filepath.Join(d.rootDir, "videos") }
|
||||
|
||||
func (d *Driver) ThumbsDir() string { return filepath.Join(d.rootDir, "thumbs") }
|
||||
|
||||
func (d *Driver) OutputDir() string { return filepath.Join(d.rootDir, "output") }
|
||||
|
||||
func (d *Driver) CrawlDir() string { return filepath.Join(d.rootDir, ".crawl") }
|
||||
|
||||
func (d *Driver) VideoPath(fileID string) (string, error) {
|
||||
return safeJoin(d.VideosDir(), fileID)
|
||||
}
|
||||
|
||||
func (d *Driver) ThumbPath(fileID string) (string, error) {
|
||||
return safeJoin(d.ThumbsDir(), fileID)
|
||||
}
|
||||
|
||||
func (d *Driver) OutputPath(fileName string) (string, error) {
|
||||
return safeJoin(d.OutputDir(), fileName)
|
||||
}
|
||||
|
||||
func (d *Driver) List(context.Context, string) ([]drives.Entry, error) {
|
||||
entries, err := os.ReadDir(d.VideosDir())
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
out := make([]drives.Entry, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, drives.Entry{
|
||||
ID: e.Name(),
|
||||
Name: e.Name(),
|
||||
Size: info.Size(),
|
||||
IsDir: false,
|
||||
ModTime: info.ModTime(),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
|
||||
path, err := d.VideoPath(fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &drives.Entry{
|
||||
ID: fileID,
|
||||
Name: fileID,
|
||||
Size: info.Size(),
|
||||
IsDir: info.IsDir(),
|
||||
ModTime: info.ModTime(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
path, err := d.VideoPath(fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.IsDir() || info.Size() == 0 {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return &drives.StreamLink{
|
||||
URL: path,
|
||||
Expires: time.Now().Add(24 * time.Hour),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
videoPath, err := d.VideoPath(fileID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Stat(videoPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
removeThumbCandidates(d.ThumbPath, strings.TrimSuffix(fileID, filepath.Ext(fileID)))
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return errors.New("scriptcrawler: refusing to remove directory")
|
||||
}
|
||||
if err := os.Remove(videoPath); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
removeThumbCandidates(d.ThumbPath, strings.TrimSuffix(fileID, filepath.Ext(fileID)))
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeThumbCandidates(pathFor func(string) (string, error), stem string) {
|
||||
stem = strings.TrimSpace(stem)
|
||||
if stem == "" {
|
||||
return
|
||||
}
|
||||
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
|
||||
path, err := pathFor(stem + ext)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}
|
||||
|
||||
func safeJoin(root, fileID string) (string, error) {
|
||||
id := strings.TrimSpace(fileID)
|
||||
if id == "" || filepath.Base(id) != id {
|
||||
return "", errors.New("scriptcrawler: invalid file id")
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return "", errors.New("scriptcrawler: empty root")
|
||||
}
|
||||
rootAbs, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
pathAbs, err := filepath.Abs(filepath.Join(rootAbs, id))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if pathAbs != rootAbs && !strings.HasPrefix(pathAbs, rootAbs+string(os.PathSeparator)) {
|
||||
return "", errors.New("scriptcrawler: file id escapes root")
|
||||
}
|
||||
return pathAbs, nil
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
@@ -0,0 +1,386 @@
|
||||
package scriptcrawler
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DryRun 在不入库的前提下试跑一个爬虫脚本:临时目录里生成 job.json,
|
||||
// 启动脚本进程,拿到第一条(或前 MaxItems 条)item 事件后立即停止,
|
||||
// 再对视频直链做一次小范围探测,验证脚本"能不能爬取到视频"。
|
||||
// 用于后台导入脚本后的"测试脚本"按钮。
|
||||
|
||||
const (
|
||||
defaultDryRunTimeout = 2 * time.Minute
|
||||
dryRunLogTailLines = 60
|
||||
dryRunMediaProbeLimit = 20 * time.Second
|
||||
dryRunStopGrace = 100 * time.Millisecond
|
||||
)
|
||||
|
||||
type DryRunConfig struct {
|
||||
PythonPath string
|
||||
ScriptPath string
|
||||
ProxyURL string
|
||||
ConfigJSON string
|
||||
// MaxItems 收到多少条 item 后停止脚本,默认 1。
|
||||
MaxItems int
|
||||
// Timeout 整个试跑的硬上限,默认 2 分钟。
|
||||
Timeout time.Duration
|
||||
// SkipMediaProbe 跳过视频直链可达性探测(单测注入用)。
|
||||
SkipMediaProbe bool
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
type DryRunItem struct {
|
||||
Title string `json:"title"`
|
||||
SourceID string `json:"sourceId,omitempty"`
|
||||
MediaURL string `json:"mediaUrl,omitempty"`
|
||||
MediaLocalFile string `json:"mediaLocalFile,omitempty"`
|
||||
ThumbnailURL string `json:"thumbnailUrl,omitempty"`
|
||||
DetailURL string `json:"detailUrl,omitempty"`
|
||||
}
|
||||
|
||||
type DryRunMediaCheck struct {
|
||||
OK bool `json:"ok"`
|
||||
Status int `json:"status,omitempty"`
|
||||
ContentType string `json:"contentType,omitempty"`
|
||||
ContentLength int64 `json:"contentLengthBytes,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type DryRunResult struct {
|
||||
OK bool `json:"ok"`
|
||||
Items []DryRunItem `json:"items"`
|
||||
MediaCheck *DryRunMediaCheck `json:"mediaCheck,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Log []string `json:"log,omitempty"`
|
||||
DurationMs int64 `json:"durationMs"`
|
||||
}
|
||||
|
||||
func DryRun(ctx context.Context, cfg DryRunConfig) *DryRunResult {
|
||||
started := time.Now()
|
||||
result := &DryRunResult{Items: []DryRunItem{}}
|
||||
defer func() { result.DurationMs = time.Since(started).Milliseconds() }()
|
||||
|
||||
scriptPath := strings.TrimSpace(cfg.ScriptPath)
|
||||
if scriptPath == "" {
|
||||
result.Error = "脚本路径为空,请先导入脚本"
|
||||
return result
|
||||
}
|
||||
if _, err := os.Stat(scriptPath); err != nil {
|
||||
result.Error = fmt.Sprintf("脚本不存在: %v", err)
|
||||
return result
|
||||
}
|
||||
pythonPath := strings.TrimSpace(cfg.PythonPath)
|
||||
if pythonPath == "" {
|
||||
pythonPath = "python3"
|
||||
}
|
||||
maxItems := cfg.MaxItems
|
||||
if maxItems <= 0 {
|
||||
maxItems = 1
|
||||
}
|
||||
timeout := cfg.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = defaultDryRunTimeout
|
||||
}
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "crawler-dryrun-")
|
||||
if err != nil {
|
||||
result.Error = fmt.Sprintf("创建临时目录失败: %v", err)
|
||||
return result
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
outputDir := filepath.Join(tmpDir, "output")
|
||||
if err := os.MkdirAll(outputDir, 0o755); err != nil {
|
||||
result.Error = fmt.Sprintf("创建输出目录失败: %v", err)
|
||||
return result
|
||||
}
|
||||
seenPath := filepath.Join(tmpDir, "seen.txt")
|
||||
if err := os.WriteFile(seenPath, nil, 0o644); err != nil {
|
||||
result.Error = fmt.Sprintf("写入 seen 文件失败: %v", err)
|
||||
return result
|
||||
}
|
||||
|
||||
configJSON := json.RawMessage([]byte("{}"))
|
||||
if raw := strings.TrimSpace(cfg.ConfigJSON); raw != "" {
|
||||
if !json.Valid([]byte(raw)) {
|
||||
result.Error = "自定义配置必须是合法 JSON"
|
||||
return result
|
||||
}
|
||||
configJSON = json.RawMessage(raw)
|
||||
}
|
||||
job := Job{
|
||||
Protocol: "crawler.v1",
|
||||
Mode: "crawl",
|
||||
RunID: "dryrun-" + started.UTC().Format("20060102T150405Z"),
|
||||
CrawlerID: "dryrun",
|
||||
TargetNew: maxItems,
|
||||
SeenSourceIDsFile: seenPath,
|
||||
OutputDir: outputDir,
|
||||
Config: configJSON,
|
||||
Network: JobNetwork{ProxyURL: strings.TrimSpace(cfg.ProxyURL)},
|
||||
}
|
||||
jobPath := filepath.Join(tmpDir, "job.json")
|
||||
jobData, err := json.MarshalIndent(job, "", " ")
|
||||
if err != nil {
|
||||
result.Error = fmt.Sprintf("生成 job 文件失败: %v", err)
|
||||
return result
|
||||
}
|
||||
if err := os.WriteFile(jobPath, jobData, 0o600); err != nil {
|
||||
result.Error = fmt.Sprintf("写入 job 文件失败: %v", err)
|
||||
return result
|
||||
}
|
||||
|
||||
runCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(runCtx, pythonPath, scriptPath, "--job", jobPath)
|
||||
cmd.Dir = filepath.Dir(scriptPath)
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
cmd.Cancel = func() error {
|
||||
return killDryRunProcess(cmd)
|
||||
}
|
||||
// 超时或提前 kill 后,脚本派生的子进程可能仍持有 stdout/stderr 管道;
|
||||
// WaitDelay 强制在宽限期后关闭管道,避免读取端永久阻塞。
|
||||
cmd.WaitDelay = 3 * time.Second
|
||||
if proxyURL := strings.TrimSpace(cfg.ProxyURL); proxyURL != "" {
|
||||
cmd.Env = append(os.Environ(),
|
||||
"HTTP_PROXY="+proxyURL,
|
||||
"HTTPS_PROXY="+proxyURL,
|
||||
"http_proxy="+proxyURL,
|
||||
"https_proxy="+proxyURL,
|
||||
"NO_PROXY=",
|
||||
"no_proxy=",
|
||||
)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
result.Error = fmt.Sprintf("启动脚本失败: %v", err)
|
||||
return result
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
_ = stdout.Close()
|
||||
result.Error = fmt.Sprintf("启动脚本失败: %v", err)
|
||||
return result
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
_ = stdout.Close()
|
||||
_ = stderr.Close()
|
||||
result.Error = fmt.Sprintf("启动脚本失败: %v", err)
|
||||
return result
|
||||
}
|
||||
|
||||
// stderr 是脚本日志,保留尾部若干行用于排错回显。
|
||||
var logMu sync.Mutex
|
||||
logTail := make([]string, 0, dryRunLogTailLines)
|
||||
stderrDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(stderrDone)
|
||||
scanner := bufio.NewScanner(stderr)
|
||||
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
logMu.Lock()
|
||||
if len(logTail) >= dryRunLogTailLines {
|
||||
logTail = logTail[1:]
|
||||
}
|
||||
logTail = append(logTail, line)
|
||||
logMu.Unlock()
|
||||
}
|
||||
}()
|
||||
|
||||
items := []DryRunItem{}
|
||||
var firstMediaHeaders map[string]string
|
||||
parseFailures := 0
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
scanner.Buffer(make([]byte, 64*1024), 4*1024*1024)
|
||||
for scanner.Scan() {
|
||||
if runCtx.Err() != nil {
|
||||
break
|
||||
}
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var event Event
|
||||
if err := json.Unmarshal([]byte(line), &event); err != nil {
|
||||
parseFailures++
|
||||
continue
|
||||
}
|
||||
eventType := strings.ToLower(strings.TrimSpace(event.Type))
|
||||
item := event.normalizedItem()
|
||||
if eventType == "" && item.hasPayload() {
|
||||
eventType = "item"
|
||||
}
|
||||
if eventType != "item" {
|
||||
continue
|
||||
}
|
||||
normalized, _, err := normalizeItemForImport(item)
|
||||
if err != nil {
|
||||
result.Error = fmt.Sprintf("item 字段不完整: %v", err)
|
||||
continue
|
||||
}
|
||||
mediaURL := strings.TrimSpace(normalized.Media.URL)
|
||||
if len(items) == 0 {
|
||||
firstMediaHeaders = normalized.Media.Headers
|
||||
}
|
||||
items = append(items, DryRunItem{
|
||||
Title: strings.TrimSpace(normalized.Title),
|
||||
SourceID: strings.TrimSpace(item.SourceID),
|
||||
MediaURL: mediaURL,
|
||||
MediaLocalFile: strings.TrimSpace(normalized.Media.LocalFile),
|
||||
ThumbnailURL: strings.TrimSpace(normalized.Thumbnail.URL),
|
||||
DetailURL: strings.TrimSpace(normalized.DetailURL),
|
||||
})
|
||||
if len(items) >= maxItems {
|
||||
break
|
||||
}
|
||||
}
|
||||
// 拿够了就停掉脚本,避免它继续翻页。给已经自然结束的脚本一个很短
|
||||
// 的宽限期,让 stderr 日志先被管道读完,避免 dry-run 回显偶发为空。
|
||||
waitDone := make(chan struct{})
|
||||
go func() {
|
||||
_ = cmd.Wait()
|
||||
close(waitDone)
|
||||
}()
|
||||
select {
|
||||
case <-waitDone:
|
||||
case <-time.After(dryRunStopGrace):
|
||||
_ = killDryRunProcess(cmd)
|
||||
<-waitDone
|
||||
}
|
||||
<-stderrDone
|
||||
|
||||
logMu.Lock()
|
||||
result.Log = append([]string{}, logTail...)
|
||||
logMu.Unlock()
|
||||
result.Items = items
|
||||
|
||||
if len(items) == 0 {
|
||||
if result.Error == "" {
|
||||
switch {
|
||||
case runCtx.Err() != nil && ctx.Err() == nil:
|
||||
result.Error = fmt.Sprintf("测试超时(%s),脚本没有输出任何视频", timeout)
|
||||
case parseFailures > 0:
|
||||
result.Error = "脚本 stdout 不是合法的 crawler.v1 JSON Lines(日志应输出到 stderr)"
|
||||
default:
|
||||
result.Error = "脚本退出但没有输出任何视频"
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
result.Error = ""
|
||||
|
||||
first := items[0]
|
||||
switch {
|
||||
case cfg.SkipMediaProbe:
|
||||
result.OK = true
|
||||
case first.MediaLocalFile != "":
|
||||
// 脚本自己下载到 output_dir 的模式:试跑用的是临时目录,
|
||||
// 文件已随目录清理,能输出合法 local_file 即视为通过。
|
||||
result.OK = true
|
||||
default:
|
||||
check := probeMediaURL(ctx, cfg, first, firstMediaHeaders)
|
||||
result.MediaCheck = check
|
||||
result.OK = check.OK
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func killDryRunProcess(cmd *exec.Cmd) error {
|
||||
if cmd == nil || cmd.Process == nil {
|
||||
return nil
|
||||
}
|
||||
if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil {
|
||||
if err == syscall.ESRCH {
|
||||
return nil
|
||||
}
|
||||
return cmd.Process.Kill()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// probeMediaURL 对视频直链发一个 Range: bytes=0-0 的小请求,
|
||||
// 验证直链可达(带上脚本给的防盗链 headers 和代理)。
|
||||
func probeMediaURL(ctx context.Context, cfg DryRunConfig, item DryRunItem, mediaHeaders map[string]string) *DryRunMediaCheck {
|
||||
check := &DryRunMediaCheck{}
|
||||
if item.MediaURL == "" {
|
||||
check.Error = "item 没有视频直链"
|
||||
return check
|
||||
}
|
||||
|
||||
client := cfg.HTTPClient
|
||||
if client == nil {
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
ResponseHeaderTimeout: dryRunMediaProbeLimit,
|
||||
}
|
||||
if err := configureExplicitProxy(transport, cfg.ProxyURL); err != nil {
|
||||
check.Error = fmt.Sprintf("代理配置无效: %v", err)
|
||||
return check
|
||||
}
|
||||
client = &http.Client{Transport: transport}
|
||||
}
|
||||
|
||||
probeCtx, cancel := context.WithTimeout(ctx, dryRunMediaProbeLimit)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(probeCtx, http.MethodGet, item.MediaURL, nil)
|
||||
if err != nil {
|
||||
check.Error = fmt.Sprintf("视频直链无效: %v", err)
|
||||
return check
|
||||
}
|
||||
req.Header.Set("User-Agent", defaultUserAgent)
|
||||
req.Header.Set("Range", "bytes=0-0")
|
||||
if item.DetailURL != "" {
|
||||
req.Header.Set("Referer", item.DetailURL)
|
||||
}
|
||||
for k, v := range mediaHeaders {
|
||||
k = strings.TrimSpace(k)
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
check.Error = fmt.Sprintf("视频直链请求失败: %v", err)
|
||||
return check
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
check.Status = resp.StatusCode
|
||||
check.ContentType = resp.Header.Get("Content-Type")
|
||||
if cr := resp.Header.Get("Content-Range"); cr != "" {
|
||||
// Content-Range: bytes 0-0/12345 → 取总大小
|
||||
if idx := strings.LastIndex(cr, "/"); idx >= 0 {
|
||||
var total int64
|
||||
if _, err := fmt.Sscanf(cr[idx+1:], "%d", &total); err == nil {
|
||||
check.ContentLength = total
|
||||
}
|
||||
}
|
||||
}
|
||||
if check.ContentLength == 0 && resp.StatusCode == http.StatusOK {
|
||||
check.ContentLength = resp.ContentLength
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||
check.Error = fmt.Sprintf("视频直链返回 HTTP %d", resp.StatusCode)
|
||||
return check
|
||||
}
|
||||
check.OK = true
|
||||
return check
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package scriptcrawler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func writeDryRunScript(t *testing.T, body string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "crawler.sh")
|
||||
if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body), 0o755); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestDryRunCollectsFirstItem(t *testing.T) {
|
||||
script := writeDryRunScript(t, `
|
||||
echo '[log] fetching list page' >&2
|
||||
echo '{"type":"item","item":{"title":"Test Video","media_url":"https://cdn.example.test/v.mp4","source_id":"123","thumbnail_url":"https://cdn.example.test/t.jpg"}}'
|
||||
echo '{"type":"done","stats":{"emitted":1}}'
|
||||
`)
|
||||
result := DryRun(context.Background(), DryRunConfig{
|
||||
PythonPath: "/bin/sh",
|
||||
ScriptPath: script,
|
||||
SkipMediaProbe: true,
|
||||
})
|
||||
if !result.OK {
|
||||
t.Fatalf("ok = false, error = %q, log = %v", result.Error, result.Log)
|
||||
}
|
||||
if len(result.Items) != 1 {
|
||||
t.Fatalf("items = %d, want 1", len(result.Items))
|
||||
}
|
||||
item := result.Items[0]
|
||||
if item.Title != "Test Video" || item.MediaURL != "https://cdn.example.test/v.mp4" || item.SourceID != "123" {
|
||||
t.Fatalf("item = %+v", item)
|
||||
}
|
||||
if len(result.Log) == 0 || !strings.Contains(result.Log[0], "fetching list page") {
|
||||
t.Fatalf("log tail = %v, want stderr captured", result.Log)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDryRunProbesMediaURL(t *testing.T) {
|
||||
var gotRange, gotReferer string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotRange = r.Header.Get("Range")
|
||||
gotReferer = r.Header.Get("Referer")
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
w.Header().Set("Content-Range", "bytes 0-0/4096")
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write([]byte("x"))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
script := writeDryRunScript(t, fmt.Sprintf(
|
||||
`echo '{"type":"item","title":"Probe Video","media_url":"%s/v.mp4","detail_url":"https://example.test/view"}'`,
|
||||
srv.URL,
|
||||
))
|
||||
result := DryRun(context.Background(), DryRunConfig{
|
||||
PythonPath: "/bin/sh",
|
||||
ScriptPath: script,
|
||||
})
|
||||
if !result.OK {
|
||||
t.Fatalf("ok = false, error = %q, mediaCheck = %+v", result.Error, result.MediaCheck)
|
||||
}
|
||||
if result.MediaCheck == nil || !result.MediaCheck.OK {
|
||||
t.Fatalf("mediaCheck = %+v, want ok", result.MediaCheck)
|
||||
}
|
||||
if result.MediaCheck.Status != http.StatusPartialContent || result.MediaCheck.ContentLength != 4096 {
|
||||
t.Fatalf("mediaCheck = %+v, want 206 with total 4096", result.MediaCheck)
|
||||
}
|
||||
if gotRange != "bytes=0-0" || gotReferer != "https://example.test/view" {
|
||||
t.Fatalf("probe headers range=%q referer=%q", gotRange, gotReferer)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDryRunReportsBrokenMediaURL(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "forbidden", http.StatusForbidden)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
script := writeDryRunScript(t, fmt.Sprintf(
|
||||
`echo '{"type":"item","title":"Dead Link","media_url":"%s/v.mp4"}'`,
|
||||
srv.URL,
|
||||
))
|
||||
result := DryRun(context.Background(), DryRunConfig{
|
||||
PythonPath: "/bin/sh",
|
||||
ScriptPath: script,
|
||||
})
|
||||
if result.OK {
|
||||
t.Fatal("ok = true, want false for HTTP 403 media url")
|
||||
}
|
||||
if result.MediaCheck == nil || result.MediaCheck.OK || result.MediaCheck.Status != http.StatusForbidden {
|
||||
t.Fatalf("mediaCheck = %+v, want failed 403", result.MediaCheck)
|
||||
}
|
||||
if len(result.Items) != 1 {
|
||||
t.Fatalf("items = %d, want item still returned for debugging", len(result.Items))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDryRunRejectsNonJSONStdout(t *testing.T) {
|
||||
script := writeDryRunScript(t, `echo 'plain text progress output'`)
|
||||
result := DryRun(context.Background(), DryRunConfig{
|
||||
PythonPath: "/bin/sh",
|
||||
ScriptPath: script,
|
||||
SkipMediaProbe: true,
|
||||
})
|
||||
if result.OK {
|
||||
t.Fatal("ok = true, want false for non-JSON stdout")
|
||||
}
|
||||
if !strings.Contains(result.Error, "JSON Lines") {
|
||||
t.Fatalf("error = %q, want JSON Lines hint", result.Error)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDryRunTimesOut(t *testing.T) {
|
||||
script := writeDryRunScript(t, `sleep 30`)
|
||||
start := time.Now()
|
||||
result := DryRun(context.Background(), DryRunConfig{
|
||||
PythonPath: "/bin/sh",
|
||||
ScriptPath: script,
|
||||
Timeout: 2 * time.Second,
|
||||
SkipMediaProbe: true,
|
||||
})
|
||||
if result.OK {
|
||||
t.Fatal("ok = true, want false on timeout")
|
||||
}
|
||||
if !strings.Contains(result.Error, "超时") {
|
||||
t.Fatalf("error = %q, want timeout message", result.Error)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed > 10*time.Second {
|
||||
t.Fatalf("dry run took %s, script was not killed", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDryRunMissingScript(t *testing.T) {
|
||||
result := DryRun(context.Background(), DryRunConfig{
|
||||
PythonPath: "/bin/sh",
|
||||
ScriptPath: filepath.Join(t.TempDir(), "missing.py"),
|
||||
})
|
||||
if result.OK || result.Error == "" {
|
||||
t.Fatalf("result = %+v, want error for missing script", result)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package scriptcrawler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const maxCrawlerNameRunes = 80
|
||||
|
||||
type Metadata struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
func ReadMetadata(scriptPath string) (Metadata, error) {
|
||||
scriptPath = strings.TrimSpace(scriptPath)
|
||||
if scriptPath == "" {
|
||||
return Metadata{}, errors.New("脚本路径为空")
|
||||
}
|
||||
if filepath.Ext(scriptPath) != ".py" {
|
||||
return Metadata{}, errors.New("目前只支持 .py 爬虫脚本")
|
||||
}
|
||||
data, err := os.ReadFile(scriptPath)
|
||||
if err != nil {
|
||||
return Metadata{}, fmt.Errorf("读取脚本失败: %w", err)
|
||||
}
|
||||
return ExtractMetadata(string(data))
|
||||
}
|
||||
|
||||
func ExtractMetadata(source string) (Metadata, error) {
|
||||
for _, line := range strings.Split(source, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" || strings.HasPrefix(trimmed, "#") {
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(trimmed, "CRAWLER_NAME") {
|
||||
continue
|
||||
}
|
||||
left, right, ok := strings.Cut(trimmed, "=")
|
||||
if !ok || strings.TrimSpace(left) != "CRAWLER_NAME" {
|
||||
continue
|
||||
}
|
||||
name, ok := parsePythonStringLiteral(right)
|
||||
if !ok {
|
||||
return Metadata{}, errors.New(`CRAWLER_NAME 必须是字符串字面量,例如 CRAWLER_NAME = "示例爬虫"`)
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return Metadata{}, errors.New("CRAWLER_NAME 不能为空")
|
||||
}
|
||||
if len([]rune(name)) > maxCrawlerNameRunes {
|
||||
return Metadata{}, fmt.Errorf("CRAWLER_NAME 不能超过 %d 个字符", maxCrawlerNameRunes)
|
||||
}
|
||||
return Metadata{Name: name}, nil
|
||||
}
|
||||
return Metadata{}, errors.New(`脚本必须声明 CRAWLER_NAME,例如 CRAWLER_NAME = "示例爬虫"`)
|
||||
}
|
||||
|
||||
func parsePythonStringLiteral(raw string) (string, bool) {
|
||||
s := strings.TrimSpace(raw)
|
||||
if s == "" {
|
||||
return "", false
|
||||
}
|
||||
rawString := false
|
||||
for len(s) > 0 {
|
||||
switch s[0] {
|
||||
case 'r', 'R':
|
||||
rawString = true
|
||||
s = strings.TrimSpace(s[1:])
|
||||
case 'u', 'U', 'b', 'B':
|
||||
s = strings.TrimSpace(s[1:])
|
||||
default:
|
||||
goto parseQuote
|
||||
}
|
||||
}
|
||||
|
||||
parseQuote:
|
||||
if len(s) < 2 || (s[0] != '"' && s[0] != '\'') {
|
||||
return "", false
|
||||
}
|
||||
quote := s[0]
|
||||
var b strings.Builder
|
||||
escaped := false
|
||||
for i := 1; i < len(s); i++ {
|
||||
ch := s[i]
|
||||
if escaped {
|
||||
switch {
|
||||
case rawString:
|
||||
b.WriteByte('\\')
|
||||
b.WriteByte(ch)
|
||||
case ch == 'n':
|
||||
b.WriteByte('\n')
|
||||
case ch == 'r':
|
||||
b.WriteByte('\r')
|
||||
case ch == 't':
|
||||
b.WriteByte('\t')
|
||||
case ch == '\\' || ch == quote || ch == '"' || ch == '\'':
|
||||
b.WriteByte(ch)
|
||||
default:
|
||||
b.WriteByte(ch)
|
||||
}
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
if ch == '\\' {
|
||||
escaped = true
|
||||
continue
|
||||
}
|
||||
if ch == quote {
|
||||
return b.String(), true
|
||||
}
|
||||
b.WriteByte(ch)
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package scriptcrawler
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestExtractMetadataReadsCrawlerName(t *testing.T) {
|
||||
meta, err := ExtractMetadata(`
|
||||
# comment
|
||||
CRAWLER_NAME = "示例爬虫"
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("extract metadata: %v", err)
|
||||
}
|
||||
if meta.Name != "示例爬虫" {
|
||||
t.Fatalf("name = %q", meta.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractMetadataRejectsMissingCrawlerName(t *testing.T) {
|
||||
_, err := ExtractMetadata(`print("hello")`)
|
||||
if err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "CRAWLER_NAME") {
|
||||
t.Fatalf("error = %v, want CRAWLER_NAME guidance", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractMetadataRejectsEmptyCrawlerName(t *testing.T) {
|
||||
_, err := ExtractMetadata(`CRAWLER_NAME = " "`)
|
||||
if err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "不能为空") {
|
||||
t.Fatalf("error = %v, want empty-name error", err)
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,12 @@ type CrawlerConfig struct {
|
||||
|
||||
// OnNewVideo 是新视频成功入库后的回调,用于触发预览视频 worker。
|
||||
OnNewVideo func(v *catalog.Video)
|
||||
// OnProgress 在抓取统计变化时触发,用于后台管理页展示实时进度。
|
||||
OnProgress func(progress CrawlProgress)
|
||||
// OnCheckedVideo 在 Python 爬虫开始检查一个列表页视频时触发。
|
||||
OnCheckedVideo func()
|
||||
// OnExtractedVideo 在 Python 爬虫提取到一个新视频直链时触发。
|
||||
OnExtractedVideo func()
|
||||
}
|
||||
|
||||
// Crawler 把 Python 爬虫产出包装成 catalog 入库流程。
|
||||
@@ -219,6 +225,16 @@ type CrawlResult struct {
|
||||
SeenFile string
|
||||
}
|
||||
|
||||
// CrawlProgress 是 RunOnce 过程中可安全对外发布的实时计数。
|
||||
type CrawlProgress struct {
|
||||
TargetNew int
|
||||
TotalEntries int
|
||||
NewVideos int
|
||||
Skipped int
|
||||
Failed int
|
||||
SeenSnapshot int
|
||||
}
|
||||
|
||||
// spiderVideoEntry 对应 spider_91porn.py 输出 JSON 中的单条视频。
|
||||
type spiderVideoEntry struct {
|
||||
Title string `json:"title"`
|
||||
@@ -266,6 +282,20 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
|
||||
result := &CrawlResult{TargetNew: targetNew, StartedAt: time.Now()}
|
||||
defer func() { result.FinishedAt = time.Now() }()
|
||||
emitProgress := func() {
|
||||
if c.cfg.OnProgress == nil {
|
||||
return
|
||||
}
|
||||
c.cfg.OnProgress(CrawlProgress{
|
||||
TargetNew: result.TargetNew,
|
||||
TotalEntries: result.TotalEntries,
|
||||
NewVideos: result.NewVideos,
|
||||
Skipped: result.Skipped,
|
||||
Failed: result.Failed,
|
||||
SeenSnapshot: result.SeenSnapshot,
|
||||
})
|
||||
}
|
||||
emitProgress()
|
||||
|
||||
// 1. 准备 .crawl/ 目录 + 已知源视频 ID 列表
|
||||
//
|
||||
@@ -291,6 +321,7 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
return result, fmt.Errorf("spider91 crawler: build seen list: %w", err)
|
||||
}
|
||||
result.SeenSnapshot = seenCount
|
||||
emitProgress()
|
||||
|
||||
// 2-3. 启动 Python 爬虫(流式 stdout 协议),并边读边处理。
|
||||
//
|
||||
@@ -321,9 +352,11 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
continue
|
||||
}
|
||||
result.TotalEntries++
|
||||
emitProgress()
|
||||
sourceID := sourceIDForItem(item)
|
||||
if sourceID == "" || strings.TrimSpace(item.VideoURL) == "" {
|
||||
result.Failed++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
if result.NewVideos >= targetNew {
|
||||
@@ -335,22 +368,27 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
if err != nil {
|
||||
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s check deleted: %v", c.cfg.Driver.ID(), item.Viewkey, sourceID, err)
|
||||
result.Failed++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
if deleted {
|
||||
result.Skipped++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
if existing, _ := c.cfg.Catalog.GetVideo(ctx, videoID); existing != nil {
|
||||
result.Skipped++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
if perr := c.processOne(ctx, videoID, item); perr != nil {
|
||||
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s failed: %v", c.cfg.Driver.ID(), item.Viewkey, sourceID, perr)
|
||||
result.Failed++
|
||||
emitProgress()
|
||||
continue
|
||||
}
|
||||
result.NewVideos++
|
||||
emitProgress()
|
||||
}
|
||||
if scerr := scanner.Err(); scerr != nil {
|
||||
log.Printf("[spider91] drive=%s stdout scan: %v", c.cfg.Driver.ID(), scerr)
|
||||
@@ -458,12 +496,12 @@ func (c *Crawler) startSpiderTargetNew(ctx context.Context, targetNew int, seenP
|
||||
return nil, nil, fmt.Errorf("start: %w", err)
|
||||
}
|
||||
// stderr 转发到 backend log。子进程退出时 reader 自动 EOF,goroutine 自然结束。
|
||||
go forwardSpiderLog(c.cfg.Driver.ID(), stderr)
|
||||
go forwardSpiderLog(c.cfg.Driver.ID(), stderr, c.cfg.OnCheckedVideo, c.cfg.OnExtractedVideo)
|
||||
return cmd, stdout, nil
|
||||
}
|
||||
|
||||
// forwardSpiderLog 把 Python stderr 逐行转发到 backend log,便于调试。
|
||||
func forwardSpiderLog(driveID string, r io.Reader) {
|
||||
func forwardSpiderLog(driveID string, r io.Reader, onCheckedVideo func(), onExtractedVideo func()) {
|
||||
scanner := bufio.NewScanner(r)
|
||||
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
@@ -472,9 +510,23 @@ func forwardSpiderLog(driveID string, r io.Reader) {
|
||||
continue
|
||||
}
|
||||
log.Printf("[spider91:py] drive=%s %s", driveID, line)
|
||||
if onCheckedVideo != nil && isSpider91CheckedVideoLogLine(line) {
|
||||
onCheckedVideo()
|
||||
}
|
||||
if onExtractedVideo != nil && isSpider91ExtractedVideoLogLine(line) {
|
||||
onExtractedVideo()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isSpider91CheckedVideoLogLine(line string) bool {
|
||||
return checkedVideoLogRE.MatchString(line)
|
||||
}
|
||||
|
||||
func isSpider91ExtractedVideoLogLine(line string) bool {
|
||||
return strings.Contains(line, "[OK] 成功提取视频直链")
|
||||
}
|
||||
|
||||
// processOne 处理单个 91 源视频:下载视频 + 封面 + 复制封面 + 入库。
|
||||
// 任一步失败会清理已写入的临时文件,不留半成品。
|
||||
func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVideoEntry) error {
|
||||
@@ -847,9 +899,10 @@ func spider91CookieHeader(cookies []*http.Cookie) string {
|
||||
}
|
||||
|
||||
var (
|
||||
strencode2RE = regexp.MustCompile(`strencode2\(["']([^"']+)["']\)`)
|
||||
srcAttrRE = regexp.MustCompile(`src=['"]([^'"]+)['"]`)
|
||||
mp4URLRE = regexp.MustCompile(`https?://[^\s"'<>]+\.mp4[^\s"'<>]*`)
|
||||
checkedVideoLogRE = regexp.MustCompile(`处理视频\s+\d+/\d+:`)
|
||||
strencode2RE = regexp.MustCompile(`strencode2\(["']([^"']+)["']\)`)
|
||||
srcAttrRE = regexp.MustCompile(`src=['"]([^'"]+)['"]`)
|
||||
mp4URLRE = regexp.MustCompile(`https?://[^\s"'<>]+\.mp4[^\s"'<>]*`)
|
||||
)
|
||||
|
||||
func parseSpider91VideoURL(html string) string {
|
||||
|
||||
@@ -707,6 +707,18 @@ func TestSpider91CookieHeader(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpider91ProgressLogLineClassifiers(t *testing.T) {
|
||||
if !isSpider91CheckedVideoLogLine("[2026-06-08 16:49:17] 处理视频 3/24: 标题") {
|
||||
t.Fatal("checked video log line was not recognized")
|
||||
}
|
||||
if isSpider91CheckedVideoLogLine("[2026-06-08 16:49:17] [页 2] 发现 24 个视频") {
|
||||
t.Fatal("page summary log line should not count as checked video")
|
||||
}
|
||||
if !isSpider91ExtractedVideoLogLine("[2026-06-08 16:49:39] [OK] 成功提取视频直链") {
|
||||
t.Fatal("extracted video log line was not recognized")
|
||||
}
|
||||
}
|
||||
|
||||
func spider91DetailHTML(videoURL string) string {
|
||||
fragment := `<video><source src="` + videoURL + `" type="video/mp4"></video>`
|
||||
return `document.write(strencode2("` + url.PathEscape(fragment) + `"));`
|
||||
|
||||
@@ -167,6 +167,46 @@ func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, er
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
videoPath, err := d.VideoPath(fileID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Stat(videoPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
removeThumbCandidates(d.ThumbPath, strings.TrimSuffix(fileID, filepath.Ext(fileID)))
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return errors.New("spider91: refusing to remove directory")
|
||||
}
|
||||
if err := os.Remove(videoPath); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
removeThumbCandidates(d.ThumbPath, strings.TrimSuffix(fileID, filepath.Ext(fileID)))
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeThumbCandidates(pathFor func(string) (string, error), stem string) {
|
||||
stem = strings.TrimSpace(stem)
|
||||
if stem == "" {
|
||||
return
|
||||
}
|
||||
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
|
||||
path, err := pathFor(stem + ext)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}
|
||||
|
||||
// safeJoin 把 fileID 拼到 root 下,保证最终路径不会逃出 root。
|
||||
// fileID 必须是单纯的文件名(不含 / 或 .. 等组件)。
|
||||
func safeJoin(root, fileID string) (string, error) {
|
||||
@@ -192,3 +232,4 @@ func safeJoin(root, fileID string) (string, error) {
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
|
||||
@@ -2,19 +2,23 @@ package wopan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
sdk "github.com/OpenListTeam/wopan-sdk-go"
|
||||
"github.com/go-resty/resty/v2"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
// Driver 封装联通沃盘
|
||||
// Driver 封装联通网盘
|
||||
type Driver struct {
|
||||
id string
|
||||
rootID string
|
||||
@@ -23,6 +27,14 @@ type Driver struct {
|
||||
refreshToken string
|
||||
client *sdk.WoClient
|
||||
onTokenUpdate func(access, refresh string)
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
listInterval time.Duration
|
||||
listCooldown time.Duration
|
||||
|
||||
fileIDMu sync.RWMutex
|
||||
fidToID map[string]string
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -47,6 +59,9 @@ func New(c Config) *Driver {
|
||||
accessToken: c.AccessToken,
|
||||
refreshToken: c.RefreshToken,
|
||||
onTokenUpdate: c.OnTokenUpdate,
|
||||
listInterval: 800 * time.Millisecond,
|
||||
listCooldown: 5 * time.Minute,
|
||||
fidToID: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,15 +93,41 @@ func (d *Driver) spaceType() string {
|
||||
}
|
||||
|
||||
func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error) {
|
||||
d.listMu.Lock()
|
||||
defer d.listMu.Unlock()
|
||||
|
||||
var result []drives.Entry
|
||||
pageNum := 0
|
||||
pageSize := 100
|
||||
for {
|
||||
data, err := d.client.QueryAllFiles(d.spaceType(), dirID, pageNum, pageSize, 0, d.familyID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wopan list: %w", err)
|
||||
var data *sdk.QueryAllFilesData
|
||||
for attempt := 0; ; attempt++ {
|
||||
if err := d.waitForListSlotLocked(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var err error
|
||||
data, err = d.client.QueryAllFiles(d.spaceType(), dirID, pageNum, pageSize, 0, d.familyID, func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
err = wopanRequestError("list", err)
|
||||
wait, ok := drives.RateLimitRetryAfter(err)
|
||||
if !ok {
|
||||
return nil, err
|
||||
}
|
||||
if wait <= 0 {
|
||||
wait = d.listCooldown
|
||||
}
|
||||
log.Printf("[wopan] list cooling down drive=%s dir=%s page=%d cooldown=%s attempt=%d err=%v",
|
||||
d.id, dirID, pageNum, wait, attempt+1, err)
|
||||
if err := sleepContext(ctx, wait); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for _, f := range data.Files {
|
||||
d.rememberFileID(f)
|
||||
result = append(result, fileToEntry(f, dirID))
|
||||
}
|
||||
if len(data.Files) < pageSize {
|
||||
@@ -103,9 +144,11 @@ func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error)
|
||||
}
|
||||
|
||||
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
data, err := d.client.GetDownloadUrlV2([]string{fileID})
|
||||
data, err := d.client.GetDownloadUrlV2([]string{fileID}, func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wopan download url: %w", err)
|
||||
return nil, wopanRequestError("download url", err)
|
||||
}
|
||||
if len(data.List) == 0 {
|
||||
return nil, fmt.Errorf("wopan download url: empty response")
|
||||
@@ -142,9 +185,151 @@ func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader,
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("wopan upload: %w", err)
|
||||
}
|
||||
if fid != "" {
|
||||
if objectID, err := d.findDeleteFileIDInParent(ctx, parentID, drives.SourceFile{
|
||||
FileID: fid,
|
||||
Name: name,
|
||||
Size: size,
|
||||
}); err == nil {
|
||||
d.rememberFIDMapping(fid, objectID)
|
||||
} else {
|
||||
log.Printf("[wopan] upload drive=%s parent=%s fid=%s resolve object id: %v", d.id, parentID, fid, err)
|
||||
}
|
||||
}
|
||||
return fid, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
if d.client == nil {
|
||||
return fmt.Errorf("wopan rename: driver not initialized")
|
||||
}
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return fmt.Errorf("wopan rename: empty file id")
|
||||
}
|
||||
newName = strings.TrimSpace(newName)
|
||||
if newName == "" {
|
||||
return fmt.Errorf("wopan rename: empty new name")
|
||||
}
|
||||
renameID := fileID
|
||||
if cached := d.cachedDeleteFileID(fileID); cached != "" {
|
||||
renameID = cached
|
||||
}
|
||||
if err := d.client.RenameFileOrDirectory(d.spaceType(), 1, renameID, newName, d.familyID, func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
}); err != nil {
|
||||
return wopanRequestError("rename", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
if d.client == nil {
|
||||
return fmt.Errorf("wopan remove: driver not initialized")
|
||||
}
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return fmt.Errorf("wopan remove: empty file id")
|
||||
}
|
||||
deleteID := fileID
|
||||
if cached := d.cachedDeleteFileID(fileID); cached != "" {
|
||||
deleteID = cached
|
||||
}
|
||||
if err := d.deleteFileByObjectID(ctx, deleteID); err != nil {
|
||||
return fmt.Errorf("wopan remove: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) RemoveSource(ctx context.Context, source drives.SourceFile) error {
|
||||
if d.client == nil {
|
||||
return fmt.Errorf("wopan remove: driver not initialized")
|
||||
}
|
||||
fileID := strings.TrimSpace(source.FileID)
|
||||
if fileID == "" {
|
||||
return fmt.Errorf("wopan remove: empty file id")
|
||||
}
|
||||
deleteID, err := d.resolveDeleteFileID(ctx, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := d.deleteFileByObjectID(ctx, deleteID); err != nil {
|
||||
return fmt.Errorf("wopan remove: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) deleteFileByObjectID(ctx context.Context, fileID string) error {
|
||||
if err := d.client.DeleteFile(d.spaceType(), nil, []string{fileID}, func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) resolveDeleteFileID(ctx context.Context, source drives.SourceFile) (string, error) {
|
||||
fileID := strings.TrimSpace(source.FileID)
|
||||
if fileID == "" {
|
||||
return "", fmt.Errorf("wopan remove: empty file id")
|
||||
}
|
||||
if cached := d.cachedDeleteFileID(fileID); cached != "" {
|
||||
return cached, nil
|
||||
}
|
||||
parentID := strings.TrimSpace(source.ParentID)
|
||||
if parentID == "" {
|
||||
return fileID, nil
|
||||
}
|
||||
return d.findDeleteFileIDInParent(ctx, parentID, source)
|
||||
}
|
||||
|
||||
func (d *Driver) findDeleteFileIDInParent(ctx context.Context, parentID string, source drives.SourceFile) (string, error) {
|
||||
d.listMu.Lock()
|
||||
defer d.listMu.Unlock()
|
||||
|
||||
pageNum := 0
|
||||
pageSize := 100
|
||||
for {
|
||||
var data *sdk.QueryAllFilesData
|
||||
for attempt := 0; ; attempt++ {
|
||||
if err := d.waitForListSlotLocked(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var err error
|
||||
data, err = d.client.QueryAllFiles(d.spaceType(), parentID, pageNum, pageSize, 0, d.familyID, func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
err = wopanRequestError("resolve delete id", err)
|
||||
wait, ok := drives.RateLimitRetryAfter(err)
|
||||
if !ok {
|
||||
return "", err
|
||||
}
|
||||
if wait <= 0 {
|
||||
wait = d.listCooldown
|
||||
}
|
||||
log.Printf("[wopan] resolve delete id cooling down drive=%s parent=%s page=%d cooldown=%s attempt=%d err=%v",
|
||||
d.id, parentID, pageNum, wait, attempt+1, err)
|
||||
if err := sleepContext(ctx, wait); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
for _, f := range data.Files {
|
||||
d.rememberFileID(f)
|
||||
if id, ok := deleteFileIDFromWopanFile(f, source); ok {
|
||||
return id, nil
|
||||
}
|
||||
}
|
||||
if len(data.Files) < pageSize {
|
||||
break
|
||||
}
|
||||
pageNum++
|
||||
}
|
||||
return "", fmt.Errorf("wopan remove: source file %q not found under parent %q", source.FileID, parentID)
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
parts := splitPath(pathFromRoot)
|
||||
currentID := d.rootID
|
||||
@@ -154,9 +339,11 @@ func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, er
|
||||
return "", err
|
||||
}
|
||||
if childID == "" {
|
||||
resp, err := d.client.CreateDirectory(d.spaceType(), currentID, name, d.familyID)
|
||||
resp, err := d.client.CreateDirectory(d.spaceType(), currentID, name, d.familyID, func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("wopan mkdir %s: %w", name, err)
|
||||
return "", wopanRequestError("mkdir "+name, err)
|
||||
}
|
||||
childID = resp.Id
|
||||
}
|
||||
@@ -190,9 +377,12 @@ func fileToEntry(f *sdk.File, parentID string) drives.Entry {
|
||||
mod, _ := time.Parse("2006-01-02 15:04:05", f.CreateTime)
|
||||
name := f.Name
|
||||
isDir := f.Type == 0
|
||||
id := f.Fid
|
||||
id := f.Id
|
||||
if !isDir && f.Fid != "" {
|
||||
id = f.Fid
|
||||
}
|
||||
if id == "" {
|
||||
id = f.Id
|
||||
id = f.Fid
|
||||
}
|
||||
if isDir && !strings.HasSuffix(name, "/") {
|
||||
// 不改 name,只标志
|
||||
@@ -208,6 +398,128 @@ func fileToEntry(f *sdk.File, parentID string) drives.Entry {
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) rememberFileID(f *sdk.File) {
|
||||
if f == nil || f.Type == 0 {
|
||||
return
|
||||
}
|
||||
objectID := strings.TrimSpace(f.Id)
|
||||
fid := strings.TrimSpace(f.Fid)
|
||||
if objectID == "" {
|
||||
return
|
||||
}
|
||||
d.fileIDMu.Lock()
|
||||
if d.fidToID == nil {
|
||||
d.fidToID = make(map[string]string)
|
||||
}
|
||||
d.fidToID[objectID] = objectID
|
||||
if fid != "" {
|
||||
d.fidToID[fid] = objectID
|
||||
}
|
||||
d.fileIDMu.Unlock()
|
||||
}
|
||||
|
||||
func (d *Driver) rememberFIDMapping(fid, objectID string) {
|
||||
fid = strings.TrimSpace(fid)
|
||||
objectID = strings.TrimSpace(objectID)
|
||||
if fid == "" || objectID == "" {
|
||||
return
|
||||
}
|
||||
d.fileIDMu.Lock()
|
||||
if d.fidToID == nil {
|
||||
d.fidToID = make(map[string]string)
|
||||
}
|
||||
d.fidToID[fid] = objectID
|
||||
d.fidToID[objectID] = objectID
|
||||
d.fileIDMu.Unlock()
|
||||
}
|
||||
|
||||
func (d *Driver) cachedDeleteFileID(fileID string) string {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return ""
|
||||
}
|
||||
d.fileIDMu.RLock()
|
||||
defer d.fileIDMu.RUnlock()
|
||||
return strings.TrimSpace(d.fidToID[fileID])
|
||||
}
|
||||
|
||||
func deleteFileIDFromWopanFile(f *sdk.File, source drives.SourceFile) (string, bool) {
|
||||
if f == nil || f.Type == 0 {
|
||||
return "", false
|
||||
}
|
||||
sourceID := strings.TrimSpace(source.FileID)
|
||||
if sourceID == "" {
|
||||
return "", false
|
||||
}
|
||||
objectID := strings.TrimSpace(f.Id)
|
||||
fid := strings.TrimSpace(f.Fid)
|
||||
if objectID == "" {
|
||||
return "", false
|
||||
}
|
||||
if sourceID != objectID && sourceID != fid {
|
||||
return "", false
|
||||
}
|
||||
return objectID, true
|
||||
}
|
||||
|
||||
func (d *Driver) waitForListSlotLocked(ctx context.Context) error {
|
||||
if d.listInterval <= 0 || d.lastListAt.IsZero() {
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
next := d.lastListAt.Add(d.listInterval)
|
||||
now := time.Now()
|
||||
if now.Before(next) {
|
||||
if err := sleepContext(ctx, next.Sub(now)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, d time.Duration) error {
|
||||
if d <= 0 {
|
||||
return ctx.Err()
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func wopanRequestError(step string, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
wrapped := fmt.Errorf("wopan %s: %w", step, err)
|
||||
if isWopanRateLimitError(err) {
|
||||
return &drives.RateLimitError{
|
||||
Provider: "wopan",
|
||||
Err: wrapped,
|
||||
}
|
||||
}
|
||||
return wrapped
|
||||
}
|
||||
|
||||
func isWopanRateLimitError(err error) bool {
|
||||
if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return false
|
||||
}
|
||||
return drives.ErrorMentionsHTTPStatus(err,
|
||||
http.StatusTooManyRequests,
|
||||
http.StatusInternalServerError,
|
||||
http.StatusBadGateway,
|
||||
http.StatusServiceUnavailable,
|
||||
http.StatusGatewayTimeout,
|
||||
509,
|
||||
)
|
||||
}
|
||||
|
||||
func guessMime(name string) string {
|
||||
ext := strings.ToLower(path.Ext(name))
|
||||
switch ext {
|
||||
@@ -229,3 +541,5 @@ func guessMime(name string) string {
|
||||
|
||||
// 确保实现接口
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
var _ drives.SourceRemover = (*Driver)(nil)
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
package wopan
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
sdk "github.com/OpenListTeam/wopan-sdk-go"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestFileToEntryUsesDirectoryIDAndFileFID(t *testing.T) {
|
||||
dir := fileToEntry(&sdk.File{
|
||||
Id: "dir-object-id",
|
||||
Fid: "0",
|
||||
Type: 0,
|
||||
Name: "collection",
|
||||
}, "root")
|
||||
if !dir.IsDir {
|
||||
t.Fatal("directory entry IsDir = false")
|
||||
}
|
||||
if dir.ID != "dir-object-id" {
|
||||
t.Fatalf("directory id = %q, want object id", dir.ID)
|
||||
}
|
||||
|
||||
file := fileToEntry(&sdk.File{
|
||||
Id: "file-object-id",
|
||||
Fid: "fid/with/slash",
|
||||
Type: 1,
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
}, "dir-object-id")
|
||||
if file.IsDir {
|
||||
t.Fatal("file entry IsDir = true")
|
||||
}
|
||||
if file.ID != "fid/with/slash" {
|
||||
t.Fatalf("file id = %q, want fid for download", file.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteFileIDFromWopanFileUsesObjectIDForFID(t *testing.T) {
|
||||
got, ok := deleteFileIDFromWopanFile(&sdk.File{
|
||||
Id: "file-object-id",
|
||||
Fid: "fid/with/slash",
|
||||
Type: 1,
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
}, drives.SourceFile{
|
||||
FileID: "fid/with/slash",
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
})
|
||||
if !ok {
|
||||
t.Fatal("delete file id not resolved")
|
||||
}
|
||||
if got != "file-object-id" {
|
||||
t.Fatalf("delete file id = %q, want object id", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteFileIDFromWopanFileAcceptsObjectID(t *testing.T) {
|
||||
got, ok := deleteFileIDFromWopanFile(&sdk.File{
|
||||
Id: "file-object-id",
|
||||
Fid: "fid-1",
|
||||
Type: 1,
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
}, drives.SourceFile{
|
||||
FileID: "file-object-id",
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
})
|
||||
if !ok {
|
||||
t.Fatal("delete file id not resolved")
|
||||
}
|
||||
if got != "file-object-id" {
|
||||
t.Fatalf("delete file id = %q, want object id", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteFileIDFromWopanFileRejectsIDMismatch(t *testing.T) {
|
||||
if _, ok := deleteFileIDFromWopanFile(&sdk.File{
|
||||
Id: "file-object-id",
|
||||
Fid: "fid-1",
|
||||
Type: 1,
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
}, drives.SourceFile{
|
||||
FileID: "other-fid",
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
}); ok {
|
||||
t.Fatal("delete file id resolved despite id mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWopanRequestErrorWrapsRateLimit(t *testing.T) {
|
||||
err := wopanRequestError("list", errors.New("request failed with status: 429 Too Many Requests"))
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.Provider != "wopan" {
|
||||
t.Fatalf("provider = %q, want wopan", rateLimit.Provider)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWopanRequestErrorLeavesNormalErrors(t *testing.T) {
|
||||
err := wopanRequestError("download url", errors.New("invalid access token"))
|
||||
var rateLimit *drives.RateLimitError
|
||||
if errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want non-rate-limit error", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
package wopan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-resty/resty/v2"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultQRCodeAPIBase = "https://panservice.mail.wo.cn/wohome/open/v1/QRCode"
|
||||
defaultQRCodeClient = "1001000021"
|
||||
)
|
||||
|
||||
type QRConfig struct {
|
||||
APIBaseURL string
|
||||
HTTPClient *http.Client
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type QRClient struct {
|
||||
apiBase string
|
||||
client *resty.Client
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type QRCodeSession struct {
|
||||
UUID string `json:"uuid"`
|
||||
QRImageDataURL string `json:"qrImageDataUrl"`
|
||||
ExpiresAt string `json:"expiresAt,omitempty"`
|
||||
}
|
||||
|
||||
type QRCodeStatus struct {
|
||||
State int `json:"state"`
|
||||
StatusText string `json:"statusText"`
|
||||
AccessToken string `json:"accessToken,omitempty"`
|
||||
RefreshToken string `json:"refreshToken,omitempty"`
|
||||
FamilyID string `json:"familyID,omitempty"`
|
||||
}
|
||||
|
||||
func NewQRClient(c QRConfig) *QRClient {
|
||||
apiBase := strings.TrimRight(strings.TrimSpace(c.APIBaseURL), "/")
|
||||
if apiBase == "" {
|
||||
apiBase = defaultQRCodeAPIBase
|
||||
}
|
||||
httpClient := c.HTTPClient
|
||||
if httpClient == nil {
|
||||
httpClient = &http.Client{Timeout: 20 * time.Second}
|
||||
}
|
||||
now := c.Now
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
return &QRClient{
|
||||
apiBase: apiBase,
|
||||
client: resty.NewWithClient(httpClient).
|
||||
SetTimeout(20*time.Second).
|
||||
SetHeader("Accept", "application/json"),
|
||||
now: now,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *QRClient) Generate(ctx context.Context) (QRCodeSession, error) {
|
||||
var envelope qrEnvelope
|
||||
res, err := c.request(ctx).
|
||||
SetResult(&envelope).
|
||||
Get(c.apiBase + "/generate")
|
||||
if err != nil {
|
||||
return QRCodeSession{}, err
|
||||
}
|
||||
if res.IsError() {
|
||||
return QRCodeSession{}, qrAPIError(envelope.message(), res.StatusCode())
|
||||
}
|
||||
|
||||
var result qrGenerateResult
|
||||
if err := decodeResult(envelope.Result, &result); err != nil {
|
||||
return QRCodeSession{}, err
|
||||
}
|
||||
result.UUID = strings.TrimSpace(result.UUID)
|
||||
result.Image = strings.TrimSpace(result.Image)
|
||||
if result.UUID == "" {
|
||||
return QRCodeSession{}, errors.New("wopan qr: empty uuid")
|
||||
}
|
||||
if result.Image == "" {
|
||||
return QRCodeSession{}, errors.New("wopan qr: empty image")
|
||||
}
|
||||
return QRCodeSession{
|
||||
UUID: result.UUID,
|
||||
QRImageDataURL: qrImageDataURL(result.Image),
|
||||
ExpiresAt: c.now().Add(60 * time.Second).Format(time.RFC3339),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *QRClient) Poll(ctx context.Context, uuid string) (QRCodeStatus, error) {
|
||||
uuid = strings.TrimSpace(uuid)
|
||||
if uuid == "" {
|
||||
return QRCodeStatus{}, errors.New("uuid is required")
|
||||
}
|
||||
|
||||
var envelope qrEnvelope
|
||||
res, err := c.request(ctx).
|
||||
SetQueryParam("uuid", uuid).
|
||||
SetResult(&envelope).
|
||||
Get(c.apiBase + "/query")
|
||||
if err != nil {
|
||||
return QRCodeStatus{}, err
|
||||
}
|
||||
if res.IsError() {
|
||||
return QRCodeStatus{}, qrAPIError(envelope.message(), res.StatusCode())
|
||||
}
|
||||
|
||||
result, err := decodeResultMap(envelope.Result)
|
||||
if err != nil {
|
||||
return QRCodeStatus{}, err
|
||||
}
|
||||
state := intValue(result["state"])
|
||||
status := QRCodeStatus{
|
||||
State: state,
|
||||
StatusText: qrStateText(state),
|
||||
}
|
||||
if state != 3 {
|
||||
return status, nil
|
||||
}
|
||||
|
||||
status.AccessToken = findStringByKeys(result, "access_token", "accessToken", "token", "tokenValue")
|
||||
status.RefreshToken = findStringByKeys(result, "refresh_token", "refreshToken")
|
||||
status.FamilyID = findStringByKeys(result, "family_id", "familyId", "familyID", "defaultFamilyId", "defaultHomeId", "homeId")
|
||||
if status.AccessToken == "" || status.RefreshToken == "" {
|
||||
missing := make([]string, 0, 2)
|
||||
if status.AccessToken == "" {
|
||||
missing = append(missing, "access_token")
|
||||
}
|
||||
if status.RefreshToken == "" {
|
||||
missing = append(missing, "refresh_token")
|
||||
}
|
||||
return QRCodeStatus{}, fmt.Errorf("wopan qr: login succeeded but missing %s; available keys: %s",
|
||||
strings.Join(missing, ", "), strings.Join(collectJSONKeys(result), ", "))
|
||||
}
|
||||
return status, nil
|
||||
}
|
||||
|
||||
func (c *QRClient) request(ctx context.Context) *resty.Request {
|
||||
return c.client.R().
|
||||
SetContext(ctx).
|
||||
SetHeaders(map[string]string{
|
||||
"client-id": defaultQRCodeClient,
|
||||
"x-yp-client-id": defaultQRCodeClient,
|
||||
"Accept": "application/json",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
})
|
||||
}
|
||||
|
||||
type qrEnvelope struct {
|
||||
Meta qrMeta `json:"meta"`
|
||||
Result json.RawMessage `json:"result"`
|
||||
Code any `json:"code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Msg string `json:"msg,omitempty"`
|
||||
}
|
||||
|
||||
type qrMeta struct {
|
||||
Code any `json:"code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Msg string `json:"msg,omitempty"`
|
||||
}
|
||||
|
||||
type qrGenerateResult struct {
|
||||
UUID string `json:"uuid"`
|
||||
Image string `json:"image"`
|
||||
}
|
||||
|
||||
func (e qrEnvelope) message() string {
|
||||
for _, s := range []string{e.Message, e.Msg, e.Meta.Message, e.Meta.Msg} {
|
||||
if strings.TrimSpace(s) != "" {
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func decodeResult(raw json.RawMessage, dst any) error {
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return errors.New("wopan qr: empty result")
|
||||
}
|
||||
if err := json.Unmarshal(raw, dst); err != nil {
|
||||
return fmt.Errorf("wopan qr: decode result: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func decodeResultMap(raw json.RawMessage) (map[string]any, error) {
|
||||
var result map[string]any
|
||||
if err := decodeResult(raw, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if result == nil {
|
||||
return nil, errors.New("wopan qr: empty result")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func qrImageDataURL(image string) string {
|
||||
image = strings.TrimSpace(image)
|
||||
if strings.HasPrefix(strings.ToLower(image), "data:image/") {
|
||||
return image
|
||||
}
|
||||
return "data:image/png;base64," + image
|
||||
}
|
||||
|
||||
func qrAPIError(message string, httpStatus int) error {
|
||||
message = strings.TrimSpace(message)
|
||||
if message == "" {
|
||||
message = fmt.Sprintf("HTTP %d", httpStatus)
|
||||
}
|
||||
return errors.New(message)
|
||||
}
|
||||
|
||||
func qrStateText(state int) string {
|
||||
switch state {
|
||||
case 1:
|
||||
return "等待扫码"
|
||||
case 2:
|
||||
return "已扫码,请在联通网盘 App 确认"
|
||||
case 3:
|
||||
return "登录成功"
|
||||
case 4:
|
||||
return "二维码已过期"
|
||||
default:
|
||||
return "未知状态"
|
||||
}
|
||||
}
|
||||
|
||||
func intValue(v any) int {
|
||||
switch x := v.(type) {
|
||||
case int:
|
||||
return x
|
||||
case int64:
|
||||
return int(x)
|
||||
case float64:
|
||||
return int(x)
|
||||
case json.Number:
|
||||
n, _ := x.Int64()
|
||||
return int(n)
|
||||
case string:
|
||||
n, _ := strconv.Atoi(strings.TrimSpace(x))
|
||||
return n
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func findStringByKeys(v any, keys ...string) string {
|
||||
targets := make(map[string]struct{}, len(keys))
|
||||
for _, key := range keys {
|
||||
targets[normalizeJSONKey(key)] = struct{}{}
|
||||
}
|
||||
return findStringByNormalizedKeys(v, targets)
|
||||
}
|
||||
|
||||
func findStringByNormalizedKeys(v any, targets map[string]struct{}) string {
|
||||
switch x := v.(type) {
|
||||
case map[string]any:
|
||||
for key, value := range x {
|
||||
if _, ok := targets[normalizeJSONKey(key)]; ok {
|
||||
if s := stringValue(value); s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, value := range x {
|
||||
if s := findStringByNormalizedKeys(value, targets); s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, value := range x {
|
||||
if s := findStringByNormalizedKeys(value, targets); s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func stringValue(v any) string {
|
||||
switch x := v.(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(x)
|
||||
case int:
|
||||
return strconv.Itoa(x)
|
||||
case int64:
|
||||
return strconv.FormatInt(x, 10)
|
||||
case float64:
|
||||
if x == float64(int64(x)) {
|
||||
return strconv.FormatInt(int64(x), 10)
|
||||
}
|
||||
return strconv.FormatFloat(x, 'f', -1, 64)
|
||||
case json.Number:
|
||||
return strings.TrimSpace(x.String())
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeJSONKey(key string) string {
|
||||
key = strings.ToLower(strings.TrimSpace(key))
|
||||
key = strings.ReplaceAll(key, "_", "")
|
||||
key = strings.ReplaceAll(key, "-", "")
|
||||
key = strings.ReplaceAll(key, " ", "")
|
||||
return key
|
||||
}
|
||||
|
||||
func collectJSONKeys(v any) []string {
|
||||
seen := map[string]struct{}{}
|
||||
var walk func(any)
|
||||
walk = func(value any) {
|
||||
switch x := value.(type) {
|
||||
case map[string]any:
|
||||
for key, child := range x {
|
||||
if strings.TrimSpace(key) != "" {
|
||||
seen[key] = struct{}{}
|
||||
}
|
||||
walk(child)
|
||||
}
|
||||
case []any:
|
||||
for _, child := range x {
|
||||
walk(child)
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(v)
|
||||
|
||||
keys := make([]string, 0, len(seen))
|
||||
for key := range seen {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
if len(keys) > 16 {
|
||||
keys = append(keys[:16], "...")
|
||||
}
|
||||
return keys
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package wopan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestQRCodeGenerateUsesServiceImage(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/QRCode/generate" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if r.Header.Get("client-id") != defaultQRCodeClient {
|
||||
t.Fatalf("client-id = %q, want %q", r.Header.Get("client-id"), defaultQRCodeClient)
|
||||
}
|
||||
if r.Header.Get("x-yp-client-id") != defaultQRCodeClient {
|
||||
t.Fatalf("x-yp-client-id = %q, want %q", r.Header.Get("x-yp-client-id"), defaultQRCodeClient)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"meta": map[string]string{"code": "0000", "message": "ok"},
|
||||
"result": map[string]string{
|
||||
"uuid": "uuid-1",
|
||||
"image": "iVBORw0KGgo=",
|
||||
},
|
||||
})
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{APIBaseURL: api.URL + "/QRCode"}).Generate(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
if got.UUID != "uuid-1" {
|
||||
t.Fatalf("uuid = %q, want uuid-1", got.UUID)
|
||||
}
|
||||
if got.QRImageDataURL != "data:image/png;base64,iVBORw0KGgo=" {
|
||||
t.Fatalf("qrImageDataUrl = %q, want PNG data URL", got.QRImageDataURL)
|
||||
}
|
||||
if got.ExpiresAt == "" {
|
||||
t.Fatalf("expiresAt is empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQRCodePollPending(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/QRCode/query" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if r.URL.Query().Get("uuid") != "uuid-1" {
|
||||
t.Fatalf("uuid query = %q, want uuid-1", r.URL.Query().Get("uuid"))
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"meta": map[string]string{"code": "0000", "message": "ok"},
|
||||
"result": map[string]any{
|
||||
"state": 1,
|
||||
"token": nil,
|
||||
"refreshToken": nil,
|
||||
},
|
||||
})
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{APIBaseURL: api.URL + "/QRCode"}).Poll(context.Background(), "uuid-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Poll() error = %v", err)
|
||||
}
|
||||
if got.State != 1 || got.StatusText != "等待扫码" || got.AccessToken != "" || got.RefreshToken != "" {
|
||||
t.Fatalf("status = %#v, want pending without tokens", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQRCodePollSuccessMapsTokenFields(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/QRCode/query" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"meta": map[string]string{"code": "0000", "message": "ok"},
|
||||
"result": map[string]any{
|
||||
"state": 3,
|
||||
"token": "access-1",
|
||||
"refreshToken": "refresh-1",
|
||||
},
|
||||
})
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{APIBaseURL: api.URL + "/QRCode"}).Poll(context.Background(), "uuid-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Poll() error = %v", err)
|
||||
}
|
||||
if got.State != 3 || got.AccessToken != "access-1" || got.RefreshToken != "refresh-1" {
|
||||
t.Fatalf("status = %#v, want token and refreshToken mapped", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQRCodePollSuccessReportsMissingTokenKeys(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"meta": map[string]string{"code": "0000", "message": "ok"},
|
||||
"result": map[string]any{
|
||||
"state": 3,
|
||||
"user": map[string]string{"name": "demo"},
|
||||
},
|
||||
})
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
_, err := NewQRClient(QRConfig{APIBaseURL: api.URL + "/QRCode"}).Poll(context.Background(), "uuid-1")
|
||||
if err == nil {
|
||||
t.Fatal("Poll() error is nil, want missing token error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "missing access_token, refresh_token") ||
|
||||
!strings.Contains(err.Error(), "available keys") {
|
||||
t.Fatalf("error = %q, want missing token keys", err.Error())
|
||||
}
|
||||
}
|
||||
@@ -149,6 +149,28 @@ func (w *Worker) Status() TaskStatus {
|
||||
return status
|
||||
}
|
||||
|
||||
// WaitIdle blocks until the fingerprint queue is empty and no item is being processed.
|
||||
func (w *Worker) WaitIdle(ctx context.Context) error {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
if w.queue.lengthExcluding("") == 0 {
|
||||
return nil
|
||||
}
|
||||
ticker := time.NewTicker(200 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
if w.queue.lengthExcluding("") == 0 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
defer w.queue.release(v.ID)
|
||||
if w.Catalog == nil || w.Drive == nil || v == nil || v.ID == "" {
|
||||
@@ -327,11 +349,74 @@ func readHTTPRange(ctx context.Context, hc *http.Client, link *drives.StreamLink
|
||||
return data, nil
|
||||
}
|
||||
}
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
|
||||
if remoteRangeResponseLooksRateLimited(link.URL, resp.StatusCode, body) {
|
||||
return nil, &drives.RateLimitError{
|
||||
Provider: "fingerprint",
|
||||
RetryAfter: parseRetryAfter(resp.Header.Get("Retry-After")),
|
||||
Err: fmt.Errorf("remote sample rate limited: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(body))),
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("fingerprint: range request got status=%d for bytes=%d-%d", resp.StatusCode, r.start, end)
|
||||
}
|
||||
return io.ReadAll(io.LimitReader(resp.Body, r.length))
|
||||
}
|
||||
|
||||
func remoteRangeResponseLooksRateLimited(rawURL string, status int, body []byte) bool {
|
||||
if status == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if isWopanMediaURL(rawURL) && (status == http.StatusForbidden || status == http.StatusTooManyRequests ||
|
||||
status == http.StatusInternalServerError || status == http.StatusBadGateway ||
|
||||
status == http.StatusServiceUnavailable || status == http.StatusGatewayTimeout ||
|
||||
status == 509) {
|
||||
return true
|
||||
}
|
||||
if isGuangYaPanMediaURL(rawURL) && (status == http.StatusForbidden || status == http.StatusTooManyRequests ||
|
||||
status == http.StatusInternalServerError || status == http.StatusBadGateway ||
|
||||
status == http.StatusServiceUnavailable || status == http.StatusGatewayTimeout ||
|
||||
status == 509) {
|
||||
return true
|
||||
}
|
||||
if status == http.StatusForbidden && isGoogleDriveMediaURL(rawURL) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isWopanMediaURL(rawURL string) bool {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
path := strings.ToLower(u.Path)
|
||||
return (strings.HasSuffix(host, "pan.wo.cn") ||
|
||||
strings.HasSuffix(host, "smartont.net") ||
|
||||
strings.Contains(host, "wo.cn")) &&
|
||||
strings.Contains(path, "/openapi/download")
|
||||
}
|
||||
|
||||
func isGuangYaPanMediaURL(rawURL string) bool {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
return strings.HasSuffix(host, "guangyacdn.com") ||
|
||||
strings.HasSuffix(host, "guangyapan.com")
|
||||
}
|
||||
|
||||
func isGoogleDriveMediaURL(rawURL string) bool {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Host)
|
||||
path := strings.ToLower(u.Path)
|
||||
return strings.Contains(host, "googleapis.com") && strings.Contains(path, "/drive/")
|
||||
}
|
||||
|
||||
func parseRetryAfter(raw string) time.Duration {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
|
||||
@@ -2,6 +2,7 @@ package fingerprint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -85,6 +86,75 @@ func TestComputeRemoteUsesRangeSamples(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeRemote429ReturnsRateLimit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Retry-After", "60")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
_, _ = w.Write([]byte(`{"error":{"code":429}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
drv := &fakeDrive{paths: map[string]string{"remote": srv.URL + "/video.mp4"}}
|
||||
_, err := Compute(ctx, drv, &catalog.Video{ID: "remote", FileID: "remote", Size: 1024 * 1024}, Config{
|
||||
SampleSizeBytes: 4,
|
||||
FullHashMaxSize: 8,
|
||||
HTTPClient: srv.Client(),
|
||||
}, srv.Client())
|
||||
if err == nil {
|
||||
t.Fatal("compute succeeded, want rate limit")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != time.Minute {
|
||||
t.Fatalf("retry after = %s, want 1m", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWopanRemoteRangeErrorsLookRateLimited(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
rawURL string
|
||||
status int
|
||||
}{
|
||||
{rawURL: "https://gxdownload.pan.wo.cn:8445/openapi/download?fid=encoded", status: http.StatusForbidden},
|
||||
{rawURL: "https://du.smartont.net:8445/openapi/download?fid=encoded", status: http.StatusServiceUnavailable},
|
||||
{rawURL: "https://du.smartont.net:8445/openapi/download?fid=encoded", status: 509},
|
||||
} {
|
||||
if !remoteRangeResponseLooksRateLimited(tc.rawURL, tc.status, nil) {
|
||||
t.Fatalf("remoteRangeResponseLooksRateLimited(%q, %d) = false, want true", tc.rawURL, tc.status)
|
||||
}
|
||||
}
|
||||
if remoteRangeResponseLooksRateLimited("https://example.com/video.mp4", http.StatusForbidden, nil) {
|
||||
t.Fatal("generic 403 should not be treated as wopan rate limit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuangYaPanRemoteRangeErrorsLookRateLimited(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
rawURL string
|
||||
status int
|
||||
}{
|
||||
{rawURL: "https://txgz02-httpdown.guangyacdn.com/download/?fid=encoded", status: http.StatusForbidden},
|
||||
{rawURL: "https://txgz02-httpdown.guangyacdn.com/download/?fid=encoded", status: http.StatusServiceUnavailable},
|
||||
{rawURL: "https://txgz02-httpdown.guangyacdn.com/download/?fid=encoded", status: 509},
|
||||
} {
|
||||
if !remoteRangeResponseLooksRateLimited(tc.rawURL, tc.status, nil) {
|
||||
t.Fatalf("remoteRangeResponseLooksRateLimited(%q, %d) = false, want true", tc.rawURL, tc.status)
|
||||
}
|
||||
}
|
||||
if remoteRangeResponseLooksRateLimited("https://example.com/video.mp4", http.StatusForbidden, nil) {
|
||||
t.Fatal("generic 403 should not be treated as guangyapan rate limit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoogleDriveRemoteRangeForbiddenLooksRateLimitedByURL(t *testing.T) {
|
||||
if !remoteRangeResponseLooksRateLimited("https://www.googleapis.com/drive/v3/files/file-1?alt=media", http.StatusForbidden, nil) {
|
||||
t.Fatal("google drive media 403 should be treated as rate limit by URL and status")
|
||||
}
|
||||
}
|
||||
|
||||
type fakeDrive struct {
|
||||
paths map[string]string
|
||||
}
|
||||
|
||||
@@ -952,15 +952,7 @@ func redactURLs(text string) string {
|
||||
}
|
||||
|
||||
func ffmpegOutputLooksRateLimited(output []byte) bool {
|
||||
text := strings.ToLower(string(output))
|
||||
if !strings.Contains(text, "429") {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "throttl") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "rate-limit") ||
|
||||
strings.Contains(text, "server returned 429")
|
||||
return drives.TextMentionsHTTPStatus(string(output), http.StatusTooManyRequests)
|
||||
}
|
||||
|
||||
// --- 本地落盘 ---
|
||||
@@ -1064,12 +1056,10 @@ type ThumbWorker struct {
|
||||
}
|
||||
|
||||
const (
|
||||
defaultTransientMediaCooldown = 5 * time.Minute
|
||||
defaultGenerationRateLimitCooldown = 5 * time.Minute
|
||||
defaultThumbTransientMediaMaxFailures = 3
|
||||
defaultWorkerQueueSize = 10000
|
||||
maxPreviewTeaserSizeBytes int64 = 5 * 1024 * 1024 * 1024
|
||||
previewStatusSkipped = "skipped"
|
||||
defaultTransientMediaCooldown = 5 * time.Minute
|
||||
defaultGenerationRateLimitCooldown = 5 * time.Minute
|
||||
defaultThumbTransientMediaMaxFailures = 3
|
||||
defaultWorkerQueueSize = 10000
|
||||
)
|
||||
|
||||
type rateLimitState struct {
|
||||
@@ -1124,6 +1114,19 @@ func (q *videoQueue) release(v *catalog.Video) {
|
||||
q.mu.Unlock()
|
||||
}
|
||||
|
||||
func (q *videoQueue) idsSnapshot() []string {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
if len(q.ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(q.ids))
|
||||
for id := range q.ids {
|
||||
out = append(out, id)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (q *videoQueue) lengthExcluding(currentID string) int {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
@@ -1251,6 +1254,13 @@ func (w *Worker) Status() TaskStatus {
|
||||
return taskStatus(&w.activity, &w.rateLimit, w.queue.lengthExcluding(currentID))
|
||||
}
|
||||
|
||||
func (w *Worker) ActiveVideoIDs() []string {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
return w.queue.idsSnapshot()
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) Status() TaskStatus {
|
||||
if w == nil {
|
||||
return TaskStatus{State: "idle"}
|
||||
@@ -1427,11 +1437,17 @@ func (w *Worker) skipIfRateLimited(v *catalog.Video) bool {
|
||||
}
|
||||
|
||||
func (w *Worker) pauseForRateLimit(err error, step, title string) bool {
|
||||
_, ok := drives.RateLimitRetryAfter(err)
|
||||
wait, ok := drives.RateLimitRetryAfter(err)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), defaultGenerationRateLimitCooldown)
|
||||
if wait <= 0 {
|
||||
wait = w.RateLimitCooldown
|
||||
if wait <= 0 {
|
||||
wait = defaultGenerationRateLimitCooldown
|
||||
}
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), wait)
|
||||
log.Printf("[preview] drive=%s rate-limited until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err)
|
||||
return true
|
||||
}
|
||||
@@ -1460,11 +1476,17 @@ func (w *ThumbWorker) skipIfRateLimited(v *catalog.Video) bool {
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) pauseForRateLimit(err error, step, title string) bool {
|
||||
_, ok := drives.RateLimitRetryAfter(err)
|
||||
wait, ok := drives.RateLimitRetryAfter(err)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), defaultGenerationRateLimitCooldown)
|
||||
if wait <= 0 {
|
||||
wait = w.RateLimitCooldown
|
||||
if wait <= 0 {
|
||||
wait = defaultGenerationRateLimitCooldown
|
||||
}
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), wait)
|
||||
log.Printf("[thumb] drive=%s rate-limited until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err)
|
||||
return true
|
||||
}
|
||||
@@ -1506,60 +1528,17 @@ func driveErrorShouldCooldown(d drives.Drive, err error) bool {
|
||||
}
|
||||
switch d.Kind() {
|
||||
case "p115":
|
||||
text := strings.ToLower(err.Error())
|
||||
return strings.Contains(text, "server returned 403") ||
|
||||
strings.Contains(text, "403 forbidden") ||
|
||||
strings.Contains(text, "server returned 405") ||
|
||||
strings.Contains(text, "405 method") ||
|
||||
strings.Contains(text, "access denied") ||
|
||||
strings.Contains(text, "moov atom not found") ||
|
||||
strings.Contains(text, "partial file") ||
|
||||
strings.Contains(text, "request has been blocked") ||
|
||||
strings.Contains(text, "访问被阻断")
|
||||
return drives.ErrorMentionsHTTPStatus(err, http.StatusForbidden, http.StatusMethodNotAllowed, http.StatusTooManyRequests)
|
||||
case "pikpak":
|
||||
// PikPak 在预览视频 / 封面生成阶段(取链或拉直链字节)可能命中:
|
||||
// - error_code=10 操作频繁
|
||||
// - HTTP 429 / 5xx / 509 限流和服务端不可用
|
||||
// - 通用文本:rate limit / too many requests / blocked
|
||||
// 命中时让 worker 冷却 5 分钟,避免连续请求加重风控。
|
||||
text := strings.ToLower(err.Error())
|
||||
return strings.Contains(text, "error_code=10") ||
|
||||
strings.Contains(text, "操作频繁") ||
|
||||
strings.Contains(text, "429") ||
|
||||
strings.Contains(text, "http 500") ||
|
||||
strings.Contains(text, "http 502") ||
|
||||
strings.Contains(text, "http 503") ||
|
||||
strings.Contains(text, "http 504") ||
|
||||
strings.Contains(text, "http 509") ||
|
||||
strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "blocked") ||
|
||||
strings.Contains(text, "partial file") ||
|
||||
strings.Contains(text, "service unavailable")
|
||||
return drives.ErrorMentionsHTTPStatus(err, http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout, 509)
|
||||
case "p123":
|
||||
// 123 云盘直链解析 / ffmpeg 读取阶段可能返回 429、5xx,或 WAF 类
|
||||
// blocked / 访问阻断文本。命中时冷却,避免封面和预览视频生成连续打接口。
|
||||
text := strings.ToLower(err.Error())
|
||||
return strings.Contains(text, "请求太频繁") ||
|
||||
strings.Contains(text, "请求过于频繁") ||
|
||||
strings.Contains(text, "请求频繁") ||
|
||||
strings.Contains(text, "操作频繁") ||
|
||||
strings.Contains(text, "频率限制") ||
|
||||
strings.Contains(text, "请求次数过多") ||
|
||||
strings.Contains(text, "429") ||
|
||||
strings.Contains(text, "http 500") ||
|
||||
strings.Contains(text, "http 502") ||
|
||||
strings.Contains(text, "http 503") ||
|
||||
strings.Contains(text, "http 504") ||
|
||||
strings.Contains(text, "server returned 403") ||
|
||||
strings.Contains(text, "403 forbidden") ||
|
||||
strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "blocked") ||
|
||||
strings.Contains(text, "访问被阻断") ||
|
||||
strings.Contains(text, "service unavailable")
|
||||
return drives.ErrorMentionsHTTPStatus(err, http.StatusForbidden, http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout)
|
||||
case "wopan":
|
||||
return drives.ErrorMentionsHTTPStatus(err, http.StatusForbidden, http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout, 509)
|
||||
case "guangyapan":
|
||||
return drives.ErrorMentionsHTTPStatus(err, http.StatusForbidden, http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout, 509)
|
||||
case "googledrive":
|
||||
return drives.ErrorMentionsHTTPStatus(err, http.StatusForbidden, http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout)
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1713,15 +1692,6 @@ func localPreviewLink(v *catalog.Video) (*drives.StreamLink, bool) {
|
||||
}
|
||||
|
||||
func (w *Worker) process(ctx context.Context, v *catalog.Video) {
|
||||
if shouldSkipTeaser(v) {
|
||||
removePreviousLocalTeaser(v.PreviewLocal, "")
|
||||
if err := w.Catalog.UpdatePreview(ctx, v.ID, "", previewStatusSkipped); err != nil {
|
||||
log.Printf("[preview] skip %s: update status: %v", v.Title, err)
|
||||
return
|
||||
}
|
||||
log.Printf("[preview] skip %s: size=%d exceeds 5GiB teaser limit", v.Title, v.Size)
|
||||
return
|
||||
}
|
||||
if w.skipIfRateLimited(v) {
|
||||
return
|
||||
}
|
||||
@@ -1774,10 +1744,6 @@ func (w *Worker) process(ctx context.Context, v *catalog.Video) {
|
||||
log.Printf("[preview] ready %s (duration=%.1fs)", v.Title, duration)
|
||||
}
|
||||
|
||||
func shouldSkipTeaser(v *catalog.Video) bool {
|
||||
return v != nil && v.Size > maxPreviewTeaserSizeBytes
|
||||
}
|
||||
|
||||
func (w *Worker) generateTeaser(ctx context.Context, v *catalog.Video, link *drives.StreamLink, duration float64) (string, error) {
|
||||
gen, ok := w.Gen.(refreshingTeaserGenerator)
|
||||
if !ok || w.Drive == nil || w.Drive.Kind() != "p115" {
|
||||
|
||||
@@ -349,42 +349,10 @@ func TestPreviewWorkerNeverCallsDriveUploadOrEnsureDir(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewWorkerSkipsTeaserForVideoLargerThanFiveGiB(t *testing.T) {
|
||||
func TestPreviewWorkerGeneratesTeaserForLargeVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "preview-large-video")
|
||||
video.Size = maxPreviewTeaserSizeBytes + 1
|
||||
if err := cat.UpsertVideo(ctx, video); err != nil {
|
||||
t.Fatalf("update video: %v", err)
|
||||
}
|
||||
|
||||
gen := &fakeTeaserGenerator{}
|
||||
drv := &previewFakeDrive{}
|
||||
worker := NewWorker(gen, cat, drv)
|
||||
|
||||
worker.process(ctx, video)
|
||||
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.PreviewStatus != previewStatusSkipped {
|
||||
t.Fatalf("preview status = %q, want skipped", got.PreviewStatus)
|
||||
}
|
||||
if got.PreviewLocal != "" {
|
||||
t.Fatalf("preview local = %q, want empty", got.PreviewLocal)
|
||||
}
|
||||
if drv.streamCalls != 0 {
|
||||
t.Fatalf("stream calls = %d, want 0", drv.streamCalls)
|
||||
}
|
||||
if gen.generateCalls != 0 {
|
||||
t.Fatalf("generate calls = %d, want 0", gen.generateCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewWorkerGeneratesTeaserAtFiveGiBBoundary(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "preview-five-gib-video")
|
||||
video.Size = maxPreviewTeaserSizeBytes
|
||||
video.Size = 6 * 1024 * 1024 * 1024
|
||||
if err := cat.UpsertVideo(ctx, video); err != nil {
|
||||
t.Fatalf("update video: %v", err)
|
||||
}
|
||||
@@ -442,7 +410,7 @@ func TestPreviewWorkerRateLimitLeavesCurrentPendingAndSkipsNextVideo(t *testing.
|
||||
if gen.generateCalls != 1 {
|
||||
t.Fatalf("generate calls = %d, want 1", gen.generateCalls)
|
||||
}
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 5*time.Minute)
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 2*time.Hour)
|
||||
|
||||
gen.generateErr = nil
|
||||
worker.process(ctx, &second)
|
||||
@@ -458,7 +426,7 @@ func TestPreviewWorkerRateLimitLeavesCurrentPendingAndSkipsNextVideo(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerRateLimitCoolsDownFiveMinutes(t *testing.T) {
|
||||
func TestThumbWorkerRateLimitHonorsRetryAfter(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-rate-limit")
|
||||
|
||||
@@ -482,12 +450,12 @@ func TestThumbWorkerRateLimitCoolsDownFiveMinutes(t *testing.T) {
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want unchanged after rate limit", got.ThumbnailURL)
|
||||
}
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 5*time.Minute)
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 2*time.Hour)
|
||||
}
|
||||
|
||||
func TestThumbWorkerP115TransientErrorFailsAfterRetryLimit(t *testing.T) {
|
||||
func TestThumbWorkerP115MessageOnlyErrorFailsWithoutCooldown(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-p115-transient")
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-p115-message-only")
|
||||
|
||||
gen := &fakeThumbGenerator{
|
||||
generateErr: errors.New("ffmpeg thumb: exit status 183, stderr: partial file Cannot determine format of input 0:0 after EOF"),
|
||||
@@ -495,69 +463,26 @@ func TestThumbWorkerP115TransientErrorFailsAfterRetryLimit(t *testing.T) {
|
||||
drv := &previewFakeDrive{kind: "p115"}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
for attempt := 1; attempt <= defaultThumbTransientMediaMaxFailures; attempt++ {
|
||||
worker.rateLimit = rateLimitState{}
|
||||
worker.process(ctx, video)
|
||||
|
||||
if attempt < defaultThumbTransientMediaMaxFailures {
|
||||
pending, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "pending", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list pending thumbnails: %v", err)
|
||||
}
|
||||
if len(pending) != 1 || pending[0].ID != video.ID {
|
||||
t.Fatalf("attempt %d pending thumbnails = %#v, want only %s", attempt, pending, video.ID)
|
||||
}
|
||||
missing, err := cat.CountVideosNeedingThumbnail(ctx, video.DriveID)
|
||||
if err != nil {
|
||||
t.Fatalf("count missing thumbnails: %v", err)
|
||||
}
|
||||
if missing != 1 {
|
||||
t.Fatalf("attempt %d missing thumbnails = %d, want 1 before retry limit", attempt, missing)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list failed thumbnails: %v", err)
|
||||
}
|
||||
if len(failed) != 1 || failed[0].ID != video.ID {
|
||||
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
|
||||
}
|
||||
missing, err := cat.CountVideosNeedingThumbnail(ctx, video.DriveID)
|
||||
if err != nil {
|
||||
t.Fatalf("count missing thumbnails: %v", err)
|
||||
}
|
||||
if missing != 0 {
|
||||
t.Fatalf("missing thumbnails = %d, want 0 after retry limit marks failed", missing)
|
||||
}
|
||||
}
|
||||
|
||||
if gen.generateCalls != defaultThumbTransientMediaMaxFailures {
|
||||
t.Fatalf("generate calls = %d, want %d", gen.generateCalls, defaultThumbTransientMediaMaxFailures)
|
||||
}
|
||||
|
||||
if err := cat.UpdateVideoMeta(ctx, video.ID, catalog.VideoMetaPatch{
|
||||
ThumbnailStatus: "pending",
|
||||
ResetThumbnailFailures: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("reset thumbnail status: %v", err)
|
||||
}
|
||||
worker.rateLimit = rateLimitState{}
|
||||
worker.process(ctx, video)
|
||||
|
||||
pending, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "pending", 0)
|
||||
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list pending thumbnails after reset: %v", err)
|
||||
t.Fatalf("list failed thumbnails: %v", err)
|
||||
}
|
||||
if len(pending) != 1 || pending[0].ID != video.ID {
|
||||
t.Fatalf("pending thumbnails after reset = %#v, want only %s", pending, video.ID)
|
||||
if len(failed) != 1 || failed[0].ID != video.ID {
|
||||
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
|
||||
}
|
||||
if !worker.Status().CooldownUntil.IsZero() {
|
||||
t.Fatalf("cooldown until = %s, want no cooldown for message-only media error", worker.Status().CooldownUntil)
|
||||
}
|
||||
if gen.generateCalls != 1 {
|
||||
t.Fatalf("generate calls = %d, want 1", gen.generateCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerRequeuesP115TransientErrorBeforeRetryLimit(t *testing.T) {
|
||||
func TestThumbWorkerDoesNotRequeueP115MessageOnlyError(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-p115-requeue")
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-p115-no-requeue")
|
||||
|
||||
gen := &fakeThumbGenerator{
|
||||
generateErr: errors.New("ffmpeg thumb: partial file Cannot determine format of input 0:0 after EOF"),
|
||||
@@ -569,11 +494,8 @@ func TestThumbWorkerRequeuesP115TransientErrorBeforeRetryLimit(t *testing.T) {
|
||||
|
||||
select {
|
||||
case queued := <-worker.ch:
|
||||
if queued.ID != video.ID {
|
||||
t.Fatalf("requeued video id = %q, want %q", queued.ID, video.ID)
|
||||
}
|
||||
t.Fatalf("unexpected requeued video id = %q", queued.ID)
|
||||
default:
|
||||
t.Fatal("expected transient thumbnail failure to requeue the same video")
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
@@ -581,14 +503,14 @@ func TestThumbWorkerRequeuesP115TransientErrorBeforeRetryLimit(t *testing.T) {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want empty after transient failure", got.ThumbnailURL)
|
||||
t.Fatalf("thumbnail = %q, want empty after message-only failure", got.ThumbnailURL)
|
||||
}
|
||||
pending, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "pending", 0)
|
||||
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list pending thumbnails: %v", err)
|
||||
t.Fatalf("list failed thumbnails: %v", err)
|
||||
}
|
||||
if len(pending) != 1 || pending[0].ID != video.ID {
|
||||
t.Fatalf("pending thumbnails = %#v, want only %s", pending, video.ID)
|
||||
if len(failed) != 1 || failed[0].ID != video.ID {
|
||||
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -649,18 +571,82 @@ func TestP123TransientErrorsShouldCooldown(t *testing.T) {
|
||||
drv := &previewFakeDrive{kind: "p123"}
|
||||
for _, err := range []error{
|
||||
errors.New("Server returned 403 Forbidden"),
|
||||
errors.New("请求太频繁"),
|
||||
errors.New("http 503 service unavailable"),
|
||||
} {
|
||||
if !driveErrorShouldCooldown(drv, err) {
|
||||
t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err)
|
||||
}
|
||||
}
|
||||
if driveErrorShouldCooldown(drv, errors.New("请求太频繁")) {
|
||||
t.Fatal("message-only throttling text should not trigger p123 cooldown")
|
||||
}
|
||||
if driveErrorShouldCooldown(drv, errors.New("invalid credential")) {
|
||||
t.Fatal("invalid credential should not trigger p123 cooldown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWopanTransientErrorsShouldCooldown(t *testing.T) {
|
||||
drv := &previewFakeDrive{kind: "wopan"}
|
||||
for _, err := range []error{
|
||||
errors.New("ffmpeg: Server returned 403 Forbidden"),
|
||||
errors.New("wopan download url: request failed with status: 429 Too Many Requests"),
|
||||
errors.New("http 503 service unavailable"),
|
||||
} {
|
||||
if !driveErrorShouldCooldown(drv, err) {
|
||||
t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err)
|
||||
}
|
||||
}
|
||||
if driveErrorShouldCooldown(drv, errors.New("操作频繁,请稍后重试")) {
|
||||
t.Fatal("message-only throttling text should not trigger wopan cooldown")
|
||||
}
|
||||
if driveErrorShouldCooldown(drv, errors.New("invalid access token")) {
|
||||
t.Fatal("invalid access token should not trigger wopan cooldown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuangYaPanTransientErrorsShouldCooldown(t *testing.T) {
|
||||
drv := &previewFakeDrive{kind: "guangyapan"}
|
||||
for _, err := range []error{
|
||||
errors.New("ffmpeg: Server returned 403 Forbidden"),
|
||||
errors.New("guangyapan api rate limited: status=429 msg=操作频繁,请稍后重试"),
|
||||
errors.New("http 503 service unavailable"),
|
||||
} {
|
||||
if !driveErrorShouldCooldown(drv, err) {
|
||||
t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err)
|
||||
}
|
||||
}
|
||||
if driveErrorShouldCooldown(drv, errors.New("操作频繁,请稍后重试")) {
|
||||
t.Fatal("message-only throttling text should not trigger guangyapan cooldown")
|
||||
}
|
||||
if driveErrorShouldCooldown(drv, errors.New("invalid access token")) {
|
||||
t.Fatal("invalid access token should not trigger guangyapan cooldown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoogleDriveMediaErrorsShouldCooldown(t *testing.T) {
|
||||
drv := &previewFakeDrive{kind: "googledrive"}
|
||||
for _, err := range []error{
|
||||
errors.New("ffmpeg: Server returned 403 Forbidden"),
|
||||
errors.New("http 503 service unavailable"),
|
||||
} {
|
||||
if !driveErrorShouldCooldown(drv, err) {
|
||||
t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err)
|
||||
}
|
||||
}
|
||||
for _, err := range []error{
|
||||
errors.New("google drive api error: usageLimits userRateLimitExceeded"),
|
||||
errors.New("downloadQuotaExceeded: The download quota for this file has been exceeded"),
|
||||
errors.New("sharingRateLimitExceeded"),
|
||||
} {
|
||||
if driveErrorShouldCooldown(drv, err) {
|
||||
t.Fatalf("message-only google drive error %v should not trigger cooldown", err)
|
||||
}
|
||||
}
|
||||
if driveErrorShouldCooldown(drv, errors.New("invalid credentials")) {
|
||||
t.Fatal("invalid credentials should not trigger googledrive cooldown")
|
||||
}
|
||||
}
|
||||
|
||||
func assertCooldownAround(t *testing.T, until time.Time, before time.Time, want time.Duration) {
|
||||
t.Helper()
|
||||
if until.IsZero() {
|
||||
|
||||
@@ -147,15 +147,19 @@ func (p *Proxy) ServeStream(w http.ResponseWriter, r *http.Request, driveID, fil
|
||||
// CDN 不校验请求头,直连可获得最佳带宽并避免占用 backend 出站
|
||||
// - onedrive:Microsoft Graph 返回的 @microsoft.graph.downloadUrl 是短期
|
||||
// 免鉴权下载 URL,不需要后端继续代传视频字节
|
||||
// - p123:123 云盘 download_info 返回的下载页会再跳 CDN;driver 已在后端
|
||||
// - p123:123网盘 download_info 返回的下载页会再跳 CDN;driver 已在后端
|
||||
// 先解出最终 Location,浏览器可直接 302 到该短期地址
|
||||
// - wopan:联通网盘 GetDownloadUrlV2 返回的是短期直链,OpenList 也是直接
|
||||
// 将该 URL 交给客户端使用;不需要后端持续代传视频字节
|
||||
// - guangyapan:光鸭 get_res_download_url 返回 signedURL / downloadUrl,
|
||||
// 浏览器可直接访问,不需要后端持续代传视频字节
|
||||
//
|
||||
// 其余网盘(如沃盘 / 夸克等)仍走反代,因为它们的下载
|
||||
// 其余网盘(如夸克等)仍走反代,因为它们的下载
|
||||
// 链接通常需要随请求带上后端持有的 Cookie / Authorization / Range
|
||||
// 的特殊处理,浏览器拿不到这些上下文。
|
||||
func shouldRedirect(d drives.Drive) bool {
|
||||
switch d.Kind() {
|
||||
case "p115", "pikpak", "onedrive", "p123":
|
||||
case "p115", "pikpak", "onedrive", "p123", "wopan", "guangyapan":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
|
||||
@@ -201,6 +201,56 @@ func TestServeStreamRedirectsP123(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamRedirectsWopan(t *testing.T) {
|
||||
reg := NewRegistry()
|
||||
drv := &proxyFakeSimpleDrive{
|
||||
kind: "wopan",
|
||||
url: "https://du.smartont.net:8445/openapi/download?fid=encoded",
|
||||
}
|
||||
reg.Set("wopan", drv)
|
||||
|
||||
p := New(reg)
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/wopan/file-1", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
p.ServeStream(rr, req, "wopan", "file-1")
|
||||
|
||||
if rr.Code != http.StatusFound {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusFound)
|
||||
}
|
||||
if got := rr.Header().Get("Location"); got != "https://du.smartont.net:8445/openapi/download?fid=encoded" {
|
||||
t.Fatalf("Location = %q", got)
|
||||
}
|
||||
if drv.calls != 1 {
|
||||
t.Fatalf("link calls = %d, want 1", drv.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamRedirectsGuangYaPan(t *testing.T) {
|
||||
reg := NewRegistry()
|
||||
drv := &proxyFakeSimpleDrive{
|
||||
kind: "guangyapan",
|
||||
url: "https://cdn.guangyapan.example/video.mp4?sign=encoded",
|
||||
}
|
||||
reg.Set("guangyapan", drv)
|
||||
|
||||
p := New(reg)
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/guangyapan/file-1", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
p.ServeStream(rr, req, "guangyapan", "file-1")
|
||||
|
||||
if rr.Code != http.StatusFound {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusFound)
|
||||
}
|
||||
if got := rr.Header().Get("Location"); got != "https://cdn.guangyapan.example/video.mp4?sign=encoded" {
|
||||
t.Fatalf("Location = %q", got)
|
||||
}
|
||||
if drv.calls != 1 {
|
||||
t.Fatalf("link calls = %d, want 1", drv.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamServesLocalFilePath(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "video.mp4")
|
||||
if err := os.WriteFile(path, []byte("0123456789"), 0o644); err != nil {
|
||||
|
||||
@@ -2,6 +2,7 @@ package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"log"
|
||||
"path"
|
||||
@@ -25,6 +26,8 @@ type Scanner struct {
|
||||
SkipDirIDs map[string]struct{}
|
||||
// 回调:新视频被加入后触发预览视频生成
|
||||
OnNewVideo func(v *catalog.Video)
|
||||
// OnProgress 在扫描进度变化时触发。回调只应读取 Stats 里的计数,不应修改 map 字段。
|
||||
OnProgress func(stats Stats)
|
||||
// ProgressInterval 控制扫描内部 heartbeat 的最小输出间隔。
|
||||
// 0 → 默认 30s;< 0 → 关闭 heartbeat(仅留外层 start / done 两行)。
|
||||
// heartbeat 单行格式:
|
||||
@@ -91,6 +94,9 @@ func (s *Scanner) Run(ctx context.Context, startDirID string) (Stats, error) {
|
||||
driveID = s.Drive.ID()
|
||||
}
|
||||
progress := func(currentDir string) {
|
||||
if s.OnProgress != nil {
|
||||
s.OnProgress(stats)
|
||||
}
|
||||
if interval < 0 {
|
||||
return
|
||||
}
|
||||
@@ -127,6 +133,9 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
}
|
||||
|
||||
for _, e := range entries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if e.IsDir {
|
||||
// 跳过 previews 目录,避免扫到自己生成的预览视频
|
||||
if strings.EqualFold(e.Name, "previews") {
|
||||
@@ -137,13 +146,15 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
continue
|
||||
}
|
||||
if err := s.walk(ctx, e.ID, e.Name, stats, progress); err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return ctxErr
|
||||
}
|
||||
stats.Errors++
|
||||
log.Printf("[scanner] walk %s error: %v", e.Name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
stats.Scanned++
|
||||
ext := strings.ToLower(path.Ext(e.Name))
|
||||
if !s.Exts[ext] {
|
||||
continue
|
||||
@@ -151,10 +162,15 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
if e.Size <= 0 {
|
||||
continue
|
||||
}
|
||||
stats.Scanned++
|
||||
progress(dirName)
|
||||
stats.SeenFileIDs[e.ID] = struct{}{}
|
||||
|
||||
id := s.Drive.Kind() + "-" + s.Drive.ID() + "-" + e.ID
|
||||
id := s.Drive.Kind() + "-" + s.Drive.ID() + "-" + videoIDFilePart(e.ID)
|
||||
if deleted, err := s.Catalog.IsDeletedVideoCandidate(ctx, id, s.Drive.ID(), e.ID, e.Hash, e.Name, e.Size); err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return ctxErr
|
||||
}
|
||||
stats.Errors++
|
||||
log.Printf("[scanner] check deleted video %s error: %v", id, err)
|
||||
continue
|
||||
@@ -170,11 +186,20 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
if matched, err := s.Catalog.MatchTags(ctx, e.Name+" "+dirName+" "+parsed.Author); err == nil {
|
||||
tags = mergeTags(tags, matched)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if label, ok, err := s.Catalog.EnsureCollectionTag(ctx, dirName); err == nil && ok {
|
||||
tags = mergeTags(tags, []string{label})
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
existing, _ := s.Catalog.GetVideo(ctx, id)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if existing != nil {
|
||||
patch := catalog.VideoMetaPatch{}
|
||||
if e.Hash != "" && existing.ContentHash == "" {
|
||||
@@ -191,12 +216,21 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
}
|
||||
if patch.Category != "" || patch.ContentHash != "" || patch.FileName != "" {
|
||||
_ = s.Catalog.UpdateVideoMeta(ctx, id, patch)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if dup := s.findDuplicate(ctx, e.Hash, e.Name, e.Size, id); dup != nil {
|
||||
continue
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !sameTags(existing.Tags, tags) {
|
||||
_ = s.Catalog.SetAutoVideoTags(ctx, id, tags)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -204,6 +238,9 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
if dup := s.findDuplicate(ctx, e.Hash, e.Name, e.Size, id); dup != nil {
|
||||
continue
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
v := &catalog.Video{
|
||||
@@ -226,10 +263,17 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if err := s.Catalog.UpsertVideo(ctx, v); err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return ctxErr
|
||||
}
|
||||
log.Printf("[scanner] upsert %s error: %v", v.Title, err)
|
||||
continue
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
stats.Added++
|
||||
progress(dirName)
|
||||
if s.OnNewVideo != nil {
|
||||
s.OnNewVideo(v)
|
||||
}
|
||||
@@ -296,3 +340,10 @@ func mergeTags(lists ...[]string) []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func videoIDFilePart(fileID string) string {
|
||||
if !strings.ContainsAny(fileID, `/\`+"\x00") {
|
||||
return fileID
|
||||
}
|
||||
return "b64_" + base64.RawURLEncoding.EncodeToString([]byte(fileID))
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package scanner
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -90,6 +91,128 @@ func TestRunIgnoresZeroSizeVideoFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunScannedCountsOnlyVideoCandidates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := &scannerFakeDrive{
|
||||
entries: []drives.Entry{
|
||||
{ID: "file-1", Name: "clip.mp4", Size: 123},
|
||||
{ID: "file-2", Name: "notes.txt", Size: 123},
|
||||
{ID: "file-3", Name: "empty.mp4", Size: 0},
|
||||
},
|
||||
}
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
|
||||
stats, err := sc.Run(ctx, "")
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Scanned != 1 {
|
||||
t.Fatalf("scanned = %d, want one non-empty video candidate", stats.Scanned)
|
||||
}
|
||||
if stats.Added != 1 {
|
||||
t.Fatalf("added = %d, want one added video", stats.Added)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUsesPathSafeVideoIDForUnsafeFileID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := &scannerFakeDrive{
|
||||
entries: []drives.Entry{{
|
||||
ID: "fid/with space",
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
}},
|
||||
}
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
|
||||
stats, err := sc.Run(ctx, "")
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Added != 1 {
|
||||
t.Fatalf("added = %d, want 1", stats.Added)
|
||||
}
|
||||
if _, ok := stats.SeenFileIDs["fid/with space"]; !ok {
|
||||
t.Fatalf("seen file ids = %#v, want original file id", stats.SeenFileIDs)
|
||||
}
|
||||
|
||||
wantID := "fake-drive-b64_ZmlkL3dpdGggc3BhY2U"
|
||||
got, err := cat.GetVideo(ctx, wantID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video %s: %v", wantID, err)
|
||||
}
|
||||
if strings.Contains(got.ID, "/") {
|
||||
t.Fatalf("video id = %q, must not contain slash", got.ID)
|
||||
}
|
||||
if got.FileID != "fid/with space" {
|
||||
t.Fatalf("file id = %q, want original", got.FileID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStopsWhenContextCanceledDuringFileLoop(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := &scannerFakeDrive{
|
||||
entries: []drives.Entry{
|
||||
{ID: "file-1", Name: "one.mp4", Size: 123},
|
||||
{ID: "file-2", Name: "two.mp4", Size: 123},
|
||||
{ID: "file-3", Name: "three.mp4", Size: 123},
|
||||
},
|
||||
}
|
||||
callbacks := 0
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, func(*catalog.Video) {
|
||||
callbacks++
|
||||
cancel()
|
||||
})
|
||||
|
||||
stats, err := sc.Run(ctx, "")
|
||||
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("scan error = %v, want context.Canceled", err)
|
||||
}
|
||||
if stats.Added != 1 || callbacks != 1 {
|
||||
t.Fatalf("added=%d callbacks=%d, want exactly one video before cancellation", stats.Added, callbacks)
|
||||
}
|
||||
if _, err := cat.GetVideo(context.Background(), "fake-drive-file-1"); err != nil {
|
||||
t.Fatalf("first video should be persisted before cancellation: %v", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(context.Background(), "fake-drive-file-2"); err != sql.ErrNoRows {
|
||||
t.Fatalf("second video lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(context.Background(), "fake-drive-file-3"); err != sql.ErrNoRows {
|
||||
t.Fatalf("third video lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSkipsAdminDeletedVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Package spider91migrate 周期性把 spider91 drive 下载到本地的视频
|
||||
// 上传到一个指定的目标 drive 目录(PikPak、115、123 或 OneDrive),上传成功后:
|
||||
// 上传到一个指定的目标 drive 目录(PikPak、115、123、OneDrive、Google Drive、联通网盘或光鸭网盘),上传成功后:
|
||||
//
|
||||
// - 改写 catalog 行:drive_id / file_id / content_hash 改成目标盘的;
|
||||
// 视频自身的 id 不变(仍是 spider91-<driveID>-<viewkey>),video_tags、
|
||||
@@ -16,6 +16,7 @@ package spider91migrate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -29,22 +30,29 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/googledrive"
|
||||
"github.com/video-site/backend/internal/drives/guangyapan"
|
||||
"github.com/video-site/backend/internal/drives/onedrive"
|
||||
"github.com/video-site/backend/internal/drives/p115"
|
||||
"github.com/video-site/backend/internal/drives/p123"
|
||||
"github.com/video-site/backend/internal/drives/pikpak"
|
||||
"github.com/video-site/backend/internal/drives/scriptcrawler"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/drives/wopan"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
)
|
||||
|
||||
// uploadTarget 是 migrator 调用目标 drive 的最小接口。任何一种"接收 spider91 上传"的
|
||||
// 网盘都要实现它;当前 PikPak、115、123 和 OneDrive 各自通过适配器满足。
|
||||
// 网盘都要实现它;当前 PikPak、115、123、OneDrive、Google Drive、联通网盘和光鸭网盘各自通过适配器满足。
|
||||
//
|
||||
// 这一层抽象把"迁移调用方"和"具体盘的 SDK 协议"解耦:
|
||||
// - PikPak 走 GCID + OSS PutObject(pikpak.UploadResult)
|
||||
// - 115 走 SHA1 + 秒传 / OSS / 分片(p115.UploadResult)
|
||||
// - 123 走 MD5 + 秒传 / S3 预签名分片(p123.UploadResult)
|
||||
// - OneDrive 走 SHA1 + 小文件 PUT / 大文件 upload session
|
||||
// - Google Drive 走 MD5 + resumable upload session
|
||||
// - 联通网盘 走 SDK Upload2C,当前上游不返回内容 hash
|
||||
// - 光鸭网盘 走 OSS 分片上传,当前上游不返回内容 hash
|
||||
//
|
||||
// 各家返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
|
||||
type uploadTarget interface {
|
||||
@@ -56,10 +64,21 @@ type uploadTarget interface {
|
||||
Rename(ctx context.Context, fileID, newName string) error
|
||||
}
|
||||
|
||||
// Spider91LocalSource is the local source interface used by the migration
|
||||
// worker. Legacy spider91.Driver and the new scriptcrawler.Driver both satisfy
|
||||
// it when they are mounted for the Spider91 built-in crawler.
|
||||
type Spider91LocalSource interface {
|
||||
drives.Drive
|
||||
VideosDir() string
|
||||
ThumbsDir() string
|
||||
VideoPath(fileID string) (string, error)
|
||||
ThumbPath(fileID string) (string, error)
|
||||
}
|
||||
|
||||
// UploadResult 是 uploadTarget.UploadAndReportHash 的归一返回。
|
||||
//
|
||||
// FileID 目标盘上的新文件 ID;
|
||||
// Hash GCID(PikPak)、MD5 HEX(123)或 SHA1 HEX(115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;
|
||||
// Hash GCID(PikPak)、MD5 HEX(123 / Google Drive)或 SHA1 HEX(115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;联通网盘和光鸭网盘暂为空;
|
||||
// Size 实际上传字节数。
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
@@ -67,9 +86,34 @@ type UploadResult struct {
|
||||
Size int64
|
||||
}
|
||||
|
||||
const spider91UploadDirName = "91 Spider"
|
||||
type UploadProgress struct {
|
||||
DriveID string
|
||||
State string
|
||||
CurrentTitle string
|
||||
QueueLength int
|
||||
DoneCount int
|
||||
TotalCount int
|
||||
}
|
||||
|
||||
// pikpakAdapter / p115Adapter / p123Adapter / onedriveAdapter 把具体 driver 包装成 uploadTarget。
|
||||
const (
|
||||
spider91UploadDirName = "91 Spider"
|
||||
scriptCrawlerUploadRootDirName = "Script Crawlers"
|
||||
)
|
||||
|
||||
type migrationPlan struct {
|
||||
source Spider91LocalSource
|
||||
row *catalog.Drive
|
||||
sourceKinds []string
|
||||
targetDriveID string
|
||||
target uploadTarget
|
||||
uploadDir string
|
||||
keepLatestN int
|
||||
requireAssetsReady bool
|
||||
requirePreviewReady bool
|
||||
legacyBackfill bool
|
||||
}
|
||||
|
||||
// pikpakAdapter / p115Adapter / p123Adapter / onedriveAdapter / googledriveAdapter / wopanAdapter / guangyapanAdapter 把具体 driver 包装成 uploadTarget。
|
||||
//
|
||||
// 之所以不让 driver 直接实现 uploadTarget:
|
||||
//
|
||||
@@ -160,6 +204,69 @@ func (a *onedriveAdapter) Rename(ctx context.Context, fileID, newName string) er
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
type googledriveAdapter struct {
|
||||
d *googledrive.Driver
|
||||
}
|
||||
|
||||
func (a *googledriveAdapter) ID() string { return a.d.ID() }
|
||||
func (a *googledriveAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *googledriveAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *googledriveAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *googledriveAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
return UploadResult{FileID: res.FileID, Hash: res.Hash, Size: res.Size}, nil
|
||||
}
|
||||
func (a *googledriveAdapter) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
type wopanAdapter struct {
|
||||
d *wopan.Driver
|
||||
}
|
||||
|
||||
func (a *wopanAdapter) ID() string { return a.d.ID() }
|
||||
func (a *wopanAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *wopanAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *wopanAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *wopanAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
fileID, err := a.d.Upload(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
return UploadResult{FileID: fileID, Size: size}, nil
|
||||
}
|
||||
func (a *wopanAdapter) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
type guangyapanAdapter struct {
|
||||
d *guangyapan.Driver
|
||||
}
|
||||
|
||||
func (a *guangyapanAdapter) ID() string { return a.d.ID() }
|
||||
func (a *guangyapanAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *guangyapanAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *guangyapanAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *guangyapanAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
fileID, err := a.d.Upload(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
return UploadResult{FileID: fileID, Size: size}, nil
|
||||
}
|
||||
func (a *guangyapanAdapter) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
// adaptUploadTarget 把通用 drive 包装成 uploadTarget。
|
||||
// 不支持的盘 kind 返回 error;调用方静默跳过。
|
||||
func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
@@ -172,6 +279,12 @@ func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
return &p123Adapter{d: v}, nil
|
||||
case *onedrive.Driver:
|
||||
return &onedriveAdapter{d: v}, nil
|
||||
case *googledrive.Driver:
|
||||
return &googledriveAdapter{d: v}, nil
|
||||
case *wopan.Driver:
|
||||
return &wopanAdapter{d: v}, nil
|
||||
case *guangyapan.Driver:
|
||||
return &guangyapanAdapter{d: v}, nil
|
||||
case uploadTarget:
|
||||
// 测试或自定义实现可以直接传入;优先使用具体类型分支以拿到适配器。
|
||||
return v, nil
|
||||
@@ -201,9 +314,10 @@ type Config struct {
|
||||
// CaptchaCooldown 是迁移 worker 在遇到 PikPak captcha 错误(error_code
|
||||
// 4002 / 9)后整体进入冷却的时长。冷却期间 runOnce 直接返回,不再发起任何
|
||||
// PikPak API 请求,避免被进一步风控。0 时默认 5 分钟;< 0 关闭冷却(仅用于测试)。
|
||||
CaptchaCooldown time.Duration
|
||||
CommonThumbDir string
|
||||
OnMigrated func(videoID string)
|
||||
CaptchaCooldown time.Duration
|
||||
CommonThumbDir string
|
||||
OnMigrated func(videoID string)
|
||||
OnUploadProgress func(UploadProgress)
|
||||
}
|
||||
|
||||
type Migrator struct {
|
||||
@@ -332,59 +446,79 @@ func (m *Migrator) runOnce(ctx context.Context) {
|
||||
log.Printf("[spider91migrate] captcha cooldown ended at %s, resuming migration", until.Format(time.RFC3339))
|
||||
}
|
||||
|
||||
target, pp, err := m.resolveTarget()
|
||||
if err != nil {
|
||||
// 没目标就静默 —— 用户选择了本地保存,或还没配 115/PikPak drive。
|
||||
plans := m.migrationPlans(ctx)
|
||||
if len(plans) == 0 {
|
||||
// 没目标就静默 —— 用户选择了本地保存,或目标盘还没挂载。
|
||||
return
|
||||
}
|
||||
|
||||
migrated := 0
|
||||
for _, src := range m.spider91Drives() {
|
||||
backfillTargets := map[string]uploadTarget{}
|
||||
for _, plan := range plans {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return
|
||||
}
|
||||
n, err := m.migrateDrive(ctx, src, target, pp)
|
||||
n, err := m.migrateDrive(ctx, plan)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] drive=%s migrate batch error: %v", src.ID(), err)
|
||||
log.Printf("[spider91migrate] drive=%s migrate batch error: %v", plan.source.ID(), err)
|
||||
}
|
||||
migrated += n
|
||||
if active, _ := m.inCooldown(); active {
|
||||
if migrated > 0 {
|
||||
log.Printf("[spider91migrate] migrated %d video(s) to drive=%s", migrated, target)
|
||||
log.Printf("[spider91migrate] migrated %d video(s)", migrated)
|
||||
}
|
||||
return
|
||||
}
|
||||
if plan.legacyBackfill {
|
||||
backfillTargets[plan.targetDriveID] = plan.target
|
||||
}
|
||||
}
|
||||
if migrated > 0 {
|
||||
log.Printf("[spider91migrate] migrated %d video(s) to drive=%s", migrated, target)
|
||||
log.Printf("[spider91migrate] migrated %d video(s)", migrated)
|
||||
}
|
||||
|
||||
// 收尾:扫每个 spider91 drive 的本地目录,把 catalog 已经迁到别处但本地
|
||||
// 收尾:扫每个本地爬虫 drive 的 videos 目录,把 catalog 已经迁到别处但本地
|
||||
// 仍有残留的孤儿文件清掉。这是纯防御性兜底——正常路径下 migrateDrive
|
||||
// 已经在迁移成功后立刻 CleanupSpider91Local,不会留孤儿。
|
||||
for _, src := range m.spider91Drives() {
|
||||
for _, plan := range plans {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return
|
||||
}
|
||||
deleted, err := m.cleanupOldLocalVideos(ctx, src)
|
||||
deleted, err := m.cleanupOldLocalVideos(ctx, plan)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] cleanup drive=%s: %v", src.ID(), err)
|
||||
log.Printf("[spider91migrate] cleanup drive=%s: %v", plan.source.ID(), err)
|
||||
}
|
||||
if deleted > 0 {
|
||||
log.Printf("[spider91migrate] cleanup drive=%s deleted %d orphan local file(s)", src.ID(), deleted)
|
||||
log.Printf("[spider91migrate] cleanup drive=%s deleted %d orphan local file(s)", plan.source.ID(), deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// 回填:把已迁移到 PikPak 的 spider91-* 视频里文件名仍是旧格式
|
||||
// (比如刚迁完没改、或人工导入)的统一改成方案 B 期望的格式。
|
||||
// 这一步幂等:已经是期望格式的不会再调 Rename。
|
||||
if renamed, err := m.backfillFileNames(ctx, target, pp); err != nil {
|
||||
log.Printf("[spider91migrate] backfill names: %v", err)
|
||||
} else if renamed > 0 {
|
||||
log.Printf("[spider91migrate] backfilled %d %s file name(s) to desired format", renamed, m.targetKindForLog())
|
||||
for targetDriveID, pp := range backfillTargets {
|
||||
if renamed, err := m.backfillFileNames(ctx, targetDriveID, pp); err != nil {
|
||||
log.Printf("[spider91migrate] backfill names: %v", err)
|
||||
} else if renamed > 0 {
|
||||
log.Printf("[spider91migrate] backfilled %d %s file name(s) to desired format", renamed, pp.Kind())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Migrator) reportUploadProgress(progress UploadProgress) {
|
||||
if m == nil || m.cfg.OnUploadProgress == nil {
|
||||
return
|
||||
}
|
||||
progress.DriveID = strings.TrimSpace(progress.DriveID)
|
||||
if progress.DriveID == "" {
|
||||
return
|
||||
}
|
||||
if progress.State == "" {
|
||||
progress.State = "idle"
|
||||
}
|
||||
m.cfg.OnUploadProgress(progress)
|
||||
}
|
||||
|
||||
// targetKindForLog 把当前目标盘 kind 转成对人友好的简称,用于日志。
|
||||
// 解析失败时回退 "target"。
|
||||
func (m *Migrator) targetKindForLog() string {
|
||||
@@ -409,9 +543,17 @@ func (m *Migrator) resolveTarget() (string, uploadTarget, error) {
|
||||
return "", nil, errors.New("no target getter")
|
||||
}
|
||||
id := m.cfg.GetTargetDriveID()
|
||||
return m.resolveTargetID(id)
|
||||
}
|
||||
|
||||
func (m *Migrator) resolveTargetID(id string) (string, uploadTarget, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
return "", nil, errors.New("target drive not configured")
|
||||
}
|
||||
if m.cfg.Registry == nil {
|
||||
return "", nil, errors.New("registry not configured")
|
||||
}
|
||||
d, ok := m.cfg.Registry.Get(id)
|
||||
if !ok {
|
||||
return "", nil, fmt.Errorf("target drive %q not in registry", id)
|
||||
@@ -423,33 +565,143 @@ func (m *Migrator) resolveTarget() (string, uploadTarget, error) {
|
||||
return id, t, nil
|
||||
}
|
||||
|
||||
// spider91Drives 返回当前注册的所有 spider91 driver。
|
||||
func (m *Migrator) spider91Drives() []*spider91.Driver {
|
||||
func (m *Migrator) migrationPlans(ctx context.Context) []migrationPlan {
|
||||
if m == nil || m.cfg.Catalog == nil || m.cfg.Registry == nil {
|
||||
return nil
|
||||
}
|
||||
all := m.cfg.Registry.All()
|
||||
out := make([]*spider91.Driver, 0, len(all))
|
||||
out := make([]migrationPlan, 0, len(all))
|
||||
for _, d := range all {
|
||||
if d.Kind() != spider91.Kind {
|
||||
if d == nil {
|
||||
continue
|
||||
}
|
||||
if sd, ok := d.(*spider91.Driver); ok {
|
||||
src, ok := d.(Spider91LocalSource)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
row, err := m.cfg.Catalog.GetDrive(ctx, d.ID())
|
||||
if (err != nil || row == nil) && d.Kind() == spider91.Kind {
|
||||
row = &catalog.Drive{ID: d.ID(), Kind: spider91.Kind, RootID: "/"}
|
||||
}
|
||||
if row == nil {
|
||||
continue
|
||||
}
|
||||
switch row.Kind {
|
||||
case scriptcrawler.Kind:
|
||||
targetID := strings.TrimSpace(row.Credentials["upload_drive_id"])
|
||||
if targetID == "" {
|
||||
continue
|
||||
}
|
||||
resolvedID, target, err := m.resolveTargetID(targetID)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] crawler=%s upload target=%q unavailable: %v", row.ID, targetID, err)
|
||||
continue
|
||||
}
|
||||
out = append(out, migrationPlan{
|
||||
source: src,
|
||||
row: row,
|
||||
sourceKinds: crawlerSourceKindsForRow(row),
|
||||
targetDriveID: resolvedID,
|
||||
target: target,
|
||||
uploadDir: scriptCrawlerUploadDir(row.ID),
|
||||
keepLatestN: 0,
|
||||
requireAssetsReady: true,
|
||||
requirePreviewReady: row.TeaserEnabled,
|
||||
})
|
||||
case spider91.Kind:
|
||||
if m.cfg.GetTargetDriveID == nil {
|
||||
continue
|
||||
}
|
||||
targetID := strings.TrimSpace(m.cfg.GetTargetDriveID())
|
||||
if targetID == "" {
|
||||
continue
|
||||
}
|
||||
resolvedID, target, err := m.resolveTargetID(targetID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, migrationPlan{
|
||||
source: src,
|
||||
row: row,
|
||||
sourceKinds: []string{spider91.Kind},
|
||||
targetDriveID: resolvedID,
|
||||
target: target,
|
||||
uploadDir: spider91UploadDirName,
|
||||
keepLatestN: m.cfg.KeepLatestN,
|
||||
legacyBackfill: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func crawlerSourceKindsForRow(d *catalog.Drive) []string {
|
||||
kinds := []string{scriptcrawler.Kind}
|
||||
if d != nil && strings.EqualFold(strings.TrimSpace(d.Credentials["builtin"]), spider91.Kind) {
|
||||
kinds = append(kinds, spider91.Kind)
|
||||
}
|
||||
return kinds
|
||||
}
|
||||
|
||||
func scriptCrawlerUploadDir(driveID string) string {
|
||||
driveID = sanitizeUploadDirSegment(driveID)
|
||||
if driveID == "" {
|
||||
driveID = "crawler"
|
||||
}
|
||||
return scriptCrawlerUploadRootDirName + "/" + driveID
|
||||
}
|
||||
|
||||
func sanitizeUploadDirSegment(raw string) string {
|
||||
clean := sanitizeTitle(raw)
|
||||
clean = strings.Trim(clean, "/")
|
||||
if clean == "." || clean == ".." {
|
||||
return ""
|
||||
}
|
||||
return clean
|
||||
}
|
||||
|
||||
// spider91Drives 返回当前注册的所有 Spider91 来源本地爬虫 driver。
|
||||
func (m *Migrator) spider91Drives(ctx context.Context) []Spider91LocalSource {
|
||||
all := m.cfg.Registry.All()
|
||||
out := make([]Spider91LocalSource, 0, len(all))
|
||||
for _, d := range all {
|
||||
if !m.isSpider91SourceDrive(ctx, d) {
|
||||
continue
|
||||
}
|
||||
if sd, ok := d.(Spider91LocalSource); ok {
|
||||
out = append(out, sd)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// migrateDrive 对单个 spider91 drive 跑一批迁移;返回成功迁移的条数。
|
||||
//
|
||||
// 策略(与"本地缓存最新 N 个"语义一致):
|
||||
// - 列出 spider91 drive 本地 videos/ 目录所有 mp4 文件,按 mtime 降序排
|
||||
// - 跳过最新 KeepLatestN 个:这些是用户希望保留在本地的最新爬取
|
||||
// - 对剩下的(更旧)逐个处理:
|
||||
// - 还没迁移(drive_id 仍是 src.ID())→ 上传到目标盘 + 改 catalog + 删本地
|
||||
// - 已经迁移过但本地还有残留 → 仅删本地(兜底)
|
||||
//
|
||||
// KeepLatestN < 0 时不保护任何本地文件,全部尝试迁移(旧行为,主要给测试用)。
|
||||
func (m *Migrator) migrateDrive(ctx context.Context, src *spider91.Driver, targetDriveID string, pp uploadTarget) (int, error) {
|
||||
keepN := m.cfg.KeepLatestN
|
||||
func (m *Migrator) isSpider91SourceDrive(ctx context.Context, d drives.Drive) bool {
|
||||
if d == nil {
|
||||
return false
|
||||
}
|
||||
if d.Kind() == spider91.Kind {
|
||||
return true
|
||||
}
|
||||
if d.Kind() != scriptcrawler.Kind || m.cfg.Catalog == nil {
|
||||
return false
|
||||
}
|
||||
row, err := m.cfg.Catalog.GetDrive(ctx, d.ID())
|
||||
if err != nil || row == nil {
|
||||
return false
|
||||
}
|
||||
if row.Kind == spider91.Kind {
|
||||
return true
|
||||
}
|
||||
return row.Kind == scriptcrawler.Kind && strings.EqualFold(strings.TrimSpace(row.Credentials["builtin"]), spider91.Kind)
|
||||
}
|
||||
|
||||
// migrateDrive 对单个本地爬虫 drive 跑一批迁移;返回成功迁移的条数。
|
||||
func (m *Migrator) migrateDrive(ctx context.Context, plan migrationPlan) (int, error) {
|
||||
src := plan.source
|
||||
if src == nil || plan.target == nil || plan.targetDriveID == "" {
|
||||
return 0, nil
|
||||
}
|
||||
keepN := plan.keepLatestN
|
||||
if keepN < 0 {
|
||||
keepN = 0
|
||||
}
|
||||
@@ -479,28 +731,46 @@ func (m *Migrator) migrateDrive(ctx context.Context, src *spider91.Driver, targe
|
||||
files = append(files, localFile{name: e.Name(), modTime: info.ModTime()})
|
||||
}
|
||||
|
||||
// 本地数量没超过 keepN 时不动任何文件 —— 这条是 KeepLatestN 语义的核心
|
||||
if m.cfg.KeepLatestN >= 0 && len(files) <= keepN {
|
||||
if plan.keepLatestN >= 0 && len(files) <= keepN {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// 按 mtime 降序:最新的排前面,保留前 keepN 个
|
||||
sort.Slice(files, func(i, j int) bool { return files[i].modTime.After(files[j].modTime) })
|
||||
|
||||
// 候选 = 跳过最新 keepN 个之外的(更旧的)。KeepLatestN < 0 时 candidates=files。
|
||||
skip := keepN
|
||||
if m.cfg.KeepLatestN < 0 {
|
||||
if plan.keepLatestN < 0 {
|
||||
skip = 0
|
||||
}
|
||||
candidates := files
|
||||
if skip < len(files) {
|
||||
candidates = files[skip:]
|
||||
} else {
|
||||
m.reportUploadProgress(UploadProgress{DriveID: src.ID(), State: "idle"})
|
||||
return 0, nil
|
||||
}
|
||||
totalCandidates := len(candidates)
|
||||
m.reportUploadProgress(UploadProgress{
|
||||
DriveID: src.ID(),
|
||||
State: "uploading",
|
||||
QueueLength: totalCandidates,
|
||||
TotalCount: totalCandidates,
|
||||
})
|
||||
defer m.reportUploadProgress(UploadProgress{DriveID: src.ID(), State: "idle"})
|
||||
|
||||
localVideos, err := m.cfg.Catalog.ListVideosByDriveID(ctx, src.ID(), 100000)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("list local catalog videos: %w", err)
|
||||
}
|
||||
byFileID := make(map[string]*catalog.Video, len(localVideos))
|
||||
for _, v := range localVideos {
|
||||
if v != nil && strings.TrimSpace(v.FileID) != "" {
|
||||
byFileID[v.FileID] = v
|
||||
}
|
||||
}
|
||||
|
||||
migrated := 0
|
||||
for _, f := range candidates {
|
||||
processed := 0
|
||||
for index, f := range candidates {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return migrated, err
|
||||
}
|
||||
@@ -508,21 +778,87 @@ func (m *Migrator) migrateDrive(ctx context.Context, src *spider91.Driver, targe
|
||||
break
|
||||
}
|
||||
|
||||
viewkey := stripExt(f.name)
|
||||
videoID := "spider91-" + src.ID() + "-" + viewkey
|
||||
v, err := m.cfg.Catalog.GetVideo(ctx, videoID)
|
||||
if err != nil || v == nil {
|
||||
// 找不到 catalog 行:保险起见保留本地,让管理员可见
|
||||
v := m.findVideoForLocalFile(ctx, plan, f.name, byFileID)
|
||||
if v == nil {
|
||||
processed++
|
||||
m.reportUploadProgress(UploadProgress{
|
||||
DriveID: src.ID(),
|
||||
State: "uploading",
|
||||
QueueLength: maxInt(totalCandidates-processed, 0),
|
||||
DoneCount: processed,
|
||||
TotalCount: totalCandidates,
|
||||
})
|
||||
continue
|
||||
}
|
||||
m.reportUploadProgress(UploadProgress{
|
||||
DriveID: src.ID(),
|
||||
State: "uploading",
|
||||
CurrentTitle: v.Title,
|
||||
QueueLength: maxInt(totalCandidates-index-1, 0),
|
||||
DoneCount: processed,
|
||||
TotalCount: totalCandidates,
|
||||
})
|
||||
|
||||
if v.DriveID != src.ID() {
|
||||
// catalog 已迁移到别的 drive,但本地还有残留 → 兜底删本地
|
||||
CleanupSpider91Local(src, v.FileID)
|
||||
CleanupSpider91Local(src, f.name)
|
||||
processed++
|
||||
m.reportUploadProgress(UploadProgress{
|
||||
DriveID: src.ID(),
|
||||
State: "uploading",
|
||||
QueueLength: maxInt(totalCandidates-processed, 0),
|
||||
DoneCount: processed,
|
||||
TotalCount: totalCandidates,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
ok, err := m.migrateOne(ctx, v, src, targetDriveID, pp)
|
||||
if targetDuplicate, err := m.cfg.Catalog.FindEquivalentVideoOnDrive(ctx, v, plan.targetDriveID); err != nil {
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
log.Printf("[spider91migrate] %s find target duplicate: %v", v.ID, err)
|
||||
}
|
||||
} else if targetDuplicate != nil {
|
||||
ok, err := m.bindToExistingTarget(ctx, v, targetDuplicate, plan)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] %s: %v", v.ID, err)
|
||||
continue
|
||||
}
|
||||
if ok {
|
||||
migrated++
|
||||
if m.cfg.OnMigrated != nil {
|
||||
m.cfg.OnMigrated(v.ID)
|
||||
}
|
||||
}
|
||||
processed++
|
||||
m.reportUploadProgress(UploadProgress{
|
||||
DriveID: src.ID(),
|
||||
State: "uploading",
|
||||
QueueLength: maxInt(totalCandidates-processed, 0),
|
||||
DoneCount: processed,
|
||||
TotalCount: totalCandidates,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
if plan.requireAssetsReady {
|
||||
ready, err := m.crawlerVideoAssetsReady(ctx, v, plan.requirePreviewReady)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] %s check generated assets: %v", v.ID, err)
|
||||
continue
|
||||
}
|
||||
if !ready {
|
||||
processed++
|
||||
m.reportUploadProgress(UploadProgress{
|
||||
DriveID: src.ID(),
|
||||
State: "uploading",
|
||||
QueueLength: maxInt(totalCandidates-processed, 0),
|
||||
DoneCount: processed,
|
||||
TotalCount: totalCandidates,
|
||||
})
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
ok, err := m.migrateOne(ctx, v, plan)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] %s: %v", v.ID, err)
|
||||
// captcha 错误(4002 / 9)说明 PikPak 当前正拒绝我们;继续在
|
||||
@@ -542,14 +878,67 @@ func (m *Migrator) migrateDrive(ctx context.Context, src *spider91.Driver, targe
|
||||
m.cfg.OnMigrated(v.ID)
|
||||
}
|
||||
}
|
||||
processed++
|
||||
m.reportUploadProgress(UploadProgress{
|
||||
DriveID: src.ID(),
|
||||
State: "uploading",
|
||||
QueueLength: maxInt(totalCandidates-processed, 0),
|
||||
DoneCount: processed,
|
||||
TotalCount: totalCandidates,
|
||||
})
|
||||
}
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
// migrateOne 把单条 spider91 视频上传到目标盘并改写 catalog。
|
||||
func maxInt(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (m *Migrator) findVideoForLocalFile(ctx context.Context, plan migrationPlan, localFile string, byFileID map[string]*catalog.Video) *catalog.Video {
|
||||
if v := byFileID[localFile]; v != nil {
|
||||
return v
|
||||
}
|
||||
sourceID := stripExt(localFile)
|
||||
driveID := ""
|
||||
if plan.source != nil {
|
||||
driveID = plan.source.ID()
|
||||
}
|
||||
for _, kind := range plan.sourceKinds {
|
||||
id := scriptcrawler.BuildVideoIDForKind(kind, driveID, sourceID)
|
||||
v, err := m.cfg.Catalog.GetVideo(ctx, id)
|
||||
if err == nil && v != nil {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Migrator) crawlerVideoAssetsReady(ctx context.Context, v *catalog.Video, requirePreview bool) (bool, error) {
|
||||
if v == nil {
|
||||
return false, nil
|
||||
}
|
||||
fingerprintReady := strings.EqualFold(strings.TrimSpace(v.FingerprintStatus), "ready") || strings.TrimSpace(v.SampledSHA256) != ""
|
||||
if !fingerprintReady {
|
||||
return false, nil
|
||||
}
|
||||
if !requirePreview {
|
||||
return true, nil
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(v.PreviewStatus), "ready") {
|
||||
return true, nil
|
||||
}
|
||||
return m.cfg.Catalog.HasReadyEquivalentPreview(ctx, v)
|
||||
}
|
||||
|
||||
// migrateOne 把单条本地爬虫视频上传到目标盘并改写 catalog。
|
||||
// 返回 (true, nil) 表示真的迁了一条;(false, nil) 表示跳过(本地文件已不在等);
|
||||
// (false, err) 表示真出错。
|
||||
func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider91.Driver, targetDriveID string, pp uploadTarget) (bool, error) {
|
||||
func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, plan migrationPlan) (bool, error) {
|
||||
src := plan.source
|
||||
pp := plan.target
|
||||
path, err := src.VideoPath(v.FileID)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("resolve local path: %w", err)
|
||||
@@ -573,20 +962,11 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// 上传到目标盘 rootID 下的固定 "91 Spider" 子目录。若用户把目标盘 rootID
|
||||
// 配成某个自定义目录,这里会在该自定义目录下查找/创建 "91 Spider"。
|
||||
// 上传名走 desiredPikPakName 算出来的方案 B 格式:
|
||||
//
|
||||
// <sanitized title>-<viewkey 后 8 位>.<ext>
|
||||
//
|
||||
// 这样网盘 Web 端列出来的文件名能直接看出是哪个视频,
|
||||
// 又用 viewkey 后 8 位避免同标题撞名。所有目标盘共用同一格式,
|
||||
// 简化前端 / catalog 的认知。
|
||||
parent, err := pp.EnsureDir(ctx, spider91UploadDirName)
|
||||
parent, err := pp.EnsureDir(ctx, plan.uploadDir)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s ensure %q dir: %w", pp.Kind(), spider91UploadDirName, err)
|
||||
return false, fmt.Errorf("%s ensure %q dir: %w", pp.Kind(), plan.uploadDir, err)
|
||||
}
|
||||
uploadName := desiredPikPakName(v.Title, extractViewKey(v.ID), v.Ext)
|
||||
uploadName := desiredPikPakName(v.Title, sourceIDForUploadName(v, plan), v.Ext)
|
||||
res, err := pp.UploadAndReportHash(ctx, parent, uploadName, f, info.Size())
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s upload: %w", pp.Kind(), err)
|
||||
@@ -596,7 +976,7 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
|
||||
}
|
||||
|
||||
// 事务性改写 catalog 行:drive_id / file_id / content_hash
|
||||
if err := m.cfg.Catalog.MigrateVideoToDrive(ctx, v.ID, targetDriveID, res.FileID, res.Hash); err != nil {
|
||||
if err := m.cfg.Catalog.MigrateVideoToDrive(ctx, v.ID, plan.targetDriveID, res.FileID, res.Hash); err != nil {
|
||||
return false, fmt.Errorf("catalog migrate: %w", err)
|
||||
}
|
||||
m.preserveCrawledThumbnail(ctx, src, v)
|
||||
@@ -608,11 +988,60 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
|
||||
// 删除本地 mp4 和源 thumb(公共 /p/thumb 副本已在 preserveCrawledThumbnail 中保留)。
|
||||
CleanupSpider91Local(src, v.FileID)
|
||||
|
||||
log.Printf("[spider91migrate] %s migrated to drive=%s(kind=%s) file=%s name=%q", v.ID, targetDriveID, pp.Kind(), res.FileID, uploadName)
|
||||
log.Printf("[spider91migrate] %s migrated to drive=%s(kind=%s) file=%s name=%q", v.ID, plan.targetDriveID, pp.Kind(), res.FileID, uploadName)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *Migrator) preserveCrawledThumbnail(ctx context.Context, src *spider91.Driver, v *catalog.Video) {
|
||||
func (m *Migrator) bindToExistingTarget(ctx context.Context, v, target *catalog.Video, plan migrationPlan) (bool, error) {
|
||||
if v == nil || target == nil || plan.source == nil {
|
||||
return false, nil
|
||||
}
|
||||
if plan.targetDriveID == "" || target.FileID == "" {
|
||||
return false, nil
|
||||
}
|
||||
if err := m.cfg.Catalog.MigrateVideoToDrive(ctx, v.ID, plan.targetDriveID, target.FileID, firstNonEmpty(target.ContentHash, v.ContentHash)); err != nil {
|
||||
return false, fmt.Errorf("catalog bind existing target: %w", err)
|
||||
}
|
||||
if target.FileName != "" {
|
||||
if err := m.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{FileName: target.FileName}); err != nil {
|
||||
log.Printf("[spider91migrate] %s update file_name after duplicate bind: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
m.preserveCrawledThumbnail(ctx, plan.source, v)
|
||||
CleanupSpider91Local(plan.source, v.FileID)
|
||||
log.Printf("[spider91migrate] %s bound to existing drive=%s(kind=%s) file=%s duplicate=%s", v.ID, plan.targetDriveID, plan.target.Kind(), target.FileID, target.ID)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func sourceIDForUploadName(v *catalog.Video, plan migrationPlan) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
if plan.legacyBackfill {
|
||||
return extractViewKey(v.ID)
|
||||
}
|
||||
for _, kind := range plan.sourceKinds {
|
||||
prefix := kind + "-" + plan.source.ID() + "-"
|
||||
if strings.HasPrefix(v.ID, prefix) {
|
||||
return strings.TrimPrefix(v.ID, prefix)
|
||||
}
|
||||
}
|
||||
if v.FileID != "" {
|
||||
return stripExt(v.FileID)
|
||||
}
|
||||
return extractViewKey(v.ID)
|
||||
}
|
||||
|
||||
func (m *Migrator) preserveCrawledThumbnail(ctx context.Context, src Spider91LocalSource, v *catalog.Video) {
|
||||
if m == nil || m.cfg.Catalog == nil || src == nil || v == nil || v.ID == "" || v.FileID == "" {
|
||||
return
|
||||
}
|
||||
@@ -651,7 +1080,7 @@ func (m *Migrator) preserveCrawledThumbnail(ctx context.Context, src *spider91.D
|
||||
v.ThumbnailURL = "/p/thumb/" + v.ID
|
||||
}
|
||||
|
||||
func findSpider91ThumbPath(src *spider91.Driver, fileID string) (string, bool) {
|
||||
func findSpider91ThumbPath(src Spider91LocalSource, fileID string) (string, bool) {
|
||||
thumbBase := stripExt(fileID)
|
||||
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
|
||||
thumbPath, err := src.ThumbPath(thumbBase + ext)
|
||||
@@ -697,7 +1126,7 @@ func copyFileAtomic(src, dst string) error {
|
||||
// 我们不知道具体是 .jpg 还是别的,逐个尝试常见后缀)。
|
||||
//
|
||||
// 暴露成包级函数方便 cleanup 模块复用(任务 6)。
|
||||
func CleanupSpider91Local(src *spider91.Driver, fileID string) {
|
||||
func CleanupSpider91Local(src Spider91LocalSource, fileID string) {
|
||||
videoPath, err := src.VideoPath(fileID)
|
||||
if err == nil {
|
||||
if err := os.Remove(videoPath); err != nil && !os.IsNotExist(err) {
|
||||
@@ -734,7 +1163,11 @@ func stripExt(name string) string {
|
||||
// 找到孤儿。
|
||||
//
|
||||
// 返回实际删除的文件个数。
|
||||
func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driver) (int, error) {
|
||||
func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, plan migrationPlan) (int, error) {
|
||||
src := plan.source
|
||||
if src == nil {
|
||||
return 0, nil
|
||||
}
|
||||
entries, err := os.ReadDir(src.VideosDir())
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
@@ -751,18 +1184,13 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driv
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
viewkey := stripExt(e.Name())
|
||||
videoID := "spider91-" + src.ID() + "-" + viewkey
|
||||
v, err := m.cfg.Catalog.GetVideo(ctx, videoID)
|
||||
if err != nil || v == nil {
|
||||
// 找不到 catalog 行:保险起见保留,等管理员处理
|
||||
v := m.findVideoForLocalFile(ctx, plan, e.Name(), nil)
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if v.DriveID == src.ID() {
|
||||
// 还没迁移,归 migrateDrive 管,不在这里动
|
||||
continue
|
||||
}
|
||||
// 已迁移到别的 drive 但本地还有 → 删
|
||||
path, perr := src.VideoPath(e.Name())
|
||||
if perr != nil {
|
||||
continue
|
||||
@@ -785,7 +1213,7 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driv
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// backfillFileNames 扫描目标 drive(PikPak、115、123 或 OneDrive)下所有 spider91-* 起始 ID 的视频,
|
||||
// backfillFileNames 扫描目标 drive(PikPak、115、123、OneDrive、Google Drive、联通网盘或光鸭网盘)下所有 spider91-* 起始 ID 的视频,
|
||||
// 对文件名不是 desiredPikPakName(...) 期望格式的,调 target.Rename 修正,
|
||||
// 并把 catalog.file_name 同步到新名字。
|
||||
//
|
||||
|
||||
@@ -14,9 +14,12 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/googledrive"
|
||||
"github.com/video-site/backend/internal/drives/p123"
|
||||
"github.com/video-site/backend/internal/drives/pikpak"
|
||||
"github.com/video-site/backend/internal/drives/scriptcrawler"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/drives/wopan"
|
||||
)
|
||||
|
||||
// fakeRegistry 是 Registry 接口的最小实现。
|
||||
@@ -342,6 +345,89 @@ func writeSpider91Video(t *testing.T, cat *catalog.Catalog, d *spider91.Driver,
|
||||
return id
|
||||
}
|
||||
|
||||
func setupScriptCrawler(t *testing.T, id string) *scriptcrawler.Driver {
|
||||
t.Helper()
|
||||
d := scriptcrawler.New(scriptcrawler.Config{ID: id, RootDir: t.TempDir()})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("scriptcrawler init: %v", err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func seedScriptCrawlerDrive(t *testing.T, cat *catalog.Catalog, d *scriptcrawler.Driver, uploadDriveID string) {
|
||||
t.Helper()
|
||||
if err := cat.UpsertDrive(context.Background(), &catalog.Drive{
|
||||
ID: d.ID(),
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Script Crawler",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"script_path": "/tmp/crawler.py",
|
||||
"upload_drive_id": uploadDriveID,
|
||||
},
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed scriptcrawler drive: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func setScriptCrawlerTeaserEnabled(t *testing.T, cat *catalog.Catalog, driveID string, enabled bool) {
|
||||
t.Helper()
|
||||
if err := cat.SetDriveTeaserEnabled(context.Background(), driveID, enabled); err != nil {
|
||||
t.Fatalf("set scriptcrawler teaser enabled: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeScriptCrawlerVideo(t *testing.T, cat *catalog.Catalog, d *scriptcrawler.Driver, sourceID, ext string, content []byte, readyAssets bool) string {
|
||||
t.Helper()
|
||||
fileID := sourceID + ext
|
||||
path, err := d.VideoPath(fileID)
|
||||
if err != nil {
|
||||
t.Fatalf("video path: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, content, 0o644); err != nil {
|
||||
t.Fatalf("write video: %v", err)
|
||||
}
|
||||
thumbPath, err := d.ThumbPath(sourceID + ".jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("thumb path: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(thumbPath, []byte("thumb"), 0o644); err != nil {
|
||||
t.Fatalf("write thumb: %v", err)
|
||||
}
|
||||
now := time.Now()
|
||||
id := scriptcrawler.BuildVideoID(d.ID(), sourceID)
|
||||
previewStatus := "pending"
|
||||
if readyAssets {
|
||||
previewStatus = "ready"
|
||||
}
|
||||
v := &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: d.ID(),
|
||||
FileID: fileID,
|
||||
FileName: fileID,
|
||||
Title: "Crawler " + sourceID,
|
||||
Author: "tester",
|
||||
Ext: strings.TrimPrefix(ext, "."),
|
||||
Quality: "HD",
|
||||
Size: int64(len(content)),
|
||||
ThumbnailURL: "/p/thumb/" + id,
|
||||
PreviewStatus: previewStatus,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if err := cat.UpsertVideo(context.Background(), v); err != nil {
|
||||
t.Fatalf("upsert scriptcrawler video: %v", err)
|
||||
}
|
||||
if readyAssets {
|
||||
if err := cat.UpdateVideoFingerprint(context.Background(), id, "sampled-"+sourceID, "ready", ""); err != nil {
|
||||
t.Fatalf("mark fingerprint ready: %v", err)
|
||||
}
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src, _ := setupSpider91(t)
|
||||
@@ -417,6 +503,215 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceMigratesReadyScriptCrawlerVideoToConfiguredUploadDrive(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src := setupScriptCrawler(t, "crawler-alpha")
|
||||
pp := newFakePikPak("pikpak-target", "pikpak-root-id")
|
||||
seedScriptCrawlerDrive(t, cat, src, pp.ID())
|
||||
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(pp)
|
||||
|
||||
id := writeScriptCrawlerVideo(t, cat, src, "source-with-dash-001", ".mp4", []byte("script video bytes"), true)
|
||||
commonThumbDir := t.TempDir()
|
||||
|
||||
m := New(Config{
|
||||
Catalog: cat,
|
||||
Registry: reg,
|
||||
CommonThumbDir: commonThumbDir,
|
||||
})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
if pp.uploadCalls != 1 {
|
||||
t.Fatalf("upload calls = %d, want 1", pp.uploadCalls)
|
||||
}
|
||||
wantDir := "Script Crawlers/crawler-alpha"
|
||||
if len(pp.ensureCalls) != 1 || pp.ensureCalls[0] != wantDir {
|
||||
t.Fatalf("ensure calls = %#v, want %q", pp.ensureCalls, wantDir)
|
||||
}
|
||||
wantName := desiredPikPakName("Crawler source-with-dash-001", "source-with-dash-001", "mp4")
|
||||
if gotParent := pp.gotParents[wantName]; gotParent != "pikpak-root-id/"+wantDir {
|
||||
t.Fatalf("upload parent = %q, want root/%s", gotParent, wantDir)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("get migrated video: %v", err)
|
||||
}
|
||||
if got.DriveID != pp.ID() {
|
||||
t.Fatalf("drive_id = %q, want %q", got.DriveID, pp.ID())
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want remote upload id", got.FileID)
|
||||
}
|
||||
if got.FileName != wantName {
|
||||
t.Fatalf("file_name = %q, want %q", got.FileName, wantName)
|
||||
}
|
||||
if got.PreviewStatus != "ready" || got.FingerprintStatus != "ready" || got.SampledSHA256 == "" {
|
||||
t.Fatalf("generated assets not preserved after migration: preview=%q fingerprint=%q sampled=%q", got.PreviewStatus, got.FingerprintStatus, got.SampledSHA256)
|
||||
}
|
||||
videoPath, _ := src.VideoPath("source-with-dash-001.mp4")
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local scriptcrawler video still exists or stat error %v", err)
|
||||
}
|
||||
thumbPath, _ := src.ThumbPath("source-with-dash-001.jpg")
|
||||
if _, err := os.Stat(thumbPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local scriptcrawler thumb still exists or stat error %v", err)
|
||||
}
|
||||
commonThumbPath := filepath.Join(commonThumbDir, id+".jpg")
|
||||
if data, err := os.ReadFile(commonThumbPath); err != nil || string(data) != "thumb" {
|
||||
t.Fatalf("common thumb = %q, %v; want copied crawled thumb", string(data), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceSkipsScriptCrawlerVideoUntilPreviewAndFingerprintReady(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src := setupScriptCrawler(t, "crawler-beta")
|
||||
pp := newFakePikPak("pikpak-target", "pikpak-root-id")
|
||||
seedScriptCrawlerDrive(t, cat, src, pp.ID())
|
||||
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(pp)
|
||||
|
||||
id := writeScriptCrawlerVideo(t, cat, src, "pending-assets", ".mp4", []byte("script video bytes"), false)
|
||||
m := New(Config{Catalog: cat, Registry: reg})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
if pp.uploadCalls != 0 {
|
||||
t.Fatalf("upload calls = %d, want 0 while generated assets are pending", pp.uploadCalls)
|
||||
}
|
||||
got, err := cat.GetVideo(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DriveID != src.ID() {
|
||||
t.Fatalf("drive_id = %q, want local crawler drive %q", got.DriveID, src.ID())
|
||||
}
|
||||
videoPath, _ := src.VideoPath("pending-assets.mp4")
|
||||
if _, err := os.Stat(videoPath); err != nil {
|
||||
t.Fatalf("local video should remain while assets pending: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceMigratesScriptCrawlerVideoWithoutPreviewWhenTeaserDisabled(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src := setupScriptCrawler(t, "crawler-no-preview")
|
||||
pp := newFakePikPak("pikpak-target", "pikpak-root-id")
|
||||
seedScriptCrawlerDrive(t, cat, src, pp.ID())
|
||||
setScriptCrawlerTeaserEnabled(t, cat, src.ID(), false)
|
||||
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(pp)
|
||||
|
||||
id := writeScriptCrawlerVideo(t, cat, src, "fingerprint-ready", ".mp4", []byte("script video bytes"), false)
|
||||
if err := cat.UpdateVideoFingerprint(context.Background(), id, "sampled-fingerprint-ready", "ready", ""); err != nil {
|
||||
t.Fatalf("mark fingerprint ready: %v", err)
|
||||
}
|
||||
if err := cat.UpdatePreview(context.Background(), id, "", "disabled"); err != nil {
|
||||
t.Fatalf("mark preview disabled: %v", err)
|
||||
}
|
||||
|
||||
m := New(Config{Catalog: cat, Registry: reg})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
if pp.uploadCalls != 1 {
|
||||
t.Fatalf("upload calls = %d, want 1 when preview generation is disabled", pp.uploadCalls)
|
||||
}
|
||||
got, err := cat.GetVideo(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("get migrated video: %v", err)
|
||||
}
|
||||
if got.DriveID != pp.ID() {
|
||||
t.Fatalf("drive_id = %q, want %q", got.DriveID, pp.ID())
|
||||
}
|
||||
if got.PreviewStatus != "disabled" || got.FingerprintStatus != "ready" || got.SampledSHA256 == "" {
|
||||
t.Fatalf("asset status after migration = preview %q fingerprint %q sampled %q, want disabled/ready/non-empty", got.PreviewStatus, got.FingerprintStatus, got.SampledSHA256)
|
||||
}
|
||||
videoPath, _ := src.VideoPath("fingerprint-ready.mp4")
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local scriptcrawler video still exists or stat error %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceBindsScriptCrawlerDuplicateToExistingTargetWithoutUpload(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src := setupScriptCrawler(t, "crawler-duplicate")
|
||||
pp := newFakePikPak("pikpak-target", "pikpak-root-id")
|
||||
seedScriptCrawlerDrive(t, cat, src, pp.ID())
|
||||
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(pp)
|
||||
|
||||
content := []byte("duplicate script video bytes")
|
||||
id := writeScriptCrawlerVideo(t, cat, src, "duplicate-source", ".mp4", content, false)
|
||||
sampled := "same-sampled-fingerprint"
|
||||
if err := cat.UpdateVideoFingerprint(context.Background(), id, sampled, "ready", ""); err != nil {
|
||||
t.Fatalf("mark source fingerprint ready: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
target := &catalog.Video{
|
||||
ID: "pikpak-existing-duplicate",
|
||||
DriveID: pp.ID(),
|
||||
FileID: "existing-target-file",
|
||||
FileName: "existing-target-name.mp4",
|
||||
ContentHash: "existing-content-hash",
|
||||
Title: "Existing duplicate",
|
||||
Ext: "mp4",
|
||||
Size: int64(len(content)),
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now.Add(-time.Hour),
|
||||
CreatedAt: now.Add(-time.Hour),
|
||||
UpdatedAt: now.Add(-time.Hour),
|
||||
}
|
||||
if err := cat.UpsertVideo(context.Background(), target); err != nil {
|
||||
t.Fatalf("upsert existing target: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(context.Background(), target.ID, sampled, "ready", ""); err != nil {
|
||||
t.Fatalf("mark target fingerprint ready: %v", err)
|
||||
}
|
||||
|
||||
commonThumbDir := t.TempDir()
|
||||
m := New(Config{Catalog: cat, Registry: reg, CommonThumbDir: commonThumbDir})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
if pp.uploadCalls != 0 {
|
||||
t.Fatalf("upload calls = %d, want 0 when equivalent target file already exists", pp.uploadCalls)
|
||||
}
|
||||
got, err := cat.GetVideo(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("get bound video: %v", err)
|
||||
}
|
||||
if got.DriveID != pp.ID() {
|
||||
t.Fatalf("drive_id = %q, want %q", got.DriveID, pp.ID())
|
||||
}
|
||||
if got.FileID != target.FileID {
|
||||
t.Fatalf("file_id = %q, want existing target file %q", got.FileID, target.FileID)
|
||||
}
|
||||
if got.FileName != target.FileName {
|
||||
t.Fatalf("file_name = %q, want existing target name %q", got.FileName, target.FileName)
|
||||
}
|
||||
if got.ContentHash != target.ContentHash {
|
||||
t.Fatalf("content_hash = %q, want %q", got.ContentHash, target.ContentHash)
|
||||
}
|
||||
videoPath, _ := src.VideoPath("duplicate-source.mp4")
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local duplicate video still exists or stat error %v", err)
|
||||
}
|
||||
thumbPath, _ := src.ThumbPath("duplicate-source.jpg")
|
||||
if _, err := os.Stat(thumbPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local duplicate thumb still exists or stat error %v", err)
|
||||
}
|
||||
commonThumbPath := filepath.Join(commonThumbDir, id+".jpg")
|
||||
if data, err := os.ReadFile(commonThumbPath); err != nil || string(data) != "thumb" {
|
||||
t.Fatalf("common thumb = %q, %v; want copied crawled thumb", string(data), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceSkipsWhenLocalFileMissing(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src, _ := setupSpider91(t)
|
||||
@@ -576,7 +871,10 @@ func TestCleanupRemovesAllAlreadyMigratedOrphans(t *testing.T) {
|
||||
GetTargetDriveID: func() string { return pp.ID() },
|
||||
})
|
||||
|
||||
deleted, err := m.cleanupOldLocalVideos(context.Background(), src)
|
||||
deleted, err := m.cleanupOldLocalVideos(context.Background(), migrationPlan{
|
||||
source: src,
|
||||
sourceKinds: []string{spider91.Kind},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("cleanup: %v", err)
|
||||
}
|
||||
@@ -598,6 +896,95 @@ func TestCleanupRemovesAllAlreadyMigratedOrphans(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceMigratesBuiltInSpider91ScriptCrawlerSource(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat := setupCatalog(t)
|
||||
src := scriptcrawler.New(scriptcrawler.Config{ID: "spider-script", RootDir: t.TempDir()})
|
||||
if err := src.Init(ctx); err != nil {
|
||||
t.Fatalf("scriptcrawler init: %v", err)
|
||||
}
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: src.ID(),
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Built-in Spider91",
|
||||
Credentials: map[string]string{
|
||||
"builtin": "spider91",
|
||||
"script_path": "/tmp/spider91.py",
|
||||
"upload_drive_id": "pikpak-target",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert source drive: %v", err)
|
||||
}
|
||||
pp := newFakePikPak("pikpak-target", "pikpak-root-id")
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(pp)
|
||||
|
||||
fileID := "vk-script.mp4"
|
||||
videoPath, err := src.VideoPath(fileID)
|
||||
if err != nil {
|
||||
t.Fatalf("video path: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(videoPath, []byte("scriptcrawler spider91 video"), 0o644); err != nil {
|
||||
t.Fatalf("write video: %v", err)
|
||||
}
|
||||
thumbPath, err := src.ThumbPath("vk-script.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("thumb path: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(thumbPath, []byte("thumb"), 0o644); err != nil {
|
||||
t.Fatalf("write thumb: %v", err)
|
||||
}
|
||||
now := time.Now()
|
||||
id := "spider91-" + src.ID() + "-vk-script"
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: src.ID(),
|
||||
FileID: fileID,
|
||||
FileName: fileID,
|
||||
Title: "Scriptcrawler Spider91",
|
||||
Author: "91porn",
|
||||
Ext: "mp4",
|
||||
Quality: "HD",
|
||||
Size: int64(len("scriptcrawler spider91 video")),
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert video: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, id, "sampled-vk-script", "ready", ""); err != nil {
|
||||
t.Fatalf("mark fingerprint ready: %v", err)
|
||||
}
|
||||
|
||||
m := New(Config{
|
||||
Catalog: cat,
|
||||
Registry: reg,
|
||||
GetTargetDriveID: func() string { return pp.ID() },
|
||||
KeepLatestN: -1,
|
||||
CommonThumbDir: t.TempDir(),
|
||||
})
|
||||
m.runOnce(ctx)
|
||||
|
||||
if pp.uploadCalls != 1 {
|
||||
t.Fatalf("upload calls = %d, want 1", pp.uploadCalls)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("get migrated video: %v", err)
|
||||
}
|
||||
if got.DriveID != pp.ID() {
|
||||
t.Fatalf("drive_id = %q, want %q", got.DriveID, pp.ID())
|
||||
}
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local video stat err = %v, want not exist", err)
|
||||
}
|
||||
if _, err := os.Stat(thumbPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local thumb stat err = %v, want not exist", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunOnceKeepsAllLocalWhenWithinKeepWindow 验证:本地文件数 ≤ KeepLatestN 时
|
||||
// 一律不上传,全部留作"最新 N"缓存。这是用户的核心需求:刚爬下来的 15 个不要立即被传走。
|
||||
func TestRunOnceKeepsAllLocalWhenWithinKeepWindow(t *testing.T) {
|
||||
@@ -1095,7 +1482,38 @@ func TestAdaptUploadTargetSupportsP123Driver(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115、123 也不是 OneDrive 时,
|
||||
func TestAdaptUploadTargetSupportsGoogleDriveDriver(t *testing.T) {
|
||||
d := googledrive.New(googledrive.Config{
|
||||
ID: "google-target",
|
||||
RootID: "root-google",
|
||||
RefreshToken: "refresh-token",
|
||||
})
|
||||
target, err := adaptUploadTarget(d)
|
||||
if err != nil {
|
||||
t.Fatalf("adaptUploadTarget() error = %v", err)
|
||||
}
|
||||
if target.ID() != "google-target" || target.Kind() != "googledrive" || target.RootID() != "root-google" {
|
||||
t.Fatalf("target id/kind/root = %q/%q/%q, want google-target/googledrive/root-google", target.ID(), target.Kind(), target.RootID())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdaptUploadTargetSupportsWopanDriver(t *testing.T) {
|
||||
d := wopan.New(wopan.Config{
|
||||
ID: "wopan-target",
|
||||
RootID: "root-wopan",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
})
|
||||
target, err := adaptUploadTarget(d)
|
||||
if err != nil {
|
||||
t.Fatalf("adaptUploadTarget() error = %v", err)
|
||||
}
|
||||
if target.ID() != "wopan-target" || target.Kind() != "wopan" || target.RootID() != "root-wopan" {
|
||||
t.Fatalf("target id/kind/root = %q/%q/%q, want wopan-target/wopan/root-wopan", target.ID(), target.Kind(), target.RootID())
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115、123、OneDrive、Google Drive、联通网盘也不是光鸭网盘时,
|
||||
// resolveTarget 拒绝并返回 error,让 runOnce 静默跳过(不会做破坏性变更)。
|
||||
func TestResolveTargetRejectsUnsupportedKind(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
// Package transcode 实现"浏览器兼容性转码":把网盘/本地存储中浏览器
|
||||
// <video> 播不动的视频(AVI/WMV/FLV、MPEG-4 Part 2、RMVB 等)转成
|
||||
// H.264 + AAC 的 MP4,并把产物上传回同一存储,播放源切到产物文件。
|
||||
//
|
||||
// 与封面/预览生成不同,转码不会自动运行——只能由管理员在网盘管理页
|
||||
// 手动开启,也可以随时手动停止。
|
||||
package transcode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MediaInfo 是 ffprobe 探测出来的、做兼容性判定所需的最小信息。
|
||||
type MediaInfo struct {
|
||||
// FormatName 是 ffprobe 的 format_name,逗号分隔的 demuxer 别名,
|
||||
// 例如 "mov,mp4,m4a,3gp,3g2,mj2" / "avi" / "matroska,webm"。
|
||||
FormatName string
|
||||
VideoCodecs []string
|
||||
AudioCodecs []string
|
||||
}
|
||||
|
||||
// browserCompatibleVideoCodecs 是主流浏览器 <video> 普遍可解码的视频编码。
|
||||
// HEVC/H.265 只有部分平台支持,保守起见不算兼容。
|
||||
var browserCompatibleVideoCodecs = map[string]bool{
|
||||
"h264": true,
|
||||
"vp8": true,
|
||||
"vp9": true,
|
||||
"av1": true,
|
||||
}
|
||||
|
||||
// browserCompatibleAudioCodecs 是主流浏览器普遍可解码的音频编码。
|
||||
var browserCompatibleAudioCodecs = map[string]bool{
|
||||
"aac": true,
|
||||
"mp3": true,
|
||||
"opus": true,
|
||||
"vorbis": true,
|
||||
"flac": true,
|
||||
}
|
||||
|
||||
// NeedsTranscode 判断这个文件是否需要转码才能在浏览器里播放。
|
||||
// ext 是 catalog 里记录的扩展名(小写、不带点),用来区分 mkv 和 webm
|
||||
// (两者的 format_name 都是 "matroska,webm")。
|
||||
func NeedsTranscode(info MediaInfo, ext string) bool {
|
||||
if !containerCompatible(info.FormatName, ext) {
|
||||
return true
|
||||
}
|
||||
for _, codec := range info.VideoCodecs {
|
||||
if !browserCompatibleVideoCodecs[strings.ToLower(codec)] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, codec := range info.AudioCodecs {
|
||||
if !browserCompatibleAudioCodecs[strings.ToLower(codec)] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containerCompatible(formatName, ext string) bool {
|
||||
format := strings.ToLower(formatName)
|
||||
for _, name := range strings.Split(format, ",") {
|
||||
if name == "mp4" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// matroska,webm:只有真 .webm 信任为浏览器可播容器;.mkv 保守转码。
|
||||
if strings.Contains(format, "webm") && strings.EqualFold(ext, "webm") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ProbeFile 用 ffprobe 探测本地文件的容器与音视频编码。
|
||||
func ProbeFile(ctx context.Context, ffprobePath, path string) (MediaInfo, error) {
|
||||
ctx2, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx2, ffprobePath,
|
||||
"-v", "error",
|
||||
"-show_entries", "format=format_name",
|
||||
"-show_entries", "stream=codec_type,codec_name",
|
||||
"-of", "json",
|
||||
path,
|
||||
)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return MediaInfo{}, fmt.Errorf("transcode: ffprobe: %w", err)
|
||||
}
|
||||
var parsed struct {
|
||||
Format struct {
|
||||
FormatName string `json:"format_name"`
|
||||
} `json:"format"`
|
||||
Streams []struct {
|
||||
CodecType string `json:"codec_type"`
|
||||
CodecName string `json:"codec_name"`
|
||||
} `json:"streams"`
|
||||
}
|
||||
if err := json.Unmarshal(out, &parsed); err != nil {
|
||||
return MediaInfo{}, fmt.Errorf("transcode: parse ffprobe output: %w", err)
|
||||
}
|
||||
info := MediaInfo{FormatName: parsed.Format.FormatName}
|
||||
for _, s := range parsed.Streams {
|
||||
switch s.CodecType {
|
||||
case "video":
|
||||
info.VideoCodecs = append(info.VideoCodecs, s.CodecName)
|
||||
case "audio":
|
||||
info.AudioCodecs = append(info.AudioCodecs, s.CodecName)
|
||||
}
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// buildFFmpegArgs 按探测结果生成转码参数:
|
||||
// - 编码本就兼容、只是容器不行(如 AVI 里装 H.264)→ 流拷贝 remux,零质量损失;
|
||||
// - 否则视频转 H.264(裁到偶数尺寸 + yuv420p 保证兼容性)、音频转 AAC。
|
||||
//
|
||||
// 两种情况都加 +faststart 把 moov 提前,便于边下边播。
|
||||
func buildFFmpegArgs(info MediaInfo, inPath, outPath string) []string {
|
||||
args := []string{"-y", "-i", inPath}
|
||||
videoOK := true
|
||||
for _, codec := range info.VideoCodecs {
|
||||
if !browserCompatibleVideoCodecs[strings.ToLower(codec)] {
|
||||
videoOK = false
|
||||
break
|
||||
}
|
||||
}
|
||||
audioOK := true
|
||||
for _, codec := range info.AudioCodecs {
|
||||
if !browserCompatibleAudioCodecs[strings.ToLower(codec)] {
|
||||
audioOK = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if videoOK {
|
||||
args = append(args, "-c:v", "copy")
|
||||
} else {
|
||||
args = append(args,
|
||||
"-c:v", "libx264",
|
||||
"-preset", "veryfast",
|
||||
"-crf", "23",
|
||||
"-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2",
|
||||
"-pix_fmt", "yuv420p",
|
||||
)
|
||||
}
|
||||
if len(info.AudioCodecs) == 0 {
|
||||
args = append(args, "-an")
|
||||
} else if audioOK {
|
||||
args = append(args, "-c:a", "copy")
|
||||
} else {
|
||||
args = append(args, "-c:a", "aac", "-b:a", "128k")
|
||||
}
|
||||
args = append(args, "-movflags", "+faststart", "-f", "mp4", outPath)
|
||||
return args
|
||||
}
|
||||
|
||||
// TranscodeFile 把本地输入文件转成浏览器可播的 MP4 写到 outPath。
|
||||
func TranscodeFile(ctx context.Context, ffmpegPath string, info MediaInfo, inPath, outPath string) error {
|
||||
args := buildFFmpegArgs(info, inPath, outPath)
|
||||
cmd := exec.CommandContext(ctx, ffmpegPath, args...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("transcode: ffmpeg: %w: %s", err, tailOf(string(out), 400))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func tailOf(s string, n int) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[len(s)-n:]
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package transcode
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
)
|
||||
|
||||
func TestNeedsTranscode(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
info MediaInfo
|
||||
ext string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "h264 aac mp4 is compatible",
|
||||
info: MediaInfo{FormatName: "mov,mp4,m4a,3gp,3g2,mj2", VideoCodecs: []string{"h264"}, AudioCodecs: []string{"aac"}},
|
||||
ext: "mp4",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "mpeg4 in avi needs transcode",
|
||||
info: MediaInfo{FormatName: "avi", VideoCodecs: []string{"mpeg4"}, AudioCodecs: []string{"mp3"}},
|
||||
ext: "avi",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "h264 in avi needs remux",
|
||||
info: MediaInfo{FormatName: "avi", VideoCodecs: []string{"h264"}, AudioCodecs: []string{"aac"}},
|
||||
ext: "avi",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "hevc in mp4 needs transcode",
|
||||
info: MediaInfo{FormatName: "mov,mp4,m4a,3gp,3g2,mj2", VideoCodecs: []string{"hevc"}, AudioCodecs: []string{"aac"}},
|
||||
ext: "mp4",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "vp9 opus webm is compatible",
|
||||
info: MediaInfo{FormatName: "matroska,webm", VideoCodecs: []string{"vp9"}, AudioCodecs: []string{"opus"}},
|
||||
ext: "webm",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "h264 in mkv is conservative transcode",
|
||||
info: MediaInfo{FormatName: "matroska,webm", VideoCodecs: []string{"h264"}, AudioCodecs: []string{"aac"}},
|
||||
ext: "mkv",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "pcm audio in mov needs transcode",
|
||||
info: MediaInfo{FormatName: "mov,mp4,m4a,3gp,3g2,mj2", VideoCodecs: []string{"h264"}, AudioCodecs: []string{"pcm_s16le"}},
|
||||
ext: "mov",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "video only h264 mp4 is compatible",
|
||||
info: MediaInfo{FormatName: "mov,mp4,m4a,3gp,3g2,mj2", VideoCodecs: []string{"h264"}},
|
||||
ext: "mp4",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := NeedsTranscode(tc.info, tc.ext); got != tc.want {
|
||||
t.Fatalf("NeedsTranscode(%+v, %q) = %v, want %v", tc.info, tc.ext, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFFmpegArgsRemuxWhenCodecsCompatible(t *testing.T) {
|
||||
// AVI 里装 H.264+AAC:只需要换容器,应该走流拷贝
|
||||
info := MediaInfo{FormatName: "avi", VideoCodecs: []string{"h264"}, AudioCodecs: []string{"aac"}}
|
||||
args := strings.Join(buildFFmpegArgs(info, "in.avi", "out.mp4"), " ")
|
||||
if !strings.Contains(args, "-c:v copy") {
|
||||
t.Fatalf("expected video stream copy, got: %s", args)
|
||||
}
|
||||
if !strings.Contains(args, "-c:a copy") {
|
||||
t.Fatalf("expected audio stream copy, got: %s", args)
|
||||
}
|
||||
if !strings.Contains(args, "+faststart") {
|
||||
t.Fatalf("expected faststart flag, got: %s", args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFFmpegArgsTranscodesIncompatibleCodecs(t *testing.T) {
|
||||
info := MediaInfo{FormatName: "avi", VideoCodecs: []string{"mpeg4"}, AudioCodecs: []string{"wmav2"}}
|
||||
args := strings.Join(buildFFmpegArgs(info, "in.avi", "out.mp4"), " ")
|
||||
if !strings.Contains(args, "-c:v libx264") {
|
||||
t.Fatalf("expected libx264 video encode, got: %s", args)
|
||||
}
|
||||
if !strings.Contains(args, "-c:a aac") {
|
||||
t.Fatalf("expected aac audio encode, got: %s", args)
|
||||
}
|
||||
if !strings.Contains(args, "yuv420p") {
|
||||
t.Fatalf("expected yuv420p pixel format, got: %s", args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFFmpegArgsDropsAudioWhenNoAudioStream(t *testing.T) {
|
||||
info := MediaInfo{FormatName: "avi", VideoCodecs: []string{"mpeg4"}}
|
||||
args := strings.Join(buildFFmpegArgs(info, "in.avi", "out.mp4"), " ")
|
||||
if !strings.Contains(args, "-an") {
|
||||
t.Fatalf("expected -an for video without audio, got: %s", args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscodedName(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
fileName, title, id, want string
|
||||
}{
|
||||
{"www.98T.la@167.avi", "www.98T.la@167", "p115-1", "www.98T.la@167.mp4"},
|
||||
{"", "标题", "p115-2", "标题.mp4"},
|
||||
{"a/b\\c.wmv", "", "p115-3", "a_b_c.mp4"},
|
||||
} {
|
||||
v := &catalog.Video{FileName: tc.fileName, Title: tc.title, ID: tc.id}
|
||||
if got := transcodedName(v); got != tc.want {
|
||||
t.Fatalf("transcodedName(%q,%q,%q) = %q, want %q", tc.fileName, tc.title, tc.id, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
package transcode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
// DefaultTargetDirName 是转码产物在网盘上的存放目录(相对根目录)。
|
||||
// worker 第一次上传前会 EnsureDir 并把该目录加进 drive 的扫描跳过列表,
|
||||
// 避免 scanner 把转码产物当成新视频重复入库。
|
||||
const DefaultTargetDirName = "91转码"
|
||||
|
||||
type Config struct {
|
||||
FFmpegPath string
|
||||
FFprobePath string
|
||||
// WorkDir 是下载原始文件 / 写转码产物的本地临时目录。
|
||||
WorkDir string
|
||||
// TargetDirName 为空时用 DefaultTargetDirName。
|
||||
TargetDirName string
|
||||
}
|
||||
|
||||
// TaskStatus 与 preview/fingerprint worker 的状态结构对齐,供 admin 展示。
|
||||
type TaskStatus struct {
|
||||
State string
|
||||
CurrentTitle string
|
||||
QueueLength int
|
||||
DoneCount int
|
||||
TotalCount int
|
||||
}
|
||||
|
||||
// Worker 串行处理一个 drive 的转码任务。生命周期与一次"开始转码"对应:
|
||||
// Run 处理完整个候选列表(或 ctx 被取消)后即结束,不常驻。
|
||||
type Worker struct {
|
||||
cfg Config
|
||||
cat *catalog.Catalog
|
||||
drv drives.Drive
|
||||
hc *http.Client
|
||||
|
||||
mu sync.Mutex
|
||||
state string
|
||||
currentTitle string
|
||||
done int
|
||||
total int
|
||||
|
||||
targetDirOnce sync.Once
|
||||
targetDirID string
|
||||
targetDirErr error
|
||||
}
|
||||
|
||||
func NewWorker(cfg Config, cat *catalog.Catalog, drv drives.Drive) *Worker {
|
||||
if cfg.FFmpegPath == "" {
|
||||
cfg.FFmpegPath = "ffmpeg"
|
||||
}
|
||||
if cfg.FFprobePath == "" {
|
||||
cfg.FFprobePath = "ffprobe"
|
||||
}
|
||||
if cfg.TargetDirName == "" {
|
||||
cfg.TargetDirName = DefaultTargetDirName
|
||||
}
|
||||
if cfg.WorkDir == "" {
|
||||
cfg.WorkDir = os.TempDir()
|
||||
}
|
||||
return &Worker{
|
||||
cfg: cfg,
|
||||
cat: cat,
|
||||
drv: drv,
|
||||
hc: &http.Client{Timeout: 0},
|
||||
state: "idle",
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) Status() TaskStatus {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
queueLen := w.total - w.done
|
||||
if w.state == "generating" && queueLen > 0 {
|
||||
// 正在处理的那条不算"排队中"
|
||||
queueLen--
|
||||
}
|
||||
if queueLen < 0 {
|
||||
queueLen = 0
|
||||
}
|
||||
return TaskStatus{
|
||||
State: w.state,
|
||||
CurrentTitle: w.currentTitle,
|
||||
QueueLength: queueLen,
|
||||
DoneCount: w.done,
|
||||
TotalCount: w.total,
|
||||
}
|
||||
}
|
||||
|
||||
// Run 串行转码整个候选列表。ctx 取消时停在当前条目边界(正在跑的 ffmpeg
|
||||
// 会被 CommandContext 杀掉),未处理的候选保持原状态,下次开始时继续。
|
||||
func (w *Worker) Run(ctx context.Context, videos []*catalog.Video) {
|
||||
w.mu.Lock()
|
||||
w.state = "generating"
|
||||
w.total = len(videos)
|
||||
w.done = 0
|
||||
w.mu.Unlock()
|
||||
|
||||
defer func() {
|
||||
w.mu.Lock()
|
||||
w.state = "idle"
|
||||
w.currentTitle = ""
|
||||
w.mu.Unlock()
|
||||
}()
|
||||
|
||||
for _, v := range videos {
|
||||
if ctx.Err() != nil {
|
||||
log.Printf("[transcode] drive=%s canceled after %d/%d", w.drv.ID(), w.doneCount(), len(videos))
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
w.currentTitle = v.Title
|
||||
w.mu.Unlock()
|
||||
|
||||
if err := w.process(ctx, v); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
// 取消导致的失败不要写 failed,保持候选状态便于下次继续
|
||||
log.Printf("[transcode] drive=%s canceled while processing %s", w.drv.ID(), v.ID)
|
||||
return
|
||||
}
|
||||
log.Printf("[transcode] drive=%s video=%s failed: %v", w.drv.ID(), v.ID, err)
|
||||
if uerr := w.cat.UpdateVideoTranscode(context.WithoutCancel(ctx), v.ID, "failed", err.Error(), "", 0); uerr != nil {
|
||||
log.Printf("[transcode] mark failed %s: %v", v.ID, uerr)
|
||||
}
|
||||
}
|
||||
w.mu.Lock()
|
||||
w.done++
|
||||
w.mu.Unlock()
|
||||
}
|
||||
log.Printf("[transcode] drive=%s finished %d videos", w.drv.ID(), len(videos))
|
||||
}
|
||||
|
||||
func (w *Worker) doneCount() int {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.done
|
||||
}
|
||||
|
||||
func (w *Worker) process(ctx context.Context, v *catalog.Video) error {
|
||||
localPath, cleanup, err := w.fetchSource(ctx, v)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch source: %w", err)
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
info, err := ProbeFile(ctx, w.cfg.FFprobePath, localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !NeedsTranscode(info, v.Ext) {
|
||||
log.Printf("[transcode] drive=%s video=%s compatible (%s), skip", w.drv.ID(), v.ID, info.FormatName)
|
||||
return w.cat.UpdateVideoTranscode(ctx, v.ID, "skipped", "", "", 0)
|
||||
}
|
||||
|
||||
outPath := filepath.Join(w.cfg.WorkDir, sanitizeFileName(v.ID)+".transcoding.mp4")
|
||||
defer os.Remove(outPath)
|
||||
if err := TranscodeFile(ctx, w.cfg.FFmpegPath, info, localPath, outPath); err != nil {
|
||||
return err
|
||||
}
|
||||
stat, err := os.Stat(outPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat transcoded output: %w", err)
|
||||
}
|
||||
|
||||
dirID, err := w.ensureTargetDir(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ensure target dir: %w", err)
|
||||
}
|
||||
f, err := os.Open(outPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
fileID, err := w.drv.Upload(ctx, dirID, transcodedName(v), f, stat.Size())
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload transcoded file: %w", err)
|
||||
}
|
||||
log.Printf("[transcode] drive=%s video=%s ready: file=%s size=%d", w.drv.ID(), v.ID, fileID, stat.Size())
|
||||
return w.cat.UpdateVideoTranscode(ctx, v.ID, "ready", "", fileID, stat.Size())
|
||||
}
|
||||
|
||||
// fetchSource 把原始文件准备成本地路径。本地存储直接复用源路径(cleanup
|
||||
// 不删除源文件);云盘则整文件下载到 WorkDir。
|
||||
func (w *Worker) fetchSource(ctx context.Context, v *catalog.Video) (string, func(), error) {
|
||||
link, err := w.drv.StreamURL(ctx, v.FileID)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
u, err := url.Parse(link.URL)
|
||||
if isLocal := err == nil && u.Scheme != "http" && u.Scheme != "https"; isLocal {
|
||||
path := link.URL
|
||||
if err == nil && u.Scheme == "file" {
|
||||
path = u.Path
|
||||
}
|
||||
return path, func() {}, nil
|
||||
}
|
||||
|
||||
tmpPath := filepath.Join(w.cfg.WorkDir, sanitizeFileName(v.ID)+".src.tmp")
|
||||
cleanup := func() { os.Remove(tmpPath) }
|
||||
if err := w.downloadTo(ctx, link, tmpPath); err != nil {
|
||||
cleanup()
|
||||
return "", nil, err
|
||||
}
|
||||
return tmpPath, cleanup, nil
|
||||
}
|
||||
|
||||
func (w *Worker) downloadTo(ctx context.Context, link *drives.StreamLink, dst string) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, link.URL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for k, vals := range link.Headers {
|
||||
for _, val := range vals {
|
||||
req.Header.Add(k, val)
|
||||
}
|
||||
}
|
||||
res, err := w.hc.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
return fmt.Errorf("download source: HTTP %d", res.StatusCode)
|
||||
}
|
||||
f, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := io.Copy(f, res.Body); err != nil {
|
||||
return fmt.Errorf("download source: %w", err)
|
||||
}
|
||||
return f.Sync()
|
||||
}
|
||||
|
||||
// ensureTargetDir 确保网盘上的转码产物目录存在,并把它写进 drive 的扫描
|
||||
// 跳过列表(幂等),避免 scanner 把产物再当新视频收进库。
|
||||
func (w *Worker) ensureTargetDir(ctx context.Context) (string, error) {
|
||||
w.targetDirOnce.Do(func() {
|
||||
dirID, err := w.drv.EnsureDir(ctx, w.cfg.TargetDirName)
|
||||
if err != nil {
|
||||
w.targetDirErr = err
|
||||
return
|
||||
}
|
||||
w.targetDirID = dirID
|
||||
if err := w.addDirToSkipList(ctx, dirID); err != nil {
|
||||
// 跳过列表更新失败不阻塞转码,只记日志(最坏情况是 scanner
|
||||
// 之后把产物扫成新视频,可手动加跳过目录修复)。
|
||||
log.Printf("[transcode] drive=%s add skip dir %s: %v", w.drv.ID(), dirID, err)
|
||||
}
|
||||
})
|
||||
return w.targetDirID, w.targetDirErr
|
||||
}
|
||||
|
||||
func (w *Worker) addDirToSkipList(ctx context.Context, dirID string) error {
|
||||
d, err := w.cat.GetDrive(ctx, w.drv.ID())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, existing := range d.SkipDirIDs {
|
||||
if existing == dirID {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return w.cat.SetDriveSkipDirIDs(ctx, w.drv.ID(), append(d.SkipDirIDs, dirID))
|
||||
}
|
||||
|
||||
// transcodedName 生成产物文件名:原文件名去掉扩展名 + .mp4。
|
||||
func transcodedName(v *catalog.Video) string {
|
||||
base := strings.TrimSpace(v.FileName)
|
||||
if base == "" {
|
||||
base = v.Title
|
||||
}
|
||||
if base == "" {
|
||||
base = v.ID
|
||||
}
|
||||
if ext := filepath.Ext(base); ext != "" {
|
||||
base = strings.TrimSuffix(base, ext)
|
||||
}
|
||||
return sanitizeFileName(base) + ".mp4"
|
||||
}
|
||||
|
||||
// sanitizeFileName 把路径分隔符等危险字符替换掉,避免拼出意外路径。
|
||||
func sanitizeFileName(name string) string {
|
||||
replacer := strings.NewReplacer(
|
||||
"/", "_", "\\", "_", ":", "_", "*", "_", "?", "_",
|
||||
"\"", "_", "<", "_", ">", "_", "|", "_", "\x00", "_",
|
||||
)
|
||||
out := strings.TrimSpace(replacer.Replace(name))
|
||||
if out == "" {
|
||||
out = fmt.Sprintf("transcoded-%d", time.Now().UnixMilli())
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -134,9 +134,9 @@ apt_install() {
|
||||
python3 python3-requests python3-bs4 python3-lxml python3-socks
|
||||
}
|
||||
|
||||
verify_spider91_python_deps() {
|
||||
command -v python3 >/dev/null 2>&1 || die "python3 is required for 91Spider"
|
||||
python3 - <<'PY' || die "missing Python modules for 91Spider: requests, bs4, lxml, socks"
|
||||
verify_crawler_python_deps() {
|
||||
command -v python3 >/dev/null 2>&1 || die "python3 is required for crawler scripts"
|
||||
python3 - <<'PY' || die "missing Python modules for crawler scripts: requests, bs4, lxml, socks"
|
||||
import importlib.util
|
||||
import sys
|
||||
|
||||
@@ -200,7 +200,7 @@ install_dependencies() {
|
||||
install_go
|
||||
command -v ffmpeg >/dev/null 2>&1 || die "ffmpeg is required"
|
||||
command -v ffprobe >/dev/null 2>&1 || die "ffprobe is required"
|
||||
verify_spider91_python_deps
|
||||
verify_crawler_python_deps
|
||||
}
|
||||
|
||||
ensure_ownership() {
|
||||
|
||||
+283
@@ -0,0 +1,283 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
光鸭网盘 - 扫码登录脚本
|
||||
========================
|
||||
1. 调用 API 获取登录二维码
|
||||
2. 保存二维码图片,等待用户扫描
|
||||
3. 扫描成功后保存用户凭证信息
|
||||
"""
|
||||
|
||||
import io
|
||||
import sys
|
||||
|
||||
# 修复 Windows 终端 GBK 编码问题
|
||||
if sys.platform == 'win32':
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace')
|
||||
|
||||
import requests
|
||||
import json
|
||||
import time
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
# ========== 配置 ==========
|
||||
API_ORIGIN = "https://account.guangyapan.com"
|
||||
CLIENT_ID = "aMe-8VSlkrbQXpUR"
|
||||
SCOPE = "user"
|
||||
QR_IMAGE_PATH = "login_qr.png"
|
||||
CREDENTIALS_PATH = "credentials.json"
|
||||
|
||||
# ========== 可选依赖 ==========
|
||||
try:
|
||||
import qrcode
|
||||
HAS_QRCODE = True
|
||||
except ImportError:
|
||||
HAS_QRCODE = False
|
||||
|
||||
try:
|
||||
from PIL import Image
|
||||
HAS_PIL = True
|
||||
except ImportError:
|
||||
HAS_PIL = False
|
||||
|
||||
|
||||
def generate_qr_image(url: str, path: str):
|
||||
"""生成二维码图片"""
|
||||
if HAS_QRCODE:
|
||||
qr = qrcode.QRCode(
|
||||
version=1,
|
||||
error_correction=qrcode.constants.ERROR_CORRECT_M,
|
||||
box_size=10,
|
||||
border=4,
|
||||
)
|
||||
qr.add_data(url)
|
||||
qr.make(fit=True)
|
||||
img = qr.make_image(fill_color="black", back_color="white")
|
||||
img.save(path)
|
||||
print(f"[✓] 二维码已保存到: {path}")
|
||||
else:
|
||||
# Fallback: 使用 qrencode 命令行工具
|
||||
import subprocess
|
||||
try:
|
||||
subprocess.run(["qrencode", "-o", path, url], check=True)
|
||||
print(f"[✓] 二维码已保存到: {path}")
|
||||
except FileNotFoundError:
|
||||
print("[✗] 需要安装 qrcode 库: pip install qrcode[pil]")
|
||||
print(f"[!] 请手动访问以下链接扫码:")
|
||||
print(f" {url}")
|
||||
return
|
||||
|
||||
# 尝试直接显示二维码到终端
|
||||
try:
|
||||
if HAS_PIL:
|
||||
img = Image.open(path)
|
||||
img.show()
|
||||
print("[✓] 二维码已在图片查看器中打开")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 终端内显示小二维码
|
||||
if HAS_QRCODE:
|
||||
try:
|
||||
qr.print_ascii(invert=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
session = requests.Session()
|
||||
session.headers.update({
|
||||
"User-Agent": "GuangYaPan-Login/1.0",
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
})
|
||||
|
||||
# ====== Step 1: 获取设备码和二维码链接 ======
|
||||
print("=" * 60)
|
||||
print("Step 1: 请求登录二维码...")
|
||||
print("=" * 60)
|
||||
|
||||
device_code_url = f"{API_ORIGIN}/v1/auth/device/code"
|
||||
device_payload = {
|
||||
"client_id": CLIENT_ID,
|
||||
"scope": SCOPE,
|
||||
}
|
||||
|
||||
try:
|
||||
resp = session.post(device_code_url, json=device_payload, timeout=30)
|
||||
resp.raise_for_status()
|
||||
device_data = resp.json()
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"[✗] 请求失败: {e}")
|
||||
if hasattr(e, 'response') and e.response is not None:
|
||||
print(f" 响应内容: {e.response.text[:500]}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"[✓] 设备码获取成功")
|
||||
print(f" device_code: {device_data.get('device_code', 'N/A')[:30]}...")
|
||||
print(f" interval: {device_data.get('interval', 5)} 秒")
|
||||
print(f" expires_in: {device_data.get('expires_in', 'N/A')} 秒")
|
||||
|
||||
device_code = device_data["device_code"]
|
||||
interval = int(device_data.get("interval", 5))
|
||||
expires_in = int(device_data.get("expires_in", 300))
|
||||
|
||||
# 二维码链接
|
||||
qr_url = device_data.get("verification_uri_complete") or device_data.get("short_uri_complete")
|
||||
if not qr_url:
|
||||
print("[✗] 响应中没有找到二维码链接")
|
||||
print(f" 完整响应: {json.dumps(device_data, indent=2, ensure_ascii=False)}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f" qr_url: {qr_url}")
|
||||
print()
|
||||
|
||||
# ====== Step 2: 生成并保存二维码 ======
|
||||
print("=" * 60)
|
||||
print("Step 2: 生成二维码图片...")
|
||||
print("=" * 60)
|
||||
|
||||
generate_qr_image(qr_url, QR_IMAGE_PATH)
|
||||
|
||||
print()
|
||||
print("!" * 60)
|
||||
print("! 请使用「光鸭APP」扫描二维码登录")
|
||||
print("!" * 60)
|
||||
print()
|
||||
|
||||
# ====== Step 3: 轮询等待用户扫描 ======
|
||||
print("=" * 60)
|
||||
print("Step 3: 等待扫码授权...")
|
||||
print("=" * 60)
|
||||
|
||||
token_url = f"{API_ORIGIN}/v1/auth/token"
|
||||
token_payload = {
|
||||
"client_id": CLIENT_ID,
|
||||
"grant_type": "urn:ietf:params:oauth:grant-type:device_code",
|
||||
"device_code": device_code,
|
||||
}
|
||||
|
||||
start_time = time.time()
|
||||
attempt = 0
|
||||
|
||||
while True:
|
||||
attempt += 1
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
# 检查是否超时
|
||||
if elapsed > expires_in:
|
||||
print(f"\n[✗] 二维码已过期({expires_in}秒),请重新运行脚本")
|
||||
sys.exit(1)
|
||||
|
||||
time.sleep(interval)
|
||||
|
||||
try:
|
||||
resp = session.post(token_url, json=token_payload, timeout=30)
|
||||
token_data = resp.json()
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"\n[!] 网络错误: {e},重试中...")
|
||||
continue
|
||||
|
||||
if "error" in token_data:
|
||||
error = token_data["error"]
|
||||
if error in ("authorization_pending", "slow_down"):
|
||||
# 用户还未扫描或确认
|
||||
dots = "." * ((attempt % 10) + 1)
|
||||
print(f"\r 等待中{dots:<10} ({int(elapsed)}s / {expires_in}s)", end="", flush=True)
|
||||
|
||||
if error == "slow_down":
|
||||
interval = min(interval * 2, 60)
|
||||
continue
|
||||
|
||||
elif error == "expired_token":
|
||||
print(f"\n[✗] 二维码已过期,请重新运行脚本")
|
||||
sys.exit(1)
|
||||
|
||||
elif error == "access_denied":
|
||||
print(f"\n[✗] 用户拒绝了授权")
|
||||
sys.exit(1)
|
||||
|
||||
else:
|
||||
print(f"\n[✗] 未知错误: {error}")
|
||||
print(f" 完整响应: {json.dumps(token_data, indent=2, ensure_ascii=False)}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
# 成功!
|
||||
print(f"\n[✓] 扫码授权成功!({int(elapsed)}s)")
|
||||
break
|
||||
|
||||
# ====== Step 4: 保存凭证 ======
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("Step 4: 保存用户凭证...")
|
||||
print("=" * 60)
|
||||
|
||||
# 保存完整 token 响应
|
||||
credentials = {
|
||||
"saved_at": datetime.now().isoformat(),
|
||||
"api_origin": API_ORIGIN,
|
||||
"client_id": CLIENT_ID,
|
||||
"token_response": token_data,
|
||||
"cookies": dict(session.cookies),
|
||||
}
|
||||
|
||||
with open(CREDENTIALS_PATH, "w", encoding="utf-8") as f:
|
||||
json.dump(credentials, f, indent=2, ensure_ascii=False)
|
||||
print(f"[✓] 完整凭证已保存到: {CREDENTIALS_PATH}")
|
||||
|
||||
# 提取关键信息
|
||||
access_token = token_data.get("access_token", "")
|
||||
refresh_token = token_data.get("refresh_token", "")
|
||||
id_token = token_data.get("id_token", "")
|
||||
token_type = token_data.get("token_type", "Bearer")
|
||||
expires_in = token_data.get("expires_in", 0)
|
||||
|
||||
print()
|
||||
print("-" * 60)
|
||||
print("凭证摘要:")
|
||||
print("-" * 60)
|
||||
print(f" access_token: {access_token[:50]}..." if access_token else " access_token: (无)")
|
||||
print(f" refresh_token: {refresh_token[:50]}..." if refresh_token else " refresh_token: (无)")
|
||||
print(f" id_token: {id_token[:50]}..." if id_token else " id_token: (无)")
|
||||
print(f" token_type: {token_type}")
|
||||
print(f" expires_in: {expires_in} 秒")
|
||||
print(f" scope: {token_data.get('scope', SCOPE)}")
|
||||
print("-" * 60)
|
||||
|
||||
# 尝试获取用户信息
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("Step 5: 获取用户信息...")
|
||||
print("=" * 60)
|
||||
|
||||
user_info_url = f"{API_ORIGIN}/v1/user/me"
|
||||
try:
|
||||
user_headers = {
|
||||
"Authorization": f"{token_type} {access_token}",
|
||||
}
|
||||
user_resp = requests.get(user_info_url, headers=user_headers, timeout=15)
|
||||
if user_resp.status_code == 200:
|
||||
user_data = user_resp.json()
|
||||
print("[✓] 用户信息获取成功:")
|
||||
print(json.dumps(user_data, indent=2, ensure_ascii=False))
|
||||
|
||||
# 追加用户信息到凭证文件
|
||||
credentials["user_info"] = user_data
|
||||
with open(CREDENTIALS_PATH, "w", encoding="utf-8") as f:
|
||||
json.dump(credentials, f, indent=2, ensure_ascii=False)
|
||||
else:
|
||||
print(f"[!] 获取用户信息返回 {user_resp.status_code}: {user_resp.text[:200]}")
|
||||
except Exception as e:
|
||||
print(f"[!] 获取用户信息失败: {e}")
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("完成!凭证文件: " + CREDENTIALS_PATH)
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+10
-2
@@ -2,8 +2,16 @@
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||
<meta name="referrer" content="no-referrer" />
|
||||
<link rel="icon" type="image/png" href="/icon.png" />
|
||||
<link rel="apple-touch-icon" href="/icon.png" />
|
||||
<link rel="manifest" href="/manifest.webmanifest" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
|
||||
<meta name="mobile-web-app-capable" content="yes" />
|
||||
<meta name="apple-mobile-web-app-capable" content="yes" />
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
||||
<meta name="apple-mobile-web-app-title" content="91" />
|
||||
<meta name="theme-color" content="#000000" />
|
||||
<meta name="description" content="91 视频站" />
|
||||
<title>91</title>
|
||||
<!-- Premium Fonts Preconnect & Links -->
|
||||
@@ -18,7 +26,7 @@
|
||||
(function () {
|
||||
try {
|
||||
var t = localStorage.getItem("video-site:theme");
|
||||
if (t === "pink" || t === "dark") {
|
||||
if (t === "pink" || t === "dark" || t === "sky") {
|
||||
document.documentElement.setAttribute("data-theme", t);
|
||||
} else {
|
||||
document.documentElement.setAttribute("data-theme", "dark");
|
||||
|
||||
+1
-1
@@ -128,7 +128,7 @@ verify_runtime_deps() {
|
||||
command -v "$cmd" >/dev/null 2>&1 || die "missing command: $cmd"
|
||||
done
|
||||
|
||||
python3 - <<'PY' || die "missing Python modules for 91Spider: requests, bs4, lxml, socks"
|
||||
python3 - <<'PY' || die "missing Python modules for crawler scripts: requests, bs4, lxml, socks"
|
||||
import importlib.util
|
||||
import sys
|
||||
|
||||
|
||||
Generated
+37
-2
@@ -1,14 +1,16 @@
|
||||
{
|
||||
"name": "video-site",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.9",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "video-site",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.9",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"artplayer": "^5.4.0",
|
||||
"hls.js": "^1.6.16",
|
||||
"lucide-react": "0.453.0",
|
||||
"react": "18.3.1",
|
||||
"react-dom": "18.3.1",
|
||||
@@ -475,6 +477,15 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/artplayer": {
|
||||
"version": "5.4.0",
|
||||
"resolved": "https://registry.npmjs.org/artplayer/-/artplayer-5.4.0.tgz",
|
||||
"integrity": "sha512-2B+plbx8N2yNsjK4nJU3+EOG8TULm1LRZk/QPkWRAMEX2Ee/MSnZG/WJYz8kcoZxZuLKcQ3uXifqLuPxZOH29A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"option-validator": "^2.0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/csstype": {
|
||||
"version": "3.2.3",
|
||||
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz",
|
||||
@@ -525,12 +536,27 @@
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/hls.js": {
|
||||
"version": "1.6.16",
|
||||
"resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.6.16.tgz",
|
||||
"integrity": "sha512-VSIRpLfRwlAAdGL4wiTucx2ScRipo0ed1FBatWkyt832jC4CReKstga6yIhYVwGu9LOBjuX9wzmRMeQdBJtzEA==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/js-tokens": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
|
||||
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/kind-of": {
|
||||
"version": "6.0.3",
|
||||
"resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
|
||||
"integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/lightningcss": {
|
||||
"version": "1.32.0",
|
||||
"resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz",
|
||||
@@ -832,6 +858,15 @@
|
||||
"node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/option-validator": {
|
||||
"version": "2.0.6",
|
||||
"resolved": "https://registry.npmjs.org/option-validator/-/option-validator-2.0.6.tgz",
|
||||
"integrity": "sha512-tmZDan2LRIRQyhUGvkff68/O0R8UmF+Btmiiz0SmSw2ng3CfPZB9wJlIjHpe/MKUZqyIZkVIXCrwr1tIN+0Dzg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"kind-of": "^6.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/picocolors": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
|
||||
|
||||
+3
-1
@@ -2,7 +2,7 @@
|
||||
"name": "video-site",
|
||||
"private": true,
|
||||
"license": "MIT",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.9",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
@@ -13,6 +13,8 @@
|
||||
"test": "node --import tsx --test tests/*.test.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"artplayer": "^5.4.0",
|
||||
"hls.js": "^1.6.16",
|
||||
"lucide-react": "0.453.0",
|
||||
"react": "18.3.1",
|
||||
"react-dom": "18.3.1",
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 212 KiB |
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"name": "91",
|
||||
"short_name": "91",
|
||||
"start_url": "/",
|
||||
"scope": "/",
|
||||
"display": "standalone",
|
||||
"display_override": ["fullscreen", "standalone"],
|
||||
"background_color": "#000000",
|
||||
"theme_color": "#000000",
|
||||
"icons": [
|
||||
{
|
||||
"src": "/icon.png",
|
||||
"sizes": "512x512",
|
||||
"type": "image/png",
|
||||
"purpose": "any maskable"
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 864 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 855 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.3 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.5 MiB |
@@ -63,8 +63,6 @@ build_package() {
|
||||
cp "$ROOT_DIR/backend/config.example.yaml" "$work/config.example.yaml"
|
||||
cp "$ROOT_DIR/install.sh" "$work/install.sh"
|
||||
cp -R "$ROOT_DIR/dist" "$work/dist"
|
||||
mkdir -p "$work/91VideoSpider"
|
||||
cp "$ROOT_DIR/91VideoSpider/spider_91porn.py" "$work/91VideoSpider/spider_91porn.py"
|
||||
|
||||
cat >"$work/README.txt" <<EOF
|
||||
$APP_NAME $VERSION
|
||||
|
||||
+67
-60
@@ -1,4 +1,5 @@
|
||||
import { Navigate, Route, Routes } from "react-router-dom";
|
||||
import { SkyStarfield } from "@/components/SkyStarfield";
|
||||
import HomePage from "@/pages/HomePage";
|
||||
import ListingPage from "@/pages/ListingPage";
|
||||
import ShortsPage from "@/pages/ShortsPage";
|
||||
@@ -8,74 +9,80 @@ import { AdminLayout } from "@/admin/AdminLayout";
|
||||
import { LoginPage } from "@/admin/LoginPage";
|
||||
import { RequireAuth } from "@/admin/RequireAuth";
|
||||
import { DrivesPage } from "@/admin/DrivesPage";
|
||||
import { CrawlersPage } from "@/admin/CrawlersPage";
|
||||
import { VideosPage } from "@/admin/VideosPage";
|
||||
import { TagsPage } from "@/admin/TagsPage";
|
||||
import { ThemePage } from "@/admin/ThemePage";
|
||||
|
||||
export default function App() {
|
||||
return (
|
||||
<Routes>
|
||||
<Route path="/login" element={<LoginPage />} />
|
||||
<>
|
||||
{/* 星空蓝主题的固定位置星星层,仅在 data-theme="sky" 下可见 */}
|
||||
<SkyStarfield />
|
||||
<Routes>
|
||||
<Route path="/login" element={<LoginPage />} />
|
||||
|
||||
{/* 主站需要登录 */}
|
||||
<Route
|
||||
path="/"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<HomePage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/list"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<ListingPage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/shorts"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<ShortsPage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/upload"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<UploadPage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/video/:id"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<VideoDetailPage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
{/* 主站需要登录 */}
|
||||
<Route
|
||||
path="/"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<HomePage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/list"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<ListingPage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/shorts"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<ShortsPage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/upload"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<UploadPage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/video/:id"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<VideoDetailPage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
|
||||
{/* 管理后台也需要登录 */}
|
||||
<Route
|
||||
path="/admin"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<AdminLayout />
|
||||
</RequireAuth>
|
||||
}
|
||||
>
|
||||
<Route index element={<Navigate to="/admin/drives" replace />} />
|
||||
<Route path="drives" element={<DrivesPage />} />
|
||||
<Route path="videos" element={<VideosPage />} />
|
||||
<Route path="tags" element={<TagsPage />} />
|
||||
<Route path="theme" element={<ThemePage />} />
|
||||
</Route>
|
||||
{/* 管理后台也需要登录 */}
|
||||
<Route
|
||||
path="/admin"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<AdminLayout />
|
||||
</RequireAuth>
|
||||
}
|
||||
>
|
||||
<Route index element={<Navigate to="/admin/drives" replace />} />
|
||||
<Route path="drives" element={<DrivesPage />} />
|
||||
<Route path="crawlers" element={<CrawlersPage />} />
|
||||
<Route path="videos" element={<VideosPage />} />
|
||||
<Route path="tags" element={<TagsPage />} />
|
||||
<Route path="theme" element={<ThemePage />} />
|
||||
</Route>
|
||||
|
||||
<Route path="*" element={<Navigate to="/" replace />} />
|
||||
</Routes>
|
||||
<Route path="*" element={<Navigate to="/" replace />} />
|
||||
</Routes>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
+77
-42
@@ -4,7 +4,6 @@ import {
|
||||
HardDrive,
|
||||
Film,
|
||||
LogOut,
|
||||
Play,
|
||||
Home,
|
||||
Tags,
|
||||
Palette,
|
||||
@@ -14,6 +13,7 @@ import {
|
||||
import * as api from "./api";
|
||||
import { useAuth } from "./AuthContext";
|
||||
import { useToast } from "./ToastContext";
|
||||
import { SpiderIcon } from "./icons/SpiderIcon";
|
||||
|
||||
export function AdminLayout() {
|
||||
const { logout } = useAuth();
|
||||
@@ -70,48 +70,80 @@ export function AdminLayout() {
|
||||
return (
|
||||
<div className="admin-shell">
|
||||
<aside className="admin-sidebar">
|
||||
<div className="admin-sidebar__brand">
|
||||
<span className="admin-sidebar__brand-mark">
|
||||
<Play size={14} fill="#000" />
|
||||
</span>
|
||||
<span className="admin-sidebar__brand-text">91后台</span>
|
||||
</div>
|
||||
<nav className="admin-nav">
|
||||
<NavLink to="/" className="admin-nav__link">
|
||||
<Home size={16} /> 返回主站
|
||||
</NavLink>
|
||||
<NavLink
|
||||
to="/admin/drives"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<HardDrive size={16} /> 网盘管理
|
||||
</NavLink>
|
||||
<NavLink
|
||||
to="/admin/videos"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<Film size={16} /> 视频管理
|
||||
</NavLink>
|
||||
<NavLink
|
||||
to="/admin/tags"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<Tags size={16} /> 标签管理
|
||||
</NavLink>
|
||||
<NavLink
|
||||
to="/admin/theme"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<Palette size={16} /> 主题外观
|
||||
</NavLink>
|
||||
<div className="admin-nav__group admin-nav__group--home">
|
||||
<span className="admin-nav__group-label">主站</span>
|
||||
<NavLink to="/" className="admin-nav__link">
|
||||
<span className="admin-nav__icon"><Home size={16} /></span>
|
||||
<span className="admin-nav__text">
|
||||
<span className="admin-nav__title">返回主站</span>
|
||||
</span>
|
||||
</NavLink>
|
||||
</div>
|
||||
<div className="admin-nav__group">
|
||||
<span className="admin-nav__group-label">资源</span>
|
||||
<NavLink
|
||||
to="/admin/drives"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<span className="admin-nav__icon"><HardDrive size={16} /></span>
|
||||
<span className="admin-nav__text">
|
||||
<span className="admin-nav__title">网盘管理</span>
|
||||
</span>
|
||||
</NavLink>
|
||||
<NavLink
|
||||
to="/admin/crawlers"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<span className="admin-nav__icon"><SpiderIcon size={16} /></span>
|
||||
<span className="admin-nav__text">
|
||||
<span className="admin-nav__title">爬虫管理</span>
|
||||
</span>
|
||||
</NavLink>
|
||||
</div>
|
||||
<div className="admin-nav__group">
|
||||
<span className="admin-nav__group-label">管理</span>
|
||||
<NavLink
|
||||
to="/admin/videos"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<span className="admin-nav__icon"><Film size={16} /></span>
|
||||
<span className="admin-nav__text">
|
||||
<span className="admin-nav__title">视频管理</span>
|
||||
</span>
|
||||
</NavLink>
|
||||
<NavLink
|
||||
to="/admin/tags"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<span className="admin-nav__icon"><Tags size={16} /></span>
|
||||
<span className="admin-nav__text">
|
||||
<span className="admin-nav__title">标签管理</span>
|
||||
</span>
|
||||
</NavLink>
|
||||
</div>
|
||||
<div className="admin-nav__group">
|
||||
<span className="admin-nav__group-label">系统</span>
|
||||
<NavLink
|
||||
to="/admin/theme"
|
||||
className={({ isActive }) =>
|
||||
`admin-nav__link ${isActive ? "is-active" : ""}`
|
||||
}
|
||||
>
|
||||
<span className="admin-nav__icon"><Palette size={16} /></span>
|
||||
<span className="admin-nav__text">
|
||||
<span className="admin-nav__title">主题外观</span>
|
||||
</span>
|
||||
</NavLink>
|
||||
</div>
|
||||
</nav>
|
||||
<div className="admin-sidebar__footer">
|
||||
<button
|
||||
@@ -139,6 +171,9 @@ export function AdminLayout() {
|
||||
<div className="admin-sidebar__mobile-overlay" onClick={() => setMobileMenuOpen(false)} />
|
||||
)}
|
||||
<div className={`admin-sidebar__mobile-panel${mobileMenuOpen ? " is-open" : ""}`}>
|
||||
<NavLink to="/" className="admin-sidebar__home" onClick={() => setMobileMenuOpen(false)}>
|
||||
<Home size={14} /> 返回主站
|
||||
</NavLink>
|
||||
<button
|
||||
className="admin-sidebar__check-update"
|
||||
onClick={() => { handleCheckUpdate(); setMobileMenuOpen(false); }}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { ReactNode } from "react";
|
||||
import { AlertTriangle } from "lucide-react";
|
||||
import { Modal } from "./Modal";
|
||||
|
||||
@@ -12,6 +13,7 @@ type ConfirmModalProps = {
|
||||
centerMessage?: boolean;
|
||||
modalClassName?: string;
|
||||
loading?: boolean;
|
||||
children?: ReactNode;
|
||||
onCancel: () => void;
|
||||
onConfirm: () => void;
|
||||
};
|
||||
@@ -27,6 +29,7 @@ export function ConfirmModal({
|
||||
centerMessage = false,
|
||||
modalClassName = "",
|
||||
loading = false,
|
||||
children,
|
||||
onCancel,
|
||||
onConfirm,
|
||||
}: ConfirmModalProps) {
|
||||
@@ -65,6 +68,7 @@ export function ConfirmModal({
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
{children}
|
||||
</div>
|
||||
</div>
|
||||
</Modal>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+133
-27
@@ -20,6 +20,7 @@ import { formatBytes } from "./storageFormat";
|
||||
import { makeUniqueDriveId } from "./driveId";
|
||||
import {
|
||||
FormState,
|
||||
driveKindAbbr,
|
||||
kindLabel,
|
||||
emptyForm,
|
||||
idleNightlyStatus,
|
||||
@@ -38,6 +39,22 @@ import { DriveForm } from "./drive/DriveForm";
|
||||
import { DeleteDriveModal } from "./drive/DeleteDriveModal";
|
||||
import { SkipDirsPanel } from "./drive/SkipDirsPanel";
|
||||
|
||||
const DRIVE_BUSY_MESSAGE = "当前存储有正在进行的任务,请稍后重试";
|
||||
const NIGHTLY_BUSY_MESSAGE = "当前有全量扫描任务正在进行,请稍后重试";
|
||||
|
||||
function isDriveBusy(d: api.AdminDrive) {
|
||||
return [
|
||||
d.scanGenerationStatus,
|
||||
d.thumbnailGenerationStatus,
|
||||
d.previewGenerationStatus,
|
||||
d.fingerprintGenerationStatus,
|
||||
d.transcodeGenerationStatus,
|
||||
].some((status) => {
|
||||
const state = status?.state || "idle";
|
||||
return state !== "idle";
|
||||
});
|
||||
}
|
||||
|
||||
export function DrivesPage() {
|
||||
const [list, setList] = useState<api.AdminDrive[]>([]);
|
||||
const [storage, setStorage] = useState<api.AdminDriveStorage | null>(null);
|
||||
@@ -58,10 +75,12 @@ export function DrivesPage() {
|
||||
const [regenFailedThumbId, setRegenFailedThumbId] = useState("");
|
||||
const [regenFailedFingerprintId, setRegenFailedFingerprintId] = useState("");
|
||||
const [togglingTeaserId, setTogglingTeaserId] = useState("");
|
||||
const [togglingTranscodeId, setTogglingTranscodeId] = useState("");
|
||||
const [scanningAll, setScanningAll] = useState(false);
|
||||
const [stoppingAll, setStoppingAll] = useState(false);
|
||||
const [trackingNightly, setTrackingNightly] = useState(false);
|
||||
const [scanningDriveId, setScanningDriveId] = useState("");
|
||||
const [scanningDriveIds, setScanningDriveIds] = useState<Record<string, boolean>>({});
|
||||
const scanningDriveIdsRef = useRef(new Set<string>());
|
||||
const [stoppingDriveId, setStoppingDriveId] = useState("");
|
||||
const [searchParams, setSearchParams] = useSearchParams();
|
||||
const selectedDriveId = searchParams.get("drive") || null;
|
||||
@@ -75,7 +94,17 @@ export function DrivesPage() {
|
||||
: hasCreateFormChanges(form, initialForm);
|
||||
|
||||
const uploadTargets = useMemo(
|
||||
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115" || d.kind === "p123" || d.kind === "onedrive"),
|
||||
() =>
|
||||
list.filter(
|
||||
(d) =>
|
||||
d.kind === "pikpak" ||
|
||||
d.kind === "p115" ||
|
||||
d.kind === "p123" ||
|
||||
d.kind === "onedrive" ||
|
||||
d.kind === "googledrive" ||
|
||||
d.kind === "wopan" ||
|
||||
d.kind === "guangyapan"
|
||||
),
|
||||
[list]
|
||||
);
|
||||
|
||||
@@ -184,7 +213,14 @@ export function DrivesPage() {
|
||||
kind: d.kind,
|
||||
name: d.name,
|
||||
rootId: d.rootId,
|
||||
creds: d.kind === "spider91" ? { proxy: d.spider91Proxy ?? "" } : {},
|
||||
creds:
|
||||
d.kind === "spider91"
|
||||
? { proxy: d.spider91Proxy ?? "" }
|
||||
: d.kind === "googledrive"
|
||||
? { use_online_api: (d.googleDriveUseOnlineAPI ?? true) ? "true" : "false" }
|
||||
: d.kind === "localstorage"
|
||||
? { strm_allow_outside_root: (d.strmAllowOutsideRoot ?? false) ? "true" : "false" }
|
||||
: {},
|
||||
spider91UploadDriveId: settings?.spider91UploadDriveId ?? "",
|
||||
};
|
||||
setForm(nextForm);
|
||||
@@ -295,25 +331,47 @@ export function DrivesPage() {
|
||||
}
|
||||
|
||||
async function handleRescan(d: api.AdminDrive) {
|
||||
if (scanningDriveId) return;
|
||||
setScanningDriveId(d.id);
|
||||
if (d.kind === "spider91") {
|
||||
show("91Spider 不再支持通过网盘运行,请到爬虫管理添加爬虫脚本", "info");
|
||||
return;
|
||||
}
|
||||
if (nightlyBusy) {
|
||||
show(nightlyBusyText(nightlyStatus) || NIGHTLY_BUSY_MESSAGE, "info");
|
||||
return;
|
||||
}
|
||||
if (isDriveBusy(d) || scanningDriveIdsRef.current.has(d.id)) {
|
||||
show(DRIVE_BUSY_MESSAGE, "info");
|
||||
return;
|
||||
}
|
||||
scanningDriveIdsRef.current.add(d.id);
|
||||
setScanningDriveIds((prev) => ({ ...prev, [d.id]: true }));
|
||||
try {
|
||||
await api.rescan(d.id);
|
||||
if (d.kind === "spider91") {
|
||||
show("已触发抓取任务,需要 2-4 分钟,可稍后刷新视频列表查看", "success");
|
||||
} else {
|
||||
show("已触发扫描,可稍后刷新视频列表查看", "success");
|
||||
const resp = await api.rescan(d.id);
|
||||
if (!resp.accepted) {
|
||||
if (resp.status) {
|
||||
setNightlyStatus(resp.status);
|
||||
}
|
||||
show(resp.message || DRIVE_BUSY_MESSAGE, "info");
|
||||
refreshDriveList();
|
||||
return;
|
||||
}
|
||||
show("已触发扫描,可稍后刷新视频列表查看", "success");
|
||||
refreshDriveList();
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "触发失败", "error");
|
||||
} finally {
|
||||
setScanningDriveId("");
|
||||
scanningDriveIdsRef.current.delete(d.id);
|
||||
setScanningDriveIds((prev) => {
|
||||
const next = { ...prev };
|
||||
delete next[d.id];
|
||||
return next;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function handleRunNightly() {
|
||||
if (nightlyBusy) {
|
||||
show(nightlyBusyText(nightlyStatus) || "当前已有扫描所有网盘任务", "info");
|
||||
show(nightlyBusyText(nightlyStatus) || NIGHTLY_BUSY_MESSAGE, "info");
|
||||
return;
|
||||
}
|
||||
setScanningAll(true);
|
||||
@@ -324,7 +382,7 @@ export function DrivesPage() {
|
||||
setTrackingNightly(!resp.status.running);
|
||||
show("已触发扫描所有网盘,耗时较长,可在任务状态和 backend 日志观察进度", "success");
|
||||
} else {
|
||||
show("当前已有扫描所有网盘任务", "info");
|
||||
show(resp.message || NIGHTLY_BUSY_MESSAGE, "info");
|
||||
}
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "触发失败", "error");
|
||||
@@ -446,6 +504,41 @@ export function DrivesPage() {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleStartTranscode(d: api.AdminDrive) {
|
||||
setTogglingTranscodeId(d.id);
|
||||
try {
|
||||
const resp = await api.startDriveTranscode(d.id);
|
||||
if (resp.accepted) {
|
||||
show(`已开始「${d.name || d.id}」的视频转码`, "success");
|
||||
} else {
|
||||
show(resp.message || "转码任务未能开启", "info");
|
||||
}
|
||||
refreshDriveList();
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "开启失败", "error");
|
||||
} finally {
|
||||
setTogglingTranscodeId("");
|
||||
}
|
||||
}
|
||||
|
||||
async function handleStopTranscode(d: api.AdminDrive) {
|
||||
setTogglingTranscodeId(d.id);
|
||||
try {
|
||||
const resp = await api.stopDriveTranscode(d.id);
|
||||
show(
|
||||
resp.stopped
|
||||
? `已停止「${d.name || d.id}」的视频转码`
|
||||
: `「${d.name || d.id}」没有正在运行的转码任务`,
|
||||
"success"
|
||||
);
|
||||
refreshDriveList();
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "停止失败", "error");
|
||||
} finally {
|
||||
setTogglingTranscodeId("");
|
||||
}
|
||||
}
|
||||
|
||||
const selectedDrive = useMemo(() => {
|
||||
return selectedDriveId ? list.find((d) => d.id === selectedDriveId) : null;
|
||||
}, [selectedDriveId, list]);
|
||||
@@ -498,10 +591,8 @@ export function DrivesPage() {
|
||||
)}
|
||||
{d.kind === "spider91" && (
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">上次抓取时间</span>
|
||||
<span className="admin-detail-value">
|
||||
{d.lastCrawlAt ? new Date(d.lastCrawlAt * 1000).toLocaleString() : "尚未抓取"}
|
||||
</span>
|
||||
<span className="admin-detail-label">配置状态</span>
|
||||
<span className="admin-detail-value">已废弃,请到爬虫管理添加</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
@@ -515,23 +606,33 @@ export function DrivesPage() {
|
||||
type="button"
|
||||
className="admin-btn is-primary"
|
||||
onClick={() => handleRescan(d)}
|
||||
disabled={!!scanningDriveId}
|
||||
disabled={d.kind === "spider91"}
|
||||
aria-disabled={d.kind === "spider91" || nightlyBusy || isDriveBusy(d) || !!scanningDriveIds[d.id]}
|
||||
title={
|
||||
d.kind === "spider91"
|
||||
? "91Spider 不再支持通过网盘运行,请到爬虫管理添加爬虫脚本"
|
||||
: nightlyBusy
|
||||
? nightlyBusyText(nightlyStatus) || NIGHTLY_BUSY_MESSAGE
|
||||
: isDriveBusy(d) || scanningDriveIds[d.id]
|
||||
? DRIVE_BUSY_MESSAGE
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
{d.kind === "spider91" ? (
|
||||
<>
|
||||
<Download size={13} className={scanningDriveId === d.id ? "admin-spin" : undefined} />
|
||||
{scanningDriveId === d.id ? "触发中..." : "立即抓取"}
|
||||
<Download size={13} className={scanningDriveIds[d.id] ? "admin-spin" : undefined} />
|
||||
已废弃
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<RefreshCw size={13} className={scanningDriveId === d.id ? "admin-spin" : undefined} />
|
||||
{scanningDriveId === d.id ? "触发中..." : "立即重扫"}
|
||||
<RefreshCw size={13} className={scanningDriveIds[d.id] ? "admin-spin" : undefined} />
|
||||
{scanningDriveIds[d.id] ? "触发中..." : "立即重扫"}
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn is-stop"
|
||||
className="admin-btn is-primary"
|
||||
onClick={() => handleStopDriveTasks(d)}
|
||||
disabled={!!stoppingDriveId}
|
||||
title="停止此网盘当前的扫描、封面、预览视频和视频指纹生成任务。"
|
||||
@@ -540,9 +641,11 @@ export function DrivesPage() {
|
||||
{stoppingDriveId === d.id ? "停止中..." : "停止所有任务"}
|
||||
</button>
|
||||
</div>
|
||||
<button type="button" className="admin-btn" onClick={() => openEdit(d)}>
|
||||
{d.kind === "spider91" ? "编辑配置" : "编辑配置凭证"}
|
||||
</button>
|
||||
{d.kind !== "spider91" && (
|
||||
<button type="button" className="admin-btn is-primary" onClick={() => openEdit(d)}>
|
||||
编辑配置凭证
|
||||
</button>
|
||||
)}
|
||||
<button type="button" className="admin-btn is-danger admin-detail-actions__danger" onClick={() => setDeleteTarget(d)}>
|
||||
<Trash2 size={13} /> 删除网盘
|
||||
</button>
|
||||
@@ -571,10 +674,13 @@ export function DrivesPage() {
|
||||
regenFailedThumbId={regenFailedThumbId}
|
||||
regenFailedFingerprintId={regenFailedFingerprintId}
|
||||
togglingTeaserId={togglingTeaserId}
|
||||
togglingTranscodeId={togglingTranscodeId}
|
||||
onToggleTeaser={() => handleToggleTeaser(d)}
|
||||
onRegenFailed={() => handleRegenFailed(d)}
|
||||
onRegenFailedThumbnails={() => handleRegenFailedThumbnails(d)}
|
||||
onRegenFailedFingerprints={() => handleRegenFailedFingerprints(d)}
|
||||
onStartTranscode={() => handleStartTranscode(d)}
|
||||
onStopTranscode={() => handleStopTranscode(d)}
|
||||
/>
|
||||
|
||||
<div className="admin-detail-card">
|
||||
@@ -720,7 +826,7 @@ export function DrivesPage() {
|
||||
<div className="admin-drive-card__header">
|
||||
<div className="admin-drive-card__title">
|
||||
<span className="admin-drive-card__brand-icon" data-kind={d.kind}>
|
||||
{d.kind.substring(0, 2)}
|
||||
{driveKindAbbr(d.kind)}
|
||||
</span>
|
||||
<span>{d.name || d.id}</span>
|
||||
</div>
|
||||
|
||||
@@ -79,9 +79,11 @@ export function LoginPage() {
|
||||
return (
|
||||
<div className="admin-login">
|
||||
<form className="admin-login__card" onSubmit={handleSubmit}>
|
||||
<h1 className="admin-login__title">
|
||||
<Play size={18} fill="currentColor" /> {setupRequired ? "首次设置管理员" : "登录"}
|
||||
</h1>
|
||||
{setupRequired && (
|
||||
<h1 className="admin-login__title">
|
||||
<Play size={18} fill="currentColor" /> 首次设置管理员
|
||||
</h1>
|
||||
)}
|
||||
<div className="admin-form">
|
||||
<div className="admin-form__row">
|
||||
<label htmlFor="admin-login-username">用户名</label>
|
||||
|
||||
+1
-6
@@ -75,12 +75,7 @@ export function Modal({ open, title, onClose, children, footer, className = "" }
|
||||
|
||||
if (!open) return null;
|
||||
return (
|
||||
<div
|
||||
className="admin-modal-backdrop"
|
||||
onMouseDown={(e) => {
|
||||
if (e.target === e.currentTarget) onClose();
|
||||
}}
|
||||
>
|
||||
<div className="admin-modal-backdrop">
|
||||
<div
|
||||
ref={dialogRef}
|
||||
className={`admin-modal${className ? ` ${className}` : ""}`}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Check, Loader2, Moon, Sparkles } from "lucide-react";
|
||||
import { Check, Loader2, Moon, Sparkles, Star } from "lucide-react";
|
||||
import * as api from "./api";
|
||||
import type { Theme } from "./api";
|
||||
import { useToast } from "./ToastContext";
|
||||
import { applyTheme, getCurrentTheme } from "@/lib/theme";
|
||||
|
||||
function isTheme(value: unknown): value is Theme {
|
||||
return value === "dark" || value === "pink";
|
||||
return value === "dark" || value === "pink" || value === "sky";
|
||||
}
|
||||
|
||||
type Option = {
|
||||
@@ -32,6 +32,13 @@ const OPTIONS: Option[] = [
|
||||
description: "柔和奶白底 + 樱花粉主色,清爽温柔,日间使用更舒适。",
|
||||
icon: Sparkles,
|
||||
},
|
||||
{
|
||||
id: "sky",
|
||||
title: "星空蓝 + 暖星黄",
|
||||
subtitle: "Starry Sky",
|
||||
description: "浅天空蓝底 + 暖星黄主色,配上淡淡的网格与点点星光,顶级美感。",
|
||||
icon: Star,
|
||||
},
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
+615
-163
@@ -1,5 +1,17 @@
|
||||
import { useEffect, useId, useState } from "react";
|
||||
import { ChevronDown, Edit, RefreshCw, Search, CheckSquare, Square, Image, Trash2 } from "lucide-react";
|
||||
import { useSearchParams } from "react-router-dom";
|
||||
import {
|
||||
ChevronDown,
|
||||
Edit,
|
||||
RefreshCw,
|
||||
Search,
|
||||
CheckSquare,
|
||||
Square,
|
||||
Image,
|
||||
Trash2,
|
||||
Ban,
|
||||
RotateCcw,
|
||||
} from "lucide-react";
|
||||
import * as api from "./api";
|
||||
import { useToast } from "./ToastContext";
|
||||
import { Modal } from "./Modal";
|
||||
@@ -9,8 +21,94 @@ import { formatBytes } from "./storageFormat";
|
||||
const DESKTOP_VIDEOS_PAGE_SIZE = 50;
|
||||
const MOBILE_VIDEOS_PAGE_SIZE = 20;
|
||||
const VIDEOS_MOBILE_QUERY = "(max-width: 640px)";
|
||||
const REGEN_PREVIEW_STATUS = "generating";
|
||||
const REGEN_PREVIEW_POLL_INTERVAL_MS = 2000;
|
||||
const REGEN_PREVIEW_TRACK_TIMEOUT_MS = 30 * 60 * 1000;
|
||||
|
||||
type TabKey = "current" | "blacklist";
|
||||
|
||||
type RegenPreviewState = {
|
||||
expiresAt: number;
|
||||
originalUpdatedAt: number;
|
||||
};
|
||||
|
||||
const TABS: { key: TabKey; label: string }[] = [
|
||||
{ key: "current", label: "当前视频" },
|
||||
{ key: "blacklist", label: "拉黑视频" },
|
||||
];
|
||||
|
||||
/**
|
||||
* 视频管理容器:顶部分段标签在「当前 / 隐藏 / 拉黑」三个视图间切换,
|
||||
* 激活标签同步到 URL ?tab=,标签上的计数来自 /videos/stats。
|
||||
*/
|
||||
export function VideosPage() {
|
||||
const [searchParams, setSearchParams] = useSearchParams();
|
||||
const rawTab = searchParams.get("tab");
|
||||
const activeTab: TabKey = rawTab === "blacklist" ? "blacklist" : "current";
|
||||
const [stats, setStats] = useState<api.VideoStats | null>(null);
|
||||
|
||||
async function refreshStats() {
|
||||
try {
|
||||
setStats(await api.getVideoStats());
|
||||
} catch {
|
||||
// 计数仅用于标签徽标,失败不阻塞主流程。
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
refreshStats();
|
||||
}, []);
|
||||
|
||||
function selectTab(key: TabKey) {
|
||||
setSearchParams(
|
||||
(prev) => {
|
||||
const next = new URLSearchParams(prev);
|
||||
if (key === "current") next.delete("tab");
|
||||
else next.set("tab", key);
|
||||
return next;
|
||||
},
|
||||
{ replace: true }
|
||||
);
|
||||
}
|
||||
|
||||
const counts: Record<TabKey, number | undefined> = {
|
||||
current: stats?.current,
|
||||
blacklist: stats?.blacklisted,
|
||||
};
|
||||
|
||||
return (
|
||||
<section>
|
||||
<header className="admin-page__header">
|
||||
<h1 className="admin-page__title">视频管理</h1>
|
||||
</header>
|
||||
|
||||
<div className="admin-video-tabs" role="tablist" aria-label="视频管理分类">
|
||||
{TABS.map((t) => (
|
||||
<button
|
||||
key={t.key}
|
||||
type="button"
|
||||
role="tab"
|
||||
aria-selected={activeTab === t.key}
|
||||
className={`admin-video-tab ${activeTab === t.key ? "is-active" : ""}`}
|
||||
onClick={() => selectTab(t.key)}
|
||||
>
|
||||
<span>{t.label}</span>
|
||||
{counts[t.key] !== undefined && (
|
||||
<span className="admin-video-tab__count">{counts[t.key]}</span>
|
||||
)}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{activeTab === "current" && <CurrentVideosTab onStatsChanged={refreshStats} />}
|
||||
{activeTab === "blacklist" && <BlacklistTab onStatsChanged={refreshStats} />}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- 当前视频 ----------
|
||||
|
||||
function CurrentVideosTab({ onStatsChanged }: { onStatsChanged: () => void }) {
|
||||
const [list, setList] = useState<api.AdminVideo[]>([]);
|
||||
const [drives, setDrives] = useState<api.AdminDrive[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
@@ -27,8 +125,11 @@ export function VideosPage() {
|
||||
const [batchRegening, setBatchRegening] = useState(false);
|
||||
const [batchDeleteOpen, setBatchDeleteOpen] = useState(false);
|
||||
const [batchDeleting, setBatchDeleting] = useState(false);
|
||||
const [batchDeleteSource, setBatchDeleteSource] = useState(false);
|
||||
const [deleteTarget, setDeleteTarget] = useState<api.AdminVideo | null>(null);
|
||||
const [deleting, setDeleting] = useState(false);
|
||||
const [deleteSource, setDeleteSource] = useState(false);
|
||||
const [regenPreviewById, setRegenPreviewById] = useState<Record<string, RegenPreviewState>>({});
|
||||
const pageSize = useVideosPageSize();
|
||||
const { show } = useToast();
|
||||
|
||||
@@ -55,6 +156,19 @@ export function VideosPage() {
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshListOnly() {
|
||||
try {
|
||||
const r = await api.listVideos({ driveId, page, size: pageSize, keyword: searchKeyword });
|
||||
setList(r.items ?? []);
|
||||
setTotal(r.total ?? 0);
|
||||
} catch {
|
||||
// Polling is only used to clear optimistic preview-generation state.
|
||||
}
|
||||
}
|
||||
|
||||
const trackedRegenCount = Object.keys(regenPreviewById).length;
|
||||
const hasGeneratingPreview = list.some((v) => v.previewStatus === REGEN_PREVIEW_STATUS);
|
||||
|
||||
useEffect(() => {
|
||||
refresh();
|
||||
}, [driveId, page, searchKeyword, pageSize]);
|
||||
@@ -72,9 +186,34 @@ export function VideosPage() {
|
||||
return () => window.clearTimeout(timer);
|
||||
}, [keyword]);
|
||||
|
||||
const driveNameMap = new Map(
|
||||
drives.map((d) => [d.id, d.name || d.id])
|
||||
);
|
||||
useEffect(() => {
|
||||
if (trackedRegenCount === 0 && !hasGeneratingPreview) return;
|
||||
const timer = window.setInterval(() => {
|
||||
refreshListOnly();
|
||||
}, REGEN_PREVIEW_POLL_INTERVAL_MS);
|
||||
return () => window.clearInterval(timer);
|
||||
}, [trackedRegenCount, hasGeneratingPreview, driveId, page, pageSize, searchKeyword]);
|
||||
|
||||
useEffect(() => {
|
||||
if (trackedRegenCount === 0) return;
|
||||
const now = Date.now();
|
||||
setRegenPreviewById((current) => {
|
||||
const next = { ...current };
|
||||
let changed = false;
|
||||
const byId = new Map(list.map((v) => [v.id, v]));
|
||||
for (const [id, state] of Object.entries(current)) {
|
||||
const video = byId.get(id);
|
||||
const updatedAt = videoUpdatedAtMs(video);
|
||||
if (!video || now >= state.expiresAt || updatedAt > state.originalUpdatedAt) {
|
||||
delete next[id];
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
return changed ? next : current;
|
||||
});
|
||||
}, [list, trackedRegenCount]);
|
||||
|
||||
const driveNameMap = new Map(drives.map((d) => [d.id, d.name || d.id]));
|
||||
|
||||
const listItems = list;
|
||||
const totalPages = Math.max(1, Math.ceil(total / pageSize));
|
||||
@@ -87,6 +226,7 @@ export function VideosPage() {
|
||||
async function handleRegen(v: api.AdminVideo) {
|
||||
try {
|
||||
await api.regenPreview(v.id);
|
||||
trackRegeneratingPreview([v]);
|
||||
show("已触发预览视频重生", "success");
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "触发失败", "error");
|
||||
@@ -100,21 +240,30 @@ export function VideosPage() {
|
||||
|
||||
async function handleBatchDelete() {
|
||||
if (selectedIds.size === 0) return;
|
||||
setBatchDeleteSource(false);
|
||||
setBatchDeleteOpen(true);
|
||||
}
|
||||
|
||||
async function confirmBatchRegen() {
|
||||
const ids = [...selectedIds];
|
||||
const videoById = new Map(listItems.map((v) => [v.id, v]));
|
||||
setBatchRegening(true);
|
||||
let success = 0;
|
||||
try {
|
||||
const results = await Promise.allSettled(
|
||||
ids.map((id) => api.regenPreview(id))
|
||||
const results = await Promise.allSettled(ids.map((id) => api.regenPreview(id)));
|
||||
const acceptedVideos: api.AdminVideo[] = [];
|
||||
results.forEach((r, index) => {
|
||||
if (r.status === "fulfilled") {
|
||||
const video = videoById.get(ids[index]);
|
||||
if (video) acceptedVideos.push(video);
|
||||
success++;
|
||||
}
|
||||
});
|
||||
trackRegeneratingPreview(acceptedVideos);
|
||||
show(
|
||||
`批量触发完成,成功 ${success} / ${ids.length} 个`,
|
||||
success === ids.length ? "success" : "info"
|
||||
);
|
||||
for (const r of results) {
|
||||
if (r.status === "fulfilled") success++;
|
||||
}
|
||||
show(`批量触发完成,成功 ${success} / ${ids.length} 个`, success === ids.length ? "success" : "info");
|
||||
setSelectedIds(new Set());
|
||||
setBatchRegenOpen(false);
|
||||
} finally {
|
||||
@@ -122,19 +271,40 @@ export function VideosPage() {
|
||||
}
|
||||
}
|
||||
|
||||
function trackRegeneratingPreview(videos: api.AdminVideo[]) {
|
||||
if (videos.length === 0) return;
|
||||
const startedAt = Date.now();
|
||||
setRegenPreviewById((current) => {
|
||||
const next = { ...current };
|
||||
for (const v of videos) {
|
||||
next[v.id] = {
|
||||
expiresAt: startedAt + REGEN_PREVIEW_TRACK_TIMEOUT_MS,
|
||||
originalUpdatedAt: videoUpdatedAtMs(v),
|
||||
};
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
function isPreviewGenerating(v: api.AdminVideo) {
|
||||
return !!regenPreviewById[v.id] || v.previewStatus === REGEN_PREVIEW_STATUS;
|
||||
}
|
||||
|
||||
async function confirmDeleteVideo() {
|
||||
if (!deleteTarget) return;
|
||||
const target = deleteTarget;
|
||||
setDeleting(true);
|
||||
try {
|
||||
const result = await api.deleteVideo(target.id);
|
||||
const result = await api.deleteVideo(target.id, { deleteSource });
|
||||
setDeleteTarget(null);
|
||||
setDeleteSource(false);
|
||||
setSelectedIds((ids) => {
|
||||
const next = new Set(ids);
|
||||
next.delete(target.id);
|
||||
return next;
|
||||
});
|
||||
show(result.deletedSource ? "已删除视频,并清理 91Spider 源文件" : "已删除视频", "success");
|
||||
show(result.deletedSource ? "已删除视频,并清理源文件" : "已删除视频", "success");
|
||||
onStatsChanged();
|
||||
if (listItems.length === 1 && page > 1) {
|
||||
setPage((p) => Math.max(1, p - 1));
|
||||
} else {
|
||||
@@ -156,7 +326,7 @@ export function VideosPage() {
|
||||
let deletedSources = 0;
|
||||
for (const id of ids) {
|
||||
try {
|
||||
const result = await api.deleteVideo(id);
|
||||
const result = await api.deleteVideo(id, { deleteSource: batchDeleteSource });
|
||||
success++;
|
||||
if (result.deletedSource) deletedSources++;
|
||||
} catch {
|
||||
@@ -165,13 +335,18 @@ export function VideosPage() {
|
||||
}
|
||||
const failed = ids.length - success;
|
||||
if (failed === 0) {
|
||||
const extra = deletedSources > 0 ? `,其中 ${deletedSources} 个清理了 91Spider 源文件` : "";
|
||||
const extra = deletedSources > 0 ? `,其中 ${deletedSources} 个清理了源文件` : "";
|
||||
show(`批量删除完成,成功 ${success} 个${extra}`, "success");
|
||||
} else {
|
||||
show(`批量删除完成,成功 ${success} / ${ids.length} 个,失败 ${failed} 个`, success > 0 ? "info" : "error");
|
||||
show(
|
||||
`批量删除完成,成功 ${success} / ${ids.length} 个,失败 ${failed} 个`,
|
||||
success > 0 ? "info" : "error"
|
||||
);
|
||||
}
|
||||
setSelectedIds(new Set());
|
||||
setBatchDeleteOpen(false);
|
||||
setBatchDeleteSource(false);
|
||||
onStatsChanged();
|
||||
if (success >= listItems.length && page > 1) {
|
||||
setPage((p) => Math.max(1, p - 1));
|
||||
} else {
|
||||
@@ -186,7 +361,7 @@ export function VideosPage() {
|
||||
if (selectedIds.size === listItems.length && listItems.length > 0) {
|
||||
setSelectedIds(new Set());
|
||||
} else {
|
||||
setSelectedIds(new Set(listItems.map(v => v.id)));
|
||||
setSelectedIds(new Set(listItems.map((v) => v.id)));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -204,52 +379,21 @@ export function VideosPage() {
|
||||
}
|
||||
|
||||
return (
|
||||
<section>
|
||||
<header className="admin-page__header">
|
||||
<h1 className="admin-page__title">视频管理</h1>
|
||||
<div className="admin-page__actions admin-videos-filter">
|
||||
<div className="admin-videos-filter__select-wrap">
|
||||
<select
|
||||
className="admin-videos-filter__select"
|
||||
value={driveId}
|
||||
onChange={(e) => {
|
||||
setDriveId(e.target.value);
|
||||
setPage(1);
|
||||
}}
|
||||
>
|
||||
<option value="">全部网盘</option>
|
||||
{drives.map((d) => (
|
||||
<option key={d.id} value={d.id}>
|
||||
{d.name || d.id}(已生成 {d.teaserReadyCount ?? 0},待生成{" "}
|
||||
{d.teaserPendingCount ?? 0})
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<ChevronDown size={15} className="admin-videos-filter__select-icon" aria-hidden="true" />
|
||||
</div>
|
||||
<form className="admin-videos-filter__search" onSubmit={handleSearchSubmit}>
|
||||
<Search size={14} className="admin-videos-filter__search-icon" />
|
||||
<input
|
||||
aria-label="搜索标题或作者"
|
||||
value={keyword}
|
||||
onChange={(e) => setKeyword(e.target.value)}
|
||||
placeholder="搜索标题 / 作者"
|
||||
/>
|
||||
</form>
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 刷新
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
<>
|
||||
<div className="admin-page__actions admin-videos-filter">
|
||||
<DriveFilter drives={drives} driveId={driveId} onChange={(id) => { setDriveId(id); setPage(1); }} withCounts />
|
||||
<SearchBox keyword={keyword} onChange={setKeyword} onSubmit={handleSearchSubmit} />
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 刷新
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{!loading && (
|
||||
<div className="admin-videos-list-toolbar">
|
||||
<div className="admin-videos-summary">{listSummary}</div>
|
||||
{selectedIds.size > 0 && (
|
||||
<div className="admin-videos-bulk-actions">
|
||||
<span className="admin-videos-bulk-actions__count">
|
||||
已选择 {selectedIds.size} 项
|
||||
</span>
|
||||
<span className="admin-videos-bulk-actions__count">已选择 {selectedIds.size} 项</span>
|
||||
<button type="button" className="admin-btn is-primary admin-videos-bulk-actions__btn" onClick={handleBatchRegen}>
|
||||
<RefreshCw size={13} /> 批量重生预览视频
|
||||
</button>
|
||||
@@ -262,18 +406,9 @@ export function VideosPage() {
|
||||
)}
|
||||
|
||||
{loading ? (
|
||||
<div className="admin-loading-state">
|
||||
<RefreshCw size={20} className="admin-spin" />
|
||||
<span>加载中...</span>
|
||||
</div>
|
||||
<LoadingState />
|
||||
) : loadError ? (
|
||||
<div className="admin-error-state">
|
||||
<strong>视频加载失败</strong>
|
||||
<span>{loadError}</span>
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 重试
|
||||
</button>
|
||||
</div>
|
||||
<ErrorState message={loadError} onRetry={refresh} />
|
||||
) : listItems.length === 0 ? (
|
||||
<div className="admin-empty-state">
|
||||
<div className="admin-empty-state__icon">
|
||||
@@ -290,14 +425,22 @@ export function VideosPage() {
|
||||
<table className="admin-table is-selectable admin-videos-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th className="is-checkbox" style={{ width: '40px' }}>
|
||||
<th className="is-checkbox" style={{ width: "40px" }}>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-table-checkbox-btn"
|
||||
onClick={toggleSelectAll}
|
||||
aria-label={selectedIds.size > 0 && selectedIds.size === listItems.length ? "清空当前页选择" : "选择当前页视频"}
|
||||
aria-label={
|
||||
selectedIds.size > 0 && selectedIds.size === listItems.length
|
||||
? "清空当前页选择"
|
||||
: "选择当前页视频"
|
||||
}
|
||||
>
|
||||
{selectedIds.size > 0 && selectedIds.size === listItems.length ? <CheckSquare size={16} /> : <Square size={16} />}
|
||||
{selectedIds.size > 0 && selectedIds.size === listItems.length ? (
|
||||
<CheckSquare size={16} />
|
||||
) : (
|
||||
<Square size={16} />
|
||||
)}
|
||||
</button>
|
||||
</th>
|
||||
<th>标题</th>
|
||||
@@ -318,40 +461,20 @@ export function VideosPage() {
|
||||
onClick={() => toggleSelect(v.id)}
|
||||
aria-label={`${selectedIds.has(v.id) ? "取消选择" : "选择"}视频 ${v.title}`}
|
||||
>
|
||||
{selectedIds.has(v.id) ? <CheckSquare size={16} color="var(--accent)" /> : <Square size={16} color="var(--border-strong)" />}
|
||||
{selectedIds.has(v.id) ? (
|
||||
<CheckSquare size={16} color="var(--accent)" />
|
||||
) : (
|
||||
<Square size={16} color="var(--border-strong)" />
|
||||
)}
|
||||
</button>
|
||||
</td>
|
||||
<td data-label="标题">
|
||||
<div className="admin-video-title-cell">
|
||||
<div className="admin-video-thumb-wrap" aria-hidden="true">
|
||||
{v.thumbnailUrl ? (
|
||||
<img className="admin-video-thumb" src={v.thumbnailUrl} alt="" />
|
||||
) : (
|
||||
<div className="admin-video-thumb-placeholder">
|
||||
<Image size={14} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="admin-video-title-body">
|
||||
<div className="admin-video-title">{v.title}</div>
|
||||
{fileMeta(v) && (
|
||||
<div className="admin-video-filemeta">{fileMeta(v)}</div>
|
||||
)}
|
||||
{(v.tags ?? []).length > 0 && (
|
||||
<div className="admin-pills admin-video-title-tags">
|
||||
{(v.tags ?? []).map((t) => (
|
||||
<span key={t} className="admin-pill">{t}</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<VideoFileMetaPills video={v} />
|
||||
</div>
|
||||
</div>
|
||||
<VideoTitleCell video={v} />
|
||||
</td>
|
||||
<td data-label="作者">{v.author || <span className="admin-text-faint">—</span>}</td>
|
||||
<td data-label="时长">{formatDur(v.durationSeconds)}</td>
|
||||
<td data-label="预览视频">
|
||||
<PreviewStatus s={v.previewStatus} />
|
||||
<PreviewStatus s={isPreviewGenerating(v) ? REGEN_PREVIEW_STATUS : v.previewStatus} />
|
||||
</td>
|
||||
<td data-label="来源" className="admin-mono-cell">
|
||||
{driveNameMap.get(v.driveId) ?? v.driveId}
|
||||
@@ -360,10 +483,24 @@ export function VideosPage() {
|
||||
<button type="button" className="admin-btn" onClick={() => setEditing(v)} title="编辑视频">
|
||||
<Edit size={13} />
|
||||
</button>{" "}
|
||||
<button type="button" className="admin-btn" onClick={() => handleRegen(v)} title="重生预览视频">
|
||||
<RefreshCw size={13} />
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => handleRegen(v)}
|
||||
disabled={isPreviewGenerating(v)}
|
||||
title={isPreviewGenerating(v) ? "预览视频正在生成" : "重生预览视频"}
|
||||
>
|
||||
<RefreshCw size={13} className={isPreviewGenerating(v) ? "admin-spin" : undefined} />
|
||||
</button>{" "}
|
||||
<button type="button" className="admin-btn is-danger" onClick={() => setDeleteTarget(v)} title="删除视频">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn is-danger"
|
||||
onClick={() => {
|
||||
setDeleteSource(false);
|
||||
setDeleteTarget(v);
|
||||
}}
|
||||
title="删除视频"
|
||||
>
|
||||
<Trash2 size={13} />
|
||||
</button>
|
||||
</td>
|
||||
@@ -371,43 +508,7 @@ export function VideosPage() {
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
<div className="admin-table-pagination">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage(1)}
|
||||
disabled={page <= 1}
|
||||
>
|
||||
首页
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage((p) => Math.max(1, p - 1))}
|
||||
disabled={page <= 1}
|
||||
>
|
||||
上一页
|
||||
</button>
|
||||
<span className="admin-table-pagination__info">
|
||||
第 {page} / {totalPages} 页,每页 {pageSize} 个
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage((p) => Math.min(totalPages, p + 1))}
|
||||
disabled={page >= totalPages}
|
||||
>
|
||||
下一页
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage(totalPages)}
|
||||
disabled={page >= totalPages}
|
||||
>
|
||||
末页
|
||||
</button>
|
||||
</div>
|
||||
<Pagination page={page} totalPages={totalPages} pageSize={pageSize} onPage={setPage} />
|
||||
</>
|
||||
)}
|
||||
|
||||
@@ -443,10 +544,15 @@ export function VideosPage() {
|
||||
modalClassName="admin-modal--delete-confirm"
|
||||
loading={deleting}
|
||||
onCancel={() => {
|
||||
if (!deleting) setDeleteTarget(null);
|
||||
if (!deleting) {
|
||||
setDeleteTarget(null);
|
||||
setDeleteSource(false);
|
||||
}
|
||||
}}
|
||||
onConfirm={confirmDeleteVideo}
|
||||
/>
|
||||
>
|
||||
<DeleteSourceOption checked={deleteSource} disabled={deleting} onChange={setDeleteSource} note="开启后会先删除源文件,失败则不会删除管理库记录。" />
|
||||
</ConfirmModal>
|
||||
<ConfirmModal
|
||||
open={batchDeleteOpen}
|
||||
title="批量删除视频"
|
||||
@@ -457,17 +563,361 @@ export function VideosPage() {
|
||||
modalClassName="admin-modal--delete-confirm"
|
||||
loading={batchDeleting}
|
||||
onCancel={() => {
|
||||
if (!batchDeleting) setBatchDeleteOpen(false);
|
||||
if (!batchDeleting) {
|
||||
setBatchDeleteOpen(false);
|
||||
setBatchDeleteSource(false);
|
||||
}
|
||||
}}
|
||||
onConfirm={confirmBatchDelete}
|
||||
>
|
||||
<DeleteSourceOption checked={batchDeleteSource} disabled={batchDeleting} onChange={setBatchDeleteSource} note="开启后会先删除源文件,失败的视频会保留管理库记录。" />
|
||||
</ConfirmModal>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- 拉黑视频 ----------
|
||||
|
||||
function BlacklistTab({ onStatsChanged }: { onStatsChanged: () => void }) {
|
||||
const [list, setList] = useState<api.AdminDeletedVideo[]>([]);
|
||||
const [drives, setDrives] = useState<api.AdminDrive[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [loadError, setLoadError] = useState("");
|
||||
const [keyword, setKeyword] = useState("");
|
||||
const [searchKeyword, setSearchKeyword] = useState("");
|
||||
const [page, setPage] = useState(1);
|
||||
const [total, setTotal] = useState(0);
|
||||
const [removeTarget, setRemoveTarget] = useState<api.AdminDeletedVideo | null>(null);
|
||||
const [removing, setRemoving] = useState(false);
|
||||
const pageSize = useVideosPageSize();
|
||||
const { show } = useToast();
|
||||
|
||||
async function refresh() {
|
||||
setLoading(true);
|
||||
setLoadError("");
|
||||
try {
|
||||
const [r, driveList] = await Promise.all([
|
||||
api.listBlacklist({ page, size: pageSize, keyword: searchKeyword }),
|
||||
api.listDrives(),
|
||||
]);
|
||||
setList(r.items ?? []);
|
||||
setTotal(r.total ?? 0);
|
||||
setDrives(driveList ?? []);
|
||||
} catch (e) {
|
||||
const message = e instanceof Error ? e.message : "加载失败";
|
||||
setLoadError(message);
|
||||
show(message, "error");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
refresh();
|
||||
}, [page, searchKeyword, pageSize]);
|
||||
|
||||
useEffect(() => {
|
||||
setPage(1);
|
||||
}, [pageSize]);
|
||||
|
||||
useEffect(() => {
|
||||
if (keyword === searchKeyword) return;
|
||||
const timer = window.setTimeout(() => {
|
||||
setSearchKeyword(keyword);
|
||||
setPage(1);
|
||||
}, 300);
|
||||
return () => window.clearTimeout(timer);
|
||||
}, [keyword]);
|
||||
|
||||
const driveNameMap = new Map(drives.map((d) => [d.id, d.name || d.id]));
|
||||
const totalPages = Math.max(1, Math.ceil(total / pageSize));
|
||||
|
||||
async function confirmRemove() {
|
||||
if (!removeTarget) return;
|
||||
const target = removeTarget;
|
||||
setRemoving(true);
|
||||
try {
|
||||
await api.removeBlacklist(target.id);
|
||||
setRemoveTarget(null);
|
||||
show("已移出黑名单,下次扫盘会重新入库", "success");
|
||||
onStatsChanged();
|
||||
if (list.length === 1 && page > 1) {
|
||||
setPage((p) => Math.max(1, p - 1));
|
||||
} else {
|
||||
refresh();
|
||||
}
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "操作失败", "error");
|
||||
} finally {
|
||||
setRemoving(false);
|
||||
}
|
||||
}
|
||||
|
||||
function handleSearchSubmit(e: React.FormEvent) {
|
||||
e.preventDefault();
|
||||
setSearchKeyword(keyword);
|
||||
setPage(1);
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="admin-tab-intro">
|
||||
被删除和被隐藏的视频会进入黑名单,扫盘时不再重新入库。这里只保留文件名等基本信息(原始记录、封面、预览已删除)。移出黑名单后,视频会在下次扫盘时被重新发现并入库
|
||||
</div>
|
||||
<div className="admin-page__actions admin-videos-filter">
|
||||
<SearchBox keyword={keyword} onChange={setKeyword} onSubmit={handleSearchSubmit} placeholder="搜索文件名" />
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 刷新
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{loading ? (
|
||||
<LoadingState />
|
||||
) : loadError ? (
|
||||
<ErrorState message={loadError} onRetry={refresh} />
|
||||
) : list.length === 0 ? (
|
||||
<div className="admin-empty-state">
|
||||
<div className="admin-empty-state__icon">
|
||||
<Ban size={48} />
|
||||
</div>
|
||||
<div className="admin-empty-state__text">黑名单为空。</div>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<div className="admin-videos-list-toolbar">
|
||||
<div className="admin-videos-summary">共 {total} 个拉黑视频</div>
|
||||
</div>
|
||||
<table className="admin-table admin-blacklist-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>文件名</th>
|
||||
<th>来源</th>
|
||||
<th>大小</th>
|
||||
<th>拉黑时间</th>
|
||||
<th className="is-actions">操作</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{list.map((v) => (
|
||||
<tr key={v.id}>
|
||||
<td data-label="文件名">
|
||||
<span className="admin-blacklist-filename">{v.fileName || <span className="admin-text-faint">(无文件名)</span>}</span>
|
||||
</td>
|
||||
<td data-label="来源" className="admin-mono-cell">
|
||||
{driveNameMap.get(v.driveId) ?? v.driveId}
|
||||
</td>
|
||||
<td data-label="大小">{v.size > 0 ? formatBytes(v.size) : <span className="admin-text-faint">—</span>}</td>
|
||||
<td data-label="拉黑时间">{formatDateTime(v.deletedAt)}</td>
|
||||
<td className="is-actions" data-label="操作">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn admin-blacklist-restore-btn"
|
||||
onClick={() => setRemoveTarget(v)}
|
||||
title="移出黑名单"
|
||||
>
|
||||
<RotateCcw size={13} /> 移出黑名单
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
<Pagination page={page} totalPages={totalPages} pageSize={pageSize} onPage={setPage} />
|
||||
</>
|
||||
)}
|
||||
|
||||
<ConfirmModal
|
||||
open={removeTarget !== null}
|
||||
title="移出黑名单"
|
||||
message={
|
||||
removeTarget
|
||||
? `确定把「${removeTarget.fileName || removeTarget.id}」移出黑名单吗?移出后它会在下次扫盘时被重新发现并入库。`
|
||||
: ""
|
||||
}
|
||||
confirmText="移出黑名单"
|
||||
centerMessage
|
||||
loading={removing}
|
||||
onCancel={() => {
|
||||
if (!removing) setRemoveTarget(null);
|
||||
}}
|
||||
onConfirm={confirmRemove}
|
||||
/>
|
||||
</section>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- 共享小组件 ----------
|
||||
|
||||
function DriveFilter({
|
||||
drives,
|
||||
driveId,
|
||||
onChange,
|
||||
withCounts = false,
|
||||
}: {
|
||||
drives: api.AdminDrive[];
|
||||
driveId: string;
|
||||
onChange: (id: string) => void;
|
||||
withCounts?: boolean;
|
||||
}) {
|
||||
return (
|
||||
<div className="admin-videos-filter__select-wrap">
|
||||
<select
|
||||
className="admin-videos-filter__select"
|
||||
value={driveId}
|
||||
onChange={(e) => onChange(e.target.value)}
|
||||
>
|
||||
<option value="">全部网盘</option>
|
||||
{drives.map((d) => (
|
||||
<option key={d.id} value={d.id}>
|
||||
{d.name || d.id}
|
||||
{withCounts ? `(已生成 ${d.teaserReadyCount ?? 0},待生成 ${d.teaserPendingCount ?? 0})` : ""}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<ChevronDown size={15} className="admin-videos-filter__select-icon" aria-hidden="true" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function SearchBox({
|
||||
keyword,
|
||||
onChange,
|
||||
onSubmit,
|
||||
placeholder = "搜索标题 / 作者",
|
||||
}: {
|
||||
keyword: string;
|
||||
onChange: (v: string) => void;
|
||||
onSubmit: (e: React.FormEvent) => void;
|
||||
placeholder?: string;
|
||||
}) {
|
||||
return (
|
||||
<form className="admin-videos-filter__search" onSubmit={onSubmit}>
|
||||
<Search size={14} className="admin-videos-filter__search-icon" />
|
||||
<input
|
||||
aria-label={placeholder}
|
||||
value={keyword}
|
||||
onChange={(e) => onChange(e.target.value)}
|
||||
placeholder={placeholder}
|
||||
/>
|
||||
</form>
|
||||
);
|
||||
}
|
||||
|
||||
function Pagination({
|
||||
page,
|
||||
totalPages,
|
||||
pageSize,
|
||||
onPage,
|
||||
}: {
|
||||
page: number;
|
||||
totalPages: number;
|
||||
pageSize: number;
|
||||
onPage: React.Dispatch<React.SetStateAction<number>>;
|
||||
}) {
|
||||
return (
|
||||
<div className="admin-table-pagination">
|
||||
<button type="button" className="admin-btn" onClick={() => onPage(() => 1)} disabled={page <= 1}>
|
||||
首页
|
||||
</button>
|
||||
<button type="button" className="admin-btn" onClick={() => onPage((p) => Math.max(1, p - 1))} disabled={page <= 1}>
|
||||
上一页
|
||||
</button>
|
||||
<span className="admin-table-pagination__info">
|
||||
第 {page} / {totalPages} 页,每页 {pageSize} 个
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => onPage((p) => Math.min(totalPages, p + 1))}
|
||||
disabled={page >= totalPages}
|
||||
>
|
||||
下一页
|
||||
</button>
|
||||
<button type="button" className="admin-btn" onClick={() => onPage(() => totalPages)} disabled={page >= totalPages}>
|
||||
末页
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function LoadingState() {
|
||||
return (
|
||||
<div className="admin-loading-state">
|
||||
<RefreshCw size={20} className="admin-spin" />
|
||||
<span>加载中...</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ErrorState({ message, onRetry }: { message: string; onRetry: () => void }) {
|
||||
return (
|
||||
<div className="admin-error-state">
|
||||
<strong>加载失败</strong>
|
||||
<span>{message}</span>
|
||||
<button type="button" className="admin-btn" onClick={onRetry}>
|
||||
<RefreshCw size={13} /> 重试
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function DeleteSourceOption({
|
||||
checked,
|
||||
disabled,
|
||||
onChange,
|
||||
note,
|
||||
}: {
|
||||
checked: boolean;
|
||||
disabled: boolean;
|
||||
onChange: (v: boolean) => void;
|
||||
note: string;
|
||||
}) {
|
||||
return (
|
||||
<label className="admin-delete-source-option">
|
||||
<input type="checkbox" checked={checked} disabled={disabled} onChange={(e) => onChange(e.target.checked)} />
|
||||
<span>
|
||||
<strong>同时删除网盘中的源文件</strong>
|
||||
<small>{note}</small>
|
||||
</span>
|
||||
</label>
|
||||
);
|
||||
}
|
||||
|
||||
function VideoTitleCell({ video: v }: { video: api.AdminVideo }) {
|
||||
return (
|
||||
<div className="admin-video-title-cell">
|
||||
<div className="admin-video-thumb-wrap" aria-hidden="true">
|
||||
{v.thumbnailUrl ? (
|
||||
<img className="admin-video-thumb" src={v.thumbnailUrl} alt="" />
|
||||
) : (
|
||||
<div className="admin-video-thumb-placeholder">
|
||||
<Image size={14} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="admin-video-title-body">
|
||||
<div className="admin-video-title">{v.title}</div>
|
||||
{fileMeta(v) && <div className="admin-video-filemeta">{fileMeta(v)}</div>}
|
||||
{(v.tags ?? []).length > 0 && (
|
||||
<div className="admin-pills admin-video-title-tags">
|
||||
{(v.tags ?? []).map((t) => (
|
||||
<span key={t} className="admin-pill">
|
||||
{t}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<VideoFileMetaPills video={v} />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function PreviewStatus({ s }: { s: string }) {
|
||||
if (s === REGEN_PREVIEW_STATUS) return <span className="admin-status is-generating">生成中</span>;
|
||||
if (s === "ready") return <span className="admin-status is-ok">就绪</span>;
|
||||
if (s === "failed") return <span className="admin-status is-error">失败</span>;
|
||||
if (s === "disabled") return <span className="admin-status">已关闭</span>;
|
||||
if (s === "skipped") return <span className="admin-status">跳过</span>;
|
||||
return <span className="admin-status is-pending">待生成</span>;
|
||||
}
|
||||
@@ -484,11 +934,7 @@ function VideoFileMetaPills({ video }: { video: api.AdminVideo }) {
|
||||
{part}
|
||||
</span>
|
||||
))}
|
||||
{category && (
|
||||
<span className="admin-video-filemeta-pill is-category">
|
||||
{category}
|
||||
</span>
|
||||
)}
|
||||
{category && <span className="admin-video-filemeta-pill is-category">{category}</span>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -500,11 +946,23 @@ function formatDur(sec: number): string {
|
||||
return `${m.toString().padStart(2, "0")}:${s.toString().padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
function formatDateTime(ms: number): string {
|
||||
if (!ms) return "—";
|
||||
const d = new Date(ms);
|
||||
if (Number.isNaN(d.getTime())) return "—";
|
||||
const pad = (n: number) => n.toString().padStart(2, "0");
|
||||
return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}`;
|
||||
}
|
||||
|
||||
function videoUpdatedAtMs(video?: api.AdminVideo): number {
|
||||
if (!video?.updatedAt) return 0;
|
||||
const value = Date.parse(video.updatedAt);
|
||||
return Number.isFinite(value) ? value : 0;
|
||||
}
|
||||
|
||||
function useVideosPageSize() {
|
||||
const [pageSize, setPageSize] = useState(() =>
|
||||
window.matchMedia(VIDEOS_MOBILE_QUERY).matches
|
||||
? MOBILE_VIDEOS_PAGE_SIZE
|
||||
: DESKTOP_VIDEOS_PAGE_SIZE
|
||||
window.matchMedia(VIDEOS_MOBILE_QUERY).matches ? MOBILE_VIDEOS_PAGE_SIZE : DESKTOP_VIDEOS_PAGE_SIZE
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -638,12 +1096,12 @@ function EditVideoModal({
|
||||
<div className="admin-thumbnail-preview">
|
||||
<input id={`${idPrefix}-video-thumbnail`} value={thumbnail} onChange={(e) => setThumbnail(e.target.value)} />
|
||||
{thumbnail && (
|
||||
<img
|
||||
src={thumbnail}
|
||||
alt="封面预览"
|
||||
className="admin-thumbnail-img"
|
||||
onError={(e) => (e.currentTarget.style.display = 'none')}
|
||||
onLoad={(e) => (e.currentTarget.style.display = 'block')}
|
||||
<img
|
||||
src={thumbnail}
|
||||
alt="封面预览"
|
||||
className="admin-thumbnail-img"
|
||||
onError={(e) => (e.currentTarget.style.display = "none")}
|
||||
onLoad={(e) => (e.currentTarget.style.display = "block")}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
@@ -685,11 +1143,7 @@ function fileMeta(v: api.AdminVideo): string {
|
||||
}
|
||||
|
||||
function fileMetaParts(v: api.AdminVideo): string[] {
|
||||
return [
|
||||
normalizeExt(v.ext),
|
||||
v.quality,
|
||||
v.size > 0 ? formatBytes(v.size) : "",
|
||||
].filter(Boolean);
|
||||
return [normalizeExt(v.ext), v.quality, v.size > 0 ? formatBytes(v.size) : ""].filter(Boolean);
|
||||
}
|
||||
|
||||
function normalizeExt(ext: string): string {
|
||||
@@ -705,7 +1159,5 @@ function splitList(s: string): string[] {
|
||||
}
|
||||
|
||||
function toggleTag(tags: string[], label: string): string[] {
|
||||
return tags.includes(label)
|
||||
? tags.filter((tag) => tag !== label)
|
||||
: [...tags, label];
|
||||
return tags.includes(label) ? tags.filter((tag) => tag !== label) : [...tags, label];
|
||||
}
|
||||
|
||||
+278
-15
@@ -12,13 +12,14 @@ async function request<T>(
|
||||
path: string,
|
||||
init: RequestInit = {}
|
||||
): Promise<T> {
|
||||
const headers = new Headers(init.headers ?? {});
|
||||
if (!(init.body instanceof FormData) && !headers.has("Content-Type")) {
|
||||
headers.set("Content-Type", "application/json");
|
||||
}
|
||||
const res = await fetch(BASE + path, {
|
||||
credentials: "include",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...(init.headers ?? {}),
|
||||
},
|
||||
...init,
|
||||
headers,
|
||||
});
|
||||
if (res.status === 401) {
|
||||
throw new UnauthorizedError();
|
||||
@@ -77,13 +78,13 @@ export function checkUpdate() {
|
||||
|
||||
export type AdminDrive = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "p123" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
kind: "quark" | "p115" | "p123" | "pikpak" | "wopan" | "guangyapan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
name: string;
|
||||
rootId: string;
|
||||
status: string;
|
||||
lastError?: string;
|
||||
hasCredential: boolean;
|
||||
/** 当前是否给该盘生成预览视频/封面(per-drive 开关,替代旧的全局 preview.enabled)。 */
|
||||
/** 当前是否给该盘生成预览视频(per-drive 开关,替代旧的全局 preview.enabled;封面不受影响)。 */
|
||||
teaserEnabled: boolean;
|
||||
/**
|
||||
* 用户在 admin 配置的"扫描跳过目录"集合(drive 侧目录 fileID 列表)。
|
||||
@@ -95,6 +96,11 @@ export type AdminDrive = {
|
||||
lastCrawlAt?: number;
|
||||
// spider91 专用代理地址;仅后台管理接口返回,用于编辑表单回显。
|
||||
spider91Proxy?: string;
|
||||
// Google Drive 是否使用 OpenList 在线续期 API;未配置时后端按 true 返回。
|
||||
googleDriveUseOnlineAPI?: boolean;
|
||||
// localstorage 的 .strm 是否允许指向存储根目录之外;未配置时后端按 false 返回。
|
||||
strmAllowOutsideRoot?: boolean;
|
||||
scanGenerationStatus?: DriveGenerationStatus;
|
||||
thumbnailGenerationStatus?: DriveGenerationStatus;
|
||||
previewGenerationStatus?: DriveGenerationStatus;
|
||||
fingerprintGenerationStatus?: DriveGenerationStatus;
|
||||
@@ -108,6 +114,12 @@ export type AdminDrive = {
|
||||
fingerprintReadyCount: number;
|
||||
fingerprintPendingCount: number;
|
||||
fingerprintFailedCount: number;
|
||||
// 浏览器兼容性转码:候选(待处理)/已转码/失败/检测后无需转码 计数与任务状态。
|
||||
transcodeGenerationStatus?: DriveGenerationStatus;
|
||||
transcodePendingCount: number;
|
||||
transcodeReadyCount: number;
|
||||
transcodeFailedCount: number;
|
||||
transcodeSkippedCount: number;
|
||||
};
|
||||
|
||||
export type DriveGenerationStatus = {
|
||||
@@ -115,6 +127,10 @@ export type DriveGenerationStatus = {
|
||||
currentTitle?: string;
|
||||
queueLength: number;
|
||||
cooldownUntil?: string;
|
||||
scannedCount: number;
|
||||
addedCount: number;
|
||||
doneCount: number;
|
||||
totalCount: number;
|
||||
};
|
||||
|
||||
export function listDrives() {
|
||||
@@ -139,7 +155,7 @@ export function getDriveStorage() {
|
||||
|
||||
export type UpsertDriveInput = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "p123" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
kind: "quark" | "p115" | "p123" | "pikpak" | "wopan" | "guangyapan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
name: string;
|
||||
rootId: string;
|
||||
credentials: Record<string, string>;
|
||||
@@ -170,7 +186,7 @@ export function deleteDrive(id: string, body: DeleteDriveInput) {
|
||||
}
|
||||
|
||||
export function rescan(id: string) {
|
||||
return request<{ ok: boolean }>(
|
||||
return request<{ ok: boolean; accepted: boolean; message?: string; status?: NightlyJobStatus }>(
|
||||
`/drives/${encodeURIComponent(id)}/rescan`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
@@ -183,6 +199,135 @@ export function stopDriveTasks(id: string) {
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- Crawlers ----------
|
||||
|
||||
export type AdminCrawler = {
|
||||
id: string;
|
||||
name: string;
|
||||
kind: "scriptcrawler" | "spider91";
|
||||
status: string;
|
||||
lastError?: string;
|
||||
scriptPath: string;
|
||||
scriptSourceUrl?: string;
|
||||
proxy?: string;
|
||||
targetNew?: string;
|
||||
uploadDriveId?: string;
|
||||
teaserEnabled: boolean;
|
||||
lastCrawlAt?: number;
|
||||
scanGenerationStatus?: DriveGenerationStatus;
|
||||
thumbnailGenerationStatus?: DriveGenerationStatus;
|
||||
previewGenerationStatus?: DriveGenerationStatus;
|
||||
fingerprintGenerationStatus?: DriveGenerationStatus;
|
||||
uploadGenerationStatus?: DriveGenerationStatus;
|
||||
thumbnailReadyCount: number;
|
||||
thumbnailPendingCount: number;
|
||||
thumbnailFailedCount: number;
|
||||
teaserReadyCount: number;
|
||||
teaserPendingCount: number;
|
||||
teaserFailedCount: number;
|
||||
fingerprintReadyCount: number;
|
||||
fingerprintPendingCount: number;
|
||||
fingerprintFailedCount: number;
|
||||
totalCrawledCount: number;
|
||||
localVideoCount: number;
|
||||
migratedVideoCount: number;
|
||||
};
|
||||
|
||||
export type UpsertCrawlerInput = {
|
||||
id?: string;
|
||||
scriptPath: string;
|
||||
scriptSourceUrl?: string;
|
||||
proxy?: string;
|
||||
targetNew?: string;
|
||||
uploadDriveId?: string;
|
||||
};
|
||||
|
||||
export type ImportCrawlerScriptResult = {
|
||||
scriptPath: string;
|
||||
name: string;
|
||||
sourceUrl?: string;
|
||||
};
|
||||
|
||||
export type CrawlerDryRunItem = {
|
||||
title: string;
|
||||
sourceId?: string;
|
||||
mediaUrl?: string;
|
||||
mediaLocalFile?: string;
|
||||
thumbnailUrl?: string;
|
||||
detailUrl?: string;
|
||||
};
|
||||
|
||||
export type CrawlerDryRunMediaCheck = {
|
||||
ok: boolean;
|
||||
status?: number;
|
||||
contentType?: string;
|
||||
contentLengthBytes?: number;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
export type CrawlerDryRunResult = {
|
||||
ok: boolean;
|
||||
items: CrawlerDryRunItem[];
|
||||
mediaCheck?: CrawlerDryRunMediaCheck;
|
||||
error?: string;
|
||||
log?: string[];
|
||||
durationMs: number;
|
||||
};
|
||||
|
||||
export function listCrawlers() {
|
||||
return request<AdminCrawler[]>("/crawlers");
|
||||
}
|
||||
|
||||
export function upsertCrawler(body: UpsertCrawlerInput) {
|
||||
return request<{ ok: boolean; id: string; warning?: string }>("/crawlers", {
|
||||
method: "POST",
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
export function importCrawlerScriptFile(file: File) {
|
||||
const form = new FormData();
|
||||
form.append("file", file);
|
||||
return request<ImportCrawlerScriptResult>("/crawlers/import-file", {
|
||||
method: "POST",
|
||||
body: form,
|
||||
});
|
||||
}
|
||||
|
||||
export function importCrawlerScriptURL(url: string) {
|
||||
return request<ImportCrawlerScriptResult>("/crawlers/import-url", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ url }),
|
||||
});
|
||||
}
|
||||
|
||||
export function testCrawlerScript(body: { scriptPath: string; proxy?: string }) {
|
||||
return request<CrawlerDryRunResult>("/crawlers/test-script", {
|
||||
method: "POST",
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
export function runCrawler(id: string) {
|
||||
return request<{ ok: boolean; accepted: boolean; message?: string; status?: NightlyJobStatus }>(
|
||||
`/crawlers/${encodeURIComponent(id)}/run`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
export function stopCrawlerTasks(id: string) {
|
||||
return request<{ ok: boolean; stopped: boolean }>(
|
||||
`/crawlers/${encodeURIComponent(id)}/tasks/stop`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
export function deleteCrawler(id: string) {
|
||||
return request<{ ok: boolean; deletedVideos: number; deletedScript?: boolean; warning?: string }>(`/crawlers/${encodeURIComponent(id)}`, {
|
||||
method: "DELETE",
|
||||
});
|
||||
}
|
||||
|
||||
export type P123QRSession = {
|
||||
loginUuid: string;
|
||||
uniID: string;
|
||||
@@ -210,6 +355,55 @@ export function getP123QRStatus(uniID: string, loginUuid: string) {
|
||||
);
|
||||
}
|
||||
|
||||
export type WopanQRSession = {
|
||||
uuid: string;
|
||||
qrImageDataUrl: string;
|
||||
expiresAt?: string;
|
||||
};
|
||||
|
||||
export type WopanQRStatus = {
|
||||
state: number;
|
||||
statusText: string;
|
||||
accessToken?: string;
|
||||
refreshToken?: string;
|
||||
familyID?: string;
|
||||
};
|
||||
|
||||
export function startWopanQRLogin() {
|
||||
return request<WopanQRSession>("/drives/wopan/qr", { method: "POST" });
|
||||
}
|
||||
|
||||
export function getWopanQRStatus(uuid: string) {
|
||||
return request<WopanQRStatus>(`/drives/wopan/qr/${encodeURIComponent(uuid)}`);
|
||||
}
|
||||
|
||||
export type GuangYaPanQRSession = {
|
||||
deviceCode: string;
|
||||
qrCodeUrl: string;
|
||||
qrImageDataUrl: string;
|
||||
intervalSeconds: number;
|
||||
expiresAt?: string;
|
||||
};
|
||||
|
||||
export type GuangYaPanQRStatus = {
|
||||
state: "pending" | "success" | "expired" | "denied" | "error";
|
||||
statusText: string;
|
||||
intervalSeconds?: number;
|
||||
accessToken?: string;
|
||||
refreshToken?: string;
|
||||
tokenType?: string;
|
||||
expiresIn?: number;
|
||||
};
|
||||
|
||||
export function startGuangYaPanQRLogin() {
|
||||
return request<GuangYaPanQRSession>("/drives/guangyapan/qr", { method: "POST" });
|
||||
}
|
||||
|
||||
export function getGuangYaPanQRStatus(deviceCode: string) {
|
||||
const qs = new URLSearchParams({ deviceCode });
|
||||
return request<GuangYaPanQRStatus>(`/drives/guangyapan/qr/status?${qs.toString()}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* 切换某个云盘的预览视频生成开关。点击网盘列表里行内的 toggle 按钮时调用。
|
||||
*
|
||||
@@ -291,6 +485,26 @@ export function regenFailedFingerprints(id: string) {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 手动开启某存储的浏览器兼容性转码(AVI/WMV 等浏览器播不动的视频转 H.264 MP4,
|
||||
* 产物上传回同一存储)。转码默认关闭、从不自动运行,这是唯一入口;
|
||||
* 任务处理完候选列表后自然结束。
|
||||
*/
|
||||
export function startDriveTranscode(id: string) {
|
||||
return request<{ ok: boolean; accepted: boolean; message?: string }>(
|
||||
`/drives/${encodeURIComponent(id)}/transcode/start`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
/** 手动停止某存储正在进行的转码任务。 */
|
||||
export function stopDriveTranscode(id: string) {
|
||||
return request<{ ok: boolean; stopped: boolean }>(
|
||||
`/drives/${encodeURIComponent(id)}/transcode/stop`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- Videos ----------
|
||||
|
||||
export type AdminVideo = {
|
||||
@@ -324,7 +538,9 @@ export type AdminVideoList = {
|
||||
size: number;
|
||||
};
|
||||
|
||||
export function listVideos(params: { driveId?: string; page?: number; size?: number; keyword?: string } = {}) {
|
||||
export function listVideos(
|
||||
params: { driveId?: string; page?: number; size?: number; keyword?: string } = {}
|
||||
) {
|
||||
const qs = new URLSearchParams();
|
||||
if (params.driveId) qs.set("driveId", params.driveId);
|
||||
if (params.page) qs.set("page", String(params.page));
|
||||
@@ -334,6 +550,50 @@ export function listVideos(params: { driveId?: string; page?: number; size?: num
|
||||
return request<AdminVideoList>(`/videos${suffix}`);
|
||||
}
|
||||
|
||||
// 后台视频管理两个标签页的计数。
|
||||
export type VideoStats = {
|
||||
current: number;
|
||||
blacklisted: number;
|
||||
};
|
||||
|
||||
export function getVideoStats() {
|
||||
return request<VideoStats>("/videos/stats");
|
||||
}
|
||||
|
||||
// 黑名单(被拉黑/手动删除、扫盘不再入库的视频)。原始记录已删除,
|
||||
// 只剩文件名/来源盘/大小/拉黑时间。
|
||||
export type AdminDeletedVideo = {
|
||||
id: string;
|
||||
driveId: string;
|
||||
fileId: string;
|
||||
fileName: string;
|
||||
size: number;
|
||||
deletedAt: number;
|
||||
};
|
||||
|
||||
export type AdminBlacklistList = {
|
||||
items: AdminDeletedVideo[];
|
||||
total: number;
|
||||
page: number;
|
||||
size: number;
|
||||
};
|
||||
|
||||
export function listBlacklist(params: { page?: number; size?: number; keyword?: string } = {}) {
|
||||
const qs = new URLSearchParams();
|
||||
if (params.page) qs.set("page", String(params.page));
|
||||
if (params.size) qs.set("size", String(params.size));
|
||||
if (params.keyword) qs.set("keyword", params.keyword);
|
||||
const suffix = qs.toString() ? `?${qs.toString()}` : "";
|
||||
return request<AdminBlacklistList>(`/blacklist${suffix}`);
|
||||
}
|
||||
|
||||
// 把视频移出黑名单(删除墓碑),下次扫盘会重新入库。
|
||||
export function removeBlacklist(id: string) {
|
||||
return request<{ ok: boolean }>(`/blacklist/${encodeURIComponent(id)}`, {
|
||||
method: "DELETE",
|
||||
});
|
||||
}
|
||||
|
||||
export type UpdateVideoInput = Partial<{
|
||||
title: string;
|
||||
author: string;
|
||||
@@ -353,10 +613,13 @@ export function updateVideo(id: string, body: UpdateVideoInput) {
|
||||
});
|
||||
}
|
||||
|
||||
export function deleteVideo(id: string) {
|
||||
export function deleteVideo(id: string, options: { deleteSource?: boolean } = {}) {
|
||||
return request<{ ok: boolean; deletedSource: boolean }>(
|
||||
`/videos/${encodeURIComponent(id)}`,
|
||||
{ method: "DELETE" }
|
||||
{
|
||||
method: "DELETE",
|
||||
body: JSON.stringify({ deleteSource: !!options.deleteSource }),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@@ -397,14 +660,14 @@ export function deleteTag(id: number) {
|
||||
|
||||
// ---------- Settings ----------
|
||||
|
||||
export type Theme = "dark" | "pink";
|
||||
export type Theme = "dark" | "pink" | "sky";
|
||||
|
||||
export type Settings = {
|
||||
theme: Theme;
|
||||
/**
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115、p123 或 onedrive drive)。
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115、p123、onedrive、googledrive 或 wopan drive)。
|
||||
* - 空字符串:本地保存,不上传到云盘。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, p123, onedrive}。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, p123, onedrive, googledrive, wopan}。
|
||||
*/
|
||||
spider91UploadDriveId: string;
|
||||
};
|
||||
@@ -448,7 +711,7 @@ export function getNightlyJobStatus() {
|
||||
}
|
||||
|
||||
export function runNightlyJob() {
|
||||
return request<{ ok: boolean; accepted: boolean; status: NightlyJobStatus }>(
|
||||
return request<{ ok: boolean; accepted: boolean; status: NightlyJobStatus; message?: string }>(
|
||||
"/jobs/nightly/run",
|
||||
{ method: "POST" }
|
||||
);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { PlayCircle, Power, PowerOff, RotateCcw } from "lucide-react";
|
||||
import { CircleStop, PlayCircle, Power, PowerOff, RotateCcw, Wand2 } from "lucide-react";
|
||||
import * as api from "../api";
|
||||
import { formatBytes } from "../storageFormat";
|
||||
import {
|
||||
@@ -101,13 +101,17 @@ export function StatusTag({
|
||||
error?: string;
|
||||
hasCred: boolean;
|
||||
}) {
|
||||
if (kind === "spider91") {
|
||||
return (
|
||||
<span className="admin-status is-error" title={error || "请到爬虫管理添加爬虫脚本"}>
|
||||
已废弃
|
||||
</span>
|
||||
);
|
||||
}
|
||||
if (kind !== "spider91" && !hasCred) {
|
||||
return <span className="admin-status is-pending">未配置凭证</span>;
|
||||
}
|
||||
if (status === "ok") {
|
||||
if (kind === "spider91") {
|
||||
return <span className="admin-status is-ok">已就绪</span>;
|
||||
}
|
||||
return <span className="admin-status is-ok">已连接</span>;
|
||||
}
|
||||
if (status === "error")
|
||||
@@ -159,20 +163,26 @@ export function DriveGenerationPanel({
|
||||
regenFailedThumbId,
|
||||
regenFailedFingerprintId,
|
||||
togglingTeaserId,
|
||||
togglingTranscodeId,
|
||||
onToggleTeaser,
|
||||
onRegenFailed,
|
||||
onRegenFailedThumbnails,
|
||||
onRegenFailedFingerprints,
|
||||
onStartTranscode,
|
||||
onStopTranscode,
|
||||
}: {
|
||||
d: api.AdminDrive;
|
||||
regenFailedId: string;
|
||||
regenFailedThumbId: string;
|
||||
regenFailedFingerprintId: string;
|
||||
togglingTeaserId: string;
|
||||
togglingTranscodeId: string;
|
||||
onToggleTeaser: () => void;
|
||||
onRegenFailed: () => void;
|
||||
onRegenFailedThumbnails: () => void;
|
||||
onRegenFailedFingerprints: () => void;
|
||||
onStartTranscode: () => void;
|
||||
onStopTranscode: () => void;
|
||||
}) {
|
||||
const canQueueThumbnails =
|
||||
(d.thumbnailFailedCount ?? 0) > 0 ||
|
||||
@@ -182,6 +192,12 @@ export function DriveGenerationPanel({
|
||||
(d.teaserFailedCount ?? 0) > 0 || (d.teaserPendingCount ?? 0) > 0;
|
||||
const canQueueFingerprints =
|
||||
(d.fingerprintFailedCount ?? 0) > 0 || (d.fingerprintPendingCount ?? 0) > 0;
|
||||
// 转码默认不运行,只能在这里手动开启/停止。
|
||||
// 候选 = 还没出结果的不兼容格式视频 + 上次失败的(重新开始会自动重试)。
|
||||
const transcodeRunning =
|
||||
(d.transcodeGenerationStatus?.state || "idle") !== "idle";
|
||||
const canStartTranscode =
|
||||
(d.transcodePendingCount ?? 0) > 0 || (d.transcodeFailedCount ?? 0) > 0;
|
||||
|
||||
return (
|
||||
<div className="admin-detail-card">
|
||||
@@ -204,6 +220,11 @@ export function DriveGenerationPanel({
|
||||
</header>
|
||||
|
||||
<div className="admin-gen-columns">
|
||||
<DriveGenCol
|
||||
label={d.kind === "spider91" ? "已废弃" : "扫盘"}
|
||||
status={d.scanGenerationStatus}
|
||||
showCounts={false}
|
||||
/>
|
||||
<DriveGenCol
|
||||
label="封面"
|
||||
status={d.thumbnailGenerationStatus}
|
||||
@@ -226,6 +247,13 @@ export function DriveGenerationPanel({
|
||||
pending={d.fingerprintPendingCount}
|
||||
failed={d.fingerprintFailedCount}
|
||||
/>
|
||||
<DriveGenCol
|
||||
label="转码"
|
||||
status={d.transcodeGenerationStatus}
|
||||
ready={d.transcodeReadyCount}
|
||||
pending={d.transcodePendingCount}
|
||||
failed={d.transcodeFailedCount}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="admin-detail-actions">
|
||||
@@ -253,6 +281,33 @@ export function DriveGenerationPanel({
|
||||
<RotateCcw size={13} />
|
||||
<span>{(d.fingerprintFailedCount ?? 0) > 0 ? "重试失败指纹" : "继续生成指纹"}</span>
|
||||
</button>
|
||||
{transcodeRunning ? (
|
||||
<button
|
||||
className="admin-btn is-stop"
|
||||
disabled={togglingTranscodeId === d.id}
|
||||
onClick={onStopTranscode}
|
||||
title="停止当前的转码任务。未处理的视频保持原状态,下次开始时继续。"
|
||||
>
|
||||
<CircleStop size={13} />
|
||||
<span>{togglingTranscodeId === d.id ? "停止中..." : "停止转码"}</span>
|
||||
</button>
|
||||
) : (
|
||||
<button
|
||||
className="admin-btn"
|
||||
disabled={!canStartTranscode || togglingTranscodeId === d.id}
|
||||
onClick={onStartTranscode}
|
||||
title="把浏览器播放不了的视频(AVI/WMV/RMVB、MPEG-4 等老格式)转码成 H.264 MP4 并上传回本存储。转码不会自动运行,只能在这里手动开启。"
|
||||
>
|
||||
<Wand2 size={13} />
|
||||
<span>
|
||||
{togglingTranscodeId === d.id
|
||||
? "开启中..."
|
||||
: (d.transcodeFailedCount ?? 0) > 0 && (d.transcodePendingCount ?? 0) === 0
|
||||
? "重试失败转码"
|
||||
: "开始转码"}
|
||||
</span>
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
@@ -265,6 +320,7 @@ function DriveGenCol({
|
||||
pending,
|
||||
failed,
|
||||
extra,
|
||||
showCounts = true,
|
||||
}: {
|
||||
label: string;
|
||||
status?: api.DriveGenerationStatus;
|
||||
@@ -272,10 +328,14 @@ function DriveGenCol({
|
||||
pending?: number;
|
||||
failed?: number;
|
||||
extra?: number;
|
||||
showCounts?: boolean;
|
||||
}) {
|
||||
const state = status?.state || "idle";
|
||||
const detail = generationDetail(status);
|
||||
const title = generationTitle(status, detail);
|
||||
const stateLabel = label === "抓取" && state === "scanning" ? "抓取中" : generationStateLabel(state);
|
||||
const showScanProgress = !showCounts && (state === "scanning" || (status?.scannedCount ?? 0) > 0 || (status?.addedCount ?? 0) > 0);
|
||||
const scannedLabel = label === "抓取" ? "已抓取" : "已扫描";
|
||||
return (
|
||||
<div className="admin-gen-col">
|
||||
<div className="admin-gen-col__head">
|
||||
@@ -284,18 +344,26 @@ function DriveGenCol({
|
||||
className={`admin-status admin-generation-state is-${generationStateClass(state)}`}
|
||||
title={title || undefined}
|
||||
>
|
||||
{generationStateLabel(state)}
|
||||
{stateLabel}
|
||||
</span>
|
||||
</div>
|
||||
{detail && <div className="admin-gen-col__detail">{detail}</div>}
|
||||
<div className="admin-gen-col__counts">
|
||||
<div className="admin-gen-col__count"><span>就绪</span><strong>{ready ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>待生成</span><strong>{pending ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>失败</span><strong>{failed ?? 0}</strong></div>
|
||||
{(extra ?? 0) > 0 && (
|
||||
<div className="admin-gen-col__count"><span>待补时长</span><strong>{extra}</strong></div>
|
||||
)}
|
||||
</div>
|
||||
{showScanProgress && (
|
||||
<div className="admin-gen-col__counts admin-gen-col__counts--scan">
|
||||
<div className="admin-gen-col__count"><span>{scannedLabel}</span><strong>{status?.scannedCount ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>预计新增</span><strong>{status?.addedCount ?? 0}</strong></div>
|
||||
</div>
|
||||
)}
|
||||
{showCounts && (
|
||||
<div className="admin-gen-col__counts">
|
||||
<div className="admin-gen-col__count"><span>就绪</span><strong>{ready ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>待生成</span><strong>{pending ?? 0}</strong></div>
|
||||
<div className="admin-gen-col__count"><span>失败</span><strong>{failed ?? 0}</strong></div>
|
||||
{(extra ?? 0) > 0 && (
|
||||
<div className="admin-gen-col__count"><span>待补时长</span><strong>{extra}</strong></div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { useId, useMemo, useState } from "react";
|
||||
import { ArrowLeft } from "lucide-react";
|
||||
import { ArrowLeft, ChevronDown } from "lucide-react";
|
||||
import { P123QRCodeLogin } from "./P123QRCodeLogin";
|
||||
import { WopanQRCodeLogin } from "./WopanQRCodeLogin";
|
||||
import { GuangYaPanQRCodeLogin } from "./GuangYaPanQRCodeLogin";
|
||||
import { Spider91UploadTargetField } from "./Spider91UploadTargetField";
|
||||
import {
|
||||
FormState,
|
||||
@@ -21,14 +23,14 @@ type DriveOption = {
|
||||
|
||||
const DRIVE_OPTIONS: DriveOption[] = [
|
||||
{ kind: "p115", label: "115 网盘", abbr: "115", desc: "302直链,不占带宽" },
|
||||
{ kind: "p123", label: "123 云盘", abbr: "123", desc: "扫码登录,302直链" },
|
||||
{ kind: "p123", label: "123网盘", abbr: "123", desc: "扫码登录,302直链" },
|
||||
{ kind: "pikpak", label: "PikPak", abbr: "Pk", desc: "302直链,稳定快速" },
|
||||
{ kind: "guangyapan", label: "光鸭网盘", abbr: "GY", desc: "扫码登录,302直链" },
|
||||
{ kind: "onedrive", label: "OneDrive", abbr: "OD", desc: "302直链,微软网盘" },
|
||||
{ kind: "googledrive", label: "Google Drive", abbr: "GD", desc: "服务器中转模式" },
|
||||
{ kind: "localstorage", label: "本地存储", abbr: "Lo", desc: "本机文件目录" },
|
||||
{ kind: "spider91", label: "91 爬虫", abbr: "91", desc: "自动抓取热门视频" },
|
||||
{ kind: "quark", label: "夸克网盘", abbr: "Qk", desc: "302直链" },
|
||||
{ kind: "wopan", label: "联通沃盘", abbr: "Wo", desc: "302直链" },
|
||||
{ kind: "wopan", label: "联通网盘", abbr: "Wo", desc: "302直链" },
|
||||
];
|
||||
|
||||
export function DriveForm({
|
||||
@@ -49,7 +51,7 @@ export function DriveForm({
|
||||
onBack?: () => void;
|
||||
}) {
|
||||
const idPrefix = useId();
|
||||
const fields = useMemo(() => credentialFields(form.kind), [form.kind]);
|
||||
const fields = useMemo(() => credentialFields(form.kind, form.creds), [form.kind, form.creds]);
|
||||
const help = credentialHelp(form.kind, isEdit);
|
||||
const [step, setStep] = useState<"type" | "form">(isEdit ? "form" : "type");
|
||||
const nameId = `${idPrefix}-drive-name`;
|
||||
@@ -178,27 +180,86 @@ export function DriveForm({
|
||||
/>
|
||||
)}
|
||||
|
||||
{form.kind === "wopan" && (
|
||||
<WopanQRCodeLogin
|
||||
onCredentials={(credentials) =>
|
||||
onChange({
|
||||
...form,
|
||||
creds: {
|
||||
...form.creds,
|
||||
access_token: credentials.accessToken,
|
||||
refresh_token: credentials.refreshToken,
|
||||
...(credentials.familyID ? { family_id: credentials.familyID } : {}),
|
||||
},
|
||||
})
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{form.kind === "guangyapan" && (
|
||||
<GuangYaPanQRCodeLogin
|
||||
onCredentials={(credentials) =>
|
||||
onChange({
|
||||
...form,
|
||||
creds: {
|
||||
...form.creds,
|
||||
access_token: credentials.accessToken,
|
||||
refresh_token: credentials.refreshToken,
|
||||
},
|
||||
})
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{fields.map((f) => (
|
||||
<div key={f.key} className="admin-form__row">
|
||||
<label htmlFor={`${idPrefix}-credential-${f.key}`}>
|
||||
{f.label}
|
||||
{f.required && " *"}
|
||||
</label>
|
||||
{f.multiline ? (
|
||||
<textarea
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
value={form.creds[f.key] ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
placeholder={f.placeholder}
|
||||
/>
|
||||
{f.type === "select" ? (
|
||||
<>
|
||||
<label htmlFor={`${idPrefix}-credential-${f.key}`}>
|
||||
{f.label}
|
||||
{f.required && " *"}
|
||||
</label>
|
||||
<div className="admin-form-select-wrap">
|
||||
<select
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
className="admin-form-select"
|
||||
value={form.creds[f.key] ?? f.defaultValue ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
>
|
||||
{(f.options ?? []).map((option) => (
|
||||
<option key={option.value} value={option.value}>
|
||||
{option.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<ChevronDown size={15} className="admin-form-select__icon" aria-hidden="true" />
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<input
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
type={credentialInputType(f.key)}
|
||||
value={form.creds[f.key] ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
placeholder={f.placeholder}
|
||||
/>
|
||||
<>
|
||||
<label htmlFor={`${idPrefix}-credential-${f.key}`}>
|
||||
{f.label}
|
||||
{f.required && " *"}
|
||||
</label>
|
||||
{f.multiline ? (
|
||||
<textarea
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
value={form.creds[f.key] ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
placeholder={f.placeholder}
|
||||
required={f.required && !isEdit}
|
||||
/>
|
||||
) : (
|
||||
<input
|
||||
id={`${idPrefix}-credential-${f.key}`}
|
||||
type={credentialInputType(f.key)}
|
||||
value={form.creds[f.key] ?? ""}
|
||||
onChange={(e) => setCred(f.key, e.target.value)}
|
||||
placeholder={f.placeholder}
|
||||
required={f.required && !isEdit}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
{f.help && <div className="admin-form__help">{f.help}</div>}
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { QrCode } from "lucide-react";
|
||||
import * as api from "../api";
|
||||
import { useToast } from "../ToastContext";
|
||||
|
||||
function guangYaPanQRStatusClass(
|
||||
status: api.GuangYaPanQRStatus | null,
|
||||
completed: boolean,
|
||||
error: string
|
||||
): string {
|
||||
if (completed || status?.state === "success") return "is-ok";
|
||||
if (error || status?.state === "expired" || status?.state === "denied" || status?.state === "error")
|
||||
return "is-error";
|
||||
return "is-pending";
|
||||
}
|
||||
|
||||
export function GuangYaPanQRCodeLogin({
|
||||
onCredentials,
|
||||
}: {
|
||||
onCredentials: (credentials: {
|
||||
accessToken: string;
|
||||
refreshToken: string;
|
||||
}) => void;
|
||||
}) {
|
||||
const { show } = useToast();
|
||||
const [session, setSession] = useState<api.GuangYaPanQRSession | null>(null);
|
||||
const [status, setStatus] = useState<api.GuangYaPanQRStatus | null>(null);
|
||||
const [starting, setStarting] = useState(false);
|
||||
const [pollingError, setPollingError] = useState("");
|
||||
const [completed, setCompleted] = useState(false);
|
||||
|
||||
async function start() {
|
||||
setStarting(true);
|
||||
setPollingError("");
|
||||
setCompleted(false);
|
||||
setStatus(null);
|
||||
try {
|
||||
const next = await api.startGuangYaPanQRLogin();
|
||||
setSession(next);
|
||||
} catch (e) {
|
||||
setSession(null);
|
||||
show(e instanceof Error ? e.message : "生成二维码失败", "error");
|
||||
} finally {
|
||||
setStarting(false);
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (!session || completed) return;
|
||||
const activeSession = session;
|
||||
let stopped = false;
|
||||
let timer: number | undefined;
|
||||
let delayMs = Math.max(1000, (activeSession.intervalSeconds || 5) * 1000);
|
||||
|
||||
async function poll() {
|
||||
if (stopped) return;
|
||||
try {
|
||||
const next = await api.getGuangYaPanQRStatus(activeSession.deviceCode);
|
||||
if (stopped) return;
|
||||
setStatus(next);
|
||||
setPollingError("");
|
||||
if (next.intervalSeconds && next.intervalSeconds > 0) {
|
||||
delayMs = Math.max(1000, next.intervalSeconds * 1000);
|
||||
}
|
||||
if (next.accessToken && next.refreshToken) {
|
||||
stopped = true;
|
||||
if (timer) window.clearTimeout(timer);
|
||||
setCompleted(true);
|
||||
onCredentials({
|
||||
accessToken: next.accessToken,
|
||||
refreshToken: next.refreshToken,
|
||||
});
|
||||
show("扫码成功,已填入 access_token 和 refresh_token,保存后生效", "success");
|
||||
return;
|
||||
}
|
||||
if (next.state === "expired" || next.state === "denied" || next.state === "error") {
|
||||
stopped = true;
|
||||
if (timer) window.clearTimeout(timer);
|
||||
return;
|
||||
}
|
||||
} catch (e) {
|
||||
if (stopped) return;
|
||||
setPollingError(e instanceof Error ? e.message : "查询扫码状态失败");
|
||||
}
|
||||
if (!stopped) {
|
||||
timer = window.setTimeout(poll, delayMs);
|
||||
}
|
||||
}
|
||||
|
||||
poll();
|
||||
return () => {
|
||||
stopped = true;
|
||||
if (timer) window.clearTimeout(timer);
|
||||
};
|
||||
}, [session, completed, onCredentials, show]);
|
||||
|
||||
const statusText = completed
|
||||
? "已获取凭证"
|
||||
: pollingError || status?.statusText || (session ? "等待扫码" : "未生成二维码");
|
||||
const statusClass = guangYaPanQRStatusClass(status, completed, pollingError);
|
||||
|
||||
return (
|
||||
<div className="admin-form__row">
|
||||
<label>扫码登录</label>
|
||||
<div className="admin-p123-qr">
|
||||
<div className="admin-p123-qr__actions">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={start}
|
||||
disabled={starting}
|
||||
>
|
||||
<QrCode size={14} />
|
||||
{starting ? "生成中..." : session ? "重新生成二维码" : "生成二维码"}
|
||||
</button>
|
||||
<span className={`admin-status ${statusClass}`}>{statusText}</span>
|
||||
</div>
|
||||
|
||||
{session && (
|
||||
<div className="admin-p123-qr__body">
|
||||
<img
|
||||
className="admin-p123-qr__image"
|
||||
src={session.qrImageDataUrl}
|
||||
alt="光鸭网盘扫码登录二维码"
|
||||
/>
|
||||
<div className="admin-p123-qr__meta">
|
||||
<div className="admin-form__help">
|
||||
使用光鸭 App 扫码并确认登录;确认后系统会自动填入 access_token 和 refresh_token。
|
||||
</div>
|
||||
{session.expiresAt && (
|
||||
<div className="admin-form__help">
|
||||
过期时间:{new Date(session.expiresAt).toLocaleTimeString("zh-CN", {
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
{(status?.state === "expired" || status?.state === "denied") && (
|
||||
<div className="admin-form__help">
|
||||
当前二维码{status.state === "denied" ? "已被拒绝" : "已过期"},请重新生成。
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -113,11 +113,11 @@ export function P123QRCodeLogin({ onToken }: { onToken: (token: string) => void
|
||||
<img
|
||||
className="admin-p123-qr__image"
|
||||
src={session.qrImageDataUrl}
|
||||
alt="123 云盘扫码登录二维码"
|
||||
alt="123网盘扫码登录二维码"
|
||||
/>
|
||||
<div className="admin-p123-qr__meta">
|
||||
<div className="admin-form__help">
|
||||
使用微信或 123 云盘 App 扫码并确认登录;确认后系统会自动填入 access_token。
|
||||
使用微信或 123网盘 App 扫码并确认登录;确认后系统会自动填入 access_token。
|
||||
</div>
|
||||
{session.expiresAt && (
|
||||
<div className="admin-form__help">
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user