mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-15 16:55:42 +08:00
Compare commits
26 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2c5a3342cc | |||
| 7ace5f8bc7 | |||
| 66b33b2a31 | |||
| 27aefc870f | |||
| 02d82e9a62 | |||
| 492431164b | |||
| 051f1555d5 | |||
| 85292ea095 | |||
| fea5e984d1 | |||
| 46bb1fa9f2 | |||
| 7f4407ac28 | |||
| 79b5b5c37d | |||
| f3180f7b3c | |||
| 353b01b8e7 | |||
| d27eae9c62 | |||
| 003efa301b | |||
| f72898f530 | |||
| 641d29e008 | |||
| fed46b51bb | |||
| 304559203c | |||
| 62ccd6a998 | |||
| 720a92af7a | |||
| e33384c786 | |||
| 2e7c761aaf | |||
| 0d6c3c6ac9 | |||
| 10426e5483 |
@@ -0,0 +1,21 @@
|
||||
.git
|
||||
.github
|
||||
.gitattributes
|
||||
.gitignore
|
||||
|
||||
node_modules
|
||||
dist
|
||||
release
|
||||
data
|
||||
backend/data
|
||||
backend/config.yaml
|
||||
config.yaml
|
||||
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
*.log
|
||||
*.tmp
|
||||
|
||||
tests
|
||||
video-site-implementation-plan.md
|
||||
@@ -0,0 +1,68 @@
|
||||
name: Docker
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
tags:
|
||||
- "v*"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract Docker metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=tag
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=sha,prefix=sha-
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
VERSION=${{ startsWith(github.ref, 'refs/tags/v') && github.ref_name || '' }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
@@ -15,6 +15,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
@@ -36,8 +38,11 @@ jobs:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ github.ref_name }}
|
||||
run: |
|
||||
if gh release view "$TAG" >/dev/null 2>&1; then
|
||||
gh release upload "$TAG" release/*.tar.gz --clobber
|
||||
else
|
||||
gh release create "$TAG" release/*.tar.gz --title "$TAG" --notes "Prebuilt Linux release packages."
|
||||
git tag -d "$TAG" >/dev/null 2>&1 || true
|
||||
git fetch --force origin "refs/tags/$TAG:refs/tags/$TAG"
|
||||
NOTES="$(git tag -l "$TAG" --format='%(contents)')"
|
||||
if [ -z "$NOTES" ]; then
|
||||
NOTES="Prebuilt Linux release packages."
|
||||
fi
|
||||
gh release delete "$TAG" --yes >/dev/null 2>&1 || true
|
||||
gh release create "$TAG" release/*.tar.gz --title "$TAG" --notes "$NOTES" --verify-tag
|
||||
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
# ---- Stage 1: Build frontend ----
|
||||
FROM node:20-slim AS frontend
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tsconfig.json vite.config.ts index.html ./
|
||||
COPY public/ public/
|
||||
COPY src/ src/
|
||||
RUN npm run build
|
||||
|
||||
# ---- Stage 2: Build backend ----
|
||||
FROM golang:1.23-bookworm AS backend
|
||||
|
||||
WORKDIR /app/backend
|
||||
|
||||
COPY backend/go.mod backend/go.sum ./
|
||||
COPY backend/vendor/ vendor/
|
||||
COPY backend/cmd/ cmd/
|
||||
COPY backend/internal/ internal/
|
||||
RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server
|
||||
|
||||
# ---- Stage 3: Runtime ----
|
||||
FROM debian:bookworm-slim AS runtime
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
ffmpeg \
|
||||
openssl \
|
||||
python3 \
|
||||
python3-bs4 \
|
||||
python3-lxml \
|
||||
python3-requests \
|
||||
tar \
|
||||
tzdata \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /opt/video-site-91
|
||||
|
||||
COPY --from=backend /out/server ./server
|
||||
COPY --from=frontend /app/dist ./dist
|
||||
COPY backend/config.example.yaml ./config.example.yaml
|
||||
COPY 91VideoSpider/ ./91VideoSpider/
|
||||
COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
ARG VERSION=
|
||||
|
||||
ENV VIDEO_CONFIG=/opt/video-site-91/data/config.yaml \
|
||||
VIDEO_FRONTEND_DIR=/opt/video-site-91/dist \
|
||||
VIDEO_GITHUB_REPO=nianzhibai/91 \
|
||||
VIDEO_IMAGE_VERSION=${VERSION} \
|
||||
VIDEO_LISTEN_PORT=9191 \
|
||||
VIDEO_VERSION_FILE=/opt/video-site-91/data/.version
|
||||
|
||||
RUN chmod +x ./server /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
VOLUME ["/opt/video-site-91/data"]
|
||||
EXPOSE 9191
|
||||
|
||||
ENTRYPOINT ["docker-entrypoint.sh"]
|
||||
CMD ["./server"]
|
||||
@@ -1,24 +1,61 @@
|
||||
## 项目说明
|
||||
个人91站<img width="30" height="30" alt="1f913" src="https://github.com/user-attachments/assets/606c8c18-e727-41fd-9431-5a053e416673" />个人91站
|
||||
<img width="120" height="120" alt="91" src="https://github.com/user-attachments/assets/5b323c94-bbd3-4dce-bbc8-adc86935b7de" />
|
||||
个人91站<img width="30" height="30" alt="1f913" src="https://github.com/user-attachments/assets/606c8c18-e727-41fd-9431-5a053e416673" />个人91站
|
||||
# 91
|
||||
|
||||
支持115云盘,PikPak云盘作为视频播放后端 ▶
|
||||
<p align="center">
|
||||
<img width="120" height="120" alt="91" src="https://github.com/user-attachments/assets/5b323c94-bbd3-4dce-bbc8-adc86935b7de" />
|
||||
</p>
|
||||
|
||||
采用115云盘和PikPak云盘的302重定向,不占用服务器带宽(也不会受服务器带宽小而影响视频播放体验)✨
|
||||
|
||||
服务器只会扫描云盘中的视频文件,给每个视频文件生成封面图和预览片段 📷
|
||||
|
||||
你可以通过封面图和预览片段在网站首页快速选择想看的视频 ✅
|
||||
|
||||
支持91爬虫,爬取91的本月最热视频 🕷
|
||||
<p align="center">
|
||||
😄个人 91 站😄
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
## 项目说明
|
||||
|
||||
支持 115 云盘、PikPak 云盘和服务器本地目录作为视频播放后端。
|
||||
|
||||
采用 115 云盘和 PikPak 云盘的 302 重定向播放,不占用服务器带宽,也不会因为服务器带宽小而影响视频播放体验。
|
||||
|
||||
服务器只负责扫描云盘或本地目录中的视频文件,并给每个视频生成封面图和预览片段。
|
||||
|
||||
你可以通过封面图和预览片段,在首页快速选择想看的视频。
|
||||
|
||||
支持 91 爬虫,爬取 91 的本月最热视频。
|
||||
|
||||
内置两种主题:黑黄主题(91 经典主题)和粉白主题。
|
||||
|
||||
支持短视频模式,一键切换成熟悉的抖音模式。
|
||||
|
||||
该项目2C2G服务器稳定跑👍👍👍
|
||||
|
||||
---
|
||||
|
||||
## 预览图
|
||||
|
||||
### 电脑端
|
||||
|
||||
<p>
|
||||
<img width="49%" alt="91 电脑端首页" src="https://github.com/user-attachments/assets/9808fceb-760b-4dd5-b7d2-8622b95b90d5" />
|
||||
<img width="49%" alt="91 电脑端播放页" src="https://github.com/user-attachments/assets/859db4aa-1fba-44f2-bb46-1db07c2f964f" />
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<img width="49%" alt="91 电脑端主题" src="https://github.com/user-attachments/assets/96bea37a-8764-413e-9b70-1856b4ae0cd2" />
|
||||
<img width="49%" alt="91 电脑端管理页" src="https://github.com/user-attachments/assets/29c1e27a-7651-4dfc-93dd-556331844214" />
|
||||
</p>
|
||||
|
||||
### 手机端
|
||||
|
||||
<p align="center">
|
||||
<img width="1284" height="1134" alt="PixPin_2026-05-29_11-54-12" src="https://github.com/user-attachments/assets/bdb7a86c-a4e5-483e-a307-e02c0bb34dac" />
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
## 快速开始
|
||||
|
||||
一键安装脚本
|
||||
### 一键安装脚本(推荐)
|
||||
|
||||
```bash
|
||||
sudo apt update
|
||||
sudo apt install -y curl ca-certificates
|
||||
@@ -31,7 +68,7 @@ sudo bash install.sh
|
||||
- 前台:`http://服务器IP:9191/`
|
||||
- 后台:`http://服务器IP:9191/admin`
|
||||
|
||||
安装后会自动创建 `91` 指令
|
||||
安装后会自动创建 `91` 指令:
|
||||
|
||||
```bash
|
||||
91 # 打开管理菜单
|
||||
@@ -44,23 +81,96 @@ sudo bash install.sh
|
||||
|
||||
同时也保留 `video-site-91` 作为同等别名。
|
||||
|
||||
**旧版本用户升级说明:**
|
||||
|
||||
如果你是在 `v0.0.2` 之前部署的项目,系统里可能还保留旧的 `91` 管理脚本。旧脚本直接运行 `91 update` 可能更新失败。先执行下面的一次性修复命令,后续再使用 `91 update` 即可:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/install.sh -o /tmp/install-91.sh
|
||||
sudo bash /tmp/install-91.sh update
|
||||
```
|
||||
|
||||
想换端口:
|
||||
|
||||
```bash
|
||||
FRONTEND_PORT=8080 sudo -E bash install.sh
|
||||
```
|
||||
|
||||
### Docker Compose 部署
|
||||
|
||||
准备目录:
|
||||
|
||||
```bash
|
||||
mkdir -p video-site-91
|
||||
cd video-site-91
|
||||
```
|
||||
|
||||
创建 `docker-compose.yml`:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
video-site-91:
|
||||
image: ghcr.io/nianzhibai/91:latest
|
||||
container_name: video-site-91
|
||||
ports:
|
||||
- "9191:9191"
|
||||
volumes:
|
||||
- ./data:/opt/video-site-91/data
|
||||
restart: unless-stopped
|
||||
```
|
||||
|
||||
启动:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
部署完成后访问:
|
||||
|
||||
- 前台:`http://服务器IP:9191/`
|
||||
- 后台:`http://服务器IP:9191/admin`
|
||||
|
||||
所有配置、数据库、封面、预览、上传文件都会保存在当前目录的 `./data` 里。更新时执行:
|
||||
|
||||
```bash
|
||||
docker compose pull
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
查看日志:
|
||||
|
||||
```bash
|
||||
docker compose logs -f
|
||||
```
|
||||
|
||||
如果只想下载仓库内置的 Compose 文件:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/docker-compose.yml -o docker-compose.yml
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 数据存放位置
|
||||
|
||||
项目会把运行数据保存在本地:
|
||||
一键安装脚本会把运行数据保存在宿主机:
|
||||
|
||||
- `/opt/video-site-91/config.yaml`:本地配置、管理员账号、网盘凭证。
|
||||
- `/opt/video-site-91/data/video-site.db`:SQLite 数据库。
|
||||
- `/opt/video-site-91/data/previews/`:本地生成的封面和 teaser。
|
||||
|
||||
Docker Compose 部署会把运行数据保存在当前目录的 `./data/`:
|
||||
|
||||
- `./data/config.yaml`:本地配置、管理员账号、网盘凭证。
|
||||
- `./data/video-site.db`:SQLite 数据库。
|
||||
- `./data/previews/`:本地生成的封面和 teaser。
|
||||
- `./data/uploads/`:本地上传的视频文件。
|
||||
- `./data/spider91/`:91 爬虫本地保存的视频文件。
|
||||
|
||||
---
|
||||
## 了解项目更多细节
|
||||
|
||||
## 了解更多
|
||||
|
||||
根目录 README 只保留项目介绍和最短上手路径。更细的实现、接口、网盘字段和部署方式可以看:
|
||||
|
||||
@@ -73,4 +183,14 @@ FRONTEND_PORT=8080 sudo -E bash install.sh
|
||||
|
||||
这个项目面向个人私有部署。请只接入你有权访问和管理的内容,并遵守对应网盘、站点服务条款以及所在地法律法规。
|
||||
|
||||
不要传播,仅限个人使用,个人视频站
|
||||
不要传播,仅限个人使用,个人视频站。
|
||||
|
||||
---
|
||||
|
||||
## 致谢
|
||||
|
||||
感谢开源项目 OpenList。
|
||||
|
||||
感谢 <a href="https://linux.do/">LinuxDo</a> 社区,学 AI 上 L 站。
|
||||
|
||||
感谢 <a href="https://nodeseek.com/">NodeSeek</a> 社区,MJJ 上 N 站。
|
||||
|
||||
+19
-5
@@ -2,7 +2,7 @@
|
||||
|
||||
视频聚合站的 Go 后端。提供三件事:
|
||||
|
||||
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive)
|
||||
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive / 本地存储)
|
||||
2. 视频元数据目录(SQLite)+ 扫描 + teaser 预生成
|
||||
3. REST API(前台)+ 管理后台 + 直链代理
|
||||
4. 标签池、视频隐藏、按网盘统计和详情页来源网盘类型展示能力
|
||||
@@ -21,6 +21,7 @@ internal/
|
||||
pikpak/ PikPak(自己实现,参考 OpenList pikpak)
|
||||
wopan/ 联通沃盘(壳子 + OpenListTeam/wopan-sdk-go)
|
||||
onedrive/ OneDrive(OpenList 在线续期 + Microsoft Graph 文件接口)
|
||||
localstorage/ 本地目录扫描(服务器已有视频目录)
|
||||
scanner/ 扫目录 → 落库
|
||||
preview/ ffmpeg 抽封面和生成多段 teaser
|
||||
proxy/ /p/stream/*、/p/preview/* 代理
|
||||
@@ -105,9 +106,10 @@ go run ./cmd/server 后端 9192
|
||||
|--------|---------------------------------------------------------------|
|
||||
| quark | `cookie` |
|
||||
| p115 | `cookie`(形如 `UID=...; CID=...; SEID=...; KID=...`) |
|
||||
| pikpak | `username`、`password`,可选 `refresh_token`、`captcha_token`、`device_id`、`platform`、`disable_media_link` |
|
||||
| pikpak | `username`、`password`(token、验证码和设备 ID 由服务端自动处理并保存) |
|
||||
| wopan | `access_token`、`refresh_token`,可选 `family_id` |
|
||||
| onedrive | `refresh_token`,可选 `access_token`、`api_url_address`、`region`、`is_sharepoint`、`site_id` |
|
||||
| onedrive | `refresh_token` |
|
||||
| localstorage | `path`(服务器上的已有视频目录,如 `/mnt/videos`) |
|
||||
|
||||
### PikPak 速度说明
|
||||
|
||||
@@ -115,7 +117,7 @@ go run ./cmd/server 后端 9192
|
||||
|
||||
当前服务器同时存在 sing-box TUN 透明代理,PikPak 默认出站会被 `tun0` 接管;但强制直连物理网卡并没有更快,慢速的主要差异来自 PikPak 取链方式。media/cache CDN 节点仍有波动,偶尔可能遇到慢节点;如果播放变慢,可重新获取直链或重新挂载 PikPak 后再测。
|
||||
|
||||
OneDrive 按 OpenList 默认方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。OpenList 代刷得到的 refresh token 可以直接填到本项目。普通 OneDrive 的 `rootId` / `scanRootId` 可填 `root`;SharePoint 文档库需要额外设置 `is_sharepoint=true` 和 `site_id`。
|
||||
OneDrive 按 OpenList 默认应用方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。后台新建 OneDrive 时只需要填 OpenList 代刷得到的 `refresh_token`;服务端会默认挂载根目录并自动回写新 token。
|
||||
|
||||
## 文件名约定
|
||||
|
||||
@@ -128,6 +130,18 @@ OneDrive 按 OpenList 默认方式调用 `https://api.oplist.org/onedrive/renewa
|
||||
|
||||
标签分隔符支持 `, , 、` 和空格。解析结果会和系统标签池匹配,常见番号类噪声会归并到 `AV` 等系统标签,避免把每个番号都变成独立标签。解析结果可在管理后台覆盖。
|
||||
|
||||
## 视频去重
|
||||
|
||||
项目有三层去重:
|
||||
|
||||
1. 同一网盘同一文件按 `(drive_id, file_id)` 形成稳定视频 ID,重复扫描只更新同一行。
|
||||
2. 扫描时优先按网盘侧 `content_hash` 去重;没有 hash 时退化为 `file_name + size_bytes`。
|
||||
3. 扫描、爬虫、本地上传或服务启动挂载网盘后,后台指纹 worker 会异步读取视频的少量 Range 片段,生成 `sampled_sha256`。前台列表、首页、搜索、推荐会按 `size_bytes + sampled_sha256` 只展示最早入库的 canonical 视频。
|
||||
|
||||
`sampled_sha256` 是文件级去重:适合识别同一个视频文件被复制到 115 / PikPak / OneDrive 等不同网盘的情况。它不会删除任何网盘文件,也不用于识别转码、裁剪、加水印后的同源视频。
|
||||
|
||||
封面和 teaser 仍然优先生成,不等待指纹完成。夜间流水线最后会做一次重复资产清理:对 `size_bytes + sampled_sha256` 命中的非 canonical 视频,只删除本机生成的重复封面和 teaser,并把对应字段重置为 `pending`。网盘原文件和视频元数据记录不会被删除;如果 canonical 视频以后被移除,这些重复项会重新进入生成队列。
|
||||
|
||||
## 管理能力
|
||||
|
||||
- `/admin/drives`:新增、编辑、删除网盘,触发扫描。
|
||||
@@ -147,7 +161,7 @@ ffmpeg -ss <起点> -headers "UA/Cookie/Referer" -i <直链> \
|
||||
|
||||
当前策略是每段固定 3 秒;30 秒以下最多 3 段,30 秒及以上固定 4 段;长视频在 20% 到 80% 区间均匀取段。生成的 teaser 和封面都只保存在本地 `data/previews/`,不会回写到网盘;旧数据中的 `preview_file_id` 会被忽略。
|
||||
|
||||
服务启动或网盘重新挂载时,如果 Teaser 开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 直链生成 teaser 时可能触发 Microsoft 429 限流;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。
|
||||
服务启动或网盘重新挂载时,如果 Teaser 开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 扫盘和直链生成 teaser / 封面时可能触发 Microsoft Graph 429、`TooManyRequests`、`activityLimitReached` 或 throttled 文本;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。扫盘阶段会按 `Retry-After` 或默认冷却时间等待后继续当前目录。
|
||||
|
||||
前端卡片的 `previewSrc` 统一指向 `/p/preview/<videoID>`,后端只从本地 `preview_local` 文件读取。
|
||||
|
||||
|
||||
+421
-123
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/config"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/localstorage"
|
||||
"github.com/video-site/backend/internal/drives/localupload"
|
||||
"github.com/video-site/backend/internal/drives/onedrive"
|
||||
"github.com/video-site/backend/internal/drives/p115"
|
||||
@@ -32,6 +33,7 @@ import (
|
||||
"github.com/video-site/backend/internal/drives/quark"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/drives/wopan"
|
||||
"github.com/video-site/backend/internal/fingerprint"
|
||||
"github.com/video-site/backend/internal/nightly"
|
||||
"github.com/video-site/backend/internal/preview"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
@@ -39,6 +41,8 @@ import (
|
||||
"github.com/video-site/backend/internal/spider91migrate"
|
||||
)
|
||||
|
||||
const fingerprintReconcileInterval = time.Minute
|
||||
|
||||
func main() {
|
||||
cfgPath := "./config.yaml"
|
||||
if v := os.Getenv("VIDEO_CONFIG"); v != "" {
|
||||
@@ -63,12 +67,13 @@ func main() {
|
||||
defer cat.Close()
|
||||
|
||||
app := &App{
|
||||
cfg: cfg,
|
||||
cat: cat,
|
||||
registry: proxy.NewRegistry(),
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
spider91Crawlers: make(map[string]*spider91.Crawler),
|
||||
cfg: cfg,
|
||||
cat: cat,
|
||||
registry: proxy.NewRegistry(),
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
fingerprintWorkers: make(map[string]*fingerprint.Worker),
|
||||
spider91Crawlers: make(map[string]*spider91.Crawler),
|
||||
}
|
||||
app.proxy = proxy.New(app.registry)
|
||||
app.spider91Migrator = spider91migrate.New(spider91migrate.Config{
|
||||
@@ -77,7 +82,8 @@ func main() {
|
||||
GetTargetDriveID: func() string { return app.Spider91UploadDriveID() },
|
||||
})
|
||||
|
||||
// 初始化现有 drives
|
||||
// 初始化本地内置盘;外部云盘放到 HTTP 服务启动后异步挂载,避免上游
|
||||
// 登录态校验拖慢端口监听。
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
@@ -86,16 +92,7 @@ func main() {
|
||||
if err := app.attachLocalUpload(ctx); err != nil {
|
||||
log.Printf("[local-upload] attach failed: %v", err)
|
||||
}
|
||||
|
||||
existing, err := cat.ListDrives(ctx)
|
||||
if err != nil {
|
||||
log.Fatalf("list drives: %v", err)
|
||||
}
|
||||
for _, d := range existing {
|
||||
if err := app.attachDrive(ctx, d); err != nil {
|
||||
log.Printf("[drive %s] attach failed: %v", d.ID, err)
|
||||
}
|
||||
}
|
||||
go app.runFingerprintReconciler(ctx)
|
||||
|
||||
authr := &auth.Authenticator{
|
||||
Username: cfg.Server.Admin.Username,
|
||||
@@ -104,6 +101,14 @@ func main() {
|
||||
}
|
||||
setupRequired := config.RequiresAdminSetup(cfg)
|
||||
var setupMu sync.Mutex
|
||||
versionFilePath := strings.TrimSpace(os.Getenv("VIDEO_VERSION_FILE"))
|
||||
if versionFilePath == "" {
|
||||
versionFilePath = filepath.Join(filepath.Dir(cfgPath), ".version")
|
||||
}
|
||||
githubRepo := strings.TrimSpace(os.Getenv("VIDEO_GITHUB_REPO"))
|
||||
if githubRepo == "" {
|
||||
githubRepo = strings.TrimSpace(os.Getenv("GITHUB_REPO"))
|
||||
}
|
||||
|
||||
apiServer := &api.Server{
|
||||
Catalog: cat,
|
||||
@@ -117,8 +122,10 @@ func main() {
|
||||
}
|
||||
|
||||
adminServer := &api.AdminServer{
|
||||
Catalog: cat,
|
||||
Auth: authr,
|
||||
Catalog: cat,
|
||||
Auth: authr,
|
||||
VersionFilePath: versionFilePath,
|
||||
GitHubRepo: githubRepo,
|
||||
SetupRequired: func() bool {
|
||||
setupMu.Lock()
|
||||
defer setupMu.Unlock()
|
||||
@@ -230,6 +237,7 @@ func main() {
|
||||
RunSpider91Crawl: app.runSpider91Crawl,
|
||||
WaitPreviewQueuesIdle: app.waitAllPreviewQueuesIdle,
|
||||
RunMigration: app.spider91Migrator.RunOnce,
|
||||
RunDedupeAssetCleanup: app.cleanupDuplicateVideoAssets,
|
||||
})
|
||||
go app.nightlyRunner.Run(ctx)
|
||||
|
||||
@@ -243,6 +251,7 @@ func main() {
|
||||
log.Fatalf("server error: %v", err)
|
||||
}
|
||||
}()
|
||||
go app.attachExistingDrives(ctx)
|
||||
|
||||
// 等待退出信号
|
||||
sigs := make(chan os.Signal, 1)
|
||||
@@ -262,20 +271,25 @@ type App struct {
|
||||
registry *proxy.Registry
|
||||
proxy *proxy.Proxy
|
||||
|
||||
mu sync.Mutex
|
||||
workers map[string]*preview.Worker
|
||||
thumbWorkers map[string]*preview.ThumbWorker
|
||||
cancels map[string]context.CancelFunc
|
||||
mu sync.Mutex
|
||||
workers map[string]*preview.Worker
|
||||
thumbWorkers map[string]*preview.ThumbWorker
|
||||
fingerprintWorkers map[string]*fingerprint.Worker
|
||||
cancels map[string]context.CancelFunc
|
||||
// spider91Crawlers 按 driveID 索引,每个 spider91 drive 独立一个 Crawler
|
||||
spider91Crawlers map[string]*spider91.Crawler
|
||||
|
||||
// driveAttachMu 串行化云盘挂载/重挂载。挂载会访问上游服务,可能较慢;
|
||||
// 串行化可以避免启动后台挂载和手动扫盘按需挂载同一个 drive 时重复创建 worker。
|
||||
driveAttachMu sync.Mutex
|
||||
|
||||
// 全站主题("dark" | "pink"),从 DB 读
|
||||
theme string
|
||||
// 显式指定的 spider91 上传目标 drive ID;
|
||||
// 未设置时由 Spider91UploadDriveID() 在所有 pikpak/p115 drive 中自动挑选唯一一个。
|
||||
// 显式指定的 spider91 上传目标 drive ID。
|
||||
// 空字符串表示本地保存不上传,不再自动挑选 pikpak/p115/onedrive drive。
|
||||
spider91UploadDriveID string
|
||||
|
||||
// spider91Migrator 周期把 spider91 视频上传到目标 drive(PikPak 或 115)。
|
||||
// spider91Migrator 周期把 spider91 视频上传到目标 drive(PikPak、115 或 OneDrive)。
|
||||
spider91Migrator *spider91migrate.Migrator
|
||||
|
||||
// nightlyRunner 是凌晨流水线调度器:每天 cron_hour 串行跑扫盘 → 91 爬虫 → 迁移。
|
||||
@@ -297,6 +311,11 @@ type App struct {
|
||||
// scanQueued 跟踪哪些 driveID 已经排队或正在跑,去重后续重复点击。
|
||||
// 一个 drive 在 scheduleScan 入队时被加入,在 runScan goroutine 结束时被移除。
|
||||
scanQueued map[string]bool
|
||||
|
||||
// fingerprintQueueing 去重每个 drive 的 pending 指纹补队列任务,避免定时
|
||||
// reconcile 和扫盘结束同时为同一批 pending 视频启动多个长时间入队 goroutine。
|
||||
fingerprintQueueMu sync.Mutex
|
||||
fingerprintQueueing map[string]bool
|
||||
}
|
||||
|
||||
// teaserEnabledForDrive 查询某个 drive 当前的 per-drive teaser 开关。
|
||||
@@ -360,43 +379,25 @@ func (a *App) loadTheme(ctx context.Context) {
|
||||
a.mu.Unlock()
|
||||
}
|
||||
|
||||
// Spider91UploadDriveID 返回当前生效的 spider91 上传目标 drive ID。
|
||||
//
|
||||
// 解析顺序:
|
||||
// 1. 管理员通过 PUT /admin/api/settings 显式设置过 → 验证该 drive 仍存在且是
|
||||
// 合法目标盘(pikpak 或 p115)→ 返回该 ID。
|
||||
// 2. 否则系统中如果只有一个合法目标盘(即 pikpak drive 数量+p115 drive 数量==1),
|
||||
// 自动返回它。这样单网盘场景"开箱即用"。
|
||||
// 3. 多个候选并存时返回空串:迁移 worker 静默跳过,等管理员显式指定。
|
||||
//
|
||||
// 注意"合法目标盘"目前是 pikpak ∪ p115。后续添加新的可上传盘要在两个分支同步加。
|
||||
// Spider91UploadDriveID 返回当前配置的 spider91 上传目标 drive ID。
|
||||
// 空字符串表示本地保存不上传;只有管理员显式选择 pikpak/p115/onedrive drive 时才迁移上传。
|
||||
func (a *App) Spider91UploadDriveID() string {
|
||||
a.mu.Lock()
|
||||
explicit := a.spider91UploadDriveID
|
||||
a.mu.Unlock()
|
||||
if explicit != "" {
|
||||
// 验证显式设置的 drive 仍然存在且 kind 合法;不在则降级到自动选取
|
||||
if d, ok := a.registry.Get(explicit); ok && isSpider91UploadKind(d.Kind()) {
|
||||
return explicit
|
||||
}
|
||||
if explicit == "" {
|
||||
return ""
|
||||
}
|
||||
var found string
|
||||
for _, d := range a.registry.All() {
|
||||
if !isSpider91UploadKind(d.Kind()) {
|
||||
continue
|
||||
}
|
||||
if found != "" {
|
||||
// 多个候选 drive 时不自动选;管理员必须显式指定
|
||||
return ""
|
||||
}
|
||||
found = d.ID()
|
||||
// 验证显式设置的 drive 仍然存在且 kind 合法;不在则视为未配置。
|
||||
if d, ok := a.registry.Get(explicit); ok && isSpider91UploadKind(d.Kind()) {
|
||||
return explicit
|
||||
}
|
||||
return found
|
||||
return ""
|
||||
}
|
||||
|
||||
// SetSpider91UploadDriveID 设置 spider91 上传目标 drive ID 并持久化。
|
||||
// 接受空字符串(清除显式设置,回退到自动模式)。
|
||||
// 设置一个不存在或 kind 不是 pikpak / p115 的 drive 会返回错误。
|
||||
// 接受空字符串(本地保存不上传)。
|
||||
// 设置一个不存在或 kind 不是 pikpak / p115 / onedrive 的 drive 会返回错误。
|
||||
func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) error {
|
||||
driveID = strings.TrimSpace(driveID)
|
||||
if driveID != "" {
|
||||
@@ -405,7 +406,7 @@ func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) erro
|
||||
return fmt.Errorf("drive %q not found", driveID)
|
||||
}
|
||||
if !isSpider91UploadKind(d.Kind()) {
|
||||
return fmt.Errorf("drive %q kind=%s, only pikpak or p115 can be spider91 upload target", driveID, d.Kind())
|
||||
return fmt.Errorf("drive %q kind=%s, only pikpak, p115 or onedrive can be spider91 upload target", driveID, d.Kind())
|
||||
}
|
||||
}
|
||||
a.mu.Lock()
|
||||
@@ -417,7 +418,7 @@ func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) erro
|
||||
// isSpider91UploadKind 是 spider91 迁移目标盘的 allowlist。
|
||||
// 与 spider91migrate.adaptUploadTarget 的支持范围保持一致。
|
||||
func isSpider91UploadKind(kind string) bool {
|
||||
return kind == "pikpak" || kind == "p115"
|
||||
return kind == "pikpak" || kind == "p115" || kind == "onedrive"
|
||||
}
|
||||
|
||||
// loadSpider91UploadDriveID 从 DB 读上传目标 drive ID 设置;不存在时使用空串。
|
||||
@@ -442,9 +443,13 @@ func (a *App) driveGenerationStatuses() map[string]api.DriveGenerationStatuses {
|
||||
for id, worker := range a.thumbWorkers {
|
||||
thumbWorkers[id] = worker
|
||||
}
|
||||
fingerprintWorkers := make(map[string]*fingerprint.Worker, len(a.fingerprintWorkers))
|
||||
for id, worker := range a.fingerprintWorkers {
|
||||
fingerprintWorkers[id] = worker
|
||||
}
|
||||
a.mu.Unlock()
|
||||
|
||||
out := make(map[string]api.DriveGenerationStatuses, len(previewWorkers)+len(thumbWorkers))
|
||||
out := make(map[string]api.DriveGenerationStatuses, len(previewWorkers)+len(thumbWorkers)+len(fingerprintWorkers))
|
||||
for id, worker := range previewWorkers {
|
||||
status := out[id]
|
||||
status.Preview = generationStatusFromPreview(worker.Status())
|
||||
@@ -455,7 +460,7 @@ func (a *App) driveGenerationStatuses() map[string]api.DriveGenerationStatuses {
|
||||
status.Thumbnail = generationStatusFromPreview(worker.Status())
|
||||
missing, err := a.cat.CountVideosNeedingThumbnail(context.Background(), id)
|
||||
if err != nil {
|
||||
log.Printf("[thumb] count missing thumbnails %s: %v", id, err)
|
||||
log.Printf("[thumb] count thumbnail work %s: %v", id, err)
|
||||
} else {
|
||||
status.Thumbnail.QueueLength = missing
|
||||
if missing > 0 && status.Thumbnail.State == "idle" {
|
||||
@@ -464,6 +469,20 @@ func (a *App) driveGenerationStatuses() map[string]api.DriveGenerationStatuses {
|
||||
}
|
||||
out[id] = status
|
||||
}
|
||||
for id, worker := range fingerprintWorkers {
|
||||
status := out[id]
|
||||
status.Fingerprint = generationStatusFromFingerprint(worker.Status())
|
||||
pending, err := a.cat.CountVideosNeedingFingerprint(context.Background(), id)
|
||||
if err != nil {
|
||||
log.Printf("[fingerprint] count pending fingerprints %s: %v", id, err)
|
||||
} else {
|
||||
status.Fingerprint.QueueLength = pending
|
||||
if pending > 0 && status.Fingerprint.State == "idle" {
|
||||
status.Fingerprint.State = "queued"
|
||||
}
|
||||
}
|
||||
out[id] = status
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -483,7 +502,67 @@ func generationStatusFromPreview(status preview.TaskStatus) api.GenerationStatus
|
||||
return out
|
||||
}
|
||||
|
||||
func generationStatusFromFingerprint(status fingerprint.TaskStatus) api.GenerationStatus {
|
||||
state := status.State
|
||||
if state == "" {
|
||||
state = "idle"
|
||||
}
|
||||
out := api.GenerationStatus{
|
||||
State: state,
|
||||
CurrentTitle: status.CurrentTitle,
|
||||
QueueLength: status.QueueLength,
|
||||
}
|
||||
if !status.CooldownUntil.IsZero() {
|
||||
out.CooldownUntil = status.CooldownUntil.Format(time.RFC3339)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *App) attachDrive(ctx context.Context, d *catalog.Drive) error {
|
||||
a.driveAttachMu.Lock()
|
||||
defer a.driveAttachMu.Unlock()
|
||||
return a.attachDriveUnlocked(ctx, d)
|
||||
}
|
||||
|
||||
func (a *App) ensureDriveAttached(ctx context.Context, driveID string) error {
|
||||
if _, ok := a.registry.Get(driveID); ok {
|
||||
return nil
|
||||
}
|
||||
a.driveAttachMu.Lock()
|
||||
defer a.driveAttachMu.Unlock()
|
||||
if _, ok := a.registry.Get(driveID); ok {
|
||||
return nil
|
||||
}
|
||||
d, err := a.cat.GetDrive(ctx, driveID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return a.attachDriveUnlocked(ctx, d)
|
||||
}
|
||||
|
||||
func (a *App) attachExistingDrives(ctx context.Context) {
|
||||
existing, err := a.cat.ListDrives(ctx)
|
||||
if err != nil {
|
||||
log.Printf("[drive] list existing drives: %v", err)
|
||||
return
|
||||
}
|
||||
log.Printf("[drive] attaching %d configured drive(s) in background", len(existing))
|
||||
for _, d := range existing {
|
||||
if err := ctx.Err(); err != nil {
|
||||
log.Printf("[drive] background attach stopped: %v", err)
|
||||
return
|
||||
}
|
||||
if err := a.attachDrive(ctx, d); err != nil {
|
||||
log.Printf("[drive %s] attach failed: %v", d.ID, err)
|
||||
}
|
||||
}
|
||||
log.Printf("[drive] background attach complete")
|
||||
}
|
||||
|
||||
func (a *App) attachDriveUnlocked(ctx context.Context, d *catalog.Drive) error {
|
||||
if d == nil {
|
||||
return errors.New("nil drive")
|
||||
}
|
||||
var drv drives.Drive
|
||||
switch d.Kind {
|
||||
case "quark":
|
||||
@@ -554,6 +633,11 @@ func (a *App) attachDrive(ctx context.Context, d *catalog.Drive) error {
|
||||
_ = a.cat.UpsertDrive(ctx, d)
|
||||
},
|
||||
})
|
||||
case localstorage.Kind:
|
||||
drv = localstorage.New(localstorage.Config{
|
||||
ID: d.ID,
|
||||
RootPath: d.Credentials["path"],
|
||||
})
|
||||
case spider91.Kind:
|
||||
drv = spider91.New(spider91.Config{
|
||||
ID: d.ID,
|
||||
@@ -587,12 +671,14 @@ func (a *App) attachDrive(ctx context.Context, d *catalog.Drive) error {
|
||||
})
|
||||
worker := preview.NewWorker(gen, a.cat, drv)
|
||||
thumbWorker := preview.NewThumbWorker(gen, a.cat, drv)
|
||||
fingerprintWorker := fingerprint.NewWorker(a.cat, drv, fingerprintConfigForDrive(drv))
|
||||
|
||||
workerCtx, cancel := context.WithCancel(ctx)
|
||||
go worker.Run(workerCtx)
|
||||
go thumbWorker.Run(workerCtx)
|
||||
go fingerprintWorker.Run(workerCtx)
|
||||
|
||||
a.registerPreviewWorkers(ctx, d.ID, worker, thumbWorker, cancel)
|
||||
a.registerPreviewWorkers(ctx, d.ID, worker, thumbWorker, fingerprintWorker, cancel)
|
||||
|
||||
// spider91 driver 还需要一个 crawler,挂在专用 map 里供 crawlerLoop 调用
|
||||
if sd, ok := drv.(*spider91.Driver); ok {
|
||||
@@ -619,12 +705,14 @@ func (a *App) attachLocalUpload(ctx context.Context) error {
|
||||
})
|
||||
worker := preview.NewWorker(gen, a.cat, drv)
|
||||
thumbWorker := preview.NewThumbWorker(gen, a.cat, drv)
|
||||
fingerprintWorker := fingerprint.NewWorker(a.cat, drv, fingerprintConfigForDrive(drv))
|
||||
|
||||
workerCtx, cancel := context.WithCancel(ctx)
|
||||
go worker.Run(workerCtx)
|
||||
go thumbWorker.Run(workerCtx)
|
||||
go fingerprintWorker.Run(workerCtx)
|
||||
|
||||
a.registerPreviewWorkers(ctx, drv.ID(), worker, thumbWorker, cancel)
|
||||
a.registerPreviewWorkers(ctx, drv.ID(), worker, thumbWorker, fingerprintWorker, cancel)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -632,6 +720,20 @@ func (a *App) localUploadDir() string {
|
||||
return filepath.Join(filepath.Dir(a.cfg.Storage.LocalPreviewDir), "uploads")
|
||||
}
|
||||
|
||||
func fingerprintConfigForDrive(drv drives.Drive) fingerprint.Config {
|
||||
cfg := fingerprint.Config{RateLimitCooldown: 5 * time.Minute}
|
||||
if drv == nil {
|
||||
return cfg
|
||||
}
|
||||
switch strings.ToLower(drv.Kind()) {
|
||||
case "p115", "onedrive":
|
||||
cfg.RateLimitCooldown = 10 * time.Minute
|
||||
case "pikpak":
|
||||
cfg.RateLimitCooldown = 5 * time.Minute
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// spider91RootDir 是所有 spider91 drive 共享的根目录。
|
||||
func (a *App) spider91RootDir() string {
|
||||
return filepath.Join(filepath.Dir(a.cfg.Storage.LocalPreviewDir), "spider91")
|
||||
@@ -716,7 +818,7 @@ func (a *App) attachSpider91Crawler(d *catalog.Drive, drv *spider91.Driver) {
|
||||
}()
|
||||
}
|
||||
|
||||
func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker *preview.Worker, thumbWorker *preview.ThumbWorker, cancel context.CancelFunc) {
|
||||
func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker *preview.Worker, thumbWorker *preview.ThumbWorker, fingerprintWorker *fingerprint.Worker, cancel context.CancelFunc) {
|
||||
a.mu.Lock()
|
||||
if a.cancels == nil {
|
||||
a.cancels = make(map[string]context.CancelFunc)
|
||||
@@ -727,6 +829,9 @@ func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker
|
||||
if a.thumbWorkers == nil {
|
||||
a.thumbWorkers = make(map[string]*preview.ThumbWorker)
|
||||
}
|
||||
if a.fingerprintWorkers == nil {
|
||||
a.fingerprintWorkers = make(map[string]*fingerprint.Worker)
|
||||
}
|
||||
if old, ok := a.cancels[driveID]; ok && old != nil {
|
||||
old()
|
||||
}
|
||||
@@ -740,6 +845,11 @@ func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker
|
||||
} else {
|
||||
delete(a.thumbWorkers, driveID)
|
||||
}
|
||||
if fingerprintWorker != nil {
|
||||
a.fingerprintWorkers[driveID] = fingerprintWorker
|
||||
} else {
|
||||
delete(a.fingerprintWorkers, driveID)
|
||||
}
|
||||
if cancel != nil {
|
||||
a.cancels[driveID] = cancel
|
||||
} else {
|
||||
@@ -747,17 +857,10 @@ func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker
|
||||
}
|
||||
a.mu.Unlock()
|
||||
|
||||
if worker != nil {
|
||||
if thumbWorker != nil {
|
||||
worker.BeforeTask = func(taskCtx context.Context) bool {
|
||||
return a.waitForThumbnailsBeforePreview(taskCtx, driveID)
|
||||
}
|
||||
} else {
|
||||
worker.BeforeTask = nil
|
||||
}
|
||||
}
|
||||
|
||||
go a.enqueueDriveGeneration(ctx, driveID, worker, thumbWorker)
|
||||
if fingerprintWorker != nil {
|
||||
a.scheduleFingerprintBackfill(ctx, driveID, fingerprintWorker)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) enqueuePending(ctx context.Context, driveID string, w *preview.Worker) {
|
||||
@@ -781,45 +884,16 @@ func (a *App) enqueuePending(ctx context.Context, driveID string, w *preview.Wor
|
||||
func (a *App) enqueueDriveGeneration(ctx context.Context, driveID string, worker *preview.Worker, thumbWorker *preview.ThumbWorker) {
|
||||
// 封面 worker 始终入队(与早期"全局 preview.enabled=false 时仍然生成封面"
|
||||
// 的行为一致);teaser worker 仅在该 drive 的 TeaserEnabled 为 true 时入队。
|
||||
// 两条队列互不等待,避免封面批量生成拖住预览视频生成。
|
||||
if thumbWorker != nil {
|
||||
a.enqueueThumbnails(ctx, driveID, thumbWorker)
|
||||
}
|
||||
if worker == nil || !a.teaserEnabledForDrive(ctx, driveID) {
|
||||
return
|
||||
}
|
||||
if thumbWorker != nil && !a.waitForThumbnailsBeforePreview(ctx, driveID) {
|
||||
return
|
||||
}
|
||||
a.enqueuePending(ctx, driveID, worker)
|
||||
}
|
||||
|
||||
func (a *App) waitForThumbnailsBeforePreview(ctx context.Context, driveID string) bool {
|
||||
const pollInterval = time.Second
|
||||
var lastLog time.Time
|
||||
for {
|
||||
missing, err := a.cat.CountVideosNeedingThumbnail(ctx, driveID)
|
||||
if err != nil {
|
||||
log.Printf("[preview] count missing thumbnails drive=%s: %v", driveID, err)
|
||||
return false
|
||||
}
|
||||
if missing == 0 {
|
||||
return true
|
||||
}
|
||||
now := time.Now()
|
||||
if lastLog.IsZero() || now.Sub(lastLog) >= time.Minute {
|
||||
log.Printf("[preview] drive=%s waiting for %d thumbnails before teaser generation", driveID, missing)
|
||||
lastLog = now
|
||||
}
|
||||
timer := time.NewTimer(pollInterval)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return false
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) enqueueThumbnails(ctx context.Context, driveID string, w *preview.ThumbWorker) {
|
||||
pending, err := a.cat.ListVideosNeedingThumbnail(ctx, driveID, 0)
|
||||
if err != nil {
|
||||
@@ -829,10 +903,81 @@ func (a *App) enqueueThumbnails(ctx context.Context, driveID string, w *preview.
|
||||
if len(pending) == 0 {
|
||||
return
|
||||
}
|
||||
log.Printf("[thumb] enqueue %d missing thumbnails for drive=%s", len(pending), driveID)
|
||||
log.Printf("[thumb] enqueue %d thumbnail/duration tasks for drive=%s", len(pending), driveID)
|
||||
for _, v := range pending {
|
||||
if !w.EnqueueBlocking(ctx, v) {
|
||||
log.Printf("[thumb] enqueue missing thumbnails canceled for drive=%s", driveID)
|
||||
log.Printf("[thumb] enqueue thumbnail/duration tasks canceled for drive=%s", driveID)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) runFingerprintReconciler(ctx context.Context) {
|
||||
ticker := time.NewTicker(fingerprintReconcileInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
a.enqueueAllPendingFingerprints(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) enqueueAllPendingFingerprints(ctx context.Context) {
|
||||
a.mu.Lock()
|
||||
workers := make(map[string]*fingerprint.Worker, len(a.fingerprintWorkers))
|
||||
for id, worker := range a.fingerprintWorkers {
|
||||
workers[id] = worker
|
||||
}
|
||||
a.mu.Unlock()
|
||||
for driveID, worker := range workers {
|
||||
a.scheduleFingerprintBackfill(ctx, driveID, worker)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) scheduleFingerprintBackfill(ctx context.Context, driveID string, w *fingerprint.Worker) {
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
a.fingerprintQueueMu.Lock()
|
||||
if a.fingerprintQueueing == nil {
|
||||
a.fingerprintQueueing = make(map[string]bool)
|
||||
}
|
||||
if a.fingerprintQueueing[driveID] {
|
||||
a.fingerprintQueueMu.Unlock()
|
||||
return
|
||||
}
|
||||
a.fingerprintQueueing[driveID] = true
|
||||
a.fingerprintQueueMu.Unlock()
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
a.fingerprintQueueMu.Lock()
|
||||
delete(a.fingerprintQueueing, driveID)
|
||||
a.fingerprintQueueMu.Unlock()
|
||||
}()
|
||||
a.enqueueFingerprints(ctx, driveID, w)
|
||||
}()
|
||||
}
|
||||
|
||||
func (a *App) enqueueFingerprints(ctx context.Context, driveID string, w *fingerprint.Worker) {
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
pending, err := a.cat.ListVideosNeedingFingerprint(ctx, driveID, 0)
|
||||
if err != nil {
|
||||
log.Printf("[fingerprint] list pending %s: %v", driveID, err)
|
||||
return
|
||||
}
|
||||
if len(pending) == 0 {
|
||||
return
|
||||
}
|
||||
log.Printf("[fingerprint] enqueue %d videos for drive=%s", len(pending), driveID)
|
||||
for _, v := range pending {
|
||||
if !w.EnqueueBlocking(ctx, v) {
|
||||
log.Printf("[fingerprint] enqueue canceled for drive=%s", driveID)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -847,6 +992,7 @@ func (a *App) detachDrive(id string) {
|
||||
}
|
||||
delete(a.workers, id)
|
||||
delete(a.thumbWorkers, id)
|
||||
delete(a.fingerprintWorkers, id)
|
||||
delete(a.spider91Crawlers, id)
|
||||
a.mu.Unlock()
|
||||
}
|
||||
@@ -941,6 +1087,10 @@ func (a *App) runScan(ctx context.Context, driveID string) {
|
||||
a.scanGlobalMu.Lock()
|
||||
defer a.scanGlobalMu.Unlock()
|
||||
|
||||
if err := a.ensureDriveAttached(ctx, driveID); err != nil {
|
||||
log.Printf("[scan] drive %s attach failed: %v", driveID, err)
|
||||
return
|
||||
}
|
||||
drv, ok := a.registry.Get(driveID)
|
||||
if !ok {
|
||||
log.Printf("[scan] drive %s not attached", driveID)
|
||||
@@ -950,14 +1100,15 @@ func (a *App) runScan(ctx context.Context, driveID string) {
|
||||
a.mu.Lock()
|
||||
worker := a.workers[driveID]
|
||||
thumbWorker := a.thumbWorkers[driveID]
|
||||
fingerprintWorker := a.fingerprintWorkers[driveID]
|
||||
a.mu.Unlock()
|
||||
|
||||
var onNew func(v *catalog.Video)
|
||||
if thumbWorker != nil {
|
||||
onNew = func(v *catalog.Video) {
|
||||
if thumbWorker != nil && v.ThumbnailURL == "" {
|
||||
thumbWorker.Enqueue(v)
|
||||
}
|
||||
onNew := func(v *catalog.Video) {
|
||||
if thumbWorker != nil && v.ThumbnailURL == "" {
|
||||
thumbWorker.Enqueue(v)
|
||||
}
|
||||
if fingerprintWorker != nil {
|
||||
fingerprintWorker.Enqueue(v)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1001,6 +1152,7 @@ func (a *App) runScan(ctx context.Context, driveID string) {
|
||||
}
|
||||
}
|
||||
}
|
||||
a.scheduleFingerprintBackfill(ctx, driveID, fingerprintWorker)
|
||||
a.enqueueDriveGeneration(ctx, driveID, worker, thumbWorker)
|
||||
}
|
||||
|
||||
@@ -1071,6 +1223,126 @@ func removeLocalVideoAssets(localDir string, v *catalog.Video) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type duplicateAssetCleanupStats struct {
|
||||
Candidates int
|
||||
VideosUpdated int
|
||||
PreviewFiles int
|
||||
ThumbnailFiles int
|
||||
MissingFiles int
|
||||
SkippedUnsafeRef int
|
||||
}
|
||||
|
||||
func (a *App) cleanupDuplicateVideoAssets(ctx context.Context) error {
|
||||
if a == nil || a.cat == nil {
|
||||
return nil
|
||||
}
|
||||
localDir := ""
|
||||
if a.cfg != nil {
|
||||
localDir = a.cfg.Storage.LocalPreviewDir
|
||||
}
|
||||
if strings.TrimSpace(localDir) == "" {
|
||||
return nil
|
||||
}
|
||||
items, err := a.cat.ListDuplicateAssetCleanupCandidates(ctx, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(items) == 0 {
|
||||
log.Printf("[dedupe-cleanup] no duplicate local assets to clean")
|
||||
return nil
|
||||
}
|
||||
|
||||
stats := duplicateAssetCleanupStats{Candidates: len(items)}
|
||||
for _, item := range items {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
clearPreview, removedPreview, missingPreview, skippedPreview, err := cleanupDuplicatePreviewAsset(localDir, item.PreviewLocal)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cleanup duplicate preview video=%s canonical=%s: %w", item.VideoID, item.CanonicalID, err)
|
||||
}
|
||||
clearThumb, removedThumb, missingThumb, err := cleanupDuplicateThumbnailAsset(localDir, item.VideoID, item.ThumbnailURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cleanup duplicate thumbnail video=%s canonical=%s: %w", item.VideoID, item.CanonicalID, err)
|
||||
}
|
||||
if skippedPreview {
|
||||
stats.SkippedUnsafeRef++
|
||||
}
|
||||
if removedPreview {
|
||||
stats.PreviewFiles++
|
||||
}
|
||||
if removedThumb {
|
||||
stats.ThumbnailFiles++
|
||||
}
|
||||
if missingPreview {
|
||||
stats.MissingFiles++
|
||||
}
|
||||
if missingThumb {
|
||||
stats.MissingFiles++
|
||||
}
|
||||
if !clearPreview && !clearThumb {
|
||||
continue
|
||||
}
|
||||
if err := a.cat.ClearGeneratedAssets(ctx, item.VideoID, clearPreview, clearThumb); err != nil {
|
||||
return fmt.Errorf("mark duplicate assets cleaned video=%s canonical=%s: %w", item.VideoID, item.CanonicalID, err)
|
||||
}
|
||||
stats.VideosUpdated++
|
||||
}
|
||||
log.Printf("[dedupe-cleanup] candidates=%d updated=%d preview_files=%d thumbnail_files=%d missing=%d skipped_unsafe_refs=%d",
|
||||
stats.Candidates, stats.VideosUpdated, stats.PreviewFiles, stats.ThumbnailFiles, stats.MissingFiles, stats.SkippedUnsafeRef)
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanupDuplicatePreviewAsset(localDir, previewLocal string) (clear bool, removed bool, missing bool, skippedUnsafe bool, err error) {
|
||||
clean, ok := localPathWithin(localDir, previewLocal)
|
||||
if !ok {
|
||||
if strings.TrimSpace(previewLocal) != "" {
|
||||
return false, false, false, true, nil
|
||||
}
|
||||
return false, false, false, false, nil
|
||||
}
|
||||
removed, missing, err = removeRegularFileIfExists(clean)
|
||||
if err != nil {
|
||||
return false, false, false, false, err
|
||||
}
|
||||
return true, removed, missing, false, nil
|
||||
}
|
||||
|
||||
func cleanupDuplicateThumbnailAsset(localDir, videoID, thumbnailURL string) (clear bool, removed bool, missing bool, err error) {
|
||||
if thumbnailURL != "/p/thumb/"+videoID {
|
||||
return false, false, false, nil
|
||||
}
|
||||
clean, ok := localPathWithin(localDir, filepath.Join(localDir, "thumbs", videoID+".jpg"))
|
||||
if !ok {
|
||||
return false, false, false, nil
|
||||
}
|
||||
removed, missing, err = removeRegularFileIfExists(clean)
|
||||
if err != nil {
|
||||
return false, false, false, err
|
||||
}
|
||||
return true, removed, missing, nil
|
||||
}
|
||||
|
||||
func removeRegularFileIfExists(path string) (removed bool, missing bool, err error) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, true, nil
|
||||
}
|
||||
return false, false, err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return false, false, nil
|
||||
}
|
||||
if err := os.Remove(path); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, true, nil
|
||||
}
|
||||
return false, false, err
|
||||
}
|
||||
return true, false, nil
|
||||
}
|
||||
|
||||
func localPathWithin(root, path string) (string, bool) {
|
||||
if strings.TrimSpace(root) == "" || strings.TrimSpace(path) == "" {
|
||||
return "", false
|
||||
@@ -1097,6 +1369,7 @@ func (a *App) enqueueUploadedVideo(ctx context.Context, v *catalog.Video) {
|
||||
a.mu.Lock()
|
||||
worker := a.workers[v.DriveID]
|
||||
thumbWorker := a.thumbWorkers[v.DriveID]
|
||||
fingerprintWorker := a.fingerprintWorkers[v.DriveID]
|
||||
a.mu.Unlock()
|
||||
|
||||
if thumbWorker != nil && v.ThumbnailURL == "" {
|
||||
@@ -1105,6 +1378,9 @@ func (a *App) enqueueUploadedVideo(ctx context.Context, v *catalog.Video) {
|
||||
if worker != nil && a.teaserEnabledForDrive(ctx, v.DriveID) {
|
||||
worker.Enqueue(v)
|
||||
}
|
||||
if fingerprintWorker != nil {
|
||||
fingerprintWorker.Enqueue(v)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) regenPreview(ctx context.Context, videoID string) {
|
||||
@@ -1231,26 +1507,36 @@ func (a *App) regenFailedThumbnails(ctx context.Context, driveID string) {
|
||||
}
|
||||
|
||||
// listScanTargetIDs 返回 nightly Phase 1 应扫描的所有 drive ID
|
||||
// (非 spider91、非 localupload)。顺序按 registry.All 给的稳定顺序。
|
||||
func (a *App) listScanTargetIDs(_ context.Context) []string {
|
||||
all := a.registry.All()
|
||||
// (非 spider91、非 localupload)。它直接读 catalog,而不是 registry,这样
|
||||
// 进程刚启动、云盘还在后台挂载时,nightly 也不会漏掉配置过的 drive。
|
||||
func (a *App) listScanTargetIDs(ctx context.Context) []string {
|
||||
all, err := a.cat.ListDrives(ctx)
|
||||
if err != nil {
|
||||
log.Printf("[nightly] list scan target drives: %v", err)
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(all))
|
||||
for _, d := range all {
|
||||
if !shouldScanDrive(d) {
|
||||
if d == nil || d.ID == localupload.DriveID || d.Kind == spider91.Kind {
|
||||
continue
|
||||
}
|
||||
out = append(out, d.ID())
|
||||
out = append(out, d.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// listSpider91DriveIDs 返回 nightly Phase 2 应触发爬取的 spider91 drive ID 列表。
|
||||
func (a *App) listSpider91DriveIDs(_ context.Context) []string {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
out := make([]string, 0, len(a.spider91Crawlers))
|
||||
for id := range a.spider91Crawlers {
|
||||
out = append(out, id)
|
||||
func (a *App) listSpider91DriveIDs(ctx context.Context) []string {
|
||||
all, err := a.cat.ListDrives(ctx)
|
||||
if err != nil {
|
||||
log.Printf("[nightly] list spider91 drives: %v", err)
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(all))
|
||||
for _, d := range all {
|
||||
if d != nil && d.Kind == spider91.Kind {
|
||||
out = append(out, d.ID)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1258,8 +1544,8 @@ func (a *App) listSpider91DriveIDs(_ context.Context) []string {
|
||||
// waitAllPreviewQueuesIdle 阻塞直到所有 drive 的封面 worker 和 teaser worker
|
||||
// 队列都为空且无 in-flight 任务。
|
||||
//
|
||||
// 顺序:先等所有 thumb worker(因为 enqueueDriveGeneration 内部已经先等当前
|
||||
// drive 的封面再入队 teaser,但这里是跨 drive 的全局同步),再等所有 teaser。
|
||||
// 顺序:先等所有 thumb worker,再等所有 teaser。两个队列生成时互不等待;
|
||||
// nightly 只在 phase 边界统一等待它们都 drain。
|
||||
// 若 ctx 在等待中被取消(软超时 / shutdown),立即返回 ctx.Err。
|
||||
func (a *App) waitAllPreviewQueuesIdle(ctx context.Context) error {
|
||||
a.mu.Lock()
|
||||
@@ -1309,7 +1595,17 @@ func (a *App) runSpider91Crawl(ctx context.Context, driveID string) {
|
||||
c := a.spider91Crawlers[driveID]
|
||||
a.mu.Unlock()
|
||||
if c == nil {
|
||||
return
|
||||
if err := a.ensureDriveAttached(ctx, driveID); err != nil {
|
||||
log.Printf("[spider91] drive=%s attach failed: %v", driveID, err)
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
c = a.spider91Crawlers[driveID]
|
||||
a.mu.Unlock()
|
||||
if c == nil {
|
||||
log.Printf("[spider91] drive=%s crawler not attached", driveID)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
d, err := a.cat.GetDrive(ctx, driveID)
|
||||
@@ -1356,7 +1652,9 @@ func (a *App) runSpider91Crawl(ctx context.Context, driveID string) {
|
||||
a.mu.Lock()
|
||||
worker := a.workers[driveID]
|
||||
thumbWorker := a.thumbWorkers[driveID]
|
||||
fingerprintWorker := a.fingerprintWorkers[driveID]
|
||||
a.mu.Unlock()
|
||||
a.scheduleFingerprintBackfill(ctx, driveID, fingerprintWorker)
|
||||
a.enqueueDriveGeneration(ctx, driveID, worker, thumbWorker)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
)
|
||||
|
||||
func TestSpider91IntCredFallbacks(t *testing.T) {
|
||||
@@ -30,3 +34,56 @@ func TestSpider91IntCredFallbacks(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
|
||||
reg := proxy.NewRegistry()
|
||||
reg.Set("p115-one", &spider91UploadTargetFakeDrive{id: "p115-one", kind: "p115"})
|
||||
reg.Set("onedrive-one", &spider91UploadTargetFakeDrive{id: "onedrive-one", kind: "onedrive"})
|
||||
|
||||
app := &App{registry: reg}
|
||||
if got := app.Spider91UploadDriveID(); got != "" {
|
||||
t.Fatalf("empty upload target selected %q, want local-only empty target", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "p115-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "p115-one" {
|
||||
t.Fatalf("explicit upload target = %q, want p115-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "onedrive-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "onedrive-one" {
|
||||
t.Fatalf("explicit onedrive upload target = %q, want onedrive-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "missing"
|
||||
if got := app.Spider91UploadDriveID(); got != "" {
|
||||
t.Fatalf("missing upload target = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
type spider91UploadTargetFakeDrive struct {
|
||||
id string
|
||||
kind string
|
||||
}
|
||||
|
||||
func (d *spider91UploadTargetFakeDrive) Kind() string { return d.kind }
|
||||
func (d *spider91UploadTargetFakeDrive) ID() string { return d.id }
|
||||
func (d *spider91UploadTargetFakeDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) RootID() string { return "root" }
|
||||
|
||||
+368
-37
@@ -13,7 +13,9 @@ import (
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/config"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/fingerprint"
|
||||
"github.com/video-site/backend/internal/preview"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
)
|
||||
|
||||
func TestRegisterPreviewWorkerBackfillsPendingWhenDriveTeaserEnabled(t *testing.T) {
|
||||
@@ -53,7 +55,7 @@ func TestRegisterPreviewWorkerBackfillsPendingWhenDriveTeaserEnabled(t *testing.
|
||||
worker := preview.NewWorker(&serverFakeTeaserGenerator{}, cat, &serverFakeDrive{})
|
||||
go worker.Run(ctx)
|
||||
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, nil, func() {})
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, nil, nil, func() {})
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
@@ -77,7 +79,7 @@ func TestRegisterPreviewWorkerBackfillsPendingWhenDriveTeaserEnabled(t *testing.
|
||||
t.Fatalf("preview status = %q, want ready", got.PreviewStatus)
|
||||
}
|
||||
|
||||
func TestRegisterPreviewWorkersGenerateThumbnailsBeforePreviews(t *testing.T) {
|
||||
func TestRegisterPreviewWorkersRunThumbnailsAndPreviewsIndependently(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
@@ -93,16 +95,18 @@ func TestRegisterPreviewWorkersGenerateThumbnailsBeforePreviews(t *testing.T) {
|
||||
|
||||
seedDriveWithTeaser(t, cat, "drive-id", true)
|
||||
now := time.Now()
|
||||
for _, v := range []*catalog.Video{
|
||||
{ID: "video-1", DriveID: "drive-id", FileID: "file-1", Title: "Clip 1", PreviewStatus: "pending"},
|
||||
{ID: "video-2", DriveID: "drive-id", FileID: "file-2", Title: "Clip 2", PreviewStatus: "pending"},
|
||||
} {
|
||||
v.PublishedAt = now
|
||||
v.CreatedAt = now
|
||||
v.UpdatedAt = now
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed video %s: %v", v.ID, err)
|
||||
}
|
||||
video := &catalog.Video{
|
||||
ID: "video-1",
|
||||
DriveID: "drive-id",
|
||||
FileID: "file-1",
|
||||
Title: "Clip 1",
|
||||
PreviewStatus: "pending",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if err := cat.UpsertVideo(ctx, video); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
app := &App{
|
||||
@@ -110,47 +114,234 @@ func TestRegisterPreviewWorkersGenerateThumbnailsBeforePreviews(t *testing.T) {
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
}
|
||||
gen := &serverFakeTeaserGenerator{}
|
||||
gen := &serverBlockingThumbGenerator{
|
||||
started: make(chan string, 1),
|
||||
release: make(chan struct{}),
|
||||
}
|
||||
drv := &serverFakeDrive{}
|
||||
worker := preview.NewWorker(gen, cat, drv)
|
||||
thumbWorker := preview.NewThumbWorker(gen, cat, drv)
|
||||
go worker.Run(ctx)
|
||||
go thumbWorker.Run(ctx)
|
||||
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, func() {})
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, nil, func() {})
|
||||
|
||||
select {
|
||||
case got := <-gen.started:
|
||||
if got != video.ID {
|
||||
t.Fatalf("thumbnail started for %q, want %q", got, video.ID)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("thumbnail generation did not start")
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
first, err := cat.GetVideo(ctx, "video-1")
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video-1: %v", err)
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
second, err := cat.GetVideo(ctx, "video-2")
|
||||
if err != nil {
|
||||
t.Fatalf("get video-2: %v", err)
|
||||
}
|
||||
if first.ThumbnailURL != "" && second.ThumbnailURL != "" &&
|
||||
first.PreviewStatus == "ready" && second.PreviewStatus == "ready" {
|
||||
events := gen.Events()
|
||||
if len(events) != 4 {
|
||||
t.Fatalf("events = %#v, want 4 generation events", events)
|
||||
}
|
||||
for i, event := range events[:2] {
|
||||
if event[:6] != "thumb:" {
|
||||
t.Fatalf("event %d = %q, want thumbnail before previews; all events=%#v", i, event, events)
|
||||
}
|
||||
}
|
||||
for i, event := range events[2:] {
|
||||
if event[:8] != "preview:" {
|
||||
t.Fatalf("event %d = %q, want previews after thumbnails; all events=%#v", i+2, event, events)
|
||||
}
|
||||
if got.PreviewStatus == "ready" {
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail url = %q, want preview ready while thumbnail is still blocked", got.ThumbnailURL)
|
||||
}
|
||||
close(gen.release)
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
t.Fatalf("generation did not finish, events=%#v", gen.Events())
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video after timeout: %v", err)
|
||||
}
|
||||
t.Fatalf("preview status=%q thumbnail=%q, want preview ready before thumbnail finishes", got.PreviewStatus, got.ThumbnailURL)
|
||||
}
|
||||
|
||||
func TestRegisterPreviewWorkersBackfillsHistoricalFingerprints(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
dataPath := filepath.Join(t.TempDir(), "video.mp4")
|
||||
data := []byte("historical video content for fingerprint")
|
||||
if err := os.WriteFile(dataPath, data, 0o644); err != nil {
|
||||
t.Fatalf("write video data: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
video := &catalog.Video{
|
||||
ID: "historical-video",
|
||||
DriveID: "drive-id",
|
||||
FileID: "file-id",
|
||||
Title: "Historical",
|
||||
Size: int64(len(data)),
|
||||
FingerprintStatus: "pending",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if err := cat.UpsertVideo(ctx, video); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
app := &App{
|
||||
cat: cat,
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
fingerprintWorkers: make(map[string]*fingerprint.Worker),
|
||||
}
|
||||
drv := &serverFingerprintFakeDrive{path: dataPath}
|
||||
fingerprintWorker := fingerprint.NewWorker(cat, drv, fingerprint.Config{})
|
||||
go fingerprintWorker.Run(ctx)
|
||||
|
||||
app.registerPreviewWorkers(ctx, "drive-id", nil, nil, fingerprintWorker, func() {})
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.SampledSHA256 != "" && got.FingerprintStatus == "ready" {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video after timeout: %v", err)
|
||||
}
|
||||
t.Fatalf("fingerprint status=%q sampled=%q, want ready with hash", got.FingerprintStatus, got.SampledSHA256)
|
||||
}
|
||||
|
||||
func TestRunScanStartsFingerprintBeforeThumbnailAndPreviewDrain(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
seedDriveWithTeaser(t, cat, "drive-id", true)
|
||||
|
||||
dataPath := filepath.Join(t.TempDir(), "scan-video.mp4")
|
||||
data := []byte("scan video content for independent fingerprint")
|
||||
if err := os.WriteFile(dataPath, data, 0o644); err != nil {
|
||||
t.Fatalf("write video data: %v", err)
|
||||
}
|
||||
|
||||
drv := &serverScanFingerprintFakeDrive{
|
||||
serverFingerprintFakeDrive: serverFingerprintFakeDrive{path: dataPath},
|
||||
entries: []drives.Entry{{
|
||||
ID: "file-id",
|
||||
Name: "scan-video.mp4",
|
||||
Size: int64(len(data)),
|
||||
ParentID: "root",
|
||||
}},
|
||||
}
|
||||
registry := proxy.NewRegistry()
|
||||
registry.Set("drive-id", drv)
|
||||
|
||||
gen := &serverFakeTeaserGenerator{}
|
||||
worker := preview.NewWorker(gen, cat, drv)
|
||||
thumbWorker := preview.NewThumbWorker(gen, cat, drv)
|
||||
fingerprintWorker := fingerprint.NewWorker(cat, drv, fingerprint.Config{})
|
||||
go fingerprintWorker.Run(ctx)
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{
|
||||
Scanner: config.Scanner{VideoExtensions: []string{".mp4"}},
|
||||
},
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
workers: map[string]*preview.Worker{"drive-id": worker},
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{"drive-id": thumbWorker},
|
||||
fingerprintWorkers: map[string]*fingerprint.Worker{"drive-id": fingerprintWorker},
|
||||
}
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
app.runScan(ctx, "drive-id")
|
||||
}()
|
||||
|
||||
videoID := "fake-drive-id-file-id"
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
got, err := cat.GetVideo(ctx, videoID)
|
||||
if err == nil && got.SampledSHA256 != "" && got.FingerprintStatus == "ready" {
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("scan did not stop after context cancel")
|
||||
}
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail url = %q, want fingerprint before thumbnail generation", got.ThumbnailURL)
|
||||
}
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("scan did not stop after context cancel")
|
||||
}
|
||||
got, err := cat.GetVideo(context.Background(), videoID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video after timeout: %v", err)
|
||||
}
|
||||
t.Fatalf("fingerprint status=%q sampled=%q, want ready before thumbnail/preview drain", got.FingerprintStatus, got.SampledSHA256)
|
||||
}
|
||||
|
||||
func TestNightlyTargetsComeFromCatalogBeforeDriveAttach(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, d := range []*catalog.Drive{
|
||||
{ID: "115", Kind: "p115", Name: "115", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "pikpak", Kind: "pikpak", Name: "PikPak", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "91-spider", Kind: "spider91", Name: "91 Spider", RootID: "0", TeaserEnabled: true},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
t.Fatalf("seed drive %s: %v", d.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
app := &App{cat: cat}
|
||||
scanIDs := app.listScanTargetIDs(ctx)
|
||||
if len(scanIDs) != 2 || scanIDs[0] != "115" || scanIDs[1] != "pikpak" {
|
||||
t.Fatalf("scan target ids = %#v, want 115 and pikpak from catalog", scanIDs)
|
||||
}
|
||||
spiderIDs := app.listSpider91DriveIDs(ctx)
|
||||
if len(spiderIDs) != 1 || spiderIDs[0] != "91-spider" {
|
||||
t.Fatalf("spider91 ids = %#v, want catalog spider drive", spiderIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedThumbnailsDoNotBlockPreviewGeneration(t *testing.T) {
|
||||
@@ -205,7 +396,7 @@ func TestFailedThumbnailsDoNotBlockPreviewGeneration(t *testing.T) {
|
||||
go worker.Run(ctx)
|
||||
go thumbWorker.Run(ctx)
|
||||
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, func() {})
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, nil, func() {})
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
@@ -480,6 +671,106 @@ func TestCleanupMissingPikPakVideosRemovesDatabaseRowsAndLocalAssets(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupDuplicateVideoAssetsRemovesOnlyDuplicateLocalAssets(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
localDir := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
canonicalPreview := filepath.Join(localDir, "canonical.mp4")
|
||||
duplicatePreview := filepath.Join(localDir, "duplicate.mp4")
|
||||
canonicalThumb := filepath.Join(localDir, "thumbs", "canonical-video.jpg")
|
||||
duplicateThumb := filepath.Join(localDir, "thumbs", "duplicate-video.jpg")
|
||||
for _, path := range []string{canonicalPreview, duplicatePreview, canonicalThumb, duplicateThumb} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("asset"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
|
||||
for _, v := range []*catalog.Video{
|
||||
{
|
||||
ID: "canonical-video",
|
||||
DriveID: "115",
|
||||
FileID: "file-a",
|
||||
Title: "Canonical",
|
||||
Size: 2048,
|
||||
ThumbnailURL: "/p/thumb/canonical-video",
|
||||
PreviewLocal: canonicalPreview,
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "duplicate-video",
|
||||
DriveID: "onedrive",
|
||||
FileID: "file-b",
|
||||
Title: "Duplicate",
|
||||
Size: 2048,
|
||||
ThumbnailURL: "/p/thumb/duplicate-video",
|
||||
PreviewLocal: duplicatePreview,
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, v.ID, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
}
|
||||
if err := app.cleanupDuplicateVideoAssets(ctx); err != nil {
|
||||
t.Fatalf("cleanup duplicate video assets: %v", err)
|
||||
}
|
||||
|
||||
for _, path := range []string{canonicalPreview, canonicalThumb} {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("canonical asset %s missing: %v", path, err)
|
||||
}
|
||||
}
|
||||
for _, path := range []string{duplicatePreview, duplicateThumb} {
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("duplicate asset %s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
dup, err := cat.GetVideo(ctx, "duplicate-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get duplicate: %v", err)
|
||||
}
|
||||
if dup.PreviewLocal != "" || dup.PreviewStatus != "pending" {
|
||||
t.Fatalf("duplicate preview local=%q status=%q, want empty pending", dup.PreviewLocal, dup.PreviewStatus)
|
||||
}
|
||||
if dup.ThumbnailURL != "" {
|
||||
t.Fatalf("duplicate thumbnail url = %q, want empty", dup.ThumbnailURL)
|
||||
}
|
||||
canon, err := cat.GetVideo(ctx, "canonical-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get canonical: %v", err)
|
||||
}
|
||||
if canon.PreviewLocal != canonicalPreview || canon.ThumbnailURL != "/p/thumb/canonical-video" {
|
||||
t.Fatalf("canonical changed: preview=%q thumb=%q", canon.PreviewLocal, canon.ThumbnailURL)
|
||||
}
|
||||
}
|
||||
|
||||
type serverFakeTeaserGenerator struct {
|
||||
mu sync.Mutex
|
||||
events []string
|
||||
@@ -520,6 +811,28 @@ func (g *serverFakeTeaserGenerator) GenerateThumbnail(_ context.Context, _ *driv
|
||||
return "/tmp/" + videoID + ".jpg", nil
|
||||
}
|
||||
|
||||
type serverBlockingThumbGenerator struct {
|
||||
serverFakeTeaserGenerator
|
||||
started chan string
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (g *serverBlockingThumbGenerator) GenerateThumbnail(ctx context.Context, _ *drives.StreamLink, videoID string, _ float64) (string, error) {
|
||||
g.record("thumb:" + videoID)
|
||||
if g.started != nil {
|
||||
select {
|
||||
case g.started <- videoID:
|
||||
default:
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-g.release:
|
||||
return "/tmp/" + videoID + ".jpg", nil
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
type serverFakeDrive struct{}
|
||||
|
||||
func (d *serverFakeDrive) Kind() string { return "fake" }
|
||||
@@ -544,6 +857,24 @@ func (d *serverFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
}
|
||||
func (d *serverFakeDrive) RootID() string { return "root" }
|
||||
|
||||
type serverFingerprintFakeDrive struct {
|
||||
serverFakeDrive
|
||||
path string
|
||||
}
|
||||
|
||||
func (d *serverFingerprintFakeDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
|
||||
return &drives.StreamLink{URL: d.path}, nil
|
||||
}
|
||||
|
||||
type serverScanFingerprintFakeDrive struct {
|
||||
serverFingerprintFakeDrive
|
||||
entries []drives.Entry
|
||||
}
|
||||
|
||||
func (d *serverScanFingerprintFakeDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return d.entries, nil
|
||||
}
|
||||
|
||||
type serverLocalUploadFakeDrive struct {
|
||||
serverFakeDrive
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ preview:
|
||||
width: 480
|
||||
|
||||
# 盘列表。上线后请通过管理后台添加,本文件可留空。
|
||||
# kind 支持 quark / p115 / pikpak / wopan / onedrive。
|
||||
# kind 支持 quark / p115 / pikpak / wopan / onedrive / localstorage。
|
||||
# OneDrive 示例:
|
||||
# - id: "my-onedrive"
|
||||
# kind: "onedrive"
|
||||
@@ -67,6 +67,12 @@ preview:
|
||||
# root_id: "root"
|
||||
# params:
|
||||
# refresh_token: "..."
|
||||
# api_url_address: "https://api.oplist.org/onedrive/renewapi"
|
||||
# region: "global"
|
||||
# 本地存储示例:
|
||||
# - id: "local-media"
|
||||
# kind: "localstorage"
|
||||
# name: "本地视频目录"
|
||||
# root_id: "/"
|
||||
# scan_root_id: "/"
|
||||
# params:
|
||||
# path: "/mnt/videos"
|
||||
drives: []
|
||||
|
||||
+154
-27
@@ -5,9 +5,12 @@ import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
@@ -18,6 +21,13 @@ import (
|
||||
type AdminServer struct {
|
||||
Catalog *catalog.Catalog
|
||||
Auth *auth.Authenticator
|
||||
// VersionFilePath points to the installer-written .version file.
|
||||
VersionFilePath string
|
||||
// GitHubRepo is the owner/name repo used for update checks.
|
||||
GitHubRepo string
|
||||
// ReleaseAPIURL and HTTPClient are injectable for tests. Production code leaves them empty.
|
||||
ReleaseAPIURL string
|
||||
HTTPClient *http.Client
|
||||
// SetupRequired 表示当前是否仍处于首次部署初始化状态。
|
||||
SetupRequired func() bool
|
||||
// OnSetup 持久化首次部署时设置的管理员账号密码,并更新运行中认证器。
|
||||
@@ -40,12 +50,12 @@ type AdminServer struct {
|
||||
// Theme 读写("dark" | "pink")
|
||||
GetTheme func() string
|
||||
SetTheme func(theme string) error
|
||||
// Spider91 → PikPak 上传目标 drive ID 读写
|
||||
// Spider91 → 115/PikPak 上传目标 drive ID 读写
|
||||
GetSpider91UploadDriveID func() string
|
||||
SetSpider91UploadDriveID func(driveID string) error
|
||||
// OnRunNightlyJob 触发一次完整的凌晨流水线(Phase1 扫盘 + Phase2 91 爬虫 +
|
||||
// Phase3 迁移)。立即返回 —— 实际任务在后台跑,admin 在日志或下次状态查询里
|
||||
// 看进度。重复点击会被 Runner.TryLock 丢弃。
|
||||
// 看进度。若流水线正在跑,Runner 最多保留一个待触发请求,当前轮结束后再跑一轮。
|
||||
OnRunNightlyJob func()
|
||||
// ListDriveDirChildren 列出某个 drive 在 parentID 目录下的直接子目录。
|
||||
// parentID 为空时使用 drive 的 RootID。返回 (子目录列表, error)。
|
||||
@@ -68,8 +78,9 @@ type GenerationStatus struct {
|
||||
}
|
||||
|
||||
type DriveGenerationStatuses struct {
|
||||
Thumbnail GenerationStatus `json:"thumbnail"`
|
||||
Preview GenerationStatus `json:"preview"`
|
||||
Thumbnail GenerationStatus `json:"thumbnail"`
|
||||
Preview GenerationStatus `json:"preview"`
|
||||
Fingerprint GenerationStatus `json:"fingerprint"`
|
||||
}
|
||||
|
||||
func (a *AdminServer) Register(r chi.Router) {
|
||||
@@ -112,11 +123,25 @@ func (a *AdminServer) Register(r chi.Router) {
|
||||
r.Put("/settings", a.handlePutSettings)
|
||||
|
||||
// 运维任务
|
||||
r.Get("/update/check", a.handleCheckUpdate)
|
||||
r.Post("/jobs/nightly/run", a.handleRunNightlyJob)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
type updateCheckDTO struct {
|
||||
CurrentVersion string `json:"currentVersion"`
|
||||
LatestVersion string `json:"latestVersion"`
|
||||
HasUpdate bool `json:"hasUpdate"`
|
||||
ReleaseURL string `json:"releaseUrl,omitempty"`
|
||||
CheckedAt string `json:"checkedAt"`
|
||||
}
|
||||
|
||||
type githubReleaseDTO struct {
|
||||
TagName string `json:"tag_name"`
|
||||
HTMLURL string `json:"html_url"`
|
||||
}
|
||||
|
||||
type loginReq struct {
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
@@ -221,6 +246,91 @@ func (a *AdminServer) handleMe(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"authenticated": ok})
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleCheckUpdate(w http.ResponseWriter, r *http.Request) {
|
||||
info, err := a.checkUpdate(r.Context())
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusBadGateway, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, http.StatusOK, info)
|
||||
}
|
||||
|
||||
func (a *AdminServer) checkUpdate(ctx context.Context) (updateCheckDTO, error) {
|
||||
current := a.installedVersion()
|
||||
if current == "" {
|
||||
current = "unknown"
|
||||
}
|
||||
release, err := a.latestRelease(ctx)
|
||||
if err != nil {
|
||||
return updateCheckDTO{
|
||||
CurrentVersion: current,
|
||||
CheckedAt: time.Now().Format(time.RFC3339),
|
||||
}, err
|
||||
}
|
||||
latest := strings.TrimSpace(release.TagName)
|
||||
return updateCheckDTO{
|
||||
CurrentVersion: current,
|
||||
LatestVersion: latest,
|
||||
HasUpdate: current != "unknown" && latest != "" && current != latest,
|
||||
ReleaseURL: release.HTMLURL,
|
||||
CheckedAt: time.Now().Format(time.RFC3339),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *AdminServer) installedVersion() string {
|
||||
path := strings.TrimSpace(a.VersionFilePath)
|
||||
if path == "" {
|
||||
path = ".version"
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
lines := strings.Split(strings.ReplaceAll(string(data), "\r\n", "\n"), "\n")
|
||||
if len(lines) == 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(lines[0])
|
||||
}
|
||||
|
||||
func (a *AdminServer) latestRelease(ctx context.Context) (githubReleaseDTO, error) {
|
||||
url := strings.TrimSpace(a.ReleaseAPIURL)
|
||||
if url == "" {
|
||||
repo := strings.TrimSpace(a.GitHubRepo)
|
||||
if repo == "" {
|
||||
repo = "nianzhibai/91"
|
||||
}
|
||||
url = "https://api.github.com/repos/" + repo + "/releases/latest"
|
||||
}
|
||||
client := a.HTTPClient
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 8 * time.Second}
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return githubReleaseDTO{}, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/vnd.github+json")
|
||||
req.Header.Set("User-Agent", "video-site-91")
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return githubReleaseDTO{}, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
return githubReleaseDTO{}, fmt.Errorf("github release check failed: HTTP %d", res.StatusCode)
|
||||
}
|
||||
var release githubReleaseDTO
|
||||
if err := json.NewDecoder(res.Body).Decode(&release); err != nil {
|
||||
return githubReleaseDTO{}, err
|
||||
}
|
||||
if strings.TrimSpace(release.TagName) == "" {
|
||||
return githubReleaseDTO{}, errors.New("github release check returned empty tag")
|
||||
}
|
||||
return release, nil
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
drives, err := a.Catalog.ListDrives(r.Context())
|
||||
if err != nil {
|
||||
@@ -237,6 +347,11 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
fingerprintCounts, err := a.Catalog.CountFingerprintsByDrive(r.Context())
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
generationStatuses := map[string]DriveGenerationStatuses{}
|
||||
if a.GetDriveGenerationStatuses != nil {
|
||||
generationStatuses = a.GetDriveGenerationStatuses()
|
||||
@@ -259,20 +374,25 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
SkipDirIDs []string `json:"skipDirIds"`
|
||||
// LastCrawlAt 是 spider91 上次成功爬取的 unix 秒(来自 credentials.last_crawl_at)。
|
||||
// 其它 kind 留 0;前端用它显示"上次抓取: N 小时前"。
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
FingerprintReadyCount int `json:"fingerprintReadyCount"`
|
||||
FingerprintPendingCount int `json:"fingerprintPendingCount"`
|
||||
FingerprintFailedCount int `json:"fingerprintFailedCount"`
|
||||
}
|
||||
list := make([]out, 0, len(drives))
|
||||
for _, d := range drives {
|
||||
counts := teaserCounts[d.ID]
|
||||
thumbCounts := thumbnailCounts[d.ID]
|
||||
fingerprintCount := fingerprintCounts[d.ID]
|
||||
generation := generationStatuses[d.ID]
|
||||
if generation.Thumbnail.State == "" {
|
||||
generation.Thumbnail.State = "idle"
|
||||
@@ -280,6 +400,9 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
if generation.Preview.State == "" {
|
||||
generation.Preview.State = "idle"
|
||||
}
|
||||
if generation.Fingerprint.State == "" {
|
||||
generation.Fingerprint.State = "idle"
|
||||
}
|
||||
// spider91 没有用户凭证概念;只要存在 drive 行就视为"已配置"。
|
||||
// last_crawl_at 是后端自动写入的运行状态字段,不计入 hasCredential 判定。
|
||||
hasCred := false
|
||||
@@ -305,18 +428,22 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
ID: d.ID, Kind: d.Kind, Name: d.Name,
|
||||
RootID: d.RootID, ScanRootID: d.ScanRootID,
|
||||
Status: d.Status, LastError: d.LastError,
|
||||
HasCredential: hasCred,
|
||||
TeaserEnabled: d.TeaserEnabled,
|
||||
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
|
||||
LastCrawlAt: lastCrawlAt,
|
||||
ThumbnailGenerationStatus: generation.Thumbnail,
|
||||
PreviewGenerationStatus: generation.Preview,
|
||||
ThumbnailReadyCount: thumbCounts.Ready,
|
||||
ThumbnailPendingCount: thumbCounts.Pending,
|
||||
ThumbnailFailedCount: thumbCounts.Failed,
|
||||
TeaserReadyCount: counts.Ready,
|
||||
TeaserPendingCount: counts.Pending,
|
||||
TeaserFailedCount: counts.Failed,
|
||||
HasCredential: hasCred,
|
||||
TeaserEnabled: d.TeaserEnabled,
|
||||
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
|
||||
LastCrawlAt: lastCrawlAt,
|
||||
ThumbnailGenerationStatus: generation.Thumbnail,
|
||||
PreviewGenerationStatus: generation.Preview,
|
||||
FingerprintGenerationStatus: generation.Fingerprint,
|
||||
ThumbnailReadyCount: thumbCounts.Ready,
|
||||
ThumbnailPendingCount: thumbCounts.Pending,
|
||||
ThumbnailFailedCount: thumbCounts.Failed,
|
||||
TeaserReadyCount: counts.Ready,
|
||||
TeaserPendingCount: counts.Pending,
|
||||
TeaserFailedCount: counts.Failed,
|
||||
FingerprintReadyCount: fingerprintCount.Ready,
|
||||
FingerprintPendingCount: fingerprintCount.Pending,
|
||||
FingerprintFailedCount: fingerprintCount.Failed,
|
||||
})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, list)
|
||||
@@ -425,7 +552,7 @@ func (a *AdminServer) handleRescan(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// handleRunNightlyJob 触发一次完整的凌晨流水线(不论当前时间,不论今日是否已跑)。
|
||||
// 立即返回 202;进度通过 backend 日志和下次 GET /admin/api/drives 的状态变化观察。
|
||||
// 流水线已在跑时 Runner 会丢弃此次触发并记日志。
|
||||
// 流水线已在跑时 Runner 最多排队一个后续触发;如果已有待触发请求,新的点击会被忽略。
|
||||
func (a *AdminServer) handleRunNightlyJob(w http.ResponseWriter, r *http.Request) {
|
||||
if a.OnRunNightlyJob != nil {
|
||||
a.OnRunNightlyJob()
|
||||
@@ -748,7 +875,7 @@ func (a *AdminServer) handleGetSettings(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
func (a *AdminServer) handlePutSettings(w http.ResponseWriter, r *http.Request) {
|
||||
// 用 map 区分"没传"和"传了空字符串"两种语义;空 spider91 上传 ID 表示
|
||||
// 清除显式设置(回退到自动模式)。
|
||||
// 本地保存不上传。
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.NewDecoder(r.Body).Decode(&raw); err != nil {
|
||||
writeErr(w, http.StatusBadRequest, err)
|
||||
|
||||
@@ -115,6 +115,85 @@ func TestHandleSetupStoresCredentialsAndCreatesSession(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleCheckUpdateReportsNewRelease(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
versionFile := filepath.Join(dir, ".version")
|
||||
if err := os.WriteFile(versionFile, []byte("v0.1.0\n2026-05-29 12:00:00\n"), 0o644); err != nil {
|
||||
t.Fatalf("write version file: %v", err)
|
||||
}
|
||||
releaseServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("User-Agent") == "" {
|
||||
http.Error(w, "missing user agent", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"tag_name": "v0.2.0",
|
||||
"html_url": "https://github.com/nianzhibai/91/releases/tag/v0.2.0",
|
||||
})
|
||||
}))
|
||||
t.Cleanup(releaseServer.Close)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/update/check", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{
|
||||
VersionFilePath: versionFile,
|
||||
ReleaseAPIURL: releaseServer.URL,
|
||||
}).handleCheckUpdate(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got updateCheckDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.CurrentVersion != "v0.1.0" {
|
||||
t.Fatalf("currentVersion = %q, want v0.1.0", got.CurrentVersion)
|
||||
}
|
||||
if got.LatestVersion != "v0.2.0" {
|
||||
t.Fatalf("latestVersion = %q, want v0.2.0", got.LatestVersion)
|
||||
}
|
||||
if !got.HasUpdate {
|
||||
t.Fatalf("hasUpdate = false, want true")
|
||||
}
|
||||
if got.ReleaseURL == "" {
|
||||
t.Fatalf("releaseUrl is empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleCheckUpdateReportsUpToDate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
versionFile := filepath.Join(dir, ".version")
|
||||
if err := os.WriteFile(versionFile, []byte("v0.2.0\n"), 0o644); err != nil {
|
||||
t.Fatalf("write version file: %v", err)
|
||||
}
|
||||
releaseServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"tag_name": "v0.2.0",
|
||||
"html_url": "https://github.com/nianzhibai/91/releases/tag/v0.2.0",
|
||||
})
|
||||
}))
|
||||
t.Cleanup(releaseServer.Close)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/update/check", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{
|
||||
VersionFilePath: versionFile,
|
||||
ReleaseAPIURL: releaseServer.URL,
|
||||
}).handleCheckUpdate(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got updateCheckDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.HasUpdate {
|
||||
t.Fatalf("hasUpdate = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertDrivePreservesExistingCredentialsWhenRequestCredentialsEmpty(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -244,11 +323,11 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
|
||||
now := time.Now()
|
||||
videos := []*catalog.Video{
|
||||
{ID: "od-ready-1", DriveID: "OneDrive", FileID: "od-file-1", Title: "OD Ready 1", ThumbnailURL: "/p/thumb/od-ready-1", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-ready-2", DriveID: "OneDrive", FileID: "od-file-2", Title: "OD Ready 2", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-pending", DriveID: "OneDrive", FileID: "od-file-3", Title: "OD Pending", PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "pp-pending", DriveID: "PikPak", FileID: "pp-file-1", Title: "PP Pending", PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "pp-failed", DriveID: "PikPak", FileID: "pp-file-2", Title: "PP Failed", ThumbnailURL: "/p/thumb/pp-failed", PreviewStatus: "failed", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-ready-1", DriveID: "OneDrive", FileID: "od-file-1", Title: "OD Ready 1", Size: 100, ThumbnailURL: "/p/thumb/od-ready-1", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-ready-2", DriveID: "OneDrive", FileID: "od-file-2", Title: "OD Ready 2", Size: 100, PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-pending", DriveID: "OneDrive", FileID: "od-file-3", Title: "OD Pending", Size: 100, PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "pp-pending", DriveID: "PikPak", FileID: "pp-file-1", Title: "PP Pending", Size: 100, PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "pp-failed", DriveID: "PikPak", FileID: "pp-file-2", Title: "PP Failed", Size: 100, ThumbnailURL: "/p/thumb/pp-failed", PreviewStatus: "failed", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
}
|
||||
for _, v := range videos {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
@@ -258,6 +337,12 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
if err := cat.UpdateVideoMeta(ctx, "od-ready-2", catalog.VideoMetaPatch{ThumbnailStatus: "failed"}); err != nil {
|
||||
t.Fatalf("mark thumbnail failed: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "od-ready-1", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "ready", ""); err != nil {
|
||||
t.Fatalf("mark fingerprint ready: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "od-ready-2", "", "failed", "sample failed"); err != nil {
|
||||
t.Fatalf("mark fingerprint failed: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/drives", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
@@ -266,8 +351,9 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
GetDriveGenerationStatuses: func() map[string]DriveGenerationStatuses {
|
||||
return map[string]DriveGenerationStatuses{
|
||||
"OneDrive": {
|
||||
Thumbnail: GenerationStatus{State: "cooling", QueueLength: 3, CooldownUntil: "2026-05-16T21:00:00+08:00"},
|
||||
Preview: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
|
||||
Thumbnail: GenerationStatus{State: "cooling", QueueLength: 3, CooldownUntil: "2026-05-16T21:00:00+08:00"},
|
||||
Preview: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
|
||||
Fingerprint: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
|
||||
},
|
||||
}
|
||||
},
|
||||
@@ -277,48 +363,64 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got []struct {
|
||||
ID string `json:"id"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
ID string `json:"id"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
FingerprintReadyCount int `json:"fingerprintReadyCount"`
|
||||
FingerprintPendingCount int `json:"fingerprintPendingCount"`
|
||||
FingerprintFailedCount int `json:"fingerprintFailedCount"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
byID := map[string]struct {
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
FingerprintReady int
|
||||
FingerprintPending int
|
||||
FingerprintFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
Fingerprint GenerationStatus
|
||||
}{}
|
||||
for _, d := range got {
|
||||
byID[d.ID] = struct {
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
FingerprintReady int
|
||||
FingerprintPending int
|
||||
FingerprintFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
Fingerprint GenerationStatus
|
||||
}{
|
||||
TeaserReady: d.TeaserReadyCount,
|
||||
TeaserPending: d.TeaserPendingCount,
|
||||
TeaserFailed: d.TeaserFailedCount,
|
||||
ThumbnailReady: d.ThumbnailReadyCount,
|
||||
ThumbnailPending: d.ThumbnailPendingCount,
|
||||
ThumbnailFailed: d.ThumbnailFailedCount,
|
||||
Thumbnail: d.ThumbnailGenerationStatus,
|
||||
Preview: d.PreviewGenerationStatus,
|
||||
TeaserReady: d.TeaserReadyCount,
|
||||
TeaserPending: d.TeaserPendingCount,
|
||||
TeaserFailed: d.TeaserFailedCount,
|
||||
ThumbnailReady: d.ThumbnailReadyCount,
|
||||
ThumbnailPending: d.ThumbnailPendingCount,
|
||||
ThumbnailFailed: d.ThumbnailFailedCount,
|
||||
FingerprintReady: d.FingerprintReadyCount,
|
||||
FingerprintPending: d.FingerprintPendingCount,
|
||||
FingerprintFailed: d.FingerprintFailedCount,
|
||||
Thumbnail: d.ThumbnailGenerationStatus,
|
||||
Preview: d.PreviewGenerationStatus,
|
||||
Fingerprint: d.FingerprintGenerationStatus,
|
||||
}
|
||||
}
|
||||
if byID["OneDrive"].TeaserReady != 2 || byID["OneDrive"].TeaserPending != 1 || byID["OneDrive"].TeaserFailed != 0 {
|
||||
@@ -330,13 +432,22 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
if byID["OneDrive"].Thumbnail.State != "cooling" || byID["OneDrive"].Preview.State != "generating" {
|
||||
t.Fatalf("OneDrive generation statuses = %#v, want thumbnail cooling and preview generating", byID["OneDrive"])
|
||||
}
|
||||
if byID["OneDrive"].FingerprintReady != 1 || byID["OneDrive"].FingerprintPending != 1 || byID["OneDrive"].FingerprintFailed != 1 {
|
||||
t.Fatalf("OneDrive fingerprint counts = %#v, want ready=1 pending=1 failed=1", byID["OneDrive"])
|
||||
}
|
||||
if byID["OneDrive"].Fingerprint.State != "generating" {
|
||||
t.Fatalf("OneDrive fingerprint status = %#v, want generating", byID["OneDrive"].Fingerprint)
|
||||
}
|
||||
if byID["PikPak"].TeaserReady != 0 || byID["PikPak"].TeaserPending != 1 || byID["PikPak"].TeaserFailed != 1 {
|
||||
t.Fatalf("PikPak counts = %#v, want ready=0 pending=1 failed=1", byID["PikPak"])
|
||||
}
|
||||
if byID["PikPak"].ThumbnailReady != 1 || byID["PikPak"].ThumbnailPending != 1 || byID["PikPak"].ThumbnailFailed != 0 {
|
||||
t.Fatalf("PikPak thumbnail counts = %#v, want ready=1 pending=1 failed=0", byID["PikPak"])
|
||||
}
|
||||
if byID["PikPak"].Thumbnail.State != "idle" || byID["PikPak"].Preview.State != "idle" {
|
||||
if byID["PikPak"].FingerprintPending != 2 {
|
||||
t.Fatalf("PikPak fingerprint counts = %#v, want pending=2", byID["PikPak"])
|
||||
}
|
||||
if byID["PikPak"].Thumbnail.State != "idle" || byID["PikPak"].Preview.State != "idle" || byID["PikPak"].Fingerprint.State != "idle" {
|
||||
t.Fatalf("PikPak generation statuses = %#v, want idle defaults", byID["PikPak"])
|
||||
}
|
||||
}
|
||||
|
||||
+164
-58
@@ -21,6 +21,7 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/auth"
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives/localstorage"
|
||||
"github.com/video-site/backend/internal/drives/localupload"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
@@ -155,26 +156,67 @@ func (s *Server) handleGetTheme(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) {
|
||||
// 拉一批候选(按发布时间倒序,覆盖最近 200 个),然后随机洗牌取前 homePageSize 个。
|
||||
// 如果库内不足 200 个会自动按实际数量返回,最后裁剪到 homePageSize。
|
||||
// 首页优先展示封面已经生成好的视频,避免新盘扫盘时大量黑封面占满首页。
|
||||
// 候选仍按发布时间覆盖最近 200 个,随后随机洗牌;封面不足时再用普通可见视频补齐。
|
||||
const candidatePool = 200
|
||||
items, _, err := s.Catalog.ListVideos(r.Context(), catalog.ListParams{
|
||||
Sort: "latest", Page: 1, PageSize: candidatePool,
|
||||
readyItems, _, err := s.Catalog.ListVideos(r.Context(), catalog.ListParams{
|
||||
Sort: "latest", Page: 1, PageSize: candidatePool, ThumbnailReadyOnly: true,
|
||||
})
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
rand.Shuffle(len(items), func(i, j int) {
|
||||
items[i], items[j] = items[j], items[i]
|
||||
rand.Shuffle(len(readyItems), func(i, j int) {
|
||||
readyItems[i], readyItems[j] = readyItems[j], readyItems[i]
|
||||
})
|
||||
|
||||
items := appendUniqueVideos(nil, readyItems, homePageSize)
|
||||
if len(items) > homePageSize {
|
||||
items = items[:homePageSize]
|
||||
}
|
||||
if len(items) < homePageSize {
|
||||
fallback, _, err := s.Catalog.ListVideos(r.Context(), catalog.ListParams{
|
||||
Sort: "latest", Page: 1, PageSize: candidatePool,
|
||||
})
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
rand.Shuffle(len(fallback), func(i, j int) {
|
||||
fallback[i], fallback[j] = fallback[j], fallback[i]
|
||||
})
|
||||
items = appendUniqueVideos(items, fallback, homePageSize)
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, http.StatusOK, mapVideos(items))
|
||||
}
|
||||
|
||||
func appendUniqueVideos(dst []*catalog.Video, candidates []*catalog.Video, limit int) []*catalog.Video {
|
||||
if len(dst) >= limit {
|
||||
return dst[:limit]
|
||||
}
|
||||
seen := make(map[string]struct{}, len(dst))
|
||||
for _, v := range dst {
|
||||
if v != nil {
|
||||
seen[v.ID] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, v := range candidates {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
dst = append(dst, v)
|
||||
seen[v.ID] = struct{}{}
|
||||
if len(dst) >= limit {
|
||||
return dst
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
func (s *Server) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
page, _ := strconv.Atoi(q.Get("page"))
|
||||
@@ -182,14 +224,18 @@ func (s *Server) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
if size <= 0 {
|
||||
size = 24
|
||||
}
|
||||
sort := q.Get("sort")
|
||||
params := catalog.ListParams{
|
||||
Keyword: q.Get("q"),
|
||||
Tag: q.Get("tag"),
|
||||
Category: q.Get("cat"),
|
||||
Sort: q.Get("sort"),
|
||||
Sort: sort,
|
||||
Page: page,
|
||||
PageSize: size,
|
||||
}
|
||||
if sort == "" || sort == "latest" {
|
||||
params.PreferReadyThumbnails = true
|
||||
}
|
||||
items, total, err := s.Catalog.ListVideos(r.Context(), params)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
@@ -241,7 +287,8 @@ func (s *Server) handleVideoDetail(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// pickRelatedVideos 选 total 个推荐视频。
|
||||
// 一半(向上取整)来自同标签命中,剩下用全库随机补齐;不会重复,也不会包含当前视频。
|
||||
// 一半来自同标签命中,剩下用全库随机补齐;两段都优先取已有封面的视频,
|
||||
// 不够时再回退到未生成封面的候选。结果不会重复,也不会包含当前视频。
|
||||
func (s *Server) pickRelatedVideos(ctx context.Context, current *catalog.Video, total int) []*catalog.Video {
|
||||
if total <= 0 || current == nil {
|
||||
return nil
|
||||
@@ -254,67 +301,124 @@ func (s *Server) pickRelatedVideos(ctx context.Context, current *catalog.Video,
|
||||
picked := make([]*catalog.Video, 0, total)
|
||||
seen := map[string]struct{}{current.ID: {}}
|
||||
|
||||
// 1) 同标签候选:对每个 tag 取一批,合并去重,洗牌后取 tagQuota 个
|
||||
// 1) 同标签候选:先取已有封面的候选,数量不够再从全部候选里补。
|
||||
if tagQuota > 0 && len(current.Tags) > 0 {
|
||||
var tagPool []*catalog.Video
|
||||
for _, tag := range current.Tags {
|
||||
if tag == "" {
|
||||
continue
|
||||
}
|
||||
items, _, err := s.Catalog.ListVideos(ctx, catalog.ListParams{
|
||||
Tag: tag, Sort: "latest", Page: 1, PageSize: 30,
|
||||
})
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, v := range items {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[v.ID] = struct{}{}
|
||||
tagPool = append(tagPool, v)
|
||||
}
|
||||
picked = appendRandomRelated(
|
||||
picked,
|
||||
s.relatedTagPool(ctx, current.Tags, seen, true),
|
||||
tagQuota,
|
||||
seen,
|
||||
)
|
||||
if len(picked) < tagQuota {
|
||||
picked = appendRandomRelated(
|
||||
picked,
|
||||
s.relatedTagPool(ctx, current.Tags, seen, false),
|
||||
tagQuota,
|
||||
seen,
|
||||
)
|
||||
}
|
||||
rand.Shuffle(len(tagPool), func(i, j int) {
|
||||
tagPool[i], tagPool[j] = tagPool[j], tagPool[i]
|
||||
})
|
||||
if len(tagPool) > tagQuota {
|
||||
tagPool = tagPool[:tagQuota]
|
||||
}
|
||||
picked = append(picked, tagPool...)
|
||||
}
|
||||
|
||||
// 2) 随机补齐:从全库取一批(避开已选 ID),洗牌后取剩下的名额
|
||||
remaining := total - len(picked)
|
||||
if remaining > 0 {
|
||||
// 2) 随机补齐:同样优先已有封面的全库候选,不够再回退。
|
||||
if len(picked) < total {
|
||||
picked = appendRandomRelated(
|
||||
picked,
|
||||
s.relatedListPool(ctx, seen, true, 200),
|
||||
total,
|
||||
seen,
|
||||
)
|
||||
}
|
||||
if len(picked) < total {
|
||||
picked = appendRandomRelated(
|
||||
picked,
|
||||
s.relatedListPool(ctx, seen, false, 200),
|
||||
total,
|
||||
seen,
|
||||
)
|
||||
}
|
||||
|
||||
return picked
|
||||
}
|
||||
|
||||
func (s *Server) relatedTagPool(ctx context.Context, tags []string, seen map[string]struct{}, readyOnly bool) []*catalog.Video {
|
||||
var pool []*catalog.Video
|
||||
poolSeen := make(map[string]struct{})
|
||||
for _, tag := range tags {
|
||||
if tag == "" {
|
||||
continue
|
||||
}
|
||||
items, _, err := s.Catalog.ListVideos(ctx, catalog.ListParams{
|
||||
Sort: "latest", Page: 1, PageSize: 200,
|
||||
Tag: tag,
|
||||
Sort: "latest",
|
||||
Page: 1,
|
||||
PageSize: 30,
|
||||
ThumbnailReadyOnly: readyOnly,
|
||||
PreferReadyThumbnails: !readyOnly,
|
||||
})
|
||||
if err == nil {
|
||||
var randomPool []*catalog.Video
|
||||
for _, v := range items {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[v.ID] = struct{}{}
|
||||
randomPool = append(randomPool, v)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, v := range items {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
rand.Shuffle(len(randomPool), func(i, j int) {
|
||||
randomPool[i], randomPool[j] = randomPool[j], randomPool[i]
|
||||
})
|
||||
if len(randomPool) > remaining {
|
||||
randomPool = randomPool[:remaining]
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
picked = append(picked, randomPool...)
|
||||
if _, ok := poolSeen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
poolSeen[v.ID] = struct{}{}
|
||||
pool = append(pool, v)
|
||||
}
|
||||
}
|
||||
return pool
|
||||
}
|
||||
|
||||
func (s *Server) relatedListPool(ctx context.Context, seen map[string]struct{}, readyOnly bool, pageSize int) []*catalog.Video {
|
||||
items, _, err := s.Catalog.ListVideos(ctx, catalog.ListParams{
|
||||
Sort: "latest",
|
||||
Page: 1,
|
||||
PageSize: pageSize,
|
||||
ThumbnailReadyOnly: readyOnly,
|
||||
PreferReadyThumbnails: !readyOnly,
|
||||
})
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
pool := make([]*catalog.Video, 0, len(items))
|
||||
for _, v := range items {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
pool = append(pool, v)
|
||||
}
|
||||
return pool
|
||||
}
|
||||
|
||||
func appendRandomRelated(picked []*catalog.Video, pool []*catalog.Video, targetLen int, seen map[string]struct{}) []*catalog.Video {
|
||||
if len(picked) >= targetLen || len(pool) == 0 {
|
||||
return picked
|
||||
}
|
||||
rand.Shuffle(len(pool), func(i, j int) {
|
||||
pool[i], pool[j] = pool[j], pool[i]
|
||||
})
|
||||
for _, v := range pool {
|
||||
if len(picked) >= targetLen {
|
||||
break
|
||||
}
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[v.ID] = struct{}{}
|
||||
picked = append(picked, v)
|
||||
}
|
||||
return picked
|
||||
}
|
||||
|
||||
@@ -796,6 +900,8 @@ func driveKindLabel(kind string) string {
|
||||
return "联通沃盘"
|
||||
case "onedrive":
|
||||
return "OneDrive"
|
||||
case localstorage.Kind:
|
||||
return "本地存储"
|
||||
case spider91.Kind:
|
||||
return "91 爬虫"
|
||||
default:
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -98,6 +99,146 @@ func TestPreviewURLFallsBackWithoutUpdatedAt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleHomePrioritizesVideosWithReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < 20; i++ {
|
||||
id := "pending-video-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
PublishedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
CreatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
UpdatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed pending video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < homePageSize+2; i++ {
|
||||
id := "ready-video-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
|
||||
PublishedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
CreatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
UpdatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/home", nil)
|
||||
(&Server{Catalog: cat}).handleHome(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got []VideoDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(got) != homePageSize {
|
||||
t.Fatalf("home items = %d, want %d", len(got), homePageSize)
|
||||
}
|
||||
for _, item := range got {
|
||||
if !strings.HasPrefix(item.ID, "ready-video-") {
|
||||
t.Fatalf("home returned %q without a ready thumbnail; items=%#v", item.ID, got)
|
||||
}
|
||||
if !strings.HasPrefix(item.Thumbnail, "https://thumb.example/") {
|
||||
t.Fatalf("thumbnail for %q = %q, want ready thumbnail URL", item.ID, item.Thumbnail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListLatestPrefersReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < 20; i++ {
|
||||
id := "pending-latest-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
PublishedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
CreatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
UpdatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed pending video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < 12; i++ {
|
||||
id := "ready-latest-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
|
||||
PublishedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
CreatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
UpdatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/list?page=1&size=12&sort=latest", nil)
|
||||
(&Server{Catalog: cat}).handleList(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
Items []VideoDTO `json:"items"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if got.Total != 32 {
|
||||
t.Fatalf("total = %d, want all matching videos included", got.Total)
|
||||
}
|
||||
if len(got.Items) != 12 {
|
||||
t.Fatalf("items = %d, want 12", len(got.Items))
|
||||
}
|
||||
for _, item := range got.Items {
|
||||
if !strings.HasPrefix(item.ID, "ready-latest-") {
|
||||
t.Fatalf("latest list returned %q before ready thumbnails; items=%#v", item.ID, got.Items)
|
||||
}
|
||||
if !strings.HasPrefix(item.Thumbnail, "https://thumb.example/") {
|
||||
t.Fatalf("thumbnail for %q = %q, want ready thumbnail URL", item.ID, item.Thumbnail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUploadVideoSavesFileVideoTagsAndQueuesPreview(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -509,6 +650,88 @@ func TestHandleVideoDetailIncludesDriveKindLabel(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleVideoDetailRecommendationsPreferReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "current-video",
|
||||
DriveID: "drive",
|
||||
FileID: "current-video",
|
||||
Title: "Current",
|
||||
Tags: []string{"same-tag"},
|
||||
ThumbnailURL: "https://thumb.example/current-video.jpg",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed current video: %v", err)
|
||||
}
|
||||
for i := 0; i < 20; i++ {
|
||||
id := "pending-related-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
Tags: []string{"same-tag"},
|
||||
PublishedAt: now.Add(time.Duration(i+1) * time.Minute),
|
||||
CreatedAt: now.Add(time.Duration(i+1) * time.Minute),
|
||||
UpdatedAt: now.Add(time.Duration(i+1) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed pending related video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < 8; i++ {
|
||||
id := "ready-related-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
Tags: []string{"same-tag"},
|
||||
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
|
||||
PublishedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
CreatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
UpdatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready related video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
req := requestWithVideoID(http.MethodGet, "/api/video/current-video", "current-video", strings.NewReader(``))
|
||||
rr := httptest.NewRecorder()
|
||||
(&Server{Catalog: cat}).handleVideoDetail(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got VideoDetailDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(got.RelatedVideos) != 6 {
|
||||
t.Fatalf("related videos = %d, want 6; items=%#v", len(got.RelatedVideos), got.RelatedVideos)
|
||||
}
|
||||
for _, item := range got.RelatedVideos {
|
||||
if !strings.HasPrefix(item.ID, "ready-related-") {
|
||||
t.Fatalf("related returned %q before ready thumbnails; items=%#v", item.ID, got.RelatedVideos)
|
||||
}
|
||||
if !strings.HasPrefix(item.Thumbnail, "https://thumb.example/") {
|
||||
t.Fatalf("thumbnail for %q = %q, want ready thumbnail URL", item.ID, item.Thumbnail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleHideVideoRemovesVideoFromPublicListAndDetail(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
@@ -41,35 +41,38 @@ func (c *Catalog) Close() error { return c.db.Close() }
|
||||
// ---------- Video ----------
|
||||
|
||||
type Video struct {
|
||||
ID string `json:"id"`
|
||||
DriveID string `json:"driveId"`
|
||||
FileID string `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
ContentHash string `json:"contentHash"`
|
||||
ParentID string `json:"parentId"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Tags []string `json:"tags"`
|
||||
DurationSeconds int `json:"durationSeconds"`
|
||||
Size int64 `json:"size"`
|
||||
Ext string `json:"ext"`
|
||||
Quality string `json:"quality"`
|
||||
ThumbnailURL string `json:"thumbnailUrl"`
|
||||
PreviewFileID string `json:"previewFileId"`
|
||||
PreviewLocal string `json:"previewLocal"`
|
||||
PreviewStatus string `json:"previewStatus"`
|
||||
Views int `json:"views"`
|
||||
Favorites int `json:"favorites"`
|
||||
Comments int `json:"comments"`
|
||||
Likes int `json:"likes"`
|
||||
Dislikes int `json:"dislikes"`
|
||||
Category string `json:"category"`
|
||||
Hidden bool `json:"hidden"`
|
||||
Badges []string `json:"badges"`
|
||||
Description string `json:"description"`
|
||||
PublishedAt time.Time `json:"publishedAt"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
ID string `json:"id"`
|
||||
DriveID string `json:"driveId"`
|
||||
FileID string `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
ContentHash string `json:"contentHash"`
|
||||
SampledSHA256 string `json:"sampledSha256"`
|
||||
FingerprintStatus string `json:"fingerprintStatus"`
|
||||
FingerprintError string `json:"fingerprintError"`
|
||||
ParentID string `json:"parentId"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Tags []string `json:"tags"`
|
||||
DurationSeconds int `json:"durationSeconds"`
|
||||
Size int64 `json:"size"`
|
||||
Ext string `json:"ext"`
|
||||
Quality string `json:"quality"`
|
||||
ThumbnailURL string `json:"thumbnailUrl"`
|
||||
PreviewFileID string `json:"previewFileId"`
|
||||
PreviewLocal string `json:"previewLocal"`
|
||||
PreviewStatus string `json:"previewStatus"`
|
||||
Views int `json:"views"`
|
||||
Favorites int `json:"favorites"`
|
||||
Comments int `json:"comments"`
|
||||
Likes int `json:"likes"`
|
||||
Dislikes int `json:"dislikes"`
|
||||
Category string `json:"category"`
|
||||
Hidden bool `json:"hidden"`
|
||||
Badges []string `json:"badges"`
|
||||
Description string `json:"description"`
|
||||
PublishedAt time.Time `json:"publishedAt"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
func (c *Catalog) UpsertVideo(ctx context.Context, v *Video) error {
|
||||
@@ -109,6 +112,18 @@ ON CONFLICT(id) DO UPDATE SET
|
||||
WHEN excluded.content_hash != '' THEN excluded.content_hash
|
||||
ELSE videos.content_hash
|
||||
END,
|
||||
sampled_sha256 = CASE
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN ''
|
||||
ELSE videos.sampled_sha256
|
||||
END,
|
||||
fingerprint_status = CASE
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN 'pending'
|
||||
ELSE COALESCE(videos.fingerprint_status, 'pending')
|
||||
END,
|
||||
fingerprint_error = CASE
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN ''
|
||||
ELSE COALESCE(videos.fingerprint_error, '')
|
||||
END,
|
||||
duration_seconds= excluded.duration_seconds,
|
||||
size_bytes = excluded.size_bytes,
|
||||
ext = excluded.ext,
|
||||
@@ -199,8 +214,8 @@ func (c *Catalog) MigrateVideoToDrive(ctx context.Context, videoID, newDriveID,
|
||||
}
|
||||
|
||||
// ListVideosByDriveID 列出指定 drive 下所有未隐藏的视频,按 published_at 倒序。
|
||||
// 给 spider91 → PikPak 迁移 worker 用:扫描 spider91 drive 下所有视频,
|
||||
// 检查哪些还有本地文件,依次上传到 PikPak。
|
||||
// 给 spider91 → 115/PikPak 迁移 worker 用:扫描 spider91 drive 下所有视频,
|
||||
// 检查哪些还有本地文件,依次上传到目标盘。
|
||||
func (c *Catalog) ListVideosByDriveID(ctx context.Context, driveID string, limit int) ([]*Video, error) {
|
||||
if driveID == "" {
|
||||
return nil, fmt.Errorf("catalog: list videos by drive: empty drive id")
|
||||
@@ -486,8 +501,14 @@ func (c *Catalog) ListVideosByThumbnailStatus(ctx context.Context, driveID, stat
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ListVideosNeedingThumbnail returns videos that still need a thumbnail attempt.
|
||||
// ListVideosNeedingThumbnail returns videos that still need thumbnail-worker work.
|
||||
// Besides missing thumbnails, this includes videos with an existing thumbnail but
|
||||
// missing duration metadata, because the thumbnail worker probes duration while
|
||||
// it already has a stream link.
|
||||
// Failed thumbnails are reported separately and should not block teaser generation.
|
||||
// Videos whose local assets were cleared because they are fingerprint duplicates
|
||||
// stay pending in the DB, but uniqueVideoWhereSQL keeps them out of this queue
|
||||
// while their canonical sibling still exists.
|
||||
func (c *Catalog) ListVideosNeedingThumbnail(ctx context.Context, driveID string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10000
|
||||
@@ -495,8 +516,11 @@ func (c *Catalog) ListVideosNeedingThumbnail(ctx context.Context, driveID string
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND COALESCE(thumbnail_url, '') = ''
|
||||
AND COALESCE(thumbnail_status, 'pending') != 'failed'
|
||||
AND (
|
||||
COALESCE(thumbnail_url, '') = ''
|
||||
OR COALESCE(duration_seconds, 0) <= 0
|
||||
)
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('failed', 'skipped')
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
AND `+uniqueVideoWhereSQL+`
|
||||
ORDER BY created_at ASC
|
||||
@@ -522,8 +546,11 @@ func (c *Catalog) CountVideosNeedingThumbnail(ctx context.Context, driveID strin
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND COALESCE(thumbnail_url, '') = ''
|
||||
AND COALESCE(thumbnail_status, 'pending') != 'failed'
|
||||
AND (
|
||||
COALESCE(thumbnail_url, '') = ''
|
||||
OR COALESCE(duration_seconds, 0) <= 0
|
||||
)
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('failed', 'skipped')
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
AND `+uniqueVideoWhereSQL,
|
||||
driveID).Scan(&count)
|
||||
@@ -668,14 +695,70 @@ func (c *Catalog) FindVideoByFileSignature(ctx context.Context, fileName string,
|
||||
return scanVideo(row)
|
||||
}
|
||||
|
||||
func (c *Catalog) ListVideosNeedingFingerprint(ctx context.Context, driveID string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10000
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND size_bytes > 0
|
||||
AND COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = 'pending'
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT ?`,
|
||||
driveID, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) UpdateVideoFingerprint(ctx context.Context, id, sampledSHA256, status, errText string) error {
|
||||
sampledSHA256 = normalizeContentHash(sampledSHA256)
|
||||
if status == "" {
|
||||
status = "pending"
|
||||
}
|
||||
if len(errText) > 500 {
|
||||
errText = errText[:500]
|
||||
}
|
||||
res, err := c.db.ExecContext(ctx,
|
||||
`UPDATE videos
|
||||
SET sampled_sha256 = ?,
|
||||
fingerprint_status = ?,
|
||||
fingerprint_error = ?,
|
||||
updated_at = ?
|
||||
WHERE id = ?`,
|
||||
sampledSHA256, status, errText, time.Now().UnixMilli(), id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rows, err := res.RowsAffected(); err == nil && rows == 0 {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type ListParams struct {
|
||||
Keyword string
|
||||
DriveID string
|
||||
Tag string
|
||||
Category string
|
||||
Sort string // latest | hot | week | long
|
||||
Page int
|
||||
PageSize int
|
||||
Keyword string
|
||||
DriveID string
|
||||
Tag string
|
||||
Category string
|
||||
Sort string // latest | hot | week | long
|
||||
ThumbnailReadyOnly bool
|
||||
PreferReadyThumbnails bool
|
||||
Page int
|
||||
PageSize int
|
||||
}
|
||||
|
||||
func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int, error) {
|
||||
@@ -710,21 +793,29 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
|
||||
where = append(where, "category = ?")
|
||||
args = append(args, p.Category)
|
||||
}
|
||||
if p.ThumbnailReadyOnly {
|
||||
where = append(where, "COALESCE(thumbnail_url, '') != ''")
|
||||
}
|
||||
where = append(where, "COALESCE(hidden, 0) = 0")
|
||||
where = append(where, uniqueVideoWhereSQL)
|
||||
|
||||
whereSQL := ""
|
||||
whereSQL = " WHERE " + strings.Join(where, " AND ")
|
||||
|
||||
orderBy := " ORDER BY published_at DESC"
|
||||
readyOrderPrefix := ""
|
||||
if p.PreferReadyThumbnails {
|
||||
readyOrderPrefix = "CASE WHEN COALESCE(thumbnail_url, '') != '' THEN 0 ELSE 1 END, "
|
||||
}
|
||||
|
||||
orderBy := " ORDER BY " + readyOrderPrefix + "published_at DESC"
|
||||
switch p.Sort {
|
||||
case "hot":
|
||||
// 热度 = 点赞数,点赞相同按最新
|
||||
orderBy = " ORDER BY likes DESC, published_at DESC"
|
||||
orderBy = " ORDER BY " + readyOrderPrefix + "likes DESC, published_at DESC"
|
||||
case "week":
|
||||
orderBy = " ORDER BY likes DESC"
|
||||
orderBy = " ORDER BY " + readyOrderPrefix + "likes DESC"
|
||||
case "long":
|
||||
orderBy = " ORDER BY duration_seconds DESC"
|
||||
orderBy = " ORDER BY " + readyOrderPrefix + "duration_seconds DESC"
|
||||
}
|
||||
|
||||
// count
|
||||
@@ -842,6 +933,12 @@ type DriveThumbnailCounts struct {
|
||||
Failed int
|
||||
}
|
||||
|
||||
type DriveFingerprintCounts struct {
|
||||
Ready int
|
||||
Pending int
|
||||
Failed int
|
||||
}
|
||||
|
||||
func (c *Catalog) CountTeasersByDrive(ctx context.Context) (map[string]DriveTeaserCounts, error) {
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT drive_id,
|
||||
@@ -904,6 +1001,52 @@ func (c *Catalog) CountThumbnailsByDrive(ctx context.Context) (map[string]DriveT
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) CountFingerprintsByDrive(ctx context.Context) (map[string]DriveFingerprintCounts, error) {
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT drive_id,
|
||||
COUNT(CASE WHEN COALESCE(sampled_sha256, '') != ''
|
||||
OR COALESCE(fingerprint_status, 'pending') = 'ready' THEN 1 END) AS ready_count,
|
||||
COUNT(CASE WHEN size_bytes > 0
|
||||
AND COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = 'pending' THEN 1 END) AS pending_count,
|
||||
COUNT(CASE WHEN COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = 'failed' THEN 1 END) AS failed_count
|
||||
FROM videos
|
||||
WHERE COALESCE(hidden, 0) = 0
|
||||
GROUP BY drive_id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make(map[string]DriveFingerprintCounts)
|
||||
for rows.Next() {
|
||||
var driveID string
|
||||
var counts DriveFingerprintCounts
|
||||
if err := rows.Scan(&driveID, &counts.Ready, &counts.Pending, &counts.Failed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[driveID] = counts
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) CountVideosNeedingFingerprint(ctx context.Context, driveID string) (int, error) {
|
||||
var count int
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND size_bytes > 0
|
||||
AND COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = 'pending'
|
||||
AND COALESCE(hidden, 0) = 0`,
|
||||
driveID).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
type LocalMediaRef struct {
|
||||
DriveID string
|
||||
VideoID string
|
||||
@@ -933,6 +1076,124 @@ func (c *Catalog) ListLocalMediaRefs(ctx context.Context) ([]LocalMediaRef, erro
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// DuplicateAssetCleanupCandidate points at a non-canonical video in a
|
||||
// size+sampled_sha256 duplicate group that still owns generated local assets.
|
||||
// The cleanup job uses this to remove duplicate thumbnails/teasers without
|
||||
// touching the original cloud file or deleting the catalog row.
|
||||
type DuplicateAssetCleanupCandidate struct {
|
||||
VideoID string
|
||||
DriveID string
|
||||
Title string
|
||||
PreviewLocal string
|
||||
ThumbnailURL string
|
||||
CanonicalID string
|
||||
SampledSHA256 string
|
||||
Size int64
|
||||
}
|
||||
|
||||
// ListDuplicateAssetCleanupCandidates returns duplicate videos whose own local
|
||||
// generated assets can be cleared. A group canonical is the same representative
|
||||
// used by uniqueVideoWhereSQL: earliest created_at, then lexicographically
|
||||
// smallest id.
|
||||
func (c *Catalog) ListDuplicateAssetCleanupCandidates(ctx context.Context, limit int) ([]DuplicateAssetCleanupCandidate, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10000
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
WITH canonical AS (
|
||||
SELECT v.id, v.size_bytes, v.sampled_sha256
|
||||
FROM videos v
|
||||
WHERE v.size_bytes > 0
|
||||
AND COALESCE(v.sampled_sha256, '') != ''
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM videos earlier
|
||||
WHERE earlier.size_bytes = v.size_bytes
|
||||
AND earlier.sampled_sha256 = v.sampled_sha256
|
||||
AND COALESCE(earlier.sampled_sha256, '') != ''
|
||||
AND earlier.size_bytes > 0
|
||||
AND (
|
||||
earlier.created_at < v.created_at
|
||||
OR (earlier.created_at = v.created_at AND earlier.id < v.id)
|
||||
)
|
||||
)
|
||||
)
|
||||
SELECT dup.id,
|
||||
dup.drive_id,
|
||||
dup.title,
|
||||
COALESCE(dup.preview_local, ''),
|
||||
COALESCE(dup.thumbnail_url, ''),
|
||||
canonical.id,
|
||||
dup.sampled_sha256,
|
||||
dup.size_bytes
|
||||
FROM videos dup
|
||||
JOIN canonical
|
||||
ON canonical.size_bytes = dup.size_bytes
|
||||
AND canonical.sampled_sha256 = dup.sampled_sha256
|
||||
WHERE dup.id != canonical.id
|
||||
AND dup.size_bytes > 0
|
||||
AND COALESCE(dup.sampled_sha256, '') != ''
|
||||
AND (
|
||||
COALESCE(dup.preview_local, '') != ''
|
||||
OR COALESCE(dup.thumbnail_url, '') = '/p/thumb/' || dup.id
|
||||
)
|
||||
ORDER BY dup.created_at ASC, dup.id ASC
|
||||
LIMIT ?`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []DuplicateAssetCleanupCandidate
|
||||
for rows.Next() {
|
||||
var item DuplicateAssetCleanupCandidate
|
||||
if err := rows.Scan(
|
||||
&item.VideoID,
|
||||
&item.DriveID,
|
||||
&item.Title,
|
||||
&item.PreviewLocal,
|
||||
&item.ThumbnailURL,
|
||||
&item.CanonicalID,
|
||||
&item.SampledSHA256,
|
||||
&item.Size,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ClearGeneratedAssets clears DB references to generated local assets for a
|
||||
// video. The statuses go back to pending so the video can regenerate assets if
|
||||
// it later becomes the canonical item after its older duplicate is removed.
|
||||
func (c *Catalog) ClearGeneratedAssets(ctx context.Context, videoID string, clearPreview, clearThumbnail bool) error {
|
||||
parts := []string{}
|
||||
args := []any{}
|
||||
if clearPreview {
|
||||
parts = append(parts, "preview_file_id = ''", "preview_local = ''", "preview_status = 'pending'")
|
||||
}
|
||||
if clearThumbnail {
|
||||
parts = append(parts, "thumbnail_url = ''", "thumbnail_status = 'pending'")
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return nil
|
||||
}
|
||||
parts = append(parts, "updated_at = ?")
|
||||
args = append(args, time.Now().UnixMilli(), videoID)
|
||||
res, err := c.db.ExecContext(ctx, `UPDATE videos SET `+strings.Join(parts, ", ")+` WHERE id = ?`, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rows, err := res.RowsAffected(); err == nil && rows == 0 {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------- Drive ----------
|
||||
|
||||
type Drive struct {
|
||||
@@ -1161,7 +1422,9 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.upd
|
||||
// ---------- helpers ----------
|
||||
|
||||
const allVideoCols = `
|
||||
id, drive_id, file_id, COALESCE(file_name, ''), COALESCE(content_hash, ''), COALESCE(parent_id, ''), title, COALESCE(author, ''), COALESCE(tags, '[]'),
|
||||
id, drive_id, file_id, COALESCE(file_name, ''), COALESCE(content_hash, ''),
|
||||
COALESCE(sampled_sha256, ''), COALESCE(fingerprint_status, 'pending'), COALESCE(fingerprint_error, ''),
|
||||
COALESCE(parent_id, ''), title, COALESCE(author, ''), COALESCE(tags, '[]'),
|
||||
duration_seconds, size_bytes, COALESCE(ext, ''), COALESCE(quality, ''), COALESCE(thumbnail_url, ''),
|
||||
COALESCE(preview_file_id, ''), COALESCE(preview_local, ''), COALESCE(preview_status, 'pending'),
|
||||
views, favorites, comments, likes, dislikes,
|
||||
@@ -1180,6 +1443,20 @@ const uniqueVideoWhereSQL = `((COALESCE(videos.content_hash, '') = ''
|
||||
OR (dup.created_at = videos.created_at AND dup.id < videos.id)
|
||||
)
|
||||
))
|
||||
AND (COALESCE(videos.sampled_sha256, '') = ''
|
||||
OR videos.size_bytes <= 0
|
||||
OR NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM videos AS dup
|
||||
WHERE dup.sampled_sha256 = videos.sampled_sha256
|
||||
AND dup.size_bytes = videos.size_bytes
|
||||
AND COALESCE(dup.sampled_sha256, '') != ''
|
||||
AND dup.size_bytes > 0
|
||||
AND (
|
||||
dup.created_at < videos.created_at
|
||||
OR (dup.created_at = videos.created_at AND dup.id < videos.id)
|
||||
)
|
||||
))
|
||||
AND (COALESCE(videos.file_name, '') = ''
|
||||
OR videos.size_bytes <= 0
|
||||
OR NOT EXISTS (
|
||||
@@ -1205,7 +1482,9 @@ func scanVideo(row rowScanner) (*Video, error) {
|
||||
var publishedAt, createdAt, updatedAt int64
|
||||
var hidden int
|
||||
err := row.Scan(
|
||||
&v.ID, &v.DriveID, &v.FileID, &v.FileName, &v.ContentHash, &v.ParentID, &v.Title, &v.Author, &tagsJSON,
|
||||
&v.ID, &v.DriveID, &v.FileID, &v.FileName, &v.ContentHash,
|
||||
&v.SampledSHA256, &v.FingerprintStatus, &v.FingerprintError,
|
||||
&v.ParentID, &v.Title, &v.Author, &tagsJSON,
|
||||
&v.DurationSeconds, &v.Size, &v.Ext, &v.Quality, &v.ThumbnailURL,
|
||||
&v.PreviewFileID, &v.PreviewLocal, &v.PreviewStatus,
|
||||
&v.Views, &v.Favorites, &v.Comments, &v.Likes, &v.Dislikes,
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestListVideosDeduplicatesBySampledSHA256(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{
|
||||
ID: "drive-a-file-a",
|
||||
DriveID: "drive-a",
|
||||
FileID: "file-a",
|
||||
FileName: "first-name.mp4",
|
||||
Title: "First",
|
||||
Size: 1234,
|
||||
PublishedAt: now.Add(-time.Minute),
|
||||
CreatedAt: now.Add(-time.Minute),
|
||||
UpdatedAt: now.Add(-time.Minute),
|
||||
},
|
||||
{
|
||||
ID: "drive-b-file-b",
|
||||
DriveID: "drive-b",
|
||||
FileID: "file-b",
|
||||
FileName: "second-name.mp4",
|
||||
Title: "Second",
|
||||
Size: 1234,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("upsert %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list before fingerprint: %v", err)
|
||||
}
|
||||
if total != 2 || len(items) != 2 {
|
||||
t.Fatalf("before fingerprint total=%d len=%d, want 2", total, len(items))
|
||||
}
|
||||
|
||||
const sampled = "abc123"
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "drive-a-file-a", sampled, "ready", ""); err != nil {
|
||||
t.Fatalf("update a fingerprint: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "drive-b-file-b", sampled, "ready", ""); err != nil {
|
||||
t.Fatalf("update b fingerprint: %v", err)
|
||||
}
|
||||
|
||||
items, total, err = cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list after fingerprint: %v", err)
|
||||
}
|
||||
if total != 1 || len(items) != 1 {
|
||||
t.Fatalf("after fingerprint total=%d len=%d, want 1", total, len(items))
|
||||
}
|
||||
if items[0].ID != "drive-a-file-a" {
|
||||
t.Fatalf("canonical id = %q, want earliest created video", items[0].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuplicateAssetCleanupCandidates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
base := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
|
||||
videos := []*Video{
|
||||
{
|
||||
ID: "drive-a-canonical",
|
||||
DriveID: "drive-a",
|
||||
FileID: "file-a",
|
||||
FileName: "canonical.mp4",
|
||||
Title: "Canonical",
|
||||
Size: 1234,
|
||||
ThumbnailURL: "/p/thumb/drive-a-canonical",
|
||||
PreviewLocal: "/tmp/previews/canonical.mp4",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: base,
|
||||
CreatedAt: base,
|
||||
UpdatedAt: base,
|
||||
},
|
||||
{
|
||||
ID: "drive-b-duplicate",
|
||||
DriveID: "drive-b",
|
||||
FileID: "file-b",
|
||||
FileName: "duplicate.mp4",
|
||||
Title: "Duplicate",
|
||||
Size: 1234,
|
||||
ThumbnailURL: "/p/thumb/drive-b-duplicate",
|
||||
PreviewLocal: "/tmp/previews/duplicate.mp4",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: base.Add(time.Second),
|
||||
CreatedAt: base.Add(time.Second),
|
||||
UpdatedAt: base.Add(time.Second),
|
||||
},
|
||||
{
|
||||
ID: "drive-c-remote-thumb",
|
||||
DriveID: "drive-c",
|
||||
FileID: "file-c",
|
||||
FileName: "remote-thumb.mp4",
|
||||
Title: "Remote Thumbnail",
|
||||
Size: 1234,
|
||||
ThumbnailURL: "https://thumb.example/file-c.jpg",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: base.Add(2 * time.Second),
|
||||
CreatedAt: base.Add(2 * time.Second),
|
||||
UpdatedAt: base.Add(2 * time.Second),
|
||||
},
|
||||
}
|
||||
for _, v := range videos {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
const sampled = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
for _, v := range videos {
|
||||
if err := cat.UpdateVideoFingerprint(ctx, v.ID, sampled, "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, err := cat.ListDuplicateAssetCleanupCandidates(ctx, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list cleanup candidates: %v", err)
|
||||
}
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("candidates = %#v, want only local duplicate", items)
|
||||
}
|
||||
item := items[0]
|
||||
if item.VideoID != "drive-b-duplicate" || item.CanonicalID != "drive-a-canonical" {
|
||||
t.Fatalf("candidate = %#v, want duplicate with canonical", item)
|
||||
}
|
||||
|
||||
if err := cat.ClearGeneratedAssets(ctx, item.VideoID, true, true); err != nil {
|
||||
t.Fatalf("clear generated assets: %v", err)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, item.VideoID)
|
||||
if err != nil {
|
||||
t.Fatalf("get duplicate: %v", err)
|
||||
}
|
||||
if got.PreviewLocal != "" || got.PreviewStatus != "pending" {
|
||||
t.Fatalf("preview after cleanup local=%q status=%q, want empty pending", got.PreviewLocal, got.PreviewStatus)
|
||||
}
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail after cleanup = %q, want empty", got.ThumbnailURL)
|
||||
}
|
||||
var thumbStatus string
|
||||
if err := cat.db.QueryRowContext(ctx, `SELECT thumbnail_status FROM videos WHERE id = ?`, item.VideoID).Scan(&thumbStatus); err != nil {
|
||||
t.Fatalf("query thumbnail status: %v", err)
|
||||
}
|
||||
if thumbStatus != "pending" {
|
||||
t.Fatalf("thumbnail_status = %q, want pending", thumbStatus)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,9 @@ CREATE TABLE IF NOT EXISTS videos (
|
||||
file_id TEXT NOT NULL,
|
||||
file_name TEXT DEFAULT '', -- 网盘侧原始文件名,用于同名同大小去重
|
||||
content_hash TEXT DEFAULT '',
|
||||
sampled_sha256 TEXT DEFAULT '', -- 跨网盘统一采样指纹(size + sampled bytes)
|
||||
fingerprint_status TEXT DEFAULT 'pending', -- pending / ready / failed
|
||||
fingerprint_error TEXT DEFAULT '',
|
||||
parent_id TEXT,
|
||||
title TEXT NOT NULL,
|
||||
author TEXT,
|
||||
@@ -61,7 +64,7 @@ CREATE INDEX IF NOT EXISTS idx_video_tags_video ON video_tags(video_id);
|
||||
-- 网盘账户
|
||||
CREATE TABLE IF NOT EXISTS drives (
|
||||
id TEXT PRIMARY KEY,
|
||||
kind TEXT NOT NULL, -- quark / p115 / pikpak / wopan / onedrive / spider91
|
||||
kind TEXT NOT NULL, -- quark / p115 / pikpak / wopan / onedrive / localstorage / spider91
|
||||
name TEXT NOT NULL,
|
||||
root_id TEXT NOT NULL DEFAULT '0',
|
||||
scan_root_id TEXT, -- 扫描起点(默认 root_id)
|
||||
|
||||
@@ -43,6 +43,15 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "content_hash", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "sampled_sha256", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "fingerprint_status", "TEXT DEFAULT 'pending'"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "fingerprint_error", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "file_name", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -83,6 +92,9 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_content_hash ON videos(content_hash)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_sampled_sha256 ON videos(size_bytes, sampled_sha256)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_hidden ON videos(hidden)`); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -7,6 +7,90 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestListVideosNeedingThumbnailIncludesExistingThumbnailMissingDuration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
videos := []*Video{
|
||||
{
|
||||
ID: "duration-only",
|
||||
DriveID: "drive",
|
||||
FileID: "file-duration-only",
|
||||
Title: "Duration Only",
|
||||
ThumbnailURL: "/p/thumb/duration-only",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "complete",
|
||||
DriveID: "drive",
|
||||
FileID: "file-complete",
|
||||
Title: "Complete",
|
||||
DurationSeconds: 12,
|
||||
ThumbnailURL: "/p/thumb/complete",
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
{
|
||||
ID: "missing-thumb",
|
||||
DriveID: "drive",
|
||||
FileID: "file-missing-thumb",
|
||||
Title: "Missing Thumb",
|
||||
DurationSeconds: 18,
|
||||
PublishedAt: now.Add(2 * time.Second),
|
||||
CreatedAt: now.Add(2 * time.Second),
|
||||
UpdatedAt: now.Add(2 * time.Second),
|
||||
},
|
||||
{
|
||||
ID: "failed",
|
||||
DriveID: "drive",
|
||||
FileID: "file-failed",
|
||||
Title: "Failed",
|
||||
PublishedAt: now.Add(3 * time.Second),
|
||||
CreatedAt: now.Add(3 * time.Second),
|
||||
UpdatedAt: now.Add(3 * time.Second),
|
||||
},
|
||||
}
|
||||
for _, v := range videos {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
if err := cat.UpdateVideoMeta(ctx, "failed", VideoMetaPatch{ThumbnailStatus: "failed"}); err != nil {
|
||||
t.Fatalf("mark failed thumbnail: %v", err)
|
||||
}
|
||||
|
||||
items, err := cat.ListVideosNeedingThumbnail(ctx, "drive", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list videos needing thumbnail: %v", err)
|
||||
}
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("items = %#v, want duration-only and missing-thumb", items)
|
||||
}
|
||||
if items[0].ID != "duration-only" || items[1].ID != "missing-thumb" {
|
||||
t.Fatalf("item ids = %q, %q; want duration-only, missing-thumb", items[0].ID, items[1].ID)
|
||||
}
|
||||
|
||||
count, err := cat.CountVideosNeedingThumbnail(ctx, "drive")
|
||||
if err != nil {
|
||||
t.Fatalf("count videos needing thumbnail: %v", err)
|
||||
}
|
||||
if count != 2 {
|
||||
t.Fatalf("count = %d, want 2", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateTagAndClassifyAddsTagToMatchingExistingVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
@@ -581,6 +665,59 @@ func TestListVideosHidesDuplicateContentHashes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestListVideosCanFilterReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{
|
||||
ID: "ready-video",
|
||||
DriveID: "drive",
|
||||
FileID: "file-ready",
|
||||
Title: "Ready",
|
||||
ThumbnailURL: "/p/thumb/ready-video",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "pending-video",
|
||||
DriveID: "drive",
|
||||
FileID: "file-pending",
|
||||
Title: "Pending",
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed video %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{
|
||||
Page: 1, PageSize: 10, ThumbnailReadyOnly: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list videos: %v", err)
|
||||
}
|
||||
if total != 1 || len(items) != 1 {
|
||||
t.Fatalf("ready videos total=%d len=%d, want 1", total, len(items))
|
||||
}
|
||||
if items[0].ID != "ready-video" {
|
||||
t.Fatalf("ready video id = %q, want ready-video", items[0].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func sameStrings(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
|
||||
@@ -202,7 +202,7 @@ type Nightly struct {
|
||||
// 这里保留 yaml 中的静态定义,用于启动时预置盘。生产建议只在 DB 里维护。
|
||||
type Drive struct {
|
||||
ID string `yaml:"id"`
|
||||
Kind string `yaml:"kind"` // quark / p115 / pikpak / wopan / onedrive
|
||||
Kind string `yaml:"kind"` // quark / p115 / pikpak / wopan / onedrive / localstorage
|
||||
Name string `yaml:"name"`
|
||||
RootID string `yaml:"root_id"`
|
||||
Params map[string]string `yaml:"params,omitempty"`
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
// Drive 是多家网盘统一抽象。上层不区分盘,只区分 Kind。
|
||||
type Drive interface {
|
||||
// Kind 返回驱动代号:"quark" / "p115" / "pikpak" / "wopan" / "onedrive"
|
||||
// Kind 返回驱动代号:"quark" / "p115" / "pikpak" / "wopan" / "onedrive" / "localstorage"
|
||||
Kind() string
|
||||
|
||||
// ID 返回该盘在 catalog 中的唯一标识
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
// Package localstorage exposes an existing server-side directory as a Drive.
|
||||
package localstorage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
const Kind = "localstorage"
|
||||
|
||||
type Config struct {
|
||||
ID string
|
||||
RootPath string
|
||||
}
|
||||
|
||||
type Driver struct {
|
||||
id string
|
||||
rootPath string
|
||||
}
|
||||
|
||||
func New(c Config) *Driver {
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
rootPath: c.RootPath,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Kind() string { return Kind }
|
||||
|
||||
func (d *Driver) ID() string { return d.id }
|
||||
|
||||
func (d *Driver) RootID() string { return "/" }
|
||||
|
||||
func (d *Driver) Init(context.Context) error {
|
||||
root, err := d.root()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("localstorage: stat root: %w", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("localstorage: root is not a directory: %s", root)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error) {
|
||||
dir, rel, err := d.pathForID(dirID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]drives.Entry, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Symlinks can escape the configured root or create cycles. Keep the
|
||||
// local storage drive predictable by scanning real files/directories only.
|
||||
if entry.Type()&os.ModeSymlink != 0 {
|
||||
continue
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !info.IsDir() && !info.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
childRel := joinRel(rel, entry.Name())
|
||||
out = append(out, drives.Entry{
|
||||
ID: encodeRel(childRel),
|
||||
Name: entry.Name(),
|
||||
Size: sizeForEntry(info),
|
||||
IsDir: info.IsDir(),
|
||||
ParentID: idForRel(rel),
|
||||
ModTime: info.ModTime(),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
|
||||
p, rel, err := d.pathForID(fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &drives.Entry{
|
||||
ID: idForRel(rel),
|
||||
Name: filepath.Base(p),
|
||||
Size: sizeForEntry(info),
|
||||
IsDir: info.IsDir(),
|
||||
ParentID: idForRel(parentRel(rel)),
|
||||
ModTime: info.ModTime(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
p, _, err := d.pathForID(fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.IsDir() || !info.Mode().IsRegular() || info.Size() <= 0 {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return &drives.StreamLink{
|
||||
URL: p,
|
||||
Expires: time.Now().Add(24 * time.Hour),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) root() (string, error) {
|
||||
raw := strings.TrimSpace(d.rootPath)
|
||||
if raw == "" {
|
||||
return "", errors.New("localstorage: empty path")
|
||||
}
|
||||
raw = os.ExpandEnv(raw)
|
||||
if strings.HasPrefix(raw, "~") {
|
||||
if home, err := os.UserHomeDir(); err == nil && home != "" {
|
||||
switch {
|
||||
case raw == "~":
|
||||
raw = home
|
||||
case strings.HasPrefix(raw, "~/") || strings.HasPrefix(raw, `~\`):
|
||||
raw = filepath.Join(home, raw[2:])
|
||||
}
|
||||
}
|
||||
}
|
||||
return filepath.Abs(raw)
|
||||
}
|
||||
|
||||
func (d *Driver) pathForID(id string) (string, string, error) {
|
||||
root, err := d.root()
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
rel, err := decodeRel(id)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if rel == "" {
|
||||
return root, "", nil
|
||||
}
|
||||
p, err := filepath.Abs(filepath.Join(root, filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if p != root && !strings.HasPrefix(p, root+string(os.PathSeparator)) {
|
||||
return "", "", errors.New("localstorage: path escapes root")
|
||||
}
|
||||
return p, rel, nil
|
||||
}
|
||||
|
||||
func decodeRel(id string) (string, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" || id == "/" {
|
||||
return "", nil
|
||||
}
|
||||
raw, err := base64.RawURLEncoding.DecodeString(id)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("localstorage: invalid file id: %w", err)
|
||||
}
|
||||
rel := filepath.ToSlash(filepath.Clean(filepath.FromSlash(string(raw))))
|
||||
if rel == "." {
|
||||
return "", nil
|
||||
}
|
||||
if strings.HasPrefix(rel, "../") || rel == ".." || strings.HasPrefix(rel, "/") {
|
||||
return "", errors.New("localstorage: invalid relative path")
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
|
||||
func encodeRel(rel string) string {
|
||||
rel = filepath.ToSlash(filepath.Clean(filepath.FromSlash(rel)))
|
||||
if rel == "." || rel == "" {
|
||||
return "/"
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(rel))
|
||||
}
|
||||
|
||||
func idForRel(rel string) string {
|
||||
if rel == "" {
|
||||
return "/"
|
||||
}
|
||||
return encodeRel(rel)
|
||||
}
|
||||
|
||||
func joinRel(parent, name string) string {
|
||||
if parent == "" {
|
||||
return filepath.ToSlash(name)
|
||||
}
|
||||
return filepath.ToSlash(filepath.Join(filepath.FromSlash(parent), name))
|
||||
}
|
||||
|
||||
func parentRel(rel string) string {
|
||||
if rel == "" {
|
||||
return ""
|
||||
}
|
||||
parent := filepath.ToSlash(filepath.Dir(filepath.FromSlash(rel)))
|
||||
if parent == "." {
|
||||
return ""
|
||||
}
|
||||
return parent
|
||||
}
|
||||
|
||||
func sizeForEntry(info os.FileInfo) int64 {
|
||||
if info == nil || info.IsDir() {
|
||||
return 0
|
||||
}
|
||||
return info.Size()
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
@@ -0,0 +1,119 @@
|
||||
package localstorage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/scanner"
|
||||
)
|
||||
|
||||
func TestListEncodesRelativePathsAndStreamURLResolvesFile(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
sub := filepath.Join(root, "clips")
|
||||
if err := os.MkdirAll(sub, 0o755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
videoPath := filepath.Join(sub, "sample.mp4")
|
||||
if err := os.WriteFile(videoPath, []byte("video"), 0o644); err != nil {
|
||||
t.Fatalf("write video: %v", err)
|
||||
}
|
||||
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
if err := drv.Init(context.Background()); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
rootEntries, err := drv.List(context.Background(), drv.RootID())
|
||||
if err != nil {
|
||||
t.Fatalf("list root: %v", err)
|
||||
}
|
||||
if len(rootEntries) != 1 || !rootEntries[0].IsDir {
|
||||
t.Fatalf("root entries = %#v, want one directory", rootEntries)
|
||||
}
|
||||
if strings.Contains(rootEntries[0].ID, "/") {
|
||||
t.Fatalf("encoded dir id contains slash: %q", rootEntries[0].ID)
|
||||
}
|
||||
|
||||
fileEntries, err := drv.List(context.Background(), rootEntries[0].ID)
|
||||
if err != nil {
|
||||
t.Fatalf("list subdir: %v", err)
|
||||
}
|
||||
if len(fileEntries) != 1 || fileEntries[0].Name != "sample.mp4" {
|
||||
t.Fatalf("file entries = %#v, want sample.mp4", fileEntries)
|
||||
}
|
||||
if strings.Contains(fileEntries[0].ID, "/") {
|
||||
t.Fatalf("encoded file id contains slash: %q", fileEntries[0].ID)
|
||||
}
|
||||
|
||||
link, err := drv.StreamURL(context.Background(), fileEntries[0].ID)
|
||||
if err != nil {
|
||||
t.Fatalf("stream url: %v", err)
|
||||
}
|
||||
if link.URL != videoPath {
|
||||
t.Fatalf("url = %q, want %q", link.URL, videoPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRejectsEscapingID(t *testing.T) {
|
||||
drv := New(Config{ID: "local", RootPath: t.TempDir()})
|
||||
escaped := base64.RawURLEncoding.EncodeToString([]byte("../secret.mp4"))
|
||||
|
||||
_, err := drv.StreamURL(context.Background(), escaped)
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid relative path") {
|
||||
t.Fatalf("error = %v, want invalid relative path", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitRequiresExistingDirectory(t *testing.T) {
|
||||
drv := New(Config{ID: "local", RootPath: filepath.Join(t.TempDir(), "missing")})
|
||||
|
||||
err := drv.Init(context.Background())
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "stat root") {
|
||||
t.Fatalf("error = %v, want stat root failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerPersistsLocalStorageVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(root, "collection"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir collection: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "collection", "clip.mp4"), []byte("video"), 0o644); err != nil {
|
||||
t.Fatalf("write video: %v", err)
|
||||
}
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
sc := scanner.New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
stats, err := sc.Run(ctx, drv.RootID())
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Added != 1 {
|
||||
t.Fatalf("added = %d, want 1", stats.Added)
|
||||
}
|
||||
|
||||
fileID := encodeRel("collection/clip.mp4")
|
||||
got, err := cat.GetVideo(ctx, Kind+"-local-"+fileID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DriveID != "local" || got.FileID != fileID || got.Category != "collection" {
|
||||
t.Fatalf("video = %#v, want local drive video in collection", got)
|
||||
}
|
||||
}
|
||||
@@ -3,14 +3,19 @@ package onedrive
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-resty/resty/v2"
|
||||
@@ -18,8 +23,17 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
maxSmallUploadSize = 250 * 1024 * 1024
|
||||
defaultRenewAPIURL = "https://api.oplist.org/onedrive/renewapi"
|
||||
maxSmallUploadSize = 250 * 1024 * 1024
|
||||
defaultUploadSessionChunk = 10 * 1024 * 1024
|
||||
uploadSessionRetryAttempts = 3
|
||||
defaultRenewAPIURL = "https://api.oplist.org/onedrive/renewapi"
|
||||
onedriveListCooldown = 5 * time.Minute
|
||||
onedriveListInterval = 1 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
smallUploadThreshold = int64(maxSmallUploadSize)
|
||||
uploadSessionChunk = int64(defaultUploadSessionChunk)
|
||||
)
|
||||
|
||||
type Driver struct {
|
||||
@@ -34,6 +48,11 @@ type Driver struct {
|
||||
renewAPIURL string
|
||||
client *resty.Client
|
||||
onTokenUpdate func(access, refresh string)
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
listInterval time.Duration
|
||||
listCooldown time.Duration
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -85,6 +104,8 @@ func New(c Config) *Driver {
|
||||
client: resty.New().
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
listInterval: onedriveListInterval,
|
||||
listCooldown: onedriveListCooldown,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,10 +127,16 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
|
||||
if dirID == "" {
|
||||
dirID = d.rootID
|
||||
}
|
||||
d.listMu.Lock()
|
||||
defer d.listMu.Unlock()
|
||||
|
||||
nextLink := d.childrenURL(dirID)
|
||||
first := true
|
||||
out := make([]drives.Entry, 0)
|
||||
for nextLink != "" {
|
||||
if err := d.waitForListSlotLocked(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var resp filesResp
|
||||
err := d.request(ctx, nextLink, http.MethodGet, func(req *resty.Request) {
|
||||
if first {
|
||||
@@ -120,6 +147,19 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
|
||||
}
|
||||
}, &resp)
|
||||
if err != nil {
|
||||
if wait, ok := drives.RateLimitRetryAfter(err); ok {
|
||||
if wait <= 0 {
|
||||
wait = d.listCooldown
|
||||
if wait <= 0 {
|
||||
wait = onedriveListCooldown
|
||||
}
|
||||
}
|
||||
log.Printf("[onedrive] list cooling down drive=%s dir=%s cooldown=%s err=%v", d.id, dirID, wait, err)
|
||||
if err := sleepContext(ctx, wait); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("onedrive list: %w", err)
|
||||
}
|
||||
for _, item := range resp.Value {
|
||||
@@ -131,6 +171,36 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *Driver) waitForListSlotLocked(ctx context.Context) error {
|
||||
if d.listInterval <= 0 || d.lastListAt.IsZero() {
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
next := d.lastListAt.Add(d.listInterval)
|
||||
now := time.Now()
|
||||
if now.Before(next) {
|
||||
if err := sleepContext(ctx, next.Sub(now)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, d time.Duration) error {
|
||||
if d <= 0 {
|
||||
return ctx.Err()
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
|
||||
var item graphItem
|
||||
if err := d.request(ctx, d.itemURL(fileID), http.MethodGet, nil, &item); err != nil {
|
||||
@@ -156,15 +226,49 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
res, err := d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return res.FileID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
parentID, name, err := d.normalizeUploadArgs(parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
threshold := smallUploadThreshold
|
||||
if threshold <= 0 {
|
||||
threshold = maxSmallUploadSize
|
||||
}
|
||||
if size <= threshold {
|
||||
return d.uploadSmallAndReportHash(ctx, parentID, name, r, size, threshold)
|
||||
}
|
||||
return d.uploadSessionAndReportHash(ctx, parentID, name, r, size)
|
||||
}
|
||||
|
||||
func (d *Driver) normalizeUploadArgs(parentID, name string, r io.Reader, size int64) (string, string, error) {
|
||||
if r == nil {
|
||||
return "", "", errors.New("onedrive upload: body is required")
|
||||
}
|
||||
if size < 0 {
|
||||
return "", "", fmt.Errorf("onedrive upload: invalid size %d", size)
|
||||
}
|
||||
if parentID == "" {
|
||||
parentID = d.rootID
|
||||
}
|
||||
if size > maxSmallUploadSize {
|
||||
return "", fmt.Errorf("onedrive upload: files over %d bytes require upload session", maxSmallUploadSize)
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return "", "", errors.New("onedrive upload: empty file name")
|
||||
}
|
||||
data, err := readSmallUpload(r)
|
||||
return parentID, name, nil
|
||||
}
|
||||
|
||||
func (d *Driver) uploadSmallAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size, limit int64) (UploadResult, error) {
|
||||
data, hash, actualSize, err := readSmallUpload(r, size, limit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return UploadResult{}, err
|
||||
}
|
||||
u := fmt.Sprintf("%s/items/%s:/%s:/content", d.driveBaseURL(), url.PathEscape(parentID), url.PathEscape(name))
|
||||
var item graphItem
|
||||
@@ -173,26 +277,159 @@ func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader,
|
||||
req.SetContentLength(true)
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("onedrive upload: %w", err)
|
||||
return UploadResult{}, fmt.Errorf("onedrive upload: %w", err)
|
||||
}
|
||||
if item.ID == "" {
|
||||
return "", errors.New("onedrive upload: empty item id")
|
||||
return UploadResult{}, errors.New("onedrive upload: empty item id")
|
||||
}
|
||||
return item.ID, nil
|
||||
return UploadResult{FileID: item.ID, Hash: hash, Size: actualSize}, nil
|
||||
}
|
||||
|
||||
func readSmallUpload(r io.Reader) ([]byte, error) {
|
||||
if r == nil {
|
||||
return nil, errors.New("onedrive upload: body is required")
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(r, maxSmallUploadSize+1))
|
||||
func (d *Driver) uploadSessionAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
session, err := d.createUploadSession(ctx, parentID, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("onedrive upload: read body: %w", err)
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if len(data) > maxSmallUploadSize {
|
||||
return nil, fmt.Errorf("onedrive upload: files over %d bytes require upload session", maxSmallUploadSize)
|
||||
if strings.TrimSpace(session.UploadURL) == "" {
|
||||
return UploadResult{}, errors.New("onedrive upload session: empty upload url")
|
||||
}
|
||||
return data, nil
|
||||
|
||||
chunkSize := uploadSessionChunk
|
||||
if chunkSize <= 0 {
|
||||
chunkSize = defaultUploadSessionChunk
|
||||
}
|
||||
buf := make([]byte, int(chunkSize))
|
||||
hasher := sha1.New()
|
||||
var finalItem graphItem
|
||||
var offset int64
|
||||
for offset < size {
|
||||
partSize := minInt64(chunkSize, size-offset)
|
||||
chunk := buf[:int(partSize)]
|
||||
n, err := io.ReadFull(r, chunk)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return UploadResult{}, fmt.Errorf("onedrive upload: size mismatch: declared %d, copied %d", size, offset+int64(n))
|
||||
}
|
||||
return UploadResult{}, fmt.Errorf("onedrive upload: read body: %w", err)
|
||||
}
|
||||
chunk = chunk[:n]
|
||||
_, _ = hasher.Write(chunk)
|
||||
item, err := d.putUploadSessionChunkWithRetry(ctx, session.UploadURL, offset, size, chunk)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if item != nil {
|
||||
finalItem = *item
|
||||
}
|
||||
offset += int64(n)
|
||||
}
|
||||
if finalItem.ID == "" {
|
||||
return UploadResult{}, errors.New("onedrive upload session: empty item id")
|
||||
}
|
||||
return UploadResult{
|
||||
FileID: finalItem.ID,
|
||||
Hash: hex.EncodeToString(hasher.Sum(nil)),
|
||||
Size: offset,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) createUploadSession(ctx context.Context, parentID, name string) (uploadSessionResp, error) {
|
||||
u := fmt.Sprintf("%s/items/%s:/%s:/createUploadSession", d.driveBaseURL(), url.PathEscape(parentID), url.PathEscape(name))
|
||||
body := map[string]any{
|
||||
"item": map[string]any{
|
||||
"@microsoft.graph.conflictBehavior": "rename",
|
||||
},
|
||||
}
|
||||
var out uploadSessionResp
|
||||
err := d.request(ctx, u, http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(body)
|
||||
}, &out)
|
||||
if err != nil {
|
||||
return uploadSessionResp{}, fmt.Errorf("onedrive upload session: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunkWithRetry(ctx context.Context, uploadURL string, start, total int64, data []byte) (*graphItem, error) {
|
||||
var last error
|
||||
for attempt := 0; attempt < uploadSessionRetryAttempts; attempt++ {
|
||||
if attempt > 0 {
|
||||
if err := sleepContext(ctx, time.Duration(attempt)*time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
item, retryable, err := d.putUploadSessionChunk(ctx, uploadURL, start, total, data)
|
||||
if err == nil {
|
||||
return item, nil
|
||||
}
|
||||
last = err
|
||||
if !retryable {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if last == nil {
|
||||
last = errors.New("onedrive upload session: retry attempts exhausted")
|
||||
}
|
||||
return nil, last
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunk(ctx context.Context, uploadURL string, start, total int64, data []byte) (*graphItem, bool, error) {
|
||||
end := start + int64(len(data)) - 1
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
req.ContentLength = int64(len(data))
|
||||
req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, total))
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK, http.StatusCreated:
|
||||
var item graphItem
|
||||
if err := json.NewDecoder(res.Body).Decode(&item); err != nil {
|
||||
return nil, false, fmt.Errorf("onedrive upload session: decode completed item: %w", err)
|
||||
}
|
||||
return &item, false, nil
|
||||
case http.StatusAccepted:
|
||||
return nil, false, nil
|
||||
default:
|
||||
body, _ := io.ReadAll(io.LimitReader(res.Body, 4096))
|
||||
err := fmt.Errorf("onedrive upload session: status=%d body=%s", res.StatusCode, strings.TrimSpace(string(body)))
|
||||
retryable := res.StatusCode == http.StatusTooManyRequests || (res.StatusCode >= 500 && res.StatusCode <= 504)
|
||||
return nil, retryable, err
|
||||
}
|
||||
}
|
||||
|
||||
func readSmallUpload(r io.Reader, declaredSize, limit int64) ([]byte, string, int64, error) {
|
||||
if r == nil {
|
||||
return nil, "", 0, errors.New("onedrive upload: body is required")
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = maxSmallUploadSize
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(r, limit+1))
|
||||
if err != nil {
|
||||
return nil, "", 0, fmt.Errorf("onedrive upload: read body: %w", err)
|
||||
}
|
||||
if int64(len(data)) > limit {
|
||||
return nil, "", 0, fmt.Errorf("onedrive upload: files over %d bytes require upload session", limit)
|
||||
}
|
||||
if declaredSize >= 0 && int64(len(data)) != declaredSize {
|
||||
return nil, "", 0, fmt.Errorf("onedrive upload: size mismatch: declared %d, copied %d", declaredSize, len(data))
|
||||
}
|
||||
sum := sha1.Sum(data)
|
||||
return data, hex.EncodeToString(sum[:]), int64(len(data)), nil
|
||||
}
|
||||
|
||||
func minInt64(a, b int64) int64 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
@@ -245,6 +482,25 @@ func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, er
|
||||
return item.ID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("onedrive rename: empty file id")
|
||||
}
|
||||
newName = strings.TrimSpace(newName)
|
||||
if newName == "" {
|
||||
return errors.New("onedrive rename: empty new name")
|
||||
}
|
||||
var item graphItem
|
||||
err := d.request(ctx, d.itemURL(fileID), http.MethodPatch, func(req *resty.Request) {
|
||||
req.SetBody(map[string]string{"name": newName})
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return fmt.Errorf("onedrive rename: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) request(ctx context.Context, rawURL, method string, configure func(*resty.Request), out any) error {
|
||||
return d.requestOnce(ctx, rawURL, method, configure, out, true)
|
||||
}
|
||||
@@ -265,7 +521,7 @@ func (d *Driver) requestOnce(ctx context.Context, rawURL, method string, configu
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isRateLimitResponse(res, graphErr.Error.Code) {
|
||||
if isRateLimitResponse(res, graphErr.Error.Code, graphErr.Error.Message) {
|
||||
return onedriveRateLimitError(res, graphErr.Error.Message)
|
||||
}
|
||||
if graphErr.Error.Code != "" {
|
||||
@@ -327,11 +583,54 @@ func (d *Driver) refresh(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func isRateLimitResponse(res *resty.Response, code string) bool {
|
||||
if code == "TooManyRequests" || code == "activityLimitReached" {
|
||||
func isRateLimitResponse(res *resty.Response, code, message string) bool {
|
||||
if isRateLimitCode(code) || isRateLimitMessage(message) {
|
||||
return true
|
||||
}
|
||||
return res != nil && res.StatusCode() == http.StatusTooManyRequests
|
||||
if res == nil {
|
||||
return false
|
||||
}
|
||||
if res.StatusCode() == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if res.Header().Get("Retry-After") == "" {
|
||||
return false
|
||||
}
|
||||
switch res.StatusCode() {
|
||||
case http.StatusServiceUnavailable, http.StatusGatewayTimeout:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isRateLimitCode(code string) bool {
|
||||
normalized := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(code), "_", ""))
|
||||
normalized = strings.ReplaceAll(normalized, "-", "")
|
||||
switch normalized {
|
||||
case "toomanyrequests",
|
||||
"activitylimitreached",
|
||||
"throttledrequest",
|
||||
"requestthrottled",
|
||||
"resourcethrottled",
|
||||
"applicationthrottled",
|
||||
"tenantthrottled":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isRateLimitMessage(message string) bool {
|
||||
text := strings.ToLower(strings.TrimSpace(message))
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "throttl") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "activity limit") ||
|
||||
strings.Contains(text, "temporarily blocked")
|
||||
}
|
||||
|
||||
func onedriveRateLimitError(res *resty.Response, message string) error {
|
||||
|
||||
@@ -2,6 +2,8 @@ package onedrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
@@ -199,7 +201,7 @@ func TestGraph429ReturnsRateLimitErrorWithRetryAfter(t *testing.T) {
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
|
||||
_, err := d.List(context.Background(), "root")
|
||||
_, err := d.StreamURL(context.Background(), "file-id")
|
||||
if err == nil {
|
||||
t.Fatal("list succeeded, want rate limit error")
|
||||
}
|
||||
@@ -212,6 +214,92 @@ func TestGraph429ReturnsRateLimitErrorWithRetryAfter(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphThrottleMessageReturnsRateLimitError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{
|
||||
"error": map[string]any{
|
||||
"code": "generalException",
|
||||
"message": "The request has been throttled. Please try again later.",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("write json: %v", err)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
|
||||
_, err := d.StreamURL(context.Background(), "file-id")
|
||||
if err == nil {
|
||||
t.Fatal("list succeeded, want rate limit error")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListCoolsDownAndRetriesOneDriveRateLimit(t *testing.T) {
|
||||
var calls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1.0/me/drive/items/root/children" {
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
|
||||
}
|
||||
calls++
|
||||
if calls == 1 {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{
|
||||
"error": map[string]any{
|
||||
"code": "TooManyRequests",
|
||||
"message": "throttled",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("write json: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
writeJSON(t, w, map[string]any{
|
||||
"value": []map[string]any{
|
||||
{
|
||||
"id": "file-id",
|
||||
"name": "demo.mp4",
|
||||
"size": 100,
|
||||
"file": map[string]any{"mimeType": "video/mp4"},
|
||||
},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
d.listInterval = 0
|
||||
d.listCooldown = time.Millisecond
|
||||
|
||||
got, err := d.List(context.Background(), "root")
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("calls = %d, want retry after rate limit", calls)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID != "file-id" {
|
||||
t.Fatalf("entries = %#v, want retried file", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatAndStreamURLUseDriveItemMetadata(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
@@ -320,6 +408,36 @@ func TestEnsureDirCreatesMissingFolders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenamePatchesDriveItemName(t *testing.T) {
|
||||
var body map[string]string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPatch || r.URL.EscapedPath() != "/v1.0/me/drive/items/file-id" {
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
|
||||
}
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
t.Fatalf("authorization = %q, want bearer token", got)
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode body: %v", err)
|
||||
}
|
||||
writeJSON(t, w, map[string]any{"id": "file-id", "name": "new name.mp4"})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
if err := d.Rename(context.Background(), "file-id", "new name.mp4"); err != nil {
|
||||
t.Fatalf("rename: %v", err)
|
||||
}
|
||||
if body["name"] != "new name.mp4" {
|
||||
t.Fatalf("rename body = %#v, want new name", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadSmallFileReturnsNewItemID(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
@@ -358,6 +476,86 @@ func TestUploadSmallFileReturnsNewItemID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadLargeFileUsesUploadSessionAndReportsHash(t *testing.T) {
|
||||
oldThreshold := smallUploadThreshold
|
||||
oldChunk := uploadSessionChunk
|
||||
smallUploadThreshold = 8
|
||||
uploadSessionChunk = 4
|
||||
t.Cleanup(func() {
|
||||
smallUploadThreshold = oldThreshold
|
||||
uploadSessionChunk = oldChunk
|
||||
})
|
||||
|
||||
body := "0123456789abc"
|
||||
var ranges []string
|
||||
var chunks []string
|
||||
var createdSession bool
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost && r.URL.EscapedPath() == "/v1.0/me/drive/items/parent-id:/big.mp4:/createUploadSession":
|
||||
createdSession = true
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
t.Fatalf("authorization = %q, want bearer token", got)
|
||||
}
|
||||
writeJSON(t, w, map[string]any{"uploadUrl": srv.URL + "/upload-session"})
|
||||
case r.Method == http.MethodPut && r.URL.Path == "/upload-session":
|
||||
ranges = append(ranges, r.Header.Get("Content-Range"))
|
||||
data, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read chunk: %v", err)
|
||||
}
|
||||
chunks = append(chunks, string(data))
|
||||
if len(ranges) < 4 {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
if _, err := w.Write([]byte(`{"nextExpectedRanges":["0-"]}`)); err != nil {
|
||||
t.Fatalf("write accepted: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{"id": "uploaded-big-id"}); err != nil {
|
||||
t.Fatalf("write final item: %v", err)
|
||||
}
|
||||
default:
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
got, err := d.UploadAndReportHash(context.Background(), "parent-id", "big.mp4", strings.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
if !createdSession {
|
||||
t.Fatal("createUploadSession was not called")
|
||||
}
|
||||
wantRanges := []string{
|
||||
"bytes 0-3/13",
|
||||
"bytes 4-7/13",
|
||||
"bytes 8-11/13",
|
||||
"bytes 12-12/13",
|
||||
}
|
||||
if strings.Join(ranges, "|") != strings.Join(wantRanges, "|") {
|
||||
t.Fatalf("ranges = %#v, want %#v", ranges, wantRanges)
|
||||
}
|
||||
if strings.Join(chunks, "") != body {
|
||||
t.Fatalf("uploaded chunks = %q, want %q", strings.Join(chunks, ""), body)
|
||||
}
|
||||
sum := sha1.Sum([]byte(body))
|
||||
if got.FileID != "uploaded-big-id" || got.Size != int64(len(body)) || got.Hash != hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("upload result = %#v, want file id/hash/size for body", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadRefreshesExpiredTokenAndReplaysBody(t *testing.T) {
|
||||
var uploadAttempts int
|
||||
var tokenRefreshes int
|
||||
|
||||
@@ -82,3 +82,13 @@ type filesResp struct {
|
||||
Value []graphItem `json:"value"`
|
||||
NextLink string `json:"@odata.nextLink"`
|
||||
}
|
||||
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
Hash string
|
||||
Size int64
|
||||
}
|
||||
|
||||
type uploadSessionResp struct {
|
||||
UploadURL string `json:"uploadUrl"`
|
||||
}
|
||||
|
||||
@@ -199,9 +199,8 @@ func (d *Driver) refreshCaptchaToken(ctx context.Context, action string, meta ma
|
||||
|
||||
// refreshCaptchaTokenOnce 调 /v1/shield/captcha/init 申请新 captcha token。
|
||||
//
|
||||
// 如果 retry=true 且服务端返回 4002(captcha_token expired,意味着 body 里
|
||||
// 携带的 d.captchaToken 已经过期),就清空缓存的 captcha_token 后再调一次;
|
||||
// 这次 body 里 captcha_token 为空,服务端会直接发一个新的。这覆盖
|
||||
// 如果 retry=true 且服务端返回 captcha 失效错误(4002 或 9),就清空缓存的
|
||||
// captcha_token 后再调一次;这次 body 里 captcha_token 为空,服务端会直接发一个新的。这覆盖
|
||||
// driver 重启后 Init() 用持久化的旧 captcha_token 调 captcha init 失败的
|
||||
// 场景。
|
||||
func (d *Driver) refreshCaptchaTokenOnce(ctx context.Context, action string, meta map[string]string, retry bool) error {
|
||||
@@ -230,7 +229,7 @@ func (d *Driver) refreshCaptchaTokenOnce(ctx context.Context, action string, met
|
||||
return err
|
||||
}
|
||||
if e.isError() {
|
||||
if retry && e.ErrorCode == 4002 && d.captchaToken != "" {
|
||||
if retry && isCaptchaTokenRejectedCode(e.ErrorCode) && d.captchaToken != "" {
|
||||
d.captchaToken = ""
|
||||
return d.refreshCaptchaTokenOnce(ctx, action, meta, false)
|
||||
}
|
||||
|
||||
@@ -96,6 +96,65 @@ func TestRefreshCaptchaTokenRecoversFrom4002(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefreshCaptchaTokenRecoversFrom9 覆盖 PikPak 返回 error_code=9
|
||||
// captcha_invalid 的路径。这个错误和 4002 一样表示当前 captcha_token 已被拒绝;
|
||||
// 重试 captcha/init 前必须先清空旧 token,否则服务端会继续拒绝。
|
||||
func TestRefreshCaptchaTokenRecoversFrom9(t *testing.T) {
|
||||
var calls int32
|
||||
type bodyShape struct {
|
||||
CaptchaToken string `json:"captcha_token"`
|
||||
}
|
||||
var (
|
||||
firstBody bodyShape
|
||||
secondBody bodyShape
|
||||
)
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v1/shield/captcha/init", func(w http.ResponseWriter, r *http.Request) {
|
||||
n := atomic.AddInt32(&calls, 1)
|
||||
switch n {
|
||||
case 1:
|
||||
_ = json.NewDecoder(r.Body).Decode(&firstBody)
|
||||
writeErrorJSON(w, `{
|
||||
"error_code": 9,
|
||||
"error": "captcha_invalid",
|
||||
"error_description": "Verification code is invalid"
|
||||
}`)
|
||||
case 2:
|
||||
_ = json.NewDecoder(r.Body).Decode(&secondBody)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"captcha_token": "fresh-captcha",
|
||||
"expires_in": 300
|
||||
}`))
|
||||
default:
|
||||
t.Errorf("unexpected captcha init call #%d", n)
|
||||
}
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
d.captchaToken = "expired-captcha"
|
||||
|
||||
if err := d.refreshCaptchaTokenAtLogin(context.Background(), "GET:/drive/v1/files", "user-1"); err != nil {
|
||||
t.Fatalf("refreshCaptchaTokenAtLogin: %v", err)
|
||||
}
|
||||
|
||||
if got := atomic.LoadInt32(&calls); got != 2 {
|
||||
t.Fatalf("captcha init called %d times, want 2", got)
|
||||
}
|
||||
if firstBody.CaptchaToken != "expired-captcha" {
|
||||
t.Errorf("first body captcha_token = %q, want \"expired-captcha\"", firstBody.CaptchaToken)
|
||||
}
|
||||
if secondBody.CaptchaToken != "" {
|
||||
t.Errorf("second body captcha_token = %q, want empty (cleared after error_code=9)", secondBody.CaptchaToken)
|
||||
}
|
||||
if d.captchaToken != "fresh-captcha" {
|
||||
t.Errorf("d.captchaToken = %q, want \"fresh-captcha\"", d.captchaToken)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefreshCaptchaTokenDoesNotLoopOn4002WithEmptyToken 防止退化成无限重试:
|
||||
// 如果调用方一开始 captchaToken 就是空,又遇上 4002,不应该再清空一次重试
|
||||
// (清空后还是空,再发会拿到同样的错误),应该直接返回错误让上层处理。
|
||||
@@ -121,6 +180,141 @@ func TestRefreshCaptchaTokenDoesNotLoopOn4002WithEmptyToken(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitWithRefreshTokenDoesNotSendPersistedCaptchaToken(t *testing.T) {
|
||||
var captchaCalls int32
|
||||
var captchaBody struct {
|
||||
CaptchaToken string `json:"captcha_token"`
|
||||
}
|
||||
var persisted struct {
|
||||
access, refresh, captcha string
|
||||
calls int
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v1/auth/token", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"access_token": "fresh-access",
|
||||
"refresh_token": "fresh-refresh",
|
||||
"sub": "user-1"
|
||||
}`))
|
||||
})
|
||||
mux.HandleFunc("/v1/shield/captcha/init", func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&captchaCalls, 1)
|
||||
_ = json.NewDecoder(r.Body).Decode(&captchaBody)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"captcha_token": "fresh-captcha",
|
||||
"expires_in": 300
|
||||
}`))
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
d.captchaToken = "persisted-stale-captcha"
|
||||
d.onTokenUpdate = func(access, refresh, captcha, deviceID string) {
|
||||
persisted.access = access
|
||||
persisted.refresh = refresh
|
||||
persisted.captcha = captcha
|
||||
persisted.calls++
|
||||
}
|
||||
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
|
||||
if got := atomic.LoadInt32(&captchaCalls); got != 1 {
|
||||
t.Fatalf("captcha init calls = %d, want 1", got)
|
||||
}
|
||||
if captchaBody.CaptchaToken != "" {
|
||||
t.Errorf("captcha init body captcha_token = %q, want empty", captchaBody.CaptchaToken)
|
||||
}
|
||||
if d.captchaToken != "fresh-captcha" {
|
||||
t.Errorf("d.captchaToken = %q, want \"fresh-captcha\"", d.captchaToken)
|
||||
}
|
||||
if persisted.access != "fresh-access" || persisted.refresh != "fresh-refresh" || persisted.captcha != "fresh-captcha" {
|
||||
t.Errorf("persisted tokens = (%q, %q, %q), want fresh values", persisted.access, persisted.refresh, persisted.captcha)
|
||||
}
|
||||
if persisted.calls < 2 {
|
||||
t.Errorf("persist callback calls = %d, want at least 2 (clear stale + persist fresh)", persisted.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitFallsBackToLoginWhenRefreshReturnsCaptchaInvalid(t *testing.T) {
|
||||
var (
|
||||
tokenCalls int32
|
||||
captchaCalls int32
|
||||
signinCalls int32
|
||||
)
|
||||
var signinBody struct {
|
||||
CaptchaToken string `json:"captcha_token"`
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v1/auth/token", func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&tokenCalls, 1)
|
||||
writeErrorJSON(w, `{
|
||||
"error_code": 4002,
|
||||
"error": "captcha_invalid",
|
||||
"error_description": "Code(4002) - captcha_token expired"
|
||||
}`)
|
||||
})
|
||||
mux.HandleFunc("/v1/shield/captcha/init", func(w http.ResponseWriter, r *http.Request) {
|
||||
n := atomic.AddInt32(&captchaCalls, 1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch n {
|
||||
case 1:
|
||||
_, _ = w.Write([]byte(`{
|
||||
"captcha_token": "login-captcha",
|
||||
"expires_in": 300
|
||||
}`))
|
||||
case 2:
|
||||
_, _ = w.Write([]byte(`{
|
||||
"captcha_token": "files-captcha",
|
||||
"expires_in": 300
|
||||
}`))
|
||||
default:
|
||||
t.Errorf("unexpected captcha init call #%d", n)
|
||||
}
|
||||
})
|
||||
mux.HandleFunc("/v1/auth/signin", func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&signinCalls, 1)
|
||||
_ = json.NewDecoder(r.Body).Decode(&signinBody)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"access_token": "login-access",
|
||||
"refresh_token": "login-refresh",
|
||||
"sub": "user-1"
|
||||
}`))
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
d.captchaToken = "persisted-stale-captcha"
|
||||
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
|
||||
if got := atomic.LoadInt32(&tokenCalls); got != 1 {
|
||||
t.Fatalf("token refresh calls = %d, want 1", got)
|
||||
}
|
||||
if got := atomic.LoadInt32(&signinCalls); got != 1 {
|
||||
t.Fatalf("signin calls = %d, want 1", got)
|
||||
}
|
||||
if got := atomic.LoadInt32(&captchaCalls); got != 2 {
|
||||
t.Fatalf("captcha init calls = %d, want 2 (login + post-login files action)", got)
|
||||
}
|
||||
if signinBody.CaptchaToken != "login-captcha" {
|
||||
t.Errorf("signin captcha_token = %q, want \"login-captcha\"", signinBody.CaptchaToken)
|
||||
}
|
||||
if d.accessToken != "login-access" || d.refreshToken != "login-refresh" || d.captchaToken != "files-captcha" {
|
||||
t.Errorf("driver tokens = (%q, %q, %q), want login/files tokens", d.accessToken, d.refreshToken, d.captchaToken)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRequestOnceRecoversFrom4002OnAPICall 验证一个普通 API 调用收到 4002
|
||||
// 时,requestOnce 会先清空 captchaToken、再走 captcha 刷新,最后用新 token
|
||||
// 重试请求,最终成功返回。
|
||||
@@ -196,6 +390,76 @@ func TestRequestOnceRecoversFrom4002OnAPICall(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRequestOnceRecoversFrom9OnAPICall 验证普通 API 调用收到 error_code=9
|
||||
// 时,会先清空旧 captchaToken,再刷新 captcha 并重试原请求。
|
||||
func TestRequestOnceRecoversFrom9OnAPICall(t *testing.T) {
|
||||
var (
|
||||
filesCalls int32
|
||||
captchaCalls int32
|
||||
)
|
||||
type capturedFiles struct {
|
||||
captchaHeader string
|
||||
}
|
||||
var firstFiles, secondFiles capturedFiles
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
n := atomic.AddInt32(&filesCalls, 1)
|
||||
switch n {
|
||||
case 1:
|
||||
firstFiles.captchaHeader = r.Header.Get("X-Captcha-Token")
|
||||
writeErrorJSON(w, `{
|
||||
"error_code": 9,
|
||||
"error": "captcha_invalid",
|
||||
"error_description": "Verification code is invalid"
|
||||
}`)
|
||||
case 2:
|
||||
secondFiles.captchaHeader = r.Header.Get("X-Captcha-Token")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"files": [], "next_page_token": ""}`))
|
||||
default:
|
||||
t.Errorf("unexpected /drive/v1/files call #%d", n)
|
||||
}
|
||||
})
|
||||
mux.HandleFunc("/v1/shield/captcha/init", func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&captchaCalls, 1)
|
||||
var body struct {
|
||||
CaptchaToken string `json:"captcha_token"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
if body.CaptchaToken != "" {
|
||||
t.Errorf("captcha init body captcha_token = %q, want empty (error_code=9 path should clear cache)", body.CaptchaToken)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"captcha_token": "fresh-captcha", "expires_in": 300}`))
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
d.captchaToken = "expired-captcha"
|
||||
|
||||
if _, err := d.List(context.Background(), "any-parent"); err != nil {
|
||||
t.Fatalf("List: %v", err)
|
||||
}
|
||||
|
||||
if got := atomic.LoadInt32(&filesCalls); got != 2 {
|
||||
t.Fatalf("/drive/v1/files calls = %d, want 2 (initial + retry)", got)
|
||||
}
|
||||
if got := atomic.LoadInt32(&captchaCalls); got != 1 {
|
||||
t.Fatalf("captcha init calls = %d, want 1", got)
|
||||
}
|
||||
if firstFiles.captchaHeader != "expired-captcha" {
|
||||
t.Errorf("first request X-Captcha-Token = %q, want \"expired-captcha\"", firstFiles.captchaHeader)
|
||||
}
|
||||
if secondFiles.captchaHeader != "fresh-captcha" {
|
||||
t.Errorf("retry X-Captcha-Token = %q, want \"fresh-captcha\"", secondFiles.captchaHeader)
|
||||
}
|
||||
if d.captchaToken != "fresh-captcha" {
|
||||
t.Errorf("d.captchaToken after recovery = %q, want \"fresh-captcha\"", d.captchaToken)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRequestOnceDoesNotRetryTwiceOn4002 验证 4002 恢复路径只重试一次;
|
||||
// 如果重试请求依然失败(哪怕是再来一个 4002),也不会再次进入恢复逻辑,
|
||||
// 而是把错误返回出去,避免无限循环。
|
||||
|
||||
@@ -121,9 +121,28 @@ func (d *Driver) ID() string { return d.id }
|
||||
func (d *Driver) RootID() string { return d.rootID }
|
||||
|
||||
func (d *Driver) Init(ctx context.Context) error {
|
||||
clearPersistedCaptcha := func() {
|
||||
if d.captchaToken == "" {
|
||||
return
|
||||
}
|
||||
d.captchaToken = ""
|
||||
d.persistTokens()
|
||||
}
|
||||
|
||||
if d.refreshToken != "" {
|
||||
if err := d.refresh(ctx, d.refreshToken); err != nil {
|
||||
return err
|
||||
if !IsCaptchaError(err) || d.username == "" || d.password == "" {
|
||||
return err
|
||||
}
|
||||
clearPersistedCaptcha()
|
||||
if err := d.login(ctx); err != nil {
|
||||
return fmt.Errorf("pikpak refresh captcha recovery login: %w", err)
|
||||
}
|
||||
} else {
|
||||
// Persisted captcha tokens are short-lived. With a refresh token we can
|
||||
// safely request a fresh captcha token after auth, and avoiding the
|
||||
// stored value prevents known-stale tokens from poisoning startup.
|
||||
clearPersistedCaptcha()
|
||||
}
|
||||
} else {
|
||||
if err := d.login(ctx); err != nil {
|
||||
@@ -336,7 +355,60 @@ func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
currentID := d.rootID
|
||||
for _, name := range splitPath(pathFromRoot) {
|
||||
childID, err := d.findChildDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if childID == "" {
|
||||
childID, err = d.makeDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
currentID = childID
|
||||
}
|
||||
return currentID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) findChildDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
entries, err := d.List(ctx, parentID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir && e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
var out file
|
||||
err := d.request(ctx, filesURL, http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(map[string]any{
|
||||
"kind": "drive#folder",
|
||||
"parent_id": parentID,
|
||||
"name": name,
|
||||
})
|
||||
}, &out)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("pikpak mkdir %s: %w", name, err)
|
||||
}
|
||||
if out.ID == "" {
|
||||
return "", fmt.Errorf("pikpak mkdir %s: empty folder id", name)
|
||||
}
|
||||
return out.ID, nil
|
||||
}
|
||||
|
||||
func splitPath(p string) []string {
|
||||
p = strings.Trim(p, "/")
|
||||
if p == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(p, "/")
|
||||
}
|
||||
|
||||
func (d *Driver) getFiles(ctx context.Context, parentID string) ([]file, error) {
|
||||
@@ -408,14 +480,15 @@ func (d *Driver) requestOnce(ctx context.Context, url, method string, configure
|
||||
// serialized. Once we hold the lock, if d.captchaToken has
|
||||
// already moved past staleToken, another goroutine has refreshed
|
||||
// it for us — we skip the refresh and just retry. Otherwise we
|
||||
// clear the cached token (4002 means "the value in the body is
|
||||
// expired"; sending it again will keep returning 4002) and ask
|
||||
// /v1/shield/captcha/init for a fresh one.
|
||||
// clear the cached token before asking /v1/shield/captcha/init
|
||||
// for a fresh one. PikPak may report stale captcha as either
|
||||
// 4002 or 9, and sending the rejected token into captcha init can
|
||||
// keep returning captcha_invalid.
|
||||
staleToken := d.captchaToken
|
||||
d.captchaMu.Lock()
|
||||
var refreshErr error
|
||||
if d.captchaToken == staleToken {
|
||||
if e.ErrorCode == 4002 {
|
||||
if d.captchaToken != "" {
|
||||
d.captchaToken = ""
|
||||
}
|
||||
refreshErr = d.refreshCaptchaTokenAtLogin(ctx, getAction(method, url), d.userID)
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package pikpak
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestNewDefaults(t *testing.T) {
|
||||
@@ -95,11 +97,85 @@ func TestFolderToEntry(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDirStillUnsupported(t *testing.T) {
|
||||
d := New(Config{ID: "pikpak-main"})
|
||||
func TestEnsureDirReusesExistingFolder(t *testing.T) {
|
||||
var postCalled bool
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
if got := r.URL.Query().Get("parent_id"); got != "root-id" {
|
||||
t.Fatalf("parent_id = %q, want root-id", got)
|
||||
}
|
||||
writePikPakJSON(t, w, map[string]any{
|
||||
"files": []map[string]any{{
|
||||
"id": "existing-folder-id",
|
||||
"kind": "drive#folder",
|
||||
"name": "91 Spider",
|
||||
}},
|
||||
})
|
||||
case http.MethodPost:
|
||||
postCalled = true
|
||||
t.Fatalf("existing folder should not be created again")
|
||||
default:
|
||||
t.Fatalf("unexpected method %s", r.Method)
|
||||
}
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
if _, err := d.EnsureDir(nil, "/previews"); err != drives.ErrNotSupported {
|
||||
t.Fatalf("EnsureDir error = %v, want ErrNotSupported", err)
|
||||
d := newTestDriver(t, srv)
|
||||
got, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure dir: %v", err)
|
||||
}
|
||||
if got != "existing-folder-id" {
|
||||
t.Fatalf("dir id = %q, want existing-folder-id", got)
|
||||
}
|
||||
if postCalled {
|
||||
t.Fatal("POST should not be called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDirCreatesMissingFolder(t *testing.T) {
|
||||
var got uploadRequestBody
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
writePikPakJSON(t, w, map[string]any{"files": []map[string]any{}})
|
||||
case http.MethodPost:
|
||||
if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode create folder body: %v", err)
|
||||
}
|
||||
writePikPakJSON(t, w, map[string]any{
|
||||
"id": "new-folder-id",
|
||||
"kind": "drive#folder",
|
||||
"name": "91 Spider",
|
||||
})
|
||||
default:
|
||||
t.Fatalf("unexpected method %s", r.Method)
|
||||
}
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
d := newTestDriver(t, srv)
|
||||
id, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure dir: %v", err)
|
||||
}
|
||||
if id != "new-folder-id" {
|
||||
t.Fatalf("dir id = %q, want new-folder-id", id)
|
||||
}
|
||||
if got.Kind != "drive#folder" || got.ParentID != "root-id" || got.Name != "91 Spider" {
|
||||
t.Fatalf("create folder body = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func writePikPakJSON(t *testing.T, w http.ResponseWriter, body any) {
|
||||
t.Helper()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(body); err != nil {
|
||||
t.Fatalf("write json: %v", err)
|
||||
}
|
||||
// Upload 的真实实现见 upload_test.go。
|
||||
}
|
||||
|
||||
@@ -59,6 +59,10 @@ func (e *errResp) Error() string {
|
||||
return fmt.Sprintf("pikpak error_code=%d error=%s description=%s", e.ErrorCode, e.ErrorMsg, e.ErrorDescription)
|
||||
}
|
||||
|
||||
func isCaptchaTokenRejectedCode(code int64) bool {
|
||||
return code == 9 || code == 4002
|
||||
}
|
||||
|
||||
// APIError is the public alias for the PikPak API error response. Callers
|
||||
// outside this package (e.g. the spider91→PikPak migrator, tests) can either
|
||||
// construct it for fakes or unwrap it via errors.As. Prefer IsCaptchaError
|
||||
@@ -76,7 +80,7 @@ func IsCaptchaError(err error) bool {
|
||||
}
|
||||
var e *errResp
|
||||
if errors.As(err, &e) {
|
||||
return e != nil && (e.ErrorCode == 4002 || e.ErrorCode == 9)
|
||||
return e != nil && isCaptchaTokenRejectedCode(e.ErrorCode)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -463,8 +463,7 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
// 网站封面下载失败的视频:spider91 drive 的 thumb worker 按设计不
|
||||
// 处理 spider91 视频(封面应是网站原图直接保存),所以没人接手。
|
||||
// 显式标 'failed' 让 CountVideosNeedingThumbnail 排除(条件 status
|
||||
// != 'failed'),否则 enqueueDriveGeneration → waitForThumbnailsBeforePreview
|
||||
// 会因为 count > 0 把 teaser 入队永远卡在等待循环里。
|
||||
// != 'failed'),避免后续封面补队列一直重复捞到这条视频。
|
||||
_ = c.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
ThumbnailStatus: "failed",
|
||||
})
|
||||
|
||||
@@ -234,13 +234,13 @@ func TestCrawlerRunOnceMissingScript(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestCrawlerThumbDownloadFailureMarksStatusFailed 验证:网站封面下载失败时
|
||||
// crawler 把 thumbnail_status 显式标 'failed',避免 enqueueDriveGeneration 的
|
||||
// waitForThumbnailsBeforePreview 因为 count > 0 把 teaser 卡死等待。
|
||||
// crawler 把 thumbnail_status 显式标 'failed',避免后续封面补队列一直重复
|
||||
// 捞到这条 spider91 视频。
|
||||
//
|
||||
// 历史 bug:之前 thumb 下载失败仅打 log,url=”, status 走 schema DEFAULT 'pending'。
|
||||
// CountVideosNeedingThumbnail 条件是 url=” AND status != 'failed' → count=1。
|
||||
// spider91 drive 的 thumb worker 按设计不处理 spider91 视频 → 没人会改 status。
|
||||
// 结果 teaser 永远卡在 [preview] waiting for 1 thumbnails before teaser generation。
|
||||
// spider91 drive 的 thumb worker 按设计不处理 spider91 视频 → 没人会改 status,
|
||||
// 后续补队列会一直认为它还缺封面。
|
||||
func TestCrawlerThumbDownloadFailureMarksStatusFailed(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell-based fake script only on unix")
|
||||
@@ -317,8 +317,7 @@ func TestCrawlerThumbDownloadFailureMarksStatusFailed(t *testing.T) {
|
||||
|
||||
// 关键断言:CountVideosNeedingThumbnail 应该返回 0。
|
||||
// 该函数的 SQL 条件是 `url = '' AND status != 'failed'`;如果 crawler 没把
|
||||
// status 标 'failed'(schema DEFAULT 'pending'),count 就会是 1,外层
|
||||
// waitForThumbnailsBeforePreview 会因为 count > 0 把 teaser 卡死等待。
|
||||
// status 标 'failed'(schema DEFAULT 'pending'),count 就会是 1。
|
||||
count, err := cat.CountVideosNeedingThumbnail(context.Background(), driveID)
|
||||
if err != nil {
|
||||
t.Fatalf("count: %v", err)
|
||||
|
||||
@@ -0,0 +1,468 @@
|
||||
package fingerprint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultSampleSizeBytes int64 = 512 * 1024
|
||||
defaultFullHashMaxSize int64 = 8 * 1024 * 1024
|
||||
defaultCooldown = 5 * time.Minute
|
||||
defaultWorkerQueueSize = 10000
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
SampleSizeBytes int64
|
||||
FullHashMaxSize int64
|
||||
RateLimitCooldown time.Duration
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
type Worker struct {
|
||||
Catalog *catalog.Catalog
|
||||
Drive drives.Drive
|
||||
Config Config
|
||||
|
||||
ch chan *catalog.Video
|
||||
queue videoQueue
|
||||
activity taskActivity
|
||||
cooldown cooldownState
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
type TaskStatus struct {
|
||||
State string
|
||||
CurrentTitle string
|
||||
QueueLength int
|
||||
CooldownUntil time.Time
|
||||
}
|
||||
|
||||
func NewWorker(cat *catalog.Catalog, drv drives.Drive, cfg Config) *Worker {
|
||||
hc := cfg.HTTPClient
|
||||
if hc == nil {
|
||||
hc = &http.Client{Timeout: 0}
|
||||
}
|
||||
if cfg.SampleSizeBytes <= 0 {
|
||||
cfg.SampleSizeBytes = defaultSampleSizeBytes
|
||||
}
|
||||
if cfg.FullHashMaxSize <= 0 {
|
||||
cfg.FullHashMaxSize = defaultFullHashMaxSize
|
||||
}
|
||||
if cfg.RateLimitCooldown <= 0 {
|
||||
cfg.RateLimitCooldown = defaultCooldown
|
||||
}
|
||||
return &Worker{
|
||||
Catalog: cat,
|
||||
Drive: drv,
|
||||
Config: cfg,
|
||||
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
|
||||
http: hc,
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) Enqueue(v *catalog.Video) bool {
|
||||
if v == nil {
|
||||
return false
|
||||
}
|
||||
if !w.queue.reserve(v.ID) {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case w.ch <- v:
|
||||
return true
|
||||
default:
|
||||
w.queue.release(v.ID)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) EnqueueBlocking(ctx context.Context, v *catalog.Video) bool {
|
||||
if v == nil {
|
||||
return false
|
||||
}
|
||||
if !w.queue.reserve(v.ID) {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case w.ch <- v:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
w.queue.release(v.ID)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) Run(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case v := <-w.ch:
|
||||
w.processQueued(ctx, v)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) Status() TaskStatus {
|
||||
if w == nil {
|
||||
return TaskStatus{State: "idle"}
|
||||
}
|
||||
currentID, currentTitle := w.activity.current()
|
||||
status := TaskStatus{
|
||||
State: "idle",
|
||||
CurrentTitle: currentTitle,
|
||||
QueueLength: w.queue.lengthExcluding(currentID),
|
||||
}
|
||||
if until, ok := w.cooldown.active(time.Now()); ok {
|
||||
status.State = "cooling"
|
||||
status.CooldownUntil = until
|
||||
return status
|
||||
}
|
||||
if currentID != "" {
|
||||
status.State = "generating"
|
||||
return status
|
||||
}
|
||||
if status.QueueLength > 0 {
|
||||
status.State = "queued"
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
defer w.queue.release(v.ID)
|
||||
if w.Catalog == nil || w.Drive == nil || v == nil || v.ID == "" {
|
||||
return
|
||||
}
|
||||
current, err := w.Catalog.GetVideo(ctx, v.ID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if current.SampledSHA256 != "" || current.FingerprintStatus == "ready" || current.Hidden {
|
||||
return
|
||||
}
|
||||
w.activity.start(current)
|
||||
defer w.activity.done()
|
||||
sum, err := Compute(ctx, w.Drive, current, w.Config, w.http)
|
||||
if err != nil {
|
||||
var rl *drives.RateLimitError
|
||||
if errors.As(err, &rl) {
|
||||
wait := rl.RetryAfter
|
||||
if wait <= 0 {
|
||||
wait = w.Config.RateLimitCooldown
|
||||
}
|
||||
until := time.Now().Add(wait)
|
||||
w.cooldown.set(until)
|
||||
log.Printf("[fingerprint] drive=%s rate limited; keep video=%s pending and cool down for %s: %v", w.Drive.ID(), current.ID, wait, err)
|
||||
sleepContext(ctx, wait)
|
||||
w.cooldown.clear(until)
|
||||
return
|
||||
}
|
||||
log.Printf("[fingerprint] video=%s failed: %v", current.ID, err)
|
||||
_ = w.Catalog.UpdateVideoFingerprint(ctx, current.ID, "", "failed", err.Error())
|
||||
return
|
||||
}
|
||||
if err := w.Catalog.UpdateVideoFingerprint(ctx, current.ID, sum, "ready", ""); err != nil {
|
||||
log.Printf("[fingerprint] update video=%s: %v", current.ID, err)
|
||||
return
|
||||
}
|
||||
log.Printf("[fingerprint] video=%s ready sampled_sha256=%s", current.ID, sum)
|
||||
}
|
||||
|
||||
func Compute(ctx context.Context, drv drives.Drive, v *catalog.Video, cfg Config, hc *http.Client) (string, error) {
|
||||
if drv == nil {
|
||||
return "", errors.New("fingerprint: nil drive")
|
||||
}
|
||||
if v == nil {
|
||||
return "", errors.New("fingerprint: nil video")
|
||||
}
|
||||
if v.Size <= 0 {
|
||||
return "", errors.New("fingerprint: video size is empty")
|
||||
}
|
||||
if cfg.SampleSizeBytes <= 0 {
|
||||
cfg.SampleSizeBytes = defaultSampleSizeBytes
|
||||
}
|
||||
if cfg.FullHashMaxSize <= 0 {
|
||||
cfg.FullHashMaxSize = defaultFullHashMaxSize
|
||||
}
|
||||
if hc == nil {
|
||||
hc = &http.Client{Timeout: 0}
|
||||
}
|
||||
link, err := drv.StreamURL(ctx, v.FileID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fingerprint: stream url: %w", err)
|
||||
}
|
||||
if link == nil || strings.TrimSpace(link.URL) == "" {
|
||||
return "", errors.New("fingerprint: empty stream url")
|
||||
}
|
||||
ranges := sampleRanges(v.Size, cfg.SampleSizeBytes, cfg.FullHashMaxSize)
|
||||
h := sha256.New()
|
||||
writeHashHeader(h, v.Size, ranges)
|
||||
for _, r := range ranges {
|
||||
data, err := readRange(ctx, hc, link, r)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if int64(len(data)) != r.length {
|
||||
return "", fmt.Errorf("fingerprint: short sample at %d: got %d want %d", r.start, len(data), r.length)
|
||||
}
|
||||
_, _ = h.Write([]byte(fmt.Sprintf("offset=%d length=%d\n", r.start, r.length)))
|
||||
_, _ = h.Write(data)
|
||||
_, _ = h.Write([]byte("\n"))
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
type byteRange struct {
|
||||
start int64
|
||||
length int64
|
||||
}
|
||||
|
||||
func sampleRanges(size, sampleSize, fullHashMax int64) []byteRange {
|
||||
if size <= fullHashMax {
|
||||
return []byteRange{{start: 0, length: size}}
|
||||
}
|
||||
if sampleSize > size {
|
||||
sampleSize = size
|
||||
}
|
||||
maxStart := size - sampleSize
|
||||
percents := []int64{0, 20, 40, 60, 80}
|
||||
out := make([]byteRange, 0, len(percents))
|
||||
seen := make(map[int64]struct{}, len(percents))
|
||||
for _, pct := range percents {
|
||||
start := maxStart * pct / 100
|
||||
if _, ok := seen[start]; ok {
|
||||
continue
|
||||
}
|
||||
seen[start] = struct{}{}
|
||||
out = append(out, byteRange{start: start, length: sampleSize})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func writeHashHeader(w io.Writer, size int64, ranges []byteRange) {
|
||||
_, _ = fmt.Fprintf(w, "video-site-sampled-sha256-v1\nsize=%d\nsamples=%d\n", size, len(ranges))
|
||||
}
|
||||
|
||||
func readRange(ctx context.Context, hc *http.Client, link *drives.StreamLink, r byteRange) ([]byte, error) {
|
||||
u, err := url.Parse(link.URL)
|
||||
if err == nil && (u.Scheme == "http" || u.Scheme == "https") {
|
||||
return readHTTPRange(ctx, hc, link, r)
|
||||
}
|
||||
path := link.URL
|
||||
if err == nil && u.Scheme == "file" {
|
||||
path = u.Path
|
||||
}
|
||||
return readLocalRange(path, r)
|
||||
}
|
||||
|
||||
func readLocalRange(path string, r byteRange) ([]byte, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fingerprint: open local stream: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
buf := make([]byte, r.length)
|
||||
n, err := f.ReadAt(buf, r.start)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, fmt.Errorf("fingerprint: read local sample: %w", err)
|
||||
}
|
||||
if int64(n) != r.length {
|
||||
return nil, fmt.Errorf("fingerprint: read local sample at %d: got %d want %d", r.start, n, r.length)
|
||||
}
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
func readHTTPRange(ctx context.Context, hc *http.Client, link *drives.StreamLink, r byteRange) ([]byte, error) {
|
||||
end := r.start + r.length - 1
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, link.URL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for k, vs := range link.Headers {
|
||||
for _, v := range vs {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", r.start, end))
|
||||
resp, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fingerprint: read remote sample: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
return nil, &drives.RateLimitError{
|
||||
Provider: "fingerprint",
|
||||
RetryAfter: parseRetryAfter(resp.Header.Get("Retry-After")),
|
||||
Err: fmt.Errorf("remote sample rate limited: status=%d", resp.StatusCode),
|
||||
}
|
||||
}
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
if resp.StatusCode == http.StatusOK && r.start == 0 {
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, r.length+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) == r.length {
|
||||
return data, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("fingerprint: range request got status=%d for bytes=%d-%d", resp.StatusCode, r.start, end)
|
||||
}
|
||||
return io.ReadAll(io.LimitReader(resp.Body, r.length))
|
||||
}
|
||||
|
||||
func parseRetryAfter(raw string) time.Duration {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return 0
|
||||
}
|
||||
if seconds, err := strconv.Atoi(raw); err == nil && seconds > 0 {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
if when, err := http.ParseTime(raw); err == nil {
|
||||
d := time.Until(when)
|
||||
if d > 0 {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, d time.Duration) bool {
|
||||
if d <= 0 {
|
||||
return true
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-timer.C:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
type taskActivity struct {
|
||||
mu sync.Mutex
|
||||
currentID string
|
||||
currentTitle string
|
||||
}
|
||||
|
||||
func (a *taskActivity) start(v *catalog.Video) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if v == nil {
|
||||
a.currentID = ""
|
||||
a.currentTitle = ""
|
||||
return
|
||||
}
|
||||
a.currentID = v.ID
|
||||
a.currentTitle = v.Title
|
||||
}
|
||||
|
||||
func (a *taskActivity) done() {
|
||||
a.mu.Lock()
|
||||
a.currentID = ""
|
||||
a.currentTitle = ""
|
||||
a.mu.Unlock()
|
||||
}
|
||||
|
||||
func (a *taskActivity) current() (string, string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.currentID, a.currentTitle
|
||||
}
|
||||
|
||||
type cooldownState struct {
|
||||
mu sync.Mutex
|
||||
until time.Time
|
||||
}
|
||||
|
||||
func (s *cooldownState) set(until time.Time) {
|
||||
s.mu.Lock()
|
||||
s.until = until
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *cooldownState) clear(until time.Time) {
|
||||
s.mu.Lock()
|
||||
if s.until.Equal(until) {
|
||||
s.until = time.Time{}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *cooldownState) active(now time.Time) (time.Time, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.until.IsZero() || !s.until.After(now) {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return s.until, true
|
||||
}
|
||||
|
||||
type videoQueue struct {
|
||||
mu sync.Mutex
|
||||
ids map[string]struct{}
|
||||
}
|
||||
|
||||
func (q *videoQueue) reserve(id string) bool {
|
||||
if id == "" {
|
||||
return true
|
||||
}
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
if q.ids == nil {
|
||||
q.ids = make(map[string]struct{})
|
||||
}
|
||||
if _, ok := q.ids[id]; ok {
|
||||
return false
|
||||
}
|
||||
q.ids[id] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
func (q *videoQueue) release(id string) {
|
||||
if id == "" {
|
||||
return
|
||||
}
|
||||
q.mu.Lock()
|
||||
delete(q.ids, id)
|
||||
q.mu.Unlock()
|
||||
}
|
||||
|
||||
func (q *videoQueue) lengthExcluding(currentID string) int {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
n := len(q.ids)
|
||||
if currentID != "" {
|
||||
if _, ok := q.ids[currentID]; ok {
|
||||
n--
|
||||
}
|
||||
}
|
||||
if n < 0 {
|
||||
return 0
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package fingerprint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestComputeLocalFilesWithSameContentMatch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
body := []byte("same video bytes")
|
||||
a := filepath.Join(dir, "a.mp4")
|
||||
b := filepath.Join(dir, "b.mp4")
|
||||
if err := os.WriteFile(a, body, 0o644); err != nil {
|
||||
t.Fatalf("write a: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(b, body, 0o644); err != nil {
|
||||
t.Fatalf("write b: %v", err)
|
||||
}
|
||||
|
||||
sumA, err := Compute(ctx, &fakeDrive{paths: map[string]string{"a": a}}, &catalog.Video{ID: "a", FileID: "a", Size: int64(len(body))}, Config{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("compute a: %v", err)
|
||||
}
|
||||
sumB, err := Compute(ctx, &fakeDrive{paths: map[string]string{"b": b}}, &catalog.Video{ID: "b", FileID: "b", Size: int64(len(body))}, Config{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("compute b: %v", err)
|
||||
}
|
||||
if sumA == "" || sumA != sumB {
|
||||
t.Fatalf("fingerprints = %q / %q, want same non-empty", sumA, sumB)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeRemoteUsesRangeSamples(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
data := make([]byte, 10*1024*1024)
|
||||
for i := range data {
|
||||
data[i] = byte(i % 251)
|
||||
}
|
||||
var ranges []string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
rawRange := r.Header.Get("Range")
|
||||
ranges = append(ranges, rawRange)
|
||||
var start, end int
|
||||
if _, err := fmt.Sscanf(rawRange, "bytes=%d-%d", &start, &end); err != nil {
|
||||
t.Fatalf("bad range %q: %v", rawRange, err)
|
||||
}
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write(data[start : end+1])
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
drv := &fakeDrive{paths: map[string]string{"remote": srv.URL + "/video.mp4"}}
|
||||
sum, err := Compute(ctx, drv, &catalog.Video{ID: "remote", FileID: "remote", Size: int64(len(data))}, Config{
|
||||
SampleSizeBytes: 4,
|
||||
FullHashMaxSize: 8,
|
||||
HTTPClient: srv.Client(),
|
||||
}, srv.Client())
|
||||
if err != nil {
|
||||
t.Fatalf("compute remote: %v", err)
|
||||
}
|
||||
if sum == "" {
|
||||
t.Fatal("fingerprint should not be empty")
|
||||
}
|
||||
want := []string{
|
||||
"bytes=0-3",
|
||||
"bytes=2097151-2097154",
|
||||
"bytes=4194302-4194305",
|
||||
"bytes=6291453-6291456",
|
||||
"bytes=8388604-8388607",
|
||||
}
|
||||
if fmt.Sprint(ranges) != fmt.Sprint(want) {
|
||||
t.Fatalf("ranges = %#v, want %#v", ranges, want)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeDrive struct {
|
||||
paths map[string]string
|
||||
}
|
||||
|
||||
func (d *fakeDrive) Kind() string { return "fake" }
|
||||
func (d *fakeDrive) ID() string { return "fake" }
|
||||
func (d *fakeDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *fakeDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeDrive) StreamURL(_ context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
return &drives.StreamLink{URL: d.paths[fileID], Expires: time.Now().Add(time.Minute)}, nil
|
||||
}
|
||||
func (d *fakeDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeDrive) RootID() string { return "root" }
|
||||
@@ -12,6 +12,8 @@
|
||||
// wait until teaser queues are idle
|
||||
// Phase 3: spider91 → cloud migration (single sweep, captcha cooldown still
|
||||
// honored within this call)
|
||||
// Phase 4: cleanup duplicate local preview/thumbnail assets after sampled
|
||||
// fingerprints have identified canonical videos
|
||||
//
|
||||
// A 6h soft deadline guards each pipeline run; phases check deadline at their
|
||||
// boundaries and exit cleanly if exceeded (no in-flight ffmpeg / upload is
|
||||
@@ -85,6 +87,11 @@ type Config struct {
|
||||
// RunMigration runs spider91migrate.Migrator.RunOnce for Phase 3.
|
||||
RunMigration func(ctx context.Context) error
|
||||
|
||||
// RunDedupeAssetCleanup removes generated local assets from non-canonical
|
||||
// videos in size+sampled_sha256 duplicate groups. It must not delete cloud
|
||||
// files or catalog rows.
|
||||
RunDedupeAssetCleanup func(ctx context.Context) error
|
||||
|
||||
// Now is injected for tests; nil → time.Now.
|
||||
Now func() time.Time
|
||||
}
|
||||
@@ -131,9 +138,10 @@ func (r *Runner) Run(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// TriggerNow asks the running loop to fire a pipeline ASAP. If a pipeline is
|
||||
// already in progress (or another trigger is already pending), the request
|
||||
// is dropped — the in-progress run will absorb the intent.
|
||||
// TriggerNow asks the running loop to fire a pipeline ASAP. The trigger channel
|
||||
// is buffered(1): if a pipeline is already in progress, one follow-up run may
|
||||
// remain pending and will start after the current run finishes. Additional
|
||||
// clicks while that follow-up is pending are ignored.
|
||||
func (r *Runner) TriggerNow() {
|
||||
select {
|
||||
case r.trigger <- struct{}{}:
|
||||
@@ -240,6 +248,7 @@ func (r *Runner) runPipeline(ctx context.Context) {
|
||||
}
|
||||
if len(spiderIDs) == 0 {
|
||||
log.Printf("[nightly] phase 2/3 skipped: no spider91 drive configured")
|
||||
r.runDedupeAssetCleanupPhase(ctx)
|
||||
return
|
||||
}
|
||||
log.Printf("[nightly] phase 2: crawling %d spider91 drive(s)", len(spiderIDs))
|
||||
@@ -266,6 +275,8 @@ func (r *Runner) runPipeline(ctx context.Context) {
|
||||
log.Printf("[nightly] phase 3 migration: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
r.runDedupeAssetCleanupPhase(ctx)
|
||||
}
|
||||
|
||||
// checkDeadline returns true when ctx is already done (runner shutting down or
|
||||
@@ -291,6 +302,19 @@ func (r *Runner) waitIdle(ctx context.Context, phase string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runner) runDedupeAssetCleanupPhase(ctx context.Context) {
|
||||
if r.checkDeadline(ctx, "phase 4") {
|
||||
return
|
||||
}
|
||||
if r.cfg.RunDedupeAssetCleanup == nil {
|
||||
return
|
||||
}
|
||||
log.Printf("[nightly] phase 4: duplicate asset cleanup")
|
||||
if err := r.cfg.RunDedupeAssetCleanup(ctx); err != nil {
|
||||
log.Printf("[nightly] phase 4 duplicate asset cleanup: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// readLastRunDate reads the persisted last_run_date or returns "" when unset.
|
||||
func (r *Runner) readLastRunDate(ctx context.Context) (string, error) {
|
||||
if r.cfg.Settings == nil {
|
||||
|
||||
@@ -114,6 +114,10 @@ func TestRunPipelineHonoursPhaseOrder(t *testing.T) {
|
||||
rec.push("migrate")
|
||||
return nil
|
||||
},
|
||||
RunDedupeAssetCleanup: func(context.Context) error {
|
||||
rec.push("dedupe-cleanup")
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
r.runPipeline(context.Background())
|
||||
@@ -128,6 +132,7 @@ func TestRunPipelineHonoursPhaseOrder(t *testing.T) {
|
||||
"crawl:sp-1",
|
||||
"wait-idle", // after phase 2
|
||||
"migrate",
|
||||
"dedupe-cleanup",
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("call sequence len = %d, want %d; got=%v", len(got), len(want), got)
|
||||
@@ -156,6 +161,10 @@ func TestRunPipelineSkipsMigrationWhenNoSpider91(t *testing.T) {
|
||||
rec.push("migrate")
|
||||
return nil
|
||||
},
|
||||
RunDedupeAssetCleanup: func(context.Context) error {
|
||||
rec.push("dedupe-cleanup")
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
r.runPipeline(context.Background())
|
||||
@@ -165,6 +174,15 @@ func TestRunPipelineSkipsMigrationWhenNoSpider91(t *testing.T) {
|
||||
t.Fatalf("phase 2/3 should be skipped when no spider91 drive, got call %q", c)
|
||||
}
|
||||
}
|
||||
foundCleanup := false
|
||||
for _, c := range rec.snapshot() {
|
||||
if c == "dedupe-cleanup" {
|
||||
foundCleanup = true
|
||||
}
|
||||
}
|
||||
if !foundCleanup {
|
||||
t.Fatalf("dedupe cleanup should still run when spider91 is absent; calls=%v", rec.snapshot())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
|
||||
@@ -186,6 +204,7 @@ func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
|
||||
RunSpider91Crawl: func(context.Context, string) { rec.push("crawl") },
|
||||
WaitPreviewQueuesIdle: func(context.Context) error { rec.push("wait-idle"); return nil },
|
||||
RunMigration: func(context.Context) error { rec.push("migrate"); return nil },
|
||||
RunDedupeAssetCleanup: func(context.Context) error { rec.push("dedupe-cleanup"); return nil },
|
||||
})
|
||||
|
||||
r.runPipeline(ctx)
|
||||
@@ -200,6 +219,9 @@ func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
|
||||
if c == "crawl" || c == "migrate" {
|
||||
t.Fatalf("subsequent phase should not run after cancel, got call %q", c)
|
||||
}
|
||||
if c == "dedupe-cleanup" {
|
||||
t.Fatalf("dedupe cleanup should not run after cancel, got call %q", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -289,7 +289,7 @@ func (g *Generator) generateThumbnailAtOffset(ctx context.Context, link *drives.
|
||||
args = append(args,
|
||||
"-i", ffmpegLink.URL,
|
||||
"-frames:v", "1",
|
||||
"-vf", fmt.Sprintf("scale=%d:-2", g.cfg.Width),
|
||||
"-vf", thumbnailVideoFilter(g.cfg.Width),
|
||||
"-q:v", "3",
|
||||
"-y", dst,
|
||||
)
|
||||
@@ -307,6 +307,12 @@ func (g *Generator) generateThumbnailAtOffset(ctx context.Context, link *drives.
|
||||
return nil
|
||||
}
|
||||
|
||||
func thumbnailVideoFilter(width int) string {
|
||||
// FFmpeg 7 rejects non-full-range YUV for MJPEG/JPEG output. Force the
|
||||
// scaled frame into a JPEG-friendly full-range pixel format before encode.
|
||||
return fmt.Sprintf("scale=%d:-2:out_range=pc,format=yuvj420p", width)
|
||||
}
|
||||
|
||||
func thumbnailOffsetFallbackAllowed(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
@@ -923,6 +929,7 @@ func ffmpegOutputLooksRateLimited(output []byte) bool {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "throttl") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "rate-limit") ||
|
||||
strings.Contains(text, "server returned 429")
|
||||
@@ -968,7 +975,6 @@ type Worker struct {
|
||||
queue videoQueue
|
||||
|
||||
RateLimitCooldown time.Duration
|
||||
BeforeTask func(context.Context) bool
|
||||
rateLimit rateLimitState
|
||||
activity taskActivity
|
||||
}
|
||||
@@ -978,7 +984,7 @@ func NewWorker(gen TeaserGenerator, cat *catalog.Catalog, drv drives.Drive) *Wor
|
||||
Gen: gen,
|
||||
Catalog: cat,
|
||||
Drive: drv,
|
||||
ch: make(chan *catalog.Video, 4096),
|
||||
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1029,6 +1035,7 @@ type ThumbWorker struct {
|
||||
const (
|
||||
defaultTransientMediaCooldown = 5 * time.Minute
|
||||
defaultGenerationRateLimitCooldown = 5 * time.Minute
|
||||
defaultWorkerQueueSize = 10000
|
||||
maxPreviewTeaserSizeBytes int64 = 5 * 1024 * 1024 * 1024
|
||||
previewStatusSkipped = "skipped"
|
||||
)
|
||||
@@ -1168,7 +1175,7 @@ func NewThumbWorker(gen ThumbnailGenerator, cat *catalog.Catalog, drv drives.Dri
|
||||
Gen: gen,
|
||||
Catalog: cat,
|
||||
Drive: drv,
|
||||
ch: make(chan *catalog.Video, 4096),
|
||||
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1323,10 +1330,6 @@ func (w *ThumbWorker) Run(ctx context.Context) {
|
||||
|
||||
func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
defer w.queue.release(v)
|
||||
if w.BeforeTask != nil && !w.BeforeTask(ctx) {
|
||||
return
|
||||
}
|
||||
|
||||
w.activity.start(v)
|
||||
defer w.activity.done()
|
||||
if !waitForRateLimitCooldown(ctx, &w.rateLimit, "preview", w.Drive) {
|
||||
@@ -1482,29 +1485,53 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
|
||||
if w.skipIfRateLimited(v) {
|
||||
return
|
||||
}
|
||||
if current, err := w.Catalog.GetVideo(ctx, v.ID); err == nil {
|
||||
if current.ThumbnailURL != "" {
|
||||
queued := v
|
||||
current := v
|
||||
if loaded, err := w.Catalog.GetVideo(ctx, v.ID); err == nil {
|
||||
if loaded.PreviewLocal == "" {
|
||||
loaded.PreviewLocal = queued.PreviewLocal
|
||||
}
|
||||
current = loaded
|
||||
v = loaded
|
||||
if loaded.ThumbnailURL != "" && loaded.DurationSeconds > 0 {
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
|
||||
return
|
||||
}
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "pending"})
|
||||
link, err := w.Drive.StreamURL(ctx, v.FileID)
|
||||
if err != nil {
|
||||
if localLink, ok := localPreviewLink(v); ok {
|
||||
link = localLink
|
||||
} else {
|
||||
if w.pauseForRecoverableError(err, "streamURL", v.Title) {
|
||||
if current.ThumbnailURL != "" {
|
||||
if current.DurationSeconds <= 0 {
|
||||
link, err := w.streamLink(ctx, current)
|
||||
if err != nil {
|
||||
if w.pauseForRecoverableError(err, "streamURL", current.Title) {
|
||||
return
|
||||
}
|
||||
log.Printf("[thumb] probe streamURL %s: %v", current.Title, err)
|
||||
} else if w.probeDuration(ctx, current, link) {
|
||||
return
|
||||
}
|
||||
log.Printf("[thumb] streamURL %s: %v", v.Title, err)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, current.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
|
||||
return
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "pending"})
|
||||
link, err := w.streamLink(ctx, v)
|
||||
if err != nil {
|
||||
if w.pauseForRecoverableError(err, "streamURL", v.Title) {
|
||||
return
|
||||
}
|
||||
log.Printf("[thumb] streamURL %s: %v", v.Title, err)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
|
||||
return
|
||||
}
|
||||
if w.probeDuration(ctx, v, link) {
|
||||
return
|
||||
}
|
||||
|
||||
if err := w.generateThumbnailFromLink(ctx, v, link); err != nil {
|
||||
if localLink, ok := localPreviewLink(v); ok && link.URL != localLink.URL {
|
||||
if w.probeDuration(ctx, v, localLink) {
|
||||
return
|
||||
}
|
||||
if localErr := w.generateThumbnailFromLink(ctx, v, localLink); localErr == nil {
|
||||
return
|
||||
}
|
||||
@@ -1518,6 +1545,38 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
|
||||
}
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) streamLink(ctx context.Context, v *catalog.Video) (*drives.StreamLink, error) {
|
||||
link, err := w.Drive.StreamURL(ctx, v.FileID)
|
||||
if err == nil {
|
||||
return link, nil
|
||||
}
|
||||
if localLink, ok := localPreviewLink(v); ok {
|
||||
return localLink, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) probeDuration(ctx context.Context, v *catalog.Video, link *drives.StreamLink) bool {
|
||||
if v.DurationSeconds > 0 {
|
||||
return false
|
||||
}
|
||||
dur, err := w.Gen.Probe(ctx, link)
|
||||
if err == nil {
|
||||
if dur > 0 {
|
||||
v.DurationSeconds = int(dur)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
DurationSeconds: int(dur),
|
||||
})
|
||||
}
|
||||
return false
|
||||
}
|
||||
if w.pauseForRecoverableError(err, "probe", v.Title) {
|
||||
return true
|
||||
}
|
||||
log.Printf("[thumb] probe %s: %v", v.Title, err)
|
||||
return false
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) generateThumbnailFromLink(ctx context.Context, v *catalog.Video, link *drives.StreamLink) error {
|
||||
if _, err := w.Gen.GenerateThumbnail(ctx, link, v.ID, 0); err != nil {
|
||||
return err
|
||||
|
||||
@@ -161,6 +161,16 @@ func TestThumbnailOffsetsUseFiveSecondsWithEarlyFallbacks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailVideoFilterUsesFullRangeJPEGPixelFormat(t *testing.T) {
|
||||
got := thumbnailVideoFilter(480)
|
||||
if !strings.Contains(got, "scale=480:-2:out_range=pc") {
|
||||
t.Fatalf("thumbnail filter = %q, want full-range scale output", got)
|
||||
}
|
||||
if !strings.Contains(got, "format=yuvj420p") {
|
||||
t.Fatalf("thumbnail filter = %q, want JPEG-friendly pixel format", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailOffsetFallbackAllowedForEmptyOutputAndTimeouts(t *testing.T) {
|
||||
for _, err := range []error{
|
||||
errors.New("ffmpeg thumb produced empty file, stderr: "),
|
||||
|
||||
@@ -13,11 +13,11 @@ import (
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
|
||||
func TestThumbWorkerUpdatesThumbnailAndDurationWithoutChangingPreviewStatus(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-video")
|
||||
|
||||
gen := &fakeThumbGenerator{}
|
||||
gen := &fakeThumbGenerator{probeDuration: 42}
|
||||
drv := &previewFakeDrive{}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
@@ -33,8 +33,8 @@ func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
|
||||
if got.PreviewStatus != "pending" {
|
||||
t.Fatalf("preview status = %q, want pending", got.PreviewStatus)
|
||||
}
|
||||
if got.DurationSeconds != 0 {
|
||||
t.Fatalf("duration = %d, want unchanged", got.DurationSeconds)
|
||||
if got.DurationSeconds != 42 {
|
||||
t.Fatalf("duration = %d, want probed duration", got.DurationSeconds)
|
||||
}
|
||||
if gen.thumbnailVideoID != video.ID {
|
||||
t.Fatalf("thumbnail video id = %q, want %q", gen.thumbnailVideoID, video.ID)
|
||||
@@ -42,14 +42,53 @@ func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
|
||||
if gen.thumbnailDuration != 0 {
|
||||
t.Fatalf("thumbnail duration = %.1f, want fixed-offset thumbnail generation", gen.thumbnailDuration)
|
||||
}
|
||||
if gen.probeCalls != 0 {
|
||||
t.Fatalf("probe calls = %d, want 0 for thumbnail generation", gen.probeCalls)
|
||||
if gen.probeCalls != 1 {
|
||||
t.Fatalf("probe calls = %d, want 1 for thumbnail generation", gen.probeCalls)
|
||||
}
|
||||
if drv.streamFileID != video.FileID {
|
||||
t.Fatalf("stream file id = %q, want %q", drv.streamFileID, video.FileID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerBackfillsDurationWhenThumbnailAlreadyExists(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-existing-thumbnail")
|
||||
video.ThumbnailURL = "/p/thumb/" + video.ID
|
||||
if err := cat.UpsertVideo(ctx, video); err != nil {
|
||||
t.Fatalf("update video: %v", err)
|
||||
}
|
||||
|
||||
gen := &fakeThumbGenerator{probeDuration: 19}
|
||||
drv := &previewFakeDrive{}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
worker.process(ctx, video)
|
||||
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DurationSeconds != 19 {
|
||||
t.Fatalf("duration = %d, want probed duration", got.DurationSeconds)
|
||||
}
|
||||
if got.ThumbnailURL != "/p/thumb/"+video.ID {
|
||||
t.Fatalf("thumbnail = %q, want unchanged existing thumbnail", got.ThumbnailURL)
|
||||
}
|
||||
ready, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "ready", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list ready thumbnails: %v", err)
|
||||
}
|
||||
if len(ready) != 1 || ready[0].ID != video.ID {
|
||||
t.Fatalf("ready thumbnails = %#v, want only %s", ready, video.ID)
|
||||
}
|
||||
if gen.probeCalls != 1 {
|
||||
t.Fatalf("probe calls = %d, want 1", gen.probeCalls)
|
||||
}
|
||||
if gen.thumbnailVideoID != "" {
|
||||
t.Fatalf("thumbnail generation video id = %q, want no regeneration", gen.thumbnailVideoID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerFallsBackToLocalPreviewWhenDriveStreamFails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-local-preview")
|
||||
@@ -469,12 +508,17 @@ type fakeThumbGenerator struct {
|
||||
thumbnailDuration float64
|
||||
thumbnailURL string
|
||||
probeCalls int
|
||||
probeDuration float64
|
||||
probeErr error
|
||||
generateErr error
|
||||
}
|
||||
|
||||
func (g *fakeThumbGenerator) Probe(context.Context, *drives.StreamLink) (float64, error) {
|
||||
g.probeCalls++
|
||||
return 42, nil
|
||||
if g.probeErr != nil {
|
||||
return 0, g.probeErr
|
||||
}
|
||||
return g.probeDuration, nil
|
||||
}
|
||||
|
||||
func (g *fakeThumbGenerator) GenerateThumbnail(_ context.Context, link *drives.StreamLink, videoID string, duration float64) (string, error) {
|
||||
@@ -568,7 +612,6 @@ func (d *previewFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
}
|
||||
func (d *previewFakeDrive) RootID() string { return "root" }
|
||||
|
||||
|
||||
func TestWorkerWaitIdleReturnsImmediatelyWhenQueueEmpty(t *testing.T) {
|
||||
worker := NewWorker(&fakeTeaserGenerator{}, nil, &previewFakeDrive{})
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -144,13 +145,15 @@ func (p *Proxy) ServeStream(w http.ResponseWriter, r *http.Request, driveID, fil
|
||||
// 302 之后浏览器用自己的 UA 直连,CDN 仍然认签名
|
||||
// - pikpak:与 OpenList 一致,WebContentLink / media link 都是自签 URL,
|
||||
// CDN 不校验请求头,直连可获得最佳带宽并避免占用 backend 出站
|
||||
// - onedrive:Microsoft Graph 返回的 @microsoft.graph.downloadUrl 是短期
|
||||
// 免鉴权下载 URL,不需要后端继续代传视频字节
|
||||
//
|
||||
// 其余网盘(如 OneDrive / 沃盘 / 夸克等)仍走反代,因为它们的下载
|
||||
// 其余网盘(如沃盘 / 夸克等)仍走反代,因为它们的下载
|
||||
// 链接通常需要随请求带上后端持有的 Cookie / Authorization / Range
|
||||
// 的特殊处理,浏览器拿不到这些上下文。
|
||||
func shouldRedirect(d drives.Drive) bool {
|
||||
switch d.Kind() {
|
||||
case "p115", "pikpak":
|
||||
case "p115", "pikpak", "onedrive":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -169,6 +172,11 @@ func (p *Proxy) serve(w http.ResponseWriter, r *http.Request, link *drives.Strea
|
||||
http.Error(w, "bad upstream url", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
if localPath, ok := localFilePath(u, link.URL); ok {
|
||||
w.Header().Set("Cache-Control", "private, max-age=300")
|
||||
http.ServeFile(w, r, localPath)
|
||||
return
|
||||
}
|
||||
req, err := http.NewRequestWithContext(r.Context(), r.Method, u.String(), nil)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
@@ -211,6 +219,19 @@ func (p *Proxy) ServeLocal(w http.ResponseWriter, r *http.Request, path string)
|
||||
http.ServeFile(w, r, path)
|
||||
}
|
||||
|
||||
func localFilePath(u *url.URL, raw string) (string, bool) {
|
||||
if u == nil {
|
||||
return "", false
|
||||
}
|
||||
if u.Scheme == "file" && u.Path != "" {
|
||||
return u.Path, true
|
||||
}
|
||||
if u.Scheme == "" && u.Host == "" && filepath.IsAbs(raw) {
|
||||
return raw, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
var errDriveNotFound = &httpError{Code: http.StatusNotFound, Msg: "drive not found"}
|
||||
|
||||
type httpError struct {
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -149,6 +151,61 @@ func TestServeStreamPikPakSetsRedirectHeaders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamRedirectsOneDrive(t *testing.T) {
|
||||
reg := NewRegistry()
|
||||
drv := &proxyFakeSimpleDrive{
|
||||
kind: "onedrive",
|
||||
url: "https://public.onedrive.example/video.mp4",
|
||||
}
|
||||
reg.Set("onedrive", drv)
|
||||
|
||||
p := New(reg)
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/onedrive/file-1", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
p.ServeStream(rr, req, "onedrive", "file-1")
|
||||
|
||||
if rr.Code != http.StatusFound {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusFound)
|
||||
}
|
||||
if got := rr.Header().Get("Location"); got != "https://public.onedrive.example/video.mp4" {
|
||||
t.Fatalf("Location = %q", got)
|
||||
}
|
||||
if drv.calls != 1 {
|
||||
t.Fatalf("link calls = %d, want 1", drv.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamServesLocalFilePath(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "video.mp4")
|
||||
if err := os.WriteFile(path, []byte("0123456789"), 0o644); err != nil {
|
||||
t.Fatalf("write local file: %v", err)
|
||||
}
|
||||
reg := NewRegistry()
|
||||
drv := &proxyFakeSimpleDrive{
|
||||
kind: "localstorage",
|
||||
url: path,
|
||||
}
|
||||
reg.Set("local", drv)
|
||||
|
||||
p := New(reg)
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/local/file-1", nil)
|
||||
req.Header.Set("Range", "bytes=2-5")
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
p.ServeStream(rr, req, "local", "file-1")
|
||||
|
||||
if rr.Code != http.StatusPartialContent {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusPartialContent)
|
||||
}
|
||||
if got := rr.Body.String(); got != "2345" {
|
||||
t.Fatalf("body = %q, want range bytes", got)
|
||||
}
|
||||
if drv.calls != 1 {
|
||||
t.Fatalf("link calls = %d, want 1", drv.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func requestPikPak(t *testing.T, p *Proxy, driveID, fileID, ua string) {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/"+driveID+"/"+fileID, nil)
|
||||
@@ -192,3 +249,36 @@ func (d *proxyFakePikPakDrive) EnsureDir(context.Context, string) (string, error
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakePikPakDrive) RootID() string { return "0" }
|
||||
|
||||
type proxyFakeSimpleDrive struct {
|
||||
kind string
|
||||
url string
|
||||
calls int
|
||||
}
|
||||
|
||||
func (d *proxyFakeSimpleDrive) Kind() string { return d.kind }
|
||||
func (d *proxyFakeSimpleDrive) ID() string { return d.kind }
|
||||
func (d *proxyFakeSimpleDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
|
||||
d.calls++
|
||||
return &drives.StreamLink{
|
||||
URL: d.url,
|
||||
Headers: http.Header{},
|
||||
Expires: time.Now().Add(10 * time.Minute),
|
||||
}, nil
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) RootID() string { return "0" }
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Package spider91migrate 周期性把 spider91 drive 下载到本地的视频
|
||||
// 上传到一个指定的目标 drive 目录(PikPak 或 115),上传成功后:
|
||||
// 上传到一个指定的目标 drive 目录(PikPak、115 或 OneDrive),上传成功后:
|
||||
//
|
||||
// - 改写 catalog 行:drive_id / file_id / content_hash 改成目标盘的;
|
||||
// 视频自身的 id 不变(仍是 spider91-<driveID>-<viewkey>),video_tags、
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/onedrive"
|
||||
"github.com/video-site/backend/internal/drives/p115"
|
||||
"github.com/video-site/backend/internal/drives/pikpak"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
@@ -39,12 +40,14 @@ import (
|
||||
// 这一层抽象把"迁移调用方"和"具体盘的 SDK 协议"解耦:
|
||||
// - PikPak 走 GCID + OSS PutObject(pikpak.UploadResult)
|
||||
// - 115 走 SHA1 + 秒传 / OSS / 分片(p115.UploadResult)
|
||||
// - OneDrive 走 SHA1 + 小文件 PUT / 大文件 upload session
|
||||
//
|
||||
// 两个返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
|
||||
// 各家返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
|
||||
type uploadTarget interface {
|
||||
ID() string
|
||||
Kind() string
|
||||
RootID() string
|
||||
EnsureDir(ctx context.Context, pathFromRoot string) (string, error)
|
||||
UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error)
|
||||
Rename(ctx context.Context, fileID, newName string) error
|
||||
}
|
||||
@@ -52,7 +55,7 @@ type uploadTarget interface {
|
||||
// UploadResult 是 uploadTarget.UploadAndReportHash 的归一返回。
|
||||
//
|
||||
// FileID 目标盘上的新文件 ID;
|
||||
// Hash GCID(PikPak)或 SHA1 HEX 大写(115),写入 catalog.content_hash 用于跨盘去重;
|
||||
// Hash GCID(PikPak)或 SHA1 HEX(115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;
|
||||
// Size 实际上传字节数。
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
@@ -60,7 +63,9 @@ type UploadResult struct {
|
||||
Size int64
|
||||
}
|
||||
|
||||
// pikpakAdapter / p115Adapter 把具体 driver 包装成 uploadTarget。
|
||||
const spider91UploadDirName = "91 Spider"
|
||||
|
||||
// pikpakAdapter / p115Adapter / onedriveAdapter 把具体 driver 包装成 uploadTarget。
|
||||
//
|
||||
// 之所以不让 driver 直接实现 uploadTarget:
|
||||
//
|
||||
@@ -74,6 +79,9 @@ type pikpakAdapter struct {
|
||||
func (a *pikpakAdapter) ID() string { return a.d.ID() }
|
||||
func (a *pikpakAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *pikpakAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *pikpakAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *pikpakAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
@@ -92,6 +100,9 @@ type p115Adapter struct {
|
||||
func (a *p115Adapter) ID() string { return a.d.ID() }
|
||||
func (a *p115Adapter) Kind() string { return a.d.Kind() }
|
||||
func (a *p115Adapter) RootID() string { return a.d.RootID() }
|
||||
func (a *p115Adapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *p115Adapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportSha1(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
@@ -103,6 +114,27 @@ func (a *p115Adapter) Rename(ctx context.Context, fileID, newName string) error
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
type onedriveAdapter struct {
|
||||
d *onedrive.Driver
|
||||
}
|
||||
|
||||
func (a *onedriveAdapter) ID() string { return a.d.ID() }
|
||||
func (a *onedriveAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *onedriveAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *onedriveAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *onedriveAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
return UploadResult{FileID: res.FileID, Hash: res.Hash, Size: res.Size}, nil
|
||||
}
|
||||
func (a *onedriveAdapter) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
// adaptUploadTarget 把通用 drive 包装成 uploadTarget。
|
||||
// 不支持的盘 kind 返回 error;调用方静默跳过。
|
||||
func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
@@ -111,6 +143,8 @@ func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
return &pikpakAdapter{d: v}, nil
|
||||
case *p115.Driver:
|
||||
return &p115Adapter{d: v}, nil
|
||||
case *onedrive.Driver:
|
||||
return &onedriveAdapter{d: v}, nil
|
||||
case uploadTarget:
|
||||
// 测试或自定义实现可以直接传入;优先使用具体类型分支以拿到适配器。
|
||||
return v, nil
|
||||
@@ -272,7 +306,7 @@ func (m *Migrator) runOnce(ctx context.Context) {
|
||||
|
||||
target, pp, err := m.resolveTarget()
|
||||
if err != nil {
|
||||
// 没目标就静默 —— 用户可能还没配 PikPak drive
|
||||
// 没目标就静默 —— 用户选择了本地保存,或还没配 115/PikPak drive。
|
||||
return
|
||||
}
|
||||
|
||||
@@ -382,7 +416,7 @@ func (m *Migrator) spider91Drives() []*spider91.Driver {
|
||||
// - 列出 spider91 drive 本地 videos/ 目录所有 mp4 文件,按 mtime 降序排
|
||||
// - 跳过最新 KeepLatestN 个:这些是用户希望保留在本地的最新爬取
|
||||
// - 对剩下的(更旧)逐个处理:
|
||||
// - 还没迁移(drive_id 仍是 src.ID())→ 上传到 PikPak + 改 catalog + 删本地
|
||||
// - 还没迁移(drive_id 仍是 src.ID())→ 上传到目标盘 + 改 catalog + 删本地
|
||||
// - 已经迁移过但本地还有残留 → 仅删本地(兜底)
|
||||
//
|
||||
// KeepLatestN < 0 时不保护任何本地文件,全部尝试迁移(旧行为,主要给测试用)。
|
||||
@@ -484,7 +518,7 @@ func (m *Migrator) migrateDrive(ctx context.Context, src *spider91.Driver, targe
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
// migrateOne 把单条 spider91 视频上传到 PikPak 并改写 catalog。
|
||||
// migrateOne 把单条 spider91 视频上传到目标盘并改写 catalog。
|
||||
// 返回 (true, nil) 表示真的迁了一条;(false, nil) 表示跳过(本地文件已不在等);
|
||||
// (false, err) 表示真出错。
|
||||
func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider91.Driver, targetDriveID string, pp uploadTarget) (bool, error) {
|
||||
@@ -511,15 +545,19 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// 上传到目标盘的根目录(用户配置的目标 drive 的 rootID)。
|
||||
// 上传到目标盘 rootID 下的固定 "91 Spider" 子目录。若用户把目标盘 rootID
|
||||
// 配成某个自定义目录,这里会在该自定义目录下查找/创建 "91 Spider"。
|
||||
// 上传名走 desiredPikPakName 算出来的方案 B 格式:
|
||||
//
|
||||
// <sanitized title>-<viewkey 后 8 位>.<ext>
|
||||
//
|
||||
// 这样网盘 Web 端列出来的文件名能直接看出是哪个视频,
|
||||
// 又用 viewkey 后 8 位避免同标题撞名。两个目标盘(PikPak / 115)共用同一格式,
|
||||
// 又用 viewkey 后 8 位避免同标题撞名。所有目标盘共用同一格式,
|
||||
// 简化前端 / catalog 的认知。
|
||||
parent := pp.RootID()
|
||||
parent, err := pp.EnsureDir(ctx, spider91UploadDirName)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s ensure %q dir: %w", pp.Kind(), spider91UploadDirName, err)
|
||||
}
|
||||
uploadName := desiredPikPakName(v.Title, extractViewKey(v.ID), v.Ext)
|
||||
res, err := pp.UploadAndReportHash(ctx, parent, uploadName, f, info.Size())
|
||||
if err != nil {
|
||||
@@ -639,7 +677,7 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driv
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// backfillFileNames 扫描目标 drive(PikPak 或 115)下所有 spider91-* 起始 ID 的视频,
|
||||
// backfillFileNames 扫描目标 drive(PikPak、115 或 OneDrive)下所有 spider91-* 起始 ID 的视频,
|
||||
// 对文件名不是 desiredPikPakName(...) 期望格式的,调 target.Rename 修正,
|
||||
// 并把 catalog.file_name 同步到新名字。
|
||||
//
|
||||
|
||||
@@ -53,6 +53,8 @@ type fakePikPak struct {
|
||||
uploadFunc func(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error)
|
||||
mu sync.Mutex
|
||||
gotBodies map[string][]byte
|
||||
gotParents map[string]string
|
||||
ensureCalls []string
|
||||
// renameCalls 记录每次 Rename 的 fileID->newName 历史,用于 backfill 测试断言。
|
||||
renameCalls map[string]string
|
||||
}
|
||||
@@ -62,6 +64,7 @@ func newFakePikPak(id, rootID string) *fakePikPak {
|
||||
id: id,
|
||||
rootID: rootID,
|
||||
gotBodies: make(map[string][]byte),
|
||||
gotParents: make(map[string]string),
|
||||
renameCalls: make(map[string]string),
|
||||
}
|
||||
}
|
||||
@@ -80,8 +83,11 @@ func (d *fakePikPak) StreamURL(context.Context, string) (*drives.StreamLink, err
|
||||
func (d *fakePikPak) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakePikPak) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
func (d *fakePikPak) EnsureDir(_ context.Context, pathFromRoot string) (string, error) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.ensureCalls = append(d.ensureCalls, pathFromRoot)
|
||||
return d.rootID + "/" + pathFromRoot, nil
|
||||
}
|
||||
func (d *fakePikPak) Rename(_ context.Context, fileID, newName string) error {
|
||||
d.mu.Lock()
|
||||
@@ -99,6 +105,7 @@ func (d *fakePikPak) UploadAndReportHash(ctx context.Context, parentID, name str
|
||||
body, _ := io.ReadAll(r)
|
||||
d.mu.Lock()
|
||||
d.gotBodies[name] = body
|
||||
d.gotParents[name] = parentID
|
||||
d.mu.Unlock()
|
||||
return UploadResult{
|
||||
FileID: "remote-" + name,
|
||||
@@ -127,6 +134,19 @@ func (d *fakeP115) Kind() string { return "p115" }
|
||||
var _ drives.Drive = (*fakeP115)(nil)
|
||||
var _ uploadTarget = (*fakeP115)(nil)
|
||||
|
||||
type fakeOneDrive struct {
|
||||
*fakePikPak
|
||||
}
|
||||
|
||||
func newFakeOneDrive(id, rootID string) *fakeOneDrive {
|
||||
return &fakeOneDrive{fakePikPak: newFakePikPak(id, rootID)}
|
||||
}
|
||||
|
||||
func (d *fakeOneDrive) Kind() string { return "onedrive" }
|
||||
|
||||
var _ drives.Drive = (*fakeOneDrive)(nil)
|
||||
var _ uploadTarget = (*fakeOneDrive)(nil)
|
||||
|
||||
// TestBackfillFileNamesRenamesOnlyMismatchedSpider91Videos 验证回填逻辑:
|
||||
//
|
||||
// - 已经是期望格式的不会再调 Rename(幂等)
|
||||
@@ -347,6 +367,12 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
|
||||
if _, ok := pp.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("PikPak did not receive expected upload name %q (got names: %v)", wantName, keysOf(pp.gotBodies))
|
||||
}
|
||||
if gotParent := pp.gotParents[wantName]; gotParent != "pikpak-root-id/"+spider91UploadDirName {
|
||||
t.Fatalf("upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(pp.ensureCalls) != 1 || pp.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("ensure calls = %#v, want %q", pp.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
@@ -588,7 +614,7 @@ func TestRunOnceKeepsAllLocalWhenWithinKeepWindow(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestRunOnceMigratesOnlyOlderFilesBeyondKeepWindow 验证:本地文件数 > KeepLatestN 时
|
||||
// 按 mtime 降序保留最新 N 个,超出部分(更旧的)才上传到 PikPak。
|
||||
// 按 mtime 降序保留最新 N 个,超出部分(更旧的)才上传到目标盘。
|
||||
func TestRunOnceMigratesOnlyOlderFilesBeyondKeepWindow(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src, _ := setupSpider91(t)
|
||||
@@ -841,7 +867,6 @@ func TestNonCaptchaErrorDoesNotTriggerCooldown(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// TestRunOnceMigratesToP115Target 验证:当目标 drive 是 115(kind="p115")时,
|
||||
// migrator 也能正确把 spider91 视频上传过去并改写 catalog。
|
||||
//
|
||||
@@ -885,6 +910,12 @@ func TestRunOnceMigratesToP115Target(t *testing.T) {
|
||||
if _, ok := target.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("p115 did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
|
||||
}
|
||||
if gotParent := target.gotParents[wantName]; gotParent != "p115-root-cid/"+spider91UploadDirName {
|
||||
t.Fatalf("p115 upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("p115 ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
@@ -906,7 +937,67 @@ func TestRunOnceMigratesToP115Target(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak 也不是 115 时,
|
||||
func TestRunOnceMigratesToOneDriveTarget(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src, _ := setupSpider91(t)
|
||||
target := newFakeOneDrive("onedrive-target", "onedrive-root")
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(target)
|
||||
|
||||
now := time.Now()
|
||||
id := writeSpider91Video(t, cat, src, "vk-od-001", ".mp4", []byte("video bytes onedrive"), now)
|
||||
|
||||
m := New(Config{
|
||||
Catalog: cat,
|
||||
Registry: reg,
|
||||
GetTargetDriveID: func() string { return target.ID() },
|
||||
KeepLatestN: -1,
|
||||
})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
if target.uploadCalls != 1 {
|
||||
t.Fatalf("onedrive upload calls = %d, want 1", target.uploadCalls)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DriveID != target.ID() {
|
||||
t.Fatalf("drive_id = %q, want %q", got.DriveID, target.ID())
|
||||
}
|
||||
wantName := "Sample vk-od-001-001.mp4"
|
||||
if _, ok := target.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("onedrive did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
|
||||
}
|
||||
if gotParent := target.gotParents[wantName]; gotParent != "onedrive-root/"+spider91UploadDirName {
|
||||
t.Fatalf("onedrive upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("onedrive ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
if got.FileName != wantName {
|
||||
t.Fatalf("file_name = %q, want %q", got.FileName, wantName)
|
||||
}
|
||||
if got.ContentHash == "" {
|
||||
t.Fatal("content_hash should be set after onedrive migration")
|
||||
}
|
||||
|
||||
videoPath, _ := src.VideoPath("vk-od-001.mp4")
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local mp4 still exists after onedrive migration or stat error: %v", err)
|
||||
}
|
||||
thumbPath, _ := src.ThumbPath("vk-od-001.jpg")
|
||||
if _, err := os.Stat(thumbPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local thumb still exists after onedrive migration or stat error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115 也不是 OneDrive 时,
|
||||
// resolveTarget 拒绝并返回 error,让 runOnce 静默跳过(不会做破坏性变更)。
|
||||
func TestResolveTargetRejectsUnsupportedKind(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
services:
|
||||
video-site-91:
|
||||
image: ghcr.io/nianzhibai/91:latest
|
||||
container_name: video-site-91
|
||||
ports:
|
||||
- "9191:9191"
|
||||
volumes:
|
||||
- ./data:/opt/video-site-91/data
|
||||
restart: unless-stopped
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
APP_DIR="/opt/video-site-91"
|
||||
DATA_DIR="${VIDEO_DATA_DIR:-$APP_DIR/data}"
|
||||
CONFIG="${VIDEO_CONFIG:-$DATA_DIR/config.yaml}"
|
||||
EXAMPLE="$APP_DIR/config.example.yaml"
|
||||
PORT="${VIDEO_LISTEN_PORT:-9191}"
|
||||
|
||||
mkdir -p "$DATA_DIR" "$DATA_DIR/previews" "$DATA_DIR/uploads" "$DATA_DIR/spider91"
|
||||
|
||||
if [ ! -f "$CONFIG" ]; then
|
||||
if [ ! -f "$EXAMPLE" ]; then
|
||||
echo "[entrypoint] missing config template: $EXAMPLE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$(dirname "$CONFIG")"
|
||||
cp "$EXAMPLE" "$CONFIG"
|
||||
|
||||
SECRET="$(openssl rand -hex 32)"
|
||||
sed -i -E "s#^([[:space:]]*listen:[[:space:]]*).*\$#\1\"0.0.0.0:${PORT}\"#" "$CONFIG"
|
||||
sed -i -E "s#^([[:space:]]*session_secret:[[:space:]]*).*\$#\1\"${SECRET}\"#" "$CONFIG"
|
||||
sed -i -E "s#^([[:space:]]*db_path:[[:space:]]*).*\$#\1\"${DATA_DIR}/video-site.db\"#" "$CONFIG"
|
||||
sed -i -E "s#^([[:space:]]*local_preview_dir:[[:space:]]*).*\$#\1\"${DATA_DIR}/previews\"#" "$CONFIG"
|
||||
chmod 600 "$CONFIG"
|
||||
|
||||
echo "[entrypoint] generated $CONFIG"
|
||||
else
|
||||
echo "[entrypoint] using existing $CONFIG"
|
||||
fi
|
||||
|
||||
if [ -n "${VIDEO_VERSION_FILE:-}" ] && [ -n "${VIDEO_IMAGE_VERSION:-}" ] && [ ! -f "$VIDEO_VERSION_FILE" ]; then
|
||||
mkdir -p "$(dirname "$VIDEO_VERSION_FILE")"
|
||||
printf '%s\n' "$VIDEO_IMAGE_VERSION" > "$VIDEO_VERSION_FILE"
|
||||
fi
|
||||
|
||||
exec "$@"
|
||||
+224
-23
@@ -11,6 +11,12 @@ VERSION="${VERSION:-latest}"
|
||||
GH_PROXY="${GH_PROXY:-}"
|
||||
CONFIGURE_UFW="${CONFIGURE_UFW:-1}"
|
||||
INSTALL_DEPS="${INSTALL_DEPS:-1}"
|
||||
SELF_UPDATE="${SELF_UPDATE:-1}"
|
||||
FORCE_UPDATE="${FORCE_UPDATE:-0}"
|
||||
INSTALL_SCRIPT_REF="${INSTALL_SCRIPT_REF:-main}"
|
||||
INSTALL_SCRIPT_URL="${INSTALL_SCRIPT_URL:-${GH_PROXY}https://raw.githubusercontent.com/${GITHUB_REPO}/${INSTALL_SCRIPT_REF}/install.sh}"
|
||||
VIDEO_SITE_SKIP_SELF_UPDATE="${VIDEO_SITE_SKIP_SELF_UPDATE:-0}"
|
||||
SERVICE_READY_TIMEOUT="${SERVICE_READY_TIMEOUT:-90}"
|
||||
VERSION_FILE="$INSTALL_PATH/.version"
|
||||
MANAGER_PATH="/usr/local/sbin/${APP_NAME}-manager"
|
||||
COMMAND_LINK="/usr/local/bin/91"
|
||||
@@ -47,7 +53,7 @@ Default action:
|
||||
|
||||
Actions:
|
||||
install Install to $INSTALL_PATH
|
||||
update Download latest release and replace program files, keeping config/data
|
||||
update Refresh manager script, download latest release, and keep config/data
|
||||
restart Restart service
|
||||
stop Stop service
|
||||
status Show service status
|
||||
@@ -62,6 +68,11 @@ Options via environment:
|
||||
GH_PROXY=$GH_PROXY
|
||||
INSTALL_DEPS=$INSTALL_DEPS
|
||||
CONFIGURE_UFW=$CONFIGURE_UFW
|
||||
SELF_UPDATE=$SELF_UPDATE
|
||||
FORCE_UPDATE=$FORCE_UPDATE
|
||||
INSTALL_SCRIPT_REF=$INSTALL_SCRIPT_REF
|
||||
INSTALL_SCRIPT_URL=$INSTALL_SCRIPT_URL
|
||||
SERVICE_READY_TIMEOUT=$SERVICE_READY_TIMEOUT
|
||||
|
||||
Examples:
|
||||
sudo bash install.sh
|
||||
@@ -158,6 +169,30 @@ download_file() {
|
||||
return 1
|
||||
}
|
||||
|
||||
backup_install_files() {
|
||||
local backup="$1"
|
||||
mkdir -p "$backup"
|
||||
cp -a "$INSTALL_PATH/server" "$backup/server"
|
||||
for item in dist config.example.yaml 91VideoSpider config.yaml .version; do
|
||||
if [[ -e "$INSTALL_PATH/$item" ]]; then
|
||||
cp -a "$INSTALL_PATH/$item" "$backup/$item"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
restore_install_files() {
|
||||
local backup="$1"
|
||||
mkdir -p "$INSTALL_PATH"
|
||||
cp -a "$backup/server" "$INSTALL_PATH/server"
|
||||
for item in dist config.example.yaml 91VideoSpider config.yaml .version; do
|
||||
rm -rf "${INSTALL_PATH:?}/$item"
|
||||
if [[ -e "$backup/$item" ]]; then
|
||||
cp -a "$backup/$item" "$INSTALL_PATH/$item"
|
||||
fi
|
||||
done
|
||||
chmod +x "$INSTALL_PATH/server"
|
||||
}
|
||||
|
||||
prepare_config() {
|
||||
local cfg="$INSTALL_PATH/config.yaml"
|
||||
local example="$INSTALL_PATH/config.example.yaml"
|
||||
@@ -200,6 +235,8 @@ RestartSec=5
|
||||
TimeoutStopSec=20
|
||||
Environment=VIDEO_CONFIG=${INSTALL_PATH}/config.yaml
|
||||
Environment=VIDEO_FRONTEND_DIR=${INSTALL_PATH}/dist
|
||||
Environment=VIDEO_VERSION_FILE=${VERSION_FILE}
|
||||
Environment=VIDEO_GITHUB_REPO=${GITHUB_REPO}
|
||||
Environment=HOME=/root
|
||||
Environment=PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
|
||||
LimitNOFILE=65536
|
||||
@@ -217,12 +254,72 @@ EOF
|
||||
install_cli() {
|
||||
local src
|
||||
src="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")"
|
||||
if [[ -f "$src" ]]; then
|
||||
cp "$src" "$MANAGER_PATH"
|
||||
chmod 755 "$MANAGER_PATH"
|
||||
ln -sf "$MANAGER_PATH" "$COMMAND_LINK"
|
||||
ln -sf "$MANAGER_PATH" "$APP_COMMAND_LINK"
|
||||
install_cli_from_file "$src"
|
||||
}
|
||||
|
||||
install_cli_from_file() {
|
||||
local src="$1"
|
||||
local tmp
|
||||
[[ -f "$src" ]] || return 0
|
||||
mkdir -p "$(dirname "$MANAGER_PATH")" "$(dirname "$COMMAND_LINK")" "$(dirname "$APP_COMMAND_LINK")"
|
||||
tmp="${MANAGER_PATH}.tmp.$$"
|
||||
cp "$src" "$tmp"
|
||||
chmod 755 "$tmp"
|
||||
mv "$tmp" "$MANAGER_PATH"
|
||||
ln -sfn "$MANAGER_PATH" "$COMMAND_LINK"
|
||||
ln -sfn "$MANAGER_PATH" "$APP_COMMAND_LINK"
|
||||
}
|
||||
|
||||
self_update_manager() {
|
||||
[[ "$SELF_UPDATE" == "1" ]] || return 1
|
||||
[[ "$VIDEO_SITE_SKIP_SELF_UPDATE" != "1" ]] || return 1
|
||||
[[ -n "$INSTALL_SCRIPT_URL" ]] || return 1
|
||||
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
log "checking latest manager script"
|
||||
if ! download_file "$INSTALL_SCRIPT_URL" "$tmp"; then
|
||||
warn "manager self-update skipped: cannot download $INSTALL_SCRIPT_URL"
|
||||
rm -f "$tmp"
|
||||
return 1
|
||||
fi
|
||||
if ! bash -n "$tmp"; then
|
||||
warn "manager self-update skipped: downloaded script has syntax errors"
|
||||
rm -f "$tmp"
|
||||
return 1
|
||||
fi
|
||||
if [[ -f "$MANAGER_PATH" ]] && cmp -s "$tmp" "$MANAGER_PATH"; then
|
||||
rm -f "$tmp"
|
||||
return 1
|
||||
fi
|
||||
|
||||
install_cli_from_file "$tmp"
|
||||
rm -f "$tmp"
|
||||
log "manager script updated"
|
||||
return 0
|
||||
}
|
||||
|
||||
exec_latest_manager_update() {
|
||||
local env_args=(
|
||||
"VIDEO_SITE_SKIP_SELF_UPDATE=1"
|
||||
"APP_NAME=$APP_NAME"
|
||||
"GITHUB_REPO=$GITHUB_REPO"
|
||||
"INSTALL_PATH=$INSTALL_PATH"
|
||||
"SERVICE_NAME=$SERVICE_NAME"
|
||||
"VERSION=$VERSION"
|
||||
"GH_PROXY=$GH_PROXY"
|
||||
"CONFIGURE_UFW=$CONFIGURE_UFW"
|
||||
"INSTALL_DEPS=$INSTALL_DEPS"
|
||||
"SELF_UPDATE=$SELF_UPDATE"
|
||||
"FORCE_UPDATE=$FORCE_UPDATE"
|
||||
"INSTALL_SCRIPT_REF=$INSTALL_SCRIPT_REF"
|
||||
"INSTALL_SCRIPT_URL=$INSTALL_SCRIPT_URL"
|
||||
"SERVICE_READY_TIMEOUT=$SERVICE_READY_TIMEOUT"
|
||||
)
|
||||
if [[ -n "$FRONTEND_PORT_WAS_SET" ]]; then
|
||||
env_args+=("FRONTEND_PORT=$FRONTEND_PORT")
|
||||
fi
|
||||
exec env "${env_args[@]}" bash "$MANAGER_PATH" update
|
||||
}
|
||||
|
||||
open_firewall_port() {
|
||||
@@ -234,6 +331,55 @@ open_firewall_port() {
|
||||
fi
|
||||
}
|
||||
|
||||
listen_port_from_config() {
|
||||
local cfg="$INSTALL_PATH/config.yaml"
|
||||
local listen="" port
|
||||
if [[ -f "$cfg" ]]; then
|
||||
listen="$(sed -nE 's/^[[:space:]]*listen:[[:space:]]*"?([^" #]+)"?.*/\1/p' "$cfg" | head -n1)"
|
||||
fi
|
||||
port="${listen##*:}"
|
||||
if [[ "$port" =~ ^[0-9]+$ ]]; then
|
||||
printf '%s' "$port"
|
||||
return
|
||||
fi
|
||||
printf '%s' "$FRONTEND_PORT"
|
||||
}
|
||||
|
||||
service_health_url() {
|
||||
printf 'http://127.0.0.1:%s/admin/api/setup' "$(listen_port_from_config)"
|
||||
}
|
||||
|
||||
wait_for_service_ready() {
|
||||
local url deadline
|
||||
url="$(service_health_url)"
|
||||
deadline=$((SECONDS + SERVICE_READY_TIMEOUT))
|
||||
log "waiting for service at $url"
|
||||
while (( SECONDS < deadline )); do
|
||||
if curl -fsS --connect-timeout 2 --max-time 5 "$url" >/dev/null 2>&1; then
|
||||
log "service is ready"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
restart_service_ready() {
|
||||
if systemctl restart "${SERVICE_NAME}.service" && wait_for_service_ready; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
warn "service did not become ready; retrying restart"
|
||||
if systemctl restart "${SERVICE_NAME}.service" && wait_for_service_ready; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
warn "service failed to become ready"
|
||||
systemctl --no-pager --full status "${SERVICE_NAME}.service" || true
|
||||
journalctl -u "${SERVICE_NAME}.service" -n 80 --no-pager || true
|
||||
return 1
|
||||
}
|
||||
|
||||
fetch_and_unpack() {
|
||||
local tmp archive url root
|
||||
tmp="$(mktemp -d)"
|
||||
@@ -271,19 +417,63 @@ fetch_and_unpack() {
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
current_version_from_github() {
|
||||
installed_version() {
|
||||
if [[ -f "$VERSION_FILE" ]]; then
|
||||
head -n1 "$VERSION_FILE" 2>/dev/null | tr -d '\r'
|
||||
fi
|
||||
}
|
||||
|
||||
target_version() {
|
||||
if [[ "$VERSION" != "latest" ]]; then
|
||||
printf '%s' "$VERSION"
|
||||
return
|
||||
fi
|
||||
curl -fsSL "https://api.github.com/repos/${GITHUB_REPO}/releases/latest" \
|
||||
|
||||
local body version effective_url
|
||||
body="$(curl -fsSL \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "User-Agent: video-site-91-installer" \
|
||||
"https://api.github.com/repos/${GITHUB_REPO}/releases/latest" 2>/dev/null || true)"
|
||||
version="$(printf '%s\n' "$body" \
|
||||
| sed -nE 's/.*"tag_name"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/p' \
|
||||
| head -n1)"
|
||||
if [[ -n "$version" ]]; then
|
||||
printf '%s' "$version"
|
||||
return
|
||||
fi
|
||||
|
||||
effective_url="$(curl -fsSLI -o /dev/null -w '%{url_effective}' "$(download_base_url)/$(asset_name)" 2>/dev/null || true)"
|
||||
printf '%s\n' "$effective_url" \
|
||||
| sed -nE 's#.*/releases/download/([^/]+)/.*#\1#p' \
|
||||
| head -n1
|
||||
}
|
||||
|
||||
should_skip_update() {
|
||||
[[ "$FORCE_UPDATE" != "1" ]] || return 1
|
||||
|
||||
local current target
|
||||
current="$(installed_version)"
|
||||
target="$(target_version || true)"
|
||||
|
||||
if [[ -z "$target" ]]; then
|
||||
warn "cannot determine target version; continuing update"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -z "$current" ]]; then
|
||||
log "installed version: unknown"
|
||||
log "target version: $target"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "installed version: $current"
|
||||
log "target version: $target"
|
||||
[[ "$current" == "$target" ]]
|
||||
}
|
||||
|
||||
record_version() {
|
||||
local version
|
||||
version="$(current_version_from_github || true)"
|
||||
version="$(target_version || true)"
|
||||
[[ -n "$version" ]] || version="$VERSION"
|
||||
{
|
||||
echo "$version"
|
||||
@@ -298,7 +488,7 @@ show_success() {
|
||||
version="$(head -n1 "$VERSION_FILE" 2>/dev/null || echo unknown)"
|
||||
|
||||
echo
|
||||
printf "${GREEN}安装完成${RESET}\n"
|
||||
printf '%b安装完成%b\n' "$GREEN" "$RESET"
|
||||
echo "版本:$version"
|
||||
[[ -n "$local_ip" ]] && echo "局域网:http://${local_ip}:${FRONTEND_PORT}/"
|
||||
[[ -n "$public_ip" ]] && echo "公网: http://${public_ip}:${FRONTEND_PORT}/"
|
||||
@@ -319,38 +509,49 @@ install_app() {
|
||||
write_service
|
||||
install_cli
|
||||
open_firewall_port
|
||||
restart_service_ready || die "service failed to start"
|
||||
record_version
|
||||
systemctl restart "${SERVICE_NAME}.service"
|
||||
show_success
|
||||
}
|
||||
|
||||
update_app() {
|
||||
check_system
|
||||
[[ -f "$INSTALL_PATH/server" ]] || die "not installed at $INSTALL_PATH"
|
||||
|
||||
if self_update_manager; then
|
||||
log "re-running update with latest manager script"
|
||||
exec_latest_manager_update
|
||||
fi
|
||||
|
||||
if should_skip_update; then
|
||||
log "already up to date; skipped app update"
|
||||
return 0
|
||||
fi
|
||||
|
||||
check_disk_space
|
||||
install_deps
|
||||
[[ -f "$INSTALL_PATH/server" ]] || die "not installed at $INSTALL_PATH"
|
||||
|
||||
local backup
|
||||
backup="$(mktemp -d)"
|
||||
cp "$INSTALL_PATH/server" "$backup/server"
|
||||
[[ -d "$INSTALL_PATH/dist" ]] && cp -R "$INSTALL_PATH/dist" "$backup/dist"
|
||||
backup_install_files "$backup"
|
||||
|
||||
systemctl stop "${SERVICE_NAME}.service" 2>/dev/null || true
|
||||
if ! fetch_and_unpack; then
|
||||
if ! (fetch_and_unpack && prepare_config && write_service && install_cli); then
|
||||
warn "update failed; restoring previous files"
|
||||
cp "$backup/server" "$INSTALL_PATH/server"
|
||||
rm -rf "$INSTALL_PATH/dist"
|
||||
[[ -d "$backup/dist" ]] && cp -R "$backup/dist" "$INSTALL_PATH/dist"
|
||||
restore_install_files "$backup"
|
||||
systemctl start "${SERVICE_NAME}.service" 2>/dev/null || true
|
||||
rm -rf "$backup"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
prepare_config
|
||||
write_service
|
||||
install_cli
|
||||
if ! restart_service_ready; then
|
||||
warn "new version failed to start; restoring previous files"
|
||||
restore_install_files "$backup"
|
||||
restart_service_ready 2>/dev/null || true
|
||||
rm -rf "$backup"
|
||||
exit 1
|
||||
fi
|
||||
record_version
|
||||
systemctl restart "${SERVICE_NAME}.service"
|
||||
rm -rf "$backup"
|
||||
log "updated"
|
||||
}
|
||||
@@ -430,7 +631,7 @@ main() {
|
||||
;;
|
||||
restart)
|
||||
need_root "$@"
|
||||
systemctl restart "${SERVICE_NAME}.service"
|
||||
restart_service_ready || die "service failed to start"
|
||||
;;
|
||||
stop)
|
||||
need_root "$@"
|
||||
|
||||
Generated
+829
-1531
File diff suppressed because it is too large
Load Diff
+4
-4
@@ -15,14 +15,14 @@
|
||||
"lucide-react": "0.453.0",
|
||||
"react": "18.3.1",
|
||||
"react-dom": "18.3.1",
|
||||
"react-router-dom": "6.26.2"
|
||||
"react-router-dom": "^6.30.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "18.3.12",
|
||||
"@types/react-dom": "18.3.1",
|
||||
"@vitejs/plugin-react": "4.3.3",
|
||||
"tsx": "^4.19.2",
|
||||
"@vitejs/plugin-react": "^6.0.2",
|
||||
"tsx": "^4.22.3",
|
||||
"typescript": "5.6.3",
|
||||
"vite": "5.4.10"
|
||||
"vite": "^8.0.14"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,6 +61,7 @@ build_package() {
|
||||
)
|
||||
|
||||
cp "$ROOT_DIR/backend/config.example.yaml" "$work/config.example.yaml"
|
||||
cp "$ROOT_DIR/install.sh" "$work/install.sh"
|
||||
cp -R "$ROOT_DIR/dist" "$work/dist"
|
||||
mkdir -p "$work/91VideoSpider"
|
||||
cp "$ROOT_DIR/91VideoSpider/spider_91porn.py" "$work/91VideoSpider/spider_91porn.py"
|
||||
@@ -69,10 +70,11 @@ build_package() {
|
||||
$APP_NAME $VERSION
|
||||
|
||||
This is a prebuilt release package.
|
||||
Use install.sh from the repository to install it on a Linux server.
|
||||
Use install.sh in this package or from the repository to install it on a Linux server.
|
||||
EOF
|
||||
|
||||
chmod +x "$work/server"
|
||||
chmod +x "$work/install.sh"
|
||||
tar -C "$OUT_DIR/.work" -czf "$OUT_DIR/$artifact.tar.gz" "$artifact"
|
||||
log "wrote $OUT_DIR/$artifact.tar.gz"
|
||||
}
|
||||
|
||||
@@ -1,5 +1,16 @@
|
||||
import { useState } from "react";
|
||||
import { NavLink, Outlet, useNavigate } from "react-router-dom";
|
||||
import { HardDrive, Film, LogOut, Play, Home, Tags, Palette } from "lucide-react";
|
||||
import {
|
||||
HardDrive,
|
||||
Film,
|
||||
LogOut,
|
||||
Play,
|
||||
Home,
|
||||
Tags,
|
||||
Palette,
|
||||
RefreshCw,
|
||||
} from "lucide-react";
|
||||
import * as api from "./api";
|
||||
import { useAuth } from "./AuthContext";
|
||||
import { useToast } from "./ToastContext";
|
||||
|
||||
@@ -7,6 +18,31 @@ export function AdminLayout() {
|
||||
const { logout } = useAuth();
|
||||
const navigate = useNavigate();
|
||||
const { show } = useToast();
|
||||
const [checkingUpdate, setCheckingUpdate] = useState(false);
|
||||
|
||||
async function handleCheckUpdate() {
|
||||
if (checkingUpdate) return;
|
||||
setCheckingUpdate(true);
|
||||
try {
|
||||
const result = await api.checkUpdate();
|
||||
if (result.hasUpdate) {
|
||||
show(
|
||||
`发现新版本 ${result.latestVersion},当前 ${result.currentVersion}`,
|
||||
"success"
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (result.currentVersion === "unknown") {
|
||||
show(`当前版本未知,GitHub 最新版本为 ${result.latestVersion}`, "info");
|
||||
return;
|
||||
}
|
||||
show(`当前已是最新版本:${result.currentVersion}`, "success");
|
||||
} catch {
|
||||
show("检查更新失败,请稍后重试", "error");
|
||||
} finally {
|
||||
setCheckingUpdate(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleLogout() {
|
||||
try {
|
||||
@@ -65,6 +101,14 @@ export function AdminLayout() {
|
||||
</NavLink>
|
||||
</nav>
|
||||
<div className="admin-sidebar__footer">
|
||||
<button
|
||||
className="admin-sidebar__check-update"
|
||||
onClick={handleCheckUpdate}
|
||||
disabled={checkingUpdate}
|
||||
>
|
||||
<RefreshCw size={14} />
|
||||
{checkingUpdate ? "检查中" : "检查更新"}
|
||||
</button>
|
||||
<button className="admin-sidebar__logout" onClick={handleLogout}>
|
||||
<LogOut size={14} style={{ verticalAlign: -2, marginRight: 4 }} />
|
||||
退出登录
|
||||
|
||||
+76
-121
@@ -26,6 +26,7 @@ const kindLabel: Record<string, string> = {
|
||||
pikpak: "PikPak",
|
||||
wopan: "联通沃盘",
|
||||
onedrive: "OneDrive",
|
||||
localstorage: "本地存储",
|
||||
spider91: "91 爬虫",
|
||||
};
|
||||
|
||||
@@ -48,7 +49,7 @@ type FormState = {
|
||||
* 单独通过 PUT /admin/api/settings 写到全局 setting。在 form state 里维护它
|
||||
* 是为了让 DriveForm 能读写同一份编辑状态。
|
||||
*
|
||||
* 空字符串 = 自动模式(系统中唯一的 pikpak/p115 drive)。
|
||||
* 空字符串 = 本地保存,不上传。
|
||||
*/
|
||||
spider91UploadDriveId: string;
|
||||
};
|
||||
@@ -80,10 +81,10 @@ export function DrivesPage() {
|
||||
const [selectedDriveId, setSelectedDriveId] = useState<string | null>(null);
|
||||
const { show } = useToast();
|
||||
|
||||
// 当前系统中可作为 spider91 上传目标的 drive 列表(pikpak ∪ p115)。
|
||||
// 用户保存 spider91 drive 时从这里挑一个;空表示走"自动"模式。
|
||||
// 当前系统中可作为 spider91 上传目标的 drive 列表(pikpak ∪ p115 ∪ onedrive)。
|
||||
// 用户保存 spider91 drive 时从这里挑一个;空表示本地保存不上传。
|
||||
const uploadTargets = useMemo(
|
||||
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115"),
|
||||
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115" || d.kind === "onedrive"),
|
||||
[list]
|
||||
);
|
||||
|
||||
@@ -126,7 +127,7 @@ export function DrivesPage() {
|
||||
|
||||
function openCreate() {
|
||||
// 创建时把全局 setting 当前值带进表单,方便用户在新建第一个 spider91 drive 时
|
||||
// 直接看到当前的上传目标选择(一般是空 = 自动)。
|
||||
// 直接看到当前的上传目标选择(一般是空 = 本地保存)。
|
||||
setForm({
|
||||
...emptyForm,
|
||||
spider91UploadDriveId: settings?.spider91UploadDriveId ?? "",
|
||||
@@ -232,7 +233,7 @@ export function DrivesPage() {
|
||||
/**
|
||||
* 立即触发完整凌晨流水线(Phase1 扫所有云盘 → Phase2 spider91 爬虫 →
|
||||
* Phase3 spider91 → 云盘迁移)。后端立即返回 202;进度看 backend 日志。
|
||||
* 已在跑时后端会丢弃此次触发,按钮再点也不会重复进入。
|
||||
* 如果当前已有流水线在跑,后端最多保留一个待触发请求,当前轮结束后再跑一轮。
|
||||
*/
|
||||
async function handleRunNightly() {
|
||||
try {
|
||||
@@ -427,13 +428,13 @@ export function DrivesPage() {
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* 右栏:Teaser / 封面 与 缓存占用 */}
|
||||
{/* 右栏:Teaser / 封面 / 指纹 与 缓存占用 */}
|
||||
<div>
|
||||
<div className="admin-detail-card">
|
||||
<header className="admin-detail-card__title">
|
||||
<div className="admin-detail-card__title-left">
|
||||
<PlayCircle size={16} />
|
||||
<span>Teaser 预览与封面</span>
|
||||
<span>生成状态</span>
|
||||
</div>
|
||||
<div className="admin-detail-actions-inline">
|
||||
<button
|
||||
@@ -481,6 +482,22 @@ export function DrivesPage() {
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">指纹状态</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationStatusLine label="指纹" status={d.fingerprintGenerationStatus} />
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">指纹数量</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationCounts
|
||||
ready={d.fingerprintReadyCount}
|
||||
pending={d.fingerprintPendingCount}
|
||||
failed={d.fingerprintFailedCount}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="admin-detail-actions">
|
||||
@@ -586,7 +603,7 @@ export function DrivesPage() {
|
||||
<div className="admin-empty">加载中...</div>
|
||||
) : list.length === 0 ? (
|
||||
<div className="admin-card admin-empty">
|
||||
还没有配置任何网盘。点击右上角「新建」,选择夸克 / 115 / PikPak / 沃盘 / OneDrive,填入凭证即可。
|
||||
还没有配置任何网盘。点击右上角「新建」,选择夸克 / 115 / PikPak / 沃盘 / OneDrive / 本地存储,填入凭证或路径即可。
|
||||
</div>
|
||||
) : (
|
||||
<div className="admin-drives-grid">
|
||||
@@ -625,6 +642,15 @@ export function DrivesPage() {
|
||||
</span>
|
||||
</strong>
|
||||
</div>
|
||||
<div className="admin-drive-card__metric">
|
||||
<span>指纹数 (就绪/失败)</span>
|
||||
<strong>
|
||||
{d.fingerprintReadyCount ?? 0}
|
||||
<span style={{ fontSize: "11px", fontWeight: "normal", color: "var(--text-faint)" }}>
|
||||
{" "}/ {d.fingerprintFailedCount ?? 0}
|
||||
</span>
|
||||
</strong>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="admin-drive-card__footer">
|
||||
@@ -837,6 +863,11 @@ function DriveForm({
|
||||
}) {
|
||||
const fields = useMemo(() => credentialFields(form.kind), [form.kind]);
|
||||
const help = credentialHelp(form.kind, isEdit);
|
||||
const showDirectoryFields =
|
||||
form.kind !== "spider91" &&
|
||||
form.kind !== "onedrive" &&
|
||||
form.kind !== "localstorage" &&
|
||||
form.kind !== "pikpak";
|
||||
|
||||
function set<K extends keyof FormState>(k: K, v: FormState[K]) {
|
||||
onChange({ ...form, [k]: v });
|
||||
@@ -873,13 +904,14 @@ function DriveForm({
|
||||
>
|
||||
<option value="p115">115 网盘</option>
|
||||
<option value="pikpak">PikPak</option>
|
||||
<option value="spider91">91 爬虫</option>
|
||||
<option value="onedrive">OneDrive</option>
|
||||
<option value="localstorage">本地存储</option>
|
||||
<option value="spider91">91 Spider</option>
|
||||
<option value="quark">夸克网盘</option>
|
||||
<option value="wopan">联通沃盘</option>
|
||||
<option value="onedrive">OneDrive</option>
|
||||
</select>
|
||||
</div>
|
||||
{form.kind !== "spider91" && (
|
||||
{showDirectoryFields && (
|
||||
<>
|
||||
<div className="admin-form__row">
|
||||
<label>根目录 ID</label>
|
||||
@@ -953,10 +985,9 @@ function DriveForm({
|
||||
* Spider91UploadTargetField 是 spider91 drive 表单专属的"上传目标"下拉。
|
||||
*
|
||||
* 行为:
|
||||
* - 选项 = "(自动)" + 系统中所有 pikpak/p115 drive
|
||||
* - "自动" 模式(value="")下,后端在迁移 worker 启动时挑唯一的目标盘;
|
||||
* 系统中如果同时挂着 pikpak 和 p115 drive 必须显式选定,否则不会上传
|
||||
* - 没有任何 pikpak/p115 drive 时给一行提示文字,告诉用户先去添加目标盘
|
||||
* - 选项 = "本地保存,不上传" + 系统中所有 pikpak/p115/onedrive drive
|
||||
* - value="" 时后端不迁移上传,视频保存在服务器本地
|
||||
* - 没有任何 pikpak/p115/onedrive drive 时仍允许选择本地保存
|
||||
* - 该字段写入的是全局 setting `spider91.upload_drive_id`,不是 drive 自己的
|
||||
* credentials —— 所有 spider91 drive 共享同一个上传目标
|
||||
*/
|
||||
@@ -969,53 +1000,20 @@ function Spider91UploadTargetField({
|
||||
onChange: (v: string) => void;
|
||||
uploadTargets: api.AdminDrive[];
|
||||
}) {
|
||||
// 文案根据系统中实际挂载的目标盘 kind 自适应:
|
||||
// - 只挂了 PikPak → 文案只讲 "PikPak"
|
||||
// - 只挂了 115 → 文案只讲 "115 网盘"
|
||||
// - 两类都挂 → 文案讲 "PikPak / 115 网盘"
|
||||
// 这样在单一类型场景下用户不会被另一类的字样干扰。
|
||||
const kindsPresent = new Set(uploadTargets.map((d) => d.kind));
|
||||
const hasPikPak = kindsPresent.has("pikpak");
|
||||
const has115 = kindsPresent.has("p115");
|
||||
const presentLabel =
|
||||
hasPikPak && has115
|
||||
? "PikPak / 115 网盘"
|
||||
: hasPikPak
|
||||
? "PikPak"
|
||||
: has115
|
||||
? "115 网盘"
|
||||
: "PikPak 或 115 网盘";
|
||||
|
||||
return (
|
||||
<div className="admin-form__row">
|
||||
<label>视频上传目标</label>
|
||||
{uploadTargets.length === 0 ? (
|
||||
<>
|
||||
<select value="" disabled>
|
||||
<option value="">(请先添加 {presentLabel})</option>
|
||||
</select>
|
||||
<div className="admin-form__help">
|
||||
目前系统里还没有 {presentLabel} drive。可以先保存 91 爬虫,之后再回来选择上传目标。
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<select value={value} onChange={(e) => onChange(e.target.value)}>
|
||||
<option value="">(自动:唯一的 {presentLabel})</option>
|
||||
{uploadTargets.map((d) => (
|
||||
<option key={d.id} value={d.id}>
|
||||
{kindLabel[d.kind] ?? d.kind} · {d.name || d.id}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<div className="admin-form__help">
|
||||
爬取后的旧视频会上传到该云盘根目录。该设置全局生效;
|
||||
{uploadTargets.length > 1
|
||||
? `如果同时挂着多个 ${presentLabel} drive,"自动"模式不会工作,必须显式选定一个。`
|
||||
: `当前只挂着 1 个 ${presentLabel},"自动"模式会直接选用它。`}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
<select value={value} onChange={(e) => onChange(e.target.value)}>
|
||||
<option value="">本地保存,不上传</option>
|
||||
{uploadTargets.map((d) => (
|
||||
<option key={d.id} value={d.id}>
|
||||
{kindLabel[d.kind] ?? d.kind} · {d.name || d.id}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<div className="admin-form__help">
|
||||
选择本地保存时,爬取视频只保存在服务器本地;选择 115 网盘、PikPak 或 OneDrive 后,较早的视频会上传到该云盘根目录下的 91 Spider 文件夹。该设置全局生效。
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1028,13 +1026,15 @@ function credentialHelp(kind: Kind, isEdit: boolean): string {
|
||||
case "p115":
|
||||
return `登录 115.com 后复制 Cookie,形如 "UID=...; CID=...; SEID=...; KID=..."。${note}`;
|
||||
case "pikpak":
|
||||
return `参考 OpenList 的 PikPak 登录方式。可填用户名和密码首次登录,也可填 refresh_token;如返回验证码链接,打开验证后把 captcha_token 粘贴回来。${note}`;
|
||||
return `填写 PikPak 账号和密码即可。平台、设备 ID、验证码 token 和 refresh token 会由服务端自动处理并保存。${note}`;
|
||||
case "wopan":
|
||||
return `需要 access_token 和 refresh_token。后续会加扫码/短信登录入口,第一版只能手工粘贴。${note}`;
|
||||
case "onedrive":
|
||||
return `按 OpenList 默认方式,通过 api.oplist.org 在线刷新 token。只需要 refresh_token;保存后会自动回写新的 access_token / refresh_token。${note}`;
|
||||
return `按 OpenList 默认应用在线挂载,只需要 refresh_token;保存时会自动刷新并保存 token。${note}`;
|
||||
case "localstorage":
|
||||
return `把服务器上的一个已有目录作为视频来源扫描。填写绝对路径,例如 /mnt/videos;系统会读取该目录及子目录中的视频,并生成封面、Teaser 和指纹。${note}`;
|
||||
case "spider91":
|
||||
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;它不是外部网盘,不需要填写 Cookie 或目录 ID。后续流水线会把较早的视频上传到你选择的 115 / PikPak 目标盘。";
|
||||
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;它不是外部网盘,不需要填写 Cookie 或目录 ID。后续流水线会把较早的视频上传到你选择的 115 / PikPak / OneDrive 目标盘。";
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
@@ -1073,42 +1073,15 @@ function credentialFields(kind: Kind): Array<{
|
||||
return [
|
||||
{
|
||||
key: "username",
|
||||
label: "用户名 / 邮箱(无 refresh_token 时必填)",
|
||||
label: "用户名 / 邮箱",
|
||||
placeholder: "user@example.com",
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
key: "password",
|
||||
label: "密码(无 refresh_token 时必填)",
|
||||
label: "密码",
|
||||
placeholder: "PikPak 密码",
|
||||
},
|
||||
{
|
||||
key: "platform",
|
||||
label: "platform",
|
||||
placeholder: "web(可选:android / web / pc)",
|
||||
help: "默认 web;如果登录或直链异常,可尝试 android 或 pc。",
|
||||
},
|
||||
{
|
||||
key: "refresh_token",
|
||||
label: "refresh_token(可选)",
|
||||
placeholder: "已有 token 时可直接粘贴",
|
||||
multiline: true,
|
||||
},
|
||||
{
|
||||
key: "captcha_token",
|
||||
label: "captcha_token(可选)",
|
||||
placeholder: "遇到验证码校验时粘贴",
|
||||
multiline: true,
|
||||
},
|
||||
{
|
||||
key: "device_id",
|
||||
label: "device_id(可选)",
|
||||
placeholder: "留空自动生成并保存",
|
||||
},
|
||||
{
|
||||
key: "disable_media_link",
|
||||
label: "disable_media_link",
|
||||
placeholder: "true",
|
||||
help: "默认 true,使用原始下载链接;填 false 可尝试使用媒体缓存链接。",
|
||||
required: true,
|
||||
},
|
||||
];
|
||||
case "wopan":
|
||||
@@ -1140,34 +1113,15 @@ function credentialFields(kind: Kind): Array<{
|
||||
multiline: true,
|
||||
required: true,
|
||||
},
|
||||
];
|
||||
case "localstorage":
|
||||
return [
|
||||
{
|
||||
key: "access_token",
|
||||
label: "access_token(可选)",
|
||||
placeholder: "留空也可以,保存时会在线刷新",
|
||||
multiline: true,
|
||||
},
|
||||
{
|
||||
key: "api_url_address",
|
||||
label: "api_url_address(可选)",
|
||||
placeholder: "https://api.oplist.org/onedrive/renewapi",
|
||||
help: "默认使用 OpenList 的在线刷新 API;除非你有自建兼容服务,否则留空。",
|
||||
},
|
||||
{
|
||||
key: "region",
|
||||
label: "region(可选)",
|
||||
placeholder: "global(可选:global / cn / us / de)",
|
||||
help: "默认 global;世纪互联填 cn,美国政府云填 us,德国云填 de。",
|
||||
},
|
||||
{
|
||||
key: "is_sharepoint",
|
||||
label: "is_sharepoint(可选)",
|
||||
placeholder: "false",
|
||||
help: "普通 OneDrive 留空或 false;SharePoint 文档库填 true,并填写 site_id。",
|
||||
},
|
||||
{
|
||||
key: "site_id",
|
||||
label: "site_id(SharePoint 必填)",
|
||||
placeholder: "SharePoint site id",
|
||||
key: "path",
|
||||
label: "本地目录路径",
|
||||
placeholder: "/mnt/videos",
|
||||
required: true,
|
||||
help: "路径必须是后端服务器上的已有目录;保存后可手动重扫,系统会递归扫描支持的视频格式。",
|
||||
},
|
||||
];
|
||||
case "spider91":
|
||||
@@ -1178,6 +1132,7 @@ function credentialFields(kind: Kind): Array<{
|
||||
function defaultRootId(kind: Kind): string {
|
||||
if (kind === "pikpak") return "";
|
||||
if (kind === "onedrive") return "root";
|
||||
if (kind === "localstorage") return "/";
|
||||
if (kind === "spider91") return "/";
|
||||
return "0";
|
||||
}
|
||||
|
||||
+22
-6
@@ -61,11 +61,23 @@ export function me() {
|
||||
return request<{ authenticated: boolean }>("/me");
|
||||
}
|
||||
|
||||
export type UpdateCheck = {
|
||||
currentVersion: string;
|
||||
latestVersion: string;
|
||||
hasUpdate: boolean;
|
||||
releaseUrl?: string;
|
||||
checkedAt: string;
|
||||
};
|
||||
|
||||
export function checkUpdate() {
|
||||
return request<UpdateCheck>("/update/check");
|
||||
}
|
||||
|
||||
// ---------- Drives ----------
|
||||
|
||||
export type AdminDrive = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "spider91";
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "localstorage" | "spider91";
|
||||
name: string;
|
||||
rootId: string;
|
||||
scanRootId: string;
|
||||
@@ -84,12 +96,16 @@ export type AdminDrive = {
|
||||
lastCrawlAt?: number;
|
||||
thumbnailGenerationStatus?: DriveGenerationStatus;
|
||||
previewGenerationStatus?: DriveGenerationStatus;
|
||||
fingerprintGenerationStatus?: DriveGenerationStatus;
|
||||
thumbnailReadyCount: number;
|
||||
thumbnailPendingCount: number;
|
||||
thumbnailFailedCount: number;
|
||||
teaserReadyCount: number;
|
||||
teaserPendingCount: number;
|
||||
teaserFailedCount: number;
|
||||
fingerprintReadyCount: number;
|
||||
fingerprintPendingCount: number;
|
||||
fingerprintFailedCount: number;
|
||||
};
|
||||
|
||||
export type DriveGenerationStatus = {
|
||||
@@ -121,7 +137,7 @@ export function getDriveStorage() {
|
||||
|
||||
export type UpsertDriveInput = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "spider91";
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "localstorage" | "spider91";
|
||||
name: string;
|
||||
rootId: string;
|
||||
scanRootId: string;
|
||||
@@ -324,9 +340,9 @@ export type Theme = "dark" | "pink";
|
||||
export type Settings = {
|
||||
theme: Theme;
|
||||
/**
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak 或 p115 drive)。
|
||||
* - 空字符串:自动模式。系统中如果只挂着一个 pikpak/p115 drive 就用它;多个并存时迁移会跳过。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115}。
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115 或 onedrive drive)。
|
||||
* - 空字符串:本地保存,不上传到云盘。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, onedrive}。
|
||||
*/
|
||||
spider91UploadDriveId: string;
|
||||
};
|
||||
@@ -355,7 +371,7 @@ export function updateSettings(body: Partial<Settings>) {
|
||||
* 立即触发一次完整的凌晨流水线(Phase1 扫盘 + Phase2 91 爬虫 + Phase3 迁移),
|
||||
* 不论当前时间或今日是否已跑。立即返回 202;进度通过 backend 日志观察。
|
||||
*
|
||||
* 流水线已在跑时后端会丢弃此次触发。
|
||||
* 流水线已在跑时后端最多保留一个待触发请求;已有待触发请求时,新的点击会被忽略。
|
||||
*/
|
||||
export function runNightlyJob() {
|
||||
return request<{ ok: boolean }>("/jobs/nightly/run", { method: "POST" });
|
||||
|
||||
@@ -294,5 +294,6 @@ function sourceKindFromLabel(label: string): string {
|
||||
if (value.includes("pikpak")) return "pikpak";
|
||||
if (value.includes("沃盘") || value.includes("wopan") || value.includes("联通")) return "wopan";
|
||||
if (value.includes("onedrive") || value.includes("one drive")) return "onedrive";
|
||||
if (value.includes("本地") || value.includes("localstorage") || value.includes("local storage")) return "localstorage";
|
||||
return "";
|
||||
}
|
||||
|
||||
@@ -72,5 +72,7 @@ function sourceKindFromLabel(label: string): string {
|
||||
if (value.includes("沃盘") || value.includes("wopan") || value.includes("联通"))
|
||||
return "wopan";
|
||||
if (value.includes("onedrive") || value.includes("one drive")) return "onedrive";
|
||||
if (value.includes("本地") || value.includes("localstorage") || value.includes("local storage"))
|
||||
return "localstorage";
|
||||
return "";
|
||||
}
|
||||
|
||||
@@ -1287,6 +1287,7 @@ function getDriveShortName(source: string): string {
|
||||
if (s.includes("quark") || s.includes("夸克")) return "Quak";
|
||||
if (s.includes("onedrive")) return "OneDrive";
|
||||
if (s.includes("wopan") || s.includes("沃盘")) return "沃盘";
|
||||
if (s.includes("localstorage") || s.includes("本地")) return "本地";
|
||||
if (s.includes("spider") || s.includes("爬虫")) return "爬虫";
|
||||
return source.substring(0, 4);
|
||||
}
|
||||
|
||||
@@ -121,10 +121,45 @@
|
||||
border-top: 1px solid var(--border-subtle);
|
||||
}
|
||||
|
||||
.admin-sidebar__check-update {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
width: 100%;
|
||||
min-height: 34px;
|
||||
padding: 8px 12px;
|
||||
border-radius: var(--radius-sm);
|
||||
color: var(--text-muted);
|
||||
font-size: var(--font-sm);
|
||||
font-weight: var(--weight-medium);
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.admin-sidebar__check-update:hover:not(:disabled) {
|
||||
color: var(--text-strong);
|
||||
background: rgba(255, 255, 255, 0.04);
|
||||
}
|
||||
|
||||
.admin-sidebar__check-update:disabled {
|
||||
opacity: 0.6;
|
||||
cursor: wait;
|
||||
}
|
||||
|
||||
.admin-sidebar__check-update:disabled svg {
|
||||
animation: admin-update-spin 0.9s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes admin-update-spin {
|
||||
to {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
|
||||
.admin-sidebar__logout {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
width: 100%;
|
||||
padding: 8px 12px;
|
||||
margin-top: 8px;
|
||||
border-radius: var(--radius-sm);
|
||||
@@ -1790,6 +1825,7 @@
|
||||
.admin-drive-card__brand-icon[data-kind="pikpak"] { background: var(--drive-pikpak); }
|
||||
.admin-drive-card__brand-icon[data-kind="wopan"] { background: var(--drive-wopan); }
|
||||
.admin-drive-card__brand-icon[data-kind="onedrive"] { background: var(--drive-onedrive); }
|
||||
.admin-drive-card__brand-icon[data-kind="localstorage"] { background: var(--drive-localstorage); }
|
||||
.admin-drive-card__brand-icon[data-kind="spider91"] { background: var(--accent); }
|
||||
|
||||
.admin-drive-card__info {
|
||||
|
||||
@@ -134,6 +134,7 @@
|
||||
--drive-pikpak: #8a6dff;
|
||||
--drive-wopan: #ff8a3c;
|
||||
--drive-onedrive: #4cabea;
|
||||
--drive-localstorage: #35b88f;
|
||||
|
||||
/* ----- 阴影 ----- */
|
||||
--shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.4);
|
||||
@@ -213,6 +214,7 @@
|
||||
--drive-pikpak: #8466e6;
|
||||
--drive-wopan: #e57a36;
|
||||
--drive-onedrive: #2f95cf;
|
||||
--drive-localstorage: #239978;
|
||||
|
||||
/* ----- 阴影(粉色柔投影 + 一层中性) ----- */
|
||||
--shadow-sm: 0 1px 2px rgba(180, 90, 120, 0.08);
|
||||
|
||||
@@ -306,6 +306,14 @@
|
||||
--drive-shadow: rgba(76, 171, 234, 0.15);
|
||||
--drive-shadow-strong: rgba(76, 171, 234, 0.4);
|
||||
}
|
||||
.source-badge[data-kind="localstorage"] {
|
||||
--drive-color: var(--drive-localstorage);
|
||||
--drive-bg: rgba(53, 184, 143, 0.12);
|
||||
--drive-text: #a8efdc;
|
||||
--drive-border: rgba(53, 184, 143, 0.35);
|
||||
--drive-shadow: rgba(53, 184, 143, 0.15);
|
||||
--drive-shadow-strong: rgba(53, 184, 143, 0.4);
|
||||
}
|
||||
|
||||
/* ----- Preview overlay ----- */
|
||||
.preview-loader {
|
||||
@@ -441,6 +449,7 @@
|
||||
.video-card__source[data-kind="pikpak"] { --source-color: var(--drive-pikpak); }
|
||||
.video-card__source[data-kind="wopan"] { --source-color: var(--drive-wopan); }
|
||||
.video-card__source[data-kind="onedrive"] { --source-color: var(--drive-onedrive); }
|
||||
.video-card__source[data-kind="localstorage"] { --source-color: var(--drive-localstorage); }
|
||||
|
||||
/* =========================================================
|
||||
* Empty / Loading / Skeleton
|
||||
|
||||
@@ -351,6 +351,12 @@
|
||||
color: var(--drive-onedrive);
|
||||
}
|
||||
|
||||
.vd-meta__chip[data-tone="localstorage"] {
|
||||
background: rgba(53, 184, 143, 0.14);
|
||||
border-color: rgba(53, 184, 143, 0.3);
|
||||
color: var(--drive-localstorage);
|
||||
}
|
||||
|
||||
/* ---------- Actions toolbar ---------- */
|
||||
.vd-actions {
|
||||
display: flex;
|
||||
|
||||
@@ -13,3 +13,84 @@ test("spider91 drive form does not expose advanced crawler credentials", () => {
|
||||
assert.doesNotMatch(drivesPageSource, /python_path/);
|
||||
assert.doesNotMatch(drivesPageSource, /script_path/);
|
||||
});
|
||||
|
||||
test("spider91 upload target uses explicit local-save option instead of auto target", () => {
|
||||
assert.match(drivesPageSource, /本地保存,不上传/);
|
||||
assert.match(
|
||||
drivesPageSource,
|
||||
/d\.kind === "pikpak" \|\| d\.kind === "p115" \|\| d\.kind === "onedrive"/
|
||||
);
|
||||
assert.doesNotMatch(drivesPageSource, /自动:唯一/);
|
||||
assert.doesNotMatch(drivesPageSource, /自动模式/);
|
||||
});
|
||||
|
||||
test("onedrive drive form only exposes required default-app fields", () => {
|
||||
assert.match(
|
||||
drivesPageSource,
|
||||
/form\.kind !== "spider91" &&\s*form\.kind !== "onedrive" &&\s*form\.kind !== "localstorage" &&\s*form\.kind !== "pikpak"/
|
||||
);
|
||||
|
||||
const match =
|
||||
/function credentialFields[\s\S]*?case "onedrive":\s*return \[([\s\S]*?)\];\s*case "spider91":/.exec(
|
||||
drivesPageSource
|
||||
);
|
||||
assert.ok(match, "onedrive credential field block should be present");
|
||||
const fields = match[1];
|
||||
|
||||
assert.match(fields, /key: "refresh_token"/);
|
||||
assert.doesNotMatch(fields, /key: "access_token"/);
|
||||
assert.doesNotMatch(fields, /key: "api_url_address"/);
|
||||
assert.doesNotMatch(fields, /key: "region"/);
|
||||
assert.doesNotMatch(fields, /key: "is_sharepoint"/);
|
||||
assert.doesNotMatch(fields, /key: "site_id"/);
|
||||
});
|
||||
|
||||
test("pikpak drive form only exposes account login fields", () => {
|
||||
const match =
|
||||
/case "pikpak":\s*return \[([\s\S]*?)\];\s*case "wopan":/.exec(
|
||||
drivesPageSource
|
||||
);
|
||||
assert.ok(match, "pikpak credential field block should be present");
|
||||
const fields = match[1];
|
||||
|
||||
assert.match(fields, /key: "username"/);
|
||||
assert.match(fields, /key: "password"/);
|
||||
assert.doesNotMatch(fields, /key: "platform"/);
|
||||
assert.doesNotMatch(fields, /key: "refresh_token"/);
|
||||
assert.doesNotMatch(fields, /key: "captcha_token"/);
|
||||
assert.doesNotMatch(fields, /key: "device_id"/);
|
||||
assert.doesNotMatch(fields, /key: "disable_media_link"/);
|
||||
});
|
||||
|
||||
test("localstorage drive form asks for a server directory path", () => {
|
||||
assert.match(drivesPageSource, /<option value="localstorage">本地存储<\/option>/);
|
||||
|
||||
const match =
|
||||
/case "localstorage":\s*return \[([\s\S]*?)\];\s*case "spider91":/.exec(
|
||||
drivesPageSource
|
||||
);
|
||||
assert.ok(match, "localstorage credential field block should be present");
|
||||
const fields = match[1];
|
||||
|
||||
assert.match(fields, /key: "path"/);
|
||||
assert.match(fields, /label: "本地目录路径"/);
|
||||
assert.match(drivesPageSource, /if \(kind === "localstorage"\) return "\/"/);
|
||||
});
|
||||
|
||||
test("drive type selector keeps primary source order", () => {
|
||||
const options = Array.from(
|
||||
drivesPageSource.matchAll(/<option value="([^"]+)">([^<]+)<\/option>/g),
|
||||
(match) => ({ value: match[1], label: match[2] })
|
||||
);
|
||||
const driveOptions = options.slice(0, 7);
|
||||
|
||||
assert.deepEqual(driveOptions, [
|
||||
{ value: "p115", label: "115 网盘" },
|
||||
{ value: "pikpak", label: "PikPak" },
|
||||
{ value: "onedrive", label: "OneDrive" },
|
||||
{ value: "localstorage", label: "本地存储" },
|
||||
{ value: "spider91", label: "91 Spider" },
|
||||
{ value: "quark", label: "夸克网盘" },
|
||||
{ value: "wopan", label: "联通沃盘" },
|
||||
]);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user