30 Commits

Author SHA1 Message Date
nianzhibai 9ada39debb chore: deploy Docker Compose from stable release image 2026-05-31 10:04:56 +08:00
nianzhibai 24d1244bc3 fix: detect Docker image version for update checks 2026-05-31 09:55:15 +08:00
nianzhibai 0dd0c45509 Update README with service restart instruction
Add note about restarting service on first access.
2026-05-30 20:26:29 +08:00
nianzhibai ef6eadd0a6 Update README.md 2026-05-30 20:17:18 +08:00
nianzhibai 2c5a3342cc feat: prepare v0.0.4 storage release 2026-05-30 20:02:02 +08:00
nianzhibai 7ace5f8bc7 feat: probe video duration during thumbnail generation 2026-05-30 18:30:22 +08:00
nianzhibai 66b33b2a31 feat: support spider91 uploads to OneDrive 2026-05-30 18:04:15 +08:00
nianzhibai 27aefc870f feat: improve media generation pipeline status 2026-05-30 17:37:31 +08:00
nianzhibai 02d82e9a62 Add Docker Compose deployment support 2026-05-30 11:09:04 +08:00
nianzhibai 492431164b Improve fingerprint dedupe maintenance 2026-05-29 23:58:36 +08:00
nianzhibai 051f1555d5 Add sampled fingerprint deduplication 2026-05-29 23:19:52 +08:00
nianzhibai 85292ea095 Simplify OneDrive setup and redirect playback 2026-05-29 22:35:02 +08:00
nianzhibai fea5e984d1 Release v0.0.3 improvements 2026-05-29 18:34:38 +08:00
nianzhibai 46bb1fa9f2 Recreate releases with assets 2026-05-29 16:46:02 +08:00
nianzhibai 7f4407ac28 Fetch annotated tag notes for releases 2026-05-29 16:39:12 +08:00
nianzhibai 79b5b5c37d Use tag notes for release body 2026-05-29 16:34:29 +08:00
nianzhibai f3180f7b3c Prioritize ready thumbnails on home 2026-05-29 16:23:13 +08:00
nianzhibai 353b01b8e7 Update README with upgrade instructions and cleanup
Added upgrade instructions for old version users and removed redundant access troubleshooting note.
2026-05-29 15:39:51 +08:00
nianzhibai d27eae9c62 Document legacy update recovery 2026-05-29 15:37:40 +08:00
nianzhibai 003efa301b Wait for service readiness after install 2026-05-29 15:34:48 +08:00
nianzhibai f72898f530 Harden installer update flow 2026-05-29 15:23:42 +08:00
nianzhibai 641d29e008 Fix PikPak captcha recovery 2026-05-29 14:49:47 +08:00
nianzhibai fed46b51bb Fix spider91 upload target and thumbnails 2026-05-29 06:28:18 +00:00
nianzhibai 304559203c Update mobile section images in README 2026-05-29 11:54:56 +08:00
nianzhibai 62ccd6a998 更新 README.md 2026-05-29 11:28:02 +08:00
nianzhibai 720a92af7a Update LinuxDo community link in README 2026-05-28 21:30:38 +08:00
nianzhibai e33384c786 Include restart command for access issues
Add troubleshooting tip for project access issues.
2026-05-28 21:26:49 +08:00
nianzhibai 2e7c761aaf Revise README content for clarity and updates
Updated the README to enhance the description and clarify features.
2026-05-28 21:23:29 +08:00
nianzhibai 0d6c3c6ac9 docs: polish README layout 2026-05-28 21:15:40 +08:00
nianzhibai 10426e5483 Enhance README with new features and preview images
Added preview images for desktop and mobile, included theme options and short video mode.
2026-05-28 21:11:13 +08:00
61 changed files with 6376 additions and 2172 deletions
+21
View File
@@ -0,0 +1,21 @@
.git
.github
.gitattributes
.gitignore
node_modules
dist
release
data
backend/data
backend/config.yaml
config.yaml
*.db
*.sqlite
*.sqlite3
*.log
*.tmp
tests
video-site-implementation-plan.md
+82
View File
@@ -0,0 +1,82 @@
name: Docker
on:
push:
branches:
- main
tags:
- "v*"
pull_request:
branches:
- main
workflow_dispatch:
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
permissions:
contents: read
packages: write
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to GHCR
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
type=ref,event=branch
type=ref,event=tag
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha,prefix=sha-
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
- name: Determine image version
id: version
shell: bash
run: |
if [[ "$GITHUB_REF" == refs/tags/v* ]]; then
version="$GITHUB_REF_NAME"
else
version="$(git describe --tags --always --dirty 2>/dev/null || git rev-parse --short=12 HEAD)"
fi
echo "version=$version" >> "$GITHUB_OUTPUT"
- name: Build and push Docker image
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm64
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
VERSION=${{ steps.version.outputs.version }}
cache-from: type=gha
cache-to: type=gha,mode=max
+9 -4
View File
@@ -15,6 +15,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@v5
@@ -36,8 +38,11 @@ jobs:
GH_TOKEN: ${{ github.token }}
TAG: ${{ github.ref_name }}
run: |
if gh release view "$TAG" >/dev/null 2>&1; then
gh release upload "$TAG" release/*.tar.gz --clobber
else
gh release create "$TAG" release/*.tar.gz --title "$TAG" --notes "Prebuilt Linux release packages."
git tag -d "$TAG" >/dev/null 2>&1 || true
git fetch --force origin "refs/tags/$TAG:refs/tags/$TAG"
NOTES="$(git tag -l "$TAG" --format='%(contents)')"
if [ -z "$NOTES" ]; then
NOTES="Prebuilt Linux release packages."
fi
gh release delete "$TAG" --yes >/dev/null 2>&1 || true
gh release create "$TAG" release/*.tar.gz --title "$TAG" --notes "$NOTES" --verify-tag
+66
View File
@@ -0,0 +1,66 @@
# ---- Stage 1: Build frontend ----
FROM node:20-slim AS frontend
WORKDIR /app
COPY package.json package-lock.json ./
RUN npm ci
COPY tsconfig.json vite.config.ts index.html ./
COPY public/ public/
COPY src/ src/
RUN npm run build
# ---- Stage 2: Build backend ----
FROM golang:1.23-bookworm AS backend
WORKDIR /app/backend
COPY backend/go.mod backend/go.sum ./
COPY backend/vendor/ vendor/
COPY backend/cmd/ cmd/
COPY backend/internal/ internal/
RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server
# ---- Stage 3: Runtime ----
FROM debian:bookworm-slim AS runtime
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
curl \
ffmpeg \
openssl \
python3 \
python3-bs4 \
python3-lxml \
python3-requests \
tar \
tzdata \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /opt/video-site-91
COPY --from=backend /out/server ./server
COPY --from=frontend /app/dist ./dist
COPY backend/config.example.yaml ./config.example.yaml
COPY 91VideoSpider/ ./91VideoSpider/
COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
ARG VERSION=dev
ENV VIDEO_CONFIG=/opt/video-site-91/data/config.yaml \
VIDEO_FRONTEND_DIR=/opt/video-site-91/dist \
VIDEO_GITHUB_REPO=nianzhibai/91 \
VIDEO_IMAGE_VERSION=${VERSION} \
VIDEO_LISTEN_PORT=9191 \
VIDEO_VERSION_FILE=/opt/video-site-91/data/.version
RUN chmod +x ./server /usr/local/bin/docker-entrypoint.sh
VOLUME ["/opt/video-site-91/data"]
EXPOSE 9191
ENTRYPOINT ["docker-entrypoint.sh"]
CMD ["./server"]
+168 -38
View File
@@ -1,76 +1,206 @@
## 项目说明
个人91站<img width="30" height="30" alt="1f913" src="https://github.com/user-attachments/assets/606c8c18-e727-41fd-9431-5a053e416673" />个人91站
<img width="120" height="120" alt="91" src="https://github.com/user-attachments/assets/5b323c94-bbd3-4dce-bbc8-adc86935b7de" />
个人91站<img width="30" height="30" alt="1f913" src="https://github.com/user-attachments/assets/606c8c18-e727-41fd-9431-5a053e416673" />个人91站
# 91
支持115云盘,PikPak云盘作为视频播放后端 ▶
<p align="center">
<img width="120" height="120" alt="91" src="https://github.com/user-attachments/assets/5b323c94-bbd3-4dce-bbc8-adc86935b7de" />
</p>
采用115云盘和PikPak云盘的302重定向,不占用服务器带宽(也不会受服务器带宽小而影响视频播放体验)✨
<p align="center">
😄 个人私有视频站 😄
</p>
服务器只会扫描云盘中的视频文件,给每个视频文件生成封面图和预览片段 📷
你可以通过封面图和预览片段在网站首页快速选择想看的视频 ✅
支持91爬虫,爬取91的本月最热视频 🕷
<p align="center">
<a href="#快速开始">快速开始</a> ·
<a href="#功能特性">功能特性</a> ·
<a href="#预览图">预览图</a> ·
<a href="#数据存放位置">数据目录</a>
</p>
---
## 功能特性
- **多后端支持** — 兼容 115 云盘、PikPak 云盘、OneDrive和本地存储
- **零带宽消耗** — 云盘视频采用 302 重定向播放,不占用服务器带宽
- **封面 & 预览片段** — 自动为每个视频生成封面图和预览片段,首页快速选片
- **91 爬虫** — 内置爬虫,支持抓取 91 本月最热视频
- **双主题** — 黑黄经典主题 / 粉白清新主题,随时切换
- **短视频模式** — 一键切换抖音风格,沉浸刷片
- **低资源占用** — 2C2G 服务器稳定运行,主要性能消耗就是封面图和预览视频的生成
---
## 预览图
### 电脑端
<p>
<img width="49%" alt="首页" src="https://github.com/user-attachments/assets/9808fceb-760b-4dd5-b7d2-8622b95b90d5" />
<img width="49%" alt="播放页" src="https://github.com/user-attachments/assets/859db4aa-1fba-44f2-bb46-1db07c2f964f" />
</p>
<p>
<img width="49%" alt="主题切换" src="https://github.com/user-attachments/assets/96bea37a-8764-413e-9b70-1856b4ae0cd2" />
<img width="49%" alt="管理页" src="https://github.com/user-attachments/assets/29c1e27a-7651-4dfc-93dd-556331844214" />
</p>
### 手机端
<p align="center">
<img width="1284" height="1134" alt="手机端" src="https://github.com/user-attachments/assets/bdb7a86c-a4e5-483e-a307-e02c0bb34dac" />
</p>
---
## 快速开始
一键安装脚本
### 方式一:一键安装脚本(推荐)
```bash
sudo apt update
sudo apt install -y curl ca-certificates
sudo apt update && sudo apt install -y curl ca-certificates
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/install.sh -o install.sh
sudo bash install.sh
```
部署完成后访问:
- 前台:`http://服务器IP:9191/`
- 后台:`http://服务器IP:9191/admin`
| 地址 | 说明 |
|------|------|
| `http://服务器IP:9191/` | 前台 |
| `http://服务器IP:9191/admin` | 后台管理 |
安装后会自动创建 `91` 指令
**注意:如果首次访问,显示502,可以运行 `91 restart` 重启一下服务**
安装后自动注册 `91` 管理命令:
```bash
91 # 打开管理菜单
91 status # 查看状态
91 logs # 查看日志
91 update # 更新
91 restart # 重启
91 stop # 停止
91 # 打开管理菜单
91 status # 查看运行状态
91 logs # 查看日志
91 update # 更新到最新版本
91 restart # 重启服务
91 stop # 停止服务
```
同时也保留 `video-site-91` 作为同等别名
> `video-site-91` 为等效别名,两者可互换使用
想换端口:
**自定义端口:**
```bash
FRONTEND_PORT=8080 sudo -E bash install.sh
```
**旧版本升级(v0.0.2 之前):**
旧版脚本直接执行 `91 update` 可能失败,先执行以下修复命令:
```bash
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/install.sh -o /tmp/install-91.sh
sudo bash /tmp/install-91.sh update
```
---
### 方式二:Docker Compose 部署
**1. 准备目录**
```bash
mkdir -p video-site-91 && cd video-site-91
```
**2. 创建 `docker-compose.yml`**
```yaml
services:
video-site-91:
image: ghcr.io/nianzhibai/91:stable
container_name: video-site-91
ports:
- "9191:9191"
volumes:
- ./data:/opt/video-site-91/data
restart: unless-stopped
```
`stable` 只会在发布 `v*` 正式 Release 时更新,不会跟随 `main` 分支开发镜像变化。
升级到最新正式版:
```bash
docker compose pull
docker compose up -d
```
如果想固定某个 Release 版本,可以改成明确的 tag,例如:
```yaml
image: ghcr.io/nianzhibai/91:v0.0.4
```
或直接拉取仓库内置配置:
```bash
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/docker-compose.yml -o docker-compose.yml
```
**3. 启动**
```bash
docker compose up -d
```
**常用命令:**
```bash
docker compose logs -f # 查看日志
docker compose pull # 拉取最新正式版 stable 镜像
docker compose up -d # 更新并重启
```
> 所有配置、数据库、封面、预览及上传文件均保存在 `./data/` 目录下。
---
## 数据存放位置
项目会把运行数据保存在本地:
### 一键脚本部署
- `/opt/video-site-91/config.yaml`:本地配置、管理员账号、网盘凭证。
- `/opt/video-site-91/data/video-site.db`SQLite 数据库。
- `/opt/video-site-91/data/previews/`:本地生成的封面和 teaser。
| 路径 | 内容 |
|------|------|
| `/opt/video-site-91/config.yaml` | 配置文件、管理员账号、网盘凭证 |
| `/opt/video-site-91/data/video-site.db` | SQLite 数据库 |
| `/opt/video-site-91/data/previews/` | 封面图和预览片段 |
---
## 了解项目更多细节
### Docker Compose 部署
根目录 README 只保留项目介绍和最短上手路径。更细的实现、接口、网盘字段和部署方式可以看:
- [backend/README.md](backend/README.md)
- [video-site-implementation-plan.md](video-site-implementation-plan.md)
| 路径 | 内容 |
|------|------|
| `./data/config.yaml` | 配置文件、管理员账号、网盘凭证 |
| `./data/video-site.db` | SQLite 数据库 |
| `./data/previews/` | 封面图和预览片段 |
| `./data/uploads/` | 本地上传的视频文件 |
| `./data/spider91/` | 91 爬虫抓取的视频文件 |
---
## 使用边界
## 更多文档
这个项目面向个人私有部署。请只接入你有权访问和管理的内容,并遵守对应网盘、站点服务条款以及所在地法律法规。
| 文档 | 内容 |
|------|------|
| [backend/README.md](backend/README.md) | 后端实现、接口说明、网盘字段 |
| [video-site-implementation-plan.md](video-site-implementation-plan.md) | 完整实现方案 |
不要传播,仅限个人使用,个人视频站
---
## 使用须知
本项目面向**个人私有部署**,请仅接入你有权访问和管理的内容,并遵守对应网盘、站点的服务条款及所在地法律法规。
> 不对外传播,仅限个人使用。
---
## 致谢
- [OpenList](https://github.com/OpenListTeam/OpenList) — 优秀的开源项目
- [LinuxDo](https://linux.do/) — 学 AI 上 L 站
- [NodeSeek](https://nodeseek.com/) — MJJ 上 N 站
+19 -5
View File
@@ -2,7 +2,7 @@
视频聚合站的 Go 后端。提供三件事:
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive / 本地存储
2. 视频元数据目录(SQLite+ 扫描 + teaser 预生成
3. REST API(前台)+ 管理后台 + 直链代理
4. 标签池、视频隐藏、按网盘统计和详情页来源网盘类型展示能力
@@ -21,6 +21,7 @@ internal/
pikpak/ PikPak(自己实现,参考 OpenList pikpak
wopan/ 联通沃盘(壳子 + OpenListTeam/wopan-sdk-go
onedrive/ OneDriveOpenList 在线续期 + Microsoft Graph 文件接口)
localstorage/ 本地目录扫描(服务器已有视频目录)
scanner/ 扫目录 → 落库
preview/ ffmpeg 抽封面和生成多段 teaser
proxy/ /p/stream/*、/p/preview/* 代理
@@ -105,9 +106,10 @@ go run ./cmd/server 后端 9192
|--------|---------------------------------------------------------------|
| quark | `cookie` |
| p115 | `cookie`(形如 `UID=...; CID=...; SEID=...; KID=...` |
| pikpak | `username`、`password`,可选 `refresh_token`、`captcha_token`、`device_id`、`platform`、`disable_media_link` |
| pikpak | `username`、`password`(token、验证码和设备 ID 由服务端自动处理并保存) |
| wopan | `access_token`、`refresh_token`,可选 `family_id` |
| onedrive | `refresh_token`,可选 `access_token`、`api_url_address`、`region`、`is_sharepoint`、`site_id` |
| onedrive | `refresh_token` |
| localstorage | `path`(服务器上的已有视频目录,如 `/mnt/videos` |
### PikPak 速度说明
@@ -115,7 +117,7 @@ go run ./cmd/server 后端 9192
当前服务器同时存在 sing-box TUN 透明代理,PikPak 默认出站会被 `tun0` 接管;但强制直连物理网卡并没有更快,慢速的主要差异来自 PikPak 取链方式。media/cache CDN 节点仍有波动,偶尔可能遇到慢节点;如果播放变慢,可重新获取直链或重新挂载 PikPak 后再测。
OneDrive 按 OpenList 默认方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。OpenList 代刷得到的 refresh token 可以直接填到本项目。普通 OneDrive 的 `rootId` / `scanRootId` 可填 `root`SharePoint 文档库需要额外设置 `is_sharepoint=true` 和 `site_id`
OneDrive 按 OpenList 默认应用方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。后台新建 OneDrive 时只需要填 OpenList 代刷得到的 `refresh_token`;服务端会默认挂载根目录并自动回写新 token
## 文件名约定
@@ -128,6 +130,18 @@ OneDrive 按 OpenList 默认方式调用 `https://api.oplist.org/onedrive/renewa
标签分隔符支持 `, ` 和空格。解析结果会和系统标签池匹配,常见番号类噪声会归并到 `AV` 等系统标签,避免把每个番号都变成独立标签。解析结果可在管理后台覆盖。
## 视频去重
项目有三层去重:
1. 同一网盘同一文件按 `(drive_id, file_id)` 形成稳定视频 ID,重复扫描只更新同一行。
2. 扫描时优先按网盘侧 `content_hash` 去重;没有 hash 时退化为 `file_name + size_bytes`。
3. 扫描、爬虫、本地上传或服务启动挂载网盘后,后台指纹 worker 会异步读取视频的少量 Range 片段,生成 `sampled_sha256`。前台列表、首页、搜索、推荐会按 `size_bytes + sampled_sha256` 只展示最早入库的 canonical 视频。
`sampled_sha256` 是文件级去重:适合识别同一个视频文件被复制到 115 / PikPak / OneDrive 等不同网盘的情况。它不会删除任何网盘文件,也不用于识别转码、裁剪、加水印后的同源视频。
封面和 teaser 仍然优先生成,不等待指纹完成。夜间流水线最后会做一次重复资产清理:对 `size_bytes + sampled_sha256` 命中的非 canonical 视频,只删除本机生成的重复封面和 teaser,并把对应字段重置为 `pending`。网盘原文件和视频元数据记录不会被删除;如果 canonical 视频以后被移除,这些重复项会重新进入生成队列。
## 管理能力
- `/admin/drives`:新增、编辑、删除网盘,触发扫描。
@@ -147,7 +161,7 @@ ffmpeg -ss <起点> -headers "UA/Cookie/Referer" -i <直链> \
当前策略是每段固定 3 秒;30 秒以下最多 3 段,30 秒及以上固定 4 段;长视频在 20% 到 80% 区间均匀取段。生成的 teaser 和封面都只保存在本地 `data/previews/`,不会回写到网盘;旧数据中的 `preview_file_id` 会被忽略。
服务启动或网盘重新挂载时,如果 Teaser 开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 直链生成 teaser 时可能触发 Microsoft 429 限流;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。
服务启动或网盘重新挂载时,如果 Teaser 开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 扫盘和直链生成 teaser / 封面时可能触发 Microsoft Graph 429、`TooManyRequests`、`activityLimitReached` 或 throttled 文本;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。扫盘阶段会按 `Retry-After` 或默认冷却时间等待后继续当前目录。
前端卡片的 `previewSrc` 统一指向 `/p/preview/<videoID>`,后端只从本地 `preview_local` 文件读取。
+422 -123
View File
@@ -25,6 +25,7 @@ import (
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/config"
"github.com/video-site/backend/internal/drives"
"github.com/video-site/backend/internal/drives/localstorage"
"github.com/video-site/backend/internal/drives/localupload"
"github.com/video-site/backend/internal/drives/onedrive"
"github.com/video-site/backend/internal/drives/p115"
@@ -32,6 +33,7 @@ import (
"github.com/video-site/backend/internal/drives/quark"
"github.com/video-site/backend/internal/drives/spider91"
"github.com/video-site/backend/internal/drives/wopan"
"github.com/video-site/backend/internal/fingerprint"
"github.com/video-site/backend/internal/nightly"
"github.com/video-site/backend/internal/preview"
"github.com/video-site/backend/internal/proxy"
@@ -39,6 +41,8 @@ import (
"github.com/video-site/backend/internal/spider91migrate"
)
const fingerprintReconcileInterval = time.Minute
func main() {
cfgPath := "./config.yaml"
if v := os.Getenv("VIDEO_CONFIG"); v != "" {
@@ -63,12 +67,13 @@ func main() {
defer cat.Close()
app := &App{
cfg: cfg,
cat: cat,
registry: proxy.NewRegistry(),
workers: make(map[string]*preview.Worker),
thumbWorkers: make(map[string]*preview.ThumbWorker),
spider91Crawlers: make(map[string]*spider91.Crawler),
cfg: cfg,
cat: cat,
registry: proxy.NewRegistry(),
workers: make(map[string]*preview.Worker),
thumbWorkers: make(map[string]*preview.ThumbWorker),
fingerprintWorkers: make(map[string]*fingerprint.Worker),
spider91Crawlers: make(map[string]*spider91.Crawler),
}
app.proxy = proxy.New(app.registry)
app.spider91Migrator = spider91migrate.New(spider91migrate.Config{
@@ -77,7 +82,8 @@ func main() {
GetTargetDriveID: func() string { return app.Spider91UploadDriveID() },
})
// 初始化现有 drives
// 初始化本地内置盘;外部云盘放到 HTTP 服务启动后异步挂载,避免上游
// 登录态校验拖慢端口监听。
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -86,16 +92,7 @@ func main() {
if err := app.attachLocalUpload(ctx); err != nil {
log.Printf("[local-upload] attach failed: %v", err)
}
existing, err := cat.ListDrives(ctx)
if err != nil {
log.Fatalf("list drives: %v", err)
}
for _, d := range existing {
if err := app.attachDrive(ctx, d); err != nil {
log.Printf("[drive %s] attach failed: %v", d.ID, err)
}
}
go app.runFingerprintReconciler(ctx)
authr := &auth.Authenticator{
Username: cfg.Server.Admin.Username,
@@ -104,6 +101,14 @@ func main() {
}
setupRequired := config.RequiresAdminSetup(cfg)
var setupMu sync.Mutex
versionFilePath := strings.TrimSpace(os.Getenv("VIDEO_VERSION_FILE"))
if versionFilePath == "" {
versionFilePath = filepath.Join(filepath.Dir(cfgPath), ".version")
}
githubRepo := strings.TrimSpace(os.Getenv("VIDEO_GITHUB_REPO"))
if githubRepo == "" {
githubRepo = strings.TrimSpace(os.Getenv("GITHUB_REPO"))
}
apiServer := &api.Server{
Catalog: cat,
@@ -117,8 +122,11 @@ func main() {
}
adminServer := &api.AdminServer{
Catalog: cat,
Auth: authr,
Catalog: cat,
Auth: authr,
VersionFilePath: versionFilePath,
ImageVersion: strings.TrimSpace(os.Getenv("VIDEO_IMAGE_VERSION")),
GitHubRepo: githubRepo,
SetupRequired: func() bool {
setupMu.Lock()
defer setupMu.Unlock()
@@ -230,6 +238,7 @@ func main() {
RunSpider91Crawl: app.runSpider91Crawl,
WaitPreviewQueuesIdle: app.waitAllPreviewQueuesIdle,
RunMigration: app.spider91Migrator.RunOnce,
RunDedupeAssetCleanup: app.cleanupDuplicateVideoAssets,
})
go app.nightlyRunner.Run(ctx)
@@ -243,6 +252,7 @@ func main() {
log.Fatalf("server error: %v", err)
}
}()
go app.attachExistingDrives(ctx)
// 等待退出信号
sigs := make(chan os.Signal, 1)
@@ -262,20 +272,25 @@ type App struct {
registry *proxy.Registry
proxy *proxy.Proxy
mu sync.Mutex
workers map[string]*preview.Worker
thumbWorkers map[string]*preview.ThumbWorker
cancels map[string]context.CancelFunc
mu sync.Mutex
workers map[string]*preview.Worker
thumbWorkers map[string]*preview.ThumbWorker
fingerprintWorkers map[string]*fingerprint.Worker
cancels map[string]context.CancelFunc
// spider91Crawlers 按 driveID 索引,每个 spider91 drive 独立一个 Crawler
spider91Crawlers map[string]*spider91.Crawler
// driveAttachMu 串行化云盘挂载/重挂载。挂载会访问上游服务,可能较慢;
// 串行化可以避免启动后台挂载和手动扫盘按需挂载同一个 drive 时重复创建 worker。
driveAttachMu sync.Mutex
// 全站主题("dark" | "pink"),从 DB 读
theme string
// 显式指定的 spider91 上传目标 drive ID
// 未设置时由 Spider91UploadDriveID() 在所有 pikpak/p115 drive 中自动挑选唯一一个
// 显式指定的 spider91 上传目标 drive ID
// 空字符串表示本地保存不上传,不再自动挑选 pikpak/p115/onedrive drive
spider91UploadDriveID string
// spider91Migrator 周期把 spider91 视频上传到目标 drivePikPak 或 115)。
// spider91Migrator 周期把 spider91 视频上传到目标 drivePikPak、115 或 OneDrive)。
spider91Migrator *spider91migrate.Migrator
// nightlyRunner 是凌晨流水线调度器:每天 cron_hour 串行跑扫盘 → 91 爬虫 → 迁移。
@@ -297,6 +312,11 @@ type App struct {
// scanQueued 跟踪哪些 driveID 已经排队或正在跑,去重后续重复点击。
// 一个 drive 在 scheduleScan 入队时被加入,在 runScan goroutine 结束时被移除。
scanQueued map[string]bool
// fingerprintQueueing 去重每个 drive 的 pending 指纹补队列任务,避免定时
// reconcile 和扫盘结束同时为同一批 pending 视频启动多个长时间入队 goroutine。
fingerprintQueueMu sync.Mutex
fingerprintQueueing map[string]bool
}
// teaserEnabledForDrive 查询某个 drive 当前的 per-drive teaser 开关。
@@ -360,43 +380,25 @@ func (a *App) loadTheme(ctx context.Context) {
a.mu.Unlock()
}
// Spider91UploadDriveID 返回当前生效的 spider91 上传目标 drive ID。
//
// 解析顺序:
// 1. 管理员通过 PUT /admin/api/settings 显式设置过 → 验证该 drive 仍存在且是
// 合法目标盘(pikpak 或 p115)→ 返回该 ID。
// 2. 否则系统中如果只有一个合法目标盘(即 pikpak drive 数量+p115 drive 数量==1),
// 自动返回它。这样单网盘场景"开箱即用"。
// 3. 多个候选并存时返回空串:迁移 worker 静默跳过,等管理员显式指定。
//
// 注意"合法目标盘"目前是 pikpak ∪ p115。后续添加新的可上传盘要在两个分支同步加。
// Spider91UploadDriveID 返回当前配置的 spider91 上传目标 drive ID。
// 空字符串表示本地保存不上传;只有管理员显式选择 pikpak/p115/onedrive drive 时才迁移上传。
func (a *App) Spider91UploadDriveID() string {
a.mu.Lock()
explicit := a.spider91UploadDriveID
a.mu.Unlock()
if explicit != "" {
// 验证显式设置的 drive 仍然存在且 kind 合法;不在则降级到自动选取
if d, ok := a.registry.Get(explicit); ok && isSpider91UploadKind(d.Kind()) {
return explicit
}
if explicit == "" {
return ""
}
var found string
for _, d := range a.registry.All() {
if !isSpider91UploadKind(d.Kind()) {
continue
}
if found != "" {
// 多个候选 drive 时不自动选;管理员必须显式指定
return ""
}
found = d.ID()
// 验证显式设置的 drive 仍然存在且 kind 合法;不在则视为未配置。
if d, ok := a.registry.Get(explicit); ok && isSpider91UploadKind(d.Kind()) {
return explicit
}
return found
return ""
}
// SetSpider91UploadDriveID 设置 spider91 上传目标 drive ID 并持久化。
// 接受空字符串(清除显式设置,回退到自动模式)。
// 设置一个不存在或 kind 不是 pikpak / p115 的 drive 会返回错误。
// 接受空字符串(本地保存不上传)。
// 设置一个不存在或 kind 不是 pikpak / p115 / onedrive 的 drive 会返回错误。
func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) error {
driveID = strings.TrimSpace(driveID)
if driveID != "" {
@@ -405,7 +407,7 @@ func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) erro
return fmt.Errorf("drive %q not found", driveID)
}
if !isSpider91UploadKind(d.Kind()) {
return fmt.Errorf("drive %q kind=%s, only pikpak or p115 can be spider91 upload target", driveID, d.Kind())
return fmt.Errorf("drive %q kind=%s, only pikpak, p115 or onedrive can be spider91 upload target", driveID, d.Kind())
}
}
a.mu.Lock()
@@ -417,7 +419,7 @@ func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) erro
// isSpider91UploadKind 是 spider91 迁移目标盘的 allowlist。
// 与 spider91migrate.adaptUploadTarget 的支持范围保持一致。
func isSpider91UploadKind(kind string) bool {
return kind == "pikpak" || kind == "p115"
return kind == "pikpak" || kind == "p115" || kind == "onedrive"
}
// loadSpider91UploadDriveID 从 DB 读上传目标 drive ID 设置;不存在时使用空串。
@@ -442,9 +444,13 @@ func (a *App) driveGenerationStatuses() map[string]api.DriveGenerationStatuses {
for id, worker := range a.thumbWorkers {
thumbWorkers[id] = worker
}
fingerprintWorkers := make(map[string]*fingerprint.Worker, len(a.fingerprintWorkers))
for id, worker := range a.fingerprintWorkers {
fingerprintWorkers[id] = worker
}
a.mu.Unlock()
out := make(map[string]api.DriveGenerationStatuses, len(previewWorkers)+len(thumbWorkers))
out := make(map[string]api.DriveGenerationStatuses, len(previewWorkers)+len(thumbWorkers)+len(fingerprintWorkers))
for id, worker := range previewWorkers {
status := out[id]
status.Preview = generationStatusFromPreview(worker.Status())
@@ -455,7 +461,7 @@ func (a *App) driveGenerationStatuses() map[string]api.DriveGenerationStatuses {
status.Thumbnail = generationStatusFromPreview(worker.Status())
missing, err := a.cat.CountVideosNeedingThumbnail(context.Background(), id)
if err != nil {
log.Printf("[thumb] count missing thumbnails %s: %v", id, err)
log.Printf("[thumb] count thumbnail work %s: %v", id, err)
} else {
status.Thumbnail.QueueLength = missing
if missing > 0 && status.Thumbnail.State == "idle" {
@@ -464,6 +470,20 @@ func (a *App) driveGenerationStatuses() map[string]api.DriveGenerationStatuses {
}
out[id] = status
}
for id, worker := range fingerprintWorkers {
status := out[id]
status.Fingerprint = generationStatusFromFingerprint(worker.Status())
pending, err := a.cat.CountVideosNeedingFingerprint(context.Background(), id)
if err != nil {
log.Printf("[fingerprint] count pending fingerprints %s: %v", id, err)
} else {
status.Fingerprint.QueueLength = pending
if pending > 0 && status.Fingerprint.State == "idle" {
status.Fingerprint.State = "queued"
}
}
out[id] = status
}
return out
}
@@ -483,7 +503,67 @@ func generationStatusFromPreview(status preview.TaskStatus) api.GenerationStatus
return out
}
func generationStatusFromFingerprint(status fingerprint.TaskStatus) api.GenerationStatus {
state := status.State
if state == "" {
state = "idle"
}
out := api.GenerationStatus{
State: state,
CurrentTitle: status.CurrentTitle,
QueueLength: status.QueueLength,
}
if !status.CooldownUntil.IsZero() {
out.CooldownUntil = status.CooldownUntil.Format(time.RFC3339)
}
return out
}
func (a *App) attachDrive(ctx context.Context, d *catalog.Drive) error {
a.driveAttachMu.Lock()
defer a.driveAttachMu.Unlock()
return a.attachDriveUnlocked(ctx, d)
}
func (a *App) ensureDriveAttached(ctx context.Context, driveID string) error {
if _, ok := a.registry.Get(driveID); ok {
return nil
}
a.driveAttachMu.Lock()
defer a.driveAttachMu.Unlock()
if _, ok := a.registry.Get(driveID); ok {
return nil
}
d, err := a.cat.GetDrive(ctx, driveID)
if err != nil {
return err
}
return a.attachDriveUnlocked(ctx, d)
}
func (a *App) attachExistingDrives(ctx context.Context) {
existing, err := a.cat.ListDrives(ctx)
if err != nil {
log.Printf("[drive] list existing drives: %v", err)
return
}
log.Printf("[drive] attaching %d configured drive(s) in background", len(existing))
for _, d := range existing {
if err := ctx.Err(); err != nil {
log.Printf("[drive] background attach stopped: %v", err)
return
}
if err := a.attachDrive(ctx, d); err != nil {
log.Printf("[drive %s] attach failed: %v", d.ID, err)
}
}
log.Printf("[drive] background attach complete")
}
func (a *App) attachDriveUnlocked(ctx context.Context, d *catalog.Drive) error {
if d == nil {
return errors.New("nil drive")
}
var drv drives.Drive
switch d.Kind {
case "quark":
@@ -554,6 +634,11 @@ func (a *App) attachDrive(ctx context.Context, d *catalog.Drive) error {
_ = a.cat.UpsertDrive(ctx, d)
},
})
case localstorage.Kind:
drv = localstorage.New(localstorage.Config{
ID: d.ID,
RootPath: d.Credentials["path"],
})
case spider91.Kind:
drv = spider91.New(spider91.Config{
ID: d.ID,
@@ -587,12 +672,14 @@ func (a *App) attachDrive(ctx context.Context, d *catalog.Drive) error {
})
worker := preview.NewWorker(gen, a.cat, drv)
thumbWorker := preview.NewThumbWorker(gen, a.cat, drv)
fingerprintWorker := fingerprint.NewWorker(a.cat, drv, fingerprintConfigForDrive(drv))
workerCtx, cancel := context.WithCancel(ctx)
go worker.Run(workerCtx)
go thumbWorker.Run(workerCtx)
go fingerprintWorker.Run(workerCtx)
a.registerPreviewWorkers(ctx, d.ID, worker, thumbWorker, cancel)
a.registerPreviewWorkers(ctx, d.ID, worker, thumbWorker, fingerprintWorker, cancel)
// spider91 driver 还需要一个 crawler,挂在专用 map 里供 crawlerLoop 调用
if sd, ok := drv.(*spider91.Driver); ok {
@@ -619,12 +706,14 @@ func (a *App) attachLocalUpload(ctx context.Context) error {
})
worker := preview.NewWorker(gen, a.cat, drv)
thumbWorker := preview.NewThumbWorker(gen, a.cat, drv)
fingerprintWorker := fingerprint.NewWorker(a.cat, drv, fingerprintConfigForDrive(drv))
workerCtx, cancel := context.WithCancel(ctx)
go worker.Run(workerCtx)
go thumbWorker.Run(workerCtx)
go fingerprintWorker.Run(workerCtx)
a.registerPreviewWorkers(ctx, drv.ID(), worker, thumbWorker, cancel)
a.registerPreviewWorkers(ctx, drv.ID(), worker, thumbWorker, fingerprintWorker, cancel)
return nil
}
@@ -632,6 +721,20 @@ func (a *App) localUploadDir() string {
return filepath.Join(filepath.Dir(a.cfg.Storage.LocalPreviewDir), "uploads")
}
func fingerprintConfigForDrive(drv drives.Drive) fingerprint.Config {
cfg := fingerprint.Config{RateLimitCooldown: 5 * time.Minute}
if drv == nil {
return cfg
}
switch strings.ToLower(drv.Kind()) {
case "p115", "onedrive":
cfg.RateLimitCooldown = 10 * time.Minute
case "pikpak":
cfg.RateLimitCooldown = 5 * time.Minute
}
return cfg
}
// spider91RootDir 是所有 spider91 drive 共享的根目录。
func (a *App) spider91RootDir() string {
return filepath.Join(filepath.Dir(a.cfg.Storage.LocalPreviewDir), "spider91")
@@ -716,7 +819,7 @@ func (a *App) attachSpider91Crawler(d *catalog.Drive, drv *spider91.Driver) {
}()
}
func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker *preview.Worker, thumbWorker *preview.ThumbWorker, cancel context.CancelFunc) {
func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker *preview.Worker, thumbWorker *preview.ThumbWorker, fingerprintWorker *fingerprint.Worker, cancel context.CancelFunc) {
a.mu.Lock()
if a.cancels == nil {
a.cancels = make(map[string]context.CancelFunc)
@@ -727,6 +830,9 @@ func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker
if a.thumbWorkers == nil {
a.thumbWorkers = make(map[string]*preview.ThumbWorker)
}
if a.fingerprintWorkers == nil {
a.fingerprintWorkers = make(map[string]*fingerprint.Worker)
}
if old, ok := a.cancels[driveID]; ok && old != nil {
old()
}
@@ -740,6 +846,11 @@ func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker
} else {
delete(a.thumbWorkers, driveID)
}
if fingerprintWorker != nil {
a.fingerprintWorkers[driveID] = fingerprintWorker
} else {
delete(a.fingerprintWorkers, driveID)
}
if cancel != nil {
a.cancels[driveID] = cancel
} else {
@@ -747,17 +858,10 @@ func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker
}
a.mu.Unlock()
if worker != nil {
if thumbWorker != nil {
worker.BeforeTask = func(taskCtx context.Context) bool {
return a.waitForThumbnailsBeforePreview(taskCtx, driveID)
}
} else {
worker.BeforeTask = nil
}
}
go a.enqueueDriveGeneration(ctx, driveID, worker, thumbWorker)
if fingerprintWorker != nil {
a.scheduleFingerprintBackfill(ctx, driveID, fingerprintWorker)
}
}
func (a *App) enqueuePending(ctx context.Context, driveID string, w *preview.Worker) {
@@ -781,45 +885,16 @@ func (a *App) enqueuePending(ctx context.Context, driveID string, w *preview.Wor
func (a *App) enqueueDriveGeneration(ctx context.Context, driveID string, worker *preview.Worker, thumbWorker *preview.ThumbWorker) {
// 封面 worker 始终入队(与早期"全局 preview.enabled=false 时仍然生成封面"
// 的行为一致);teaser worker 仅在该 drive 的 TeaserEnabled 为 true 时入队。
// 两条队列互不等待,避免封面批量生成拖住预览视频生成。
if thumbWorker != nil {
a.enqueueThumbnails(ctx, driveID, thumbWorker)
}
if worker == nil || !a.teaserEnabledForDrive(ctx, driveID) {
return
}
if thumbWorker != nil && !a.waitForThumbnailsBeforePreview(ctx, driveID) {
return
}
a.enqueuePending(ctx, driveID, worker)
}
func (a *App) waitForThumbnailsBeforePreview(ctx context.Context, driveID string) bool {
const pollInterval = time.Second
var lastLog time.Time
for {
missing, err := a.cat.CountVideosNeedingThumbnail(ctx, driveID)
if err != nil {
log.Printf("[preview] count missing thumbnails drive=%s: %v", driveID, err)
return false
}
if missing == 0 {
return true
}
now := time.Now()
if lastLog.IsZero() || now.Sub(lastLog) >= time.Minute {
log.Printf("[preview] drive=%s waiting for %d thumbnails before teaser generation", driveID, missing)
lastLog = now
}
timer := time.NewTimer(pollInterval)
select {
case <-ctx.Done():
timer.Stop()
return false
case <-timer.C:
}
}
}
func (a *App) enqueueThumbnails(ctx context.Context, driveID string, w *preview.ThumbWorker) {
pending, err := a.cat.ListVideosNeedingThumbnail(ctx, driveID, 0)
if err != nil {
@@ -829,10 +904,81 @@ func (a *App) enqueueThumbnails(ctx context.Context, driveID string, w *preview.
if len(pending) == 0 {
return
}
log.Printf("[thumb] enqueue %d missing thumbnails for drive=%s", len(pending), driveID)
log.Printf("[thumb] enqueue %d thumbnail/duration tasks for drive=%s", len(pending), driveID)
for _, v := range pending {
if !w.EnqueueBlocking(ctx, v) {
log.Printf("[thumb] enqueue missing thumbnails canceled for drive=%s", driveID)
log.Printf("[thumb] enqueue thumbnail/duration tasks canceled for drive=%s", driveID)
return
}
}
}
func (a *App) runFingerprintReconciler(ctx context.Context) {
ticker := time.NewTicker(fingerprintReconcileInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
a.enqueueAllPendingFingerprints(ctx)
}
}
}
func (a *App) enqueueAllPendingFingerprints(ctx context.Context) {
a.mu.Lock()
workers := make(map[string]*fingerprint.Worker, len(a.fingerprintWorkers))
for id, worker := range a.fingerprintWorkers {
workers[id] = worker
}
a.mu.Unlock()
for driveID, worker := range workers {
a.scheduleFingerprintBackfill(ctx, driveID, worker)
}
}
func (a *App) scheduleFingerprintBackfill(ctx context.Context, driveID string, w *fingerprint.Worker) {
if w == nil {
return
}
a.fingerprintQueueMu.Lock()
if a.fingerprintQueueing == nil {
a.fingerprintQueueing = make(map[string]bool)
}
if a.fingerprintQueueing[driveID] {
a.fingerprintQueueMu.Unlock()
return
}
a.fingerprintQueueing[driveID] = true
a.fingerprintQueueMu.Unlock()
go func() {
defer func() {
a.fingerprintQueueMu.Lock()
delete(a.fingerprintQueueing, driveID)
a.fingerprintQueueMu.Unlock()
}()
a.enqueueFingerprints(ctx, driveID, w)
}()
}
func (a *App) enqueueFingerprints(ctx context.Context, driveID string, w *fingerprint.Worker) {
if w == nil {
return
}
pending, err := a.cat.ListVideosNeedingFingerprint(ctx, driveID, 0)
if err != nil {
log.Printf("[fingerprint] list pending %s: %v", driveID, err)
return
}
if len(pending) == 0 {
return
}
log.Printf("[fingerprint] enqueue %d videos for drive=%s", len(pending), driveID)
for _, v := range pending {
if !w.EnqueueBlocking(ctx, v) {
log.Printf("[fingerprint] enqueue canceled for drive=%s", driveID)
return
}
}
@@ -847,6 +993,7 @@ func (a *App) detachDrive(id string) {
}
delete(a.workers, id)
delete(a.thumbWorkers, id)
delete(a.fingerprintWorkers, id)
delete(a.spider91Crawlers, id)
a.mu.Unlock()
}
@@ -941,6 +1088,10 @@ func (a *App) runScan(ctx context.Context, driveID string) {
a.scanGlobalMu.Lock()
defer a.scanGlobalMu.Unlock()
if err := a.ensureDriveAttached(ctx, driveID); err != nil {
log.Printf("[scan] drive %s attach failed: %v", driveID, err)
return
}
drv, ok := a.registry.Get(driveID)
if !ok {
log.Printf("[scan] drive %s not attached", driveID)
@@ -950,14 +1101,15 @@ func (a *App) runScan(ctx context.Context, driveID string) {
a.mu.Lock()
worker := a.workers[driveID]
thumbWorker := a.thumbWorkers[driveID]
fingerprintWorker := a.fingerprintWorkers[driveID]
a.mu.Unlock()
var onNew func(v *catalog.Video)
if thumbWorker != nil {
onNew = func(v *catalog.Video) {
if thumbWorker != nil && v.ThumbnailURL == "" {
thumbWorker.Enqueue(v)
}
onNew := func(v *catalog.Video) {
if thumbWorker != nil && v.ThumbnailURL == "" {
thumbWorker.Enqueue(v)
}
if fingerprintWorker != nil {
fingerprintWorker.Enqueue(v)
}
}
@@ -1001,6 +1153,7 @@ func (a *App) runScan(ctx context.Context, driveID string) {
}
}
}
a.scheduleFingerprintBackfill(ctx, driveID, fingerprintWorker)
a.enqueueDriveGeneration(ctx, driveID, worker, thumbWorker)
}
@@ -1071,6 +1224,126 @@ func removeLocalVideoAssets(localDir string, v *catalog.Video) error {
return nil
}
type duplicateAssetCleanupStats struct {
Candidates int
VideosUpdated int
PreviewFiles int
ThumbnailFiles int
MissingFiles int
SkippedUnsafeRef int
}
func (a *App) cleanupDuplicateVideoAssets(ctx context.Context) error {
if a == nil || a.cat == nil {
return nil
}
localDir := ""
if a.cfg != nil {
localDir = a.cfg.Storage.LocalPreviewDir
}
if strings.TrimSpace(localDir) == "" {
return nil
}
items, err := a.cat.ListDuplicateAssetCleanupCandidates(ctx, 0)
if err != nil {
return err
}
if len(items) == 0 {
log.Printf("[dedupe-cleanup] no duplicate local assets to clean")
return nil
}
stats := duplicateAssetCleanupStats{Candidates: len(items)}
for _, item := range items {
if err := ctx.Err(); err != nil {
return err
}
clearPreview, removedPreview, missingPreview, skippedPreview, err := cleanupDuplicatePreviewAsset(localDir, item.PreviewLocal)
if err != nil {
return fmt.Errorf("cleanup duplicate preview video=%s canonical=%s: %w", item.VideoID, item.CanonicalID, err)
}
clearThumb, removedThumb, missingThumb, err := cleanupDuplicateThumbnailAsset(localDir, item.VideoID, item.ThumbnailURL)
if err != nil {
return fmt.Errorf("cleanup duplicate thumbnail video=%s canonical=%s: %w", item.VideoID, item.CanonicalID, err)
}
if skippedPreview {
stats.SkippedUnsafeRef++
}
if removedPreview {
stats.PreviewFiles++
}
if removedThumb {
stats.ThumbnailFiles++
}
if missingPreview {
stats.MissingFiles++
}
if missingThumb {
stats.MissingFiles++
}
if !clearPreview && !clearThumb {
continue
}
if err := a.cat.ClearGeneratedAssets(ctx, item.VideoID, clearPreview, clearThumb); err != nil {
return fmt.Errorf("mark duplicate assets cleaned video=%s canonical=%s: %w", item.VideoID, item.CanonicalID, err)
}
stats.VideosUpdated++
}
log.Printf("[dedupe-cleanup] candidates=%d updated=%d preview_files=%d thumbnail_files=%d missing=%d skipped_unsafe_refs=%d",
stats.Candidates, stats.VideosUpdated, stats.PreviewFiles, stats.ThumbnailFiles, stats.MissingFiles, stats.SkippedUnsafeRef)
return nil
}
func cleanupDuplicatePreviewAsset(localDir, previewLocal string) (clear bool, removed bool, missing bool, skippedUnsafe bool, err error) {
clean, ok := localPathWithin(localDir, previewLocal)
if !ok {
if strings.TrimSpace(previewLocal) != "" {
return false, false, false, true, nil
}
return false, false, false, false, nil
}
removed, missing, err = removeRegularFileIfExists(clean)
if err != nil {
return false, false, false, false, err
}
return true, removed, missing, false, nil
}
func cleanupDuplicateThumbnailAsset(localDir, videoID, thumbnailURL string) (clear bool, removed bool, missing bool, err error) {
if thumbnailURL != "/p/thumb/"+videoID {
return false, false, false, nil
}
clean, ok := localPathWithin(localDir, filepath.Join(localDir, "thumbs", videoID+".jpg"))
if !ok {
return false, false, false, nil
}
removed, missing, err = removeRegularFileIfExists(clean)
if err != nil {
return false, false, false, err
}
return true, removed, missing, nil
}
func removeRegularFileIfExists(path string) (removed bool, missing bool, err error) {
info, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return false, true, nil
}
return false, false, err
}
if !info.Mode().IsRegular() {
return false, false, nil
}
if err := os.Remove(path); err != nil {
if os.IsNotExist(err) {
return false, true, nil
}
return false, false, err
}
return true, false, nil
}
func localPathWithin(root, path string) (string, bool) {
if strings.TrimSpace(root) == "" || strings.TrimSpace(path) == "" {
return "", false
@@ -1097,6 +1370,7 @@ func (a *App) enqueueUploadedVideo(ctx context.Context, v *catalog.Video) {
a.mu.Lock()
worker := a.workers[v.DriveID]
thumbWorker := a.thumbWorkers[v.DriveID]
fingerprintWorker := a.fingerprintWorkers[v.DriveID]
a.mu.Unlock()
if thumbWorker != nil && v.ThumbnailURL == "" {
@@ -1105,6 +1379,9 @@ func (a *App) enqueueUploadedVideo(ctx context.Context, v *catalog.Video) {
if worker != nil && a.teaserEnabledForDrive(ctx, v.DriveID) {
worker.Enqueue(v)
}
if fingerprintWorker != nil {
fingerprintWorker.Enqueue(v)
}
}
func (a *App) regenPreview(ctx context.Context, videoID string) {
@@ -1231,26 +1508,36 @@ func (a *App) regenFailedThumbnails(ctx context.Context, driveID string) {
}
// listScanTargetIDs 返回 nightly Phase 1 应扫描的所有 drive ID
// (非 spider91、非 localupload)。顺序按 registry.All 给的稳定顺序。
func (a *App) listScanTargetIDs(_ context.Context) []string {
all := a.registry.All()
// (非 spider91、非 localupload)。它直接读 catalog,而不是 registry,这样
// 进程刚启动、云盘还在后台挂载时,nightly 也不会漏掉配置过的 drive。
func (a *App) listScanTargetIDs(ctx context.Context) []string {
all, err := a.cat.ListDrives(ctx)
if err != nil {
log.Printf("[nightly] list scan target drives: %v", err)
return nil
}
out := make([]string, 0, len(all))
for _, d := range all {
if !shouldScanDrive(d) {
if d == nil || d.ID == localupload.DriveID || d.Kind == spider91.Kind {
continue
}
out = append(out, d.ID())
out = append(out, d.ID)
}
return out
}
// listSpider91DriveIDs 返回 nightly Phase 2 应触发爬取的 spider91 drive ID 列表。
func (a *App) listSpider91DriveIDs(_ context.Context) []string {
a.mu.Lock()
defer a.mu.Unlock()
out := make([]string, 0, len(a.spider91Crawlers))
for id := range a.spider91Crawlers {
out = append(out, id)
func (a *App) listSpider91DriveIDs(ctx context.Context) []string {
all, err := a.cat.ListDrives(ctx)
if err != nil {
log.Printf("[nightly] list spider91 drives: %v", err)
return nil
}
out := make([]string, 0, len(all))
for _, d := range all {
if d != nil && d.Kind == spider91.Kind {
out = append(out, d.ID)
}
}
return out
}
@@ -1258,8 +1545,8 @@ func (a *App) listSpider91DriveIDs(_ context.Context) []string {
// waitAllPreviewQueuesIdle 阻塞直到所有 drive 的封面 worker 和 teaser worker
// 队列都为空且无 in-flight 任务。
//
// 顺序:先等所有 thumb worker(因为 enqueueDriveGeneration 内部已经先等当前
// drive 的封面再入队 teaser,但这里是跨 drive 的全局同步),再等所有 teaser
// 顺序:先等所有 thumb worker,再等所有 teaser。两个队列生成时互不等待;
// nightly 只在 phase 边界统一等待它们都 drain
// 若 ctx 在等待中被取消(软超时 / shutdown),立即返回 ctx.Err。
func (a *App) waitAllPreviewQueuesIdle(ctx context.Context) error {
a.mu.Lock()
@@ -1309,7 +1596,17 @@ func (a *App) runSpider91Crawl(ctx context.Context, driveID string) {
c := a.spider91Crawlers[driveID]
a.mu.Unlock()
if c == nil {
return
if err := a.ensureDriveAttached(ctx, driveID); err != nil {
log.Printf("[spider91] drive=%s attach failed: %v", driveID, err)
return
}
a.mu.Lock()
c = a.spider91Crawlers[driveID]
a.mu.Unlock()
if c == nil {
log.Printf("[spider91] drive=%s crawler not attached", driveID)
return
}
}
d, err := a.cat.GetDrive(ctx, driveID)
@@ -1356,7 +1653,9 @@ func (a *App) runSpider91Crawl(ctx context.Context, driveID string) {
a.mu.Lock()
worker := a.workers[driveID]
thumbWorker := a.thumbWorkers[driveID]
fingerprintWorker := a.fingerprintWorkers[driveID]
a.mu.Unlock()
a.scheduleFingerprintBackfill(ctx, driveID, fingerprintWorker)
a.enqueueDriveGeneration(ctx, driveID, worker, thumbWorker)
}
+57
View File
@@ -1,9 +1,13 @@
package main
import (
"context"
"io"
"testing"
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/drives"
"github.com/video-site/backend/internal/proxy"
)
func TestSpider91IntCredFallbacks(t *testing.T) {
@@ -30,3 +34,56 @@ func TestSpider91IntCredFallbacks(t *testing.T) {
})
}
}
func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
reg := proxy.NewRegistry()
reg.Set("p115-one", &spider91UploadTargetFakeDrive{id: "p115-one", kind: "p115"})
reg.Set("onedrive-one", &spider91UploadTargetFakeDrive{id: "onedrive-one", kind: "onedrive"})
app := &App{registry: reg}
if got := app.Spider91UploadDriveID(); got != "" {
t.Fatalf("empty upload target selected %q, want local-only empty target", got)
}
app.spider91UploadDriveID = "p115-one"
if got := app.Spider91UploadDriveID(); got != "p115-one" {
t.Fatalf("explicit upload target = %q, want p115-one", got)
}
app.spider91UploadDriveID = "onedrive-one"
if got := app.Spider91UploadDriveID(); got != "onedrive-one" {
t.Fatalf("explicit onedrive upload target = %q, want onedrive-one", got)
}
app.spider91UploadDriveID = "missing"
if got := app.Spider91UploadDriveID(); got != "" {
t.Fatalf("missing upload target = %q, want empty", got)
}
}
type spider91UploadTargetFakeDrive struct {
id string
kind string
}
func (d *spider91UploadTargetFakeDrive) Kind() string { return d.kind }
func (d *spider91UploadTargetFakeDrive) ID() string { return d.id }
func (d *spider91UploadTargetFakeDrive) Init(context.Context) error {
return nil
}
func (d *spider91UploadTargetFakeDrive) List(context.Context, string) ([]drives.Entry, error) {
return nil, nil
}
func (d *spider91UploadTargetFakeDrive) Stat(context.Context, string) (*drives.Entry, error) {
return nil, drives.ErrNotSupported
}
func (d *spider91UploadTargetFakeDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
return nil, drives.ErrNotSupported
}
func (d *spider91UploadTargetFakeDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
return "", drives.ErrNotSupported
}
func (d *spider91UploadTargetFakeDrive) EnsureDir(context.Context, string) (string, error) {
return "", drives.ErrNotSupported
}
func (d *spider91UploadTargetFakeDrive) RootID() string { return "root" }
+368 -37
View File
@@ -13,7 +13,9 @@ import (
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/config"
"github.com/video-site/backend/internal/drives"
"github.com/video-site/backend/internal/fingerprint"
"github.com/video-site/backend/internal/preview"
"github.com/video-site/backend/internal/proxy"
)
func TestRegisterPreviewWorkerBackfillsPendingWhenDriveTeaserEnabled(t *testing.T) {
@@ -53,7 +55,7 @@ func TestRegisterPreviewWorkerBackfillsPendingWhenDriveTeaserEnabled(t *testing.
worker := preview.NewWorker(&serverFakeTeaserGenerator{}, cat, &serverFakeDrive{})
go worker.Run(ctx)
app.registerPreviewWorkers(ctx, "drive-id", worker, nil, func() {})
app.registerPreviewWorkers(ctx, "drive-id", worker, nil, nil, func() {})
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
@@ -77,7 +79,7 @@ func TestRegisterPreviewWorkerBackfillsPendingWhenDriveTeaserEnabled(t *testing.
t.Fatalf("preview status = %q, want ready", got.PreviewStatus)
}
func TestRegisterPreviewWorkersGenerateThumbnailsBeforePreviews(t *testing.T) {
func TestRegisterPreviewWorkersRunThumbnailsAndPreviewsIndependently(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -93,16 +95,18 @@ func TestRegisterPreviewWorkersGenerateThumbnailsBeforePreviews(t *testing.T) {
seedDriveWithTeaser(t, cat, "drive-id", true)
now := time.Now()
for _, v := range []*catalog.Video{
{ID: "video-1", DriveID: "drive-id", FileID: "file-1", Title: "Clip 1", PreviewStatus: "pending"},
{ID: "video-2", DriveID: "drive-id", FileID: "file-2", Title: "Clip 2", PreviewStatus: "pending"},
} {
v.PublishedAt = now
v.CreatedAt = now
v.UpdatedAt = now
if err := cat.UpsertVideo(ctx, v); err != nil {
t.Fatalf("seed video %s: %v", v.ID, err)
}
video := &catalog.Video{
ID: "video-1",
DriveID: "drive-id",
FileID: "file-1",
Title: "Clip 1",
PreviewStatus: "pending",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
}
if err := cat.UpsertVideo(ctx, video); err != nil {
t.Fatalf("seed video: %v", err)
}
app := &App{
@@ -110,47 +114,234 @@ func TestRegisterPreviewWorkersGenerateThumbnailsBeforePreviews(t *testing.T) {
workers: make(map[string]*preview.Worker),
thumbWorkers: make(map[string]*preview.ThumbWorker),
}
gen := &serverFakeTeaserGenerator{}
gen := &serverBlockingThumbGenerator{
started: make(chan string, 1),
release: make(chan struct{}),
}
drv := &serverFakeDrive{}
worker := preview.NewWorker(gen, cat, drv)
thumbWorker := preview.NewThumbWorker(gen, cat, drv)
go worker.Run(ctx)
go thumbWorker.Run(ctx)
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, func() {})
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, nil, func() {})
select {
case got := <-gen.started:
if got != video.ID {
t.Fatalf("thumbnail started for %q, want %q", got, video.ID)
}
case <-time.After(2 * time.Second):
t.Fatal("thumbnail generation did not start")
}
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
first, err := cat.GetVideo(ctx, "video-1")
got, err := cat.GetVideo(ctx, video.ID)
if err != nil {
t.Fatalf("get video-1: %v", err)
t.Fatalf("get video: %v", err)
}
second, err := cat.GetVideo(ctx, "video-2")
if err != nil {
t.Fatalf("get video-2: %v", err)
}
if first.ThumbnailURL != "" && second.ThumbnailURL != "" &&
first.PreviewStatus == "ready" && second.PreviewStatus == "ready" {
events := gen.Events()
if len(events) != 4 {
t.Fatalf("events = %#v, want 4 generation events", events)
}
for i, event := range events[:2] {
if event[:6] != "thumb:" {
t.Fatalf("event %d = %q, want thumbnail before previews; all events=%#v", i, event, events)
}
}
for i, event := range events[2:] {
if event[:8] != "preview:" {
t.Fatalf("event %d = %q, want previews after thumbnails; all events=%#v", i+2, event, events)
}
if got.PreviewStatus == "ready" {
if got.ThumbnailURL != "" {
t.Fatalf("thumbnail url = %q, want preview ready while thumbnail is still blocked", got.ThumbnailURL)
}
close(gen.release)
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatalf("generation did not finish, events=%#v", gen.Events())
got, err := cat.GetVideo(ctx, video.ID)
if err != nil {
t.Fatalf("get video after timeout: %v", err)
}
t.Fatalf("preview status=%q thumbnail=%q, want preview ready before thumbnail finishes", got.PreviewStatus, got.ThumbnailURL)
}
func TestRegisterPreviewWorkersBackfillsHistoricalFingerprints(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
dataPath := filepath.Join(t.TempDir(), "video.mp4")
data := []byte("historical video content for fingerprint")
if err := os.WriteFile(dataPath, data, 0o644); err != nil {
t.Fatalf("write video data: %v", err)
}
now := time.Now()
video := &catalog.Video{
ID: "historical-video",
DriveID: "drive-id",
FileID: "file-id",
Title: "Historical",
Size: int64(len(data)),
FingerprintStatus: "pending",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
}
if err := cat.UpsertVideo(ctx, video); err != nil {
t.Fatalf("seed video: %v", err)
}
app := &App{
cat: cat,
workers: make(map[string]*preview.Worker),
thumbWorkers: make(map[string]*preview.ThumbWorker),
fingerprintWorkers: make(map[string]*fingerprint.Worker),
}
drv := &serverFingerprintFakeDrive{path: dataPath}
fingerprintWorker := fingerprint.NewWorker(cat, drv, fingerprint.Config{})
go fingerprintWorker.Run(ctx)
app.registerPreviewWorkers(ctx, "drive-id", nil, nil, fingerprintWorker, func() {})
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
got, err := cat.GetVideo(ctx, video.ID)
if err != nil {
t.Fatalf("get video: %v", err)
}
if got.SampledSHA256 != "" && got.FingerprintStatus == "ready" {
return
}
time.Sleep(10 * time.Millisecond)
}
got, err := cat.GetVideo(ctx, video.ID)
if err != nil {
t.Fatalf("get video after timeout: %v", err)
}
t.Fatalf("fingerprint status=%q sampled=%q, want ready with hash", got.FingerprintStatus, got.SampledSHA256)
}
func TestRunScanStartsFingerprintBeforeThumbnailAndPreviewDrain(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
seedDriveWithTeaser(t, cat, "drive-id", true)
dataPath := filepath.Join(t.TempDir(), "scan-video.mp4")
data := []byte("scan video content for independent fingerprint")
if err := os.WriteFile(dataPath, data, 0o644); err != nil {
t.Fatalf("write video data: %v", err)
}
drv := &serverScanFingerprintFakeDrive{
serverFingerprintFakeDrive: serverFingerprintFakeDrive{path: dataPath},
entries: []drives.Entry{{
ID: "file-id",
Name: "scan-video.mp4",
Size: int64(len(data)),
ParentID: "root",
}},
}
registry := proxy.NewRegistry()
registry.Set("drive-id", drv)
gen := &serverFakeTeaserGenerator{}
worker := preview.NewWorker(gen, cat, drv)
thumbWorker := preview.NewThumbWorker(gen, cat, drv)
fingerprintWorker := fingerprint.NewWorker(cat, drv, fingerprint.Config{})
go fingerprintWorker.Run(ctx)
app := &App{
cfg: &config.Config{
Scanner: config.Scanner{VideoExtensions: []string{".mp4"}},
},
cat: cat,
registry: registry,
workers: map[string]*preview.Worker{"drive-id": worker},
thumbWorkers: map[string]*preview.ThumbWorker{"drive-id": thumbWorker},
fingerprintWorkers: map[string]*fingerprint.Worker{"drive-id": fingerprintWorker},
}
done := make(chan struct{})
go func() {
defer close(done)
app.runScan(ctx, "drive-id")
}()
videoID := "fake-drive-id-file-id"
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
got, err := cat.GetVideo(ctx, videoID)
if err == nil && got.SampledSHA256 != "" && got.FingerprintStatus == "ready" {
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("scan did not stop after context cancel")
}
if got.ThumbnailURL != "" {
t.Fatalf("thumbnail url = %q, want fingerprint before thumbnail generation", got.ThumbnailURL)
}
return
}
time.Sleep(10 * time.Millisecond)
}
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("scan did not stop after context cancel")
}
got, err := cat.GetVideo(context.Background(), videoID)
if err != nil {
t.Fatalf("get video after timeout: %v", err)
}
t.Fatalf("fingerprint status=%q sampled=%q, want ready before thumbnail/preview drain", got.FingerprintStatus, got.SampledSHA256)
}
func TestNightlyTargetsComeFromCatalogBeforeDriveAttach(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
for _, d := range []*catalog.Drive{
{ID: "115", Kind: "p115", Name: "115", RootID: "0", TeaserEnabled: true},
{ID: "pikpak", Kind: "pikpak", Name: "PikPak", RootID: "0", TeaserEnabled: true},
{ID: "91-spider", Kind: "spider91", Name: "91 Spider", RootID: "0", TeaserEnabled: true},
} {
if err := cat.UpsertDrive(ctx, d); err != nil {
t.Fatalf("seed drive %s: %v", d.ID, err)
}
}
app := &App{cat: cat}
scanIDs := app.listScanTargetIDs(ctx)
if len(scanIDs) != 2 || scanIDs[0] != "115" || scanIDs[1] != "pikpak" {
t.Fatalf("scan target ids = %#v, want 115 and pikpak from catalog", scanIDs)
}
spiderIDs := app.listSpider91DriveIDs(ctx)
if len(spiderIDs) != 1 || spiderIDs[0] != "91-spider" {
t.Fatalf("spider91 ids = %#v, want catalog spider drive", spiderIDs)
}
}
func TestFailedThumbnailsDoNotBlockPreviewGeneration(t *testing.T) {
@@ -205,7 +396,7 @@ func TestFailedThumbnailsDoNotBlockPreviewGeneration(t *testing.T) {
go worker.Run(ctx)
go thumbWorker.Run(ctx)
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, func() {})
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, nil, func() {})
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
@@ -480,6 +671,106 @@ func TestCleanupMissingPikPakVideosRemovesDatabaseRowsAndLocalAssets(t *testing.
}
}
func TestCleanupDuplicateVideoAssetsRemovesOnlyDuplicateLocalAssets(t *testing.T) {
ctx := context.Background()
localDir := t.TempDir()
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
canonicalPreview := filepath.Join(localDir, "canonical.mp4")
duplicatePreview := filepath.Join(localDir, "duplicate.mp4")
canonicalThumb := filepath.Join(localDir, "thumbs", "canonical-video.jpg")
duplicateThumb := filepath.Join(localDir, "thumbs", "duplicate-video.jpg")
for _, path := range []string{canonicalPreview, duplicatePreview, canonicalThumb, duplicateThumb} {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", path, err)
}
if err := os.WriteFile(path, []byte("asset"), 0o644); err != nil {
t.Fatalf("write %s: %v", path, err)
}
}
now := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
for _, v := range []*catalog.Video{
{
ID: "canonical-video",
DriveID: "115",
FileID: "file-a",
Title: "Canonical",
Size: 2048,
ThumbnailURL: "/p/thumb/canonical-video",
PreviewLocal: canonicalPreview,
PreviewStatus: "ready",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
},
{
ID: "duplicate-video",
DriveID: "onedrive",
FileID: "file-b",
Title: "Duplicate",
Size: 2048,
ThumbnailURL: "/p/thumb/duplicate-video",
PreviewLocal: duplicatePreview,
PreviewStatus: "ready",
PublishedAt: now.Add(time.Second),
CreatedAt: now.Add(time.Second),
UpdatedAt: now.Add(time.Second),
},
} {
if err := cat.UpsertVideo(ctx, v); err != nil {
t.Fatalf("seed %s: %v", v.ID, err)
}
if err := cat.UpdateVideoFingerprint(ctx, v.ID, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "ready", ""); err != nil {
t.Fatalf("fingerprint %s: %v", v.ID, err)
}
}
app := &App{
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
cat: cat,
}
if err := app.cleanupDuplicateVideoAssets(ctx); err != nil {
t.Fatalf("cleanup duplicate video assets: %v", err)
}
for _, path := range []string{canonicalPreview, canonicalThumb} {
if _, err := os.Stat(path); err != nil {
t.Fatalf("canonical asset %s missing: %v", path, err)
}
}
for _, path := range []string{duplicatePreview, duplicateThumb} {
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("duplicate asset %s still exists, stat err=%v", path, err)
}
}
dup, err := cat.GetVideo(ctx, "duplicate-video")
if err != nil {
t.Fatalf("get duplicate: %v", err)
}
if dup.PreviewLocal != "" || dup.PreviewStatus != "pending" {
t.Fatalf("duplicate preview local=%q status=%q, want empty pending", dup.PreviewLocal, dup.PreviewStatus)
}
if dup.ThumbnailURL != "" {
t.Fatalf("duplicate thumbnail url = %q, want empty", dup.ThumbnailURL)
}
canon, err := cat.GetVideo(ctx, "canonical-video")
if err != nil {
t.Fatalf("get canonical: %v", err)
}
if canon.PreviewLocal != canonicalPreview || canon.ThumbnailURL != "/p/thumb/canonical-video" {
t.Fatalf("canonical changed: preview=%q thumb=%q", canon.PreviewLocal, canon.ThumbnailURL)
}
}
type serverFakeTeaserGenerator struct {
mu sync.Mutex
events []string
@@ -520,6 +811,28 @@ func (g *serverFakeTeaserGenerator) GenerateThumbnail(_ context.Context, _ *driv
return "/tmp/" + videoID + ".jpg", nil
}
type serverBlockingThumbGenerator struct {
serverFakeTeaserGenerator
started chan string
release chan struct{}
}
func (g *serverBlockingThumbGenerator) GenerateThumbnail(ctx context.Context, _ *drives.StreamLink, videoID string, _ float64) (string, error) {
g.record("thumb:" + videoID)
if g.started != nil {
select {
case g.started <- videoID:
default:
}
}
select {
case <-g.release:
return "/tmp/" + videoID + ".jpg", nil
case <-ctx.Done():
return "", ctx.Err()
}
}
type serverFakeDrive struct{}
func (d *serverFakeDrive) Kind() string { return "fake" }
@@ -544,6 +857,24 @@ func (d *serverFakeDrive) EnsureDir(context.Context, string) (string, error) {
}
func (d *serverFakeDrive) RootID() string { return "root" }
type serverFingerprintFakeDrive struct {
serverFakeDrive
path string
}
func (d *serverFingerprintFakeDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
return &drives.StreamLink{URL: d.path}, nil
}
type serverScanFingerprintFakeDrive struct {
serverFingerprintFakeDrive
entries []drives.Entry
}
func (d *serverScanFingerprintFakeDrive) List(context.Context, string) ([]drives.Entry, error) {
return d.entries, nil
}
type serverLocalUploadFakeDrive struct {
serverFakeDrive
}
+9 -3
View File
@@ -59,7 +59,7 @@ preview:
width: 480
# 盘列表。上线后请通过管理后台添加,本文件可留空。
# kind 支持 quark / p115 / pikpak / wopan / onedrive。
# kind 支持 quark / p115 / pikpak / wopan / onedrive / localstorage
# OneDrive 示例:
# - id: "my-onedrive"
# kind: "onedrive"
@@ -67,6 +67,12 @@ preview:
# root_id: "root"
# params:
# refresh_token: "..."
# api_url_address: "https://api.oplist.org/onedrive/renewapi"
# region: "global"
# 本地存储示例:
# - id: "local-media"
# kind: "localstorage"
# name: "本地视频目录"
# root_id: "/"
# scan_root_id: "/"
# params:
# path: "/mnt/videos"
drives: []
+161 -27
View File
@@ -5,9 +5,12 @@ import (
"database/sql"
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"strconv"
"strings"
"time"
"github.com/go-chi/chi/v5"
@@ -18,6 +21,17 @@ import (
type AdminServer struct {
Catalog *catalog.Catalog
Auth *auth.Authenticator
// VersionFilePath points to the installer-written .version file.
VersionFilePath string
// ImageVersion is the Docker image version injected at build/runtime.
// It takes precedence over VersionFilePath because Docker data volumes can
// keep an older .version file across image upgrades.
ImageVersion string
// GitHubRepo is the owner/name repo used for update checks.
GitHubRepo string
// ReleaseAPIURL and HTTPClient are injectable for tests. Production code leaves them empty.
ReleaseAPIURL string
HTTPClient *http.Client
// SetupRequired 表示当前是否仍处于首次部署初始化状态。
SetupRequired func() bool
// OnSetup 持久化首次部署时设置的管理员账号密码,并更新运行中认证器。
@@ -40,12 +54,12 @@ type AdminServer struct {
// Theme 读写("dark" | "pink"
GetTheme func() string
SetTheme func(theme string) error
// Spider91 → PikPak 上传目标 drive ID 读写
// Spider91 → 115/PikPak 上传目标 drive ID 读写
GetSpider91UploadDriveID func() string
SetSpider91UploadDriveID func(driveID string) error
// OnRunNightlyJob 触发一次完整的凌晨流水线(Phase1 扫盘 + Phase2 91 爬虫 +
// Phase3 迁移)。立即返回 —— 实际任务在后台跑,admin 在日志或下次状态查询里
// 看进度。重复点击会被 Runner.TryLock 丢弃
// 看进度。若流水线正在跑,Runner 最多保留一个待触发请求,当前轮结束后再跑一轮
OnRunNightlyJob func()
// ListDriveDirChildren 列出某个 drive 在 parentID 目录下的直接子目录。
// parentID 为空时使用 drive 的 RootID。返回 (子目录列表, error)。
@@ -68,8 +82,9 @@ type GenerationStatus struct {
}
type DriveGenerationStatuses struct {
Thumbnail GenerationStatus `json:"thumbnail"`
Preview GenerationStatus `json:"preview"`
Thumbnail GenerationStatus `json:"thumbnail"`
Preview GenerationStatus `json:"preview"`
Fingerprint GenerationStatus `json:"fingerprint"`
}
func (a *AdminServer) Register(r chi.Router) {
@@ -112,11 +127,25 @@ func (a *AdminServer) Register(r chi.Router) {
r.Put("/settings", a.handlePutSettings)
// 运维任务
r.Get("/update/check", a.handleCheckUpdate)
r.Post("/jobs/nightly/run", a.handleRunNightlyJob)
})
})
}
type updateCheckDTO struct {
CurrentVersion string `json:"currentVersion"`
LatestVersion string `json:"latestVersion"`
HasUpdate bool `json:"hasUpdate"`
ReleaseURL string `json:"releaseUrl,omitempty"`
CheckedAt string `json:"checkedAt"`
}
type githubReleaseDTO struct {
TagName string `json:"tag_name"`
HTMLURL string `json:"html_url"`
}
type loginReq struct {
Username string `json:"username"`
Password string `json:"password"`
@@ -221,6 +250,94 @@ func (a *AdminServer) handleMe(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]any{"authenticated": ok})
}
func (a *AdminServer) handleCheckUpdate(w http.ResponseWriter, r *http.Request) {
info, err := a.checkUpdate(r.Context())
if err != nil {
writeErr(w, http.StatusBadGateway, err)
return
}
w.Header().Set("Cache-Control", "no-store")
writeJSON(w, http.StatusOK, info)
}
func (a *AdminServer) checkUpdate(ctx context.Context) (updateCheckDTO, error) {
current := a.installedVersion()
if current == "" {
current = "unknown"
}
release, err := a.latestRelease(ctx)
if err != nil {
return updateCheckDTO{
CurrentVersion: current,
CheckedAt: time.Now().Format(time.RFC3339),
}, err
}
latest := strings.TrimSpace(release.TagName)
return updateCheckDTO{
CurrentVersion: current,
LatestVersion: latest,
HasUpdate: current != "unknown" && latest != "" && current != latest,
ReleaseURL: release.HTMLURL,
CheckedAt: time.Now().Format(time.RFC3339),
}, nil
}
func (a *AdminServer) installedVersion() string {
if version := strings.TrimSpace(a.ImageVersion); version != "" {
return version
}
path := strings.TrimSpace(a.VersionFilePath)
if path == "" {
path = ".version"
}
data, err := os.ReadFile(path)
if err != nil {
return ""
}
lines := strings.Split(strings.ReplaceAll(string(data), "\r\n", "\n"), "\n")
if len(lines) == 0 {
return ""
}
return strings.TrimSpace(lines[0])
}
func (a *AdminServer) latestRelease(ctx context.Context) (githubReleaseDTO, error) {
url := strings.TrimSpace(a.ReleaseAPIURL)
if url == "" {
repo := strings.TrimSpace(a.GitHubRepo)
if repo == "" {
repo = "nianzhibai/91"
}
url = "https://api.github.com/repos/" + repo + "/releases/latest"
}
client := a.HTTPClient
if client == nil {
client = &http.Client{Timeout: 8 * time.Second}
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return githubReleaseDTO{}, err
}
req.Header.Set("Accept", "application/vnd.github+json")
req.Header.Set("User-Agent", "video-site-91")
res, err := client.Do(req)
if err != nil {
return githubReleaseDTO{}, err
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
return githubReleaseDTO{}, fmt.Errorf("github release check failed: HTTP %d", res.StatusCode)
}
var release githubReleaseDTO
if err := json.NewDecoder(res.Body).Decode(&release); err != nil {
return githubReleaseDTO{}, err
}
if strings.TrimSpace(release.TagName) == "" {
return githubReleaseDTO{}, errors.New("github release check returned empty tag")
}
return release, nil
}
func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
drives, err := a.Catalog.ListDrives(r.Context())
if err != nil {
@@ -237,6 +354,11 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
writeErr(w, http.StatusInternalServerError, err)
return
}
fingerprintCounts, err := a.Catalog.CountFingerprintsByDrive(r.Context())
if err != nil {
writeErr(w, http.StatusInternalServerError, err)
return
}
generationStatuses := map[string]DriveGenerationStatuses{}
if a.GetDriveGenerationStatuses != nil {
generationStatuses = a.GetDriveGenerationStatuses()
@@ -259,20 +381,25 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
SkipDirIDs []string `json:"skipDirIds"`
// LastCrawlAt 是 spider91 上次成功爬取的 unix 秒(来自 credentials.last_crawl_at)。
// 其它 kind 留 0;前端用它显示"上次抓取: N 小时前"。
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
TeaserReadyCount int `json:"teaserReadyCount"`
TeaserPendingCount int `json:"teaserPendingCount"`
TeaserFailedCount int `json:"teaserFailedCount"`
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
TeaserReadyCount int `json:"teaserReadyCount"`
TeaserPendingCount int `json:"teaserPendingCount"`
TeaserFailedCount int `json:"teaserFailedCount"`
FingerprintReadyCount int `json:"fingerprintReadyCount"`
FingerprintPendingCount int `json:"fingerprintPendingCount"`
FingerprintFailedCount int `json:"fingerprintFailedCount"`
}
list := make([]out, 0, len(drives))
for _, d := range drives {
counts := teaserCounts[d.ID]
thumbCounts := thumbnailCounts[d.ID]
fingerprintCount := fingerprintCounts[d.ID]
generation := generationStatuses[d.ID]
if generation.Thumbnail.State == "" {
generation.Thumbnail.State = "idle"
@@ -280,6 +407,9 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
if generation.Preview.State == "" {
generation.Preview.State = "idle"
}
if generation.Fingerprint.State == "" {
generation.Fingerprint.State = "idle"
}
// spider91 没有用户凭证概念;只要存在 drive 行就视为"已配置"。
// last_crawl_at 是后端自动写入的运行状态字段,不计入 hasCredential 判定。
hasCred := false
@@ -305,18 +435,22 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
ID: d.ID, Kind: d.Kind, Name: d.Name,
RootID: d.RootID, ScanRootID: d.ScanRootID,
Status: d.Status, LastError: d.LastError,
HasCredential: hasCred,
TeaserEnabled: d.TeaserEnabled,
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
LastCrawlAt: lastCrawlAt,
ThumbnailGenerationStatus: generation.Thumbnail,
PreviewGenerationStatus: generation.Preview,
ThumbnailReadyCount: thumbCounts.Ready,
ThumbnailPendingCount: thumbCounts.Pending,
ThumbnailFailedCount: thumbCounts.Failed,
TeaserReadyCount: counts.Ready,
TeaserPendingCount: counts.Pending,
TeaserFailedCount: counts.Failed,
HasCredential: hasCred,
TeaserEnabled: d.TeaserEnabled,
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
LastCrawlAt: lastCrawlAt,
ThumbnailGenerationStatus: generation.Thumbnail,
PreviewGenerationStatus: generation.Preview,
FingerprintGenerationStatus: generation.Fingerprint,
ThumbnailReadyCount: thumbCounts.Ready,
ThumbnailPendingCount: thumbCounts.Pending,
ThumbnailFailedCount: thumbCounts.Failed,
TeaserReadyCount: counts.Ready,
TeaserPendingCount: counts.Pending,
TeaserFailedCount: counts.Failed,
FingerprintReadyCount: fingerprintCount.Ready,
FingerprintPendingCount: fingerprintCount.Pending,
FingerprintFailedCount: fingerprintCount.Failed,
})
}
writeJSON(w, http.StatusOK, list)
@@ -425,7 +559,7 @@ func (a *AdminServer) handleRescan(w http.ResponseWriter, r *http.Request) {
// handleRunNightlyJob 触发一次完整的凌晨流水线(不论当前时间,不论今日是否已跑)。
// 立即返回 202;进度通过 backend 日志和下次 GET /admin/api/drives 的状态变化观察。
// 流水线已在跑时 Runner 会丢弃此次触发并记日志
// 流水线已在跑时 Runner 最多排队一个后续触发;如果已有待触发请求,新的点击会被忽略
func (a *AdminServer) handleRunNightlyJob(w http.ResponseWriter, r *http.Request) {
if a.OnRunNightlyJob != nil {
a.OnRunNightlyJob()
@@ -748,7 +882,7 @@ func (a *AdminServer) handleGetSettings(w http.ResponseWriter, r *http.Request)
func (a *AdminServer) handlePutSettings(w http.ResponseWriter, r *http.Request) {
// 用 map 区分"没传"和"传了空字符串"两种语义;空 spider91 上传 ID 表示
// 清除显式设置(回退到自动模式)
// 本地保存不上传
var raw map[string]json.RawMessage
if err := json.NewDecoder(r.Body).Decode(&raw); err != nil {
writeErr(w, http.StatusBadRequest, err)
+200 -41
View File
@@ -115,6 +115,133 @@ func TestHandleSetupStoresCredentialsAndCreatesSession(t *testing.T) {
}
}
func TestHandleCheckUpdateReportsNewRelease(t *testing.T) {
dir := t.TempDir()
versionFile := filepath.Join(dir, ".version")
if err := os.WriteFile(versionFile, []byte("v0.1.0\n2026-05-29 12:00:00\n"), 0o644); err != nil {
t.Fatalf("write version file: %v", err)
}
releaseServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("User-Agent") == "" {
http.Error(w, "missing user agent", http.StatusBadRequest)
return
}
writeJSON(w, http.StatusOK, map[string]any{
"tag_name": "v0.2.0",
"html_url": "https://github.com/nianzhibai/91/releases/tag/v0.2.0",
})
}))
t.Cleanup(releaseServer.Close)
req := httptest.NewRequest(http.MethodGet, "/admin/api/update/check", nil)
rr := httptest.NewRecorder()
(&AdminServer{
VersionFilePath: versionFile,
ReleaseAPIURL: releaseServer.URL,
}).handleCheckUpdate(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
var got updateCheckDTO
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
t.Fatalf("decode: %v", err)
}
if got.CurrentVersion != "v0.1.0" {
t.Fatalf("currentVersion = %q, want v0.1.0", got.CurrentVersion)
}
if got.LatestVersion != "v0.2.0" {
t.Fatalf("latestVersion = %q, want v0.2.0", got.LatestVersion)
}
if !got.HasUpdate {
t.Fatalf("hasUpdate = false, want true")
}
if got.ReleaseURL == "" {
t.Fatalf("releaseUrl is empty")
}
}
func TestHandleCheckUpdateReportsUpToDate(t *testing.T) {
dir := t.TempDir()
versionFile := filepath.Join(dir, ".version")
if err := os.WriteFile(versionFile, []byte("v0.2.0\n"), 0o644); err != nil {
t.Fatalf("write version file: %v", err)
}
releaseServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]any{
"tag_name": "v0.2.0",
"html_url": "https://github.com/nianzhibai/91/releases/tag/v0.2.0",
})
}))
t.Cleanup(releaseServer.Close)
req := httptest.NewRequest(http.MethodGet, "/admin/api/update/check", nil)
rr := httptest.NewRecorder()
(&AdminServer{
VersionFilePath: versionFile,
ReleaseAPIURL: releaseServer.URL,
}).handleCheckUpdate(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
var got updateCheckDTO
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
t.Fatalf("decode: %v", err)
}
if got.HasUpdate {
t.Fatalf("hasUpdate = true, want false")
}
}
func TestHandleCheckUpdateUsesDockerImageVersion(t *testing.T) {
releaseServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]any{
"tag_name": "v0.2.0",
"html_url": "https://github.com/nianzhibai/91/releases/tag/v0.2.0",
})
}))
t.Cleanup(releaseServer.Close)
req := httptest.NewRequest(http.MethodGet, "/admin/api/update/check", nil)
rr := httptest.NewRecorder()
(&AdminServer{
ImageVersion: "v0.1.0",
ReleaseAPIURL: releaseServer.URL,
}).handleCheckUpdate(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
var got updateCheckDTO
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
t.Fatalf("decode: %v", err)
}
if got.CurrentVersion != "v0.1.0" {
t.Fatalf("currentVersion = %q, want v0.1.0", got.CurrentVersion)
}
if !got.HasUpdate {
t.Fatalf("hasUpdate = false, want true")
}
}
func TestInstalledVersionPrefersDockerImageVersionOverVersionFile(t *testing.T) {
dir := t.TempDir()
versionFile := filepath.Join(dir, ".version")
if err := os.WriteFile(versionFile, []byte("v0.1.0\n"), 0o644); err != nil {
t.Fatalf("write version file: %v", err)
}
got := (&AdminServer{
VersionFilePath: versionFile,
ImageVersion: "v0.2.0",
}).installedVersion()
if got != "v0.2.0" {
t.Fatalf("installedVersion = %q, want v0.2.0", got)
}
}
func TestHandleUpsertDrivePreservesExistingCredentialsWhenRequestCredentialsEmpty(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
@@ -244,11 +371,11 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
now := time.Now()
videos := []*catalog.Video{
{ID: "od-ready-1", DriveID: "OneDrive", FileID: "od-file-1", Title: "OD Ready 1", ThumbnailURL: "/p/thumb/od-ready-1", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "od-ready-2", DriveID: "OneDrive", FileID: "od-file-2", Title: "OD Ready 2", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "od-pending", DriveID: "OneDrive", FileID: "od-file-3", Title: "OD Pending", PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "pp-pending", DriveID: "PikPak", FileID: "pp-file-1", Title: "PP Pending", PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "pp-failed", DriveID: "PikPak", FileID: "pp-file-2", Title: "PP Failed", ThumbnailURL: "/p/thumb/pp-failed", PreviewStatus: "failed", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "od-ready-1", DriveID: "OneDrive", FileID: "od-file-1", Title: "OD Ready 1", Size: 100, ThumbnailURL: "/p/thumb/od-ready-1", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "od-ready-2", DriveID: "OneDrive", FileID: "od-file-2", Title: "OD Ready 2", Size: 100, PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "od-pending", DriveID: "OneDrive", FileID: "od-file-3", Title: "OD Pending", Size: 100, PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "pp-pending", DriveID: "PikPak", FileID: "pp-file-1", Title: "PP Pending", Size: 100, PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "pp-failed", DriveID: "PikPak", FileID: "pp-file-2", Title: "PP Failed", Size: 100, ThumbnailURL: "/p/thumb/pp-failed", PreviewStatus: "failed", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
}
for _, v := range videos {
if err := cat.UpsertVideo(ctx, v); err != nil {
@@ -258,6 +385,12 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
if err := cat.UpdateVideoMeta(ctx, "od-ready-2", catalog.VideoMetaPatch{ThumbnailStatus: "failed"}); err != nil {
t.Fatalf("mark thumbnail failed: %v", err)
}
if err := cat.UpdateVideoFingerprint(ctx, "od-ready-1", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "ready", ""); err != nil {
t.Fatalf("mark fingerprint ready: %v", err)
}
if err := cat.UpdateVideoFingerprint(ctx, "od-ready-2", "", "failed", "sample failed"); err != nil {
t.Fatalf("mark fingerprint failed: %v", err)
}
req := httptest.NewRequest(http.MethodGet, "/admin/api/drives", nil)
rr := httptest.NewRecorder()
@@ -266,8 +399,9 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
GetDriveGenerationStatuses: func() map[string]DriveGenerationStatuses {
return map[string]DriveGenerationStatuses{
"OneDrive": {
Thumbnail: GenerationStatus{State: "cooling", QueueLength: 3, CooldownUntil: "2026-05-16T21:00:00+08:00"},
Preview: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
Thumbnail: GenerationStatus{State: "cooling", QueueLength: 3, CooldownUntil: "2026-05-16T21:00:00+08:00"},
Preview: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
Fingerprint: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
},
}
},
@@ -277,48 +411,64 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
var got []struct {
ID string `json:"id"`
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
TeaserReadyCount int `json:"teaserReadyCount"`
TeaserPendingCount int `json:"teaserPendingCount"`
TeaserFailedCount int `json:"teaserFailedCount"`
ID string `json:"id"`
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
TeaserReadyCount int `json:"teaserReadyCount"`
TeaserPendingCount int `json:"teaserPendingCount"`
TeaserFailedCount int `json:"teaserFailedCount"`
FingerprintReadyCount int `json:"fingerprintReadyCount"`
FingerprintPendingCount int `json:"fingerprintPendingCount"`
FingerprintFailedCount int `json:"fingerprintFailedCount"`
}
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
t.Fatalf("decode: %v", err)
}
byID := map[string]struct {
TeaserReady int
TeaserPending int
TeaserFailed int
ThumbnailReady int
ThumbnailPending int
ThumbnailFailed int
Thumbnail GenerationStatus
Preview GenerationStatus
TeaserReady int
TeaserPending int
TeaserFailed int
ThumbnailReady int
ThumbnailPending int
ThumbnailFailed int
FingerprintReady int
FingerprintPending int
FingerprintFailed int
Thumbnail GenerationStatus
Preview GenerationStatus
Fingerprint GenerationStatus
}{}
for _, d := range got {
byID[d.ID] = struct {
TeaserReady int
TeaserPending int
TeaserFailed int
ThumbnailReady int
ThumbnailPending int
ThumbnailFailed int
Thumbnail GenerationStatus
Preview GenerationStatus
TeaserReady int
TeaserPending int
TeaserFailed int
ThumbnailReady int
ThumbnailPending int
ThumbnailFailed int
FingerprintReady int
FingerprintPending int
FingerprintFailed int
Thumbnail GenerationStatus
Preview GenerationStatus
Fingerprint GenerationStatus
}{
TeaserReady: d.TeaserReadyCount,
TeaserPending: d.TeaserPendingCount,
TeaserFailed: d.TeaserFailedCount,
ThumbnailReady: d.ThumbnailReadyCount,
ThumbnailPending: d.ThumbnailPendingCount,
ThumbnailFailed: d.ThumbnailFailedCount,
Thumbnail: d.ThumbnailGenerationStatus,
Preview: d.PreviewGenerationStatus,
TeaserReady: d.TeaserReadyCount,
TeaserPending: d.TeaserPendingCount,
TeaserFailed: d.TeaserFailedCount,
ThumbnailReady: d.ThumbnailReadyCount,
ThumbnailPending: d.ThumbnailPendingCount,
ThumbnailFailed: d.ThumbnailFailedCount,
FingerprintReady: d.FingerprintReadyCount,
FingerprintPending: d.FingerprintPendingCount,
FingerprintFailed: d.FingerprintFailedCount,
Thumbnail: d.ThumbnailGenerationStatus,
Preview: d.PreviewGenerationStatus,
Fingerprint: d.FingerprintGenerationStatus,
}
}
if byID["OneDrive"].TeaserReady != 2 || byID["OneDrive"].TeaserPending != 1 || byID["OneDrive"].TeaserFailed != 0 {
@@ -330,13 +480,22 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
if byID["OneDrive"].Thumbnail.State != "cooling" || byID["OneDrive"].Preview.State != "generating" {
t.Fatalf("OneDrive generation statuses = %#v, want thumbnail cooling and preview generating", byID["OneDrive"])
}
if byID["OneDrive"].FingerprintReady != 1 || byID["OneDrive"].FingerprintPending != 1 || byID["OneDrive"].FingerprintFailed != 1 {
t.Fatalf("OneDrive fingerprint counts = %#v, want ready=1 pending=1 failed=1", byID["OneDrive"])
}
if byID["OneDrive"].Fingerprint.State != "generating" {
t.Fatalf("OneDrive fingerprint status = %#v, want generating", byID["OneDrive"].Fingerprint)
}
if byID["PikPak"].TeaserReady != 0 || byID["PikPak"].TeaserPending != 1 || byID["PikPak"].TeaserFailed != 1 {
t.Fatalf("PikPak counts = %#v, want ready=0 pending=1 failed=1", byID["PikPak"])
}
if byID["PikPak"].ThumbnailReady != 1 || byID["PikPak"].ThumbnailPending != 1 || byID["PikPak"].ThumbnailFailed != 0 {
t.Fatalf("PikPak thumbnail counts = %#v, want ready=1 pending=1 failed=0", byID["PikPak"])
}
if byID["PikPak"].Thumbnail.State != "idle" || byID["PikPak"].Preview.State != "idle" {
if byID["PikPak"].FingerprintPending != 2 {
t.Fatalf("PikPak fingerprint counts = %#v, want pending=2", byID["PikPak"])
}
if byID["PikPak"].Thumbnail.State != "idle" || byID["PikPak"].Preview.State != "idle" || byID["PikPak"].Fingerprint.State != "idle" {
t.Fatalf("PikPak generation statuses = %#v, want idle defaults", byID["PikPak"])
}
}
+164 -58
View File
@@ -21,6 +21,7 @@ import (
"github.com/video-site/backend/internal/auth"
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/drives/localstorage"
"github.com/video-site/backend/internal/drives/localupload"
"github.com/video-site/backend/internal/drives/spider91"
"github.com/video-site/backend/internal/proxy"
@@ -155,26 +156,67 @@ func (s *Server) handleGetTheme(w http.ResponseWriter, r *http.Request) {
}
func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) {
// 拉一批候选(按发布时间倒序,覆盖最近 200 个),然后随机洗牌取前 homePageSize 个
// 如果库内不足 200 个会自动按实际数量返回,最后裁剪到 homePageSize
// 首页优先展示封面已经生成好的视频,避免新盘扫盘时大量黑封面占满首页
// 候选仍按发布时间覆盖最近 200 个,随后随机洗牌;封面不足时再用普通可见视频补齐
const candidatePool = 200
items, _, err := s.Catalog.ListVideos(r.Context(), catalog.ListParams{
Sort: "latest", Page: 1, PageSize: candidatePool,
readyItems, _, err := s.Catalog.ListVideos(r.Context(), catalog.ListParams{
Sort: "latest", Page: 1, PageSize: candidatePool, ThumbnailReadyOnly: true,
})
if err != nil {
writeErr(w, http.StatusInternalServerError, err)
return
}
rand.Shuffle(len(items), func(i, j int) {
items[i], items[j] = items[j], items[i]
rand.Shuffle(len(readyItems), func(i, j int) {
readyItems[i], readyItems[j] = readyItems[j], readyItems[i]
})
items := appendUniqueVideos(nil, readyItems, homePageSize)
if len(items) > homePageSize {
items = items[:homePageSize]
}
if len(items) < homePageSize {
fallback, _, err := s.Catalog.ListVideos(r.Context(), catalog.ListParams{
Sort: "latest", Page: 1, PageSize: candidatePool,
})
if err != nil {
writeErr(w, http.StatusInternalServerError, err)
return
}
rand.Shuffle(len(fallback), func(i, j int) {
fallback[i], fallback[j] = fallback[j], fallback[i]
})
items = appendUniqueVideos(items, fallback, homePageSize)
}
w.Header().Set("Cache-Control", "no-store")
writeJSON(w, http.StatusOK, mapVideos(items))
}
func appendUniqueVideos(dst []*catalog.Video, candidates []*catalog.Video, limit int) []*catalog.Video {
if len(dst) >= limit {
return dst[:limit]
}
seen := make(map[string]struct{}, len(dst))
for _, v := range dst {
if v != nil {
seen[v.ID] = struct{}{}
}
}
for _, v := range candidates {
if v == nil {
continue
}
if _, ok := seen[v.ID]; ok {
continue
}
dst = append(dst, v)
seen[v.ID] = struct{}{}
if len(dst) >= limit {
return dst
}
}
return dst
}
func (s *Server) handleList(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
page, _ := strconv.Atoi(q.Get("page"))
@@ -182,14 +224,18 @@ func (s *Server) handleList(w http.ResponseWriter, r *http.Request) {
if size <= 0 {
size = 24
}
sort := q.Get("sort")
params := catalog.ListParams{
Keyword: q.Get("q"),
Tag: q.Get("tag"),
Category: q.Get("cat"),
Sort: q.Get("sort"),
Sort: sort,
Page: page,
PageSize: size,
}
if sort == "" || sort == "latest" {
params.PreferReadyThumbnails = true
}
items, total, err := s.Catalog.ListVideos(r.Context(), params)
if err != nil {
writeErr(w, http.StatusInternalServerError, err)
@@ -241,7 +287,8 @@ func (s *Server) handleVideoDetail(w http.ResponseWriter, r *http.Request) {
}
// pickRelatedVideos 选 total 个推荐视频。
// 一半(向上取整)来自同标签命中,剩下用全库随机补齐;不会重复,也不会包含当前视频
// 一半来自同标签命中,剩下用全库随机补齐;两段都优先取已有封面的视频
// 不够时再回退到未生成封面的候选。结果不会重复,也不会包含当前视频。
func (s *Server) pickRelatedVideos(ctx context.Context, current *catalog.Video, total int) []*catalog.Video {
if total <= 0 || current == nil {
return nil
@@ -254,67 +301,124 @@ func (s *Server) pickRelatedVideos(ctx context.Context, current *catalog.Video,
picked := make([]*catalog.Video, 0, total)
seen := map[string]struct{}{current.ID: {}}
// 1) 同标签候选:对每个 tag 取一批,合并去重,洗牌后取 tagQuota 个
// 1) 同标签候选:先取已有封面的候选,数量不够再从全部候选里补。
if tagQuota > 0 && len(current.Tags) > 0 {
var tagPool []*catalog.Video
for _, tag := range current.Tags {
if tag == "" {
continue
}
items, _, err := s.Catalog.ListVideos(ctx, catalog.ListParams{
Tag: tag, Sort: "latest", Page: 1, PageSize: 30,
})
if err != nil {
continue
}
for _, v := range items {
if v == nil {
continue
}
if _, ok := seen[v.ID]; ok {
continue
}
seen[v.ID] = struct{}{}
tagPool = append(tagPool, v)
}
picked = appendRandomRelated(
picked,
s.relatedTagPool(ctx, current.Tags, seen, true),
tagQuota,
seen,
)
if len(picked) < tagQuota {
picked = appendRandomRelated(
picked,
s.relatedTagPool(ctx, current.Tags, seen, false),
tagQuota,
seen,
)
}
rand.Shuffle(len(tagPool), func(i, j int) {
tagPool[i], tagPool[j] = tagPool[j], tagPool[i]
})
if len(tagPool) > tagQuota {
tagPool = tagPool[:tagQuota]
}
picked = append(picked, tagPool...)
}
// 2) 随机补齐:从全库取一批(避开已选 ID),洗牌后取剩下的名额
remaining := total - len(picked)
if remaining > 0 {
// 2) 随机补齐:同样优先已有封面的全库候选,不够再回退。
if len(picked) < total {
picked = appendRandomRelated(
picked,
s.relatedListPool(ctx, seen, true, 200),
total,
seen,
)
}
if len(picked) < total {
picked = appendRandomRelated(
picked,
s.relatedListPool(ctx, seen, false, 200),
total,
seen,
)
}
return picked
}
func (s *Server) relatedTagPool(ctx context.Context, tags []string, seen map[string]struct{}, readyOnly bool) []*catalog.Video {
var pool []*catalog.Video
poolSeen := make(map[string]struct{})
for _, tag := range tags {
if tag == "" {
continue
}
items, _, err := s.Catalog.ListVideos(ctx, catalog.ListParams{
Sort: "latest", Page: 1, PageSize: 200,
Tag: tag,
Sort: "latest",
Page: 1,
PageSize: 30,
ThumbnailReadyOnly: readyOnly,
PreferReadyThumbnails: !readyOnly,
})
if err == nil {
var randomPool []*catalog.Video
for _, v := range items {
if v == nil {
continue
}
if _, ok := seen[v.ID]; ok {
continue
}
seen[v.ID] = struct{}{}
randomPool = append(randomPool, v)
if err != nil {
continue
}
for _, v := range items {
if v == nil {
continue
}
rand.Shuffle(len(randomPool), func(i, j int) {
randomPool[i], randomPool[j] = randomPool[j], randomPool[i]
})
if len(randomPool) > remaining {
randomPool = randomPool[:remaining]
if _, ok := seen[v.ID]; ok {
continue
}
picked = append(picked, randomPool...)
if _, ok := poolSeen[v.ID]; ok {
continue
}
poolSeen[v.ID] = struct{}{}
pool = append(pool, v)
}
}
return pool
}
func (s *Server) relatedListPool(ctx context.Context, seen map[string]struct{}, readyOnly bool, pageSize int) []*catalog.Video {
items, _, err := s.Catalog.ListVideos(ctx, catalog.ListParams{
Sort: "latest",
Page: 1,
PageSize: pageSize,
ThumbnailReadyOnly: readyOnly,
PreferReadyThumbnails: !readyOnly,
})
if err != nil {
return nil
}
pool := make([]*catalog.Video, 0, len(items))
for _, v := range items {
if v == nil {
continue
}
if _, ok := seen[v.ID]; ok {
continue
}
pool = append(pool, v)
}
return pool
}
func appendRandomRelated(picked []*catalog.Video, pool []*catalog.Video, targetLen int, seen map[string]struct{}) []*catalog.Video {
if len(picked) >= targetLen || len(pool) == 0 {
return picked
}
rand.Shuffle(len(pool), func(i, j int) {
pool[i], pool[j] = pool[j], pool[i]
})
for _, v := range pool {
if len(picked) >= targetLen {
break
}
if v == nil {
continue
}
if _, ok := seen[v.ID]; ok {
continue
}
seen[v.ID] = struct{}{}
picked = append(picked, v)
}
return picked
}
@@ -796,6 +900,8 @@ func driveKindLabel(kind string) string {
return "联通沃盘"
case "onedrive":
return "OneDrive"
case localstorage.Kind:
return "本地存储"
case spider91.Kind:
return "91 爬虫"
default:
+223
View File
@@ -9,6 +9,7 @@ import (
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
@@ -98,6 +99,146 @@ func TestPreviewURLFallsBackWithoutUpdatedAt(t *testing.T) {
}
}
func TestHandleHomePrioritizesVideosWithReadyThumbnails(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
now := time.Now()
for i := 0; i < 20; i++ {
id := "pending-video-" + strconv.Itoa(i)
if err := cat.UpsertVideo(ctx, &catalog.Video{
ID: id,
DriveID: "drive",
FileID: id,
Title: id,
PublishedAt: now.Add(time.Duration(i) * time.Minute),
CreatedAt: now.Add(time.Duration(i) * time.Minute),
UpdatedAt: now.Add(time.Duration(i) * time.Minute),
}); err != nil {
t.Fatalf("seed pending video %s: %v", id, err)
}
}
for i := 0; i < homePageSize+2; i++ {
id := "ready-video-" + strconv.Itoa(i)
if err := cat.UpsertVideo(ctx, &catalog.Video{
ID: id,
DriveID: "drive",
FileID: id,
Title: id,
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
PublishedAt: now.Add(-time.Duration(i+1) * time.Hour),
CreatedAt: now.Add(-time.Duration(i+1) * time.Hour),
UpdatedAt: now.Add(-time.Duration(i+1) * time.Hour),
}); err != nil {
t.Fatalf("seed ready video %s: %v", id, err)
}
}
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/home", nil)
(&Server{Catalog: cat}).handleHome(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
var got []VideoDTO
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
t.Fatalf("decode response: %v", err)
}
if len(got) != homePageSize {
t.Fatalf("home items = %d, want %d", len(got), homePageSize)
}
for _, item := range got {
if !strings.HasPrefix(item.ID, "ready-video-") {
t.Fatalf("home returned %q without a ready thumbnail; items=%#v", item.ID, got)
}
if !strings.HasPrefix(item.Thumbnail, "https://thumb.example/") {
t.Fatalf("thumbnail for %q = %q, want ready thumbnail URL", item.ID, item.Thumbnail)
}
}
}
func TestHandleListLatestPrefersReadyThumbnails(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
now := time.Now()
for i := 0; i < 20; i++ {
id := "pending-latest-" + strconv.Itoa(i)
if err := cat.UpsertVideo(ctx, &catalog.Video{
ID: id,
DriveID: "drive",
FileID: id,
Title: id,
PublishedAt: now.Add(time.Duration(i) * time.Minute),
CreatedAt: now.Add(time.Duration(i) * time.Minute),
UpdatedAt: now.Add(time.Duration(i) * time.Minute),
}); err != nil {
t.Fatalf("seed pending video %s: %v", id, err)
}
}
for i := 0; i < 12; i++ {
id := "ready-latest-" + strconv.Itoa(i)
if err := cat.UpsertVideo(ctx, &catalog.Video{
ID: id,
DriveID: "drive",
FileID: id,
Title: id,
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
PublishedAt: now.Add(-time.Duration(i+1) * time.Hour),
CreatedAt: now.Add(-time.Duration(i+1) * time.Hour),
UpdatedAt: now.Add(-time.Duration(i+1) * time.Hour),
}); err != nil {
t.Fatalf("seed ready video %s: %v", id, err)
}
}
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/list?page=1&size=12&sort=latest", nil)
(&Server{Catalog: cat}).handleList(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
var got struct {
Items []VideoDTO `json:"items"`
Total int `json:"total"`
}
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
t.Fatalf("decode response: %v", err)
}
if got.Total != 32 {
t.Fatalf("total = %d, want all matching videos included", got.Total)
}
if len(got.Items) != 12 {
t.Fatalf("items = %d, want 12", len(got.Items))
}
for _, item := range got.Items {
if !strings.HasPrefix(item.ID, "ready-latest-") {
t.Fatalf("latest list returned %q before ready thumbnails; items=%#v", item.ID, got.Items)
}
if !strings.HasPrefix(item.Thumbnail, "https://thumb.example/") {
t.Fatalf("thumbnail for %q = %q, want ready thumbnail URL", item.ID, item.Thumbnail)
}
}
}
func TestHandleUploadVideoSavesFileVideoTagsAndQueuesPreview(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
@@ -509,6 +650,88 @@ func TestHandleVideoDetailIncludesDriveKindLabel(t *testing.T) {
}
}
func TestHandleVideoDetailRecommendationsPreferReadyThumbnails(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
now := time.Now()
if err := cat.UpsertVideo(ctx, &catalog.Video{
ID: "current-video",
DriveID: "drive",
FileID: "current-video",
Title: "Current",
Tags: []string{"same-tag"},
ThumbnailURL: "https://thumb.example/current-video.jpg",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
}); err != nil {
t.Fatalf("seed current video: %v", err)
}
for i := 0; i < 20; i++ {
id := "pending-related-" + strconv.Itoa(i)
if err := cat.UpsertVideo(ctx, &catalog.Video{
ID: id,
DriveID: "drive",
FileID: id,
Title: id,
Tags: []string{"same-tag"},
PublishedAt: now.Add(time.Duration(i+1) * time.Minute),
CreatedAt: now.Add(time.Duration(i+1) * time.Minute),
UpdatedAt: now.Add(time.Duration(i+1) * time.Minute),
}); err != nil {
t.Fatalf("seed pending related video %s: %v", id, err)
}
}
for i := 0; i < 8; i++ {
id := "ready-related-" + strconv.Itoa(i)
if err := cat.UpsertVideo(ctx, &catalog.Video{
ID: id,
DriveID: "drive",
FileID: id,
Title: id,
Tags: []string{"same-tag"},
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
PublishedAt: now.Add(-time.Duration(i+1) * time.Hour),
CreatedAt: now.Add(-time.Duration(i+1) * time.Hour),
UpdatedAt: now.Add(-time.Duration(i+1) * time.Hour),
}); err != nil {
t.Fatalf("seed ready related video %s: %v", id, err)
}
}
req := requestWithVideoID(http.MethodGet, "/api/video/current-video", "current-video", strings.NewReader(``))
rr := httptest.NewRecorder()
(&Server{Catalog: cat}).handleVideoDetail(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
var got VideoDetailDTO
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
t.Fatalf("decode: %v", err)
}
if len(got.RelatedVideos) != 6 {
t.Fatalf("related videos = %d, want 6; items=%#v", len(got.RelatedVideos), got.RelatedVideos)
}
for _, item := range got.RelatedVideos {
if !strings.HasPrefix(item.ID, "ready-related-") {
t.Fatalf("related returned %q before ready thumbnails; items=%#v", item.ID, got.RelatedVideos)
}
if !strings.HasPrefix(item.Thumbnail, "https://thumb.example/") {
t.Fatalf("thumbnail for %q = %q, want ready thumbnail URL", item.ID, item.Thumbnail)
}
}
}
func TestHandleHideVideoRemovesVideoFromPublicListAndDetail(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
+328 -49
View File
@@ -41,35 +41,38 @@ func (c *Catalog) Close() error { return c.db.Close() }
// ---------- Video ----------
type Video struct {
ID string `json:"id"`
DriveID string `json:"driveId"`
FileID string `json:"fileId"`
FileName string `json:"fileName"`
ContentHash string `json:"contentHash"`
ParentID string `json:"parentId"`
Title string `json:"title"`
Author string `json:"author"`
Tags []string `json:"tags"`
DurationSeconds int `json:"durationSeconds"`
Size int64 `json:"size"`
Ext string `json:"ext"`
Quality string `json:"quality"`
ThumbnailURL string `json:"thumbnailUrl"`
PreviewFileID string `json:"previewFileId"`
PreviewLocal string `json:"previewLocal"`
PreviewStatus string `json:"previewStatus"`
Views int `json:"views"`
Favorites int `json:"favorites"`
Comments int `json:"comments"`
Likes int `json:"likes"`
Dislikes int `json:"dislikes"`
Category string `json:"category"`
Hidden bool `json:"hidden"`
Badges []string `json:"badges"`
Description string `json:"description"`
PublishedAt time.Time `json:"publishedAt"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ID string `json:"id"`
DriveID string `json:"driveId"`
FileID string `json:"fileId"`
FileName string `json:"fileName"`
ContentHash string `json:"contentHash"`
SampledSHA256 string `json:"sampledSha256"`
FingerprintStatus string `json:"fingerprintStatus"`
FingerprintError string `json:"fingerprintError"`
ParentID string `json:"parentId"`
Title string `json:"title"`
Author string `json:"author"`
Tags []string `json:"tags"`
DurationSeconds int `json:"durationSeconds"`
Size int64 `json:"size"`
Ext string `json:"ext"`
Quality string `json:"quality"`
ThumbnailURL string `json:"thumbnailUrl"`
PreviewFileID string `json:"previewFileId"`
PreviewLocal string `json:"previewLocal"`
PreviewStatus string `json:"previewStatus"`
Views int `json:"views"`
Favorites int `json:"favorites"`
Comments int `json:"comments"`
Likes int `json:"likes"`
Dislikes int `json:"dislikes"`
Category string `json:"category"`
Hidden bool `json:"hidden"`
Badges []string `json:"badges"`
Description string `json:"description"`
PublishedAt time.Time `json:"publishedAt"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
func (c *Catalog) UpsertVideo(ctx context.Context, v *Video) error {
@@ -109,6 +112,18 @@ ON CONFLICT(id) DO UPDATE SET
WHEN excluded.content_hash != '' THEN excluded.content_hash
ELSE videos.content_hash
END,
sampled_sha256 = CASE
WHEN videos.size_bytes != excluded.size_bytes THEN ''
ELSE videos.sampled_sha256
END,
fingerprint_status = CASE
WHEN videos.size_bytes != excluded.size_bytes THEN 'pending'
ELSE COALESCE(videos.fingerprint_status, 'pending')
END,
fingerprint_error = CASE
WHEN videos.size_bytes != excluded.size_bytes THEN ''
ELSE COALESCE(videos.fingerprint_error, '')
END,
duration_seconds= excluded.duration_seconds,
size_bytes = excluded.size_bytes,
ext = excluded.ext,
@@ -199,8 +214,8 @@ func (c *Catalog) MigrateVideoToDrive(ctx context.Context, videoID, newDriveID,
}
// ListVideosByDriveID 列出指定 drive 下所有未隐藏的视频,按 published_at 倒序。
// 给 spider91 → PikPak 迁移 worker 用:扫描 spider91 drive 下所有视频,
// 检查哪些还有本地文件,依次上传到 PikPak
// 给 spider91 → 115/PikPak 迁移 worker 用:扫描 spider91 drive 下所有视频,
// 检查哪些还有本地文件,依次上传到目标盘
func (c *Catalog) ListVideosByDriveID(ctx context.Context, driveID string, limit int) ([]*Video, error) {
if driveID == "" {
return nil, fmt.Errorf("catalog: list videos by drive: empty drive id")
@@ -486,8 +501,14 @@ func (c *Catalog) ListVideosByThumbnailStatus(ctx context.Context, driveID, stat
return out, nil
}
// ListVideosNeedingThumbnail returns videos that still need a thumbnail attempt.
// ListVideosNeedingThumbnail returns videos that still need thumbnail-worker work.
// Besides missing thumbnails, this includes videos with an existing thumbnail but
// missing duration metadata, because the thumbnail worker probes duration while
// it already has a stream link.
// Failed thumbnails are reported separately and should not block teaser generation.
// Videos whose local assets were cleared because they are fingerprint duplicates
// stay pending in the DB, but uniqueVideoWhereSQL keeps them out of this queue
// while their canonical sibling still exists.
func (c *Catalog) ListVideosNeedingThumbnail(ctx context.Context, driveID string, limit int) ([]*Video, error) {
if limit <= 0 {
limit = 10000
@@ -495,8 +516,11 @@ func (c *Catalog) ListVideosNeedingThumbnail(ctx context.Context, driveID string
rows, err := c.db.QueryContext(ctx,
`SELECT `+allVideoCols+` FROM videos
WHERE drive_id = ?
AND COALESCE(thumbnail_url, '') = ''
AND COALESCE(thumbnail_status, 'pending') != 'failed'
AND (
COALESCE(thumbnail_url, '') = ''
OR COALESCE(duration_seconds, 0) <= 0
)
AND COALESCE(thumbnail_status, 'pending') NOT IN ('failed', 'skipped')
AND COALESCE(hidden, 0) = 0
AND `+uniqueVideoWhereSQL+`
ORDER BY created_at ASC
@@ -522,8 +546,11 @@ func (c *Catalog) CountVideosNeedingThumbnail(ctx context.Context, driveID strin
err := c.db.QueryRowContext(ctx,
`SELECT COUNT(*) FROM videos
WHERE drive_id = ?
AND COALESCE(thumbnail_url, '') = ''
AND COALESCE(thumbnail_status, 'pending') != 'failed'
AND (
COALESCE(thumbnail_url, '') = ''
OR COALESCE(duration_seconds, 0) <= 0
)
AND COALESCE(thumbnail_status, 'pending') NOT IN ('failed', 'skipped')
AND COALESCE(hidden, 0) = 0
AND `+uniqueVideoWhereSQL,
driveID).Scan(&count)
@@ -668,14 +695,70 @@ func (c *Catalog) FindVideoByFileSignature(ctx context.Context, fileName string,
return scanVideo(row)
}
func (c *Catalog) ListVideosNeedingFingerprint(ctx context.Context, driveID string, limit int) ([]*Video, error) {
if limit <= 0 {
limit = 10000
}
rows, err := c.db.QueryContext(ctx,
`SELECT `+allVideoCols+` FROM videos
WHERE drive_id = ?
AND size_bytes > 0
AND COALESCE(sampled_sha256, '') = ''
AND COALESCE(fingerprint_status, 'pending') = 'pending'
AND COALESCE(hidden, 0) = 0
ORDER BY created_at ASC, id ASC
LIMIT ?`,
driveID, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []*Video
for rows.Next() {
v, err := scanVideo(rows)
if err != nil {
return nil, err
}
out = append(out, v)
}
return out, rows.Err()
}
func (c *Catalog) UpdateVideoFingerprint(ctx context.Context, id, sampledSHA256, status, errText string) error {
sampledSHA256 = normalizeContentHash(sampledSHA256)
if status == "" {
status = "pending"
}
if len(errText) > 500 {
errText = errText[:500]
}
res, err := c.db.ExecContext(ctx,
`UPDATE videos
SET sampled_sha256 = ?,
fingerprint_status = ?,
fingerprint_error = ?,
updated_at = ?
WHERE id = ?`,
sampledSHA256, status, errText, time.Now().UnixMilli(), id)
if err != nil {
return err
}
if rows, err := res.RowsAffected(); err == nil && rows == 0 {
return sql.ErrNoRows
}
return nil
}
type ListParams struct {
Keyword string
DriveID string
Tag string
Category string
Sort string // latest | hot | week | long
Page int
PageSize int
Keyword string
DriveID string
Tag string
Category string
Sort string // latest | hot | week | long
ThumbnailReadyOnly bool
PreferReadyThumbnails bool
Page int
PageSize int
}
func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int, error) {
@@ -710,21 +793,29 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
where = append(where, "category = ?")
args = append(args, p.Category)
}
if p.ThumbnailReadyOnly {
where = append(where, "COALESCE(thumbnail_url, '') != ''")
}
where = append(where, "COALESCE(hidden, 0) = 0")
where = append(where, uniqueVideoWhereSQL)
whereSQL := ""
whereSQL = " WHERE " + strings.Join(where, " AND ")
orderBy := " ORDER BY published_at DESC"
readyOrderPrefix := ""
if p.PreferReadyThumbnails {
readyOrderPrefix = "CASE WHEN COALESCE(thumbnail_url, '') != '' THEN 0 ELSE 1 END, "
}
orderBy := " ORDER BY " + readyOrderPrefix + "published_at DESC"
switch p.Sort {
case "hot":
// 热度 = 点赞数,点赞相同按最新
orderBy = " ORDER BY likes DESC, published_at DESC"
orderBy = " ORDER BY " + readyOrderPrefix + "likes DESC, published_at DESC"
case "week":
orderBy = " ORDER BY likes DESC"
orderBy = " ORDER BY " + readyOrderPrefix + "likes DESC"
case "long":
orderBy = " ORDER BY duration_seconds DESC"
orderBy = " ORDER BY " + readyOrderPrefix + "duration_seconds DESC"
}
// count
@@ -842,6 +933,12 @@ type DriveThumbnailCounts struct {
Failed int
}
type DriveFingerprintCounts struct {
Ready int
Pending int
Failed int
}
func (c *Catalog) CountTeasersByDrive(ctx context.Context) (map[string]DriveTeaserCounts, error) {
rows, err := c.db.QueryContext(ctx,
`SELECT drive_id,
@@ -904,6 +1001,52 @@ func (c *Catalog) CountThumbnailsByDrive(ctx context.Context) (map[string]DriveT
return out, nil
}
func (c *Catalog) CountFingerprintsByDrive(ctx context.Context) (map[string]DriveFingerprintCounts, error) {
rows, err := c.db.QueryContext(ctx,
`SELECT drive_id,
COUNT(CASE WHEN COALESCE(sampled_sha256, '') != ''
OR COALESCE(fingerprint_status, 'pending') = 'ready' THEN 1 END) AS ready_count,
COUNT(CASE WHEN size_bytes > 0
AND COALESCE(sampled_sha256, '') = ''
AND COALESCE(fingerprint_status, 'pending') = 'pending' THEN 1 END) AS pending_count,
COUNT(CASE WHEN COALESCE(sampled_sha256, '') = ''
AND COALESCE(fingerprint_status, 'pending') = 'failed' THEN 1 END) AS failed_count
FROM videos
WHERE COALESCE(hidden, 0) = 0
GROUP BY drive_id`)
if err != nil {
return nil, err
}
defer rows.Close()
out := make(map[string]DriveFingerprintCounts)
for rows.Next() {
var driveID string
var counts DriveFingerprintCounts
if err := rows.Scan(&driveID, &counts.Ready, &counts.Pending, &counts.Failed); err != nil {
return nil, err
}
out[driveID] = counts
}
if err := rows.Err(); err != nil {
return nil, err
}
return out, nil
}
func (c *Catalog) CountVideosNeedingFingerprint(ctx context.Context, driveID string) (int, error) {
var count int
err := c.db.QueryRowContext(ctx,
`SELECT COUNT(*) FROM videos
WHERE drive_id = ?
AND size_bytes > 0
AND COALESCE(sampled_sha256, '') = ''
AND COALESCE(fingerprint_status, 'pending') = 'pending'
AND COALESCE(hidden, 0) = 0`,
driveID).Scan(&count)
return count, err
}
type LocalMediaRef struct {
DriveID string
VideoID string
@@ -933,6 +1076,124 @@ func (c *Catalog) ListLocalMediaRefs(ctx context.Context) ([]LocalMediaRef, erro
return out, nil
}
// DuplicateAssetCleanupCandidate points at a non-canonical video in a
// size+sampled_sha256 duplicate group that still owns generated local assets.
// The cleanup job uses this to remove duplicate thumbnails/teasers without
// touching the original cloud file or deleting the catalog row.
type DuplicateAssetCleanupCandidate struct {
VideoID string
DriveID string
Title string
PreviewLocal string
ThumbnailURL string
CanonicalID string
SampledSHA256 string
Size int64
}
// ListDuplicateAssetCleanupCandidates returns duplicate videos whose own local
// generated assets can be cleared. A group canonical is the same representative
// used by uniqueVideoWhereSQL: earliest created_at, then lexicographically
// smallest id.
func (c *Catalog) ListDuplicateAssetCleanupCandidates(ctx context.Context, limit int) ([]DuplicateAssetCleanupCandidate, error) {
if limit <= 0 {
limit = 10000
}
rows, err := c.db.QueryContext(ctx, `
WITH canonical AS (
SELECT v.id, v.size_bytes, v.sampled_sha256
FROM videos v
WHERE v.size_bytes > 0
AND COALESCE(v.sampled_sha256, '') != ''
AND NOT EXISTS (
SELECT 1
FROM videos earlier
WHERE earlier.size_bytes = v.size_bytes
AND earlier.sampled_sha256 = v.sampled_sha256
AND COALESCE(earlier.sampled_sha256, '') != ''
AND earlier.size_bytes > 0
AND (
earlier.created_at < v.created_at
OR (earlier.created_at = v.created_at AND earlier.id < v.id)
)
)
)
SELECT dup.id,
dup.drive_id,
dup.title,
COALESCE(dup.preview_local, ''),
COALESCE(dup.thumbnail_url, ''),
canonical.id,
dup.sampled_sha256,
dup.size_bytes
FROM videos dup
JOIN canonical
ON canonical.size_bytes = dup.size_bytes
AND canonical.sampled_sha256 = dup.sampled_sha256
WHERE dup.id != canonical.id
AND dup.size_bytes > 0
AND COALESCE(dup.sampled_sha256, '') != ''
AND (
COALESCE(dup.preview_local, '') != ''
OR COALESCE(dup.thumbnail_url, '') = '/p/thumb/' || dup.id
)
ORDER BY dup.created_at ASC, dup.id ASC
LIMIT ?`, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []DuplicateAssetCleanupCandidate
for rows.Next() {
var item DuplicateAssetCleanupCandidate
if err := rows.Scan(
&item.VideoID,
&item.DriveID,
&item.Title,
&item.PreviewLocal,
&item.ThumbnailURL,
&item.CanonicalID,
&item.SampledSHA256,
&item.Size,
); err != nil {
return nil, err
}
out = append(out, item)
}
if err := rows.Err(); err != nil {
return nil, err
}
return out, nil
}
// ClearGeneratedAssets clears DB references to generated local assets for a
// video. The statuses go back to pending so the video can regenerate assets if
// it later becomes the canonical item after its older duplicate is removed.
func (c *Catalog) ClearGeneratedAssets(ctx context.Context, videoID string, clearPreview, clearThumbnail bool) error {
parts := []string{}
args := []any{}
if clearPreview {
parts = append(parts, "preview_file_id = ''", "preview_local = ''", "preview_status = 'pending'")
}
if clearThumbnail {
parts = append(parts, "thumbnail_url = ''", "thumbnail_status = 'pending'")
}
if len(parts) == 0 {
return nil
}
parts = append(parts, "updated_at = ?")
args = append(args, time.Now().UnixMilli(), videoID)
res, err := c.db.ExecContext(ctx, `UPDATE videos SET `+strings.Join(parts, ", ")+` WHERE id = ?`, args...)
if err != nil {
return err
}
if rows, err := res.RowsAffected(); err == nil && rows == 0 {
return sql.ErrNoRows
}
return nil
}
// ---------- Drive ----------
type Drive struct {
@@ -1161,7 +1422,9 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.upd
// ---------- helpers ----------
const allVideoCols = `
id, drive_id, file_id, COALESCE(file_name, ''), COALESCE(content_hash, ''), COALESCE(parent_id, ''), title, COALESCE(author, ''), COALESCE(tags, '[]'),
id, drive_id, file_id, COALESCE(file_name, ''), COALESCE(content_hash, ''),
COALESCE(sampled_sha256, ''), COALESCE(fingerprint_status, 'pending'), COALESCE(fingerprint_error, ''),
COALESCE(parent_id, ''), title, COALESCE(author, ''), COALESCE(tags, '[]'),
duration_seconds, size_bytes, COALESCE(ext, ''), COALESCE(quality, ''), COALESCE(thumbnail_url, ''),
COALESCE(preview_file_id, ''), COALESCE(preview_local, ''), COALESCE(preview_status, 'pending'),
views, favorites, comments, likes, dislikes,
@@ -1180,6 +1443,20 @@ const uniqueVideoWhereSQL = `((COALESCE(videos.content_hash, '') = ''
OR (dup.created_at = videos.created_at AND dup.id < videos.id)
)
))
AND (COALESCE(videos.sampled_sha256, '') = ''
OR videos.size_bytes <= 0
OR NOT EXISTS (
SELECT 1
FROM videos AS dup
WHERE dup.sampled_sha256 = videos.sampled_sha256
AND dup.size_bytes = videos.size_bytes
AND COALESCE(dup.sampled_sha256, '') != ''
AND dup.size_bytes > 0
AND (
dup.created_at < videos.created_at
OR (dup.created_at = videos.created_at AND dup.id < videos.id)
)
))
AND (COALESCE(videos.file_name, '') = ''
OR videos.size_bytes <= 0
OR NOT EXISTS (
@@ -1205,7 +1482,9 @@ func scanVideo(row rowScanner) (*Video, error) {
var publishedAt, createdAt, updatedAt int64
var hidden int
err := row.Scan(
&v.ID, &v.DriveID, &v.FileID, &v.FileName, &v.ContentHash, &v.ParentID, &v.Title, &v.Author, &tagsJSON,
&v.ID, &v.DriveID, &v.FileID, &v.FileName, &v.ContentHash,
&v.SampledSHA256, &v.FingerprintStatus, &v.FingerprintError,
&v.ParentID, &v.Title, &v.Author, &tagsJSON,
&v.DurationSeconds, &v.Size, &v.Ext, &v.Quality, &v.ThumbnailURL,
&v.PreviewFileID, &v.PreviewLocal, &v.PreviewStatus,
&v.Views, &v.Favorites, &v.Comments, &v.Likes, &v.Dislikes,
@@ -0,0 +1,179 @@
package catalog
import (
"context"
"testing"
"time"
)
func TestListVideosDeduplicatesBySampledSHA256(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
now := time.Now()
for _, v := range []*Video{
{
ID: "drive-a-file-a",
DriveID: "drive-a",
FileID: "file-a",
FileName: "first-name.mp4",
Title: "First",
Size: 1234,
PublishedAt: now.Add(-time.Minute),
CreatedAt: now.Add(-time.Minute),
UpdatedAt: now.Add(-time.Minute),
},
{
ID: "drive-b-file-b",
DriveID: "drive-b",
FileID: "file-b",
FileName: "second-name.mp4",
Title: "Second",
Size: 1234,
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
},
} {
if err := cat.UpsertVideo(ctx, v); err != nil {
t.Fatalf("upsert %s: %v", v.ID, err)
}
}
items, total, err := cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 10})
if err != nil {
t.Fatalf("list before fingerprint: %v", err)
}
if total != 2 || len(items) != 2 {
t.Fatalf("before fingerprint total=%d len=%d, want 2", total, len(items))
}
const sampled = "abc123"
if err := cat.UpdateVideoFingerprint(ctx, "drive-a-file-a", sampled, "ready", ""); err != nil {
t.Fatalf("update a fingerprint: %v", err)
}
if err := cat.UpdateVideoFingerprint(ctx, "drive-b-file-b", sampled, "ready", ""); err != nil {
t.Fatalf("update b fingerprint: %v", err)
}
items, total, err = cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 10})
if err != nil {
t.Fatalf("list after fingerprint: %v", err)
}
if total != 1 || len(items) != 1 {
t.Fatalf("after fingerprint total=%d len=%d, want 1", total, len(items))
}
if items[0].ID != "drive-a-file-a" {
t.Fatalf("canonical id = %q, want earliest created video", items[0].ID)
}
}
func TestDuplicateAssetCleanupCandidates(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
base := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
videos := []*Video{
{
ID: "drive-a-canonical",
DriveID: "drive-a",
FileID: "file-a",
FileName: "canonical.mp4",
Title: "Canonical",
Size: 1234,
ThumbnailURL: "/p/thumb/drive-a-canonical",
PreviewLocal: "/tmp/previews/canonical.mp4",
PreviewStatus: "ready",
PublishedAt: base,
CreatedAt: base,
UpdatedAt: base,
},
{
ID: "drive-b-duplicate",
DriveID: "drive-b",
FileID: "file-b",
FileName: "duplicate.mp4",
Title: "Duplicate",
Size: 1234,
ThumbnailURL: "/p/thumb/drive-b-duplicate",
PreviewLocal: "/tmp/previews/duplicate.mp4",
PreviewStatus: "ready",
PublishedAt: base.Add(time.Second),
CreatedAt: base.Add(time.Second),
UpdatedAt: base.Add(time.Second),
},
{
ID: "drive-c-remote-thumb",
DriveID: "drive-c",
FileID: "file-c",
FileName: "remote-thumb.mp4",
Title: "Remote Thumbnail",
Size: 1234,
ThumbnailURL: "https://thumb.example/file-c.jpg",
PreviewStatus: "ready",
PublishedAt: base.Add(2 * time.Second),
CreatedAt: base.Add(2 * time.Second),
UpdatedAt: base.Add(2 * time.Second),
},
}
for _, v := range videos {
if err := cat.UpsertVideo(ctx, v); err != nil {
t.Fatalf("seed %s: %v", v.ID, err)
}
}
const sampled = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
for _, v := range videos {
if err := cat.UpdateVideoFingerprint(ctx, v.ID, sampled, "ready", ""); err != nil {
t.Fatalf("fingerprint %s: %v", v.ID, err)
}
}
items, err := cat.ListDuplicateAssetCleanupCandidates(ctx, 0)
if err != nil {
t.Fatalf("list cleanup candidates: %v", err)
}
if len(items) != 1 {
t.Fatalf("candidates = %#v, want only local duplicate", items)
}
item := items[0]
if item.VideoID != "drive-b-duplicate" || item.CanonicalID != "drive-a-canonical" {
t.Fatalf("candidate = %#v, want duplicate with canonical", item)
}
if err := cat.ClearGeneratedAssets(ctx, item.VideoID, true, true); err != nil {
t.Fatalf("clear generated assets: %v", err)
}
got, err := cat.GetVideo(ctx, item.VideoID)
if err != nil {
t.Fatalf("get duplicate: %v", err)
}
if got.PreviewLocal != "" || got.PreviewStatus != "pending" {
t.Fatalf("preview after cleanup local=%q status=%q, want empty pending", got.PreviewLocal, got.PreviewStatus)
}
if got.ThumbnailURL != "" {
t.Fatalf("thumbnail after cleanup = %q, want empty", got.ThumbnailURL)
}
var thumbStatus string
if err := cat.db.QueryRowContext(ctx, `SELECT thumbnail_status FROM videos WHERE id = ?`, item.VideoID).Scan(&thumbStatus); err != nil {
t.Fatalf("query thumbnail status: %v", err)
}
if thumbStatus != "pending" {
t.Fatalf("thumbnail_status = %q, want pending", thumbStatus)
}
}
+4 -1
View File
@@ -5,6 +5,9 @@ CREATE TABLE IF NOT EXISTS videos (
file_id TEXT NOT NULL,
file_name TEXT DEFAULT '', -- 网盘侧原始文件名,用于同名同大小去重
content_hash TEXT DEFAULT '',
sampled_sha256 TEXT DEFAULT '', -- 跨网盘统一采样指纹(size + sampled bytes
fingerprint_status TEXT DEFAULT 'pending', -- pending / ready / failed
fingerprint_error TEXT DEFAULT '',
parent_id TEXT,
title TEXT NOT NULL,
author TEXT,
@@ -61,7 +64,7 @@ CREATE INDEX IF NOT EXISTS idx_video_tags_video ON video_tags(video_id);
-- 网盘账户
CREATE TABLE IF NOT EXISTS drives (
id TEXT PRIMARY KEY,
kind TEXT NOT NULL, -- quark / p115 / pikpak / wopan / onedrive / spider91
kind TEXT NOT NULL, -- quark / p115 / pikpak / wopan / onedrive / localstorage / spider91
name TEXT NOT NULL,
root_id TEXT NOT NULL DEFAULT '0',
scan_root_id TEXT, -- 扫描起点(默认 root_id
+12
View File
@@ -43,6 +43,15 @@ func (c *Catalog) migrate(ctx context.Context) error {
if err := c.addColumnIfMissing(ctx, "videos", "content_hash", "TEXT DEFAULT ''"); err != nil {
return err
}
if err := c.addColumnIfMissing(ctx, "videos", "sampled_sha256", "TEXT DEFAULT ''"); err != nil {
return err
}
if err := c.addColumnIfMissing(ctx, "videos", "fingerprint_status", "TEXT DEFAULT 'pending'"); err != nil {
return err
}
if err := c.addColumnIfMissing(ctx, "videos", "fingerprint_error", "TEXT DEFAULT ''"); err != nil {
return err
}
if err := c.addColumnIfMissing(ctx, "videos", "file_name", "TEXT DEFAULT ''"); err != nil {
return err
}
@@ -83,6 +92,9 @@ func (c *Catalog) migrate(ctx context.Context) error {
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_content_hash ON videos(content_hash)`); err != nil {
return err
}
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_sampled_sha256 ON videos(size_bytes, sampled_sha256)`); err != nil {
return err
}
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_hidden ON videos(hidden)`); err != nil {
return err
}
+137
View File
@@ -7,6 +7,90 @@ import (
"time"
)
func TestListVideosNeedingThumbnailIncludesExistingThumbnailMissingDuration(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
now := time.Now()
videos := []*Video{
{
ID: "duration-only",
DriveID: "drive",
FileID: "file-duration-only",
Title: "Duration Only",
ThumbnailURL: "/p/thumb/duration-only",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
},
{
ID: "complete",
DriveID: "drive",
FileID: "file-complete",
Title: "Complete",
DurationSeconds: 12,
ThumbnailURL: "/p/thumb/complete",
PublishedAt: now.Add(time.Second),
CreatedAt: now.Add(time.Second),
UpdatedAt: now.Add(time.Second),
},
{
ID: "missing-thumb",
DriveID: "drive",
FileID: "file-missing-thumb",
Title: "Missing Thumb",
DurationSeconds: 18,
PublishedAt: now.Add(2 * time.Second),
CreatedAt: now.Add(2 * time.Second),
UpdatedAt: now.Add(2 * time.Second),
},
{
ID: "failed",
DriveID: "drive",
FileID: "file-failed",
Title: "Failed",
PublishedAt: now.Add(3 * time.Second),
CreatedAt: now.Add(3 * time.Second),
UpdatedAt: now.Add(3 * time.Second),
},
}
for _, v := range videos {
if err := cat.UpsertVideo(ctx, v); err != nil {
t.Fatalf("seed %s: %v", v.ID, err)
}
}
if err := cat.UpdateVideoMeta(ctx, "failed", VideoMetaPatch{ThumbnailStatus: "failed"}); err != nil {
t.Fatalf("mark failed thumbnail: %v", err)
}
items, err := cat.ListVideosNeedingThumbnail(ctx, "drive", 0)
if err != nil {
t.Fatalf("list videos needing thumbnail: %v", err)
}
if len(items) != 2 {
t.Fatalf("items = %#v, want duration-only and missing-thumb", items)
}
if items[0].ID != "duration-only" || items[1].ID != "missing-thumb" {
t.Fatalf("item ids = %q, %q; want duration-only, missing-thumb", items[0].ID, items[1].ID)
}
count, err := cat.CountVideosNeedingThumbnail(ctx, "drive")
if err != nil {
t.Fatalf("count videos needing thumbnail: %v", err)
}
if count != 2 {
t.Fatalf("count = %d, want 2", count)
}
}
func TestCreateTagAndClassifyAddsTagToMatchingExistingVideos(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
@@ -581,6 +665,59 @@ func TestListVideosHidesDuplicateContentHashes(t *testing.T) {
}
}
func TestListVideosCanFilterReadyThumbnails(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
now := time.Now()
for _, v := range []*Video{
{
ID: "ready-video",
DriveID: "drive",
FileID: "file-ready",
Title: "Ready",
ThumbnailURL: "/p/thumb/ready-video",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
},
{
ID: "pending-video",
DriveID: "drive",
FileID: "file-pending",
Title: "Pending",
PublishedAt: now.Add(time.Second),
CreatedAt: now.Add(time.Second),
UpdatedAt: now.Add(time.Second),
},
} {
if err := cat.UpsertVideo(ctx, v); err != nil {
t.Fatalf("seed video %s: %v", v.ID, err)
}
}
items, total, err := cat.ListVideos(ctx, ListParams{
Page: 1, PageSize: 10, ThumbnailReadyOnly: true,
})
if err != nil {
t.Fatalf("list videos: %v", err)
}
if total != 1 || len(items) != 1 {
t.Fatalf("ready videos total=%d len=%d, want 1", total, len(items))
}
if items[0].ID != "ready-video" {
t.Fatalf("ready video id = %q, want ready-video", items[0].ID)
}
}
func sameStrings(a, b []string) bool {
if len(a) != len(b) {
return false
+1 -1
View File
@@ -202,7 +202,7 @@ type Nightly struct {
// 这里保留 yaml 中的静态定义,用于启动时预置盘。生产建议只在 DB 里维护。
type Drive struct {
ID string `yaml:"id"`
Kind string `yaml:"kind"` // quark / p115 / pikpak / wopan / onedrive
Kind string `yaml:"kind"` // quark / p115 / pikpak / wopan / onedrive / localstorage
Name string `yaml:"name"`
RootID string `yaml:"root_id"`
Params map[string]string `yaml:"params,omitempty"`
+1 -1
View File
@@ -10,7 +10,7 @@ import (
// Drive 是多家网盘统一抽象。上层不区分盘,只区分 Kind。
type Drive interface {
// Kind 返回驱动代号:"quark" / "p115" / "pikpak" / "wopan" / "onedrive"
// Kind 返回驱动代号:"quark" / "p115" / "pikpak" / "wopan" / "onedrive" / "localstorage"
Kind() string
// ID 返回该盘在 catalog 中的唯一标识
@@ -0,0 +1,242 @@
// Package localstorage exposes an existing server-side directory as a Drive.
package localstorage
import (
"context"
"encoding/base64"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/video-site/backend/internal/drives"
)
const Kind = "localstorage"
type Config struct {
ID string
RootPath string
}
type Driver struct {
id string
rootPath string
}
func New(c Config) *Driver {
return &Driver{
id: c.ID,
rootPath: c.RootPath,
}
}
func (d *Driver) Kind() string { return Kind }
func (d *Driver) ID() string { return d.id }
func (d *Driver) RootID() string { return "/" }
func (d *Driver) Init(context.Context) error {
root, err := d.root()
if err != nil {
return err
}
info, err := os.Stat(root)
if err != nil {
return fmt.Errorf("localstorage: stat root: %w", err)
}
if !info.IsDir() {
return fmt.Errorf("localstorage: root is not a directory: %s", root)
}
return nil
}
func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error) {
dir, rel, err := d.pathForID(dirID)
if err != nil {
return nil, err
}
entries, err := os.ReadDir(dir)
if err != nil {
return nil, err
}
out := make([]drives.Entry, 0, len(entries))
for _, entry := range entries {
if err := ctx.Err(); err != nil {
return nil, err
}
// Symlinks can escape the configured root or create cycles. Keep the
// local storage drive predictable by scanning real files/directories only.
if entry.Type()&os.ModeSymlink != 0 {
continue
}
info, err := entry.Info()
if err != nil {
continue
}
if !info.IsDir() && !info.Mode().IsRegular() {
continue
}
childRel := joinRel(rel, entry.Name())
out = append(out, drives.Entry{
ID: encodeRel(childRel),
Name: entry.Name(),
Size: sizeForEntry(info),
IsDir: info.IsDir(),
ParentID: idForRel(rel),
ModTime: info.ModTime(),
})
}
return out, nil
}
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
p, rel, err := d.pathForID(fileID)
if err != nil {
return nil, err
}
info, err := os.Stat(p)
if err != nil {
return nil, err
}
return &drives.Entry{
ID: idForRel(rel),
Name: filepath.Base(p),
Size: sizeForEntry(info),
IsDir: info.IsDir(),
ParentID: idForRel(parentRel(rel)),
ModTime: info.ModTime(),
}, nil
}
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
p, _, err := d.pathForID(fileID)
if err != nil {
return nil, err
}
info, err := os.Stat(p)
if err != nil {
return nil, err
}
if info.IsDir() || !info.Mode().IsRegular() || info.Size() <= 0 {
return nil, os.ErrNotExist
}
return &drives.StreamLink{
URL: p,
Expires: time.Now().Add(24 * time.Hour),
}, nil
}
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
return "", drives.ErrNotSupported
}
func (d *Driver) EnsureDir(context.Context, string) (string, error) {
return "", drives.ErrNotSupported
}
func (d *Driver) root() (string, error) {
raw := strings.TrimSpace(d.rootPath)
if raw == "" {
return "", errors.New("localstorage: empty path")
}
raw = os.ExpandEnv(raw)
if strings.HasPrefix(raw, "~") {
if home, err := os.UserHomeDir(); err == nil && home != "" {
switch {
case raw == "~":
raw = home
case strings.HasPrefix(raw, "~/") || strings.HasPrefix(raw, `~\`):
raw = filepath.Join(home, raw[2:])
}
}
}
return filepath.Abs(raw)
}
func (d *Driver) pathForID(id string) (string, string, error) {
root, err := d.root()
if err != nil {
return "", "", err
}
rel, err := decodeRel(id)
if err != nil {
return "", "", err
}
if rel == "" {
return root, "", nil
}
p, err := filepath.Abs(filepath.Join(root, filepath.FromSlash(rel)))
if err != nil {
return "", "", err
}
if p != root && !strings.HasPrefix(p, root+string(os.PathSeparator)) {
return "", "", errors.New("localstorage: path escapes root")
}
return p, rel, nil
}
func decodeRel(id string) (string, error) {
id = strings.TrimSpace(id)
if id == "" || id == "/" {
return "", nil
}
raw, err := base64.RawURLEncoding.DecodeString(id)
if err != nil {
return "", fmt.Errorf("localstorage: invalid file id: %w", err)
}
rel := filepath.ToSlash(filepath.Clean(filepath.FromSlash(string(raw))))
if rel == "." {
return "", nil
}
if strings.HasPrefix(rel, "../") || rel == ".." || strings.HasPrefix(rel, "/") {
return "", errors.New("localstorage: invalid relative path")
}
return rel, nil
}
func encodeRel(rel string) string {
rel = filepath.ToSlash(filepath.Clean(filepath.FromSlash(rel)))
if rel == "." || rel == "" {
return "/"
}
return base64.RawURLEncoding.EncodeToString([]byte(rel))
}
func idForRel(rel string) string {
if rel == "" {
return "/"
}
return encodeRel(rel)
}
func joinRel(parent, name string) string {
if parent == "" {
return filepath.ToSlash(name)
}
return filepath.ToSlash(filepath.Join(filepath.FromSlash(parent), name))
}
func parentRel(rel string) string {
if rel == "" {
return ""
}
parent := filepath.ToSlash(filepath.Dir(filepath.FromSlash(rel)))
if parent == "." {
return ""
}
return parent
}
func sizeForEntry(info os.FileInfo) int64 {
if info == nil || info.IsDir() {
return 0
}
return info.Size()
}
var _ drives.Drive = (*Driver)(nil)
@@ -0,0 +1,119 @@
package localstorage
import (
"context"
"encoding/base64"
"os"
"path/filepath"
"strings"
"testing"
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/scanner"
)
func TestListEncodesRelativePathsAndStreamURLResolvesFile(t *testing.T) {
root := t.TempDir()
sub := filepath.Join(root, "clips")
if err := os.MkdirAll(sub, 0o755); err != nil {
t.Fatalf("mkdir: %v", err)
}
videoPath := filepath.Join(sub, "sample.mp4")
if err := os.WriteFile(videoPath, []byte("video"), 0o644); err != nil {
t.Fatalf("write video: %v", err)
}
drv := New(Config{ID: "local", RootPath: root})
if err := drv.Init(context.Background()); err != nil {
t.Fatalf("init: %v", err)
}
rootEntries, err := drv.List(context.Background(), drv.RootID())
if err != nil {
t.Fatalf("list root: %v", err)
}
if len(rootEntries) != 1 || !rootEntries[0].IsDir {
t.Fatalf("root entries = %#v, want one directory", rootEntries)
}
if strings.Contains(rootEntries[0].ID, "/") {
t.Fatalf("encoded dir id contains slash: %q", rootEntries[0].ID)
}
fileEntries, err := drv.List(context.Background(), rootEntries[0].ID)
if err != nil {
t.Fatalf("list subdir: %v", err)
}
if len(fileEntries) != 1 || fileEntries[0].Name != "sample.mp4" {
t.Fatalf("file entries = %#v, want sample.mp4", fileEntries)
}
if strings.Contains(fileEntries[0].ID, "/") {
t.Fatalf("encoded file id contains slash: %q", fileEntries[0].ID)
}
link, err := drv.StreamURL(context.Background(), fileEntries[0].ID)
if err != nil {
t.Fatalf("stream url: %v", err)
}
if link.URL != videoPath {
t.Fatalf("url = %q, want %q", link.URL, videoPath)
}
}
func TestStreamURLRejectsEscapingID(t *testing.T) {
drv := New(Config{ID: "local", RootPath: t.TempDir()})
escaped := base64.RawURLEncoding.EncodeToString([]byte("../secret.mp4"))
_, err := drv.StreamURL(context.Background(), escaped)
if err == nil || !strings.Contains(err.Error(), "invalid relative path") {
t.Fatalf("error = %v, want invalid relative path", err)
}
}
func TestInitRequiresExistingDirectory(t *testing.T) {
drv := New(Config{ID: "local", RootPath: filepath.Join(t.TempDir(), "missing")})
err := drv.Init(context.Background())
if err == nil || !strings.Contains(err.Error(), "stat root") {
t.Fatalf("error = %v, want stat root failure", err)
}
}
func TestScannerPersistsLocalStorageVideo(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
if err := os.MkdirAll(filepath.Join(root, "collection"), 0o755); err != nil {
t.Fatalf("mkdir collection: %v", err)
}
if err := os.WriteFile(filepath.Join(root, "collection", "clip.mp4"), []byte("video"), 0o644); err != nil {
t.Fatalf("write video: %v", err)
}
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
drv := New(Config{ID: "local", RootPath: root})
sc := scanner.New(cat, drv, []string{".mp4"}, nil, nil)
stats, err := sc.Run(ctx, drv.RootID())
if err != nil {
t.Fatalf("scan: %v", err)
}
if stats.Added != 1 {
t.Fatalf("added = %d, want 1", stats.Added)
}
fileID := encodeRel("collection/clip.mp4")
got, err := cat.GetVideo(ctx, Kind+"-local-"+fileID)
if err != nil {
t.Fatalf("get video: %v", err)
}
if got.DriveID != "local" || got.FileID != fileID || got.Category != "collection" {
t.Fatalf("video = %#v, want local drive video in collection", got)
}
}
+321 -22
View File
@@ -3,14 +3,19 @@ package onedrive
import (
"bytes"
"context"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"sync"
"time"
"github.com/go-resty/resty/v2"
@@ -18,8 +23,17 @@ import (
)
const (
maxSmallUploadSize = 250 * 1024 * 1024
defaultRenewAPIURL = "https://api.oplist.org/onedrive/renewapi"
maxSmallUploadSize = 250 * 1024 * 1024
defaultUploadSessionChunk = 10 * 1024 * 1024
uploadSessionRetryAttempts = 3
defaultRenewAPIURL = "https://api.oplist.org/onedrive/renewapi"
onedriveListCooldown = 5 * time.Minute
onedriveListInterval = 1 * time.Second
)
var (
smallUploadThreshold = int64(maxSmallUploadSize)
uploadSessionChunk = int64(defaultUploadSessionChunk)
)
type Driver struct {
@@ -34,6 +48,11 @@ type Driver struct {
renewAPIURL string
client *resty.Client
onTokenUpdate func(access, refresh string)
listMu sync.Mutex
lastListAt time.Time
listInterval time.Duration
listCooldown time.Duration
}
type Config struct {
@@ -85,6 +104,8 @@ func New(c Config) *Driver {
client: resty.New().
SetTimeout(30*time.Second).
SetHeader("Accept", "application/json, text/plain, */*"),
listInterval: onedriveListInterval,
listCooldown: onedriveListCooldown,
}
}
@@ -106,10 +127,16 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
if dirID == "" {
dirID = d.rootID
}
d.listMu.Lock()
defer d.listMu.Unlock()
nextLink := d.childrenURL(dirID)
first := true
out := make([]drives.Entry, 0)
for nextLink != "" {
if err := d.waitForListSlotLocked(ctx); err != nil {
return nil, err
}
var resp filesResp
err := d.request(ctx, nextLink, http.MethodGet, func(req *resty.Request) {
if first {
@@ -120,6 +147,19 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
}
}, &resp)
if err != nil {
if wait, ok := drives.RateLimitRetryAfter(err); ok {
if wait <= 0 {
wait = d.listCooldown
if wait <= 0 {
wait = onedriveListCooldown
}
}
log.Printf("[onedrive] list cooling down drive=%s dir=%s cooldown=%s err=%v", d.id, dirID, wait, err)
if err := sleepContext(ctx, wait); err != nil {
return nil, err
}
continue
}
return nil, fmt.Errorf("onedrive list: %w", err)
}
for _, item := range resp.Value {
@@ -131,6 +171,36 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
return out, nil
}
func (d *Driver) waitForListSlotLocked(ctx context.Context) error {
if d.listInterval <= 0 || d.lastListAt.IsZero() {
d.lastListAt = time.Now()
return ctx.Err()
}
next := d.lastListAt.Add(d.listInterval)
now := time.Now()
if now.Before(next) {
if err := sleepContext(ctx, next.Sub(now)); err != nil {
return err
}
}
d.lastListAt = time.Now()
return ctx.Err()
}
func sleepContext(ctx context.Context, d time.Duration) error {
if d <= 0 {
return ctx.Err()
}
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
var item graphItem
if err := d.request(ctx, d.itemURL(fileID), http.MethodGet, nil, &item); err != nil {
@@ -156,15 +226,49 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
}
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
res, err := d.UploadAndReportHash(ctx, parentID, name, r, size)
if err != nil {
return "", err
}
return res.FileID, nil
}
func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
parentID, name, err := d.normalizeUploadArgs(parentID, name, r, size)
if err != nil {
return UploadResult{}, err
}
threshold := smallUploadThreshold
if threshold <= 0 {
threshold = maxSmallUploadSize
}
if size <= threshold {
return d.uploadSmallAndReportHash(ctx, parentID, name, r, size, threshold)
}
return d.uploadSessionAndReportHash(ctx, parentID, name, r, size)
}
func (d *Driver) normalizeUploadArgs(parentID, name string, r io.Reader, size int64) (string, string, error) {
if r == nil {
return "", "", errors.New("onedrive upload: body is required")
}
if size < 0 {
return "", "", fmt.Errorf("onedrive upload: invalid size %d", size)
}
if parentID == "" {
parentID = d.rootID
}
if size > maxSmallUploadSize {
return "", fmt.Errorf("onedrive upload: files over %d bytes require upload session", maxSmallUploadSize)
name = strings.TrimSpace(name)
if name == "" {
return "", "", errors.New("onedrive upload: empty file name")
}
data, err := readSmallUpload(r)
return parentID, name, nil
}
func (d *Driver) uploadSmallAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size, limit int64) (UploadResult, error) {
data, hash, actualSize, err := readSmallUpload(r, size, limit)
if err != nil {
return "", err
return UploadResult{}, err
}
u := fmt.Sprintf("%s/items/%s:/%s:/content", d.driveBaseURL(), url.PathEscape(parentID), url.PathEscape(name))
var item graphItem
@@ -173,26 +277,159 @@ func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader,
req.SetContentLength(true)
}, &item)
if err != nil {
return "", fmt.Errorf("onedrive upload: %w", err)
return UploadResult{}, fmt.Errorf("onedrive upload: %w", err)
}
if item.ID == "" {
return "", errors.New("onedrive upload: empty item id")
return UploadResult{}, errors.New("onedrive upload: empty item id")
}
return item.ID, nil
return UploadResult{FileID: item.ID, Hash: hash, Size: actualSize}, nil
}
func readSmallUpload(r io.Reader) ([]byte, error) {
if r == nil {
return nil, errors.New("onedrive upload: body is required")
}
data, err := io.ReadAll(io.LimitReader(r, maxSmallUploadSize+1))
func (d *Driver) uploadSessionAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
session, err := d.createUploadSession(ctx, parentID, name)
if err != nil {
return nil, fmt.Errorf("onedrive upload: read body: %w", err)
return UploadResult{}, err
}
if len(data) > maxSmallUploadSize {
return nil, fmt.Errorf("onedrive upload: files over %d bytes require upload session", maxSmallUploadSize)
if strings.TrimSpace(session.UploadURL) == "" {
return UploadResult{}, errors.New("onedrive upload session: empty upload url")
}
return data, nil
chunkSize := uploadSessionChunk
if chunkSize <= 0 {
chunkSize = defaultUploadSessionChunk
}
buf := make([]byte, int(chunkSize))
hasher := sha1.New()
var finalItem graphItem
var offset int64
for offset < size {
partSize := minInt64(chunkSize, size-offset)
chunk := buf[:int(partSize)]
n, err := io.ReadFull(r, chunk)
if err != nil {
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
return UploadResult{}, fmt.Errorf("onedrive upload: size mismatch: declared %d, copied %d", size, offset+int64(n))
}
return UploadResult{}, fmt.Errorf("onedrive upload: read body: %w", err)
}
chunk = chunk[:n]
_, _ = hasher.Write(chunk)
item, err := d.putUploadSessionChunkWithRetry(ctx, session.UploadURL, offset, size, chunk)
if err != nil {
return UploadResult{}, err
}
if item != nil {
finalItem = *item
}
offset += int64(n)
}
if finalItem.ID == "" {
return UploadResult{}, errors.New("onedrive upload session: empty item id")
}
return UploadResult{
FileID: finalItem.ID,
Hash: hex.EncodeToString(hasher.Sum(nil)),
Size: offset,
}, nil
}
func (d *Driver) createUploadSession(ctx context.Context, parentID, name string) (uploadSessionResp, error) {
u := fmt.Sprintf("%s/items/%s:/%s:/createUploadSession", d.driveBaseURL(), url.PathEscape(parentID), url.PathEscape(name))
body := map[string]any{
"item": map[string]any{
"@microsoft.graph.conflictBehavior": "rename",
},
}
var out uploadSessionResp
err := d.request(ctx, u, http.MethodPost, func(req *resty.Request) {
req.SetBody(body)
}, &out)
if err != nil {
return uploadSessionResp{}, fmt.Errorf("onedrive upload session: %w", err)
}
return out, nil
}
func (d *Driver) putUploadSessionChunkWithRetry(ctx context.Context, uploadURL string, start, total int64, data []byte) (*graphItem, error) {
var last error
for attempt := 0; attempt < uploadSessionRetryAttempts; attempt++ {
if attempt > 0 {
if err := sleepContext(ctx, time.Duration(attempt)*time.Second); err != nil {
return nil, err
}
}
item, retryable, err := d.putUploadSessionChunk(ctx, uploadURL, start, total, data)
if err == nil {
return item, nil
}
last = err
if !retryable {
return nil, err
}
}
if last == nil {
last = errors.New("onedrive upload session: retry attempts exhausted")
}
return nil, last
}
func (d *Driver) putUploadSessionChunk(ctx context.Context, uploadURL string, start, total int64, data []byte) (*graphItem, bool, error) {
end := start + int64(len(data)) - 1
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(data))
if err != nil {
return nil, false, err
}
req.ContentLength = int64(len(data))
req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, total))
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, true, err
}
defer res.Body.Close()
switch res.StatusCode {
case http.StatusOK, http.StatusCreated:
var item graphItem
if err := json.NewDecoder(res.Body).Decode(&item); err != nil {
return nil, false, fmt.Errorf("onedrive upload session: decode completed item: %w", err)
}
return &item, false, nil
case http.StatusAccepted:
return nil, false, nil
default:
body, _ := io.ReadAll(io.LimitReader(res.Body, 4096))
err := fmt.Errorf("onedrive upload session: status=%d body=%s", res.StatusCode, strings.TrimSpace(string(body)))
retryable := res.StatusCode == http.StatusTooManyRequests || (res.StatusCode >= 500 && res.StatusCode <= 504)
return nil, retryable, err
}
}
func readSmallUpload(r io.Reader, declaredSize, limit int64) ([]byte, string, int64, error) {
if r == nil {
return nil, "", 0, errors.New("onedrive upload: body is required")
}
if limit <= 0 {
limit = maxSmallUploadSize
}
data, err := io.ReadAll(io.LimitReader(r, limit+1))
if err != nil {
return nil, "", 0, fmt.Errorf("onedrive upload: read body: %w", err)
}
if int64(len(data)) > limit {
return nil, "", 0, fmt.Errorf("onedrive upload: files over %d bytes require upload session", limit)
}
if declaredSize >= 0 && int64(len(data)) != declaredSize {
return nil, "", 0, fmt.Errorf("onedrive upload: size mismatch: declared %d, copied %d", declaredSize, len(data))
}
sum := sha1.Sum(data)
return data, hex.EncodeToString(sum[:]), int64(len(data)), nil
}
func minInt64(a, b int64) int64 {
if a < b {
return a
}
return b
}
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
@@ -245,6 +482,25 @@ func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, er
return item.ID, nil
}
func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
fileID = strings.TrimSpace(fileID)
if fileID == "" {
return errors.New("onedrive rename: empty file id")
}
newName = strings.TrimSpace(newName)
if newName == "" {
return errors.New("onedrive rename: empty new name")
}
var item graphItem
err := d.request(ctx, d.itemURL(fileID), http.MethodPatch, func(req *resty.Request) {
req.SetBody(map[string]string{"name": newName})
}, &item)
if err != nil {
return fmt.Errorf("onedrive rename: %w", err)
}
return nil
}
func (d *Driver) request(ctx context.Context, rawURL, method string, configure func(*resty.Request), out any) error {
return d.requestOnce(ctx, rawURL, method, configure, out, true)
}
@@ -265,7 +521,7 @@ func (d *Driver) requestOnce(ctx context.Context, rawURL, method string, configu
if err != nil {
return err
}
if isRateLimitResponse(res, graphErr.Error.Code) {
if isRateLimitResponse(res, graphErr.Error.Code, graphErr.Error.Message) {
return onedriveRateLimitError(res, graphErr.Error.Message)
}
if graphErr.Error.Code != "" {
@@ -327,11 +583,54 @@ func (d *Driver) refresh(ctx context.Context) error {
return nil
}
func isRateLimitResponse(res *resty.Response, code string) bool {
if code == "TooManyRequests" || code == "activityLimitReached" {
func isRateLimitResponse(res *resty.Response, code, message string) bool {
if isRateLimitCode(code) || isRateLimitMessage(message) {
return true
}
return res != nil && res.StatusCode() == http.StatusTooManyRequests
if res == nil {
return false
}
if res.StatusCode() == http.StatusTooManyRequests {
return true
}
if res.Header().Get("Retry-After") == "" {
return false
}
switch res.StatusCode() {
case http.StatusServiceUnavailable, http.StatusGatewayTimeout:
return true
default:
return false
}
}
func isRateLimitCode(code string) bool {
normalized := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(code), "_", ""))
normalized = strings.ReplaceAll(normalized, "-", "")
switch normalized {
case "toomanyrequests",
"activitylimitreached",
"throttledrequest",
"requestthrottled",
"resourcethrottled",
"applicationthrottled",
"tenantthrottled":
return true
default:
return false
}
}
func isRateLimitMessage(message string) bool {
text := strings.ToLower(strings.TrimSpace(message))
if text == "" {
return false
}
return strings.Contains(text, "too many requests") ||
strings.Contains(text, "throttl") ||
strings.Contains(text, "rate limit") ||
strings.Contains(text, "activity limit") ||
strings.Contains(text, "temporarily blocked")
}
func onedriveRateLimitError(res *resty.Response, message string) error {
+199 -1
View File
@@ -2,6 +2,8 @@ package onedrive
import (
"context"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"errors"
"io"
@@ -199,7 +201,7 @@ func TestGraph429ReturnsRateLimitErrorWithRetryAfter(t *testing.T) {
APIBaseURL: srv.URL,
})
_, err := d.List(context.Background(), "root")
_, err := d.StreamURL(context.Background(), "file-id")
if err == nil {
t.Fatal("list succeeded, want rate limit error")
}
@@ -212,6 +214,92 @@ func TestGraph429ReturnsRateLimitErrorWithRetryAfter(t *testing.T) {
}
}
func TestGraphThrottleMessageReturnsRateLimitError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusForbidden)
if err := json.NewEncoder(w).Encode(map[string]any{
"error": map[string]any{
"code": "generalException",
"message": "The request has been throttled. Please try again later.",
},
}); err != nil {
t.Fatalf("write json: %v", err)
}
}))
defer srv.Close()
d := New(Config{
ID: "od-main",
AccessToken: "access-token",
RefreshToken: "refresh-token",
APIBaseURL: srv.URL,
})
_, err := d.StreamURL(context.Background(), "file-id")
if err == nil {
t.Fatal("list succeeded, want rate limit error")
}
var rateLimit *drives.RateLimitError
if !errors.As(err, &rateLimit) {
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
}
}
func TestListCoolsDownAndRetriesOneDriveRateLimit(t *testing.T) {
var calls int
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1.0/me/drive/items/root/children" {
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
}
calls++
if calls == 1 {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusTooManyRequests)
if err := json.NewEncoder(w).Encode(map[string]any{
"error": map[string]any{
"code": "TooManyRequests",
"message": "throttled",
},
}); err != nil {
t.Fatalf("write json: %v", err)
}
return
}
writeJSON(t, w, map[string]any{
"value": []map[string]any{
{
"id": "file-id",
"name": "demo.mp4",
"size": 100,
"file": map[string]any{"mimeType": "video/mp4"},
},
},
})
}))
defer srv.Close()
d := New(Config{
ID: "od-main",
AccessToken: "access-token",
RefreshToken: "refresh-token",
APIBaseURL: srv.URL,
})
d.listInterval = 0
d.listCooldown = time.Millisecond
got, err := d.List(context.Background(), "root")
if err != nil {
t.Fatalf("list: %v", err)
}
if calls != 2 {
t.Fatalf("calls = %d, want retry after rate limit", calls)
}
if len(got) != 1 || got[0].ID != "file-id" {
t.Fatalf("entries = %#v, want retried file", got)
}
}
func TestStatAndStreamURLUseDriveItemMetadata(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
@@ -320,6 +408,36 @@ func TestEnsureDirCreatesMissingFolders(t *testing.T) {
}
}
func TestRenamePatchesDriveItemName(t *testing.T) {
var body map[string]string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPatch || r.URL.EscapedPath() != "/v1.0/me/drive/items/file-id" {
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
}
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
t.Fatalf("authorization = %q, want bearer token", got)
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatalf("decode body: %v", err)
}
writeJSON(t, w, map[string]any{"id": "file-id", "name": "new name.mp4"})
}))
defer srv.Close()
d := New(Config{
ID: "od-main",
AccessToken: "access-token",
RefreshToken: "refresh-token",
APIBaseURL: srv.URL,
})
if err := d.Rename(context.Background(), "file-id", "new name.mp4"); err != nil {
t.Fatalf("rename: %v", err)
}
if body["name"] != "new name.mp4" {
t.Fatalf("rename body = %#v, want new name", body)
}
}
func TestUploadSmallFileReturnsNewItemID(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
@@ -358,6 +476,86 @@ func TestUploadSmallFileReturnsNewItemID(t *testing.T) {
}
}
func TestUploadLargeFileUsesUploadSessionAndReportsHash(t *testing.T) {
oldThreshold := smallUploadThreshold
oldChunk := uploadSessionChunk
smallUploadThreshold = 8
uploadSessionChunk = 4
t.Cleanup(func() {
smallUploadThreshold = oldThreshold
uploadSessionChunk = oldChunk
})
body := "0123456789abc"
var ranges []string
var chunks []string
var createdSession bool
var srv *httptest.Server
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodPost && r.URL.EscapedPath() == "/v1.0/me/drive/items/parent-id:/big.mp4:/createUploadSession":
createdSession = true
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
t.Fatalf("authorization = %q, want bearer token", got)
}
writeJSON(t, w, map[string]any{"uploadUrl": srv.URL + "/upload-session"})
case r.Method == http.MethodPut && r.URL.Path == "/upload-session":
ranges = append(ranges, r.Header.Get("Content-Range"))
data, err := io.ReadAll(r.Body)
if err != nil {
t.Fatalf("read chunk: %v", err)
}
chunks = append(chunks, string(data))
if len(ranges) < 4 {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusAccepted)
if _, err := w.Write([]byte(`{"nextExpectedRanges":["0-"]}`)); err != nil {
t.Fatalf("write accepted: %v", err)
}
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusCreated)
if err := json.NewEncoder(w).Encode(map[string]any{"id": "uploaded-big-id"}); err != nil {
t.Fatalf("write final item: %v", err)
}
default:
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
}
}))
defer srv.Close()
d := New(Config{
ID: "od-main",
AccessToken: "access-token",
RefreshToken: "refresh-token",
APIBaseURL: srv.URL,
})
got, err := d.UploadAndReportHash(context.Background(), "parent-id", "big.mp4", strings.NewReader(body), int64(len(body)))
if err != nil {
t.Fatalf("upload: %v", err)
}
if !createdSession {
t.Fatal("createUploadSession was not called")
}
wantRanges := []string{
"bytes 0-3/13",
"bytes 4-7/13",
"bytes 8-11/13",
"bytes 12-12/13",
}
if strings.Join(ranges, "|") != strings.Join(wantRanges, "|") {
t.Fatalf("ranges = %#v, want %#v", ranges, wantRanges)
}
if strings.Join(chunks, "") != body {
t.Fatalf("uploaded chunks = %q, want %q", strings.Join(chunks, ""), body)
}
sum := sha1.Sum([]byte(body))
if got.FileID != "uploaded-big-id" || got.Size != int64(len(body)) || got.Hash != hex.EncodeToString(sum[:]) {
t.Fatalf("upload result = %#v, want file id/hash/size for body", got)
}
}
func TestUploadRefreshesExpiredTokenAndReplaysBody(t *testing.T) {
var uploadAttempts int
var tokenRefreshes int
+10
View File
@@ -82,3 +82,13 @@ type filesResp struct {
Value []graphItem `json:"value"`
NextLink string `json:"@odata.nextLink"`
}
type UploadResult struct {
FileID string
Hash string
Size int64
}
type uploadSessionResp struct {
UploadURL string `json:"uploadUrl"`
}
+3 -4
View File
@@ -199,9 +199,8 @@ func (d *Driver) refreshCaptchaToken(ctx context.Context, action string, meta ma
// refreshCaptchaTokenOnce 调 /v1/shield/captcha/init 申请新 captcha token。
//
// 如果 retry=true 且服务端返回 4002captcha_token expired,意味着 body 里
// 携带的 d.captchaToken 已经过期),就清空缓存的 captcha_token 后再调一次;
// 这次 body 里 captcha_token 为空,服务端会直接发一个新的。这覆盖
// 如果 retry=true 且服务端返回 captcha 失效错误(4002 或 9),就清空缓存的
// captcha_token 后再调一次;这次 body 里 captcha_token 为空,服务端会直接发一个新的。这覆盖
// driver 重启后 Init() 用持久化的旧 captcha_token 调 captcha init 失败的
// 场景。
func (d *Driver) refreshCaptchaTokenOnce(ctx context.Context, action string, meta map[string]string, retry bool) error {
@@ -230,7 +229,7 @@ func (d *Driver) refreshCaptchaTokenOnce(ctx context.Context, action string, met
return err
}
if e.isError() {
if retry && e.ErrorCode == 4002 && d.captchaToken != "" {
if retry && isCaptchaTokenRejectedCode(e.ErrorCode) && d.captchaToken != "" {
d.captchaToken = ""
return d.refreshCaptchaTokenOnce(ctx, action, meta, false)
}
@@ -96,6 +96,65 @@ func TestRefreshCaptchaTokenRecoversFrom4002(t *testing.T) {
}
}
// TestRefreshCaptchaTokenRecoversFrom9 覆盖 PikPak 返回 error_code=9
// captcha_invalid 的路径。这个错误和 4002 一样表示当前 captcha_token 已被拒绝;
// 重试 captcha/init 前必须先清空旧 token,否则服务端会继续拒绝。
func TestRefreshCaptchaTokenRecoversFrom9(t *testing.T) {
var calls int32
type bodyShape struct {
CaptchaToken string `json:"captcha_token"`
}
var (
firstBody bodyShape
secondBody bodyShape
)
mux := http.NewServeMux()
mux.HandleFunc("/v1/shield/captcha/init", func(w http.ResponseWriter, r *http.Request) {
n := atomic.AddInt32(&calls, 1)
switch n {
case 1:
_ = json.NewDecoder(r.Body).Decode(&firstBody)
writeErrorJSON(w, `{
"error_code": 9,
"error": "captcha_invalid",
"error_description": "Verification code is invalid"
}`)
case 2:
_ = json.NewDecoder(r.Body).Decode(&secondBody)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"captcha_token": "fresh-captcha",
"expires_in": 300
}`))
default:
t.Errorf("unexpected captcha init call #%d", n)
}
})
server := httptest.NewServer(mux)
defer server.Close()
d := newTestDriver(t, server)
d.captchaToken = "expired-captcha"
if err := d.refreshCaptchaTokenAtLogin(context.Background(), "GET:/drive/v1/files", "user-1"); err != nil {
t.Fatalf("refreshCaptchaTokenAtLogin: %v", err)
}
if got := atomic.LoadInt32(&calls); got != 2 {
t.Fatalf("captcha init called %d times, want 2", got)
}
if firstBody.CaptchaToken != "expired-captcha" {
t.Errorf("first body captcha_token = %q, want \"expired-captcha\"", firstBody.CaptchaToken)
}
if secondBody.CaptchaToken != "" {
t.Errorf("second body captcha_token = %q, want empty (cleared after error_code=9)", secondBody.CaptchaToken)
}
if d.captchaToken != "fresh-captcha" {
t.Errorf("d.captchaToken = %q, want \"fresh-captcha\"", d.captchaToken)
}
}
// TestRefreshCaptchaTokenDoesNotLoopOn4002WithEmptyToken 防止退化成无限重试:
// 如果调用方一开始 captchaToken 就是空,又遇上 4002,不应该再清空一次重试
// (清空后还是空,再发会拿到同样的错误),应该直接返回错误让上层处理。
@@ -121,6 +180,141 @@ func TestRefreshCaptchaTokenDoesNotLoopOn4002WithEmptyToken(t *testing.T) {
}
}
func TestInitWithRefreshTokenDoesNotSendPersistedCaptchaToken(t *testing.T) {
var captchaCalls int32
var captchaBody struct {
CaptchaToken string `json:"captcha_token"`
}
var persisted struct {
access, refresh, captcha string
calls int
}
mux := http.NewServeMux()
mux.HandleFunc("/v1/auth/token", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"access_token": "fresh-access",
"refresh_token": "fresh-refresh",
"sub": "user-1"
}`))
})
mux.HandleFunc("/v1/shield/captcha/init", func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&captchaCalls, 1)
_ = json.NewDecoder(r.Body).Decode(&captchaBody)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"captcha_token": "fresh-captcha",
"expires_in": 300
}`))
})
server := httptest.NewServer(mux)
defer server.Close()
d := newTestDriver(t, server)
d.captchaToken = "persisted-stale-captcha"
d.onTokenUpdate = func(access, refresh, captcha, deviceID string) {
persisted.access = access
persisted.refresh = refresh
persisted.captcha = captcha
persisted.calls++
}
if err := d.Init(context.Background()); err != nil {
t.Fatalf("Init: %v", err)
}
if got := atomic.LoadInt32(&captchaCalls); got != 1 {
t.Fatalf("captcha init calls = %d, want 1", got)
}
if captchaBody.CaptchaToken != "" {
t.Errorf("captcha init body captcha_token = %q, want empty", captchaBody.CaptchaToken)
}
if d.captchaToken != "fresh-captcha" {
t.Errorf("d.captchaToken = %q, want \"fresh-captcha\"", d.captchaToken)
}
if persisted.access != "fresh-access" || persisted.refresh != "fresh-refresh" || persisted.captcha != "fresh-captcha" {
t.Errorf("persisted tokens = (%q, %q, %q), want fresh values", persisted.access, persisted.refresh, persisted.captcha)
}
if persisted.calls < 2 {
t.Errorf("persist callback calls = %d, want at least 2 (clear stale + persist fresh)", persisted.calls)
}
}
func TestInitFallsBackToLoginWhenRefreshReturnsCaptchaInvalid(t *testing.T) {
var (
tokenCalls int32
captchaCalls int32
signinCalls int32
)
var signinBody struct {
CaptchaToken string `json:"captcha_token"`
}
mux := http.NewServeMux()
mux.HandleFunc("/v1/auth/token", func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&tokenCalls, 1)
writeErrorJSON(w, `{
"error_code": 4002,
"error": "captcha_invalid",
"error_description": "Code(4002) - captcha_token expired"
}`)
})
mux.HandleFunc("/v1/shield/captcha/init", func(w http.ResponseWriter, r *http.Request) {
n := atomic.AddInt32(&captchaCalls, 1)
w.Header().Set("Content-Type", "application/json")
switch n {
case 1:
_, _ = w.Write([]byte(`{
"captcha_token": "login-captcha",
"expires_in": 300
}`))
case 2:
_, _ = w.Write([]byte(`{
"captcha_token": "files-captcha",
"expires_in": 300
}`))
default:
t.Errorf("unexpected captcha init call #%d", n)
}
})
mux.HandleFunc("/v1/auth/signin", func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&signinCalls, 1)
_ = json.NewDecoder(r.Body).Decode(&signinBody)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"access_token": "login-access",
"refresh_token": "login-refresh",
"sub": "user-1"
}`))
})
server := httptest.NewServer(mux)
defer server.Close()
d := newTestDriver(t, server)
d.captchaToken = "persisted-stale-captcha"
if err := d.Init(context.Background()); err != nil {
t.Fatalf("Init: %v", err)
}
if got := atomic.LoadInt32(&tokenCalls); got != 1 {
t.Fatalf("token refresh calls = %d, want 1", got)
}
if got := atomic.LoadInt32(&signinCalls); got != 1 {
t.Fatalf("signin calls = %d, want 1", got)
}
if got := atomic.LoadInt32(&captchaCalls); got != 2 {
t.Fatalf("captcha init calls = %d, want 2 (login + post-login files action)", got)
}
if signinBody.CaptchaToken != "login-captcha" {
t.Errorf("signin captcha_token = %q, want \"login-captcha\"", signinBody.CaptchaToken)
}
if d.accessToken != "login-access" || d.refreshToken != "login-refresh" || d.captchaToken != "files-captcha" {
t.Errorf("driver tokens = (%q, %q, %q), want login/files tokens", d.accessToken, d.refreshToken, d.captchaToken)
}
}
// TestRequestOnceRecoversFrom4002OnAPICall 验证一个普通 API 调用收到 4002
// 时,requestOnce 会先清空 captchaToken、再走 captcha 刷新,最后用新 token
// 重试请求,最终成功返回。
@@ -196,6 +390,76 @@ func TestRequestOnceRecoversFrom4002OnAPICall(t *testing.T) {
}
}
// TestRequestOnceRecoversFrom9OnAPICall 验证普通 API 调用收到 error_code=9
// 时,会先清空旧 captchaToken,再刷新 captcha 并重试原请求。
func TestRequestOnceRecoversFrom9OnAPICall(t *testing.T) {
var (
filesCalls int32
captchaCalls int32
)
type capturedFiles struct {
captchaHeader string
}
var firstFiles, secondFiles capturedFiles
mux := http.NewServeMux()
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
n := atomic.AddInt32(&filesCalls, 1)
switch n {
case 1:
firstFiles.captchaHeader = r.Header.Get("X-Captcha-Token")
writeErrorJSON(w, `{
"error_code": 9,
"error": "captcha_invalid",
"error_description": "Verification code is invalid"
}`)
case 2:
secondFiles.captchaHeader = r.Header.Get("X-Captcha-Token")
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"files": [], "next_page_token": ""}`))
default:
t.Errorf("unexpected /drive/v1/files call #%d", n)
}
})
mux.HandleFunc("/v1/shield/captcha/init", func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&captchaCalls, 1)
var body struct {
CaptchaToken string `json:"captcha_token"`
}
_ = json.NewDecoder(r.Body).Decode(&body)
if body.CaptchaToken != "" {
t.Errorf("captcha init body captcha_token = %q, want empty (error_code=9 path should clear cache)", body.CaptchaToken)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"captcha_token": "fresh-captcha", "expires_in": 300}`))
})
server := httptest.NewServer(mux)
defer server.Close()
d := newTestDriver(t, server)
d.captchaToken = "expired-captcha"
if _, err := d.List(context.Background(), "any-parent"); err != nil {
t.Fatalf("List: %v", err)
}
if got := atomic.LoadInt32(&filesCalls); got != 2 {
t.Fatalf("/drive/v1/files calls = %d, want 2 (initial + retry)", got)
}
if got := atomic.LoadInt32(&captchaCalls); got != 1 {
t.Fatalf("captcha init calls = %d, want 1", got)
}
if firstFiles.captchaHeader != "expired-captcha" {
t.Errorf("first request X-Captcha-Token = %q, want \"expired-captcha\"", firstFiles.captchaHeader)
}
if secondFiles.captchaHeader != "fresh-captcha" {
t.Errorf("retry X-Captcha-Token = %q, want \"fresh-captcha\"", secondFiles.captchaHeader)
}
if d.captchaToken != "fresh-captcha" {
t.Errorf("d.captchaToken after recovery = %q, want \"fresh-captcha\"", d.captchaToken)
}
}
// TestRequestOnceDoesNotRetryTwiceOn4002 验证 4002 恢复路径只重试一次;
// 如果重试请求依然失败(哪怕是再来一个 4002),也不会再次进入恢复逻辑,
// 而是把错误返回出去,避免无限循环。
+79 -6
View File
@@ -121,9 +121,28 @@ func (d *Driver) ID() string { return d.id }
func (d *Driver) RootID() string { return d.rootID }
func (d *Driver) Init(ctx context.Context) error {
clearPersistedCaptcha := func() {
if d.captchaToken == "" {
return
}
d.captchaToken = ""
d.persistTokens()
}
if d.refreshToken != "" {
if err := d.refresh(ctx, d.refreshToken); err != nil {
return err
if !IsCaptchaError(err) || d.username == "" || d.password == "" {
return err
}
clearPersistedCaptcha()
if err := d.login(ctx); err != nil {
return fmt.Errorf("pikpak refresh captcha recovery login: %w", err)
}
} else {
// Persisted captcha tokens are short-lived. With a refresh token we can
// safely request a fresh captcha token after auth, and avoiding the
// stored value prevents known-stale tokens from poisoning startup.
clearPersistedCaptcha()
}
} else {
if err := d.login(ctx); err != nil {
@@ -336,7 +355,60 @@ func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
}
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
return "", drives.ErrNotSupported
currentID := d.rootID
for _, name := range splitPath(pathFromRoot) {
childID, err := d.findChildDir(ctx, currentID, name)
if err != nil {
return "", err
}
if childID == "" {
childID, err = d.makeDir(ctx, currentID, name)
if err != nil {
return "", err
}
}
currentID = childID
}
return currentID, nil
}
func (d *Driver) findChildDir(ctx context.Context, parentID, name string) (string, error) {
entries, err := d.List(ctx, parentID)
if err != nil {
return "", err
}
for _, e := range entries {
if e.IsDir && e.Name == name {
return e.ID, nil
}
}
return "", nil
}
func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, error) {
var out file
err := d.request(ctx, filesURL, http.MethodPost, func(req *resty.Request) {
req.SetBody(map[string]any{
"kind": "drive#folder",
"parent_id": parentID,
"name": name,
})
}, &out)
if err != nil {
return "", fmt.Errorf("pikpak mkdir %s: %w", name, err)
}
if out.ID == "" {
return "", fmt.Errorf("pikpak mkdir %s: empty folder id", name)
}
return out.ID, nil
}
func splitPath(p string) []string {
p = strings.Trim(p, "/")
if p == "" {
return nil
}
return strings.Split(p, "/")
}
func (d *Driver) getFiles(ctx context.Context, parentID string) ([]file, error) {
@@ -408,14 +480,15 @@ func (d *Driver) requestOnce(ctx context.Context, url, method string, configure
// serialized. Once we hold the lock, if d.captchaToken has
// already moved past staleToken, another goroutine has refreshed
// it for us — we skip the refresh and just retry. Otherwise we
// clear the cached token (4002 means "the value in the body is
// expired"; sending it again will keep returning 4002) and ask
// /v1/shield/captcha/init for a fresh one.
// clear the cached token before asking /v1/shield/captcha/init
// for a fresh one. PikPak may report stale captcha as either
// 4002 or 9, and sending the rejected token into captcha init can
// keep returning captcha_invalid.
staleToken := d.captchaToken
d.captchaMu.Lock()
var refreshErr error
if d.captchaToken == staleToken {
if e.ErrorCode == 4002 {
if d.captchaToken != "" {
d.captchaToken = ""
}
refreshErr = d.refreshCaptchaTokenAtLogin(ctx, getAction(method, url), d.userID)
+83 -7
View File
@@ -1,10 +1,12 @@
package pikpak
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/video-site/backend/internal/drives"
)
func TestNewDefaults(t *testing.T) {
@@ -95,11 +97,85 @@ func TestFolderToEntry(t *testing.T) {
}
}
func TestEnsureDirStillUnsupported(t *testing.T) {
d := New(Config{ID: "pikpak-main"})
func TestEnsureDirReusesExistingFolder(t *testing.T) {
var postCalled bool
mux := http.NewServeMux()
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
if got := r.URL.Query().Get("parent_id"); got != "root-id" {
t.Fatalf("parent_id = %q, want root-id", got)
}
writePikPakJSON(t, w, map[string]any{
"files": []map[string]any{{
"id": "existing-folder-id",
"kind": "drive#folder",
"name": "91 Spider",
}},
})
case http.MethodPost:
postCalled = true
t.Fatalf("existing folder should not be created again")
default:
t.Fatalf("unexpected method %s", r.Method)
}
})
srv := httptest.NewServer(mux)
defer srv.Close()
if _, err := d.EnsureDir(nil, "/previews"); err != drives.ErrNotSupported {
t.Fatalf("EnsureDir error = %v, want ErrNotSupported", err)
d := newTestDriver(t, srv)
got, err := d.EnsureDir(context.Background(), "91 Spider")
if err != nil {
t.Fatalf("ensure dir: %v", err)
}
if got != "existing-folder-id" {
t.Fatalf("dir id = %q, want existing-folder-id", got)
}
if postCalled {
t.Fatal("POST should not be called")
}
}
func TestEnsureDirCreatesMissingFolder(t *testing.T) {
var got uploadRequestBody
mux := http.NewServeMux()
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
writePikPakJSON(t, w, map[string]any{"files": []map[string]any{}})
case http.MethodPost:
if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
t.Fatalf("decode create folder body: %v", err)
}
writePikPakJSON(t, w, map[string]any{
"id": "new-folder-id",
"kind": "drive#folder",
"name": "91 Spider",
})
default:
t.Fatalf("unexpected method %s", r.Method)
}
})
srv := httptest.NewServer(mux)
defer srv.Close()
d := newTestDriver(t, srv)
id, err := d.EnsureDir(context.Background(), "91 Spider")
if err != nil {
t.Fatalf("ensure dir: %v", err)
}
if id != "new-folder-id" {
t.Fatalf("dir id = %q, want new-folder-id", id)
}
if got.Kind != "drive#folder" || got.ParentID != "root-id" || got.Name != "91 Spider" {
t.Fatalf("create folder body = %#v", got)
}
}
func writePikPakJSON(t *testing.T, w http.ResponseWriter, body any) {
t.Helper()
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(body); err != nil {
t.Fatalf("write json: %v", err)
}
// Upload 的真实实现见 upload_test.go。
}
+5 -1
View File
@@ -59,6 +59,10 @@ func (e *errResp) Error() string {
return fmt.Sprintf("pikpak error_code=%d error=%s description=%s", e.ErrorCode, e.ErrorMsg, e.ErrorDescription)
}
func isCaptchaTokenRejectedCode(code int64) bool {
return code == 9 || code == 4002
}
// APIError is the public alias for the PikPak API error response. Callers
// outside this package (e.g. the spider91→PikPak migrator, tests) can either
// construct it for fakes or unwrap it via errors.As. Prefer IsCaptchaError
@@ -76,7 +80,7 @@ func IsCaptchaError(err error) bool {
}
var e *errResp
if errors.As(err, &e) {
return e != nil && (e.ErrorCode == 4002 || e.ErrorCode == 9)
return e != nil && isCaptchaTokenRejectedCode(e.ErrorCode)
}
return false
}
+1 -2
View File
@@ -463,8 +463,7 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
// 网站封面下载失败的视频:spider91 drive 的 thumb worker 按设计不
// 处理 spider91 视频(封面应是网站原图直接保存),所以没人接手。
// 显式标 'failed' 让 CountVideosNeedingThumbnail 排除(条件 status
// != 'failed'),否则 enqueueDriveGeneration → waitForThumbnailsBeforePreview
// 会因为 count > 0 把 teaser 入队永远卡在等待循环里。
// != 'failed'),避免后续封面补队列一直重复捞到这条视频。
_ = c.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
ThumbnailStatus: "failed",
})
@@ -234,13 +234,13 @@ func TestCrawlerRunOnceMissingScript(t *testing.T) {
}
// TestCrawlerThumbDownloadFailureMarksStatusFailed 验证:网站封面下载失败时
// crawler 把 thumbnail_status 显式标 'failed',避免 enqueueDriveGeneration 的
// waitForThumbnailsBeforePreview 因为 count > 0 把 teaser 卡死等待
// crawler 把 thumbnail_status 显式标 'failed',避免后续封面补队列一直重复
// 捞到这条 spider91 视频
//
// 历史 bug:之前 thumb 下载失败仅打 logurl=”, status 走 schema DEFAULT 'pending'。
// CountVideosNeedingThumbnail 条件是 url=” AND status != 'failed' → count=1。
// spider91 drive 的 thumb worker 按设计不处理 spider91 视频 → 没人会改 status
// 结果 teaser 永远卡在 [preview] waiting for 1 thumbnails before teaser generation
// spider91 drive 的 thumb worker 按设计不处理 spider91 视频 → 没人会改 status
// 后续补队列会一直认为它还缺封面
func TestCrawlerThumbDownloadFailureMarksStatusFailed(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("shell-based fake script only on unix")
@@ -317,8 +317,7 @@ func TestCrawlerThumbDownloadFailureMarksStatusFailed(t *testing.T) {
// 关键断言:CountVideosNeedingThumbnail 应该返回 0。
// 该函数的 SQL 条件是 `url = '' AND status != 'failed'`;如果 crawler 没把
// status 标 'failed'schema DEFAULT 'pending'),count 就会是 1,外层
// waitForThumbnailsBeforePreview 会因为 count > 0 把 teaser 卡死等待。
// status 标 'failed'schema DEFAULT 'pending'),count 就会是 1
count, err := cat.CountVideosNeedingThumbnail(context.Background(), driveID)
if err != nil {
t.Fatalf("count: %v", err)
+468
View File
@@ -0,0 +1,468 @@
package fingerprint
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/drives"
)
const (
defaultSampleSizeBytes int64 = 512 * 1024
defaultFullHashMaxSize int64 = 8 * 1024 * 1024
defaultCooldown = 5 * time.Minute
defaultWorkerQueueSize = 10000
)
type Config struct {
SampleSizeBytes int64
FullHashMaxSize int64
RateLimitCooldown time.Duration
HTTPClient *http.Client
}
type Worker struct {
Catalog *catalog.Catalog
Drive drives.Drive
Config Config
ch chan *catalog.Video
queue videoQueue
activity taskActivity
cooldown cooldownState
http *http.Client
}
type TaskStatus struct {
State string
CurrentTitle string
QueueLength int
CooldownUntil time.Time
}
func NewWorker(cat *catalog.Catalog, drv drives.Drive, cfg Config) *Worker {
hc := cfg.HTTPClient
if hc == nil {
hc = &http.Client{Timeout: 0}
}
if cfg.SampleSizeBytes <= 0 {
cfg.SampleSizeBytes = defaultSampleSizeBytes
}
if cfg.FullHashMaxSize <= 0 {
cfg.FullHashMaxSize = defaultFullHashMaxSize
}
if cfg.RateLimitCooldown <= 0 {
cfg.RateLimitCooldown = defaultCooldown
}
return &Worker{
Catalog: cat,
Drive: drv,
Config: cfg,
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
http: hc,
}
}
func (w *Worker) Enqueue(v *catalog.Video) bool {
if v == nil {
return false
}
if !w.queue.reserve(v.ID) {
return true
}
select {
case w.ch <- v:
return true
default:
w.queue.release(v.ID)
return false
}
}
func (w *Worker) EnqueueBlocking(ctx context.Context, v *catalog.Video) bool {
if v == nil {
return false
}
if !w.queue.reserve(v.ID) {
return true
}
select {
case w.ch <- v:
return true
case <-ctx.Done():
w.queue.release(v.ID)
return false
}
}
func (w *Worker) Run(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case v := <-w.ch:
w.processQueued(ctx, v)
select {
case <-ctx.Done():
return
case <-time.After(500 * time.Millisecond):
}
}
}
}
func (w *Worker) Status() TaskStatus {
if w == nil {
return TaskStatus{State: "idle"}
}
currentID, currentTitle := w.activity.current()
status := TaskStatus{
State: "idle",
CurrentTitle: currentTitle,
QueueLength: w.queue.lengthExcluding(currentID),
}
if until, ok := w.cooldown.active(time.Now()); ok {
status.State = "cooling"
status.CooldownUntil = until
return status
}
if currentID != "" {
status.State = "generating"
return status
}
if status.QueueLength > 0 {
status.State = "queued"
}
return status
}
func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
defer w.queue.release(v.ID)
if w.Catalog == nil || w.Drive == nil || v == nil || v.ID == "" {
return
}
current, err := w.Catalog.GetVideo(ctx, v.ID)
if err != nil {
return
}
if current.SampledSHA256 != "" || current.FingerprintStatus == "ready" || current.Hidden {
return
}
w.activity.start(current)
defer w.activity.done()
sum, err := Compute(ctx, w.Drive, current, w.Config, w.http)
if err != nil {
var rl *drives.RateLimitError
if errors.As(err, &rl) {
wait := rl.RetryAfter
if wait <= 0 {
wait = w.Config.RateLimitCooldown
}
until := time.Now().Add(wait)
w.cooldown.set(until)
log.Printf("[fingerprint] drive=%s rate limited; keep video=%s pending and cool down for %s: %v", w.Drive.ID(), current.ID, wait, err)
sleepContext(ctx, wait)
w.cooldown.clear(until)
return
}
log.Printf("[fingerprint] video=%s failed: %v", current.ID, err)
_ = w.Catalog.UpdateVideoFingerprint(ctx, current.ID, "", "failed", err.Error())
return
}
if err := w.Catalog.UpdateVideoFingerprint(ctx, current.ID, sum, "ready", ""); err != nil {
log.Printf("[fingerprint] update video=%s: %v", current.ID, err)
return
}
log.Printf("[fingerprint] video=%s ready sampled_sha256=%s", current.ID, sum)
}
func Compute(ctx context.Context, drv drives.Drive, v *catalog.Video, cfg Config, hc *http.Client) (string, error) {
if drv == nil {
return "", errors.New("fingerprint: nil drive")
}
if v == nil {
return "", errors.New("fingerprint: nil video")
}
if v.Size <= 0 {
return "", errors.New("fingerprint: video size is empty")
}
if cfg.SampleSizeBytes <= 0 {
cfg.SampleSizeBytes = defaultSampleSizeBytes
}
if cfg.FullHashMaxSize <= 0 {
cfg.FullHashMaxSize = defaultFullHashMaxSize
}
if hc == nil {
hc = &http.Client{Timeout: 0}
}
link, err := drv.StreamURL(ctx, v.FileID)
if err != nil {
return "", fmt.Errorf("fingerprint: stream url: %w", err)
}
if link == nil || strings.TrimSpace(link.URL) == "" {
return "", errors.New("fingerprint: empty stream url")
}
ranges := sampleRanges(v.Size, cfg.SampleSizeBytes, cfg.FullHashMaxSize)
h := sha256.New()
writeHashHeader(h, v.Size, ranges)
for _, r := range ranges {
data, err := readRange(ctx, hc, link, r)
if err != nil {
return "", err
}
if int64(len(data)) != r.length {
return "", fmt.Errorf("fingerprint: short sample at %d: got %d want %d", r.start, len(data), r.length)
}
_, _ = h.Write([]byte(fmt.Sprintf("offset=%d length=%d\n", r.start, r.length)))
_, _ = h.Write(data)
_, _ = h.Write([]byte("\n"))
}
return hex.EncodeToString(h.Sum(nil)), nil
}
type byteRange struct {
start int64
length int64
}
func sampleRanges(size, sampleSize, fullHashMax int64) []byteRange {
if size <= fullHashMax {
return []byteRange{{start: 0, length: size}}
}
if sampleSize > size {
sampleSize = size
}
maxStart := size - sampleSize
percents := []int64{0, 20, 40, 60, 80}
out := make([]byteRange, 0, len(percents))
seen := make(map[int64]struct{}, len(percents))
for _, pct := range percents {
start := maxStart * pct / 100
if _, ok := seen[start]; ok {
continue
}
seen[start] = struct{}{}
out = append(out, byteRange{start: start, length: sampleSize})
}
return out
}
func writeHashHeader(w io.Writer, size int64, ranges []byteRange) {
_, _ = fmt.Fprintf(w, "video-site-sampled-sha256-v1\nsize=%d\nsamples=%d\n", size, len(ranges))
}
func readRange(ctx context.Context, hc *http.Client, link *drives.StreamLink, r byteRange) ([]byte, error) {
u, err := url.Parse(link.URL)
if err == nil && (u.Scheme == "http" || u.Scheme == "https") {
return readHTTPRange(ctx, hc, link, r)
}
path := link.URL
if err == nil && u.Scheme == "file" {
path = u.Path
}
return readLocalRange(path, r)
}
func readLocalRange(path string, r byteRange) ([]byte, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("fingerprint: open local stream: %w", err)
}
defer f.Close()
buf := make([]byte, r.length)
n, err := f.ReadAt(buf, r.start)
if err != nil && !errors.Is(err, io.EOF) {
return nil, fmt.Errorf("fingerprint: read local sample: %w", err)
}
if int64(n) != r.length {
return nil, fmt.Errorf("fingerprint: read local sample at %d: got %d want %d", r.start, n, r.length)
}
return buf, nil
}
func readHTTPRange(ctx context.Context, hc *http.Client, link *drives.StreamLink, r byteRange) ([]byte, error) {
end := r.start + r.length - 1
req, err := http.NewRequestWithContext(ctx, http.MethodGet, link.URL, nil)
if err != nil {
return nil, err
}
for k, vs := range link.Headers {
for _, v := range vs {
req.Header.Add(k, v)
}
}
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", r.start, end))
resp, err := hc.Do(req)
if err != nil {
return nil, fmt.Errorf("fingerprint: read remote sample: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusTooManyRequests {
return nil, &drives.RateLimitError{
Provider: "fingerprint",
RetryAfter: parseRetryAfter(resp.Header.Get("Retry-After")),
Err: fmt.Errorf("remote sample rate limited: status=%d", resp.StatusCode),
}
}
if resp.StatusCode != http.StatusPartialContent {
if resp.StatusCode == http.StatusOK && r.start == 0 {
data, err := io.ReadAll(io.LimitReader(resp.Body, r.length+1))
if err != nil {
return nil, err
}
if int64(len(data)) == r.length {
return data, nil
}
}
return nil, fmt.Errorf("fingerprint: range request got status=%d for bytes=%d-%d", resp.StatusCode, r.start, end)
}
return io.ReadAll(io.LimitReader(resp.Body, r.length))
}
func parseRetryAfter(raw string) time.Duration {
raw = strings.TrimSpace(raw)
if raw == "" {
return 0
}
if seconds, err := strconv.Atoi(raw); err == nil && seconds > 0 {
return time.Duration(seconds) * time.Second
}
if when, err := http.ParseTime(raw); err == nil {
d := time.Until(when)
if d > 0 {
return d
}
}
return 0
}
func sleepContext(ctx context.Context, d time.Duration) bool {
if d <= 0 {
return true
}
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return false
case <-timer.C:
return true
}
}
type taskActivity struct {
mu sync.Mutex
currentID string
currentTitle string
}
func (a *taskActivity) start(v *catalog.Video) {
a.mu.Lock()
defer a.mu.Unlock()
if v == nil {
a.currentID = ""
a.currentTitle = ""
return
}
a.currentID = v.ID
a.currentTitle = v.Title
}
func (a *taskActivity) done() {
a.mu.Lock()
a.currentID = ""
a.currentTitle = ""
a.mu.Unlock()
}
func (a *taskActivity) current() (string, string) {
a.mu.Lock()
defer a.mu.Unlock()
return a.currentID, a.currentTitle
}
type cooldownState struct {
mu sync.Mutex
until time.Time
}
func (s *cooldownState) set(until time.Time) {
s.mu.Lock()
s.until = until
s.mu.Unlock()
}
func (s *cooldownState) clear(until time.Time) {
s.mu.Lock()
if s.until.Equal(until) {
s.until = time.Time{}
}
s.mu.Unlock()
}
func (s *cooldownState) active(now time.Time) (time.Time, bool) {
s.mu.Lock()
defer s.mu.Unlock()
if s.until.IsZero() || !s.until.After(now) {
return time.Time{}, false
}
return s.until, true
}
type videoQueue struct {
mu sync.Mutex
ids map[string]struct{}
}
func (q *videoQueue) reserve(id string) bool {
if id == "" {
return true
}
q.mu.Lock()
defer q.mu.Unlock()
if q.ids == nil {
q.ids = make(map[string]struct{})
}
if _, ok := q.ids[id]; ok {
return false
}
q.ids[id] = struct{}{}
return true
}
func (q *videoQueue) release(id string) {
if id == "" {
return
}
q.mu.Lock()
delete(q.ids, id)
q.mu.Unlock()
}
func (q *videoQueue) lengthExcluding(currentID string) int {
q.mu.Lock()
defer q.mu.Unlock()
n := len(q.ids)
if currentID != "" {
if _, ok := q.ids[currentID]; ok {
n--
}
}
if n < 0 {
return 0
}
return n
}
+112
View File
@@ -0,0 +1,112 @@
package fingerprint
import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"time"
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/drives"
)
func TestComputeLocalFilesWithSameContentMatch(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
body := []byte("same video bytes")
a := filepath.Join(dir, "a.mp4")
b := filepath.Join(dir, "b.mp4")
if err := os.WriteFile(a, body, 0o644); err != nil {
t.Fatalf("write a: %v", err)
}
if err := os.WriteFile(b, body, 0o644); err != nil {
t.Fatalf("write b: %v", err)
}
sumA, err := Compute(ctx, &fakeDrive{paths: map[string]string{"a": a}}, &catalog.Video{ID: "a", FileID: "a", Size: int64(len(body))}, Config{}, nil)
if err != nil {
t.Fatalf("compute a: %v", err)
}
sumB, err := Compute(ctx, &fakeDrive{paths: map[string]string{"b": b}}, &catalog.Video{ID: "b", FileID: "b", Size: int64(len(body))}, Config{}, nil)
if err != nil {
t.Fatalf("compute b: %v", err)
}
if sumA == "" || sumA != sumB {
t.Fatalf("fingerprints = %q / %q, want same non-empty", sumA, sumB)
}
}
func TestComputeRemoteUsesRangeSamples(t *testing.T) {
ctx := context.Background()
data := make([]byte, 10*1024*1024)
for i := range data {
data[i] = byte(i % 251)
}
var ranges []string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rawRange := r.Header.Get("Range")
ranges = append(ranges, rawRange)
var start, end int
if _, err := fmt.Sscanf(rawRange, "bytes=%d-%d", &start, &end); err != nil {
t.Fatalf("bad range %q: %v", rawRange, err)
}
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
w.WriteHeader(http.StatusPartialContent)
_, _ = w.Write(data[start : end+1])
}))
defer srv.Close()
drv := &fakeDrive{paths: map[string]string{"remote": srv.URL + "/video.mp4"}}
sum, err := Compute(ctx, drv, &catalog.Video{ID: "remote", FileID: "remote", Size: int64(len(data))}, Config{
SampleSizeBytes: 4,
FullHashMaxSize: 8,
HTTPClient: srv.Client(),
}, srv.Client())
if err != nil {
t.Fatalf("compute remote: %v", err)
}
if sum == "" {
t.Fatal("fingerprint should not be empty")
}
want := []string{
"bytes=0-3",
"bytes=2097151-2097154",
"bytes=4194302-4194305",
"bytes=6291453-6291456",
"bytes=8388604-8388607",
}
if fmt.Sprint(ranges) != fmt.Sprint(want) {
t.Fatalf("ranges = %#v, want %#v", ranges, want)
}
}
type fakeDrive struct {
paths map[string]string
}
func (d *fakeDrive) Kind() string { return "fake" }
func (d *fakeDrive) ID() string { return "fake" }
func (d *fakeDrive) Init(context.Context) error {
return nil
}
func (d *fakeDrive) List(context.Context, string) ([]drives.Entry, error) {
return nil, drives.ErrNotSupported
}
func (d *fakeDrive) Stat(context.Context, string) (*drives.Entry, error) {
return nil, drives.ErrNotSupported
}
func (d *fakeDrive) StreamURL(_ context.Context, fileID string) (*drives.StreamLink, error) {
return &drives.StreamLink{URL: d.paths[fileID], Expires: time.Now().Add(time.Minute)}, nil
}
func (d *fakeDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
return "", drives.ErrNotSupported
}
func (d *fakeDrive) EnsureDir(context.Context, string) (string, error) {
return "", drives.ErrNotSupported
}
func (d *fakeDrive) RootID() string { return "root" }
+27 -3
View File
@@ -12,6 +12,8 @@
// wait until teaser queues are idle
// Phase 3: spider91 → cloud migration (single sweep, captcha cooldown still
// honored within this call)
// Phase 4: cleanup duplicate local preview/thumbnail assets after sampled
// fingerprints have identified canonical videos
//
// A 6h soft deadline guards each pipeline run; phases check deadline at their
// boundaries and exit cleanly if exceeded (no in-flight ffmpeg / upload is
@@ -85,6 +87,11 @@ type Config struct {
// RunMigration runs spider91migrate.Migrator.RunOnce for Phase 3.
RunMigration func(ctx context.Context) error
// RunDedupeAssetCleanup removes generated local assets from non-canonical
// videos in size+sampled_sha256 duplicate groups. It must not delete cloud
// files or catalog rows.
RunDedupeAssetCleanup func(ctx context.Context) error
// Now is injected for tests; nil → time.Now.
Now func() time.Time
}
@@ -131,9 +138,10 @@ func (r *Runner) Run(ctx context.Context) {
}
}
// TriggerNow asks the running loop to fire a pipeline ASAP. If a pipeline is
// already in progress (or another trigger is already pending), the request
// is dropped — the in-progress run will absorb the intent.
// TriggerNow asks the running loop to fire a pipeline ASAP. The trigger channel
// is buffered(1): if a pipeline is already in progress, one follow-up run may
// remain pending and will start after the current run finishes. Additional
// clicks while that follow-up is pending are ignored.
func (r *Runner) TriggerNow() {
select {
case r.trigger <- struct{}{}:
@@ -240,6 +248,7 @@ func (r *Runner) runPipeline(ctx context.Context) {
}
if len(spiderIDs) == 0 {
log.Printf("[nightly] phase 2/3 skipped: no spider91 drive configured")
r.runDedupeAssetCleanupPhase(ctx)
return
}
log.Printf("[nightly] phase 2: crawling %d spider91 drive(s)", len(spiderIDs))
@@ -266,6 +275,8 @@ func (r *Runner) runPipeline(ctx context.Context) {
log.Printf("[nightly] phase 3 migration: %v", err)
}
}
r.runDedupeAssetCleanupPhase(ctx)
}
// checkDeadline returns true when ctx is already done (runner shutting down or
@@ -291,6 +302,19 @@ func (r *Runner) waitIdle(ctx context.Context, phase string) error {
return nil
}
func (r *Runner) runDedupeAssetCleanupPhase(ctx context.Context) {
if r.checkDeadline(ctx, "phase 4") {
return
}
if r.cfg.RunDedupeAssetCleanup == nil {
return
}
log.Printf("[nightly] phase 4: duplicate asset cleanup")
if err := r.cfg.RunDedupeAssetCleanup(ctx); err != nil {
log.Printf("[nightly] phase 4 duplicate asset cleanup: %v", err)
}
}
// readLastRunDate reads the persisted last_run_date or returns "" when unset.
func (r *Runner) readLastRunDate(ctx context.Context) (string, error) {
if r.cfg.Settings == nil {
+22
View File
@@ -114,6 +114,10 @@ func TestRunPipelineHonoursPhaseOrder(t *testing.T) {
rec.push("migrate")
return nil
},
RunDedupeAssetCleanup: func(context.Context) error {
rec.push("dedupe-cleanup")
return nil
},
})
r.runPipeline(context.Background())
@@ -128,6 +132,7 @@ func TestRunPipelineHonoursPhaseOrder(t *testing.T) {
"crawl:sp-1",
"wait-idle", // after phase 2
"migrate",
"dedupe-cleanup",
}
if len(got) != len(want) {
t.Fatalf("call sequence len = %d, want %d; got=%v", len(got), len(want), got)
@@ -156,6 +161,10 @@ func TestRunPipelineSkipsMigrationWhenNoSpider91(t *testing.T) {
rec.push("migrate")
return nil
},
RunDedupeAssetCleanup: func(context.Context) error {
rec.push("dedupe-cleanup")
return nil
},
})
r.runPipeline(context.Background())
@@ -165,6 +174,15 @@ func TestRunPipelineSkipsMigrationWhenNoSpider91(t *testing.T) {
t.Fatalf("phase 2/3 should be skipped when no spider91 drive, got call %q", c)
}
}
foundCleanup := false
for _, c := range rec.snapshot() {
if c == "dedupe-cleanup" {
foundCleanup = true
}
}
if !foundCleanup {
t.Fatalf("dedupe cleanup should still run when spider91 is absent; calls=%v", rec.snapshot())
}
}
func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
@@ -186,6 +204,7 @@ func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
RunSpider91Crawl: func(context.Context, string) { rec.push("crawl") },
WaitPreviewQueuesIdle: func(context.Context) error { rec.push("wait-idle"); return nil },
RunMigration: func(context.Context) error { rec.push("migrate"); return nil },
RunDedupeAssetCleanup: func(context.Context) error { rec.push("dedupe-cleanup"); return nil },
})
r.runPipeline(ctx)
@@ -200,6 +219,9 @@ func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
if c == "crawl" || c == "migrate" {
t.Fatalf("subsequent phase should not run after cancel, got call %q", c)
}
if c == "dedupe-cleanup" {
t.Fatalf("dedupe cleanup should not run after cancel, got call %q", c)
}
}
}
+78 -19
View File
@@ -289,7 +289,7 @@ func (g *Generator) generateThumbnailAtOffset(ctx context.Context, link *drives.
args = append(args,
"-i", ffmpegLink.URL,
"-frames:v", "1",
"-vf", fmt.Sprintf("scale=%d:-2", g.cfg.Width),
"-vf", thumbnailVideoFilter(g.cfg.Width),
"-q:v", "3",
"-y", dst,
)
@@ -307,6 +307,12 @@ func (g *Generator) generateThumbnailAtOffset(ctx context.Context, link *drives.
return nil
}
func thumbnailVideoFilter(width int) string {
// FFmpeg 7 rejects non-full-range YUV for MJPEG/JPEG output. Force the
// scaled frame into a JPEG-friendly full-range pixel format before encode.
return fmt.Sprintf("scale=%d:-2:out_range=pc,format=yuvj420p", width)
}
func thumbnailOffsetFallbackAllowed(err error) bool {
if err == nil {
return false
@@ -923,6 +929,7 @@ func ffmpegOutputLooksRateLimited(output []byte) bool {
return false
}
return strings.Contains(text, "too many requests") ||
strings.Contains(text, "throttl") ||
strings.Contains(text, "rate limit") ||
strings.Contains(text, "rate-limit") ||
strings.Contains(text, "server returned 429")
@@ -968,7 +975,6 @@ type Worker struct {
queue videoQueue
RateLimitCooldown time.Duration
BeforeTask func(context.Context) bool
rateLimit rateLimitState
activity taskActivity
}
@@ -978,7 +984,7 @@ func NewWorker(gen TeaserGenerator, cat *catalog.Catalog, drv drives.Drive) *Wor
Gen: gen,
Catalog: cat,
Drive: drv,
ch: make(chan *catalog.Video, 4096),
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
}
}
@@ -1029,6 +1035,7 @@ type ThumbWorker struct {
const (
defaultTransientMediaCooldown = 5 * time.Minute
defaultGenerationRateLimitCooldown = 5 * time.Minute
defaultWorkerQueueSize = 10000
maxPreviewTeaserSizeBytes int64 = 5 * 1024 * 1024 * 1024
previewStatusSkipped = "skipped"
)
@@ -1168,7 +1175,7 @@ func NewThumbWorker(gen ThumbnailGenerator, cat *catalog.Catalog, drv drives.Dri
Gen: gen,
Catalog: cat,
Drive: drv,
ch: make(chan *catalog.Video, 4096),
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
}
}
@@ -1323,10 +1330,6 @@ func (w *ThumbWorker) Run(ctx context.Context) {
func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
defer w.queue.release(v)
if w.BeforeTask != nil && !w.BeforeTask(ctx) {
return
}
w.activity.start(v)
defer w.activity.done()
if !waitForRateLimitCooldown(ctx, &w.rateLimit, "preview", w.Drive) {
@@ -1482,29 +1485,53 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
if w.skipIfRateLimited(v) {
return
}
if current, err := w.Catalog.GetVideo(ctx, v.ID); err == nil {
if current.ThumbnailURL != "" {
queued := v
current := v
if loaded, err := w.Catalog.GetVideo(ctx, v.ID); err == nil {
if loaded.PreviewLocal == "" {
loaded.PreviewLocal = queued.PreviewLocal
}
current = loaded
v = loaded
if loaded.ThumbnailURL != "" && loaded.DurationSeconds > 0 {
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
return
}
}
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "pending"})
link, err := w.Drive.StreamURL(ctx, v.FileID)
if err != nil {
if localLink, ok := localPreviewLink(v); ok {
link = localLink
} else {
if w.pauseForRecoverableError(err, "streamURL", v.Title) {
if current.ThumbnailURL != "" {
if current.DurationSeconds <= 0 {
link, err := w.streamLink(ctx, current)
if err != nil {
if w.pauseForRecoverableError(err, "streamURL", current.Title) {
return
}
log.Printf("[thumb] probe streamURL %s: %v", current.Title, err)
} else if w.probeDuration(ctx, current, link) {
return
}
log.Printf("[thumb] streamURL %s: %v", v.Title, err)
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
}
_ = w.Catalog.UpdateVideoMeta(ctx, current.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
return
}
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "pending"})
link, err := w.streamLink(ctx, v)
if err != nil {
if w.pauseForRecoverableError(err, "streamURL", v.Title) {
return
}
log.Printf("[thumb] streamURL %s: %v", v.Title, err)
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
return
}
if w.probeDuration(ctx, v, link) {
return
}
if err := w.generateThumbnailFromLink(ctx, v, link); err != nil {
if localLink, ok := localPreviewLink(v); ok && link.URL != localLink.URL {
if w.probeDuration(ctx, v, localLink) {
return
}
if localErr := w.generateThumbnailFromLink(ctx, v, localLink); localErr == nil {
return
}
@@ -1518,6 +1545,38 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
}
}
func (w *ThumbWorker) streamLink(ctx context.Context, v *catalog.Video) (*drives.StreamLink, error) {
link, err := w.Drive.StreamURL(ctx, v.FileID)
if err == nil {
return link, nil
}
if localLink, ok := localPreviewLink(v); ok {
return localLink, nil
}
return nil, err
}
func (w *ThumbWorker) probeDuration(ctx context.Context, v *catalog.Video, link *drives.StreamLink) bool {
if v.DurationSeconds > 0 {
return false
}
dur, err := w.Gen.Probe(ctx, link)
if err == nil {
if dur > 0 {
v.DurationSeconds = int(dur)
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
DurationSeconds: int(dur),
})
}
return false
}
if w.pauseForRecoverableError(err, "probe", v.Title) {
return true
}
log.Printf("[thumb] probe %s: %v", v.Title, err)
return false
}
func (w *ThumbWorker) generateThumbnailFromLink(ctx context.Context, v *catalog.Video, link *drives.StreamLink) error {
if _, err := w.Gen.GenerateThumbnail(ctx, link, v.ID, 0); err != nil {
return err
+10
View File
@@ -161,6 +161,16 @@ func TestThumbnailOffsetsUseFiveSecondsWithEarlyFallbacks(t *testing.T) {
}
}
func TestThumbnailVideoFilterUsesFullRangeJPEGPixelFormat(t *testing.T) {
got := thumbnailVideoFilter(480)
if !strings.Contains(got, "scale=480:-2:out_range=pc") {
t.Fatalf("thumbnail filter = %q, want full-range scale output", got)
}
if !strings.Contains(got, "format=yuvj420p") {
t.Fatalf("thumbnail filter = %q, want JPEG-friendly pixel format", got)
}
}
func TestThumbnailOffsetFallbackAllowedForEmptyOutputAndTimeouts(t *testing.T) {
for _, err := range []error{
errors.New("ffmpeg thumb produced empty file, stderr: "),
+51 -8
View File
@@ -13,11 +13,11 @@ import (
"github.com/video-site/backend/internal/drives"
)
func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
func TestThumbWorkerUpdatesThumbnailAndDurationWithoutChangingPreviewStatus(t *testing.T) {
ctx := context.Background()
cat, video := seedPreviewTestVideo(t, "thumb-worker-video")
gen := &fakeThumbGenerator{}
gen := &fakeThumbGenerator{probeDuration: 42}
drv := &previewFakeDrive{}
worker := NewThumbWorker(gen, cat, drv)
@@ -33,8 +33,8 @@ func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
if got.PreviewStatus != "pending" {
t.Fatalf("preview status = %q, want pending", got.PreviewStatus)
}
if got.DurationSeconds != 0 {
t.Fatalf("duration = %d, want unchanged", got.DurationSeconds)
if got.DurationSeconds != 42 {
t.Fatalf("duration = %d, want probed duration", got.DurationSeconds)
}
if gen.thumbnailVideoID != video.ID {
t.Fatalf("thumbnail video id = %q, want %q", gen.thumbnailVideoID, video.ID)
@@ -42,14 +42,53 @@ func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
if gen.thumbnailDuration != 0 {
t.Fatalf("thumbnail duration = %.1f, want fixed-offset thumbnail generation", gen.thumbnailDuration)
}
if gen.probeCalls != 0 {
t.Fatalf("probe calls = %d, want 0 for thumbnail generation", gen.probeCalls)
if gen.probeCalls != 1 {
t.Fatalf("probe calls = %d, want 1 for thumbnail generation", gen.probeCalls)
}
if drv.streamFileID != video.FileID {
t.Fatalf("stream file id = %q, want %q", drv.streamFileID, video.FileID)
}
}
func TestThumbWorkerBackfillsDurationWhenThumbnailAlreadyExists(t *testing.T) {
ctx := context.Background()
cat, video := seedPreviewTestVideo(t, "thumb-worker-existing-thumbnail")
video.ThumbnailURL = "/p/thumb/" + video.ID
if err := cat.UpsertVideo(ctx, video); err != nil {
t.Fatalf("update video: %v", err)
}
gen := &fakeThumbGenerator{probeDuration: 19}
drv := &previewFakeDrive{}
worker := NewThumbWorker(gen, cat, drv)
worker.process(ctx, video)
got, err := cat.GetVideo(ctx, video.ID)
if err != nil {
t.Fatalf("get video: %v", err)
}
if got.DurationSeconds != 19 {
t.Fatalf("duration = %d, want probed duration", got.DurationSeconds)
}
if got.ThumbnailURL != "/p/thumb/"+video.ID {
t.Fatalf("thumbnail = %q, want unchanged existing thumbnail", got.ThumbnailURL)
}
ready, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "ready", 0)
if err != nil {
t.Fatalf("list ready thumbnails: %v", err)
}
if len(ready) != 1 || ready[0].ID != video.ID {
t.Fatalf("ready thumbnails = %#v, want only %s", ready, video.ID)
}
if gen.probeCalls != 1 {
t.Fatalf("probe calls = %d, want 1", gen.probeCalls)
}
if gen.thumbnailVideoID != "" {
t.Fatalf("thumbnail generation video id = %q, want no regeneration", gen.thumbnailVideoID)
}
}
func TestThumbWorkerFallsBackToLocalPreviewWhenDriveStreamFails(t *testing.T) {
ctx := context.Background()
cat, video := seedPreviewTestVideo(t, "thumb-worker-local-preview")
@@ -469,12 +508,17 @@ type fakeThumbGenerator struct {
thumbnailDuration float64
thumbnailURL string
probeCalls int
probeDuration float64
probeErr error
generateErr error
}
func (g *fakeThumbGenerator) Probe(context.Context, *drives.StreamLink) (float64, error) {
g.probeCalls++
return 42, nil
if g.probeErr != nil {
return 0, g.probeErr
}
return g.probeDuration, nil
}
func (g *fakeThumbGenerator) GenerateThumbnail(_ context.Context, link *drives.StreamLink, videoID string, duration float64) (string, error) {
@@ -568,7 +612,6 @@ func (d *previewFakeDrive) EnsureDir(context.Context, string) (string, error) {
}
func (d *previewFakeDrive) RootID() string { return "root" }
func TestWorkerWaitIdleReturnsImmediatelyWhenQueueEmpty(t *testing.T) {
worker := NewWorker(&fakeTeaserGenerator{}, nil, &previewFakeDrive{})
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+23 -2
View File
@@ -5,6 +5,7 @@ import (
"io"
"net/http"
"net/url"
"path/filepath"
"sync"
"time"
@@ -144,13 +145,15 @@ func (p *Proxy) ServeStream(w http.ResponseWriter, r *http.Request, driveID, fil
// 302 之后浏览器用自己的 UA 直连,CDN 仍然认签名
// - pikpak:与 OpenList 一致,WebContentLink / media link 都是自签 URL
// CDN 不校验请求头,直连可获得最佳带宽并避免占用 backend 出站
// - onedriveMicrosoft Graph 返回的 @microsoft.graph.downloadUrl 是短期
// 免鉴权下载 URL,不需要后端继续代传视频字节
//
// 其余网盘(如 OneDrive / 沃盘 / 夸克等)仍走反代,因为它们的下载
// 其余网盘(如沃盘 / 夸克等)仍走反代,因为它们的下载
// 链接通常需要随请求带上后端持有的 Cookie / Authorization / Range
// 的特殊处理,浏览器拿不到这些上下文。
func shouldRedirect(d drives.Drive) bool {
switch d.Kind() {
case "p115", "pikpak":
case "p115", "pikpak", "onedrive":
return true
}
return false
@@ -169,6 +172,11 @@ func (p *Proxy) serve(w http.ResponseWriter, r *http.Request, link *drives.Strea
http.Error(w, "bad upstream url", http.StatusBadGateway)
return
}
if localPath, ok := localFilePath(u, link.URL); ok {
w.Header().Set("Cache-Control", "private, max-age=300")
http.ServeFile(w, r, localPath)
return
}
req, err := http.NewRequestWithContext(r.Context(), r.Method, u.String(), nil)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -211,6 +219,19 @@ func (p *Proxy) ServeLocal(w http.ResponseWriter, r *http.Request, path string)
http.ServeFile(w, r, path)
}
func localFilePath(u *url.URL, raw string) (string, bool) {
if u == nil {
return "", false
}
if u.Scheme == "file" && u.Path != "" {
return u.Path, true
}
if u.Scheme == "" && u.Host == "" && filepath.IsAbs(raw) {
return raw, true
}
return "", false
}
var errDriveNotFound = &httpError{Code: http.StatusNotFound, Msg: "drive not found"}
type httpError struct {
+90
View File
@@ -5,6 +5,8 @@ import (
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"time"
@@ -149,6 +151,61 @@ func TestServeStreamPikPakSetsRedirectHeaders(t *testing.T) {
}
}
func TestServeStreamRedirectsOneDrive(t *testing.T) {
reg := NewRegistry()
drv := &proxyFakeSimpleDrive{
kind: "onedrive",
url: "https://public.onedrive.example/video.mp4",
}
reg.Set("onedrive", drv)
p := New(reg)
req := httptest.NewRequest(http.MethodGet, "/p/stream/onedrive/file-1", nil)
rr := httptest.NewRecorder()
p.ServeStream(rr, req, "onedrive", "file-1")
if rr.Code != http.StatusFound {
t.Fatalf("status = %d, want %d", rr.Code, http.StatusFound)
}
if got := rr.Header().Get("Location"); got != "https://public.onedrive.example/video.mp4" {
t.Fatalf("Location = %q", got)
}
if drv.calls != 1 {
t.Fatalf("link calls = %d, want 1", drv.calls)
}
}
func TestServeStreamServesLocalFilePath(t *testing.T) {
path := filepath.Join(t.TempDir(), "video.mp4")
if err := os.WriteFile(path, []byte("0123456789"), 0o644); err != nil {
t.Fatalf("write local file: %v", err)
}
reg := NewRegistry()
drv := &proxyFakeSimpleDrive{
kind: "localstorage",
url: path,
}
reg.Set("local", drv)
p := New(reg)
req := httptest.NewRequest(http.MethodGet, "/p/stream/local/file-1", nil)
req.Header.Set("Range", "bytes=2-5")
rr := httptest.NewRecorder()
p.ServeStream(rr, req, "local", "file-1")
if rr.Code != http.StatusPartialContent {
t.Fatalf("status = %d, want %d", rr.Code, http.StatusPartialContent)
}
if got := rr.Body.String(); got != "2345" {
t.Fatalf("body = %q, want range bytes", got)
}
if drv.calls != 1 {
t.Fatalf("link calls = %d, want 1", drv.calls)
}
}
func requestPikPak(t *testing.T, p *Proxy, driveID, fileID, ua string) {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "/p/stream/"+driveID+"/"+fileID, nil)
@@ -192,3 +249,36 @@ func (d *proxyFakePikPakDrive) EnsureDir(context.Context, string) (string, error
return "", drives.ErrNotSupported
}
func (d *proxyFakePikPakDrive) RootID() string { return "0" }
type proxyFakeSimpleDrive struct {
kind string
url string
calls int
}
func (d *proxyFakeSimpleDrive) Kind() string { return d.kind }
func (d *proxyFakeSimpleDrive) ID() string { return d.kind }
func (d *proxyFakeSimpleDrive) Init(context.Context) error {
return nil
}
func (d *proxyFakeSimpleDrive) List(context.Context, string) ([]drives.Entry, error) {
return nil, drives.ErrNotSupported
}
func (d *proxyFakeSimpleDrive) Stat(context.Context, string) (*drives.Entry, error) {
return nil, drives.ErrNotSupported
}
func (d *proxyFakeSimpleDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
d.calls++
return &drives.StreamLink{
URL: d.url,
Headers: http.Header{},
Expires: time.Now().Add(10 * time.Minute),
}, nil
}
func (d *proxyFakeSimpleDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
return "", drives.ErrNotSupported
}
func (d *proxyFakeSimpleDrive) EnsureDir(context.Context, string) (string, error) {
return "", drives.ErrNotSupported
}
func (d *proxyFakeSimpleDrive) RootID() string { return "0" }
+49 -11
View File
@@ -1,5 +1,5 @@
// Package spider91migrate 周期性把 spider91 drive 下载到本地的视频
// 上传到一个指定的目标 drive 目录(PikPak 或 115),上传成功后:
// 上传到一个指定的目标 drive 目录(PikPak、115 或 OneDrive),上传成功后:
//
// - 改写 catalog 行:drive_id / file_id / content_hash 改成目标盘的;
// 视频自身的 id 不变(仍是 spider91-<driveID>-<viewkey>),video_tags、
@@ -28,6 +28,7 @@ import (
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/drives"
"github.com/video-site/backend/internal/drives/onedrive"
"github.com/video-site/backend/internal/drives/p115"
"github.com/video-site/backend/internal/drives/pikpak"
"github.com/video-site/backend/internal/drives/spider91"
@@ -39,12 +40,14 @@ import (
// 这一层抽象把"迁移调用方"和"具体盘的 SDK 协议"解耦:
// - PikPak 走 GCID + OSS PutObjectpikpak.UploadResult
// - 115 走 SHA1 + 秒传 / OSS / 分片(p115.UploadResult
// - OneDrive 走 SHA1 + 小文件 PUT / 大文件 upload session
//
// 两个返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
// 各家返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
type uploadTarget interface {
ID() string
Kind() string
RootID() string
EnsureDir(ctx context.Context, pathFromRoot string) (string, error)
UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error)
Rename(ctx context.Context, fileID, newName string) error
}
@@ -52,7 +55,7 @@ type uploadTarget interface {
// UploadResult 是 uploadTarget.UploadAndReportHash 的归一返回。
//
// FileID 目标盘上的新文件 ID;
// Hash GCIDPikPak)或 SHA1 HEX 大写115),写入 catalog.content_hash 用于跨盘去重;
// Hash GCIDPikPak)或 SHA1 HEX115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;
// Size 实际上传字节数。
type UploadResult struct {
FileID string
@@ -60,7 +63,9 @@ type UploadResult struct {
Size int64
}
// pikpakAdapter / p115Adapter 把具体 driver 包装成 uploadTarget。
const spider91UploadDirName = "91 Spider"
// pikpakAdapter / p115Adapter / onedriveAdapter 把具体 driver 包装成 uploadTarget。
//
// 之所以不让 driver 直接实现 uploadTarget
//
@@ -74,6 +79,9 @@ type pikpakAdapter struct {
func (a *pikpakAdapter) ID() string { return a.d.ID() }
func (a *pikpakAdapter) Kind() string { return a.d.Kind() }
func (a *pikpakAdapter) RootID() string { return a.d.RootID() }
func (a *pikpakAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
return a.d.EnsureDir(ctx, pathFromRoot)
}
func (a *pikpakAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
if err != nil {
@@ -92,6 +100,9 @@ type p115Adapter struct {
func (a *p115Adapter) ID() string { return a.d.ID() }
func (a *p115Adapter) Kind() string { return a.d.Kind() }
func (a *p115Adapter) RootID() string { return a.d.RootID() }
func (a *p115Adapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
return a.d.EnsureDir(ctx, pathFromRoot)
}
func (a *p115Adapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
res, err := a.d.UploadAndReportSha1(ctx, parentID, name, r, size)
if err != nil {
@@ -103,6 +114,27 @@ func (a *p115Adapter) Rename(ctx context.Context, fileID, newName string) error
return a.d.Rename(ctx, fileID, newName)
}
type onedriveAdapter struct {
d *onedrive.Driver
}
func (a *onedriveAdapter) ID() string { return a.d.ID() }
func (a *onedriveAdapter) Kind() string { return a.d.Kind() }
func (a *onedriveAdapter) RootID() string { return a.d.RootID() }
func (a *onedriveAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
return a.d.EnsureDir(ctx, pathFromRoot)
}
func (a *onedriveAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
if err != nil {
return UploadResult{}, err
}
return UploadResult{FileID: res.FileID, Hash: res.Hash, Size: res.Size}, nil
}
func (a *onedriveAdapter) Rename(ctx context.Context, fileID, newName string) error {
return a.d.Rename(ctx, fileID, newName)
}
// adaptUploadTarget 把通用 drive 包装成 uploadTarget。
// 不支持的盘 kind 返回 error;调用方静默跳过。
func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
@@ -111,6 +143,8 @@ func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
return &pikpakAdapter{d: v}, nil
case *p115.Driver:
return &p115Adapter{d: v}, nil
case *onedrive.Driver:
return &onedriveAdapter{d: v}, nil
case uploadTarget:
// 测试或自定义实现可以直接传入;优先使用具体类型分支以拿到适配器。
return v, nil
@@ -272,7 +306,7 @@ func (m *Migrator) runOnce(ctx context.Context) {
target, pp, err := m.resolveTarget()
if err != nil {
// 没目标就静默 —— 用户可能还没配 PikPak drive
// 没目标就静默 —— 用户选择了本地保存,或还没配 115/PikPak drive
return
}
@@ -382,7 +416,7 @@ func (m *Migrator) spider91Drives() []*spider91.Driver {
// - 列出 spider91 drive 本地 videos/ 目录所有 mp4 文件,按 mtime 降序排
// - 跳过最新 KeepLatestN 个:这些是用户希望保留在本地的最新爬取
// - 对剩下的(更旧)逐个处理:
// - 还没迁移(drive_id 仍是 src.ID())→ 上传到 PikPak + 改 catalog + 删本地
// - 还没迁移(drive_id 仍是 src.ID())→ 上传到目标盘 + 改 catalog + 删本地
// - 已经迁移过但本地还有残留 → 仅删本地(兜底)
//
// KeepLatestN < 0 时不保护任何本地文件,全部尝试迁移(旧行为,主要给测试用)。
@@ -484,7 +518,7 @@ func (m *Migrator) migrateDrive(ctx context.Context, src *spider91.Driver, targe
return migrated, nil
}
// migrateOne 把单条 spider91 视频上传到 PikPak 并改写 catalog。
// migrateOne 把单条 spider91 视频上传到目标盘并改写 catalog。
// 返回 (true, nil) 表示真的迁了一条;(false, nil) 表示跳过(本地文件已不在等);
// (false, err) 表示真出错。
func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider91.Driver, targetDriveID string, pp uploadTarget) (bool, error) {
@@ -511,15 +545,19 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
}
defer f.Close()
// 上传到目标盘的根目录(用户配置的目标 drive 的 rootID)。
// 上传到目标盘 rootID 下的固定 "91 Spider" 子目录。若用户把目标盘 rootID
// 配成某个自定义目录,这里会在该自定义目录下查找/创建 "91 Spider"。
// 上传名走 desiredPikPakName 算出来的方案 B 格式:
//
// <sanitized title>-<viewkey 后 8 位>.<ext>
//
// 这样网盘 Web 端列出来的文件名能直接看出是哪个视频,
// 又用 viewkey 后 8 位避免同标题撞名。两个目标盘PikPak / 115共用同一格式,
// 又用 viewkey 后 8 位避免同标题撞名。所有目标盘共用同一格式,
// 简化前端 / catalog 的认知。
parent := pp.RootID()
parent, err := pp.EnsureDir(ctx, spider91UploadDirName)
if err != nil {
return false, fmt.Errorf("%s ensure %q dir: %w", pp.Kind(), spider91UploadDirName, err)
}
uploadName := desiredPikPakName(v.Title, extractViewKey(v.ID), v.Ext)
res, err := pp.UploadAndReportHash(ctx, parent, uploadName, f, info.Size())
if err != nil {
@@ -639,7 +677,7 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driv
return deleted, nil
}
// backfillFileNames 扫描目标 drivePikPak 或 115)下所有 spider91-* 起始 ID 的视频,
// backfillFileNames 扫描目标 drivePikPak、115 或 OneDrive)下所有 spider91-* 起始 ID 的视频,
// 对文件名不是 desiredPikPakName(...) 期望格式的,调 target.Rename 修正,
// 并把 catalog.file_name 同步到新名字。
//
@@ -53,6 +53,8 @@ type fakePikPak struct {
uploadFunc func(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error)
mu sync.Mutex
gotBodies map[string][]byte
gotParents map[string]string
ensureCalls []string
// renameCalls 记录每次 Rename 的 fileID->newName 历史,用于 backfill 测试断言。
renameCalls map[string]string
}
@@ -62,6 +64,7 @@ func newFakePikPak(id, rootID string) *fakePikPak {
id: id,
rootID: rootID,
gotBodies: make(map[string][]byte),
gotParents: make(map[string]string),
renameCalls: make(map[string]string),
}
}
@@ -80,8 +83,11 @@ func (d *fakePikPak) StreamURL(context.Context, string) (*drives.StreamLink, err
func (d *fakePikPak) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
return "", drives.ErrNotSupported
}
func (d *fakePikPak) EnsureDir(context.Context, string) (string, error) {
return "", drives.ErrNotSupported
func (d *fakePikPak) EnsureDir(_ context.Context, pathFromRoot string) (string, error) {
d.mu.Lock()
defer d.mu.Unlock()
d.ensureCalls = append(d.ensureCalls, pathFromRoot)
return d.rootID + "/" + pathFromRoot, nil
}
func (d *fakePikPak) Rename(_ context.Context, fileID, newName string) error {
d.mu.Lock()
@@ -99,6 +105,7 @@ func (d *fakePikPak) UploadAndReportHash(ctx context.Context, parentID, name str
body, _ := io.ReadAll(r)
d.mu.Lock()
d.gotBodies[name] = body
d.gotParents[name] = parentID
d.mu.Unlock()
return UploadResult{
FileID: "remote-" + name,
@@ -127,6 +134,19 @@ func (d *fakeP115) Kind() string { return "p115" }
var _ drives.Drive = (*fakeP115)(nil)
var _ uploadTarget = (*fakeP115)(nil)
type fakeOneDrive struct {
*fakePikPak
}
func newFakeOneDrive(id, rootID string) *fakeOneDrive {
return &fakeOneDrive{fakePikPak: newFakePikPak(id, rootID)}
}
func (d *fakeOneDrive) Kind() string { return "onedrive" }
var _ drives.Drive = (*fakeOneDrive)(nil)
var _ uploadTarget = (*fakeOneDrive)(nil)
// TestBackfillFileNamesRenamesOnlyMismatchedSpider91Videos 验证回填逻辑:
//
// - 已经是期望格式的不会再调 Rename(幂等)
@@ -347,6 +367,12 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
if _, ok := pp.gotBodies[wantName]; !ok {
t.Fatalf("PikPak did not receive expected upload name %q (got names: %v)", wantName, keysOf(pp.gotBodies))
}
if gotParent := pp.gotParents[wantName]; gotParent != "pikpak-root-id/"+spider91UploadDirName {
t.Fatalf("upload parent = %q, want root/91 Spider", gotParent)
}
if len(pp.ensureCalls) != 1 || pp.ensureCalls[0] != spider91UploadDirName {
t.Fatalf("ensure calls = %#v, want %q", pp.ensureCalls, spider91UploadDirName)
}
if got.FileID != "remote-"+wantName {
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
}
@@ -588,7 +614,7 @@ func TestRunOnceKeepsAllLocalWhenWithinKeepWindow(t *testing.T) {
}
// TestRunOnceMigratesOnlyOlderFilesBeyondKeepWindow 验证:本地文件数 > KeepLatestN 时
// 按 mtime 降序保留最新 N 个,超出部分(更旧的)才上传到 PikPak
// 按 mtime 降序保留最新 N 个,超出部分(更旧的)才上传到目标盘
func TestRunOnceMigratesOnlyOlderFilesBeyondKeepWindow(t *testing.T) {
cat := setupCatalog(t)
src, _ := setupSpider91(t)
@@ -841,7 +867,6 @@ func TestNonCaptchaErrorDoesNotTriggerCooldown(t *testing.T) {
}
}
// TestRunOnceMigratesToP115Target 验证:当目标 drive 是 115kind="p115")时,
// migrator 也能正确把 spider91 视频上传过去并改写 catalog。
//
@@ -885,6 +910,12 @@ func TestRunOnceMigratesToP115Target(t *testing.T) {
if _, ok := target.gotBodies[wantName]; !ok {
t.Fatalf("p115 did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
}
if gotParent := target.gotParents[wantName]; gotParent != "p115-root-cid/"+spider91UploadDirName {
t.Fatalf("p115 upload parent = %q, want root/91 Spider", gotParent)
}
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
t.Fatalf("p115 ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
}
if got.FileID != "remote-"+wantName {
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
}
@@ -906,7 +937,67 @@ func TestRunOnceMigratesToP115Target(t *testing.T) {
}
}
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak 也不是 115 时,
func TestRunOnceMigratesToOneDriveTarget(t *testing.T) {
cat := setupCatalog(t)
src, _ := setupSpider91(t)
target := newFakeOneDrive("onedrive-target", "onedrive-root")
reg := newFakeRegistry()
reg.Add(src)
reg.Add(target)
now := time.Now()
id := writeSpider91Video(t, cat, src, "vk-od-001", ".mp4", []byte("video bytes onedrive"), now)
m := New(Config{
Catalog: cat,
Registry: reg,
GetTargetDriveID: func() string { return target.ID() },
KeepLatestN: -1,
})
m.runOnce(context.Background())
if target.uploadCalls != 1 {
t.Fatalf("onedrive upload calls = %d, want 1", target.uploadCalls)
}
got, err := cat.GetVideo(context.Background(), id)
if err != nil {
t.Fatalf("get video: %v", err)
}
if got.DriveID != target.ID() {
t.Fatalf("drive_id = %q, want %q", got.DriveID, target.ID())
}
wantName := "Sample vk-od-001-001.mp4"
if _, ok := target.gotBodies[wantName]; !ok {
t.Fatalf("onedrive did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
}
if gotParent := target.gotParents[wantName]; gotParent != "onedrive-root/"+spider91UploadDirName {
t.Fatalf("onedrive upload parent = %q, want root/91 Spider", gotParent)
}
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
t.Fatalf("onedrive ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
}
if got.FileID != "remote-"+wantName {
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
}
if got.FileName != wantName {
t.Fatalf("file_name = %q, want %q", got.FileName, wantName)
}
if got.ContentHash == "" {
t.Fatal("content_hash should be set after onedrive migration")
}
videoPath, _ := src.VideoPath("vk-od-001.mp4")
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
t.Fatalf("local mp4 still exists after onedrive migration or stat error: %v", err)
}
thumbPath, _ := src.ThumbPath("vk-od-001.jpg")
if _, err := os.Stat(thumbPath); !os.IsNotExist(err) {
t.Fatalf("local thumb still exists after onedrive migration or stat error: %v", err)
}
}
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115 也不是 OneDrive 时,
// resolveTarget 拒绝并返回 error,让 runOnce 静默跳过(不会做破坏性变更)。
func TestResolveTargetRejectsUnsupportedKind(t *testing.T) {
cat := setupCatalog(t)
+9
View File
@@ -0,0 +1,9 @@
services:
video-site-91:
image: ghcr.io/nianzhibai/91:stable
container_name: video-site-91
ports:
- "9191:9191"
volumes:
- ./data:/opt/video-site-91/data
restart: unless-stopped
+38
View File
@@ -0,0 +1,38 @@
#!/bin/sh
set -eu
APP_DIR="/opt/video-site-91"
DATA_DIR="${VIDEO_DATA_DIR:-$APP_DIR/data}"
CONFIG="${VIDEO_CONFIG:-$DATA_DIR/config.yaml}"
EXAMPLE="$APP_DIR/config.example.yaml"
PORT="${VIDEO_LISTEN_PORT:-9191}"
mkdir -p "$DATA_DIR" "$DATA_DIR/previews" "$DATA_DIR/uploads" "$DATA_DIR/spider91"
if [ ! -f "$CONFIG" ]; then
if [ ! -f "$EXAMPLE" ]; then
echo "[entrypoint] missing config template: $EXAMPLE" >&2
exit 1
fi
mkdir -p "$(dirname "$CONFIG")"
cp "$EXAMPLE" "$CONFIG"
SECRET="$(openssl rand -hex 32)"
sed -i -E "s#^([[:space:]]*listen:[[:space:]]*).*\$#\1\"0.0.0.0:${PORT}\"#" "$CONFIG"
sed -i -E "s#^([[:space:]]*session_secret:[[:space:]]*).*\$#\1\"${SECRET}\"#" "$CONFIG"
sed -i -E "s#^([[:space:]]*db_path:[[:space:]]*).*\$#\1\"${DATA_DIR}/video-site.db\"#" "$CONFIG"
sed -i -E "s#^([[:space:]]*local_preview_dir:[[:space:]]*).*\$#\1\"${DATA_DIR}/previews\"#" "$CONFIG"
chmod 600 "$CONFIG"
echo "[entrypoint] generated $CONFIG"
else
echo "[entrypoint] using existing $CONFIG"
fi
if [ -n "${VIDEO_VERSION_FILE:-}" ] && [ -n "${VIDEO_IMAGE_VERSION:-}" ]; then
mkdir -p "$(dirname "$VIDEO_VERSION_FILE")"
printf '%s\n' "$VIDEO_IMAGE_VERSION" > "$VIDEO_VERSION_FILE"
fi
exec "$@"
+224 -23
View File
@@ -11,6 +11,12 @@ VERSION="${VERSION:-latest}"
GH_PROXY="${GH_PROXY:-}"
CONFIGURE_UFW="${CONFIGURE_UFW:-1}"
INSTALL_DEPS="${INSTALL_DEPS:-1}"
SELF_UPDATE="${SELF_UPDATE:-1}"
FORCE_UPDATE="${FORCE_UPDATE:-0}"
INSTALL_SCRIPT_REF="${INSTALL_SCRIPT_REF:-main}"
INSTALL_SCRIPT_URL="${INSTALL_SCRIPT_URL:-${GH_PROXY}https://raw.githubusercontent.com/${GITHUB_REPO}/${INSTALL_SCRIPT_REF}/install.sh}"
VIDEO_SITE_SKIP_SELF_UPDATE="${VIDEO_SITE_SKIP_SELF_UPDATE:-0}"
SERVICE_READY_TIMEOUT="${SERVICE_READY_TIMEOUT:-90}"
VERSION_FILE="$INSTALL_PATH/.version"
MANAGER_PATH="/usr/local/sbin/${APP_NAME}-manager"
COMMAND_LINK="/usr/local/bin/91"
@@ -47,7 +53,7 @@ Default action:
Actions:
install Install to $INSTALL_PATH
update Download latest release and replace program files, keeping config/data
update Refresh manager script, download latest release, and keep config/data
restart Restart service
stop Stop service
status Show service status
@@ -62,6 +68,11 @@ Options via environment:
GH_PROXY=$GH_PROXY
INSTALL_DEPS=$INSTALL_DEPS
CONFIGURE_UFW=$CONFIGURE_UFW
SELF_UPDATE=$SELF_UPDATE
FORCE_UPDATE=$FORCE_UPDATE
INSTALL_SCRIPT_REF=$INSTALL_SCRIPT_REF
INSTALL_SCRIPT_URL=$INSTALL_SCRIPT_URL
SERVICE_READY_TIMEOUT=$SERVICE_READY_TIMEOUT
Examples:
sudo bash install.sh
@@ -158,6 +169,30 @@ download_file() {
return 1
}
backup_install_files() {
local backup="$1"
mkdir -p "$backup"
cp -a "$INSTALL_PATH/server" "$backup/server"
for item in dist config.example.yaml 91VideoSpider config.yaml .version; do
if [[ -e "$INSTALL_PATH/$item" ]]; then
cp -a "$INSTALL_PATH/$item" "$backup/$item"
fi
done
}
restore_install_files() {
local backup="$1"
mkdir -p "$INSTALL_PATH"
cp -a "$backup/server" "$INSTALL_PATH/server"
for item in dist config.example.yaml 91VideoSpider config.yaml .version; do
rm -rf "${INSTALL_PATH:?}/$item"
if [[ -e "$backup/$item" ]]; then
cp -a "$backup/$item" "$INSTALL_PATH/$item"
fi
done
chmod +x "$INSTALL_PATH/server"
}
prepare_config() {
local cfg="$INSTALL_PATH/config.yaml"
local example="$INSTALL_PATH/config.example.yaml"
@@ -200,6 +235,8 @@ RestartSec=5
TimeoutStopSec=20
Environment=VIDEO_CONFIG=${INSTALL_PATH}/config.yaml
Environment=VIDEO_FRONTEND_DIR=${INSTALL_PATH}/dist
Environment=VIDEO_VERSION_FILE=${VERSION_FILE}
Environment=VIDEO_GITHUB_REPO=${GITHUB_REPO}
Environment=HOME=/root
Environment=PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
LimitNOFILE=65536
@@ -217,12 +254,72 @@ EOF
install_cli() {
local src
src="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")"
if [[ -f "$src" ]]; then
cp "$src" "$MANAGER_PATH"
chmod 755 "$MANAGER_PATH"
ln -sf "$MANAGER_PATH" "$COMMAND_LINK"
ln -sf "$MANAGER_PATH" "$APP_COMMAND_LINK"
install_cli_from_file "$src"
}
install_cli_from_file() {
local src="$1"
local tmp
[[ -f "$src" ]] || return 0
mkdir -p "$(dirname "$MANAGER_PATH")" "$(dirname "$COMMAND_LINK")" "$(dirname "$APP_COMMAND_LINK")"
tmp="${MANAGER_PATH}.tmp.$$"
cp "$src" "$tmp"
chmod 755 "$tmp"
mv "$tmp" "$MANAGER_PATH"
ln -sfn "$MANAGER_PATH" "$COMMAND_LINK"
ln -sfn "$MANAGER_PATH" "$APP_COMMAND_LINK"
}
self_update_manager() {
[[ "$SELF_UPDATE" == "1" ]] || return 1
[[ "$VIDEO_SITE_SKIP_SELF_UPDATE" != "1" ]] || return 1
[[ -n "$INSTALL_SCRIPT_URL" ]] || return 1
local tmp
tmp="$(mktemp)"
log "checking latest manager script"
if ! download_file "$INSTALL_SCRIPT_URL" "$tmp"; then
warn "manager self-update skipped: cannot download $INSTALL_SCRIPT_URL"
rm -f "$tmp"
return 1
fi
if ! bash -n "$tmp"; then
warn "manager self-update skipped: downloaded script has syntax errors"
rm -f "$tmp"
return 1
fi
if [[ -f "$MANAGER_PATH" ]] && cmp -s "$tmp" "$MANAGER_PATH"; then
rm -f "$tmp"
return 1
fi
install_cli_from_file "$tmp"
rm -f "$tmp"
log "manager script updated"
return 0
}
exec_latest_manager_update() {
local env_args=(
"VIDEO_SITE_SKIP_SELF_UPDATE=1"
"APP_NAME=$APP_NAME"
"GITHUB_REPO=$GITHUB_REPO"
"INSTALL_PATH=$INSTALL_PATH"
"SERVICE_NAME=$SERVICE_NAME"
"VERSION=$VERSION"
"GH_PROXY=$GH_PROXY"
"CONFIGURE_UFW=$CONFIGURE_UFW"
"INSTALL_DEPS=$INSTALL_DEPS"
"SELF_UPDATE=$SELF_UPDATE"
"FORCE_UPDATE=$FORCE_UPDATE"
"INSTALL_SCRIPT_REF=$INSTALL_SCRIPT_REF"
"INSTALL_SCRIPT_URL=$INSTALL_SCRIPT_URL"
"SERVICE_READY_TIMEOUT=$SERVICE_READY_TIMEOUT"
)
if [[ -n "$FRONTEND_PORT_WAS_SET" ]]; then
env_args+=("FRONTEND_PORT=$FRONTEND_PORT")
fi
exec env "${env_args[@]}" bash "$MANAGER_PATH" update
}
open_firewall_port() {
@@ -234,6 +331,55 @@ open_firewall_port() {
fi
}
listen_port_from_config() {
local cfg="$INSTALL_PATH/config.yaml"
local listen="" port
if [[ -f "$cfg" ]]; then
listen="$(sed -nE 's/^[[:space:]]*listen:[[:space:]]*"?([^" #]+)"?.*/\1/p' "$cfg" | head -n1)"
fi
port="${listen##*:}"
if [[ "$port" =~ ^[0-9]+$ ]]; then
printf '%s' "$port"
return
fi
printf '%s' "$FRONTEND_PORT"
}
service_health_url() {
printf 'http://127.0.0.1:%s/admin/api/setup' "$(listen_port_from_config)"
}
wait_for_service_ready() {
local url deadline
url="$(service_health_url)"
deadline=$((SECONDS + SERVICE_READY_TIMEOUT))
log "waiting for service at $url"
while (( SECONDS < deadline )); do
if curl -fsS --connect-timeout 2 --max-time 5 "$url" >/dev/null 2>&1; then
log "service is ready"
return 0
fi
sleep 2
done
return 1
}
restart_service_ready() {
if systemctl restart "${SERVICE_NAME}.service" && wait_for_service_ready; then
return 0
fi
warn "service did not become ready; retrying restart"
if systemctl restart "${SERVICE_NAME}.service" && wait_for_service_ready; then
return 0
fi
warn "service failed to become ready"
systemctl --no-pager --full status "${SERVICE_NAME}.service" || true
journalctl -u "${SERVICE_NAME}.service" -n 80 --no-pager || true
return 1
}
fetch_and_unpack() {
local tmp archive url root
tmp="$(mktemp -d)"
@@ -271,19 +417,63 @@ fetch_and_unpack() {
rm -rf "$tmp"
}
current_version_from_github() {
installed_version() {
if [[ -f "$VERSION_FILE" ]]; then
head -n1 "$VERSION_FILE" 2>/dev/null | tr -d '\r'
fi
}
target_version() {
if [[ "$VERSION" != "latest" ]]; then
printf '%s' "$VERSION"
return
fi
curl -fsSL "https://api.github.com/repos/${GITHUB_REPO}/releases/latest" \
local body version effective_url
body="$(curl -fsSL \
-H "Accept: application/vnd.github+json" \
-H "User-Agent: video-site-91-installer" \
"https://api.github.com/repos/${GITHUB_REPO}/releases/latest" 2>/dev/null || true)"
version="$(printf '%s\n' "$body" \
| sed -nE 's/.*"tag_name"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/p' \
| head -n1)"
if [[ -n "$version" ]]; then
printf '%s' "$version"
return
fi
effective_url="$(curl -fsSLI -o /dev/null -w '%{url_effective}' "$(download_base_url)/$(asset_name)" 2>/dev/null || true)"
printf '%s\n' "$effective_url" \
| sed -nE 's#.*/releases/download/([^/]+)/.*#\1#p' \
| head -n1
}
should_skip_update() {
[[ "$FORCE_UPDATE" != "1" ]] || return 1
local current target
current="$(installed_version)"
target="$(target_version || true)"
if [[ -z "$target" ]]; then
warn "cannot determine target version; continuing update"
return 1
fi
if [[ -z "$current" ]]; then
log "installed version: unknown"
log "target version: $target"
return 1
fi
log "installed version: $current"
log "target version: $target"
[[ "$current" == "$target" ]]
}
record_version() {
local version
version="$(current_version_from_github || true)"
version="$(target_version || true)"
[[ -n "$version" ]] || version="$VERSION"
{
echo "$version"
@@ -298,7 +488,7 @@ show_success() {
version="$(head -n1 "$VERSION_FILE" 2>/dev/null || echo unknown)"
echo
printf "${GREEN}安装完成${RESET}\n"
printf '%b安装完成%b\n' "$GREEN" "$RESET"
echo "版本:$version"
[[ -n "$local_ip" ]] && echo "局域网:http://${local_ip}:${FRONTEND_PORT}/"
[[ -n "$public_ip" ]] && echo "公网: http://${public_ip}:${FRONTEND_PORT}/"
@@ -319,38 +509,49 @@ install_app() {
write_service
install_cli
open_firewall_port
restart_service_ready || die "service failed to start"
record_version
systemctl restart "${SERVICE_NAME}.service"
show_success
}
update_app() {
check_system
[[ -f "$INSTALL_PATH/server" ]] || die "not installed at $INSTALL_PATH"
if self_update_manager; then
log "re-running update with latest manager script"
exec_latest_manager_update
fi
if should_skip_update; then
log "already up to date; skipped app update"
return 0
fi
check_disk_space
install_deps
[[ -f "$INSTALL_PATH/server" ]] || die "not installed at $INSTALL_PATH"
local backup
backup="$(mktemp -d)"
cp "$INSTALL_PATH/server" "$backup/server"
[[ -d "$INSTALL_PATH/dist" ]] && cp -R "$INSTALL_PATH/dist" "$backup/dist"
backup_install_files "$backup"
systemctl stop "${SERVICE_NAME}.service" 2>/dev/null || true
if ! fetch_and_unpack; then
if ! (fetch_and_unpack && prepare_config && write_service && install_cli); then
warn "update failed; restoring previous files"
cp "$backup/server" "$INSTALL_PATH/server"
rm -rf "$INSTALL_PATH/dist"
[[ -d "$backup/dist" ]] && cp -R "$backup/dist" "$INSTALL_PATH/dist"
restore_install_files "$backup"
systemctl start "${SERVICE_NAME}.service" 2>/dev/null || true
rm -rf "$backup"
exit 1
fi
prepare_config
write_service
install_cli
if ! restart_service_ready; then
warn "new version failed to start; restoring previous files"
restore_install_files "$backup"
restart_service_ready 2>/dev/null || true
rm -rf "$backup"
exit 1
fi
record_version
systemctl restart "${SERVICE_NAME}.service"
rm -rf "$backup"
log "updated"
}
@@ -430,7 +631,7 @@ main() {
;;
restart)
need_root "$@"
systemctl restart "${SERVICE_NAME}.service"
restart_service_ready || die "service failed to start"
;;
stop)
need_root "$@"
+829 -1531
View File
File diff suppressed because it is too large Load Diff
+4 -4
View File
@@ -15,14 +15,14 @@
"lucide-react": "0.453.0",
"react": "18.3.1",
"react-dom": "18.3.1",
"react-router-dom": "6.26.2"
"react-router-dom": "^6.30.3"
},
"devDependencies": {
"@types/react": "18.3.12",
"@types/react-dom": "18.3.1",
"@vitejs/plugin-react": "4.3.3",
"tsx": "^4.19.2",
"@vitejs/plugin-react": "^6.0.2",
"tsx": "^4.22.3",
"typescript": "5.6.3",
"vite": "5.4.10"
"vite": "^8.0.14"
}
}
+3 -1
View File
@@ -61,6 +61,7 @@ build_package() {
)
cp "$ROOT_DIR/backend/config.example.yaml" "$work/config.example.yaml"
cp "$ROOT_DIR/install.sh" "$work/install.sh"
cp -R "$ROOT_DIR/dist" "$work/dist"
mkdir -p "$work/91VideoSpider"
cp "$ROOT_DIR/91VideoSpider/spider_91porn.py" "$work/91VideoSpider/spider_91porn.py"
@@ -69,10 +70,11 @@ build_package() {
$APP_NAME $VERSION
This is a prebuilt release package.
Use install.sh from the repository to install it on a Linux server.
Use install.sh in this package or from the repository to install it on a Linux server.
EOF
chmod +x "$work/server"
chmod +x "$work/install.sh"
tar -C "$OUT_DIR/.work" -czf "$OUT_DIR/$artifact.tar.gz" "$artifact"
log "wrote $OUT_DIR/$artifact.tar.gz"
}
+45 -1
View File
@@ -1,5 +1,16 @@
import { useState } from "react";
import { NavLink, Outlet, useNavigate } from "react-router-dom";
import { HardDrive, Film, LogOut, Play, Home, Tags, Palette } from "lucide-react";
import {
HardDrive,
Film,
LogOut,
Play,
Home,
Tags,
Palette,
RefreshCw,
} from "lucide-react";
import * as api from "./api";
import { useAuth } from "./AuthContext";
import { useToast } from "./ToastContext";
@@ -7,6 +18,31 @@ export function AdminLayout() {
const { logout } = useAuth();
const navigate = useNavigate();
const { show } = useToast();
const [checkingUpdate, setCheckingUpdate] = useState(false);
async function handleCheckUpdate() {
if (checkingUpdate) return;
setCheckingUpdate(true);
try {
const result = await api.checkUpdate();
if (result.hasUpdate) {
show(
`发现新版本 ${result.latestVersion},当前 ${result.currentVersion}`,
"success"
);
return;
}
if (result.currentVersion === "unknown") {
show(`当前版本未知,GitHub 最新版本为 ${result.latestVersion}`, "info");
return;
}
show(`当前已是最新版本:${result.currentVersion}`, "success");
} catch {
show("检查更新失败,请稍后重试", "error");
} finally {
setCheckingUpdate(false);
}
}
async function handleLogout() {
try {
@@ -65,6 +101,14 @@ export function AdminLayout() {
</NavLink>
</nav>
<div className="admin-sidebar__footer">
<button
className="admin-sidebar__check-update"
onClick={handleCheckUpdate}
disabled={checkingUpdate}
>
<RefreshCw size={14} />
{checkingUpdate ? "检查中" : "检查更新"}
</button>
<button className="admin-sidebar__logout" onClick={handleLogout}>
<LogOut size={14} style={{ verticalAlign: -2, marginRight: 4 }} />
退
+76 -121
View File
@@ -26,6 +26,7 @@ const kindLabel: Record<string, string> = {
pikpak: "PikPak",
wopan: "联通沃盘",
onedrive: "OneDrive",
localstorage: "本地存储",
spider91: "91 爬虫",
};
@@ -48,7 +49,7 @@ type FormState = {
* PUT /admin/api/settings setting form state
* DriveForm
*
* = pikpak/p115 drive
* =
*/
spider91UploadDriveId: string;
};
@@ -80,10 +81,10 @@ export function DrivesPage() {
const [selectedDriveId, setSelectedDriveId] = useState<string | null>(null);
const { show } = useToast();
// 当前系统中可作为 spider91 上传目标的 drive 列表(pikpak p115)。
// 用户保存 spider91 drive 时从这里挑一个;空表示走"自动"模式
// 当前系统中可作为 spider91 上传目标的 drive 列表(pikpak p115 onedrive)。
// 用户保存 spider91 drive 时从这里挑一个;空表示本地保存不上传
const uploadTargets = useMemo(
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115"),
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115" || d.kind === "onedrive"),
[list]
);
@@ -126,7 +127,7 @@ export function DrivesPage() {
function openCreate() {
// 创建时把全局 setting 当前值带进表单,方便用户在新建第一个 spider91 drive 时
// 直接看到当前的上传目标选择(一般是空 = 自动)。
// 直接看到当前的上传目标选择(一般是空 = 本地保存)。
setForm({
...emptyForm,
spider91UploadDriveId: settings?.spider91UploadDriveId ?? "",
@@ -232,7 +233,7 @@ export function DrivesPage() {
/**
* 线Phase1 Phase2 spider91
* Phase3 spider91 202 backend
*
* 线
*/
async function handleRunNightly() {
try {
@@ -427,13 +428,13 @@ export function DrivesPage() {
)}
</div>
{/* 右栏:Teaser / 封面 与 缓存占用 */}
{/* 右栏:Teaser / 封面 / 指纹 与 缓存占用 */}
<div>
<div className="admin-detail-card">
<header className="admin-detail-card__title">
<div className="admin-detail-card__title-left">
<PlayCircle size={16} />
<span>Teaser </span>
<span></span>
</div>
<div className="admin-detail-actions-inline">
<button
@@ -481,6 +482,22 @@ export function DrivesPage() {
/>
</div>
</div>
<div className="admin-detail-row">
<span className="admin-detail-label"></span>
<div className="admin-detail-value">
<GenerationStatusLine label="指纹" status={d.fingerprintGenerationStatus} />
</div>
</div>
<div className="admin-detail-row">
<span className="admin-detail-label"></span>
<div className="admin-detail-value">
<GenerationCounts
ready={d.fingerprintReadyCount}
pending={d.fingerprintPendingCount}
failed={d.fingerprintFailedCount}
/>
</div>
</div>
</div>
<div className="admin-detail-actions">
@@ -586,7 +603,7 @@ export function DrivesPage() {
<div className="admin-empty">...</div>
) : list.length === 0 ? (
<div className="admin-card admin-empty">
/ 115 / PikPak / / OneDrive
/ 115 / PikPak / / OneDrive /
</div>
) : (
<div className="admin-drives-grid">
@@ -625,6 +642,15 @@ export function DrivesPage() {
</span>
</strong>
</div>
<div className="admin-drive-card__metric">
<span> (/)</span>
<strong>
{d.fingerprintReadyCount ?? 0}
<span style={{ fontSize: "11px", fontWeight: "normal", color: "var(--text-faint)" }}>
{" "}/ {d.fingerprintFailedCount ?? 0}
</span>
</strong>
</div>
</div>
<div className="admin-drive-card__footer">
@@ -837,6 +863,11 @@ function DriveForm({
}) {
const fields = useMemo(() => credentialFields(form.kind), [form.kind]);
const help = credentialHelp(form.kind, isEdit);
const showDirectoryFields =
form.kind !== "spider91" &&
form.kind !== "onedrive" &&
form.kind !== "localstorage" &&
form.kind !== "pikpak";
function set<K extends keyof FormState>(k: K, v: FormState[K]) {
onChange({ ...form, [k]: v });
@@ -873,13 +904,14 @@ function DriveForm({
>
<option value="p115">115 </option>
<option value="pikpak">PikPak</option>
<option value="spider91">91 </option>
<option value="onedrive">OneDrive</option>
<option value="localstorage"></option>
<option value="spider91">91 Spider</option>
<option value="quark"></option>
<option value="wopan"></option>
<option value="onedrive">OneDrive</option>
</select>
</div>
{form.kind !== "spider91" && (
{showDirectoryFields && (
<>
<div className="admin-form__row">
<label> ID</label>
@@ -953,10 +985,9 @@ function DriveForm({
* Spider91UploadTargetField spider91 drive "上传目标"
*
*
* - = "(自动)" + pikpak/p115 drive
* - "自动" value="" worker
* pikpak p115 drive
* - pikpak/p115 drive
* - = "本地保存,不上传" + pikpak/p115/onedrive drive
* - value=""
* - pikpak/p115/onedrive drive
* - setting `spider91.upload_drive_id` drive
* credentials spider91 drive
*/
@@ -969,53 +1000,20 @@ function Spider91UploadTargetField({
onChange: (v: string) => void;
uploadTargets: api.AdminDrive[];
}) {
// 文案根据系统中实际挂载的目标盘 kind 自适应:
// - 只挂了 PikPak → 文案只讲 "PikPak"
// - 只挂了 115 → 文案只讲 "115 网盘"
// - 两类都挂 → 文案讲 "PikPak / 115 网盘"
// 这样在单一类型场景下用户不会被另一类的字样干扰。
const kindsPresent = new Set(uploadTargets.map((d) => d.kind));
const hasPikPak = kindsPresent.has("pikpak");
const has115 = kindsPresent.has("p115");
const presentLabel =
hasPikPak && has115
? "PikPak / 115 网盘"
: hasPikPak
? "PikPak"
: has115
? "115 网盘"
: "PikPak 或 115 网盘";
return (
<div className="admin-form__row">
<label></label>
{uploadTargets.length === 0 ? (
<>
<select value="" disabled>
<option value=""> {presentLabel}</option>
</select>
<div className="admin-form__help">
{presentLabel} drive 91
</div>
</>
) : (
<>
<select value={value} onChange={(e) => onChange(e.target.value)}>
<option value=""> {presentLabel}</option>
{uploadTargets.map((d) => (
<option key={d.id} value={d.id}>
{kindLabel[d.kind] ?? d.kind} · {d.name || d.id}
</option>
))}
</select>
<div className="admin-form__help">
{uploadTargets.length > 1
? `如果同时挂着多个 ${presentLabel} drive"自动"模式不会工作,必须显式选定一个。`
: `当前只挂着 1 个 ${presentLabel},"自动"模式会直接选用它。`}
</div>
</>
)}
<select value={value} onChange={(e) => onChange(e.target.value)}>
<option value=""></option>
{uploadTargets.map((d) => (
<option key={d.id} value={d.id}>
{kindLabel[d.kind] ?? d.kind} · {d.name || d.id}
</option>
))}
</select>
<div className="admin-form__help">
115 PikPak OneDrive 91 Spider
</div>
</div>
);
}
@@ -1028,13 +1026,15 @@ function credentialHelp(kind: Kind, isEdit: boolean): string {
case "p115":
return `登录 115.com 后复制 Cookie,形如 "UID=...; CID=...; SEID=...; KID=..."。${note}`;
case "pikpak":
return `参考 OpenList 的 PikPak 登录方式。可填用户名和密码首次登录,也可填 refresh_token;如返回验证码链接,打开验证后把 captcha_token 粘贴回来${note}`;
return `填写 PikPak 账号和密码即可。平台、设备 ID、验证码 token 和 refresh token 会由服务端自动处理并保存${note}`;
case "wopan":
return `需要 access_token 和 refresh_token。后续会加扫码/短信登录入口,第一版只能手工粘贴。${note}`;
case "onedrive":
return `按 OpenList 默认方式,通过 api.oplist.org 在线刷新 token。只需要 refresh_token;保存会自动回写新的 access_token / refresh_token。${note}`;
return `按 OpenList 默认应用在线挂载,只需要 refresh_token;保存会自动刷新并保存 token。${note}`;
case "localstorage":
return `把服务器上的一个已有目录作为视频来源扫描。填写绝对路径,例如 /mnt/videos;系统会读取该目录及子目录中的视频,并生成封面、Teaser 和指纹。${note}`;
case "spider91":
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;它不是外部网盘,不需要填写 Cookie 或目录 ID。后续流水线会把较早的视频上传到你选择的 115 / PikPak 目标盘。";
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;它不是外部网盘,不需要填写 Cookie 或目录 ID。后续流水线会把较早的视频上传到你选择的 115 / PikPak / OneDrive 目标盘。";
default:
return "";
}
@@ -1073,42 +1073,15 @@ function credentialFields(kind: Kind): Array<{
return [
{
key: "username",
label: "用户名 / 邮箱(无 refresh_token 时必填)",
label: "用户名 / 邮箱",
placeholder: "user@example.com",
required: true,
},
{
key: "password",
label: "密码(无 refresh_token 时必填)",
label: "密码",
placeholder: "PikPak 密码",
},
{
key: "platform",
label: "platform",
placeholder: "web(可选:android / web / pc",
help: "默认 web;如果登录或直链异常,可尝试 android 或 pc。",
},
{
key: "refresh_token",
label: "refresh_token(可选)",
placeholder: "已有 token 时可直接粘贴",
multiline: true,
},
{
key: "captcha_token",
label: "captcha_token(可选)",
placeholder: "遇到验证码校验时粘贴",
multiline: true,
},
{
key: "device_id",
label: "device_id(可选)",
placeholder: "留空自动生成并保存",
},
{
key: "disable_media_link",
label: "disable_media_link",
placeholder: "true",
help: "默认 true,使用原始下载链接;填 false 可尝试使用媒体缓存链接。",
required: true,
},
];
case "wopan":
@@ -1140,34 +1113,15 @@ function credentialFields(kind: Kind): Array<{
multiline: true,
required: true,
},
];
case "localstorage":
return [
{
key: "access_token",
label: "access_token(可选)",
placeholder: "留空也可以,保存时会在线刷新",
multiline: true,
},
{
key: "api_url_address",
label: "api_url_address(可选)",
placeholder: "https://api.oplist.org/onedrive/renewapi",
help: "默认使用 OpenList 的在线刷新 API;除非你有自建兼容服务,否则留空。",
},
{
key: "region",
label: "region(可选)",
placeholder: "global(可选:global / cn / us / de",
help: "默认 global;世纪互联填 cn,美国政府云填 us,德国云填 de。",
},
{
key: "is_sharepoint",
label: "is_sharepoint(可选)",
placeholder: "false",
help: "普通 OneDrive 留空或 falseSharePoint 文档库填 true,并填写 site_id。",
},
{
key: "site_id",
label: "site_idSharePoint 必填)",
placeholder: "SharePoint site id",
key: "path",
label: "本地目录路径",
placeholder: "/mnt/videos",
required: true,
help: "路径必须是后端服务器上的已有目录;保存后可手动重扫,系统会递归扫描支持的视频格式。",
},
];
case "spider91":
@@ -1178,6 +1132,7 @@ function credentialFields(kind: Kind): Array<{
function defaultRootId(kind: Kind): string {
if (kind === "pikpak") return "";
if (kind === "onedrive") return "root";
if (kind === "localstorage") return "/";
if (kind === "spider91") return "/";
return "0";
}
+22 -6
View File
@@ -61,11 +61,23 @@ export function me() {
return request<{ authenticated: boolean }>("/me");
}
export type UpdateCheck = {
currentVersion: string;
latestVersion: string;
hasUpdate: boolean;
releaseUrl?: string;
checkedAt: string;
};
export function checkUpdate() {
return request<UpdateCheck>("/update/check");
}
// ---------- Drives ----------
export type AdminDrive = {
id: string;
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "spider91";
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "localstorage" | "spider91";
name: string;
rootId: string;
scanRootId: string;
@@ -84,12 +96,16 @@ export type AdminDrive = {
lastCrawlAt?: number;
thumbnailGenerationStatus?: DriveGenerationStatus;
previewGenerationStatus?: DriveGenerationStatus;
fingerprintGenerationStatus?: DriveGenerationStatus;
thumbnailReadyCount: number;
thumbnailPendingCount: number;
thumbnailFailedCount: number;
teaserReadyCount: number;
teaserPendingCount: number;
teaserFailedCount: number;
fingerprintReadyCount: number;
fingerprintPendingCount: number;
fingerprintFailedCount: number;
};
export type DriveGenerationStatus = {
@@ -121,7 +137,7 @@ export function getDriveStorage() {
export type UpsertDriveInput = {
id: string;
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "spider91";
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "localstorage" | "spider91";
name: string;
rootId: string;
scanRootId: string;
@@ -324,9 +340,9 @@ export type Theme = "dark" | "pink";
export type Settings = {
theme: Theme;
/**
* spider91 drive ID pikpak p115 drive
* - pikpak/p115 drive
* - drive kind {pikpak, p115}
* spider91 drive ID pikpakp115 onedrive drive
* -
* - drive kind {pikpak, p115, onedrive}
*/
spider91UploadDriveId: string;
};
@@ -355,7 +371,7 @@ export function updateSettings(body: Partial<Settings>) {
* 线Phase1 + Phase2 91 + Phase3
* 202 backend
*
* 线
* 线
*/
export function runNightlyJob() {
return request<{ ok: boolean }>("/jobs/nightly/run", { method: "POST" });
+1
View File
@@ -294,5 +294,6 @@ function sourceKindFromLabel(label: string): string {
if (value.includes("pikpak")) return "pikpak";
if (value.includes("沃盘") || value.includes("wopan") || value.includes("联通")) return "wopan";
if (value.includes("onedrive") || value.includes("one drive")) return "onedrive";
if (value.includes("本地") || value.includes("localstorage") || value.includes("local storage")) return "localstorage";
return "";
}
+2
View File
@@ -72,5 +72,7 @@ function sourceKindFromLabel(label: string): string {
if (value.includes("沃盘") || value.includes("wopan") || value.includes("联通"))
return "wopan";
if (value.includes("onedrive") || value.includes("one drive")) return "onedrive";
if (value.includes("本地") || value.includes("localstorage") || value.includes("local storage"))
return "localstorage";
return "";
}
+1
View File
@@ -1287,6 +1287,7 @@ function getDriveShortName(source: string): string {
if (s.includes("quark") || s.includes("夸克")) return "Quak";
if (s.includes("onedrive")) return "OneDrive";
if (s.includes("wopan") || s.includes("沃盘")) return "沃盘";
if (s.includes("localstorage") || s.includes("本地")) return "本地";
if (s.includes("spider") || s.includes("爬虫")) return "爬虫";
return source.substring(0, 4);
}
+36
View File
@@ -121,10 +121,45 @@
border-top: 1px solid var(--border-subtle);
}
.admin-sidebar__check-update {
display: inline-flex;
align-items: center;
gap: 6px;
width: 100%;
min-height: 34px;
padding: 8px 12px;
border-radius: var(--radius-sm);
color: var(--text-muted);
font-size: var(--font-sm);
font-weight: var(--weight-medium);
text-align: left;
}
.admin-sidebar__check-update:hover:not(:disabled) {
color: var(--text-strong);
background: rgba(255, 255, 255, 0.04);
}
.admin-sidebar__check-update:disabled {
opacity: 0.6;
cursor: wait;
}
.admin-sidebar__check-update:disabled svg {
animation: admin-update-spin 0.9s linear infinite;
}
@keyframes admin-update-spin {
to {
transform: rotate(360deg);
}
}
.admin-sidebar__logout {
display: inline-flex;
align-items: center;
gap: 6px;
width: 100%;
padding: 8px 12px;
margin-top: 8px;
border-radius: var(--radius-sm);
@@ -1790,6 +1825,7 @@
.admin-drive-card__brand-icon[data-kind="pikpak"] { background: var(--drive-pikpak); }
.admin-drive-card__brand-icon[data-kind="wopan"] { background: var(--drive-wopan); }
.admin-drive-card__brand-icon[data-kind="onedrive"] { background: var(--drive-onedrive); }
.admin-drive-card__brand-icon[data-kind="localstorage"] { background: var(--drive-localstorage); }
.admin-drive-card__brand-icon[data-kind="spider91"] { background: var(--accent); }
.admin-drive-card__info {
+2
View File
@@ -134,6 +134,7 @@
--drive-pikpak: #8a6dff;
--drive-wopan: #ff8a3c;
--drive-onedrive: #4cabea;
--drive-localstorage: #35b88f;
/* ----- 阴影 ----- */
--shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.4);
@@ -213,6 +214,7 @@
--drive-pikpak: #8466e6;
--drive-wopan: #e57a36;
--drive-onedrive: #2f95cf;
--drive-localstorage: #239978;
/* ----- 阴影(粉色柔投影 + 一层中性) ----- */
--shadow-sm: 0 1px 2px rgba(180, 90, 120, 0.08);
+9
View File
@@ -306,6 +306,14 @@
--drive-shadow: rgba(76, 171, 234, 0.15);
--drive-shadow-strong: rgba(76, 171, 234, 0.4);
}
.source-badge[data-kind="localstorage"] {
--drive-color: var(--drive-localstorage);
--drive-bg: rgba(53, 184, 143, 0.12);
--drive-text: #a8efdc;
--drive-border: rgba(53, 184, 143, 0.35);
--drive-shadow: rgba(53, 184, 143, 0.15);
--drive-shadow-strong: rgba(53, 184, 143, 0.4);
}
/* ----- Preview overlay ----- */
.preview-loader {
@@ -441,6 +449,7 @@
.video-card__source[data-kind="pikpak"] { --source-color: var(--drive-pikpak); }
.video-card__source[data-kind="wopan"] { --source-color: var(--drive-wopan); }
.video-card__source[data-kind="onedrive"] { --source-color: var(--drive-onedrive); }
.video-card__source[data-kind="localstorage"] { --source-color: var(--drive-localstorage); }
/* =========================================================
* Empty / Loading / Skeleton
+6
View File
@@ -351,6 +351,12 @@
color: var(--drive-onedrive);
}
.vd-meta__chip[data-tone="localstorage"] {
background: rgba(53, 184, 143, 0.14);
border-color: rgba(53, 184, 143, 0.3);
color: var(--drive-localstorage);
}
/* ---------- Actions toolbar ---------- */
.vd-actions {
display: flex;
+81
View File
@@ -13,3 +13,84 @@ test("spider91 drive form does not expose advanced crawler credentials", () => {
assert.doesNotMatch(drivesPageSource, /python_path/);
assert.doesNotMatch(drivesPageSource, /script_path/);
});
test("spider91 upload target uses explicit local-save option instead of auto target", () => {
assert.match(drivesPageSource, /本地保存,不上传/);
assert.match(
drivesPageSource,
/d\.kind === "pikpak" \|\| d\.kind === "p115" \|\| d\.kind === "onedrive"/
);
assert.doesNotMatch(drivesPageSource, /自动:唯一/);
assert.doesNotMatch(drivesPageSource, /自动模式/);
});
test("onedrive drive form only exposes required default-app fields", () => {
assert.match(
drivesPageSource,
/form\.kind !== "spider91" &&\s*form\.kind !== "onedrive" &&\s*form\.kind !== "localstorage" &&\s*form\.kind !== "pikpak"/
);
const match =
/function credentialFields[\s\S]*?case "onedrive":\s*return \[([\s\S]*?)\];\s*case "spider91":/.exec(
drivesPageSource
);
assert.ok(match, "onedrive credential field block should be present");
const fields = match[1];
assert.match(fields, /key: "refresh_token"/);
assert.doesNotMatch(fields, /key: "access_token"/);
assert.doesNotMatch(fields, /key: "api_url_address"/);
assert.doesNotMatch(fields, /key: "region"/);
assert.doesNotMatch(fields, /key: "is_sharepoint"/);
assert.doesNotMatch(fields, /key: "site_id"/);
});
test("pikpak drive form only exposes account login fields", () => {
const match =
/case "pikpak":\s*return \[([\s\S]*?)\];\s*case "wopan":/.exec(
drivesPageSource
);
assert.ok(match, "pikpak credential field block should be present");
const fields = match[1];
assert.match(fields, /key: "username"/);
assert.match(fields, /key: "password"/);
assert.doesNotMatch(fields, /key: "platform"/);
assert.doesNotMatch(fields, /key: "refresh_token"/);
assert.doesNotMatch(fields, /key: "captcha_token"/);
assert.doesNotMatch(fields, /key: "device_id"/);
assert.doesNotMatch(fields, /key: "disable_media_link"/);
});
test("localstorage drive form asks for a server directory path", () => {
assert.match(drivesPageSource, /<option value="localstorage">本地存储<\/option>/);
const match =
/case "localstorage":\s*return \[([\s\S]*?)\];\s*case "spider91":/.exec(
drivesPageSource
);
assert.ok(match, "localstorage credential field block should be present");
const fields = match[1];
assert.match(fields, /key: "path"/);
assert.match(fields, /label: "本地目录路径"/);
assert.match(drivesPageSource, /if \(kind === "localstorage"\) return "\/"/);
});
test("drive type selector keeps primary source order", () => {
const options = Array.from(
drivesPageSource.matchAll(/<option value="([^"]+)">([^<]+)<\/option>/g),
(match) => ({ value: match[1], label: match[2] })
);
const driveOptions = options.slice(0, 7);
assert.deepEqual(driveOptions, [
{ value: "p115", label: "115 网盘" },
{ value: "pikpak", label: "PikPak" },
{ value: "onedrive", label: "OneDrive" },
{ value: "localstorage", label: "本地存储" },
{ value: "spider91", label: "91 Spider" },
{ value: "quark", label: "夸克网盘" },
{ value: "wopan", label: "联通沃盘" },
]);
});