12 Commits

Author SHA1 Message Date
nianzhibai 9ada39debb chore: deploy Docker Compose from stable release image 2026-05-31 10:04:56 +08:00
nianzhibai 24d1244bc3 fix: detect Docker image version for update checks 2026-05-31 09:55:15 +08:00
nianzhibai 0dd0c45509 Update README with service restart instruction
Add note about restarting service on first access.
2026-05-30 20:26:29 +08:00
nianzhibai ef6eadd0a6 Update README.md 2026-05-30 20:17:18 +08:00
nianzhibai 2c5a3342cc feat: prepare v0.0.4 storage release 2026-05-30 20:02:02 +08:00
nianzhibai 7ace5f8bc7 feat: probe video duration during thumbnail generation 2026-05-30 18:30:22 +08:00
nianzhibai 66b33b2a31 feat: support spider91 uploads to OneDrive 2026-05-30 18:04:15 +08:00
nianzhibai 27aefc870f feat: improve media generation pipeline status 2026-05-30 17:37:31 +08:00
nianzhibai 02d82e9a62 Add Docker Compose deployment support 2026-05-30 11:09:04 +08:00
nianzhibai 492431164b Improve fingerprint dedupe maintenance 2026-05-29 23:58:36 +08:00
nianzhibai 051f1555d5 Add sampled fingerprint deduplication 2026-05-29 23:19:52 +08:00
nianzhibai 85292ea095 Simplify OneDrive setup and redirect playback 2026-05-29 22:35:02 +08:00
50 changed files with 4106 additions and 452 deletions
+21
View File
@@ -0,0 +1,21 @@
.git
.github
.gitattributes
.gitignore
node_modules
dist
release
data
backend/data
backend/config.yaml
config.yaml
*.db
*.sqlite
*.sqlite3
*.log
*.tmp
tests
video-site-implementation-plan.md
+82
View File
@@ -0,0 +1,82 @@
name: Docker
on:
push:
branches:
- main
tags:
- "v*"
pull_request:
branches:
- main
workflow_dispatch:
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
permissions:
contents: read
packages: write
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to GHCR
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
type=ref,event=branch
type=ref,event=tag
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha,prefix=sha-
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
- name: Determine image version
id: version
shell: bash
run: |
if [[ "$GITHUB_REF" == refs/tags/v* ]]; then
version="$GITHUB_REF_NAME"
else
version="$(git describe --tags --always --dirty 2>/dev/null || git rev-parse --short=12 HEAD)"
fi
echo "version=$version" >> "$GITHUB_OUTPUT"
- name: Build and push Docker image
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm64
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
VERSION=${{ steps.version.outputs.version }}
cache-from: type=gha
cache-to: type=gha,mode=max
+66
View File
@@ -0,0 +1,66 @@
# ---- Stage 1: Build frontend ----
FROM node:20-slim AS frontend
WORKDIR /app
COPY package.json package-lock.json ./
RUN npm ci
COPY tsconfig.json vite.config.ts index.html ./
COPY public/ public/
COPY src/ src/
RUN npm run build
# ---- Stage 2: Build backend ----
FROM golang:1.23-bookworm AS backend
WORKDIR /app/backend
COPY backend/go.mod backend/go.sum ./
COPY backend/vendor/ vendor/
COPY backend/cmd/ cmd/
COPY backend/internal/ internal/
RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server
# ---- Stage 3: Runtime ----
FROM debian:bookworm-slim AS runtime
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
curl \
ffmpeg \
openssl \
python3 \
python3-bs4 \
python3-lxml \
python3-requests \
tar \
tzdata \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /opt/video-site-91
COPY --from=backend /out/server ./server
COPY --from=frontend /app/dist ./dist
COPY backend/config.example.yaml ./config.example.yaml
COPY 91VideoSpider/ ./91VideoSpider/
COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
ARG VERSION=dev
ENV VIDEO_CONFIG=/opt/video-site-91/data/config.yaml \
VIDEO_FRONTEND_DIR=/opt/video-site-91/dist \
VIDEO_GITHUB_REPO=nianzhibai/91 \
VIDEO_IMAGE_VERSION=${VERSION} \
VIDEO_LISTEN_PORT=9191 \
VIDEO_VERSION_FILE=/opt/video-site-91/data/.version
RUN chmod +x ./server /usr/local/bin/docker-entrypoint.sh
VOLUME ["/opt/video-site-91/data"]
EXPOSE 9191
ENTRYPOINT ["docker-entrypoint.sh"]
CMD ["./server"]
+128 -56
View File
@@ -5,28 +5,27 @@
</p>
<p align="center">
😄个人 91 站😄
😄 个人私有视频站 😄
</p>
<p align="center">
<a href="#快速开始">快速开始</a> ·
<a href="#功能特性">功能特性</a> ·
<a href="#预览图">预览图</a> ·
<a href="#数据存放位置">数据目录</a>
</p>
---
## 项目说明
## 功能特性
支持 115 云盘、PikPak 云盘作为视频播放后端。
采用 115 云盘和 PikPak 云盘的 302 重定向播放,不占用服务器带宽,也不会因为服务器带宽小而影响视频播放体验。
服务器只负责扫描云盘中的视频文件,并给每个视频生成封面图和预览片段。
你可以通过封面图和预览片段,在首页快速选择想看的视频。
支持 91 爬虫,爬取 91 的本月最热视频。
内置两种主题:黑黄主题(91 经典主题)和粉白主题。
支持短视频模式,一键切换成熟悉的抖音模式。
该项目2C2G服务器稳定跑👍👍👍
- **多后端支持** — 兼容 115 云盘、PikPak 云盘、OneDrive和本地存储
- **零带宽消耗** — 云盘视频采用 302 重定向播放,不占用服务器带宽
- **封面 & 预览片段** — 自动为每个视频生成封面图和预览片段,首页快速选片
- **91 爬虫** — 内置爬虫,支持抓取 91 本月最热视频
- **双主题** — 黑黄经典主题 / 粉白清新主题,随时切换
- **短视频模式** — 一键切换抖音风格,沉浸刷片
- **低资源占用** — 2C2G 服务器稳定运行,主要性能消耗就是封面图和预览视频的生成
---
@@ -35,100 +34,173 @@
### 电脑端
<p>
<img width="49%" alt="91 电脑端首页" src="https://github.com/user-attachments/assets/9808fceb-760b-4dd5-b7d2-8622b95b90d5" />
<img width="49%" alt="91 电脑端播放页" src="https://github.com/user-attachments/assets/859db4aa-1fba-44f2-bb46-1db07c2f964f" />
<img width="49%" alt="首页" src="https://github.com/user-attachments/assets/9808fceb-760b-4dd5-b7d2-8622b95b90d5" />
<img width="49%" alt="播放页" src="https://github.com/user-attachments/assets/859db4aa-1fba-44f2-bb46-1db07c2f964f" />
</p>
<p>
<img width="49%" alt="91 电脑端主题" src="https://github.com/user-attachments/assets/96bea37a-8764-413e-9b70-1856b4ae0cd2" />
<img width="49%" alt="91 电脑端管理页" src="https://github.com/user-attachments/assets/29c1e27a-7651-4dfc-93dd-556331844214" />
<img width="49%" alt="主题切换" src="https://github.com/user-attachments/assets/96bea37a-8764-413e-9b70-1856b4ae0cd2" />
<img width="49%" alt="管理页" src="https://github.com/user-attachments/assets/29c1e27a-7651-4dfc-93dd-556331844214" />
</p>
### 手机端
<p align="center">
<img width="1284" height="1134" alt="PixPin_2026-05-29_11-54-12" src="https://github.com/user-attachments/assets/bdb7a86c-a4e5-483e-a307-e02c0bb34dac" />
<img width="1284" height="1134" alt="手机端" src="https://github.com/user-attachments/assets/bdb7a86c-a4e5-483e-a307-e02c0bb34dac" />
</p>
---
## 快速开始
一键安装:
### 方式一:一键安装脚本(推荐)
```bash
sudo apt update
sudo apt install -y curl ca-certificates
sudo apt update && sudo apt install -y curl ca-certificates
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/install.sh -o install.sh
sudo bash install.sh
```
部署完成后访问:
- 前台:`http://服务器IP:9191/`
- 后台:`http://服务器IP:9191/admin`
| 地址 | 说明 |
|------|------|
| `http://服务器IP:9191/` | 前台 |
| `http://服务器IP:9191/admin` | 后台管理 |
安装后会自动创建 `91` 指令:
**注意:如果首次访问,显示502,可以运行 `91 restart` 重启一下服务**
安装后自动注册 `91` 管理命令:
```bash
91 # 打开管理菜单
91 status # 查看状态
91 logs # 查看日志
91 update # 更新
91 restart # 重启
91 stop # 停止
91 # 打开管理菜单
91 status # 查看运行状态
91 logs # 查看日志
91 update # 更新到最新版本
91 restart # 重启服务
91 stop # 停止服务
```
同时也保留 `video-site-91` 作为同等别名
> `video-site-91` 为等效别名,两者可互换使用
**旧版本用户升级说明**
**自定义端口**
如果你是在 `v0.0.2` 之前部署的项目,系统里可能还保留旧的 `91` 管理脚本。旧脚本直接运行 `91 update` 可能更新失败。先执行下面的一次性修复命令,后续再使用 `91 update` 即可:
```bash
FRONTEND_PORT=8080 sudo -E bash install.sh
```
**旧版本升级(v0.0.2 之前):**
旧版脚本直接执行 `91 update` 可能失败,先执行以下修复命令:
```bash
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/install.sh -o /tmp/install-91.sh
sudo bash /tmp/install-91.sh update
```
想换端口:
---
### 方式二:Docker Compose 部署
**1. 准备目录**
```bash
FRONTEND_PORT=8080 sudo -E bash install.sh
mkdir -p video-site-91 && cd video-site-91
```
**2. 创建 `docker-compose.yml`**
```yaml
services:
video-site-91:
image: ghcr.io/nianzhibai/91:stable
container_name: video-site-91
ports:
- "9191:9191"
volumes:
- ./data:/opt/video-site-91/data
restart: unless-stopped
```
`stable` 只会在发布 `v*` 正式 Release 时更新,不会跟随 `main` 分支开发镜像变化。
升级到最新正式版:
```bash
docker compose pull
docker compose up -d
```
如果想固定某个 Release 版本,可以改成明确的 tag,例如:
```yaml
image: ghcr.io/nianzhibai/91:v0.0.4
```
或直接拉取仓库内置配置:
```bash
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/docker-compose.yml -o docker-compose.yml
```
**3. 启动**
```bash
docker compose up -d
```
**常用命令:**
```bash
docker compose logs -f # 查看日志
docker compose pull # 拉取最新正式版 stable 镜像
docker compose up -d # 更新并重启
```
> 所有配置、数据库、封面、预览及上传文件均保存在 `./data/` 目录下。
---
## 数据存放位置
项目会把运行数据保存在本地:
### 一键脚本部署
- `/opt/video-site-91/config.yaml`:本地配置、管理员账号、网盘凭证。
- `/opt/video-site-91/data/video-site.db`SQLite 数据库。
- `/opt/video-site-91/data/previews/`:本地生成的封面和 teaser。
| 路径 | 内容 |
|------|------|
| `/opt/video-site-91/config.yaml` | 配置文件、管理员账号、网盘凭证 |
| `/opt/video-site-91/data/video-site.db` | SQLite 数据库 |
| `/opt/video-site-91/data/previews/` | 封面图和预览片段 |
### Docker Compose 部署
| 路径 | 内容 |
|------|------|
| `./data/config.yaml` | 配置文件、管理员账号、网盘凭证 |
| `./data/video-site.db` | SQLite 数据库 |
| `./data/previews/` | 封面图和预览片段 |
| `./data/uploads/` | 本地上传的视频文件 |
| `./data/spider91/` | 91 爬虫抓取的视频文件 |
---
## 了解更多
## 更多文档
根目录 README 只保留项目介绍和最短上手路径。更细的实现、接口、网盘字段和部署方式可以看:
- [backend/README.md](backend/README.md)
- [video-site-implementation-plan.md](video-site-implementation-plan.md)
| 文档 | 内容 |
|------|------|
| [backend/README.md](backend/README.md) | 后端实现、接口说明、网盘字段 |
| [video-site-implementation-plan.md](video-site-implementation-plan.md) | 完整实现方案 |
---
## 使用边界
## 使用须知
这个项目面向个人私有部署。请只接入你有权访问和管理的内容,并遵守对应网盘、站点服务条款及所在地法律法规。
项目面向**个人私有部署**,请仅接入你有权访问和管理的内容,并遵守对应网盘、站点服务条款及所在地法律法规。
不要传播,仅限个人使用,个人视频站
> 不对外传播,仅限个人使用。
---
## 致谢
感谢开源项目 OpenList。
感谢 <a href="https://linux.do/">LinuxDo</a> 社区,学 AIL
感谢 <a href="https://nodeseek.com/">NodeSeek</a> 社区,MJJ 上 N 站。
- [OpenList](https://github.com/OpenListTeam/OpenList) — 优秀的开源项目
- [LinuxDo](https://linux.do/) — 学 AI 上 L 站
- [NodeSeek](https://nodeseek.com/) — MJJN
+19 -5
View File
@@ -2,7 +2,7 @@
视频聚合站的 Go 后端。提供三件事:
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive / 本地存储
2. 视频元数据目录(SQLite+ 扫描 + teaser 预生成
3. REST API(前台)+ 管理后台 + 直链代理
4. 标签池、视频隐藏、按网盘统计和详情页来源网盘类型展示能力
@@ -21,6 +21,7 @@ internal/
pikpak/ PikPak(自己实现,参考 OpenList pikpak
wopan/ 联通沃盘(壳子 + OpenListTeam/wopan-sdk-go
onedrive/ OneDriveOpenList 在线续期 + Microsoft Graph 文件接口)
localstorage/ 本地目录扫描(服务器已有视频目录)
scanner/ 扫目录 → 落库
preview/ ffmpeg 抽封面和生成多段 teaser
proxy/ /p/stream/*、/p/preview/* 代理
@@ -105,9 +106,10 @@ go run ./cmd/server 后端 9192
|--------|---------------------------------------------------------------|
| quark | `cookie` |
| p115 | `cookie`(形如 `UID=...; CID=...; SEID=...; KID=...` |
| pikpak | `username`、`password`,可选 `refresh_token`、`captcha_token`、`device_id`、`platform`、`disable_media_link` |
| pikpak | `username`、`password`(token、验证码和设备 ID 由服务端自动处理并保存) |
| wopan | `access_token`、`refresh_token`,可选 `family_id` |
| onedrive | `refresh_token`,可选 `access_token`、`api_url_address`、`region`、`is_sharepoint`、`site_id` |
| onedrive | `refresh_token` |
| localstorage | `path`(服务器上的已有视频目录,如 `/mnt/videos` |
### PikPak 速度说明
@@ -115,7 +117,7 @@ go run ./cmd/server 后端 9192
当前服务器同时存在 sing-box TUN 透明代理,PikPak 默认出站会被 `tun0` 接管;但强制直连物理网卡并没有更快,慢速的主要差异来自 PikPak 取链方式。media/cache CDN 节点仍有波动,偶尔可能遇到慢节点;如果播放变慢,可重新获取直链或重新挂载 PikPak 后再测。
OneDrive 按 OpenList 默认方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。OpenList 代刷得到的 refresh token 可以直接填到本项目。普通 OneDrive 的 `rootId` / `scanRootId` 可填 `root`SharePoint 文档库需要额外设置 `is_sharepoint=true` 和 `site_id`
OneDrive 按 OpenList 默认应用方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。后台新建 OneDrive 时只需要填 OpenList 代刷得到的 `refresh_token`;服务端会默认挂载根目录并自动回写新 token
## 文件名约定
@@ -128,6 +130,18 @@ OneDrive 按 OpenList 默认方式调用 `https://api.oplist.org/onedrive/renewa
标签分隔符支持 `, ` 和空格。解析结果会和系统标签池匹配,常见番号类噪声会归并到 `AV` 等系统标签,避免把每个番号都变成独立标签。解析结果可在管理后台覆盖。
## 视频去重
项目有三层去重:
1. 同一网盘同一文件按 `(drive_id, file_id)` 形成稳定视频 ID,重复扫描只更新同一行。
2. 扫描时优先按网盘侧 `content_hash` 去重;没有 hash 时退化为 `file_name + size_bytes`。
3. 扫描、爬虫、本地上传或服务启动挂载网盘后,后台指纹 worker 会异步读取视频的少量 Range 片段,生成 `sampled_sha256`。前台列表、首页、搜索、推荐会按 `size_bytes + sampled_sha256` 只展示最早入库的 canonical 视频。
`sampled_sha256` 是文件级去重:适合识别同一个视频文件被复制到 115 / PikPak / OneDrive 等不同网盘的情况。它不会删除任何网盘文件,也不用于识别转码、裁剪、加水印后的同源视频。
封面和 teaser 仍然优先生成,不等待指纹完成。夜间流水线最后会做一次重复资产清理:对 `size_bytes + sampled_sha256` 命中的非 canonical 视频,只删除本机生成的重复封面和 teaser,并把对应字段重置为 `pending`。网盘原文件和视频元数据记录不会被删除;如果 canonical 视频以后被移除,这些重复项会重新进入生成队列。
## 管理能力
- `/admin/drives`:新增、编辑、删除网盘,触发扫描。
@@ -147,7 +161,7 @@ ffmpeg -ss <起点> -headers "UA/Cookie/Referer" -i <直链> \
当前策略是每段固定 3 秒;30 秒以下最多 3 段,30 秒及以上固定 4 段;长视频在 20% 到 80% 区间均匀取段。生成的 teaser 和封面都只保存在本地 `data/previews/`,不会回写到网盘;旧数据中的 `preview_file_id` 会被忽略。
服务启动或网盘重新挂载时,如果 Teaser 开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 直链生成 teaser 时可能触发 Microsoft 429 限流;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。
服务启动或网盘重新挂载时,如果 Teaser 开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 扫盘和直链生成 teaser / 封面时可能触发 Microsoft Graph 429、`TooManyRequests`、`activityLimitReached` 或 throttled 文本;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。扫盘阶段会按 `Retry-After` 或默认冷却时间等待后继续当前目录。
前端卡片的 `previewSrc` 统一指向 `/p/preview/<videoID>`,后端只从本地 `preview_local` 文件读取。
+401 -94
View File
@@ -25,6 +25,7 @@ import (
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/config"
"github.com/video-site/backend/internal/drives"
"github.com/video-site/backend/internal/drives/localstorage"
"github.com/video-site/backend/internal/drives/localupload"
"github.com/video-site/backend/internal/drives/onedrive"
"github.com/video-site/backend/internal/drives/p115"
@@ -32,6 +33,7 @@ import (
"github.com/video-site/backend/internal/drives/quark"
"github.com/video-site/backend/internal/drives/spider91"
"github.com/video-site/backend/internal/drives/wopan"
"github.com/video-site/backend/internal/fingerprint"
"github.com/video-site/backend/internal/nightly"
"github.com/video-site/backend/internal/preview"
"github.com/video-site/backend/internal/proxy"
@@ -39,6 +41,8 @@ import (
"github.com/video-site/backend/internal/spider91migrate"
)
const fingerprintReconcileInterval = time.Minute
func main() {
cfgPath := "./config.yaml"
if v := os.Getenv("VIDEO_CONFIG"); v != "" {
@@ -63,12 +67,13 @@ func main() {
defer cat.Close()
app := &App{
cfg: cfg,
cat: cat,
registry: proxy.NewRegistry(),
workers: make(map[string]*preview.Worker),
thumbWorkers: make(map[string]*preview.ThumbWorker),
spider91Crawlers: make(map[string]*spider91.Crawler),
cfg: cfg,
cat: cat,
registry: proxy.NewRegistry(),
workers: make(map[string]*preview.Worker),
thumbWorkers: make(map[string]*preview.ThumbWorker),
fingerprintWorkers: make(map[string]*fingerprint.Worker),
spider91Crawlers: make(map[string]*spider91.Crawler),
}
app.proxy = proxy.New(app.registry)
app.spider91Migrator = spider91migrate.New(spider91migrate.Config{
@@ -77,7 +82,8 @@ func main() {
GetTargetDriveID: func() string { return app.Spider91UploadDriveID() },
})
// 初始化现有 drives
// 初始化本地内置盘;外部云盘放到 HTTP 服务启动后异步挂载,避免上游
// 登录态校验拖慢端口监听。
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -86,16 +92,7 @@ func main() {
if err := app.attachLocalUpload(ctx); err != nil {
log.Printf("[local-upload] attach failed: %v", err)
}
existing, err := cat.ListDrives(ctx)
if err != nil {
log.Fatalf("list drives: %v", err)
}
for _, d := range existing {
if err := app.attachDrive(ctx, d); err != nil {
log.Printf("[drive %s] attach failed: %v", d.ID, err)
}
}
go app.runFingerprintReconciler(ctx)
authr := &auth.Authenticator{
Username: cfg.Server.Admin.Username,
@@ -128,6 +125,7 @@ func main() {
Catalog: cat,
Auth: authr,
VersionFilePath: versionFilePath,
ImageVersion: strings.TrimSpace(os.Getenv("VIDEO_IMAGE_VERSION")),
GitHubRepo: githubRepo,
SetupRequired: func() bool {
setupMu.Lock()
@@ -240,6 +238,7 @@ func main() {
RunSpider91Crawl: app.runSpider91Crawl,
WaitPreviewQueuesIdle: app.waitAllPreviewQueuesIdle,
RunMigration: app.spider91Migrator.RunOnce,
RunDedupeAssetCleanup: app.cleanupDuplicateVideoAssets,
})
go app.nightlyRunner.Run(ctx)
@@ -253,6 +252,7 @@ func main() {
log.Fatalf("server error: %v", err)
}
}()
go app.attachExistingDrives(ctx)
// 等待退出信号
sigs := make(chan os.Signal, 1)
@@ -272,20 +272,25 @@ type App struct {
registry *proxy.Registry
proxy *proxy.Proxy
mu sync.Mutex
workers map[string]*preview.Worker
thumbWorkers map[string]*preview.ThumbWorker
cancels map[string]context.CancelFunc
mu sync.Mutex
workers map[string]*preview.Worker
thumbWorkers map[string]*preview.ThumbWorker
fingerprintWorkers map[string]*fingerprint.Worker
cancels map[string]context.CancelFunc
// spider91Crawlers 按 driveID 索引,每个 spider91 drive 独立一个 Crawler
spider91Crawlers map[string]*spider91.Crawler
// driveAttachMu 串行化云盘挂载/重挂载。挂载会访问上游服务,可能较慢;
// 串行化可以避免启动后台挂载和手动扫盘按需挂载同一个 drive 时重复创建 worker。
driveAttachMu sync.Mutex
// 全站主题("dark" | "pink"),从 DB 读
theme string
// 显式指定的 spider91 上传目标 drive ID。
// 空字符串表示本地保存不上传,不再自动挑选 pikpak/p115 drive。
// 空字符串表示本地保存不上传,不再自动挑选 pikpak/p115/onedrive drive。
spider91UploadDriveID string
// spider91Migrator 周期把 spider91 视频上传到目标 drivePikPak 或 115)。
// spider91Migrator 周期把 spider91 视频上传到目标 drivePikPak、115 或 OneDrive)。
spider91Migrator *spider91migrate.Migrator
// nightlyRunner 是凌晨流水线调度器:每天 cron_hour 串行跑扫盘 → 91 爬虫 → 迁移。
@@ -307,6 +312,11 @@ type App struct {
// scanQueued 跟踪哪些 driveID 已经排队或正在跑,去重后续重复点击。
// 一个 drive 在 scheduleScan 入队时被加入,在 runScan goroutine 结束时被移除。
scanQueued map[string]bool
// fingerprintQueueing 去重每个 drive 的 pending 指纹补队列任务,避免定时
// reconcile 和扫盘结束同时为同一批 pending 视频启动多个长时间入队 goroutine。
fingerprintQueueMu sync.Mutex
fingerprintQueueing map[string]bool
}
// teaserEnabledForDrive 查询某个 drive 当前的 per-drive teaser 开关。
@@ -371,7 +381,7 @@ func (a *App) loadTheme(ctx context.Context) {
}
// Spider91UploadDriveID 返回当前配置的 spider91 上传目标 drive ID。
// 空字符串表示本地保存不上传;只有管理员显式选择 pikpak/p115 drive 时才迁移上传。
// 空字符串表示本地保存不上传;只有管理员显式选择 pikpak/p115/onedrive drive 时才迁移上传。
func (a *App) Spider91UploadDriveID() string {
a.mu.Lock()
explicit := a.spider91UploadDriveID
@@ -388,7 +398,7 @@ func (a *App) Spider91UploadDriveID() string {
// SetSpider91UploadDriveID 设置 spider91 上传目标 drive ID 并持久化。
// 接受空字符串(本地保存不上传)。
// 设置一个不存在或 kind 不是 pikpak / p115 的 drive 会返回错误。
// 设置一个不存在或 kind 不是 pikpak / p115 / onedrive 的 drive 会返回错误。
func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) error {
driveID = strings.TrimSpace(driveID)
if driveID != "" {
@@ -397,7 +407,7 @@ func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) erro
return fmt.Errorf("drive %q not found", driveID)
}
if !isSpider91UploadKind(d.Kind()) {
return fmt.Errorf("drive %q kind=%s, only pikpak or p115 can be spider91 upload target", driveID, d.Kind())
return fmt.Errorf("drive %q kind=%s, only pikpak, p115 or onedrive can be spider91 upload target", driveID, d.Kind())
}
}
a.mu.Lock()
@@ -409,7 +419,7 @@ func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) erro
// isSpider91UploadKind 是 spider91 迁移目标盘的 allowlist。
// 与 spider91migrate.adaptUploadTarget 的支持范围保持一致。
func isSpider91UploadKind(kind string) bool {
return kind == "pikpak" || kind == "p115"
return kind == "pikpak" || kind == "p115" || kind == "onedrive"
}
// loadSpider91UploadDriveID 从 DB 读上传目标 drive ID 设置;不存在时使用空串。
@@ -434,9 +444,13 @@ func (a *App) driveGenerationStatuses() map[string]api.DriveGenerationStatuses {
for id, worker := range a.thumbWorkers {
thumbWorkers[id] = worker
}
fingerprintWorkers := make(map[string]*fingerprint.Worker, len(a.fingerprintWorkers))
for id, worker := range a.fingerprintWorkers {
fingerprintWorkers[id] = worker
}
a.mu.Unlock()
out := make(map[string]api.DriveGenerationStatuses, len(previewWorkers)+len(thumbWorkers))
out := make(map[string]api.DriveGenerationStatuses, len(previewWorkers)+len(thumbWorkers)+len(fingerprintWorkers))
for id, worker := range previewWorkers {
status := out[id]
status.Preview = generationStatusFromPreview(worker.Status())
@@ -447,7 +461,7 @@ func (a *App) driveGenerationStatuses() map[string]api.DriveGenerationStatuses {
status.Thumbnail = generationStatusFromPreview(worker.Status())
missing, err := a.cat.CountVideosNeedingThumbnail(context.Background(), id)
if err != nil {
log.Printf("[thumb] count missing thumbnails %s: %v", id, err)
log.Printf("[thumb] count thumbnail work %s: %v", id, err)
} else {
status.Thumbnail.QueueLength = missing
if missing > 0 && status.Thumbnail.State == "idle" {
@@ -456,6 +470,20 @@ func (a *App) driveGenerationStatuses() map[string]api.DriveGenerationStatuses {
}
out[id] = status
}
for id, worker := range fingerprintWorkers {
status := out[id]
status.Fingerprint = generationStatusFromFingerprint(worker.Status())
pending, err := a.cat.CountVideosNeedingFingerprint(context.Background(), id)
if err != nil {
log.Printf("[fingerprint] count pending fingerprints %s: %v", id, err)
} else {
status.Fingerprint.QueueLength = pending
if pending > 0 && status.Fingerprint.State == "idle" {
status.Fingerprint.State = "queued"
}
}
out[id] = status
}
return out
}
@@ -475,7 +503,67 @@ func generationStatusFromPreview(status preview.TaskStatus) api.GenerationStatus
return out
}
func generationStatusFromFingerprint(status fingerprint.TaskStatus) api.GenerationStatus {
state := status.State
if state == "" {
state = "idle"
}
out := api.GenerationStatus{
State: state,
CurrentTitle: status.CurrentTitle,
QueueLength: status.QueueLength,
}
if !status.CooldownUntil.IsZero() {
out.CooldownUntil = status.CooldownUntil.Format(time.RFC3339)
}
return out
}
func (a *App) attachDrive(ctx context.Context, d *catalog.Drive) error {
a.driveAttachMu.Lock()
defer a.driveAttachMu.Unlock()
return a.attachDriveUnlocked(ctx, d)
}
func (a *App) ensureDriveAttached(ctx context.Context, driveID string) error {
if _, ok := a.registry.Get(driveID); ok {
return nil
}
a.driveAttachMu.Lock()
defer a.driveAttachMu.Unlock()
if _, ok := a.registry.Get(driveID); ok {
return nil
}
d, err := a.cat.GetDrive(ctx, driveID)
if err != nil {
return err
}
return a.attachDriveUnlocked(ctx, d)
}
func (a *App) attachExistingDrives(ctx context.Context) {
existing, err := a.cat.ListDrives(ctx)
if err != nil {
log.Printf("[drive] list existing drives: %v", err)
return
}
log.Printf("[drive] attaching %d configured drive(s) in background", len(existing))
for _, d := range existing {
if err := ctx.Err(); err != nil {
log.Printf("[drive] background attach stopped: %v", err)
return
}
if err := a.attachDrive(ctx, d); err != nil {
log.Printf("[drive %s] attach failed: %v", d.ID, err)
}
}
log.Printf("[drive] background attach complete")
}
func (a *App) attachDriveUnlocked(ctx context.Context, d *catalog.Drive) error {
if d == nil {
return errors.New("nil drive")
}
var drv drives.Drive
switch d.Kind {
case "quark":
@@ -546,6 +634,11 @@ func (a *App) attachDrive(ctx context.Context, d *catalog.Drive) error {
_ = a.cat.UpsertDrive(ctx, d)
},
})
case localstorage.Kind:
drv = localstorage.New(localstorage.Config{
ID: d.ID,
RootPath: d.Credentials["path"],
})
case spider91.Kind:
drv = spider91.New(spider91.Config{
ID: d.ID,
@@ -579,12 +672,14 @@ func (a *App) attachDrive(ctx context.Context, d *catalog.Drive) error {
})
worker := preview.NewWorker(gen, a.cat, drv)
thumbWorker := preview.NewThumbWorker(gen, a.cat, drv)
fingerprintWorker := fingerprint.NewWorker(a.cat, drv, fingerprintConfigForDrive(drv))
workerCtx, cancel := context.WithCancel(ctx)
go worker.Run(workerCtx)
go thumbWorker.Run(workerCtx)
go fingerprintWorker.Run(workerCtx)
a.registerPreviewWorkers(ctx, d.ID, worker, thumbWorker, cancel)
a.registerPreviewWorkers(ctx, d.ID, worker, thumbWorker, fingerprintWorker, cancel)
// spider91 driver 还需要一个 crawler,挂在专用 map 里供 crawlerLoop 调用
if sd, ok := drv.(*spider91.Driver); ok {
@@ -611,12 +706,14 @@ func (a *App) attachLocalUpload(ctx context.Context) error {
})
worker := preview.NewWorker(gen, a.cat, drv)
thumbWorker := preview.NewThumbWorker(gen, a.cat, drv)
fingerprintWorker := fingerprint.NewWorker(a.cat, drv, fingerprintConfigForDrive(drv))
workerCtx, cancel := context.WithCancel(ctx)
go worker.Run(workerCtx)
go thumbWorker.Run(workerCtx)
go fingerprintWorker.Run(workerCtx)
a.registerPreviewWorkers(ctx, drv.ID(), worker, thumbWorker, cancel)
a.registerPreviewWorkers(ctx, drv.ID(), worker, thumbWorker, fingerprintWorker, cancel)
return nil
}
@@ -624,6 +721,20 @@ func (a *App) localUploadDir() string {
return filepath.Join(filepath.Dir(a.cfg.Storage.LocalPreviewDir), "uploads")
}
func fingerprintConfigForDrive(drv drives.Drive) fingerprint.Config {
cfg := fingerprint.Config{RateLimitCooldown: 5 * time.Minute}
if drv == nil {
return cfg
}
switch strings.ToLower(drv.Kind()) {
case "p115", "onedrive":
cfg.RateLimitCooldown = 10 * time.Minute
case "pikpak":
cfg.RateLimitCooldown = 5 * time.Minute
}
return cfg
}
// spider91RootDir 是所有 spider91 drive 共享的根目录。
func (a *App) spider91RootDir() string {
return filepath.Join(filepath.Dir(a.cfg.Storage.LocalPreviewDir), "spider91")
@@ -708,7 +819,7 @@ func (a *App) attachSpider91Crawler(d *catalog.Drive, drv *spider91.Driver) {
}()
}
func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker *preview.Worker, thumbWorker *preview.ThumbWorker, cancel context.CancelFunc) {
func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker *preview.Worker, thumbWorker *preview.ThumbWorker, fingerprintWorker *fingerprint.Worker, cancel context.CancelFunc) {
a.mu.Lock()
if a.cancels == nil {
a.cancels = make(map[string]context.CancelFunc)
@@ -719,6 +830,9 @@ func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker
if a.thumbWorkers == nil {
a.thumbWorkers = make(map[string]*preview.ThumbWorker)
}
if a.fingerprintWorkers == nil {
a.fingerprintWorkers = make(map[string]*fingerprint.Worker)
}
if old, ok := a.cancels[driveID]; ok && old != nil {
old()
}
@@ -732,6 +846,11 @@ func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker
} else {
delete(a.thumbWorkers, driveID)
}
if fingerprintWorker != nil {
a.fingerprintWorkers[driveID] = fingerprintWorker
} else {
delete(a.fingerprintWorkers, driveID)
}
if cancel != nil {
a.cancels[driveID] = cancel
} else {
@@ -739,17 +858,10 @@ func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker
}
a.mu.Unlock()
if worker != nil {
if thumbWorker != nil {
worker.BeforeTask = func(taskCtx context.Context) bool {
return a.waitForThumbnailsBeforePreview(taskCtx, driveID)
}
} else {
worker.BeforeTask = nil
}
}
go a.enqueueDriveGeneration(ctx, driveID, worker, thumbWorker)
if fingerprintWorker != nil {
a.scheduleFingerprintBackfill(ctx, driveID, fingerprintWorker)
}
}
func (a *App) enqueuePending(ctx context.Context, driveID string, w *preview.Worker) {
@@ -773,45 +885,16 @@ func (a *App) enqueuePending(ctx context.Context, driveID string, w *preview.Wor
func (a *App) enqueueDriveGeneration(ctx context.Context, driveID string, worker *preview.Worker, thumbWorker *preview.ThumbWorker) {
// 封面 worker 始终入队(与早期"全局 preview.enabled=false 时仍然生成封面"
// 的行为一致);teaser worker 仅在该 drive 的 TeaserEnabled 为 true 时入队。
// 两条队列互不等待,避免封面批量生成拖住预览视频生成。
if thumbWorker != nil {
a.enqueueThumbnails(ctx, driveID, thumbWorker)
}
if worker == nil || !a.teaserEnabledForDrive(ctx, driveID) {
return
}
if thumbWorker != nil && !a.waitForThumbnailsBeforePreview(ctx, driveID) {
return
}
a.enqueuePending(ctx, driveID, worker)
}
func (a *App) waitForThumbnailsBeforePreview(ctx context.Context, driveID string) bool {
const pollInterval = time.Second
var lastLog time.Time
for {
missing, err := a.cat.CountVideosNeedingThumbnail(ctx, driveID)
if err != nil {
log.Printf("[preview] count missing thumbnails drive=%s: %v", driveID, err)
return false
}
if missing == 0 {
return true
}
now := time.Now()
if lastLog.IsZero() || now.Sub(lastLog) >= time.Minute {
log.Printf("[preview] drive=%s waiting for %d thumbnails before teaser generation", driveID, missing)
lastLog = now
}
timer := time.NewTimer(pollInterval)
select {
case <-ctx.Done():
timer.Stop()
return false
case <-timer.C:
}
}
}
func (a *App) enqueueThumbnails(ctx context.Context, driveID string, w *preview.ThumbWorker) {
pending, err := a.cat.ListVideosNeedingThumbnail(ctx, driveID, 0)
if err != nil {
@@ -821,10 +904,81 @@ func (a *App) enqueueThumbnails(ctx context.Context, driveID string, w *preview.
if len(pending) == 0 {
return
}
log.Printf("[thumb] enqueue %d missing thumbnails for drive=%s", len(pending), driveID)
log.Printf("[thumb] enqueue %d thumbnail/duration tasks for drive=%s", len(pending), driveID)
for _, v := range pending {
if !w.EnqueueBlocking(ctx, v) {
log.Printf("[thumb] enqueue missing thumbnails canceled for drive=%s", driveID)
log.Printf("[thumb] enqueue thumbnail/duration tasks canceled for drive=%s", driveID)
return
}
}
}
func (a *App) runFingerprintReconciler(ctx context.Context) {
ticker := time.NewTicker(fingerprintReconcileInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
a.enqueueAllPendingFingerprints(ctx)
}
}
}
func (a *App) enqueueAllPendingFingerprints(ctx context.Context) {
a.mu.Lock()
workers := make(map[string]*fingerprint.Worker, len(a.fingerprintWorkers))
for id, worker := range a.fingerprintWorkers {
workers[id] = worker
}
a.mu.Unlock()
for driveID, worker := range workers {
a.scheduleFingerprintBackfill(ctx, driveID, worker)
}
}
func (a *App) scheduleFingerprintBackfill(ctx context.Context, driveID string, w *fingerprint.Worker) {
if w == nil {
return
}
a.fingerprintQueueMu.Lock()
if a.fingerprintQueueing == nil {
a.fingerprintQueueing = make(map[string]bool)
}
if a.fingerprintQueueing[driveID] {
a.fingerprintQueueMu.Unlock()
return
}
a.fingerprintQueueing[driveID] = true
a.fingerprintQueueMu.Unlock()
go func() {
defer func() {
a.fingerprintQueueMu.Lock()
delete(a.fingerprintQueueing, driveID)
a.fingerprintQueueMu.Unlock()
}()
a.enqueueFingerprints(ctx, driveID, w)
}()
}
func (a *App) enqueueFingerprints(ctx context.Context, driveID string, w *fingerprint.Worker) {
if w == nil {
return
}
pending, err := a.cat.ListVideosNeedingFingerprint(ctx, driveID, 0)
if err != nil {
log.Printf("[fingerprint] list pending %s: %v", driveID, err)
return
}
if len(pending) == 0 {
return
}
log.Printf("[fingerprint] enqueue %d videos for drive=%s", len(pending), driveID)
for _, v := range pending {
if !w.EnqueueBlocking(ctx, v) {
log.Printf("[fingerprint] enqueue canceled for drive=%s", driveID)
return
}
}
@@ -839,6 +993,7 @@ func (a *App) detachDrive(id string) {
}
delete(a.workers, id)
delete(a.thumbWorkers, id)
delete(a.fingerprintWorkers, id)
delete(a.spider91Crawlers, id)
a.mu.Unlock()
}
@@ -933,6 +1088,10 @@ func (a *App) runScan(ctx context.Context, driveID string) {
a.scanGlobalMu.Lock()
defer a.scanGlobalMu.Unlock()
if err := a.ensureDriveAttached(ctx, driveID); err != nil {
log.Printf("[scan] drive %s attach failed: %v", driveID, err)
return
}
drv, ok := a.registry.Get(driveID)
if !ok {
log.Printf("[scan] drive %s not attached", driveID)
@@ -942,14 +1101,15 @@ func (a *App) runScan(ctx context.Context, driveID string) {
a.mu.Lock()
worker := a.workers[driveID]
thumbWorker := a.thumbWorkers[driveID]
fingerprintWorker := a.fingerprintWorkers[driveID]
a.mu.Unlock()
var onNew func(v *catalog.Video)
if thumbWorker != nil {
onNew = func(v *catalog.Video) {
if thumbWorker != nil && v.ThumbnailURL == "" {
thumbWorker.Enqueue(v)
}
onNew := func(v *catalog.Video) {
if thumbWorker != nil && v.ThumbnailURL == "" {
thumbWorker.Enqueue(v)
}
if fingerprintWorker != nil {
fingerprintWorker.Enqueue(v)
}
}
@@ -993,6 +1153,7 @@ func (a *App) runScan(ctx context.Context, driveID string) {
}
}
}
a.scheduleFingerprintBackfill(ctx, driveID, fingerprintWorker)
a.enqueueDriveGeneration(ctx, driveID, worker, thumbWorker)
}
@@ -1063,6 +1224,126 @@ func removeLocalVideoAssets(localDir string, v *catalog.Video) error {
return nil
}
type duplicateAssetCleanupStats struct {
Candidates int
VideosUpdated int
PreviewFiles int
ThumbnailFiles int
MissingFiles int
SkippedUnsafeRef int
}
func (a *App) cleanupDuplicateVideoAssets(ctx context.Context) error {
if a == nil || a.cat == nil {
return nil
}
localDir := ""
if a.cfg != nil {
localDir = a.cfg.Storage.LocalPreviewDir
}
if strings.TrimSpace(localDir) == "" {
return nil
}
items, err := a.cat.ListDuplicateAssetCleanupCandidates(ctx, 0)
if err != nil {
return err
}
if len(items) == 0 {
log.Printf("[dedupe-cleanup] no duplicate local assets to clean")
return nil
}
stats := duplicateAssetCleanupStats{Candidates: len(items)}
for _, item := range items {
if err := ctx.Err(); err != nil {
return err
}
clearPreview, removedPreview, missingPreview, skippedPreview, err := cleanupDuplicatePreviewAsset(localDir, item.PreviewLocal)
if err != nil {
return fmt.Errorf("cleanup duplicate preview video=%s canonical=%s: %w", item.VideoID, item.CanonicalID, err)
}
clearThumb, removedThumb, missingThumb, err := cleanupDuplicateThumbnailAsset(localDir, item.VideoID, item.ThumbnailURL)
if err != nil {
return fmt.Errorf("cleanup duplicate thumbnail video=%s canonical=%s: %w", item.VideoID, item.CanonicalID, err)
}
if skippedPreview {
stats.SkippedUnsafeRef++
}
if removedPreview {
stats.PreviewFiles++
}
if removedThumb {
stats.ThumbnailFiles++
}
if missingPreview {
stats.MissingFiles++
}
if missingThumb {
stats.MissingFiles++
}
if !clearPreview && !clearThumb {
continue
}
if err := a.cat.ClearGeneratedAssets(ctx, item.VideoID, clearPreview, clearThumb); err != nil {
return fmt.Errorf("mark duplicate assets cleaned video=%s canonical=%s: %w", item.VideoID, item.CanonicalID, err)
}
stats.VideosUpdated++
}
log.Printf("[dedupe-cleanup] candidates=%d updated=%d preview_files=%d thumbnail_files=%d missing=%d skipped_unsafe_refs=%d",
stats.Candidates, stats.VideosUpdated, stats.PreviewFiles, stats.ThumbnailFiles, stats.MissingFiles, stats.SkippedUnsafeRef)
return nil
}
func cleanupDuplicatePreviewAsset(localDir, previewLocal string) (clear bool, removed bool, missing bool, skippedUnsafe bool, err error) {
clean, ok := localPathWithin(localDir, previewLocal)
if !ok {
if strings.TrimSpace(previewLocal) != "" {
return false, false, false, true, nil
}
return false, false, false, false, nil
}
removed, missing, err = removeRegularFileIfExists(clean)
if err != nil {
return false, false, false, false, err
}
return true, removed, missing, false, nil
}
func cleanupDuplicateThumbnailAsset(localDir, videoID, thumbnailURL string) (clear bool, removed bool, missing bool, err error) {
if thumbnailURL != "/p/thumb/"+videoID {
return false, false, false, nil
}
clean, ok := localPathWithin(localDir, filepath.Join(localDir, "thumbs", videoID+".jpg"))
if !ok {
return false, false, false, nil
}
removed, missing, err = removeRegularFileIfExists(clean)
if err != nil {
return false, false, false, err
}
return true, removed, missing, nil
}
func removeRegularFileIfExists(path string) (removed bool, missing bool, err error) {
info, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return false, true, nil
}
return false, false, err
}
if !info.Mode().IsRegular() {
return false, false, nil
}
if err := os.Remove(path); err != nil {
if os.IsNotExist(err) {
return false, true, nil
}
return false, false, err
}
return true, false, nil
}
func localPathWithin(root, path string) (string, bool) {
if strings.TrimSpace(root) == "" || strings.TrimSpace(path) == "" {
return "", false
@@ -1089,6 +1370,7 @@ func (a *App) enqueueUploadedVideo(ctx context.Context, v *catalog.Video) {
a.mu.Lock()
worker := a.workers[v.DriveID]
thumbWorker := a.thumbWorkers[v.DriveID]
fingerprintWorker := a.fingerprintWorkers[v.DriveID]
a.mu.Unlock()
if thumbWorker != nil && v.ThumbnailURL == "" {
@@ -1097,6 +1379,9 @@ func (a *App) enqueueUploadedVideo(ctx context.Context, v *catalog.Video) {
if worker != nil && a.teaserEnabledForDrive(ctx, v.DriveID) {
worker.Enqueue(v)
}
if fingerprintWorker != nil {
fingerprintWorker.Enqueue(v)
}
}
func (a *App) regenPreview(ctx context.Context, videoID string) {
@@ -1223,26 +1508,36 @@ func (a *App) regenFailedThumbnails(ctx context.Context, driveID string) {
}
// listScanTargetIDs 返回 nightly Phase 1 应扫描的所有 drive ID
// (非 spider91、非 localupload)。顺序按 registry.All 给的稳定顺序。
func (a *App) listScanTargetIDs(_ context.Context) []string {
all := a.registry.All()
// (非 spider91、非 localupload)。它直接读 catalog,而不是 registry,这样
// 进程刚启动、云盘还在后台挂载时,nightly 也不会漏掉配置过的 drive。
func (a *App) listScanTargetIDs(ctx context.Context) []string {
all, err := a.cat.ListDrives(ctx)
if err != nil {
log.Printf("[nightly] list scan target drives: %v", err)
return nil
}
out := make([]string, 0, len(all))
for _, d := range all {
if !shouldScanDrive(d) {
if d == nil || d.ID == localupload.DriveID || d.Kind == spider91.Kind {
continue
}
out = append(out, d.ID())
out = append(out, d.ID)
}
return out
}
// listSpider91DriveIDs 返回 nightly Phase 2 应触发爬取的 spider91 drive ID 列表。
func (a *App) listSpider91DriveIDs(_ context.Context) []string {
a.mu.Lock()
defer a.mu.Unlock()
out := make([]string, 0, len(a.spider91Crawlers))
for id := range a.spider91Crawlers {
out = append(out, id)
func (a *App) listSpider91DriveIDs(ctx context.Context) []string {
all, err := a.cat.ListDrives(ctx)
if err != nil {
log.Printf("[nightly] list spider91 drives: %v", err)
return nil
}
out := make([]string, 0, len(all))
for _, d := range all {
if d != nil && d.Kind == spider91.Kind {
out = append(out, d.ID)
}
}
return out
}
@@ -1250,8 +1545,8 @@ func (a *App) listSpider91DriveIDs(_ context.Context) []string {
// waitAllPreviewQueuesIdle 阻塞直到所有 drive 的封面 worker 和 teaser worker
// 队列都为空且无 in-flight 任务。
//
// 顺序:先等所有 thumb worker(因为 enqueueDriveGeneration 内部已经先等当前
// drive 的封面再入队 teaser,但这里是跨 drive 的全局同步),再等所有 teaser
// 顺序:先等所有 thumb worker,再等所有 teaser。两个队列生成时互不等待;
// nightly 只在 phase 边界统一等待它们都 drain
// 若 ctx 在等待中被取消(软超时 / shutdown),立即返回 ctx.Err。
func (a *App) waitAllPreviewQueuesIdle(ctx context.Context) error {
a.mu.Lock()
@@ -1301,7 +1596,17 @@ func (a *App) runSpider91Crawl(ctx context.Context, driveID string) {
c := a.spider91Crawlers[driveID]
a.mu.Unlock()
if c == nil {
return
if err := a.ensureDriveAttached(ctx, driveID); err != nil {
log.Printf("[spider91] drive=%s attach failed: %v", driveID, err)
return
}
a.mu.Lock()
c = a.spider91Crawlers[driveID]
a.mu.Unlock()
if c == nil {
log.Printf("[spider91] drive=%s crawler not attached", driveID)
return
}
}
d, err := a.cat.GetDrive(ctx, driveID)
@@ -1348,7 +1653,9 @@ func (a *App) runSpider91Crawl(ctx context.Context, driveID string) {
a.mu.Lock()
worker := a.workers[driveID]
thumbWorker := a.thumbWorkers[driveID]
fingerprintWorker := a.fingerprintWorkers[driveID]
a.mu.Unlock()
a.scheduleFingerprintBackfill(ctx, driveID, fingerprintWorker)
a.enqueueDriveGeneration(ctx, driveID, worker, thumbWorker)
}
+6
View File
@@ -38,6 +38,7 @@ func TestSpider91IntCredFallbacks(t *testing.T) {
func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
reg := proxy.NewRegistry()
reg.Set("p115-one", &spider91UploadTargetFakeDrive{id: "p115-one", kind: "p115"})
reg.Set("onedrive-one", &spider91UploadTargetFakeDrive{id: "onedrive-one", kind: "onedrive"})
app := &App{registry: reg}
if got := app.Spider91UploadDriveID(); got != "" {
@@ -49,6 +50,11 @@ func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
t.Fatalf("explicit upload target = %q, want p115-one", got)
}
app.spider91UploadDriveID = "onedrive-one"
if got := app.Spider91UploadDriveID(); got != "onedrive-one" {
t.Fatalf("explicit onedrive upload target = %q, want onedrive-one", got)
}
app.spider91UploadDriveID = "missing"
if got := app.Spider91UploadDriveID(); got != "" {
t.Fatalf("missing upload target = %q, want empty", got)
+368 -37
View File
@@ -13,7 +13,9 @@ import (
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/config"
"github.com/video-site/backend/internal/drives"
"github.com/video-site/backend/internal/fingerprint"
"github.com/video-site/backend/internal/preview"
"github.com/video-site/backend/internal/proxy"
)
func TestRegisterPreviewWorkerBackfillsPendingWhenDriveTeaserEnabled(t *testing.T) {
@@ -53,7 +55,7 @@ func TestRegisterPreviewWorkerBackfillsPendingWhenDriveTeaserEnabled(t *testing.
worker := preview.NewWorker(&serverFakeTeaserGenerator{}, cat, &serverFakeDrive{})
go worker.Run(ctx)
app.registerPreviewWorkers(ctx, "drive-id", worker, nil, func() {})
app.registerPreviewWorkers(ctx, "drive-id", worker, nil, nil, func() {})
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
@@ -77,7 +79,7 @@ func TestRegisterPreviewWorkerBackfillsPendingWhenDriveTeaserEnabled(t *testing.
t.Fatalf("preview status = %q, want ready", got.PreviewStatus)
}
func TestRegisterPreviewWorkersGenerateThumbnailsBeforePreviews(t *testing.T) {
func TestRegisterPreviewWorkersRunThumbnailsAndPreviewsIndependently(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -93,16 +95,18 @@ func TestRegisterPreviewWorkersGenerateThumbnailsBeforePreviews(t *testing.T) {
seedDriveWithTeaser(t, cat, "drive-id", true)
now := time.Now()
for _, v := range []*catalog.Video{
{ID: "video-1", DriveID: "drive-id", FileID: "file-1", Title: "Clip 1", PreviewStatus: "pending"},
{ID: "video-2", DriveID: "drive-id", FileID: "file-2", Title: "Clip 2", PreviewStatus: "pending"},
} {
v.PublishedAt = now
v.CreatedAt = now
v.UpdatedAt = now
if err := cat.UpsertVideo(ctx, v); err != nil {
t.Fatalf("seed video %s: %v", v.ID, err)
}
video := &catalog.Video{
ID: "video-1",
DriveID: "drive-id",
FileID: "file-1",
Title: "Clip 1",
PreviewStatus: "pending",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
}
if err := cat.UpsertVideo(ctx, video); err != nil {
t.Fatalf("seed video: %v", err)
}
app := &App{
@@ -110,47 +114,234 @@ func TestRegisterPreviewWorkersGenerateThumbnailsBeforePreviews(t *testing.T) {
workers: make(map[string]*preview.Worker),
thumbWorkers: make(map[string]*preview.ThumbWorker),
}
gen := &serverFakeTeaserGenerator{}
gen := &serverBlockingThumbGenerator{
started: make(chan string, 1),
release: make(chan struct{}),
}
drv := &serverFakeDrive{}
worker := preview.NewWorker(gen, cat, drv)
thumbWorker := preview.NewThumbWorker(gen, cat, drv)
go worker.Run(ctx)
go thumbWorker.Run(ctx)
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, func() {})
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, nil, func() {})
select {
case got := <-gen.started:
if got != video.ID {
t.Fatalf("thumbnail started for %q, want %q", got, video.ID)
}
case <-time.After(2 * time.Second):
t.Fatal("thumbnail generation did not start")
}
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
first, err := cat.GetVideo(ctx, "video-1")
got, err := cat.GetVideo(ctx, video.ID)
if err != nil {
t.Fatalf("get video-1: %v", err)
t.Fatalf("get video: %v", err)
}
second, err := cat.GetVideo(ctx, "video-2")
if err != nil {
t.Fatalf("get video-2: %v", err)
}
if first.ThumbnailURL != "" && second.ThumbnailURL != "" &&
first.PreviewStatus == "ready" && second.PreviewStatus == "ready" {
events := gen.Events()
if len(events) != 4 {
t.Fatalf("events = %#v, want 4 generation events", events)
}
for i, event := range events[:2] {
if event[:6] != "thumb:" {
t.Fatalf("event %d = %q, want thumbnail before previews; all events=%#v", i, event, events)
}
}
for i, event := range events[2:] {
if event[:8] != "preview:" {
t.Fatalf("event %d = %q, want previews after thumbnails; all events=%#v", i+2, event, events)
}
if got.PreviewStatus == "ready" {
if got.ThumbnailURL != "" {
t.Fatalf("thumbnail url = %q, want preview ready while thumbnail is still blocked", got.ThumbnailURL)
}
close(gen.release)
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatalf("generation did not finish, events=%#v", gen.Events())
got, err := cat.GetVideo(ctx, video.ID)
if err != nil {
t.Fatalf("get video after timeout: %v", err)
}
t.Fatalf("preview status=%q thumbnail=%q, want preview ready before thumbnail finishes", got.PreviewStatus, got.ThumbnailURL)
}
func TestRegisterPreviewWorkersBackfillsHistoricalFingerprints(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
dataPath := filepath.Join(t.TempDir(), "video.mp4")
data := []byte("historical video content for fingerprint")
if err := os.WriteFile(dataPath, data, 0o644); err != nil {
t.Fatalf("write video data: %v", err)
}
now := time.Now()
video := &catalog.Video{
ID: "historical-video",
DriveID: "drive-id",
FileID: "file-id",
Title: "Historical",
Size: int64(len(data)),
FingerprintStatus: "pending",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
}
if err := cat.UpsertVideo(ctx, video); err != nil {
t.Fatalf("seed video: %v", err)
}
app := &App{
cat: cat,
workers: make(map[string]*preview.Worker),
thumbWorkers: make(map[string]*preview.ThumbWorker),
fingerprintWorkers: make(map[string]*fingerprint.Worker),
}
drv := &serverFingerprintFakeDrive{path: dataPath}
fingerprintWorker := fingerprint.NewWorker(cat, drv, fingerprint.Config{})
go fingerprintWorker.Run(ctx)
app.registerPreviewWorkers(ctx, "drive-id", nil, nil, fingerprintWorker, func() {})
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
got, err := cat.GetVideo(ctx, video.ID)
if err != nil {
t.Fatalf("get video: %v", err)
}
if got.SampledSHA256 != "" && got.FingerprintStatus == "ready" {
return
}
time.Sleep(10 * time.Millisecond)
}
got, err := cat.GetVideo(ctx, video.ID)
if err != nil {
t.Fatalf("get video after timeout: %v", err)
}
t.Fatalf("fingerprint status=%q sampled=%q, want ready with hash", got.FingerprintStatus, got.SampledSHA256)
}
func TestRunScanStartsFingerprintBeforeThumbnailAndPreviewDrain(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
seedDriveWithTeaser(t, cat, "drive-id", true)
dataPath := filepath.Join(t.TempDir(), "scan-video.mp4")
data := []byte("scan video content for independent fingerprint")
if err := os.WriteFile(dataPath, data, 0o644); err != nil {
t.Fatalf("write video data: %v", err)
}
drv := &serverScanFingerprintFakeDrive{
serverFingerprintFakeDrive: serverFingerprintFakeDrive{path: dataPath},
entries: []drives.Entry{{
ID: "file-id",
Name: "scan-video.mp4",
Size: int64(len(data)),
ParentID: "root",
}},
}
registry := proxy.NewRegistry()
registry.Set("drive-id", drv)
gen := &serverFakeTeaserGenerator{}
worker := preview.NewWorker(gen, cat, drv)
thumbWorker := preview.NewThumbWorker(gen, cat, drv)
fingerprintWorker := fingerprint.NewWorker(cat, drv, fingerprint.Config{})
go fingerprintWorker.Run(ctx)
app := &App{
cfg: &config.Config{
Scanner: config.Scanner{VideoExtensions: []string{".mp4"}},
},
cat: cat,
registry: registry,
workers: map[string]*preview.Worker{"drive-id": worker},
thumbWorkers: map[string]*preview.ThumbWorker{"drive-id": thumbWorker},
fingerprintWorkers: map[string]*fingerprint.Worker{"drive-id": fingerprintWorker},
}
done := make(chan struct{})
go func() {
defer close(done)
app.runScan(ctx, "drive-id")
}()
videoID := "fake-drive-id-file-id"
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
got, err := cat.GetVideo(ctx, videoID)
if err == nil && got.SampledSHA256 != "" && got.FingerprintStatus == "ready" {
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("scan did not stop after context cancel")
}
if got.ThumbnailURL != "" {
t.Fatalf("thumbnail url = %q, want fingerprint before thumbnail generation", got.ThumbnailURL)
}
return
}
time.Sleep(10 * time.Millisecond)
}
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("scan did not stop after context cancel")
}
got, err := cat.GetVideo(context.Background(), videoID)
if err != nil {
t.Fatalf("get video after timeout: %v", err)
}
t.Fatalf("fingerprint status=%q sampled=%q, want ready before thumbnail/preview drain", got.FingerprintStatus, got.SampledSHA256)
}
func TestNightlyTargetsComeFromCatalogBeforeDriveAttach(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
for _, d := range []*catalog.Drive{
{ID: "115", Kind: "p115", Name: "115", RootID: "0", TeaserEnabled: true},
{ID: "pikpak", Kind: "pikpak", Name: "PikPak", RootID: "0", TeaserEnabled: true},
{ID: "91-spider", Kind: "spider91", Name: "91 Spider", RootID: "0", TeaserEnabled: true},
} {
if err := cat.UpsertDrive(ctx, d); err != nil {
t.Fatalf("seed drive %s: %v", d.ID, err)
}
}
app := &App{cat: cat}
scanIDs := app.listScanTargetIDs(ctx)
if len(scanIDs) != 2 || scanIDs[0] != "115" || scanIDs[1] != "pikpak" {
t.Fatalf("scan target ids = %#v, want 115 and pikpak from catalog", scanIDs)
}
spiderIDs := app.listSpider91DriveIDs(ctx)
if len(spiderIDs) != 1 || spiderIDs[0] != "91-spider" {
t.Fatalf("spider91 ids = %#v, want catalog spider drive", spiderIDs)
}
}
func TestFailedThumbnailsDoNotBlockPreviewGeneration(t *testing.T) {
@@ -205,7 +396,7 @@ func TestFailedThumbnailsDoNotBlockPreviewGeneration(t *testing.T) {
go worker.Run(ctx)
go thumbWorker.Run(ctx)
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, func() {})
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, nil, func() {})
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
@@ -480,6 +671,106 @@ func TestCleanupMissingPikPakVideosRemovesDatabaseRowsAndLocalAssets(t *testing.
}
}
func TestCleanupDuplicateVideoAssetsRemovesOnlyDuplicateLocalAssets(t *testing.T) {
ctx := context.Background()
localDir := t.TempDir()
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
canonicalPreview := filepath.Join(localDir, "canonical.mp4")
duplicatePreview := filepath.Join(localDir, "duplicate.mp4")
canonicalThumb := filepath.Join(localDir, "thumbs", "canonical-video.jpg")
duplicateThumb := filepath.Join(localDir, "thumbs", "duplicate-video.jpg")
for _, path := range []string{canonicalPreview, duplicatePreview, canonicalThumb, duplicateThumb} {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", path, err)
}
if err := os.WriteFile(path, []byte("asset"), 0o644); err != nil {
t.Fatalf("write %s: %v", path, err)
}
}
now := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
for _, v := range []*catalog.Video{
{
ID: "canonical-video",
DriveID: "115",
FileID: "file-a",
Title: "Canonical",
Size: 2048,
ThumbnailURL: "/p/thumb/canonical-video",
PreviewLocal: canonicalPreview,
PreviewStatus: "ready",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
},
{
ID: "duplicate-video",
DriveID: "onedrive",
FileID: "file-b",
Title: "Duplicate",
Size: 2048,
ThumbnailURL: "/p/thumb/duplicate-video",
PreviewLocal: duplicatePreview,
PreviewStatus: "ready",
PublishedAt: now.Add(time.Second),
CreatedAt: now.Add(time.Second),
UpdatedAt: now.Add(time.Second),
},
} {
if err := cat.UpsertVideo(ctx, v); err != nil {
t.Fatalf("seed %s: %v", v.ID, err)
}
if err := cat.UpdateVideoFingerprint(ctx, v.ID, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "ready", ""); err != nil {
t.Fatalf("fingerprint %s: %v", v.ID, err)
}
}
app := &App{
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
cat: cat,
}
if err := app.cleanupDuplicateVideoAssets(ctx); err != nil {
t.Fatalf("cleanup duplicate video assets: %v", err)
}
for _, path := range []string{canonicalPreview, canonicalThumb} {
if _, err := os.Stat(path); err != nil {
t.Fatalf("canonical asset %s missing: %v", path, err)
}
}
for _, path := range []string{duplicatePreview, duplicateThumb} {
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("duplicate asset %s still exists, stat err=%v", path, err)
}
}
dup, err := cat.GetVideo(ctx, "duplicate-video")
if err != nil {
t.Fatalf("get duplicate: %v", err)
}
if dup.PreviewLocal != "" || dup.PreviewStatus != "pending" {
t.Fatalf("duplicate preview local=%q status=%q, want empty pending", dup.PreviewLocal, dup.PreviewStatus)
}
if dup.ThumbnailURL != "" {
t.Fatalf("duplicate thumbnail url = %q, want empty", dup.ThumbnailURL)
}
canon, err := cat.GetVideo(ctx, "canonical-video")
if err != nil {
t.Fatalf("get canonical: %v", err)
}
if canon.PreviewLocal != canonicalPreview || canon.ThumbnailURL != "/p/thumb/canonical-video" {
t.Fatalf("canonical changed: preview=%q thumb=%q", canon.PreviewLocal, canon.ThumbnailURL)
}
}
type serverFakeTeaserGenerator struct {
mu sync.Mutex
events []string
@@ -520,6 +811,28 @@ func (g *serverFakeTeaserGenerator) GenerateThumbnail(_ context.Context, _ *driv
return "/tmp/" + videoID + ".jpg", nil
}
type serverBlockingThumbGenerator struct {
serverFakeTeaserGenerator
started chan string
release chan struct{}
}
func (g *serverBlockingThumbGenerator) GenerateThumbnail(ctx context.Context, _ *drives.StreamLink, videoID string, _ float64) (string, error) {
g.record("thumb:" + videoID)
if g.started != nil {
select {
case g.started <- videoID:
default:
}
}
select {
case <-g.release:
return "/tmp/" + videoID + ".jpg", nil
case <-ctx.Done():
return "", ctx.Err()
}
}
type serverFakeDrive struct{}
func (d *serverFakeDrive) Kind() string { return "fake" }
@@ -544,6 +857,24 @@ func (d *serverFakeDrive) EnsureDir(context.Context, string) (string, error) {
}
func (d *serverFakeDrive) RootID() string { return "root" }
type serverFingerprintFakeDrive struct {
serverFakeDrive
path string
}
func (d *serverFingerprintFakeDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
return &drives.StreamLink{URL: d.path}, nil
}
type serverScanFingerprintFakeDrive struct {
serverFingerprintFakeDrive
entries []drives.Entry
}
func (d *serverScanFingerprintFakeDrive) List(context.Context, string) ([]drives.Entry, error) {
return d.entries, nil
}
type serverLocalUploadFakeDrive struct {
serverFakeDrive
}
+9 -3
View File
@@ -59,7 +59,7 @@ preview:
width: 480
# 盘列表。上线后请通过管理后台添加,本文件可留空。
# kind 支持 quark / p115 / pikpak / wopan / onedrive。
# kind 支持 quark / p115 / pikpak / wopan / onedrive / localstorage
# OneDrive 示例:
# - id: "my-onedrive"
# kind: "onedrive"
@@ -67,6 +67,12 @@ preview:
# root_id: "root"
# params:
# refresh_token: "..."
# api_url_address: "https://api.oplist.org/onedrive/renewapi"
# region: "global"
# 本地存储示例:
# - id: "local-media"
# kind: "localstorage"
# name: "本地视频目录"
# root_id: "/"
# scan_root_id: "/"
# params:
# path: "/mnt/videos"
drives: []
+48 -23
View File
@@ -23,6 +23,10 @@ type AdminServer struct {
Auth *auth.Authenticator
// VersionFilePath points to the installer-written .version file.
VersionFilePath string
// ImageVersion is the Docker image version injected at build/runtime.
// It takes precedence over VersionFilePath because Docker data volumes can
// keep an older .version file across image upgrades.
ImageVersion string
// GitHubRepo is the owner/name repo used for update checks.
GitHubRepo string
// ReleaseAPIURL and HTTPClient are injectable for tests. Production code leaves them empty.
@@ -78,8 +82,9 @@ type GenerationStatus struct {
}
type DriveGenerationStatuses struct {
Thumbnail GenerationStatus `json:"thumbnail"`
Preview GenerationStatus `json:"preview"`
Thumbnail GenerationStatus `json:"thumbnail"`
Preview GenerationStatus `json:"preview"`
Fingerprint GenerationStatus `json:"fingerprint"`
}
func (a *AdminServer) Register(r chi.Router) {
@@ -278,6 +283,9 @@ func (a *AdminServer) checkUpdate(ctx context.Context) (updateCheckDTO, error) {
}
func (a *AdminServer) installedVersion() string {
if version := strings.TrimSpace(a.ImageVersion); version != "" {
return version
}
path := strings.TrimSpace(a.VersionFilePath)
if path == "" {
path = ".version"
@@ -346,6 +354,11 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
writeErr(w, http.StatusInternalServerError, err)
return
}
fingerprintCounts, err := a.Catalog.CountFingerprintsByDrive(r.Context())
if err != nil {
writeErr(w, http.StatusInternalServerError, err)
return
}
generationStatuses := map[string]DriveGenerationStatuses{}
if a.GetDriveGenerationStatuses != nil {
generationStatuses = a.GetDriveGenerationStatuses()
@@ -368,20 +381,25 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
SkipDirIDs []string `json:"skipDirIds"`
// LastCrawlAt 是 spider91 上次成功爬取的 unix 秒(来自 credentials.last_crawl_at)。
// 其它 kind 留 0;前端用它显示"上次抓取: N 小时前"。
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
TeaserReadyCount int `json:"teaserReadyCount"`
TeaserPendingCount int `json:"teaserPendingCount"`
TeaserFailedCount int `json:"teaserFailedCount"`
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
TeaserReadyCount int `json:"teaserReadyCount"`
TeaserPendingCount int `json:"teaserPendingCount"`
TeaserFailedCount int `json:"teaserFailedCount"`
FingerprintReadyCount int `json:"fingerprintReadyCount"`
FingerprintPendingCount int `json:"fingerprintPendingCount"`
FingerprintFailedCount int `json:"fingerprintFailedCount"`
}
list := make([]out, 0, len(drives))
for _, d := range drives {
counts := teaserCounts[d.ID]
thumbCounts := thumbnailCounts[d.ID]
fingerprintCount := fingerprintCounts[d.ID]
generation := generationStatuses[d.ID]
if generation.Thumbnail.State == "" {
generation.Thumbnail.State = "idle"
@@ -389,6 +407,9 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
if generation.Preview.State == "" {
generation.Preview.State = "idle"
}
if generation.Fingerprint.State == "" {
generation.Fingerprint.State = "idle"
}
// spider91 没有用户凭证概念;只要存在 drive 行就视为"已配置"。
// last_crawl_at 是后端自动写入的运行状态字段,不计入 hasCredential 判定。
hasCred := false
@@ -414,18 +435,22 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
ID: d.ID, Kind: d.Kind, Name: d.Name,
RootID: d.RootID, ScanRootID: d.ScanRootID,
Status: d.Status, LastError: d.LastError,
HasCredential: hasCred,
TeaserEnabled: d.TeaserEnabled,
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
LastCrawlAt: lastCrawlAt,
ThumbnailGenerationStatus: generation.Thumbnail,
PreviewGenerationStatus: generation.Preview,
ThumbnailReadyCount: thumbCounts.Ready,
ThumbnailPendingCount: thumbCounts.Pending,
ThumbnailFailedCount: thumbCounts.Failed,
TeaserReadyCount: counts.Ready,
TeaserPendingCount: counts.Pending,
TeaserFailedCount: counts.Failed,
HasCredential: hasCred,
TeaserEnabled: d.TeaserEnabled,
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
LastCrawlAt: lastCrawlAt,
ThumbnailGenerationStatus: generation.Thumbnail,
PreviewGenerationStatus: generation.Preview,
FingerprintGenerationStatus: generation.Fingerprint,
ThumbnailReadyCount: thumbCounts.Ready,
ThumbnailPendingCount: thumbCounts.Pending,
ThumbnailFailedCount: thumbCounts.Failed,
TeaserReadyCount: counts.Ready,
TeaserPendingCount: counts.Pending,
TeaserFailedCount: counts.Failed,
FingerprintReadyCount: fingerprintCount.Ready,
FingerprintPendingCount: fingerprintCount.Pending,
FingerprintFailedCount: fingerprintCount.Failed,
})
}
writeJSON(w, http.StatusOK, list)
+121 -41
View File
@@ -194,6 +194,54 @@ func TestHandleCheckUpdateReportsUpToDate(t *testing.T) {
}
}
func TestHandleCheckUpdateUsesDockerImageVersion(t *testing.T) {
releaseServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]any{
"tag_name": "v0.2.0",
"html_url": "https://github.com/nianzhibai/91/releases/tag/v0.2.0",
})
}))
t.Cleanup(releaseServer.Close)
req := httptest.NewRequest(http.MethodGet, "/admin/api/update/check", nil)
rr := httptest.NewRecorder()
(&AdminServer{
ImageVersion: "v0.1.0",
ReleaseAPIURL: releaseServer.URL,
}).handleCheckUpdate(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
var got updateCheckDTO
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
t.Fatalf("decode: %v", err)
}
if got.CurrentVersion != "v0.1.0" {
t.Fatalf("currentVersion = %q, want v0.1.0", got.CurrentVersion)
}
if !got.HasUpdate {
t.Fatalf("hasUpdate = false, want true")
}
}
func TestInstalledVersionPrefersDockerImageVersionOverVersionFile(t *testing.T) {
dir := t.TempDir()
versionFile := filepath.Join(dir, ".version")
if err := os.WriteFile(versionFile, []byte("v0.1.0\n"), 0o644); err != nil {
t.Fatalf("write version file: %v", err)
}
got := (&AdminServer{
VersionFilePath: versionFile,
ImageVersion: "v0.2.0",
}).installedVersion()
if got != "v0.2.0" {
t.Fatalf("installedVersion = %q, want v0.2.0", got)
}
}
func TestHandleUpsertDrivePreservesExistingCredentialsWhenRequestCredentialsEmpty(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
@@ -323,11 +371,11 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
now := time.Now()
videos := []*catalog.Video{
{ID: "od-ready-1", DriveID: "OneDrive", FileID: "od-file-1", Title: "OD Ready 1", ThumbnailURL: "/p/thumb/od-ready-1", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "od-ready-2", DriveID: "OneDrive", FileID: "od-file-2", Title: "OD Ready 2", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "od-pending", DriveID: "OneDrive", FileID: "od-file-3", Title: "OD Pending", PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "pp-pending", DriveID: "PikPak", FileID: "pp-file-1", Title: "PP Pending", PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "pp-failed", DriveID: "PikPak", FileID: "pp-file-2", Title: "PP Failed", ThumbnailURL: "/p/thumb/pp-failed", PreviewStatus: "failed", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "od-ready-1", DriveID: "OneDrive", FileID: "od-file-1", Title: "OD Ready 1", Size: 100, ThumbnailURL: "/p/thumb/od-ready-1", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "od-ready-2", DriveID: "OneDrive", FileID: "od-file-2", Title: "OD Ready 2", Size: 100, PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "od-pending", DriveID: "OneDrive", FileID: "od-file-3", Title: "OD Pending", Size: 100, PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "pp-pending", DriveID: "PikPak", FileID: "pp-file-1", Title: "PP Pending", Size: 100, PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
{ID: "pp-failed", DriveID: "PikPak", FileID: "pp-file-2", Title: "PP Failed", Size: 100, ThumbnailURL: "/p/thumb/pp-failed", PreviewStatus: "failed", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
}
for _, v := range videos {
if err := cat.UpsertVideo(ctx, v); err != nil {
@@ -337,6 +385,12 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
if err := cat.UpdateVideoMeta(ctx, "od-ready-2", catalog.VideoMetaPatch{ThumbnailStatus: "failed"}); err != nil {
t.Fatalf("mark thumbnail failed: %v", err)
}
if err := cat.UpdateVideoFingerprint(ctx, "od-ready-1", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "ready", ""); err != nil {
t.Fatalf("mark fingerprint ready: %v", err)
}
if err := cat.UpdateVideoFingerprint(ctx, "od-ready-2", "", "failed", "sample failed"); err != nil {
t.Fatalf("mark fingerprint failed: %v", err)
}
req := httptest.NewRequest(http.MethodGet, "/admin/api/drives", nil)
rr := httptest.NewRecorder()
@@ -345,8 +399,9 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
GetDriveGenerationStatuses: func() map[string]DriveGenerationStatuses {
return map[string]DriveGenerationStatuses{
"OneDrive": {
Thumbnail: GenerationStatus{State: "cooling", QueueLength: 3, CooldownUntil: "2026-05-16T21:00:00+08:00"},
Preview: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
Thumbnail: GenerationStatus{State: "cooling", QueueLength: 3, CooldownUntil: "2026-05-16T21:00:00+08:00"},
Preview: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
Fingerprint: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
},
}
},
@@ -356,48 +411,64 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
var got []struct {
ID string `json:"id"`
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
TeaserReadyCount int `json:"teaserReadyCount"`
TeaserPendingCount int `json:"teaserPendingCount"`
TeaserFailedCount int `json:"teaserFailedCount"`
ID string `json:"id"`
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
TeaserReadyCount int `json:"teaserReadyCount"`
TeaserPendingCount int `json:"teaserPendingCount"`
TeaserFailedCount int `json:"teaserFailedCount"`
FingerprintReadyCount int `json:"fingerprintReadyCount"`
FingerprintPendingCount int `json:"fingerprintPendingCount"`
FingerprintFailedCount int `json:"fingerprintFailedCount"`
}
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
t.Fatalf("decode: %v", err)
}
byID := map[string]struct {
TeaserReady int
TeaserPending int
TeaserFailed int
ThumbnailReady int
ThumbnailPending int
ThumbnailFailed int
Thumbnail GenerationStatus
Preview GenerationStatus
TeaserReady int
TeaserPending int
TeaserFailed int
ThumbnailReady int
ThumbnailPending int
ThumbnailFailed int
FingerprintReady int
FingerprintPending int
FingerprintFailed int
Thumbnail GenerationStatus
Preview GenerationStatus
Fingerprint GenerationStatus
}{}
for _, d := range got {
byID[d.ID] = struct {
TeaserReady int
TeaserPending int
TeaserFailed int
ThumbnailReady int
ThumbnailPending int
ThumbnailFailed int
Thumbnail GenerationStatus
Preview GenerationStatus
TeaserReady int
TeaserPending int
TeaserFailed int
ThumbnailReady int
ThumbnailPending int
ThumbnailFailed int
FingerprintReady int
FingerprintPending int
FingerprintFailed int
Thumbnail GenerationStatus
Preview GenerationStatus
Fingerprint GenerationStatus
}{
TeaserReady: d.TeaserReadyCount,
TeaserPending: d.TeaserPendingCount,
TeaserFailed: d.TeaserFailedCount,
ThumbnailReady: d.ThumbnailReadyCount,
ThumbnailPending: d.ThumbnailPendingCount,
ThumbnailFailed: d.ThumbnailFailedCount,
Thumbnail: d.ThumbnailGenerationStatus,
Preview: d.PreviewGenerationStatus,
TeaserReady: d.TeaserReadyCount,
TeaserPending: d.TeaserPendingCount,
TeaserFailed: d.TeaserFailedCount,
ThumbnailReady: d.ThumbnailReadyCount,
ThumbnailPending: d.ThumbnailPendingCount,
ThumbnailFailed: d.ThumbnailFailedCount,
FingerprintReady: d.FingerprintReadyCount,
FingerprintPending: d.FingerprintPendingCount,
FingerprintFailed: d.FingerprintFailedCount,
Thumbnail: d.ThumbnailGenerationStatus,
Preview: d.PreviewGenerationStatus,
Fingerprint: d.FingerprintGenerationStatus,
}
}
if byID["OneDrive"].TeaserReady != 2 || byID["OneDrive"].TeaserPending != 1 || byID["OneDrive"].TeaserFailed != 0 {
@@ -409,13 +480,22 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
if byID["OneDrive"].Thumbnail.State != "cooling" || byID["OneDrive"].Preview.State != "generating" {
t.Fatalf("OneDrive generation statuses = %#v, want thumbnail cooling and preview generating", byID["OneDrive"])
}
if byID["OneDrive"].FingerprintReady != 1 || byID["OneDrive"].FingerprintPending != 1 || byID["OneDrive"].FingerprintFailed != 1 {
t.Fatalf("OneDrive fingerprint counts = %#v, want ready=1 pending=1 failed=1", byID["OneDrive"])
}
if byID["OneDrive"].Fingerprint.State != "generating" {
t.Fatalf("OneDrive fingerprint status = %#v, want generating", byID["OneDrive"].Fingerprint)
}
if byID["PikPak"].TeaserReady != 0 || byID["PikPak"].TeaserPending != 1 || byID["PikPak"].TeaserFailed != 1 {
t.Fatalf("PikPak counts = %#v, want ready=0 pending=1 failed=1", byID["PikPak"])
}
if byID["PikPak"].ThumbnailReady != 1 || byID["PikPak"].ThumbnailPending != 1 || byID["PikPak"].ThumbnailFailed != 0 {
t.Fatalf("PikPak thumbnail counts = %#v, want ready=1 pending=1 failed=0", byID["PikPak"])
}
if byID["PikPak"].Thumbnail.State != "idle" || byID["PikPak"].Preview.State != "idle" {
if byID["PikPak"].FingerprintPending != 2 {
t.Fatalf("PikPak fingerprint counts = %#v, want pending=2", byID["PikPak"])
}
if byID["PikPak"].Thumbnail.State != "idle" || byID["PikPak"].Preview.State != "idle" || byID["PikPak"].Fingerprint.State != "idle" {
t.Fatalf("PikPak generation statuses = %#v, want idle defaults", byID["PikPak"])
}
}
+3
View File
@@ -21,6 +21,7 @@ import (
"github.com/video-site/backend/internal/auth"
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/drives/localstorage"
"github.com/video-site/backend/internal/drives/localupload"
"github.com/video-site/backend/internal/drives/spider91"
"github.com/video-site/backend/internal/proxy"
@@ -899,6 +900,8 @@ func driveKindLabel(kind string) string {
return "联通沃盘"
case "onedrive":
return "OneDrive"
case localstorage.Kind:
return "本地存储"
case spider91.Kind:
return "91 爬虫"
default:
+305 -36
View File
@@ -41,35 +41,38 @@ func (c *Catalog) Close() error { return c.db.Close() }
// ---------- Video ----------
type Video struct {
ID string `json:"id"`
DriveID string `json:"driveId"`
FileID string `json:"fileId"`
FileName string `json:"fileName"`
ContentHash string `json:"contentHash"`
ParentID string `json:"parentId"`
Title string `json:"title"`
Author string `json:"author"`
Tags []string `json:"tags"`
DurationSeconds int `json:"durationSeconds"`
Size int64 `json:"size"`
Ext string `json:"ext"`
Quality string `json:"quality"`
ThumbnailURL string `json:"thumbnailUrl"`
PreviewFileID string `json:"previewFileId"`
PreviewLocal string `json:"previewLocal"`
PreviewStatus string `json:"previewStatus"`
Views int `json:"views"`
Favorites int `json:"favorites"`
Comments int `json:"comments"`
Likes int `json:"likes"`
Dislikes int `json:"dislikes"`
Category string `json:"category"`
Hidden bool `json:"hidden"`
Badges []string `json:"badges"`
Description string `json:"description"`
PublishedAt time.Time `json:"publishedAt"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ID string `json:"id"`
DriveID string `json:"driveId"`
FileID string `json:"fileId"`
FileName string `json:"fileName"`
ContentHash string `json:"contentHash"`
SampledSHA256 string `json:"sampledSha256"`
FingerprintStatus string `json:"fingerprintStatus"`
FingerprintError string `json:"fingerprintError"`
ParentID string `json:"parentId"`
Title string `json:"title"`
Author string `json:"author"`
Tags []string `json:"tags"`
DurationSeconds int `json:"durationSeconds"`
Size int64 `json:"size"`
Ext string `json:"ext"`
Quality string `json:"quality"`
ThumbnailURL string `json:"thumbnailUrl"`
PreviewFileID string `json:"previewFileId"`
PreviewLocal string `json:"previewLocal"`
PreviewStatus string `json:"previewStatus"`
Views int `json:"views"`
Favorites int `json:"favorites"`
Comments int `json:"comments"`
Likes int `json:"likes"`
Dislikes int `json:"dislikes"`
Category string `json:"category"`
Hidden bool `json:"hidden"`
Badges []string `json:"badges"`
Description string `json:"description"`
PublishedAt time.Time `json:"publishedAt"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
func (c *Catalog) UpsertVideo(ctx context.Context, v *Video) error {
@@ -109,6 +112,18 @@ ON CONFLICT(id) DO UPDATE SET
WHEN excluded.content_hash != '' THEN excluded.content_hash
ELSE videos.content_hash
END,
sampled_sha256 = CASE
WHEN videos.size_bytes != excluded.size_bytes THEN ''
ELSE videos.sampled_sha256
END,
fingerprint_status = CASE
WHEN videos.size_bytes != excluded.size_bytes THEN 'pending'
ELSE COALESCE(videos.fingerprint_status, 'pending')
END,
fingerprint_error = CASE
WHEN videos.size_bytes != excluded.size_bytes THEN ''
ELSE COALESCE(videos.fingerprint_error, '')
END,
duration_seconds= excluded.duration_seconds,
size_bytes = excluded.size_bytes,
ext = excluded.ext,
@@ -486,8 +501,14 @@ func (c *Catalog) ListVideosByThumbnailStatus(ctx context.Context, driveID, stat
return out, nil
}
// ListVideosNeedingThumbnail returns videos that still need a thumbnail attempt.
// ListVideosNeedingThumbnail returns videos that still need thumbnail-worker work.
// Besides missing thumbnails, this includes videos with an existing thumbnail but
// missing duration metadata, because the thumbnail worker probes duration while
// it already has a stream link.
// Failed thumbnails are reported separately and should not block teaser generation.
// Videos whose local assets were cleared because they are fingerprint duplicates
// stay pending in the DB, but uniqueVideoWhereSQL keeps them out of this queue
// while their canonical sibling still exists.
func (c *Catalog) ListVideosNeedingThumbnail(ctx context.Context, driveID string, limit int) ([]*Video, error) {
if limit <= 0 {
limit = 10000
@@ -495,8 +516,11 @@ func (c *Catalog) ListVideosNeedingThumbnail(ctx context.Context, driveID string
rows, err := c.db.QueryContext(ctx,
`SELECT `+allVideoCols+` FROM videos
WHERE drive_id = ?
AND COALESCE(thumbnail_url, '') = ''
AND COALESCE(thumbnail_status, 'pending') != 'failed'
AND (
COALESCE(thumbnail_url, '') = ''
OR COALESCE(duration_seconds, 0) <= 0
)
AND COALESCE(thumbnail_status, 'pending') NOT IN ('failed', 'skipped')
AND COALESCE(hidden, 0) = 0
AND `+uniqueVideoWhereSQL+`
ORDER BY created_at ASC
@@ -522,8 +546,11 @@ func (c *Catalog) CountVideosNeedingThumbnail(ctx context.Context, driveID strin
err := c.db.QueryRowContext(ctx,
`SELECT COUNT(*) FROM videos
WHERE drive_id = ?
AND COALESCE(thumbnail_url, '') = ''
AND COALESCE(thumbnail_status, 'pending') != 'failed'
AND (
COALESCE(thumbnail_url, '') = ''
OR COALESCE(duration_seconds, 0) <= 0
)
AND COALESCE(thumbnail_status, 'pending') NOT IN ('failed', 'skipped')
AND COALESCE(hidden, 0) = 0
AND `+uniqueVideoWhereSQL,
driveID).Scan(&count)
@@ -668,6 +695,60 @@ func (c *Catalog) FindVideoByFileSignature(ctx context.Context, fileName string,
return scanVideo(row)
}
func (c *Catalog) ListVideosNeedingFingerprint(ctx context.Context, driveID string, limit int) ([]*Video, error) {
if limit <= 0 {
limit = 10000
}
rows, err := c.db.QueryContext(ctx,
`SELECT `+allVideoCols+` FROM videos
WHERE drive_id = ?
AND size_bytes > 0
AND COALESCE(sampled_sha256, '') = ''
AND COALESCE(fingerprint_status, 'pending') = 'pending'
AND COALESCE(hidden, 0) = 0
ORDER BY created_at ASC, id ASC
LIMIT ?`,
driveID, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []*Video
for rows.Next() {
v, err := scanVideo(rows)
if err != nil {
return nil, err
}
out = append(out, v)
}
return out, rows.Err()
}
func (c *Catalog) UpdateVideoFingerprint(ctx context.Context, id, sampledSHA256, status, errText string) error {
sampledSHA256 = normalizeContentHash(sampledSHA256)
if status == "" {
status = "pending"
}
if len(errText) > 500 {
errText = errText[:500]
}
res, err := c.db.ExecContext(ctx,
`UPDATE videos
SET sampled_sha256 = ?,
fingerprint_status = ?,
fingerprint_error = ?,
updated_at = ?
WHERE id = ?`,
sampledSHA256, status, errText, time.Now().UnixMilli(), id)
if err != nil {
return err
}
if rows, err := res.RowsAffected(); err == nil && rows == 0 {
return sql.ErrNoRows
}
return nil
}
type ListParams struct {
Keyword string
DriveID string
@@ -852,6 +933,12 @@ type DriveThumbnailCounts struct {
Failed int
}
type DriveFingerprintCounts struct {
Ready int
Pending int
Failed int
}
func (c *Catalog) CountTeasersByDrive(ctx context.Context) (map[string]DriveTeaserCounts, error) {
rows, err := c.db.QueryContext(ctx,
`SELECT drive_id,
@@ -914,6 +1001,52 @@ func (c *Catalog) CountThumbnailsByDrive(ctx context.Context) (map[string]DriveT
return out, nil
}
func (c *Catalog) CountFingerprintsByDrive(ctx context.Context) (map[string]DriveFingerprintCounts, error) {
rows, err := c.db.QueryContext(ctx,
`SELECT drive_id,
COUNT(CASE WHEN COALESCE(sampled_sha256, '') != ''
OR COALESCE(fingerprint_status, 'pending') = 'ready' THEN 1 END) AS ready_count,
COUNT(CASE WHEN size_bytes > 0
AND COALESCE(sampled_sha256, '') = ''
AND COALESCE(fingerprint_status, 'pending') = 'pending' THEN 1 END) AS pending_count,
COUNT(CASE WHEN COALESCE(sampled_sha256, '') = ''
AND COALESCE(fingerprint_status, 'pending') = 'failed' THEN 1 END) AS failed_count
FROM videos
WHERE COALESCE(hidden, 0) = 0
GROUP BY drive_id`)
if err != nil {
return nil, err
}
defer rows.Close()
out := make(map[string]DriveFingerprintCounts)
for rows.Next() {
var driveID string
var counts DriveFingerprintCounts
if err := rows.Scan(&driveID, &counts.Ready, &counts.Pending, &counts.Failed); err != nil {
return nil, err
}
out[driveID] = counts
}
if err := rows.Err(); err != nil {
return nil, err
}
return out, nil
}
func (c *Catalog) CountVideosNeedingFingerprint(ctx context.Context, driveID string) (int, error) {
var count int
err := c.db.QueryRowContext(ctx,
`SELECT COUNT(*) FROM videos
WHERE drive_id = ?
AND size_bytes > 0
AND COALESCE(sampled_sha256, '') = ''
AND COALESCE(fingerprint_status, 'pending') = 'pending'
AND COALESCE(hidden, 0) = 0`,
driveID).Scan(&count)
return count, err
}
type LocalMediaRef struct {
DriveID string
VideoID string
@@ -943,6 +1076,124 @@ func (c *Catalog) ListLocalMediaRefs(ctx context.Context) ([]LocalMediaRef, erro
return out, nil
}
// DuplicateAssetCleanupCandidate points at a non-canonical video in a
// size+sampled_sha256 duplicate group that still owns generated local assets.
// The cleanup job uses this to remove duplicate thumbnails/teasers without
// touching the original cloud file or deleting the catalog row.
type DuplicateAssetCleanupCandidate struct {
VideoID string
DriveID string
Title string
PreviewLocal string
ThumbnailURL string
CanonicalID string
SampledSHA256 string
Size int64
}
// ListDuplicateAssetCleanupCandidates returns duplicate videos whose own local
// generated assets can be cleared. A group canonical is the same representative
// used by uniqueVideoWhereSQL: earliest created_at, then lexicographically
// smallest id.
func (c *Catalog) ListDuplicateAssetCleanupCandidates(ctx context.Context, limit int) ([]DuplicateAssetCleanupCandidate, error) {
if limit <= 0 {
limit = 10000
}
rows, err := c.db.QueryContext(ctx, `
WITH canonical AS (
SELECT v.id, v.size_bytes, v.sampled_sha256
FROM videos v
WHERE v.size_bytes > 0
AND COALESCE(v.sampled_sha256, '') != ''
AND NOT EXISTS (
SELECT 1
FROM videos earlier
WHERE earlier.size_bytes = v.size_bytes
AND earlier.sampled_sha256 = v.sampled_sha256
AND COALESCE(earlier.sampled_sha256, '') != ''
AND earlier.size_bytes > 0
AND (
earlier.created_at < v.created_at
OR (earlier.created_at = v.created_at AND earlier.id < v.id)
)
)
)
SELECT dup.id,
dup.drive_id,
dup.title,
COALESCE(dup.preview_local, ''),
COALESCE(dup.thumbnail_url, ''),
canonical.id,
dup.sampled_sha256,
dup.size_bytes
FROM videos dup
JOIN canonical
ON canonical.size_bytes = dup.size_bytes
AND canonical.sampled_sha256 = dup.sampled_sha256
WHERE dup.id != canonical.id
AND dup.size_bytes > 0
AND COALESCE(dup.sampled_sha256, '') != ''
AND (
COALESCE(dup.preview_local, '') != ''
OR COALESCE(dup.thumbnail_url, '') = '/p/thumb/' || dup.id
)
ORDER BY dup.created_at ASC, dup.id ASC
LIMIT ?`, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []DuplicateAssetCleanupCandidate
for rows.Next() {
var item DuplicateAssetCleanupCandidate
if err := rows.Scan(
&item.VideoID,
&item.DriveID,
&item.Title,
&item.PreviewLocal,
&item.ThumbnailURL,
&item.CanonicalID,
&item.SampledSHA256,
&item.Size,
); err != nil {
return nil, err
}
out = append(out, item)
}
if err := rows.Err(); err != nil {
return nil, err
}
return out, nil
}
// ClearGeneratedAssets clears DB references to generated local assets for a
// video. The statuses go back to pending so the video can regenerate assets if
// it later becomes the canonical item after its older duplicate is removed.
func (c *Catalog) ClearGeneratedAssets(ctx context.Context, videoID string, clearPreview, clearThumbnail bool) error {
parts := []string{}
args := []any{}
if clearPreview {
parts = append(parts, "preview_file_id = ''", "preview_local = ''", "preview_status = 'pending'")
}
if clearThumbnail {
parts = append(parts, "thumbnail_url = ''", "thumbnail_status = 'pending'")
}
if len(parts) == 0 {
return nil
}
parts = append(parts, "updated_at = ?")
args = append(args, time.Now().UnixMilli(), videoID)
res, err := c.db.ExecContext(ctx, `UPDATE videos SET `+strings.Join(parts, ", ")+` WHERE id = ?`, args...)
if err != nil {
return err
}
if rows, err := res.RowsAffected(); err == nil && rows == 0 {
return sql.ErrNoRows
}
return nil
}
// ---------- Drive ----------
type Drive struct {
@@ -1171,7 +1422,9 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.upd
// ---------- helpers ----------
const allVideoCols = `
id, drive_id, file_id, COALESCE(file_name, ''), COALESCE(content_hash, ''), COALESCE(parent_id, ''), title, COALESCE(author, ''), COALESCE(tags, '[]'),
id, drive_id, file_id, COALESCE(file_name, ''), COALESCE(content_hash, ''),
COALESCE(sampled_sha256, ''), COALESCE(fingerprint_status, 'pending'), COALESCE(fingerprint_error, ''),
COALESCE(parent_id, ''), title, COALESCE(author, ''), COALESCE(tags, '[]'),
duration_seconds, size_bytes, COALESCE(ext, ''), COALESCE(quality, ''), COALESCE(thumbnail_url, ''),
COALESCE(preview_file_id, ''), COALESCE(preview_local, ''), COALESCE(preview_status, 'pending'),
views, favorites, comments, likes, dislikes,
@@ -1190,6 +1443,20 @@ const uniqueVideoWhereSQL = `((COALESCE(videos.content_hash, '') = ''
OR (dup.created_at = videos.created_at AND dup.id < videos.id)
)
))
AND (COALESCE(videos.sampled_sha256, '') = ''
OR videos.size_bytes <= 0
OR NOT EXISTS (
SELECT 1
FROM videos AS dup
WHERE dup.sampled_sha256 = videos.sampled_sha256
AND dup.size_bytes = videos.size_bytes
AND COALESCE(dup.sampled_sha256, '') != ''
AND dup.size_bytes > 0
AND (
dup.created_at < videos.created_at
OR (dup.created_at = videos.created_at AND dup.id < videos.id)
)
))
AND (COALESCE(videos.file_name, '') = ''
OR videos.size_bytes <= 0
OR NOT EXISTS (
@@ -1215,7 +1482,9 @@ func scanVideo(row rowScanner) (*Video, error) {
var publishedAt, createdAt, updatedAt int64
var hidden int
err := row.Scan(
&v.ID, &v.DriveID, &v.FileID, &v.FileName, &v.ContentHash, &v.ParentID, &v.Title, &v.Author, &tagsJSON,
&v.ID, &v.DriveID, &v.FileID, &v.FileName, &v.ContentHash,
&v.SampledSHA256, &v.FingerprintStatus, &v.FingerprintError,
&v.ParentID, &v.Title, &v.Author, &tagsJSON,
&v.DurationSeconds, &v.Size, &v.Ext, &v.Quality, &v.ThumbnailURL,
&v.PreviewFileID, &v.PreviewLocal, &v.PreviewStatus,
&v.Views, &v.Favorites, &v.Comments, &v.Likes, &v.Dislikes,
@@ -0,0 +1,179 @@
package catalog
import (
"context"
"testing"
"time"
)
func TestListVideosDeduplicatesBySampledSHA256(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
now := time.Now()
for _, v := range []*Video{
{
ID: "drive-a-file-a",
DriveID: "drive-a",
FileID: "file-a",
FileName: "first-name.mp4",
Title: "First",
Size: 1234,
PublishedAt: now.Add(-time.Minute),
CreatedAt: now.Add(-time.Minute),
UpdatedAt: now.Add(-time.Minute),
},
{
ID: "drive-b-file-b",
DriveID: "drive-b",
FileID: "file-b",
FileName: "second-name.mp4",
Title: "Second",
Size: 1234,
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
},
} {
if err := cat.UpsertVideo(ctx, v); err != nil {
t.Fatalf("upsert %s: %v", v.ID, err)
}
}
items, total, err := cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 10})
if err != nil {
t.Fatalf("list before fingerprint: %v", err)
}
if total != 2 || len(items) != 2 {
t.Fatalf("before fingerprint total=%d len=%d, want 2", total, len(items))
}
const sampled = "abc123"
if err := cat.UpdateVideoFingerprint(ctx, "drive-a-file-a", sampled, "ready", ""); err != nil {
t.Fatalf("update a fingerprint: %v", err)
}
if err := cat.UpdateVideoFingerprint(ctx, "drive-b-file-b", sampled, "ready", ""); err != nil {
t.Fatalf("update b fingerprint: %v", err)
}
items, total, err = cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 10})
if err != nil {
t.Fatalf("list after fingerprint: %v", err)
}
if total != 1 || len(items) != 1 {
t.Fatalf("after fingerprint total=%d len=%d, want 1", total, len(items))
}
if items[0].ID != "drive-a-file-a" {
t.Fatalf("canonical id = %q, want earliest created video", items[0].ID)
}
}
func TestDuplicateAssetCleanupCandidates(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
base := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
videos := []*Video{
{
ID: "drive-a-canonical",
DriveID: "drive-a",
FileID: "file-a",
FileName: "canonical.mp4",
Title: "Canonical",
Size: 1234,
ThumbnailURL: "/p/thumb/drive-a-canonical",
PreviewLocal: "/tmp/previews/canonical.mp4",
PreviewStatus: "ready",
PublishedAt: base,
CreatedAt: base,
UpdatedAt: base,
},
{
ID: "drive-b-duplicate",
DriveID: "drive-b",
FileID: "file-b",
FileName: "duplicate.mp4",
Title: "Duplicate",
Size: 1234,
ThumbnailURL: "/p/thumb/drive-b-duplicate",
PreviewLocal: "/tmp/previews/duplicate.mp4",
PreviewStatus: "ready",
PublishedAt: base.Add(time.Second),
CreatedAt: base.Add(time.Second),
UpdatedAt: base.Add(time.Second),
},
{
ID: "drive-c-remote-thumb",
DriveID: "drive-c",
FileID: "file-c",
FileName: "remote-thumb.mp4",
Title: "Remote Thumbnail",
Size: 1234,
ThumbnailURL: "https://thumb.example/file-c.jpg",
PreviewStatus: "ready",
PublishedAt: base.Add(2 * time.Second),
CreatedAt: base.Add(2 * time.Second),
UpdatedAt: base.Add(2 * time.Second),
},
}
for _, v := range videos {
if err := cat.UpsertVideo(ctx, v); err != nil {
t.Fatalf("seed %s: %v", v.ID, err)
}
}
const sampled = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
for _, v := range videos {
if err := cat.UpdateVideoFingerprint(ctx, v.ID, sampled, "ready", ""); err != nil {
t.Fatalf("fingerprint %s: %v", v.ID, err)
}
}
items, err := cat.ListDuplicateAssetCleanupCandidates(ctx, 0)
if err != nil {
t.Fatalf("list cleanup candidates: %v", err)
}
if len(items) != 1 {
t.Fatalf("candidates = %#v, want only local duplicate", items)
}
item := items[0]
if item.VideoID != "drive-b-duplicate" || item.CanonicalID != "drive-a-canonical" {
t.Fatalf("candidate = %#v, want duplicate with canonical", item)
}
if err := cat.ClearGeneratedAssets(ctx, item.VideoID, true, true); err != nil {
t.Fatalf("clear generated assets: %v", err)
}
got, err := cat.GetVideo(ctx, item.VideoID)
if err != nil {
t.Fatalf("get duplicate: %v", err)
}
if got.PreviewLocal != "" || got.PreviewStatus != "pending" {
t.Fatalf("preview after cleanup local=%q status=%q, want empty pending", got.PreviewLocal, got.PreviewStatus)
}
if got.ThumbnailURL != "" {
t.Fatalf("thumbnail after cleanup = %q, want empty", got.ThumbnailURL)
}
var thumbStatus string
if err := cat.db.QueryRowContext(ctx, `SELECT thumbnail_status FROM videos WHERE id = ?`, item.VideoID).Scan(&thumbStatus); err != nil {
t.Fatalf("query thumbnail status: %v", err)
}
if thumbStatus != "pending" {
t.Fatalf("thumbnail_status = %q, want pending", thumbStatus)
}
}
+4 -1
View File
@@ -5,6 +5,9 @@ CREATE TABLE IF NOT EXISTS videos (
file_id TEXT NOT NULL,
file_name TEXT DEFAULT '', -- 网盘侧原始文件名,用于同名同大小去重
content_hash TEXT DEFAULT '',
sampled_sha256 TEXT DEFAULT '', -- 跨网盘统一采样指纹(size + sampled bytes
fingerprint_status TEXT DEFAULT 'pending', -- pending / ready / failed
fingerprint_error TEXT DEFAULT '',
parent_id TEXT,
title TEXT NOT NULL,
author TEXT,
@@ -61,7 +64,7 @@ CREATE INDEX IF NOT EXISTS idx_video_tags_video ON video_tags(video_id);
-- 网盘账户
CREATE TABLE IF NOT EXISTS drives (
id TEXT PRIMARY KEY,
kind TEXT NOT NULL, -- quark / p115 / pikpak / wopan / onedrive / spider91
kind TEXT NOT NULL, -- quark / p115 / pikpak / wopan / onedrive / localstorage / spider91
name TEXT NOT NULL,
root_id TEXT NOT NULL DEFAULT '0',
scan_root_id TEXT, -- 扫描起点(默认 root_id
+12
View File
@@ -43,6 +43,15 @@ func (c *Catalog) migrate(ctx context.Context) error {
if err := c.addColumnIfMissing(ctx, "videos", "content_hash", "TEXT DEFAULT ''"); err != nil {
return err
}
if err := c.addColumnIfMissing(ctx, "videos", "sampled_sha256", "TEXT DEFAULT ''"); err != nil {
return err
}
if err := c.addColumnIfMissing(ctx, "videos", "fingerprint_status", "TEXT DEFAULT 'pending'"); err != nil {
return err
}
if err := c.addColumnIfMissing(ctx, "videos", "fingerprint_error", "TEXT DEFAULT ''"); err != nil {
return err
}
if err := c.addColumnIfMissing(ctx, "videos", "file_name", "TEXT DEFAULT ''"); err != nil {
return err
}
@@ -83,6 +92,9 @@ func (c *Catalog) migrate(ctx context.Context) error {
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_content_hash ON videos(content_hash)`); err != nil {
return err
}
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_sampled_sha256 ON videos(size_bytes, sampled_sha256)`); err != nil {
return err
}
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_hidden ON videos(hidden)`); err != nil {
return err
}
+84
View File
@@ -7,6 +7,90 @@ import (
"time"
)
func TestListVideosNeedingThumbnailIncludesExistingThumbnailMissingDuration(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
now := time.Now()
videos := []*Video{
{
ID: "duration-only",
DriveID: "drive",
FileID: "file-duration-only",
Title: "Duration Only",
ThumbnailURL: "/p/thumb/duration-only",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
},
{
ID: "complete",
DriveID: "drive",
FileID: "file-complete",
Title: "Complete",
DurationSeconds: 12,
ThumbnailURL: "/p/thumb/complete",
PublishedAt: now.Add(time.Second),
CreatedAt: now.Add(time.Second),
UpdatedAt: now.Add(time.Second),
},
{
ID: "missing-thumb",
DriveID: "drive",
FileID: "file-missing-thumb",
Title: "Missing Thumb",
DurationSeconds: 18,
PublishedAt: now.Add(2 * time.Second),
CreatedAt: now.Add(2 * time.Second),
UpdatedAt: now.Add(2 * time.Second),
},
{
ID: "failed",
DriveID: "drive",
FileID: "file-failed",
Title: "Failed",
PublishedAt: now.Add(3 * time.Second),
CreatedAt: now.Add(3 * time.Second),
UpdatedAt: now.Add(3 * time.Second),
},
}
for _, v := range videos {
if err := cat.UpsertVideo(ctx, v); err != nil {
t.Fatalf("seed %s: %v", v.ID, err)
}
}
if err := cat.UpdateVideoMeta(ctx, "failed", VideoMetaPatch{ThumbnailStatus: "failed"}); err != nil {
t.Fatalf("mark failed thumbnail: %v", err)
}
items, err := cat.ListVideosNeedingThumbnail(ctx, "drive", 0)
if err != nil {
t.Fatalf("list videos needing thumbnail: %v", err)
}
if len(items) != 2 {
t.Fatalf("items = %#v, want duration-only and missing-thumb", items)
}
if items[0].ID != "duration-only" || items[1].ID != "missing-thumb" {
t.Fatalf("item ids = %q, %q; want duration-only, missing-thumb", items[0].ID, items[1].ID)
}
count, err := cat.CountVideosNeedingThumbnail(ctx, "drive")
if err != nil {
t.Fatalf("count videos needing thumbnail: %v", err)
}
if count != 2 {
t.Fatalf("count = %d, want 2", count)
}
}
func TestCreateTagAndClassifyAddsTagToMatchingExistingVideos(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
+1 -1
View File
@@ -202,7 +202,7 @@ type Nightly struct {
// 这里保留 yaml 中的静态定义,用于启动时预置盘。生产建议只在 DB 里维护。
type Drive struct {
ID string `yaml:"id"`
Kind string `yaml:"kind"` // quark / p115 / pikpak / wopan / onedrive
Kind string `yaml:"kind"` // quark / p115 / pikpak / wopan / onedrive / localstorage
Name string `yaml:"name"`
RootID string `yaml:"root_id"`
Params map[string]string `yaml:"params,omitempty"`
+1 -1
View File
@@ -10,7 +10,7 @@ import (
// Drive 是多家网盘统一抽象。上层不区分盘,只区分 Kind。
type Drive interface {
// Kind 返回驱动代号:"quark" / "p115" / "pikpak" / "wopan" / "onedrive"
// Kind 返回驱动代号:"quark" / "p115" / "pikpak" / "wopan" / "onedrive" / "localstorage"
Kind() string
// ID 返回该盘在 catalog 中的唯一标识
@@ -0,0 +1,242 @@
// Package localstorage exposes an existing server-side directory as a Drive.
package localstorage
import (
"context"
"encoding/base64"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/video-site/backend/internal/drives"
)
const Kind = "localstorage"
type Config struct {
ID string
RootPath string
}
type Driver struct {
id string
rootPath string
}
func New(c Config) *Driver {
return &Driver{
id: c.ID,
rootPath: c.RootPath,
}
}
func (d *Driver) Kind() string { return Kind }
func (d *Driver) ID() string { return d.id }
func (d *Driver) RootID() string { return "/" }
func (d *Driver) Init(context.Context) error {
root, err := d.root()
if err != nil {
return err
}
info, err := os.Stat(root)
if err != nil {
return fmt.Errorf("localstorage: stat root: %w", err)
}
if !info.IsDir() {
return fmt.Errorf("localstorage: root is not a directory: %s", root)
}
return nil
}
func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error) {
dir, rel, err := d.pathForID(dirID)
if err != nil {
return nil, err
}
entries, err := os.ReadDir(dir)
if err != nil {
return nil, err
}
out := make([]drives.Entry, 0, len(entries))
for _, entry := range entries {
if err := ctx.Err(); err != nil {
return nil, err
}
// Symlinks can escape the configured root or create cycles. Keep the
// local storage drive predictable by scanning real files/directories only.
if entry.Type()&os.ModeSymlink != 0 {
continue
}
info, err := entry.Info()
if err != nil {
continue
}
if !info.IsDir() && !info.Mode().IsRegular() {
continue
}
childRel := joinRel(rel, entry.Name())
out = append(out, drives.Entry{
ID: encodeRel(childRel),
Name: entry.Name(),
Size: sizeForEntry(info),
IsDir: info.IsDir(),
ParentID: idForRel(rel),
ModTime: info.ModTime(),
})
}
return out, nil
}
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
p, rel, err := d.pathForID(fileID)
if err != nil {
return nil, err
}
info, err := os.Stat(p)
if err != nil {
return nil, err
}
return &drives.Entry{
ID: idForRel(rel),
Name: filepath.Base(p),
Size: sizeForEntry(info),
IsDir: info.IsDir(),
ParentID: idForRel(parentRel(rel)),
ModTime: info.ModTime(),
}, nil
}
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
p, _, err := d.pathForID(fileID)
if err != nil {
return nil, err
}
info, err := os.Stat(p)
if err != nil {
return nil, err
}
if info.IsDir() || !info.Mode().IsRegular() || info.Size() <= 0 {
return nil, os.ErrNotExist
}
return &drives.StreamLink{
URL: p,
Expires: time.Now().Add(24 * time.Hour),
}, nil
}
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
return "", drives.ErrNotSupported
}
func (d *Driver) EnsureDir(context.Context, string) (string, error) {
return "", drives.ErrNotSupported
}
func (d *Driver) root() (string, error) {
raw := strings.TrimSpace(d.rootPath)
if raw == "" {
return "", errors.New("localstorage: empty path")
}
raw = os.ExpandEnv(raw)
if strings.HasPrefix(raw, "~") {
if home, err := os.UserHomeDir(); err == nil && home != "" {
switch {
case raw == "~":
raw = home
case strings.HasPrefix(raw, "~/") || strings.HasPrefix(raw, `~\`):
raw = filepath.Join(home, raw[2:])
}
}
}
return filepath.Abs(raw)
}
func (d *Driver) pathForID(id string) (string, string, error) {
root, err := d.root()
if err != nil {
return "", "", err
}
rel, err := decodeRel(id)
if err != nil {
return "", "", err
}
if rel == "" {
return root, "", nil
}
p, err := filepath.Abs(filepath.Join(root, filepath.FromSlash(rel)))
if err != nil {
return "", "", err
}
if p != root && !strings.HasPrefix(p, root+string(os.PathSeparator)) {
return "", "", errors.New("localstorage: path escapes root")
}
return p, rel, nil
}
func decodeRel(id string) (string, error) {
id = strings.TrimSpace(id)
if id == "" || id == "/" {
return "", nil
}
raw, err := base64.RawURLEncoding.DecodeString(id)
if err != nil {
return "", fmt.Errorf("localstorage: invalid file id: %w", err)
}
rel := filepath.ToSlash(filepath.Clean(filepath.FromSlash(string(raw))))
if rel == "." {
return "", nil
}
if strings.HasPrefix(rel, "../") || rel == ".." || strings.HasPrefix(rel, "/") {
return "", errors.New("localstorage: invalid relative path")
}
return rel, nil
}
func encodeRel(rel string) string {
rel = filepath.ToSlash(filepath.Clean(filepath.FromSlash(rel)))
if rel == "." || rel == "" {
return "/"
}
return base64.RawURLEncoding.EncodeToString([]byte(rel))
}
func idForRel(rel string) string {
if rel == "" {
return "/"
}
return encodeRel(rel)
}
func joinRel(parent, name string) string {
if parent == "" {
return filepath.ToSlash(name)
}
return filepath.ToSlash(filepath.Join(filepath.FromSlash(parent), name))
}
func parentRel(rel string) string {
if rel == "" {
return ""
}
parent := filepath.ToSlash(filepath.Dir(filepath.FromSlash(rel)))
if parent == "." {
return ""
}
return parent
}
func sizeForEntry(info os.FileInfo) int64 {
if info == nil || info.IsDir() {
return 0
}
return info.Size()
}
var _ drives.Drive = (*Driver)(nil)
@@ -0,0 +1,119 @@
package localstorage
import (
"context"
"encoding/base64"
"os"
"path/filepath"
"strings"
"testing"
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/scanner"
)
func TestListEncodesRelativePathsAndStreamURLResolvesFile(t *testing.T) {
root := t.TempDir()
sub := filepath.Join(root, "clips")
if err := os.MkdirAll(sub, 0o755); err != nil {
t.Fatalf("mkdir: %v", err)
}
videoPath := filepath.Join(sub, "sample.mp4")
if err := os.WriteFile(videoPath, []byte("video"), 0o644); err != nil {
t.Fatalf("write video: %v", err)
}
drv := New(Config{ID: "local", RootPath: root})
if err := drv.Init(context.Background()); err != nil {
t.Fatalf("init: %v", err)
}
rootEntries, err := drv.List(context.Background(), drv.RootID())
if err != nil {
t.Fatalf("list root: %v", err)
}
if len(rootEntries) != 1 || !rootEntries[0].IsDir {
t.Fatalf("root entries = %#v, want one directory", rootEntries)
}
if strings.Contains(rootEntries[0].ID, "/") {
t.Fatalf("encoded dir id contains slash: %q", rootEntries[0].ID)
}
fileEntries, err := drv.List(context.Background(), rootEntries[0].ID)
if err != nil {
t.Fatalf("list subdir: %v", err)
}
if len(fileEntries) != 1 || fileEntries[0].Name != "sample.mp4" {
t.Fatalf("file entries = %#v, want sample.mp4", fileEntries)
}
if strings.Contains(fileEntries[0].ID, "/") {
t.Fatalf("encoded file id contains slash: %q", fileEntries[0].ID)
}
link, err := drv.StreamURL(context.Background(), fileEntries[0].ID)
if err != nil {
t.Fatalf("stream url: %v", err)
}
if link.URL != videoPath {
t.Fatalf("url = %q, want %q", link.URL, videoPath)
}
}
func TestStreamURLRejectsEscapingID(t *testing.T) {
drv := New(Config{ID: "local", RootPath: t.TempDir()})
escaped := base64.RawURLEncoding.EncodeToString([]byte("../secret.mp4"))
_, err := drv.StreamURL(context.Background(), escaped)
if err == nil || !strings.Contains(err.Error(), "invalid relative path") {
t.Fatalf("error = %v, want invalid relative path", err)
}
}
func TestInitRequiresExistingDirectory(t *testing.T) {
drv := New(Config{ID: "local", RootPath: filepath.Join(t.TempDir(), "missing")})
err := drv.Init(context.Background())
if err == nil || !strings.Contains(err.Error(), "stat root") {
t.Fatalf("error = %v, want stat root failure", err)
}
}
func TestScannerPersistsLocalStorageVideo(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
if err := os.MkdirAll(filepath.Join(root, "collection"), 0o755); err != nil {
t.Fatalf("mkdir collection: %v", err)
}
if err := os.WriteFile(filepath.Join(root, "collection", "clip.mp4"), []byte("video"), 0o644); err != nil {
t.Fatalf("write video: %v", err)
}
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
drv := New(Config{ID: "local", RootPath: root})
sc := scanner.New(cat, drv, []string{".mp4"}, nil, nil)
stats, err := sc.Run(ctx, drv.RootID())
if err != nil {
t.Fatalf("scan: %v", err)
}
if stats.Added != 1 {
t.Fatalf("added = %d, want 1", stats.Added)
}
fileID := encodeRel("collection/clip.mp4")
got, err := cat.GetVideo(ctx, Kind+"-local-"+fileID)
if err != nil {
t.Fatalf("get video: %v", err)
}
if got.DriveID != "local" || got.FileID != fileID || got.Category != "collection" {
t.Fatalf("video = %#v, want local drive video in collection", got)
}
}
+321 -22
View File
@@ -3,14 +3,19 @@ package onedrive
import (
"bytes"
"context"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"sync"
"time"
"github.com/go-resty/resty/v2"
@@ -18,8 +23,17 @@ import (
)
const (
maxSmallUploadSize = 250 * 1024 * 1024
defaultRenewAPIURL = "https://api.oplist.org/onedrive/renewapi"
maxSmallUploadSize = 250 * 1024 * 1024
defaultUploadSessionChunk = 10 * 1024 * 1024
uploadSessionRetryAttempts = 3
defaultRenewAPIURL = "https://api.oplist.org/onedrive/renewapi"
onedriveListCooldown = 5 * time.Minute
onedriveListInterval = 1 * time.Second
)
var (
smallUploadThreshold = int64(maxSmallUploadSize)
uploadSessionChunk = int64(defaultUploadSessionChunk)
)
type Driver struct {
@@ -34,6 +48,11 @@ type Driver struct {
renewAPIURL string
client *resty.Client
onTokenUpdate func(access, refresh string)
listMu sync.Mutex
lastListAt time.Time
listInterval time.Duration
listCooldown time.Duration
}
type Config struct {
@@ -85,6 +104,8 @@ func New(c Config) *Driver {
client: resty.New().
SetTimeout(30*time.Second).
SetHeader("Accept", "application/json, text/plain, */*"),
listInterval: onedriveListInterval,
listCooldown: onedriveListCooldown,
}
}
@@ -106,10 +127,16 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
if dirID == "" {
dirID = d.rootID
}
d.listMu.Lock()
defer d.listMu.Unlock()
nextLink := d.childrenURL(dirID)
first := true
out := make([]drives.Entry, 0)
for nextLink != "" {
if err := d.waitForListSlotLocked(ctx); err != nil {
return nil, err
}
var resp filesResp
err := d.request(ctx, nextLink, http.MethodGet, func(req *resty.Request) {
if first {
@@ -120,6 +147,19 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
}
}, &resp)
if err != nil {
if wait, ok := drives.RateLimitRetryAfter(err); ok {
if wait <= 0 {
wait = d.listCooldown
if wait <= 0 {
wait = onedriveListCooldown
}
}
log.Printf("[onedrive] list cooling down drive=%s dir=%s cooldown=%s err=%v", d.id, dirID, wait, err)
if err := sleepContext(ctx, wait); err != nil {
return nil, err
}
continue
}
return nil, fmt.Errorf("onedrive list: %w", err)
}
for _, item := range resp.Value {
@@ -131,6 +171,36 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
return out, nil
}
func (d *Driver) waitForListSlotLocked(ctx context.Context) error {
if d.listInterval <= 0 || d.lastListAt.IsZero() {
d.lastListAt = time.Now()
return ctx.Err()
}
next := d.lastListAt.Add(d.listInterval)
now := time.Now()
if now.Before(next) {
if err := sleepContext(ctx, next.Sub(now)); err != nil {
return err
}
}
d.lastListAt = time.Now()
return ctx.Err()
}
func sleepContext(ctx context.Context, d time.Duration) error {
if d <= 0 {
return ctx.Err()
}
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
var item graphItem
if err := d.request(ctx, d.itemURL(fileID), http.MethodGet, nil, &item); err != nil {
@@ -156,15 +226,49 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
}
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
res, err := d.UploadAndReportHash(ctx, parentID, name, r, size)
if err != nil {
return "", err
}
return res.FileID, nil
}
func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
parentID, name, err := d.normalizeUploadArgs(parentID, name, r, size)
if err != nil {
return UploadResult{}, err
}
threshold := smallUploadThreshold
if threshold <= 0 {
threshold = maxSmallUploadSize
}
if size <= threshold {
return d.uploadSmallAndReportHash(ctx, parentID, name, r, size, threshold)
}
return d.uploadSessionAndReportHash(ctx, parentID, name, r, size)
}
func (d *Driver) normalizeUploadArgs(parentID, name string, r io.Reader, size int64) (string, string, error) {
if r == nil {
return "", "", errors.New("onedrive upload: body is required")
}
if size < 0 {
return "", "", fmt.Errorf("onedrive upload: invalid size %d", size)
}
if parentID == "" {
parentID = d.rootID
}
if size > maxSmallUploadSize {
return "", fmt.Errorf("onedrive upload: files over %d bytes require upload session", maxSmallUploadSize)
name = strings.TrimSpace(name)
if name == "" {
return "", "", errors.New("onedrive upload: empty file name")
}
data, err := readSmallUpload(r)
return parentID, name, nil
}
func (d *Driver) uploadSmallAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size, limit int64) (UploadResult, error) {
data, hash, actualSize, err := readSmallUpload(r, size, limit)
if err != nil {
return "", err
return UploadResult{}, err
}
u := fmt.Sprintf("%s/items/%s:/%s:/content", d.driveBaseURL(), url.PathEscape(parentID), url.PathEscape(name))
var item graphItem
@@ -173,26 +277,159 @@ func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader,
req.SetContentLength(true)
}, &item)
if err != nil {
return "", fmt.Errorf("onedrive upload: %w", err)
return UploadResult{}, fmt.Errorf("onedrive upload: %w", err)
}
if item.ID == "" {
return "", errors.New("onedrive upload: empty item id")
return UploadResult{}, errors.New("onedrive upload: empty item id")
}
return item.ID, nil
return UploadResult{FileID: item.ID, Hash: hash, Size: actualSize}, nil
}
func readSmallUpload(r io.Reader) ([]byte, error) {
if r == nil {
return nil, errors.New("onedrive upload: body is required")
}
data, err := io.ReadAll(io.LimitReader(r, maxSmallUploadSize+1))
func (d *Driver) uploadSessionAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
session, err := d.createUploadSession(ctx, parentID, name)
if err != nil {
return nil, fmt.Errorf("onedrive upload: read body: %w", err)
return UploadResult{}, err
}
if len(data) > maxSmallUploadSize {
return nil, fmt.Errorf("onedrive upload: files over %d bytes require upload session", maxSmallUploadSize)
if strings.TrimSpace(session.UploadURL) == "" {
return UploadResult{}, errors.New("onedrive upload session: empty upload url")
}
return data, nil
chunkSize := uploadSessionChunk
if chunkSize <= 0 {
chunkSize = defaultUploadSessionChunk
}
buf := make([]byte, int(chunkSize))
hasher := sha1.New()
var finalItem graphItem
var offset int64
for offset < size {
partSize := minInt64(chunkSize, size-offset)
chunk := buf[:int(partSize)]
n, err := io.ReadFull(r, chunk)
if err != nil {
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
return UploadResult{}, fmt.Errorf("onedrive upload: size mismatch: declared %d, copied %d", size, offset+int64(n))
}
return UploadResult{}, fmt.Errorf("onedrive upload: read body: %w", err)
}
chunk = chunk[:n]
_, _ = hasher.Write(chunk)
item, err := d.putUploadSessionChunkWithRetry(ctx, session.UploadURL, offset, size, chunk)
if err != nil {
return UploadResult{}, err
}
if item != nil {
finalItem = *item
}
offset += int64(n)
}
if finalItem.ID == "" {
return UploadResult{}, errors.New("onedrive upload session: empty item id")
}
return UploadResult{
FileID: finalItem.ID,
Hash: hex.EncodeToString(hasher.Sum(nil)),
Size: offset,
}, nil
}
func (d *Driver) createUploadSession(ctx context.Context, parentID, name string) (uploadSessionResp, error) {
u := fmt.Sprintf("%s/items/%s:/%s:/createUploadSession", d.driveBaseURL(), url.PathEscape(parentID), url.PathEscape(name))
body := map[string]any{
"item": map[string]any{
"@microsoft.graph.conflictBehavior": "rename",
},
}
var out uploadSessionResp
err := d.request(ctx, u, http.MethodPost, func(req *resty.Request) {
req.SetBody(body)
}, &out)
if err != nil {
return uploadSessionResp{}, fmt.Errorf("onedrive upload session: %w", err)
}
return out, nil
}
func (d *Driver) putUploadSessionChunkWithRetry(ctx context.Context, uploadURL string, start, total int64, data []byte) (*graphItem, error) {
var last error
for attempt := 0; attempt < uploadSessionRetryAttempts; attempt++ {
if attempt > 0 {
if err := sleepContext(ctx, time.Duration(attempt)*time.Second); err != nil {
return nil, err
}
}
item, retryable, err := d.putUploadSessionChunk(ctx, uploadURL, start, total, data)
if err == nil {
return item, nil
}
last = err
if !retryable {
return nil, err
}
}
if last == nil {
last = errors.New("onedrive upload session: retry attempts exhausted")
}
return nil, last
}
func (d *Driver) putUploadSessionChunk(ctx context.Context, uploadURL string, start, total int64, data []byte) (*graphItem, bool, error) {
end := start + int64(len(data)) - 1
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(data))
if err != nil {
return nil, false, err
}
req.ContentLength = int64(len(data))
req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, total))
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, true, err
}
defer res.Body.Close()
switch res.StatusCode {
case http.StatusOK, http.StatusCreated:
var item graphItem
if err := json.NewDecoder(res.Body).Decode(&item); err != nil {
return nil, false, fmt.Errorf("onedrive upload session: decode completed item: %w", err)
}
return &item, false, nil
case http.StatusAccepted:
return nil, false, nil
default:
body, _ := io.ReadAll(io.LimitReader(res.Body, 4096))
err := fmt.Errorf("onedrive upload session: status=%d body=%s", res.StatusCode, strings.TrimSpace(string(body)))
retryable := res.StatusCode == http.StatusTooManyRequests || (res.StatusCode >= 500 && res.StatusCode <= 504)
return nil, retryable, err
}
}
func readSmallUpload(r io.Reader, declaredSize, limit int64) ([]byte, string, int64, error) {
if r == nil {
return nil, "", 0, errors.New("onedrive upload: body is required")
}
if limit <= 0 {
limit = maxSmallUploadSize
}
data, err := io.ReadAll(io.LimitReader(r, limit+1))
if err != nil {
return nil, "", 0, fmt.Errorf("onedrive upload: read body: %w", err)
}
if int64(len(data)) > limit {
return nil, "", 0, fmt.Errorf("onedrive upload: files over %d bytes require upload session", limit)
}
if declaredSize >= 0 && int64(len(data)) != declaredSize {
return nil, "", 0, fmt.Errorf("onedrive upload: size mismatch: declared %d, copied %d", declaredSize, len(data))
}
sum := sha1.Sum(data)
return data, hex.EncodeToString(sum[:]), int64(len(data)), nil
}
func minInt64(a, b int64) int64 {
if a < b {
return a
}
return b
}
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
@@ -245,6 +482,25 @@ func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, er
return item.ID, nil
}
func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
fileID = strings.TrimSpace(fileID)
if fileID == "" {
return errors.New("onedrive rename: empty file id")
}
newName = strings.TrimSpace(newName)
if newName == "" {
return errors.New("onedrive rename: empty new name")
}
var item graphItem
err := d.request(ctx, d.itemURL(fileID), http.MethodPatch, func(req *resty.Request) {
req.SetBody(map[string]string{"name": newName})
}, &item)
if err != nil {
return fmt.Errorf("onedrive rename: %w", err)
}
return nil
}
func (d *Driver) request(ctx context.Context, rawURL, method string, configure func(*resty.Request), out any) error {
return d.requestOnce(ctx, rawURL, method, configure, out, true)
}
@@ -265,7 +521,7 @@ func (d *Driver) requestOnce(ctx context.Context, rawURL, method string, configu
if err != nil {
return err
}
if isRateLimitResponse(res, graphErr.Error.Code) {
if isRateLimitResponse(res, graphErr.Error.Code, graphErr.Error.Message) {
return onedriveRateLimitError(res, graphErr.Error.Message)
}
if graphErr.Error.Code != "" {
@@ -327,11 +583,54 @@ func (d *Driver) refresh(ctx context.Context) error {
return nil
}
func isRateLimitResponse(res *resty.Response, code string) bool {
if code == "TooManyRequests" || code == "activityLimitReached" {
func isRateLimitResponse(res *resty.Response, code, message string) bool {
if isRateLimitCode(code) || isRateLimitMessage(message) {
return true
}
return res != nil && res.StatusCode() == http.StatusTooManyRequests
if res == nil {
return false
}
if res.StatusCode() == http.StatusTooManyRequests {
return true
}
if res.Header().Get("Retry-After") == "" {
return false
}
switch res.StatusCode() {
case http.StatusServiceUnavailable, http.StatusGatewayTimeout:
return true
default:
return false
}
}
func isRateLimitCode(code string) bool {
normalized := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(code), "_", ""))
normalized = strings.ReplaceAll(normalized, "-", "")
switch normalized {
case "toomanyrequests",
"activitylimitreached",
"throttledrequest",
"requestthrottled",
"resourcethrottled",
"applicationthrottled",
"tenantthrottled":
return true
default:
return false
}
}
func isRateLimitMessage(message string) bool {
text := strings.ToLower(strings.TrimSpace(message))
if text == "" {
return false
}
return strings.Contains(text, "too many requests") ||
strings.Contains(text, "throttl") ||
strings.Contains(text, "rate limit") ||
strings.Contains(text, "activity limit") ||
strings.Contains(text, "temporarily blocked")
}
func onedriveRateLimitError(res *resty.Response, message string) error {
+199 -1
View File
@@ -2,6 +2,8 @@ package onedrive
import (
"context"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"errors"
"io"
@@ -199,7 +201,7 @@ func TestGraph429ReturnsRateLimitErrorWithRetryAfter(t *testing.T) {
APIBaseURL: srv.URL,
})
_, err := d.List(context.Background(), "root")
_, err := d.StreamURL(context.Background(), "file-id")
if err == nil {
t.Fatal("list succeeded, want rate limit error")
}
@@ -212,6 +214,92 @@ func TestGraph429ReturnsRateLimitErrorWithRetryAfter(t *testing.T) {
}
}
func TestGraphThrottleMessageReturnsRateLimitError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusForbidden)
if err := json.NewEncoder(w).Encode(map[string]any{
"error": map[string]any{
"code": "generalException",
"message": "The request has been throttled. Please try again later.",
},
}); err != nil {
t.Fatalf("write json: %v", err)
}
}))
defer srv.Close()
d := New(Config{
ID: "od-main",
AccessToken: "access-token",
RefreshToken: "refresh-token",
APIBaseURL: srv.URL,
})
_, err := d.StreamURL(context.Background(), "file-id")
if err == nil {
t.Fatal("list succeeded, want rate limit error")
}
var rateLimit *drives.RateLimitError
if !errors.As(err, &rateLimit) {
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
}
}
func TestListCoolsDownAndRetriesOneDriveRateLimit(t *testing.T) {
var calls int
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1.0/me/drive/items/root/children" {
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
}
calls++
if calls == 1 {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusTooManyRequests)
if err := json.NewEncoder(w).Encode(map[string]any{
"error": map[string]any{
"code": "TooManyRequests",
"message": "throttled",
},
}); err != nil {
t.Fatalf("write json: %v", err)
}
return
}
writeJSON(t, w, map[string]any{
"value": []map[string]any{
{
"id": "file-id",
"name": "demo.mp4",
"size": 100,
"file": map[string]any{"mimeType": "video/mp4"},
},
},
})
}))
defer srv.Close()
d := New(Config{
ID: "od-main",
AccessToken: "access-token",
RefreshToken: "refresh-token",
APIBaseURL: srv.URL,
})
d.listInterval = 0
d.listCooldown = time.Millisecond
got, err := d.List(context.Background(), "root")
if err != nil {
t.Fatalf("list: %v", err)
}
if calls != 2 {
t.Fatalf("calls = %d, want retry after rate limit", calls)
}
if len(got) != 1 || got[0].ID != "file-id" {
t.Fatalf("entries = %#v, want retried file", got)
}
}
func TestStatAndStreamURLUseDriveItemMetadata(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
@@ -320,6 +408,36 @@ func TestEnsureDirCreatesMissingFolders(t *testing.T) {
}
}
func TestRenamePatchesDriveItemName(t *testing.T) {
var body map[string]string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPatch || r.URL.EscapedPath() != "/v1.0/me/drive/items/file-id" {
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
}
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
t.Fatalf("authorization = %q, want bearer token", got)
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatalf("decode body: %v", err)
}
writeJSON(t, w, map[string]any{"id": "file-id", "name": "new name.mp4"})
}))
defer srv.Close()
d := New(Config{
ID: "od-main",
AccessToken: "access-token",
RefreshToken: "refresh-token",
APIBaseURL: srv.URL,
})
if err := d.Rename(context.Background(), "file-id", "new name.mp4"); err != nil {
t.Fatalf("rename: %v", err)
}
if body["name"] != "new name.mp4" {
t.Fatalf("rename body = %#v, want new name", body)
}
}
func TestUploadSmallFileReturnsNewItemID(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
@@ -358,6 +476,86 @@ func TestUploadSmallFileReturnsNewItemID(t *testing.T) {
}
}
func TestUploadLargeFileUsesUploadSessionAndReportsHash(t *testing.T) {
oldThreshold := smallUploadThreshold
oldChunk := uploadSessionChunk
smallUploadThreshold = 8
uploadSessionChunk = 4
t.Cleanup(func() {
smallUploadThreshold = oldThreshold
uploadSessionChunk = oldChunk
})
body := "0123456789abc"
var ranges []string
var chunks []string
var createdSession bool
var srv *httptest.Server
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodPost && r.URL.EscapedPath() == "/v1.0/me/drive/items/parent-id:/big.mp4:/createUploadSession":
createdSession = true
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
t.Fatalf("authorization = %q, want bearer token", got)
}
writeJSON(t, w, map[string]any{"uploadUrl": srv.URL + "/upload-session"})
case r.Method == http.MethodPut && r.URL.Path == "/upload-session":
ranges = append(ranges, r.Header.Get("Content-Range"))
data, err := io.ReadAll(r.Body)
if err != nil {
t.Fatalf("read chunk: %v", err)
}
chunks = append(chunks, string(data))
if len(ranges) < 4 {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusAccepted)
if _, err := w.Write([]byte(`{"nextExpectedRanges":["0-"]}`)); err != nil {
t.Fatalf("write accepted: %v", err)
}
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusCreated)
if err := json.NewEncoder(w).Encode(map[string]any{"id": "uploaded-big-id"}); err != nil {
t.Fatalf("write final item: %v", err)
}
default:
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
}
}))
defer srv.Close()
d := New(Config{
ID: "od-main",
AccessToken: "access-token",
RefreshToken: "refresh-token",
APIBaseURL: srv.URL,
})
got, err := d.UploadAndReportHash(context.Background(), "parent-id", "big.mp4", strings.NewReader(body), int64(len(body)))
if err != nil {
t.Fatalf("upload: %v", err)
}
if !createdSession {
t.Fatal("createUploadSession was not called")
}
wantRanges := []string{
"bytes 0-3/13",
"bytes 4-7/13",
"bytes 8-11/13",
"bytes 12-12/13",
}
if strings.Join(ranges, "|") != strings.Join(wantRanges, "|") {
t.Fatalf("ranges = %#v, want %#v", ranges, wantRanges)
}
if strings.Join(chunks, "") != body {
t.Fatalf("uploaded chunks = %q, want %q", strings.Join(chunks, ""), body)
}
sum := sha1.Sum([]byte(body))
if got.FileID != "uploaded-big-id" || got.Size != int64(len(body)) || got.Hash != hex.EncodeToString(sum[:]) {
t.Fatalf("upload result = %#v, want file id/hash/size for body", got)
}
}
func TestUploadRefreshesExpiredTokenAndReplaysBody(t *testing.T) {
var uploadAttempts int
var tokenRefreshes int
+10
View File
@@ -82,3 +82,13 @@ type filesResp struct {
Value []graphItem `json:"value"`
NextLink string `json:"@odata.nextLink"`
}
type UploadResult struct {
FileID string
Hash string
Size int64
}
type uploadSessionResp struct {
UploadURL string `json:"uploadUrl"`
}
+54 -1
View File
@@ -355,7 +355,60 @@ func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
}
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
return "", drives.ErrNotSupported
currentID := d.rootID
for _, name := range splitPath(pathFromRoot) {
childID, err := d.findChildDir(ctx, currentID, name)
if err != nil {
return "", err
}
if childID == "" {
childID, err = d.makeDir(ctx, currentID, name)
if err != nil {
return "", err
}
}
currentID = childID
}
return currentID, nil
}
func (d *Driver) findChildDir(ctx context.Context, parentID, name string) (string, error) {
entries, err := d.List(ctx, parentID)
if err != nil {
return "", err
}
for _, e := range entries {
if e.IsDir && e.Name == name {
return e.ID, nil
}
}
return "", nil
}
func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, error) {
var out file
err := d.request(ctx, filesURL, http.MethodPost, func(req *resty.Request) {
req.SetBody(map[string]any{
"kind": "drive#folder",
"parent_id": parentID,
"name": name,
})
}, &out)
if err != nil {
return "", fmt.Errorf("pikpak mkdir %s: %w", name, err)
}
if out.ID == "" {
return "", fmt.Errorf("pikpak mkdir %s: empty folder id", name)
}
return out.ID, nil
}
func splitPath(p string) []string {
p = strings.Trim(p, "/")
if p == "" {
return nil
}
return strings.Split(p, "/")
}
func (d *Driver) getFiles(ctx context.Context, parentID string) ([]file, error) {
+83 -7
View File
@@ -1,10 +1,12 @@
package pikpak
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/video-site/backend/internal/drives"
)
func TestNewDefaults(t *testing.T) {
@@ -95,11 +97,85 @@ func TestFolderToEntry(t *testing.T) {
}
}
func TestEnsureDirStillUnsupported(t *testing.T) {
d := New(Config{ID: "pikpak-main"})
func TestEnsureDirReusesExistingFolder(t *testing.T) {
var postCalled bool
mux := http.NewServeMux()
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
if got := r.URL.Query().Get("parent_id"); got != "root-id" {
t.Fatalf("parent_id = %q, want root-id", got)
}
writePikPakJSON(t, w, map[string]any{
"files": []map[string]any{{
"id": "existing-folder-id",
"kind": "drive#folder",
"name": "91 Spider",
}},
})
case http.MethodPost:
postCalled = true
t.Fatalf("existing folder should not be created again")
default:
t.Fatalf("unexpected method %s", r.Method)
}
})
srv := httptest.NewServer(mux)
defer srv.Close()
if _, err := d.EnsureDir(nil, "/previews"); err != drives.ErrNotSupported {
t.Fatalf("EnsureDir error = %v, want ErrNotSupported", err)
d := newTestDriver(t, srv)
got, err := d.EnsureDir(context.Background(), "91 Spider")
if err != nil {
t.Fatalf("ensure dir: %v", err)
}
if got != "existing-folder-id" {
t.Fatalf("dir id = %q, want existing-folder-id", got)
}
if postCalled {
t.Fatal("POST should not be called")
}
}
func TestEnsureDirCreatesMissingFolder(t *testing.T) {
var got uploadRequestBody
mux := http.NewServeMux()
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
writePikPakJSON(t, w, map[string]any{"files": []map[string]any{}})
case http.MethodPost:
if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
t.Fatalf("decode create folder body: %v", err)
}
writePikPakJSON(t, w, map[string]any{
"id": "new-folder-id",
"kind": "drive#folder",
"name": "91 Spider",
})
default:
t.Fatalf("unexpected method %s", r.Method)
}
})
srv := httptest.NewServer(mux)
defer srv.Close()
d := newTestDriver(t, srv)
id, err := d.EnsureDir(context.Background(), "91 Spider")
if err != nil {
t.Fatalf("ensure dir: %v", err)
}
if id != "new-folder-id" {
t.Fatalf("dir id = %q, want new-folder-id", id)
}
if got.Kind != "drive#folder" || got.ParentID != "root-id" || got.Name != "91 Spider" {
t.Fatalf("create folder body = %#v", got)
}
}
func writePikPakJSON(t *testing.T, w http.ResponseWriter, body any) {
t.Helper()
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(body); err != nil {
t.Fatalf("write json: %v", err)
}
// Upload 的真实实现见 upload_test.go。
}
+1 -2
View File
@@ -463,8 +463,7 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
// 网站封面下载失败的视频:spider91 drive 的 thumb worker 按设计不
// 处理 spider91 视频(封面应是网站原图直接保存),所以没人接手。
// 显式标 'failed' 让 CountVideosNeedingThumbnail 排除(条件 status
// != 'failed'),否则 enqueueDriveGeneration → waitForThumbnailsBeforePreview
// 会因为 count > 0 把 teaser 入队永远卡在等待循环里。
// != 'failed'),避免后续封面补队列一直重复捞到这条视频。
_ = c.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
ThumbnailStatus: "failed",
})
@@ -234,13 +234,13 @@ func TestCrawlerRunOnceMissingScript(t *testing.T) {
}
// TestCrawlerThumbDownloadFailureMarksStatusFailed 验证:网站封面下载失败时
// crawler 把 thumbnail_status 显式标 'failed',避免 enqueueDriveGeneration 的
// waitForThumbnailsBeforePreview 因为 count > 0 把 teaser 卡死等待
// crawler 把 thumbnail_status 显式标 'failed',避免后续封面补队列一直重复
// 捞到这条 spider91 视频
//
// 历史 bug:之前 thumb 下载失败仅打 logurl=”, status 走 schema DEFAULT 'pending'。
// CountVideosNeedingThumbnail 条件是 url=” AND status != 'failed' → count=1。
// spider91 drive 的 thumb worker 按设计不处理 spider91 视频 → 没人会改 status
// 结果 teaser 永远卡在 [preview] waiting for 1 thumbnails before teaser generation
// spider91 drive 的 thumb worker 按设计不处理 spider91 视频 → 没人会改 status
// 后续补队列会一直认为它还缺封面
func TestCrawlerThumbDownloadFailureMarksStatusFailed(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("shell-based fake script only on unix")
@@ -317,8 +317,7 @@ func TestCrawlerThumbDownloadFailureMarksStatusFailed(t *testing.T) {
// 关键断言:CountVideosNeedingThumbnail 应该返回 0。
// 该函数的 SQL 条件是 `url = '' AND status != 'failed'`;如果 crawler 没把
// status 标 'failed'schema DEFAULT 'pending'),count 就会是 1,外层
// waitForThumbnailsBeforePreview 会因为 count > 0 把 teaser 卡死等待。
// status 标 'failed'schema DEFAULT 'pending'),count 就会是 1
count, err := cat.CountVideosNeedingThumbnail(context.Background(), driveID)
if err != nil {
t.Fatalf("count: %v", err)
+468
View File
@@ -0,0 +1,468 @@
package fingerprint
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/drives"
)
const (
defaultSampleSizeBytes int64 = 512 * 1024
defaultFullHashMaxSize int64 = 8 * 1024 * 1024
defaultCooldown = 5 * time.Minute
defaultWorkerQueueSize = 10000
)
type Config struct {
SampleSizeBytes int64
FullHashMaxSize int64
RateLimitCooldown time.Duration
HTTPClient *http.Client
}
type Worker struct {
Catalog *catalog.Catalog
Drive drives.Drive
Config Config
ch chan *catalog.Video
queue videoQueue
activity taskActivity
cooldown cooldownState
http *http.Client
}
type TaskStatus struct {
State string
CurrentTitle string
QueueLength int
CooldownUntil time.Time
}
func NewWorker(cat *catalog.Catalog, drv drives.Drive, cfg Config) *Worker {
hc := cfg.HTTPClient
if hc == nil {
hc = &http.Client{Timeout: 0}
}
if cfg.SampleSizeBytes <= 0 {
cfg.SampleSizeBytes = defaultSampleSizeBytes
}
if cfg.FullHashMaxSize <= 0 {
cfg.FullHashMaxSize = defaultFullHashMaxSize
}
if cfg.RateLimitCooldown <= 0 {
cfg.RateLimitCooldown = defaultCooldown
}
return &Worker{
Catalog: cat,
Drive: drv,
Config: cfg,
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
http: hc,
}
}
func (w *Worker) Enqueue(v *catalog.Video) bool {
if v == nil {
return false
}
if !w.queue.reserve(v.ID) {
return true
}
select {
case w.ch <- v:
return true
default:
w.queue.release(v.ID)
return false
}
}
func (w *Worker) EnqueueBlocking(ctx context.Context, v *catalog.Video) bool {
if v == nil {
return false
}
if !w.queue.reserve(v.ID) {
return true
}
select {
case w.ch <- v:
return true
case <-ctx.Done():
w.queue.release(v.ID)
return false
}
}
func (w *Worker) Run(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case v := <-w.ch:
w.processQueued(ctx, v)
select {
case <-ctx.Done():
return
case <-time.After(500 * time.Millisecond):
}
}
}
}
func (w *Worker) Status() TaskStatus {
if w == nil {
return TaskStatus{State: "idle"}
}
currentID, currentTitle := w.activity.current()
status := TaskStatus{
State: "idle",
CurrentTitle: currentTitle,
QueueLength: w.queue.lengthExcluding(currentID),
}
if until, ok := w.cooldown.active(time.Now()); ok {
status.State = "cooling"
status.CooldownUntil = until
return status
}
if currentID != "" {
status.State = "generating"
return status
}
if status.QueueLength > 0 {
status.State = "queued"
}
return status
}
func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
defer w.queue.release(v.ID)
if w.Catalog == nil || w.Drive == nil || v == nil || v.ID == "" {
return
}
current, err := w.Catalog.GetVideo(ctx, v.ID)
if err != nil {
return
}
if current.SampledSHA256 != "" || current.FingerprintStatus == "ready" || current.Hidden {
return
}
w.activity.start(current)
defer w.activity.done()
sum, err := Compute(ctx, w.Drive, current, w.Config, w.http)
if err != nil {
var rl *drives.RateLimitError
if errors.As(err, &rl) {
wait := rl.RetryAfter
if wait <= 0 {
wait = w.Config.RateLimitCooldown
}
until := time.Now().Add(wait)
w.cooldown.set(until)
log.Printf("[fingerprint] drive=%s rate limited; keep video=%s pending and cool down for %s: %v", w.Drive.ID(), current.ID, wait, err)
sleepContext(ctx, wait)
w.cooldown.clear(until)
return
}
log.Printf("[fingerprint] video=%s failed: %v", current.ID, err)
_ = w.Catalog.UpdateVideoFingerprint(ctx, current.ID, "", "failed", err.Error())
return
}
if err := w.Catalog.UpdateVideoFingerprint(ctx, current.ID, sum, "ready", ""); err != nil {
log.Printf("[fingerprint] update video=%s: %v", current.ID, err)
return
}
log.Printf("[fingerprint] video=%s ready sampled_sha256=%s", current.ID, sum)
}
func Compute(ctx context.Context, drv drives.Drive, v *catalog.Video, cfg Config, hc *http.Client) (string, error) {
if drv == nil {
return "", errors.New("fingerprint: nil drive")
}
if v == nil {
return "", errors.New("fingerprint: nil video")
}
if v.Size <= 0 {
return "", errors.New("fingerprint: video size is empty")
}
if cfg.SampleSizeBytes <= 0 {
cfg.SampleSizeBytes = defaultSampleSizeBytes
}
if cfg.FullHashMaxSize <= 0 {
cfg.FullHashMaxSize = defaultFullHashMaxSize
}
if hc == nil {
hc = &http.Client{Timeout: 0}
}
link, err := drv.StreamURL(ctx, v.FileID)
if err != nil {
return "", fmt.Errorf("fingerprint: stream url: %w", err)
}
if link == nil || strings.TrimSpace(link.URL) == "" {
return "", errors.New("fingerprint: empty stream url")
}
ranges := sampleRanges(v.Size, cfg.SampleSizeBytes, cfg.FullHashMaxSize)
h := sha256.New()
writeHashHeader(h, v.Size, ranges)
for _, r := range ranges {
data, err := readRange(ctx, hc, link, r)
if err != nil {
return "", err
}
if int64(len(data)) != r.length {
return "", fmt.Errorf("fingerprint: short sample at %d: got %d want %d", r.start, len(data), r.length)
}
_, _ = h.Write([]byte(fmt.Sprintf("offset=%d length=%d\n", r.start, r.length)))
_, _ = h.Write(data)
_, _ = h.Write([]byte("\n"))
}
return hex.EncodeToString(h.Sum(nil)), nil
}
type byteRange struct {
start int64
length int64
}
func sampleRanges(size, sampleSize, fullHashMax int64) []byteRange {
if size <= fullHashMax {
return []byteRange{{start: 0, length: size}}
}
if sampleSize > size {
sampleSize = size
}
maxStart := size - sampleSize
percents := []int64{0, 20, 40, 60, 80}
out := make([]byteRange, 0, len(percents))
seen := make(map[int64]struct{}, len(percents))
for _, pct := range percents {
start := maxStart * pct / 100
if _, ok := seen[start]; ok {
continue
}
seen[start] = struct{}{}
out = append(out, byteRange{start: start, length: sampleSize})
}
return out
}
func writeHashHeader(w io.Writer, size int64, ranges []byteRange) {
_, _ = fmt.Fprintf(w, "video-site-sampled-sha256-v1\nsize=%d\nsamples=%d\n", size, len(ranges))
}
func readRange(ctx context.Context, hc *http.Client, link *drives.StreamLink, r byteRange) ([]byte, error) {
u, err := url.Parse(link.URL)
if err == nil && (u.Scheme == "http" || u.Scheme == "https") {
return readHTTPRange(ctx, hc, link, r)
}
path := link.URL
if err == nil && u.Scheme == "file" {
path = u.Path
}
return readLocalRange(path, r)
}
func readLocalRange(path string, r byteRange) ([]byte, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("fingerprint: open local stream: %w", err)
}
defer f.Close()
buf := make([]byte, r.length)
n, err := f.ReadAt(buf, r.start)
if err != nil && !errors.Is(err, io.EOF) {
return nil, fmt.Errorf("fingerprint: read local sample: %w", err)
}
if int64(n) != r.length {
return nil, fmt.Errorf("fingerprint: read local sample at %d: got %d want %d", r.start, n, r.length)
}
return buf, nil
}
func readHTTPRange(ctx context.Context, hc *http.Client, link *drives.StreamLink, r byteRange) ([]byte, error) {
end := r.start + r.length - 1
req, err := http.NewRequestWithContext(ctx, http.MethodGet, link.URL, nil)
if err != nil {
return nil, err
}
for k, vs := range link.Headers {
for _, v := range vs {
req.Header.Add(k, v)
}
}
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", r.start, end))
resp, err := hc.Do(req)
if err != nil {
return nil, fmt.Errorf("fingerprint: read remote sample: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusTooManyRequests {
return nil, &drives.RateLimitError{
Provider: "fingerprint",
RetryAfter: parseRetryAfter(resp.Header.Get("Retry-After")),
Err: fmt.Errorf("remote sample rate limited: status=%d", resp.StatusCode),
}
}
if resp.StatusCode != http.StatusPartialContent {
if resp.StatusCode == http.StatusOK && r.start == 0 {
data, err := io.ReadAll(io.LimitReader(resp.Body, r.length+1))
if err != nil {
return nil, err
}
if int64(len(data)) == r.length {
return data, nil
}
}
return nil, fmt.Errorf("fingerprint: range request got status=%d for bytes=%d-%d", resp.StatusCode, r.start, end)
}
return io.ReadAll(io.LimitReader(resp.Body, r.length))
}
func parseRetryAfter(raw string) time.Duration {
raw = strings.TrimSpace(raw)
if raw == "" {
return 0
}
if seconds, err := strconv.Atoi(raw); err == nil && seconds > 0 {
return time.Duration(seconds) * time.Second
}
if when, err := http.ParseTime(raw); err == nil {
d := time.Until(when)
if d > 0 {
return d
}
}
return 0
}
func sleepContext(ctx context.Context, d time.Duration) bool {
if d <= 0 {
return true
}
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return false
case <-timer.C:
return true
}
}
type taskActivity struct {
mu sync.Mutex
currentID string
currentTitle string
}
func (a *taskActivity) start(v *catalog.Video) {
a.mu.Lock()
defer a.mu.Unlock()
if v == nil {
a.currentID = ""
a.currentTitle = ""
return
}
a.currentID = v.ID
a.currentTitle = v.Title
}
func (a *taskActivity) done() {
a.mu.Lock()
a.currentID = ""
a.currentTitle = ""
a.mu.Unlock()
}
func (a *taskActivity) current() (string, string) {
a.mu.Lock()
defer a.mu.Unlock()
return a.currentID, a.currentTitle
}
type cooldownState struct {
mu sync.Mutex
until time.Time
}
func (s *cooldownState) set(until time.Time) {
s.mu.Lock()
s.until = until
s.mu.Unlock()
}
func (s *cooldownState) clear(until time.Time) {
s.mu.Lock()
if s.until.Equal(until) {
s.until = time.Time{}
}
s.mu.Unlock()
}
func (s *cooldownState) active(now time.Time) (time.Time, bool) {
s.mu.Lock()
defer s.mu.Unlock()
if s.until.IsZero() || !s.until.After(now) {
return time.Time{}, false
}
return s.until, true
}
type videoQueue struct {
mu sync.Mutex
ids map[string]struct{}
}
func (q *videoQueue) reserve(id string) bool {
if id == "" {
return true
}
q.mu.Lock()
defer q.mu.Unlock()
if q.ids == nil {
q.ids = make(map[string]struct{})
}
if _, ok := q.ids[id]; ok {
return false
}
q.ids[id] = struct{}{}
return true
}
func (q *videoQueue) release(id string) {
if id == "" {
return
}
q.mu.Lock()
delete(q.ids, id)
q.mu.Unlock()
}
func (q *videoQueue) lengthExcluding(currentID string) int {
q.mu.Lock()
defer q.mu.Unlock()
n := len(q.ids)
if currentID != "" {
if _, ok := q.ids[currentID]; ok {
n--
}
}
if n < 0 {
return 0
}
return n
}
+112
View File
@@ -0,0 +1,112 @@
package fingerprint
import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"time"
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/drives"
)
func TestComputeLocalFilesWithSameContentMatch(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
body := []byte("same video bytes")
a := filepath.Join(dir, "a.mp4")
b := filepath.Join(dir, "b.mp4")
if err := os.WriteFile(a, body, 0o644); err != nil {
t.Fatalf("write a: %v", err)
}
if err := os.WriteFile(b, body, 0o644); err != nil {
t.Fatalf("write b: %v", err)
}
sumA, err := Compute(ctx, &fakeDrive{paths: map[string]string{"a": a}}, &catalog.Video{ID: "a", FileID: "a", Size: int64(len(body))}, Config{}, nil)
if err != nil {
t.Fatalf("compute a: %v", err)
}
sumB, err := Compute(ctx, &fakeDrive{paths: map[string]string{"b": b}}, &catalog.Video{ID: "b", FileID: "b", Size: int64(len(body))}, Config{}, nil)
if err != nil {
t.Fatalf("compute b: %v", err)
}
if sumA == "" || sumA != sumB {
t.Fatalf("fingerprints = %q / %q, want same non-empty", sumA, sumB)
}
}
func TestComputeRemoteUsesRangeSamples(t *testing.T) {
ctx := context.Background()
data := make([]byte, 10*1024*1024)
for i := range data {
data[i] = byte(i % 251)
}
var ranges []string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rawRange := r.Header.Get("Range")
ranges = append(ranges, rawRange)
var start, end int
if _, err := fmt.Sscanf(rawRange, "bytes=%d-%d", &start, &end); err != nil {
t.Fatalf("bad range %q: %v", rawRange, err)
}
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
w.WriteHeader(http.StatusPartialContent)
_, _ = w.Write(data[start : end+1])
}))
defer srv.Close()
drv := &fakeDrive{paths: map[string]string{"remote": srv.URL + "/video.mp4"}}
sum, err := Compute(ctx, drv, &catalog.Video{ID: "remote", FileID: "remote", Size: int64(len(data))}, Config{
SampleSizeBytes: 4,
FullHashMaxSize: 8,
HTTPClient: srv.Client(),
}, srv.Client())
if err != nil {
t.Fatalf("compute remote: %v", err)
}
if sum == "" {
t.Fatal("fingerprint should not be empty")
}
want := []string{
"bytes=0-3",
"bytes=2097151-2097154",
"bytes=4194302-4194305",
"bytes=6291453-6291456",
"bytes=8388604-8388607",
}
if fmt.Sprint(ranges) != fmt.Sprint(want) {
t.Fatalf("ranges = %#v, want %#v", ranges, want)
}
}
type fakeDrive struct {
paths map[string]string
}
func (d *fakeDrive) Kind() string { return "fake" }
func (d *fakeDrive) ID() string { return "fake" }
func (d *fakeDrive) Init(context.Context) error {
return nil
}
func (d *fakeDrive) List(context.Context, string) ([]drives.Entry, error) {
return nil, drives.ErrNotSupported
}
func (d *fakeDrive) Stat(context.Context, string) (*drives.Entry, error) {
return nil, drives.ErrNotSupported
}
func (d *fakeDrive) StreamURL(_ context.Context, fileID string) (*drives.StreamLink, error) {
return &drives.StreamLink{URL: d.paths[fileID], Expires: time.Now().Add(time.Minute)}, nil
}
func (d *fakeDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
return "", drives.ErrNotSupported
}
func (d *fakeDrive) EnsureDir(context.Context, string) (string, error) {
return "", drives.ErrNotSupported
}
func (d *fakeDrive) RootID() string { return "root" }
+23
View File
@@ -12,6 +12,8 @@
// wait until teaser queues are idle
// Phase 3: spider91 → cloud migration (single sweep, captcha cooldown still
// honored within this call)
// Phase 4: cleanup duplicate local preview/thumbnail assets after sampled
// fingerprints have identified canonical videos
//
// A 6h soft deadline guards each pipeline run; phases check deadline at their
// boundaries and exit cleanly if exceeded (no in-flight ffmpeg / upload is
@@ -85,6 +87,11 @@ type Config struct {
// RunMigration runs spider91migrate.Migrator.RunOnce for Phase 3.
RunMigration func(ctx context.Context) error
// RunDedupeAssetCleanup removes generated local assets from non-canonical
// videos in size+sampled_sha256 duplicate groups. It must not delete cloud
// files or catalog rows.
RunDedupeAssetCleanup func(ctx context.Context) error
// Now is injected for tests; nil → time.Now.
Now func() time.Time
}
@@ -241,6 +248,7 @@ func (r *Runner) runPipeline(ctx context.Context) {
}
if len(spiderIDs) == 0 {
log.Printf("[nightly] phase 2/3 skipped: no spider91 drive configured")
r.runDedupeAssetCleanupPhase(ctx)
return
}
log.Printf("[nightly] phase 2: crawling %d spider91 drive(s)", len(spiderIDs))
@@ -267,6 +275,8 @@ func (r *Runner) runPipeline(ctx context.Context) {
log.Printf("[nightly] phase 3 migration: %v", err)
}
}
r.runDedupeAssetCleanupPhase(ctx)
}
// checkDeadline returns true when ctx is already done (runner shutting down or
@@ -292,6 +302,19 @@ func (r *Runner) waitIdle(ctx context.Context, phase string) error {
return nil
}
func (r *Runner) runDedupeAssetCleanupPhase(ctx context.Context) {
if r.checkDeadline(ctx, "phase 4") {
return
}
if r.cfg.RunDedupeAssetCleanup == nil {
return
}
log.Printf("[nightly] phase 4: duplicate asset cleanup")
if err := r.cfg.RunDedupeAssetCleanup(ctx); err != nil {
log.Printf("[nightly] phase 4 duplicate asset cleanup: %v", err)
}
}
// readLastRunDate reads the persisted last_run_date or returns "" when unset.
func (r *Runner) readLastRunDate(ctx context.Context) (string, error) {
if r.cfg.Settings == nil {
+22
View File
@@ -114,6 +114,10 @@ func TestRunPipelineHonoursPhaseOrder(t *testing.T) {
rec.push("migrate")
return nil
},
RunDedupeAssetCleanup: func(context.Context) error {
rec.push("dedupe-cleanup")
return nil
},
})
r.runPipeline(context.Background())
@@ -128,6 +132,7 @@ func TestRunPipelineHonoursPhaseOrder(t *testing.T) {
"crawl:sp-1",
"wait-idle", // after phase 2
"migrate",
"dedupe-cleanup",
}
if len(got) != len(want) {
t.Fatalf("call sequence len = %d, want %d; got=%v", len(got), len(want), got)
@@ -156,6 +161,10 @@ func TestRunPipelineSkipsMigrationWhenNoSpider91(t *testing.T) {
rec.push("migrate")
return nil
},
RunDedupeAssetCleanup: func(context.Context) error {
rec.push("dedupe-cleanup")
return nil
},
})
r.runPipeline(context.Background())
@@ -165,6 +174,15 @@ func TestRunPipelineSkipsMigrationWhenNoSpider91(t *testing.T) {
t.Fatalf("phase 2/3 should be skipped when no spider91 drive, got call %q", c)
}
}
foundCleanup := false
for _, c := range rec.snapshot() {
if c == "dedupe-cleanup" {
foundCleanup = true
}
}
if !foundCleanup {
t.Fatalf("dedupe cleanup should still run when spider91 is absent; calls=%v", rec.snapshot())
}
}
func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
@@ -186,6 +204,7 @@ func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
RunSpider91Crawl: func(context.Context, string) { rec.push("crawl") },
WaitPreviewQueuesIdle: func(context.Context) error { rec.push("wait-idle"); return nil },
RunMigration: func(context.Context) error { rec.push("migrate"); return nil },
RunDedupeAssetCleanup: func(context.Context) error { rec.push("dedupe-cleanup"); return nil },
})
r.runPipeline(ctx)
@@ -200,6 +219,9 @@ func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
if c == "crawl" || c == "migrate" {
t.Fatalf("subsequent phase should not run after cancel, got call %q", c)
}
if c == "dedupe-cleanup" {
t.Fatalf("dedupe cleanup should not run after cancel, got call %q", c)
}
}
}
+71 -18
View File
@@ -929,6 +929,7 @@ func ffmpegOutputLooksRateLimited(output []byte) bool {
return false
}
return strings.Contains(text, "too many requests") ||
strings.Contains(text, "throttl") ||
strings.Contains(text, "rate limit") ||
strings.Contains(text, "rate-limit") ||
strings.Contains(text, "server returned 429")
@@ -974,7 +975,6 @@ type Worker struct {
queue videoQueue
RateLimitCooldown time.Duration
BeforeTask func(context.Context) bool
rateLimit rateLimitState
activity taskActivity
}
@@ -984,7 +984,7 @@ func NewWorker(gen TeaserGenerator, cat *catalog.Catalog, drv drives.Drive) *Wor
Gen: gen,
Catalog: cat,
Drive: drv,
ch: make(chan *catalog.Video, 4096),
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
}
}
@@ -1035,6 +1035,7 @@ type ThumbWorker struct {
const (
defaultTransientMediaCooldown = 5 * time.Minute
defaultGenerationRateLimitCooldown = 5 * time.Minute
defaultWorkerQueueSize = 10000
maxPreviewTeaserSizeBytes int64 = 5 * 1024 * 1024 * 1024
previewStatusSkipped = "skipped"
)
@@ -1174,7 +1175,7 @@ func NewThumbWorker(gen ThumbnailGenerator, cat *catalog.Catalog, drv drives.Dri
Gen: gen,
Catalog: cat,
Drive: drv,
ch: make(chan *catalog.Video, 4096),
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
}
}
@@ -1329,10 +1330,6 @@ func (w *ThumbWorker) Run(ctx context.Context) {
func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
defer w.queue.release(v)
if w.BeforeTask != nil && !w.BeforeTask(ctx) {
return
}
w.activity.start(v)
defer w.activity.done()
if !waitForRateLimitCooldown(ctx, &w.rateLimit, "preview", w.Drive) {
@@ -1488,29 +1485,53 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
if w.skipIfRateLimited(v) {
return
}
if current, err := w.Catalog.GetVideo(ctx, v.ID); err == nil {
if current.ThumbnailURL != "" {
queued := v
current := v
if loaded, err := w.Catalog.GetVideo(ctx, v.ID); err == nil {
if loaded.PreviewLocal == "" {
loaded.PreviewLocal = queued.PreviewLocal
}
current = loaded
v = loaded
if loaded.ThumbnailURL != "" && loaded.DurationSeconds > 0 {
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
return
}
}
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "pending"})
link, err := w.Drive.StreamURL(ctx, v.FileID)
if err != nil {
if localLink, ok := localPreviewLink(v); ok {
link = localLink
} else {
if w.pauseForRecoverableError(err, "streamURL", v.Title) {
if current.ThumbnailURL != "" {
if current.DurationSeconds <= 0 {
link, err := w.streamLink(ctx, current)
if err != nil {
if w.pauseForRecoverableError(err, "streamURL", current.Title) {
return
}
log.Printf("[thumb] probe streamURL %s: %v", current.Title, err)
} else if w.probeDuration(ctx, current, link) {
return
}
log.Printf("[thumb] streamURL %s: %v", v.Title, err)
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
}
_ = w.Catalog.UpdateVideoMeta(ctx, current.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
return
}
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "pending"})
link, err := w.streamLink(ctx, v)
if err != nil {
if w.pauseForRecoverableError(err, "streamURL", v.Title) {
return
}
log.Printf("[thumb] streamURL %s: %v", v.Title, err)
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
return
}
if w.probeDuration(ctx, v, link) {
return
}
if err := w.generateThumbnailFromLink(ctx, v, link); err != nil {
if localLink, ok := localPreviewLink(v); ok && link.URL != localLink.URL {
if w.probeDuration(ctx, v, localLink) {
return
}
if localErr := w.generateThumbnailFromLink(ctx, v, localLink); localErr == nil {
return
}
@@ -1524,6 +1545,38 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
}
}
func (w *ThumbWorker) streamLink(ctx context.Context, v *catalog.Video) (*drives.StreamLink, error) {
link, err := w.Drive.StreamURL(ctx, v.FileID)
if err == nil {
return link, nil
}
if localLink, ok := localPreviewLink(v); ok {
return localLink, nil
}
return nil, err
}
func (w *ThumbWorker) probeDuration(ctx context.Context, v *catalog.Video, link *drives.StreamLink) bool {
if v.DurationSeconds > 0 {
return false
}
dur, err := w.Gen.Probe(ctx, link)
if err == nil {
if dur > 0 {
v.DurationSeconds = int(dur)
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
DurationSeconds: int(dur),
})
}
return false
}
if w.pauseForRecoverableError(err, "probe", v.Title) {
return true
}
log.Printf("[thumb] probe %s: %v", v.Title, err)
return false
}
func (w *ThumbWorker) generateThumbnailFromLink(ctx context.Context, v *catalog.Video, link *drives.StreamLink) error {
if _, err := w.Gen.GenerateThumbnail(ctx, link, v.ID, 0); err != nil {
return err
+51 -8
View File
@@ -13,11 +13,11 @@ import (
"github.com/video-site/backend/internal/drives"
)
func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
func TestThumbWorkerUpdatesThumbnailAndDurationWithoutChangingPreviewStatus(t *testing.T) {
ctx := context.Background()
cat, video := seedPreviewTestVideo(t, "thumb-worker-video")
gen := &fakeThumbGenerator{}
gen := &fakeThumbGenerator{probeDuration: 42}
drv := &previewFakeDrive{}
worker := NewThumbWorker(gen, cat, drv)
@@ -33,8 +33,8 @@ func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
if got.PreviewStatus != "pending" {
t.Fatalf("preview status = %q, want pending", got.PreviewStatus)
}
if got.DurationSeconds != 0 {
t.Fatalf("duration = %d, want unchanged", got.DurationSeconds)
if got.DurationSeconds != 42 {
t.Fatalf("duration = %d, want probed duration", got.DurationSeconds)
}
if gen.thumbnailVideoID != video.ID {
t.Fatalf("thumbnail video id = %q, want %q", gen.thumbnailVideoID, video.ID)
@@ -42,14 +42,53 @@ func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
if gen.thumbnailDuration != 0 {
t.Fatalf("thumbnail duration = %.1f, want fixed-offset thumbnail generation", gen.thumbnailDuration)
}
if gen.probeCalls != 0 {
t.Fatalf("probe calls = %d, want 0 for thumbnail generation", gen.probeCalls)
if gen.probeCalls != 1 {
t.Fatalf("probe calls = %d, want 1 for thumbnail generation", gen.probeCalls)
}
if drv.streamFileID != video.FileID {
t.Fatalf("stream file id = %q, want %q", drv.streamFileID, video.FileID)
}
}
func TestThumbWorkerBackfillsDurationWhenThumbnailAlreadyExists(t *testing.T) {
ctx := context.Background()
cat, video := seedPreviewTestVideo(t, "thumb-worker-existing-thumbnail")
video.ThumbnailURL = "/p/thumb/" + video.ID
if err := cat.UpsertVideo(ctx, video); err != nil {
t.Fatalf("update video: %v", err)
}
gen := &fakeThumbGenerator{probeDuration: 19}
drv := &previewFakeDrive{}
worker := NewThumbWorker(gen, cat, drv)
worker.process(ctx, video)
got, err := cat.GetVideo(ctx, video.ID)
if err != nil {
t.Fatalf("get video: %v", err)
}
if got.DurationSeconds != 19 {
t.Fatalf("duration = %d, want probed duration", got.DurationSeconds)
}
if got.ThumbnailURL != "/p/thumb/"+video.ID {
t.Fatalf("thumbnail = %q, want unchanged existing thumbnail", got.ThumbnailURL)
}
ready, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "ready", 0)
if err != nil {
t.Fatalf("list ready thumbnails: %v", err)
}
if len(ready) != 1 || ready[0].ID != video.ID {
t.Fatalf("ready thumbnails = %#v, want only %s", ready, video.ID)
}
if gen.probeCalls != 1 {
t.Fatalf("probe calls = %d, want 1", gen.probeCalls)
}
if gen.thumbnailVideoID != "" {
t.Fatalf("thumbnail generation video id = %q, want no regeneration", gen.thumbnailVideoID)
}
}
func TestThumbWorkerFallsBackToLocalPreviewWhenDriveStreamFails(t *testing.T) {
ctx := context.Background()
cat, video := seedPreviewTestVideo(t, "thumb-worker-local-preview")
@@ -469,12 +508,17 @@ type fakeThumbGenerator struct {
thumbnailDuration float64
thumbnailURL string
probeCalls int
probeDuration float64
probeErr error
generateErr error
}
func (g *fakeThumbGenerator) Probe(context.Context, *drives.StreamLink) (float64, error) {
g.probeCalls++
return 42, nil
if g.probeErr != nil {
return 0, g.probeErr
}
return g.probeDuration, nil
}
func (g *fakeThumbGenerator) GenerateThumbnail(_ context.Context, link *drives.StreamLink, videoID string, duration float64) (string, error) {
@@ -568,7 +612,6 @@ func (d *previewFakeDrive) EnsureDir(context.Context, string) (string, error) {
}
func (d *previewFakeDrive) RootID() string { return "root" }
func TestWorkerWaitIdleReturnsImmediatelyWhenQueueEmpty(t *testing.T) {
worker := NewWorker(&fakeTeaserGenerator{}, nil, &previewFakeDrive{})
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+23 -2
View File
@@ -5,6 +5,7 @@ import (
"io"
"net/http"
"net/url"
"path/filepath"
"sync"
"time"
@@ -144,13 +145,15 @@ func (p *Proxy) ServeStream(w http.ResponseWriter, r *http.Request, driveID, fil
// 302 之后浏览器用自己的 UA 直连,CDN 仍然认签名
// - pikpak:与 OpenList 一致,WebContentLink / media link 都是自签 URL
// CDN 不校验请求头,直连可获得最佳带宽并避免占用 backend 出站
// - onedriveMicrosoft Graph 返回的 @microsoft.graph.downloadUrl 是短期
// 免鉴权下载 URL,不需要后端继续代传视频字节
//
// 其余网盘(如 OneDrive / 沃盘 / 夸克等)仍走反代,因为它们的下载
// 其余网盘(如沃盘 / 夸克等)仍走反代,因为它们的下载
// 链接通常需要随请求带上后端持有的 Cookie / Authorization / Range
// 的特殊处理,浏览器拿不到这些上下文。
func shouldRedirect(d drives.Drive) bool {
switch d.Kind() {
case "p115", "pikpak":
case "p115", "pikpak", "onedrive":
return true
}
return false
@@ -169,6 +172,11 @@ func (p *Proxy) serve(w http.ResponseWriter, r *http.Request, link *drives.Strea
http.Error(w, "bad upstream url", http.StatusBadGateway)
return
}
if localPath, ok := localFilePath(u, link.URL); ok {
w.Header().Set("Cache-Control", "private, max-age=300")
http.ServeFile(w, r, localPath)
return
}
req, err := http.NewRequestWithContext(r.Context(), r.Method, u.String(), nil)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -211,6 +219,19 @@ func (p *Proxy) ServeLocal(w http.ResponseWriter, r *http.Request, path string)
http.ServeFile(w, r, path)
}
func localFilePath(u *url.URL, raw string) (string, bool) {
if u == nil {
return "", false
}
if u.Scheme == "file" && u.Path != "" {
return u.Path, true
}
if u.Scheme == "" && u.Host == "" && filepath.IsAbs(raw) {
return raw, true
}
return "", false
}
var errDriveNotFound = &httpError{Code: http.StatusNotFound, Msg: "drive not found"}
type httpError struct {
+90
View File
@@ -5,6 +5,8 @@ import (
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"time"
@@ -149,6 +151,61 @@ func TestServeStreamPikPakSetsRedirectHeaders(t *testing.T) {
}
}
func TestServeStreamRedirectsOneDrive(t *testing.T) {
reg := NewRegistry()
drv := &proxyFakeSimpleDrive{
kind: "onedrive",
url: "https://public.onedrive.example/video.mp4",
}
reg.Set("onedrive", drv)
p := New(reg)
req := httptest.NewRequest(http.MethodGet, "/p/stream/onedrive/file-1", nil)
rr := httptest.NewRecorder()
p.ServeStream(rr, req, "onedrive", "file-1")
if rr.Code != http.StatusFound {
t.Fatalf("status = %d, want %d", rr.Code, http.StatusFound)
}
if got := rr.Header().Get("Location"); got != "https://public.onedrive.example/video.mp4" {
t.Fatalf("Location = %q", got)
}
if drv.calls != 1 {
t.Fatalf("link calls = %d, want 1", drv.calls)
}
}
func TestServeStreamServesLocalFilePath(t *testing.T) {
path := filepath.Join(t.TempDir(), "video.mp4")
if err := os.WriteFile(path, []byte("0123456789"), 0o644); err != nil {
t.Fatalf("write local file: %v", err)
}
reg := NewRegistry()
drv := &proxyFakeSimpleDrive{
kind: "localstorage",
url: path,
}
reg.Set("local", drv)
p := New(reg)
req := httptest.NewRequest(http.MethodGet, "/p/stream/local/file-1", nil)
req.Header.Set("Range", "bytes=2-5")
rr := httptest.NewRecorder()
p.ServeStream(rr, req, "local", "file-1")
if rr.Code != http.StatusPartialContent {
t.Fatalf("status = %d, want %d", rr.Code, http.StatusPartialContent)
}
if got := rr.Body.String(); got != "2345" {
t.Fatalf("body = %q, want range bytes", got)
}
if drv.calls != 1 {
t.Fatalf("link calls = %d, want 1", drv.calls)
}
}
func requestPikPak(t *testing.T, p *Proxy, driveID, fileID, ua string) {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "/p/stream/"+driveID+"/"+fileID, nil)
@@ -192,3 +249,36 @@ func (d *proxyFakePikPakDrive) EnsureDir(context.Context, string) (string, error
return "", drives.ErrNotSupported
}
func (d *proxyFakePikPakDrive) RootID() string { return "0" }
type proxyFakeSimpleDrive struct {
kind string
url string
calls int
}
func (d *proxyFakeSimpleDrive) Kind() string { return d.kind }
func (d *proxyFakeSimpleDrive) ID() string { return d.kind }
func (d *proxyFakeSimpleDrive) Init(context.Context) error {
return nil
}
func (d *proxyFakeSimpleDrive) List(context.Context, string) ([]drives.Entry, error) {
return nil, drives.ErrNotSupported
}
func (d *proxyFakeSimpleDrive) Stat(context.Context, string) (*drives.Entry, error) {
return nil, drives.ErrNotSupported
}
func (d *proxyFakeSimpleDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
d.calls++
return &drives.StreamLink{
URL: d.url,
Headers: http.Header{},
Expires: time.Now().Add(10 * time.Minute),
}, nil
}
func (d *proxyFakeSimpleDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
return "", drives.ErrNotSupported
}
func (d *proxyFakeSimpleDrive) EnsureDir(context.Context, string) (string, error) {
return "", drives.ErrNotSupported
}
func (d *proxyFakeSimpleDrive) RootID() string { return "0" }
+46 -8
View File
@@ -1,5 +1,5 @@
// Package spider91migrate 周期性把 spider91 drive 下载到本地的视频
// 上传到一个指定的目标 drive 目录(PikPak 或 115),上传成功后:
// 上传到一个指定的目标 drive 目录(PikPak、115 或 OneDrive),上传成功后:
//
// - 改写 catalog 行:drive_id / file_id / content_hash 改成目标盘的;
// 视频自身的 id 不变(仍是 spider91-<driveID>-<viewkey>),video_tags、
@@ -28,6 +28,7 @@ import (
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/drives"
"github.com/video-site/backend/internal/drives/onedrive"
"github.com/video-site/backend/internal/drives/p115"
"github.com/video-site/backend/internal/drives/pikpak"
"github.com/video-site/backend/internal/drives/spider91"
@@ -39,12 +40,14 @@ import (
// 这一层抽象把"迁移调用方"和"具体盘的 SDK 协议"解耦:
// - PikPak 走 GCID + OSS PutObjectpikpak.UploadResult
// - 115 走 SHA1 + 秒传 / OSS / 分片(p115.UploadResult
// - OneDrive 走 SHA1 + 小文件 PUT / 大文件 upload session
//
// 两个返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
// 各家返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
type uploadTarget interface {
ID() string
Kind() string
RootID() string
EnsureDir(ctx context.Context, pathFromRoot string) (string, error)
UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error)
Rename(ctx context.Context, fileID, newName string) error
}
@@ -52,7 +55,7 @@ type uploadTarget interface {
// UploadResult 是 uploadTarget.UploadAndReportHash 的归一返回。
//
// FileID 目标盘上的新文件 ID;
// Hash GCIDPikPak)或 SHA1 HEX 大写115),写入 catalog.content_hash 用于跨盘去重;
// Hash GCIDPikPak)或 SHA1 HEX115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;
// Size 实际上传字节数。
type UploadResult struct {
FileID string
@@ -60,7 +63,9 @@ type UploadResult struct {
Size int64
}
// pikpakAdapter / p115Adapter 把具体 driver 包装成 uploadTarget。
const spider91UploadDirName = "91 Spider"
// pikpakAdapter / p115Adapter / onedriveAdapter 把具体 driver 包装成 uploadTarget。
//
// 之所以不让 driver 直接实现 uploadTarget
//
@@ -74,6 +79,9 @@ type pikpakAdapter struct {
func (a *pikpakAdapter) ID() string { return a.d.ID() }
func (a *pikpakAdapter) Kind() string { return a.d.Kind() }
func (a *pikpakAdapter) RootID() string { return a.d.RootID() }
func (a *pikpakAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
return a.d.EnsureDir(ctx, pathFromRoot)
}
func (a *pikpakAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
if err != nil {
@@ -92,6 +100,9 @@ type p115Adapter struct {
func (a *p115Adapter) ID() string { return a.d.ID() }
func (a *p115Adapter) Kind() string { return a.d.Kind() }
func (a *p115Adapter) RootID() string { return a.d.RootID() }
func (a *p115Adapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
return a.d.EnsureDir(ctx, pathFromRoot)
}
func (a *p115Adapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
res, err := a.d.UploadAndReportSha1(ctx, parentID, name, r, size)
if err != nil {
@@ -103,6 +114,27 @@ func (a *p115Adapter) Rename(ctx context.Context, fileID, newName string) error
return a.d.Rename(ctx, fileID, newName)
}
type onedriveAdapter struct {
d *onedrive.Driver
}
func (a *onedriveAdapter) ID() string { return a.d.ID() }
func (a *onedriveAdapter) Kind() string { return a.d.Kind() }
func (a *onedriveAdapter) RootID() string { return a.d.RootID() }
func (a *onedriveAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
return a.d.EnsureDir(ctx, pathFromRoot)
}
func (a *onedriveAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
if err != nil {
return UploadResult{}, err
}
return UploadResult{FileID: res.FileID, Hash: res.Hash, Size: res.Size}, nil
}
func (a *onedriveAdapter) Rename(ctx context.Context, fileID, newName string) error {
return a.d.Rename(ctx, fileID, newName)
}
// adaptUploadTarget 把通用 drive 包装成 uploadTarget。
// 不支持的盘 kind 返回 error;调用方静默跳过。
func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
@@ -111,6 +143,8 @@ func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
return &pikpakAdapter{d: v}, nil
case *p115.Driver:
return &p115Adapter{d: v}, nil
case *onedrive.Driver:
return &onedriveAdapter{d: v}, nil
case uploadTarget:
// 测试或自定义实现可以直接传入;优先使用具体类型分支以拿到适配器。
return v, nil
@@ -511,15 +545,19 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
}
defer f.Close()
// 上传到目标盘的根目录(用户配置的目标 drive 的 rootID)。
// 上传到目标盘 rootID 下的固定 "91 Spider" 子目录。若用户把目标盘 rootID
// 配成某个自定义目录,这里会在该自定义目录下查找/创建 "91 Spider"。
// 上传名走 desiredPikPakName 算出来的方案 B 格式:
//
// <sanitized title>-<viewkey 后 8 位>.<ext>
//
// 这样网盘 Web 端列出来的文件名能直接看出是哪个视频,
// 又用 viewkey 后 8 位避免同标题撞名。两个目标盘PikPak / 115共用同一格式,
// 又用 viewkey 后 8 位避免同标题撞名。所有目标盘共用同一格式,
// 简化前端 / catalog 的认知。
parent := pp.RootID()
parent, err := pp.EnsureDir(ctx, spider91UploadDirName)
if err != nil {
return false, fmt.Errorf("%s ensure %q dir: %w", pp.Kind(), spider91UploadDirName, err)
}
uploadName := desiredPikPakName(v.Title, extractViewKey(v.ID), v.Ext)
res, err := pp.UploadAndReportHash(ctx, parent, uploadName, f, info.Size())
if err != nil {
@@ -639,7 +677,7 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driv
return deleted, nil
}
// backfillFileNames 扫描目标 drivePikPak 或 115)下所有 spider91-* 起始 ID 的视频,
// backfillFileNames 扫描目标 drivePikPak、115 或 OneDrive)下所有 spider91-* 起始 ID 的视频,
// 对文件名不是 desiredPikPakName(...) 期望格式的,调 target.Rename 修正,
// 并把 catalog.file_name 同步到新名字。
//
@@ -53,6 +53,8 @@ type fakePikPak struct {
uploadFunc func(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error)
mu sync.Mutex
gotBodies map[string][]byte
gotParents map[string]string
ensureCalls []string
// renameCalls 记录每次 Rename 的 fileID->newName 历史,用于 backfill 测试断言。
renameCalls map[string]string
}
@@ -62,6 +64,7 @@ func newFakePikPak(id, rootID string) *fakePikPak {
id: id,
rootID: rootID,
gotBodies: make(map[string][]byte),
gotParents: make(map[string]string),
renameCalls: make(map[string]string),
}
}
@@ -80,8 +83,11 @@ func (d *fakePikPak) StreamURL(context.Context, string) (*drives.StreamLink, err
func (d *fakePikPak) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
return "", drives.ErrNotSupported
}
func (d *fakePikPak) EnsureDir(context.Context, string) (string, error) {
return "", drives.ErrNotSupported
func (d *fakePikPak) EnsureDir(_ context.Context, pathFromRoot string) (string, error) {
d.mu.Lock()
defer d.mu.Unlock()
d.ensureCalls = append(d.ensureCalls, pathFromRoot)
return d.rootID + "/" + pathFromRoot, nil
}
func (d *fakePikPak) Rename(_ context.Context, fileID, newName string) error {
d.mu.Lock()
@@ -99,6 +105,7 @@ func (d *fakePikPak) UploadAndReportHash(ctx context.Context, parentID, name str
body, _ := io.ReadAll(r)
d.mu.Lock()
d.gotBodies[name] = body
d.gotParents[name] = parentID
d.mu.Unlock()
return UploadResult{
FileID: "remote-" + name,
@@ -127,6 +134,19 @@ func (d *fakeP115) Kind() string { return "p115" }
var _ drives.Drive = (*fakeP115)(nil)
var _ uploadTarget = (*fakeP115)(nil)
type fakeOneDrive struct {
*fakePikPak
}
func newFakeOneDrive(id, rootID string) *fakeOneDrive {
return &fakeOneDrive{fakePikPak: newFakePikPak(id, rootID)}
}
func (d *fakeOneDrive) Kind() string { return "onedrive" }
var _ drives.Drive = (*fakeOneDrive)(nil)
var _ uploadTarget = (*fakeOneDrive)(nil)
// TestBackfillFileNamesRenamesOnlyMismatchedSpider91Videos 验证回填逻辑:
//
// - 已经是期望格式的不会再调 Rename(幂等)
@@ -347,6 +367,12 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
if _, ok := pp.gotBodies[wantName]; !ok {
t.Fatalf("PikPak did not receive expected upload name %q (got names: %v)", wantName, keysOf(pp.gotBodies))
}
if gotParent := pp.gotParents[wantName]; gotParent != "pikpak-root-id/"+spider91UploadDirName {
t.Fatalf("upload parent = %q, want root/91 Spider", gotParent)
}
if len(pp.ensureCalls) != 1 || pp.ensureCalls[0] != spider91UploadDirName {
t.Fatalf("ensure calls = %#v, want %q", pp.ensureCalls, spider91UploadDirName)
}
if got.FileID != "remote-"+wantName {
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
}
@@ -884,6 +910,12 @@ func TestRunOnceMigratesToP115Target(t *testing.T) {
if _, ok := target.gotBodies[wantName]; !ok {
t.Fatalf("p115 did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
}
if gotParent := target.gotParents[wantName]; gotParent != "p115-root-cid/"+spider91UploadDirName {
t.Fatalf("p115 upload parent = %q, want root/91 Spider", gotParent)
}
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
t.Fatalf("p115 ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
}
if got.FileID != "remote-"+wantName {
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
}
@@ -905,7 +937,67 @@ func TestRunOnceMigratesToP115Target(t *testing.T) {
}
}
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak 也不是 115 时,
func TestRunOnceMigratesToOneDriveTarget(t *testing.T) {
cat := setupCatalog(t)
src, _ := setupSpider91(t)
target := newFakeOneDrive("onedrive-target", "onedrive-root")
reg := newFakeRegistry()
reg.Add(src)
reg.Add(target)
now := time.Now()
id := writeSpider91Video(t, cat, src, "vk-od-001", ".mp4", []byte("video bytes onedrive"), now)
m := New(Config{
Catalog: cat,
Registry: reg,
GetTargetDriveID: func() string { return target.ID() },
KeepLatestN: -1,
})
m.runOnce(context.Background())
if target.uploadCalls != 1 {
t.Fatalf("onedrive upload calls = %d, want 1", target.uploadCalls)
}
got, err := cat.GetVideo(context.Background(), id)
if err != nil {
t.Fatalf("get video: %v", err)
}
if got.DriveID != target.ID() {
t.Fatalf("drive_id = %q, want %q", got.DriveID, target.ID())
}
wantName := "Sample vk-od-001-001.mp4"
if _, ok := target.gotBodies[wantName]; !ok {
t.Fatalf("onedrive did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
}
if gotParent := target.gotParents[wantName]; gotParent != "onedrive-root/"+spider91UploadDirName {
t.Fatalf("onedrive upload parent = %q, want root/91 Spider", gotParent)
}
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
t.Fatalf("onedrive ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
}
if got.FileID != "remote-"+wantName {
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
}
if got.FileName != wantName {
t.Fatalf("file_name = %q, want %q", got.FileName, wantName)
}
if got.ContentHash == "" {
t.Fatal("content_hash should be set after onedrive migration")
}
videoPath, _ := src.VideoPath("vk-od-001.mp4")
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
t.Fatalf("local mp4 still exists after onedrive migration or stat error: %v", err)
}
thumbPath, _ := src.ThumbPath("vk-od-001.jpg")
if _, err := os.Stat(thumbPath); !os.IsNotExist(err) {
t.Fatalf("local thumb still exists after onedrive migration or stat error: %v", err)
}
}
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115 也不是 OneDrive 时,
// resolveTarget 拒绝并返回 error,让 runOnce 静默跳过(不会做破坏性变更)。
func TestResolveTargetRejectsUnsupportedKind(t *testing.T) {
cat := setupCatalog(t)
+9
View File
@@ -0,0 +1,9 @@
services:
video-site-91:
image: ghcr.io/nianzhibai/91:stable
container_name: video-site-91
ports:
- "9191:9191"
volumes:
- ./data:/opt/video-site-91/data
restart: unless-stopped
+38
View File
@@ -0,0 +1,38 @@
#!/bin/sh
set -eu
APP_DIR="/opt/video-site-91"
DATA_DIR="${VIDEO_DATA_DIR:-$APP_DIR/data}"
CONFIG="${VIDEO_CONFIG:-$DATA_DIR/config.yaml}"
EXAMPLE="$APP_DIR/config.example.yaml"
PORT="${VIDEO_LISTEN_PORT:-9191}"
mkdir -p "$DATA_DIR" "$DATA_DIR/previews" "$DATA_DIR/uploads" "$DATA_DIR/spider91"
if [ ! -f "$CONFIG" ]; then
if [ ! -f "$EXAMPLE" ]; then
echo "[entrypoint] missing config template: $EXAMPLE" >&2
exit 1
fi
mkdir -p "$(dirname "$CONFIG")"
cp "$EXAMPLE" "$CONFIG"
SECRET="$(openssl rand -hex 32)"
sed -i -E "s#^([[:space:]]*listen:[[:space:]]*).*\$#\1\"0.0.0.0:${PORT}\"#" "$CONFIG"
sed -i -E "s#^([[:space:]]*session_secret:[[:space:]]*).*\$#\1\"${SECRET}\"#" "$CONFIG"
sed -i -E "s#^([[:space:]]*db_path:[[:space:]]*).*\$#\1\"${DATA_DIR}/video-site.db\"#" "$CONFIG"
sed -i -E "s#^([[:space:]]*local_preview_dir:[[:space:]]*).*\$#\1\"${DATA_DIR}/previews\"#" "$CONFIG"
chmod 600 "$CONFIG"
echo "[entrypoint] generated $CONFIG"
else
echo "[entrypoint] using existing $CONFIG"
fi
if [ -n "${VIDEO_VERSION_FILE:-}" ] && [ -n "${VIDEO_IMAGE_VERSION:-}" ]; then
mkdir -p "$(dirname "$VIDEO_VERSION_FILE")"
printf '%s\n' "$VIDEO_IMAGE_VERSION" > "$VIDEO_VERSION_FILE"
fi
exec "$@"
+61 -72
View File
@@ -26,6 +26,7 @@ const kindLabel: Record<string, string> = {
pikpak: "PikPak",
wopan: "联通沃盘",
onedrive: "OneDrive",
localstorage: "本地存储",
spider91: "91 爬虫",
};
@@ -80,10 +81,10 @@ export function DrivesPage() {
const [selectedDriveId, setSelectedDriveId] = useState<string | null>(null);
const { show } = useToast();
// 当前系统中可作为 spider91 上传目标的 drive 列表(pikpak p115)。
// 当前系统中可作为 spider91 上传目标的 drive 列表(pikpak p115 onedrive)。
// 用户保存 spider91 drive 时从这里挑一个;空表示本地保存不上传。
const uploadTargets = useMemo(
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115"),
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115" || d.kind === "onedrive"),
[list]
);
@@ -427,13 +428,13 @@ export function DrivesPage() {
)}
</div>
{/* 右栏:Teaser / 封面 与 缓存占用 */}
{/* 右栏:Teaser / 封面 / 指纹 与 缓存占用 */}
<div>
<div className="admin-detail-card">
<header className="admin-detail-card__title">
<div className="admin-detail-card__title-left">
<PlayCircle size={16} />
<span>Teaser </span>
<span></span>
</div>
<div className="admin-detail-actions-inline">
<button
@@ -481,6 +482,22 @@ export function DrivesPage() {
/>
</div>
</div>
<div className="admin-detail-row">
<span className="admin-detail-label"></span>
<div className="admin-detail-value">
<GenerationStatusLine label="指纹" status={d.fingerprintGenerationStatus} />
</div>
</div>
<div className="admin-detail-row">
<span className="admin-detail-label"></span>
<div className="admin-detail-value">
<GenerationCounts
ready={d.fingerprintReadyCount}
pending={d.fingerprintPendingCount}
failed={d.fingerprintFailedCount}
/>
</div>
</div>
</div>
<div className="admin-detail-actions">
@@ -586,7 +603,7 @@ export function DrivesPage() {
<div className="admin-empty">...</div>
) : list.length === 0 ? (
<div className="admin-card admin-empty">
/ 115 / PikPak / / OneDrive
/ 115 / PikPak / / OneDrive /
</div>
) : (
<div className="admin-drives-grid">
@@ -625,6 +642,15 @@ export function DrivesPage() {
</span>
</strong>
</div>
<div className="admin-drive-card__metric">
<span> (/)</span>
<strong>
{d.fingerprintReadyCount ?? 0}
<span style={{ fontSize: "11px", fontWeight: "normal", color: "var(--text-faint)" }}>
{" "}/ {d.fingerprintFailedCount ?? 0}
</span>
</strong>
</div>
</div>
<div className="admin-drive-card__footer">
@@ -837,6 +863,11 @@ function DriveForm({
}) {
const fields = useMemo(() => credentialFields(form.kind), [form.kind]);
const help = credentialHelp(form.kind, isEdit);
const showDirectoryFields =
form.kind !== "spider91" &&
form.kind !== "onedrive" &&
form.kind !== "localstorage" &&
form.kind !== "pikpak";
function set<K extends keyof FormState>(k: K, v: FormState[K]) {
onChange({ ...form, [k]: v });
@@ -873,13 +904,14 @@ function DriveForm({
>
<option value="p115">115 </option>
<option value="pikpak">PikPak</option>
<option value="spider91">91 </option>
<option value="onedrive">OneDrive</option>
<option value="localstorage"></option>
<option value="spider91">91 Spider</option>
<option value="quark"></option>
<option value="wopan"></option>
<option value="onedrive">OneDrive</option>
</select>
</div>
{form.kind !== "spider91" && (
{showDirectoryFields && (
<>
<div className="admin-form__row">
<label> ID</label>
@@ -953,9 +985,9 @@ function DriveForm({
* Spider91UploadTargetField spider91 drive "上传目标"
*
*
* - = "本地保存,不上传" + pikpak/p115 drive
* - = "本地保存,不上传" + pikpak/p115/onedrive drive
* - value=""
* - pikpak/p115 drive
* - pikpak/p115/onedrive drive
* - setting `spider91.upload_drive_id` drive
* credentials spider91 drive
*/
@@ -980,7 +1012,7 @@ function Spider91UploadTargetField({
))}
</select>
<div className="admin-form__help">
115 PikPak
115 PikPak OneDrive 91 Spider
</div>
</div>
);
@@ -994,13 +1026,15 @@ function credentialHelp(kind: Kind, isEdit: boolean): string {
case "p115":
return `登录 115.com 后复制 Cookie,形如 "UID=...; CID=...; SEID=...; KID=..."。${note}`;
case "pikpak":
return `参考 OpenList 的 PikPak 登录方式。可填用户名和密码首次登录,也可填 refresh_token;如返回验证码链接,打开验证后把 captcha_token 粘贴回来${note}`;
return `填写 PikPak 账号和密码即可。平台、设备 ID、验证码 token 和 refresh token 会由服务端自动处理并保存${note}`;
case "wopan":
return `需要 access_token 和 refresh_token。后续会加扫码/短信登录入口,第一版只能手工粘贴。${note}`;
case "onedrive":
return `按 OpenList 默认方式,通过 api.oplist.org 在线刷新 token。只需要 refresh_token;保存会自动回写新的 access_token / refresh_token。${note}`;
return `按 OpenList 默认应用在线挂载,只需要 refresh_token;保存会自动刷新并保存 token。${note}`;
case "localstorage":
return `把服务器上的一个已有目录作为视频来源扫描。填写绝对路径,例如 /mnt/videos;系统会读取该目录及子目录中的视频,并生成封面、Teaser 和指纹。${note}`;
case "spider91":
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;它不是外部网盘,不需要填写 Cookie 或目录 ID。后续流水线会把较早的视频上传到你选择的 115 / PikPak 目标盘。";
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;它不是外部网盘,不需要填写 Cookie 或目录 ID。后续流水线会把较早的视频上传到你选择的 115 / PikPak / OneDrive 目标盘。";
default:
return "";
}
@@ -1039,42 +1073,15 @@ function credentialFields(kind: Kind): Array<{
return [
{
key: "username",
label: "用户名 / 邮箱(无 refresh_token 时必填)",
label: "用户名 / 邮箱",
placeholder: "user@example.com",
required: true,
},
{
key: "password",
label: "密码(无 refresh_token 时必填)",
label: "密码",
placeholder: "PikPak 密码",
},
{
key: "platform",
label: "platform",
placeholder: "web(可选:android / web / pc",
help: "默认 web;如果登录或直链异常,可尝试 android 或 pc。",
},
{
key: "refresh_token",
label: "refresh_token(可选)",
placeholder: "已有 token 时可直接粘贴",
multiline: true,
},
{
key: "captcha_token",
label: "captcha_token(可选)",
placeholder: "遇到验证码校验时粘贴",
multiline: true,
},
{
key: "device_id",
label: "device_id(可选)",
placeholder: "留空自动生成并保存",
},
{
key: "disable_media_link",
label: "disable_media_link",
placeholder: "true",
help: "默认 true,使用原始下载链接;填 false 可尝试使用媒体缓存链接。",
required: true,
},
];
case "wopan":
@@ -1106,34 +1113,15 @@ function credentialFields(kind: Kind): Array<{
multiline: true,
required: true,
},
];
case "localstorage":
return [
{
key: "access_token",
label: "access_token(可选)",
placeholder: "留空也可以,保存时会在线刷新",
multiline: true,
},
{
key: "api_url_address",
label: "api_url_address(可选)",
placeholder: "https://api.oplist.org/onedrive/renewapi",
help: "默认使用 OpenList 的在线刷新 API;除非你有自建兼容服务,否则留空。",
},
{
key: "region",
label: "region(可选)",
placeholder: "global(可选:global / cn / us / de",
help: "默认 global;世纪互联填 cn,美国政府云填 us,德国云填 de。",
},
{
key: "is_sharepoint",
label: "is_sharepoint(可选)",
placeholder: "false",
help: "普通 OneDrive 留空或 falseSharePoint 文档库填 true,并填写 site_id。",
},
{
key: "site_id",
label: "site_idSharePoint 必填)",
placeholder: "SharePoint site id",
key: "path",
label: "本地目录路径",
placeholder: "/mnt/videos",
required: true,
help: "路径必须是后端服务器上的已有目录;保存后可手动重扫,系统会递归扫描支持的视频格式。",
},
];
case "spider91":
@@ -1144,6 +1132,7 @@ function credentialFields(kind: Kind): Array<{
function defaultRootId(kind: Kind): string {
if (kind === "pikpak") return "";
if (kind === "onedrive") return "root";
if (kind === "localstorage") return "/";
if (kind === "spider91") return "/";
return "0";
}
+8 -4
View File
@@ -77,7 +77,7 @@ export function checkUpdate() {
export type AdminDrive = {
id: string;
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "spider91";
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "localstorage" | "spider91";
name: string;
rootId: string;
scanRootId: string;
@@ -96,12 +96,16 @@ export type AdminDrive = {
lastCrawlAt?: number;
thumbnailGenerationStatus?: DriveGenerationStatus;
previewGenerationStatus?: DriveGenerationStatus;
fingerprintGenerationStatus?: DriveGenerationStatus;
thumbnailReadyCount: number;
thumbnailPendingCount: number;
thumbnailFailedCount: number;
teaserReadyCount: number;
teaserPendingCount: number;
teaserFailedCount: number;
fingerprintReadyCount: number;
fingerprintPendingCount: number;
fingerprintFailedCount: number;
};
export type DriveGenerationStatus = {
@@ -133,7 +137,7 @@ export function getDriveStorage() {
export type UpsertDriveInput = {
id: string;
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "spider91";
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "localstorage" | "spider91";
name: string;
rootId: string;
scanRootId: string;
@@ -336,9 +340,9 @@ export type Theme = "dark" | "pink";
export type Settings = {
theme: Theme;
/**
* spider91 drive ID pikpak p115 drive
* spider91 drive ID pikpakp115 onedrive drive
* -
* - drive kind {pikpak, p115}
* - drive kind {pikpak, p115, onedrive}
*/
spider91UploadDriveId: string;
};
+1
View File
@@ -294,5 +294,6 @@ function sourceKindFromLabel(label: string): string {
if (value.includes("pikpak")) return "pikpak";
if (value.includes("沃盘") || value.includes("wopan") || value.includes("联通")) return "wopan";
if (value.includes("onedrive") || value.includes("one drive")) return "onedrive";
if (value.includes("本地") || value.includes("localstorage") || value.includes("local storage")) return "localstorage";
return "";
}
+2
View File
@@ -72,5 +72,7 @@ function sourceKindFromLabel(label: string): string {
if (value.includes("沃盘") || value.includes("wopan") || value.includes("联通"))
return "wopan";
if (value.includes("onedrive") || value.includes("one drive")) return "onedrive";
if (value.includes("本地") || value.includes("localstorage") || value.includes("local storage"))
return "localstorage";
return "";
}
+1
View File
@@ -1287,6 +1287,7 @@ function getDriveShortName(source: string): string {
if (s.includes("quark") || s.includes("夸克")) return "Quak";
if (s.includes("onedrive")) return "OneDrive";
if (s.includes("wopan") || s.includes("沃盘")) return "沃盘";
if (s.includes("localstorage") || s.includes("本地")) return "本地";
if (s.includes("spider") || s.includes("爬虫")) return "爬虫";
return source.substring(0, 4);
}
+1
View File
@@ -1825,6 +1825,7 @@
.admin-drive-card__brand-icon[data-kind="pikpak"] { background: var(--drive-pikpak); }
.admin-drive-card__brand-icon[data-kind="wopan"] { background: var(--drive-wopan); }
.admin-drive-card__brand-icon[data-kind="onedrive"] { background: var(--drive-onedrive); }
.admin-drive-card__brand-icon[data-kind="localstorage"] { background: var(--drive-localstorage); }
.admin-drive-card__brand-icon[data-kind="spider91"] { background: var(--accent); }
.admin-drive-card__info {
+2
View File
@@ -134,6 +134,7 @@
--drive-pikpak: #8a6dff;
--drive-wopan: #ff8a3c;
--drive-onedrive: #4cabea;
--drive-localstorage: #35b88f;
/* ----- 阴影 ----- */
--shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.4);
@@ -213,6 +214,7 @@
--drive-pikpak: #8466e6;
--drive-wopan: #e57a36;
--drive-onedrive: #2f95cf;
--drive-localstorage: #239978;
/* ----- 阴影(粉色柔投影 + 一层中性) ----- */
--shadow-sm: 0 1px 2px rgba(180, 90, 120, 0.08);
+9
View File
@@ -306,6 +306,14 @@
--drive-shadow: rgba(76, 171, 234, 0.15);
--drive-shadow-strong: rgba(76, 171, 234, 0.4);
}
.source-badge[data-kind="localstorage"] {
--drive-color: var(--drive-localstorage);
--drive-bg: rgba(53, 184, 143, 0.12);
--drive-text: #a8efdc;
--drive-border: rgba(53, 184, 143, 0.35);
--drive-shadow: rgba(53, 184, 143, 0.15);
--drive-shadow-strong: rgba(53, 184, 143, 0.4);
}
/* ----- Preview overlay ----- */
.preview-loader {
@@ -441,6 +449,7 @@
.video-card__source[data-kind="pikpak"] { --source-color: var(--drive-pikpak); }
.video-card__source[data-kind="wopan"] { --source-color: var(--drive-wopan); }
.video-card__source[data-kind="onedrive"] { --source-color: var(--drive-onedrive); }
.video-card__source[data-kind="localstorage"] { --source-color: var(--drive-localstorage); }
/* =========================================================
* Empty / Loading / Skeleton
+6
View File
@@ -351,6 +351,12 @@
color: var(--drive-onedrive);
}
.vd-meta__chip[data-tone="localstorage"] {
background: rgba(53, 184, 143, 0.14);
border-color: rgba(53, 184, 143, 0.3);
color: var(--drive-localstorage);
}
/* ---------- Actions toolbar ---------- */
.vd-actions {
display: flex;
+75
View File
@@ -16,6 +16,81 @@ test("spider91 drive form does not expose advanced crawler credentials", () => {
test("spider91 upload target uses explicit local-save option instead of auto target", () => {
assert.match(drivesPageSource, /本地保存,不上传/);
assert.match(
drivesPageSource,
/d\.kind === "pikpak" \|\| d\.kind === "p115" \|\| d\.kind === "onedrive"/
);
assert.doesNotMatch(drivesPageSource, /自动:唯一/);
assert.doesNotMatch(drivesPageSource, /自动模式/);
});
test("onedrive drive form only exposes required default-app fields", () => {
assert.match(
drivesPageSource,
/form\.kind !== "spider91" &&\s*form\.kind !== "onedrive" &&\s*form\.kind !== "localstorage" &&\s*form\.kind !== "pikpak"/
);
const match =
/function credentialFields[\s\S]*?case "onedrive":\s*return \[([\s\S]*?)\];\s*case "spider91":/.exec(
drivesPageSource
);
assert.ok(match, "onedrive credential field block should be present");
const fields = match[1];
assert.match(fields, /key: "refresh_token"/);
assert.doesNotMatch(fields, /key: "access_token"/);
assert.doesNotMatch(fields, /key: "api_url_address"/);
assert.doesNotMatch(fields, /key: "region"/);
assert.doesNotMatch(fields, /key: "is_sharepoint"/);
assert.doesNotMatch(fields, /key: "site_id"/);
});
test("pikpak drive form only exposes account login fields", () => {
const match =
/case "pikpak":\s*return \[([\s\S]*?)\];\s*case "wopan":/.exec(
drivesPageSource
);
assert.ok(match, "pikpak credential field block should be present");
const fields = match[1];
assert.match(fields, /key: "username"/);
assert.match(fields, /key: "password"/);
assert.doesNotMatch(fields, /key: "platform"/);
assert.doesNotMatch(fields, /key: "refresh_token"/);
assert.doesNotMatch(fields, /key: "captcha_token"/);
assert.doesNotMatch(fields, /key: "device_id"/);
assert.doesNotMatch(fields, /key: "disable_media_link"/);
});
test("localstorage drive form asks for a server directory path", () => {
assert.match(drivesPageSource, /<option value="localstorage">本地存储<\/option>/);
const match =
/case "localstorage":\s*return \[([\s\S]*?)\];\s*case "spider91":/.exec(
drivesPageSource
);
assert.ok(match, "localstorage credential field block should be present");
const fields = match[1];
assert.match(fields, /key: "path"/);
assert.match(fields, /label: "本地目录路径"/);
assert.match(drivesPageSource, /if \(kind === "localstorage"\) return "\/"/);
});
test("drive type selector keeps primary source order", () => {
const options = Array.from(
drivesPageSource.matchAll(/<option value="([^"]+)">([^<]+)<\/option>/g),
(match) => ({ value: match[1], label: match[2] })
);
const driveOptions = options.slice(0, 7);
assert.deepEqual(driveOptions, [
{ value: "p115", label: "115 网盘" },
{ value: "pikpak", label: "PikPak" },
{ value: "onedrive", label: "OneDrive" },
{ value: "localstorage", label: "本地存储" },
{ value: "spider91", label: "91 Spider" },
{ value: "quark", label: "夸克网盘" },
{ value: "wopan", label: "联通沃盘" },
]);
});