mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-15 16:55:42 +08:00
Compare commits
12 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9ada39debb | |||
| 24d1244bc3 | |||
| 0dd0c45509 | |||
| ef6eadd0a6 | |||
| 2c5a3342cc | |||
| 7ace5f8bc7 | |||
| 66b33b2a31 | |||
| 27aefc870f | |||
| 02d82e9a62 | |||
| 492431164b | |||
| 051f1555d5 | |||
| 85292ea095 |
@@ -0,0 +1,21 @@
|
||||
.git
|
||||
.github
|
||||
.gitattributes
|
||||
.gitignore
|
||||
|
||||
node_modules
|
||||
dist
|
||||
release
|
||||
data
|
||||
backend/data
|
||||
backend/config.yaml
|
||||
config.yaml
|
||||
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
*.log
|
||||
*.tmp
|
||||
|
||||
tests
|
||||
video-site-implementation-plan.md
|
||||
@@ -0,0 +1,82 @@
|
||||
name: Docker
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
tags:
|
||||
- "v*"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract Docker metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=tag
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=sha,prefix=sha-
|
||||
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
|
||||
- name: Determine image version
|
||||
id: version
|
||||
shell: bash
|
||||
run: |
|
||||
if [[ "$GITHUB_REF" == refs/tags/v* ]]; then
|
||||
version="$GITHUB_REF_NAME"
|
||||
else
|
||||
version="$(git describe --tags --always --dirty 2>/dev/null || git rev-parse --short=12 HEAD)"
|
||||
fi
|
||||
echo "version=$version" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
VERSION=${{ steps.version.outputs.version }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
# ---- Stage 1: Build frontend ----
|
||||
FROM node:20-slim AS frontend
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tsconfig.json vite.config.ts index.html ./
|
||||
COPY public/ public/
|
||||
COPY src/ src/
|
||||
RUN npm run build
|
||||
|
||||
# ---- Stage 2: Build backend ----
|
||||
FROM golang:1.23-bookworm AS backend
|
||||
|
||||
WORKDIR /app/backend
|
||||
|
||||
COPY backend/go.mod backend/go.sum ./
|
||||
COPY backend/vendor/ vendor/
|
||||
COPY backend/cmd/ cmd/
|
||||
COPY backend/internal/ internal/
|
||||
RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server
|
||||
|
||||
# ---- Stage 3: Runtime ----
|
||||
FROM debian:bookworm-slim AS runtime
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
ffmpeg \
|
||||
openssl \
|
||||
python3 \
|
||||
python3-bs4 \
|
||||
python3-lxml \
|
||||
python3-requests \
|
||||
tar \
|
||||
tzdata \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /opt/video-site-91
|
||||
|
||||
COPY --from=backend /out/server ./server
|
||||
COPY --from=frontend /app/dist ./dist
|
||||
COPY backend/config.example.yaml ./config.example.yaml
|
||||
COPY 91VideoSpider/ ./91VideoSpider/
|
||||
COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
ENV VIDEO_CONFIG=/opt/video-site-91/data/config.yaml \
|
||||
VIDEO_FRONTEND_DIR=/opt/video-site-91/dist \
|
||||
VIDEO_GITHUB_REPO=nianzhibai/91 \
|
||||
VIDEO_IMAGE_VERSION=${VERSION} \
|
||||
VIDEO_LISTEN_PORT=9191 \
|
||||
VIDEO_VERSION_FILE=/opt/video-site-91/data/.version
|
||||
|
||||
RUN chmod +x ./server /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
VOLUME ["/opt/video-site-91/data"]
|
||||
EXPOSE 9191
|
||||
|
||||
ENTRYPOINT ["docker-entrypoint.sh"]
|
||||
CMD ["./server"]
|
||||
@@ -5,28 +5,27 @@
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
😄个人 91 站😄
|
||||
😄 个人私有视频站 😄
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="#快速开始">快速开始</a> ·
|
||||
<a href="#功能特性">功能特性</a> ·
|
||||
<a href="#预览图">预览图</a> ·
|
||||
<a href="#数据存放位置">数据目录</a>
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
## 项目说明
|
||||
## 功能特性
|
||||
|
||||
支持 115 云盘、PikPak 云盘作为视频播放后端。
|
||||
|
||||
采用 115 云盘和 PikPak 云盘的 302 重定向播放,不占用服务器带宽,也不会因为服务器带宽小而影响视频播放体验。
|
||||
|
||||
服务器只负责扫描云盘中的视频文件,并给每个视频生成封面图和预览片段。
|
||||
|
||||
你可以通过封面图和预览片段,在首页快速选择想看的视频。
|
||||
|
||||
支持 91 爬虫,爬取 91 的本月最热视频。
|
||||
|
||||
内置两种主题:黑黄主题(91 经典主题)和粉白主题。
|
||||
|
||||
支持短视频模式,一键切换成熟悉的抖音模式。
|
||||
|
||||
该项目2C2G服务器稳定跑👍👍👍
|
||||
- **多后端支持** — 兼容 115 云盘、PikPak 云盘、OneDrive和本地存储
|
||||
- **零带宽消耗** — 云盘视频采用 302 重定向播放,不占用服务器带宽
|
||||
- **封面 & 预览片段** — 自动为每个视频生成封面图和预览片段,首页快速选片
|
||||
- **91 爬虫** — 内置爬虫,支持抓取 91 本月最热视频
|
||||
- **双主题** — 黑黄经典主题 / 粉白清新主题,随时切换
|
||||
- **短视频模式** — 一键切换抖音风格,沉浸刷片
|
||||
- **低资源占用** — 2C2G 服务器稳定运行,主要性能消耗就是封面图和预览视频的生成
|
||||
|
||||
---
|
||||
|
||||
@@ -35,100 +34,173 @@
|
||||
### 电脑端
|
||||
|
||||
<p>
|
||||
<img width="49%" alt="91 电脑端首页" src="https://github.com/user-attachments/assets/9808fceb-760b-4dd5-b7d2-8622b95b90d5" />
|
||||
<img width="49%" alt="91 电脑端播放页" src="https://github.com/user-attachments/assets/859db4aa-1fba-44f2-bb46-1db07c2f964f" />
|
||||
<img width="49%" alt="首页" src="https://github.com/user-attachments/assets/9808fceb-760b-4dd5-b7d2-8622b95b90d5" />
|
||||
<img width="49%" alt="播放页" src="https://github.com/user-attachments/assets/859db4aa-1fba-44f2-bb46-1db07c2f964f" />
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<img width="49%" alt="91 电脑端主题" src="https://github.com/user-attachments/assets/96bea37a-8764-413e-9b70-1856b4ae0cd2" />
|
||||
<img width="49%" alt="91 电脑端管理页" src="https://github.com/user-attachments/assets/29c1e27a-7651-4dfc-93dd-556331844214" />
|
||||
<img width="49%" alt="主题切换" src="https://github.com/user-attachments/assets/96bea37a-8764-413e-9b70-1856b4ae0cd2" />
|
||||
<img width="49%" alt="管理页" src="https://github.com/user-attachments/assets/29c1e27a-7651-4dfc-93dd-556331844214" />
|
||||
</p>
|
||||
|
||||
### 手机端
|
||||
|
||||
<p align="center">
|
||||
<img width="1284" height="1134" alt="PixPin_2026-05-29_11-54-12" src="https://github.com/user-attachments/assets/bdb7a86c-a4e5-483e-a307-e02c0bb34dac" />
|
||||
<img width="1284" height="1134" alt="手机端" src="https://github.com/user-attachments/assets/bdb7a86c-a4e5-483e-a307-e02c0bb34dac" />
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
## 快速开始
|
||||
|
||||
一键安装:
|
||||
### 方式一:一键安装脚本(推荐)
|
||||
|
||||
```bash
|
||||
sudo apt update
|
||||
sudo apt install -y curl ca-certificates
|
||||
sudo apt update && sudo apt install -y curl ca-certificates
|
||||
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/install.sh -o install.sh
|
||||
sudo bash install.sh
|
||||
```
|
||||
|
||||
部署完成后访问:
|
||||
|
||||
- 前台:`http://服务器IP:9191/`
|
||||
- 后台:`http://服务器IP:9191/admin`
|
||||
| 地址 | 说明 |
|
||||
|------|------|
|
||||
| `http://服务器IP:9191/` | 前台 |
|
||||
| `http://服务器IP:9191/admin` | 后台管理 |
|
||||
|
||||
安装后会自动创建 `91` 指令:
|
||||
**注意:如果首次访问,显示502,可以运行 `91 restart` 重启一下服务**
|
||||
|
||||
安装后自动注册 `91` 管理命令:
|
||||
|
||||
```bash
|
||||
91 # 打开管理菜单
|
||||
91 status # 查看状态
|
||||
91 logs # 查看日志
|
||||
91 update # 更新
|
||||
91 restart # 重启
|
||||
91 stop # 停止
|
||||
91 # 打开管理菜单
|
||||
91 status # 查看运行状态
|
||||
91 logs # 查看日志
|
||||
91 update # 更新到最新版本
|
||||
91 restart # 重启服务
|
||||
91 stop # 停止服务
|
||||
```
|
||||
|
||||
同时也保留 `video-site-91` 作为同等别名。
|
||||
> `video-site-91` 为等效别名,两者可互换使用。
|
||||
|
||||
**旧版本用户升级说明:**
|
||||
**自定义端口:**
|
||||
|
||||
如果你是在 `v0.0.2` 之前部署的项目,系统里可能还保留旧的 `91` 管理脚本。旧脚本直接运行 `91 update` 可能更新失败。先执行下面的一次性修复命令,后续再使用 `91 update` 即可:
|
||||
```bash
|
||||
FRONTEND_PORT=8080 sudo -E bash install.sh
|
||||
```
|
||||
|
||||
**旧版本升级(v0.0.2 之前):**
|
||||
|
||||
旧版脚本直接执行 `91 update` 可能失败,先执行以下修复命令:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/install.sh -o /tmp/install-91.sh
|
||||
sudo bash /tmp/install-91.sh update
|
||||
```
|
||||
|
||||
想换端口:
|
||||
---
|
||||
|
||||
### 方式二:Docker Compose 部署
|
||||
|
||||
**1. 准备目录**
|
||||
|
||||
```bash
|
||||
FRONTEND_PORT=8080 sudo -E bash install.sh
|
||||
mkdir -p video-site-91 && cd video-site-91
|
||||
```
|
||||
|
||||
**2. 创建 `docker-compose.yml`**
|
||||
|
||||
```yaml
|
||||
services:
|
||||
video-site-91:
|
||||
image: ghcr.io/nianzhibai/91:stable
|
||||
container_name: video-site-91
|
||||
ports:
|
||||
- "9191:9191"
|
||||
volumes:
|
||||
- ./data:/opt/video-site-91/data
|
||||
restart: unless-stopped
|
||||
```
|
||||
|
||||
`stable` 只会在发布 `v*` 正式 Release 时更新,不会跟随 `main` 分支开发镜像变化。
|
||||
升级到最新正式版:
|
||||
|
||||
```bash
|
||||
docker compose pull
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
如果想固定某个 Release 版本,可以改成明确的 tag,例如:
|
||||
|
||||
```yaml
|
||||
image: ghcr.io/nianzhibai/91:v0.0.4
|
||||
```
|
||||
|
||||
或直接拉取仓库内置配置:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/docker-compose.yml -o docker-compose.yml
|
||||
```
|
||||
|
||||
**3. 启动**
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
**常用命令:**
|
||||
|
||||
```bash
|
||||
docker compose logs -f # 查看日志
|
||||
docker compose pull # 拉取最新正式版 stable 镜像
|
||||
docker compose up -d # 更新并重启
|
||||
```
|
||||
|
||||
> 所有配置、数据库、封面、预览及上传文件均保存在 `./data/` 目录下。
|
||||
|
||||
---
|
||||
|
||||
## 数据存放位置
|
||||
|
||||
项目会把运行数据保存在本地:
|
||||
### 一键脚本部署
|
||||
|
||||
- `/opt/video-site-91/config.yaml`:本地配置、管理员账号、网盘凭证。
|
||||
- `/opt/video-site-91/data/video-site.db`:SQLite 数据库。
|
||||
- `/opt/video-site-91/data/previews/`:本地生成的封面和 teaser。
|
||||
| 路径 | 内容 |
|
||||
|------|------|
|
||||
| `/opt/video-site-91/config.yaml` | 配置文件、管理员账号、网盘凭证 |
|
||||
| `/opt/video-site-91/data/video-site.db` | SQLite 数据库 |
|
||||
| `/opt/video-site-91/data/previews/` | 封面图和预览片段 |
|
||||
|
||||
### Docker Compose 部署
|
||||
|
||||
| 路径 | 内容 |
|
||||
|------|------|
|
||||
| `./data/config.yaml` | 配置文件、管理员账号、网盘凭证 |
|
||||
| `./data/video-site.db` | SQLite 数据库 |
|
||||
| `./data/previews/` | 封面图和预览片段 |
|
||||
| `./data/uploads/` | 本地上传的视频文件 |
|
||||
| `./data/spider91/` | 91 爬虫抓取的视频文件 |
|
||||
|
||||
---
|
||||
|
||||
## 了解更多
|
||||
## 更多文档
|
||||
|
||||
根目录 README 只保留项目介绍和最短上手路径。更细的实现、接口、网盘字段和部署方式可以看:
|
||||
|
||||
- [backend/README.md](backend/README.md)
|
||||
- [video-site-implementation-plan.md](video-site-implementation-plan.md)
|
||||
| 文档 | 内容 |
|
||||
|------|------|
|
||||
| [backend/README.md](backend/README.md) | 后端实现、接口说明、网盘字段 |
|
||||
| [video-site-implementation-plan.md](video-site-implementation-plan.md) | 完整实现方案 |
|
||||
|
||||
---
|
||||
|
||||
## 使用边界
|
||||
## 使用须知
|
||||
|
||||
这个项目面向个人私有部署。请只接入你有权访问和管理的内容,并遵守对应网盘、站点服务条款以及所在地法律法规。
|
||||
本项目面向**个人私有部署**,请仅接入你有权访问和管理的内容,并遵守对应网盘、站点的服务条款及所在地法律法规。
|
||||
|
||||
不要传播,仅限个人使用,个人视频站。
|
||||
> 不对外传播,仅限个人使用。
|
||||
|
||||
---
|
||||
|
||||
## 致谢
|
||||
|
||||
感谢开源项目 OpenList。
|
||||
|
||||
感谢 <a href="https://linux.do/">LinuxDo</a> 社区,学 AI 上 L 站。
|
||||
|
||||
感谢 <a href="https://nodeseek.com/">NodeSeek</a> 社区,MJJ 上 N 站。
|
||||
- [OpenList](https://github.com/OpenListTeam/OpenList) — 优秀的开源项目
|
||||
- [LinuxDo](https://linux.do/) — 学 AI 上 L 站
|
||||
- [NodeSeek](https://nodeseek.com/) — MJJ 上 N 站
|
||||
|
||||
+19
-5
@@ -2,7 +2,7 @@
|
||||
|
||||
视频聚合站的 Go 后端。提供三件事:
|
||||
|
||||
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive)
|
||||
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive / 本地存储)
|
||||
2. 视频元数据目录(SQLite)+ 扫描 + teaser 预生成
|
||||
3. REST API(前台)+ 管理后台 + 直链代理
|
||||
4. 标签池、视频隐藏、按网盘统计和详情页来源网盘类型展示能力
|
||||
@@ -21,6 +21,7 @@ internal/
|
||||
pikpak/ PikPak(自己实现,参考 OpenList pikpak)
|
||||
wopan/ 联通沃盘(壳子 + OpenListTeam/wopan-sdk-go)
|
||||
onedrive/ OneDrive(OpenList 在线续期 + Microsoft Graph 文件接口)
|
||||
localstorage/ 本地目录扫描(服务器已有视频目录)
|
||||
scanner/ 扫目录 → 落库
|
||||
preview/ ffmpeg 抽封面和生成多段 teaser
|
||||
proxy/ /p/stream/*、/p/preview/* 代理
|
||||
@@ -105,9 +106,10 @@ go run ./cmd/server 后端 9192
|
||||
|--------|---------------------------------------------------------------|
|
||||
| quark | `cookie` |
|
||||
| p115 | `cookie`(形如 `UID=...; CID=...; SEID=...; KID=...`) |
|
||||
| pikpak | `username`、`password`,可选 `refresh_token`、`captcha_token`、`device_id`、`platform`、`disable_media_link` |
|
||||
| pikpak | `username`、`password`(token、验证码和设备 ID 由服务端自动处理并保存) |
|
||||
| wopan | `access_token`、`refresh_token`,可选 `family_id` |
|
||||
| onedrive | `refresh_token`,可选 `access_token`、`api_url_address`、`region`、`is_sharepoint`、`site_id` |
|
||||
| onedrive | `refresh_token` |
|
||||
| localstorage | `path`(服务器上的已有视频目录,如 `/mnt/videos`) |
|
||||
|
||||
### PikPak 速度说明
|
||||
|
||||
@@ -115,7 +117,7 @@ go run ./cmd/server 后端 9192
|
||||
|
||||
当前服务器同时存在 sing-box TUN 透明代理,PikPak 默认出站会被 `tun0` 接管;但强制直连物理网卡并没有更快,慢速的主要差异来自 PikPak 取链方式。media/cache CDN 节点仍有波动,偶尔可能遇到慢节点;如果播放变慢,可重新获取直链或重新挂载 PikPak 后再测。
|
||||
|
||||
OneDrive 按 OpenList 默认方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。OpenList 代刷得到的 refresh token 可以直接填到本项目。普通 OneDrive 的 `rootId` / `scanRootId` 可填 `root`;SharePoint 文档库需要额外设置 `is_sharepoint=true` 和 `site_id`。
|
||||
OneDrive 按 OpenList 默认应用方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。后台新建 OneDrive 时只需要填 OpenList 代刷得到的 `refresh_token`;服务端会默认挂载根目录并自动回写新 token。
|
||||
|
||||
## 文件名约定
|
||||
|
||||
@@ -128,6 +130,18 @@ OneDrive 按 OpenList 默认方式调用 `https://api.oplist.org/onedrive/renewa
|
||||
|
||||
标签分隔符支持 `, , 、` 和空格。解析结果会和系统标签池匹配,常见番号类噪声会归并到 `AV` 等系统标签,避免把每个番号都变成独立标签。解析结果可在管理后台覆盖。
|
||||
|
||||
## 视频去重
|
||||
|
||||
项目有三层去重:
|
||||
|
||||
1. 同一网盘同一文件按 `(drive_id, file_id)` 形成稳定视频 ID,重复扫描只更新同一行。
|
||||
2. 扫描时优先按网盘侧 `content_hash` 去重;没有 hash 时退化为 `file_name + size_bytes`。
|
||||
3. 扫描、爬虫、本地上传或服务启动挂载网盘后,后台指纹 worker 会异步读取视频的少量 Range 片段,生成 `sampled_sha256`。前台列表、首页、搜索、推荐会按 `size_bytes + sampled_sha256` 只展示最早入库的 canonical 视频。
|
||||
|
||||
`sampled_sha256` 是文件级去重:适合识别同一个视频文件被复制到 115 / PikPak / OneDrive 等不同网盘的情况。它不会删除任何网盘文件,也不用于识别转码、裁剪、加水印后的同源视频。
|
||||
|
||||
封面和 teaser 仍然优先生成,不等待指纹完成。夜间流水线最后会做一次重复资产清理:对 `size_bytes + sampled_sha256` 命中的非 canonical 视频,只删除本机生成的重复封面和 teaser,并把对应字段重置为 `pending`。网盘原文件和视频元数据记录不会被删除;如果 canonical 视频以后被移除,这些重复项会重新进入生成队列。
|
||||
|
||||
## 管理能力
|
||||
|
||||
- `/admin/drives`:新增、编辑、删除网盘,触发扫描。
|
||||
@@ -147,7 +161,7 @@ ffmpeg -ss <起点> -headers "UA/Cookie/Referer" -i <直链> \
|
||||
|
||||
当前策略是每段固定 3 秒;30 秒以下最多 3 段,30 秒及以上固定 4 段;长视频在 20% 到 80% 区间均匀取段。生成的 teaser 和封面都只保存在本地 `data/previews/`,不会回写到网盘;旧数据中的 `preview_file_id` 会被忽略。
|
||||
|
||||
服务启动或网盘重新挂载时,如果 Teaser 开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 直链生成 teaser 时可能触发 Microsoft 429 限流;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。
|
||||
服务启动或网盘重新挂载时,如果 Teaser 开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 扫盘和直链生成 teaser / 封面时可能触发 Microsoft Graph 429、`TooManyRequests`、`activityLimitReached` 或 throttled 文本;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。扫盘阶段会按 `Retry-After` 或默认冷却时间等待后继续当前目录。
|
||||
|
||||
前端卡片的 `previewSrc` 统一指向 `/p/preview/<videoID>`,后端只从本地 `preview_local` 文件读取。
|
||||
|
||||
|
||||
+401
-94
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/config"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/localstorage"
|
||||
"github.com/video-site/backend/internal/drives/localupload"
|
||||
"github.com/video-site/backend/internal/drives/onedrive"
|
||||
"github.com/video-site/backend/internal/drives/p115"
|
||||
@@ -32,6 +33,7 @@ import (
|
||||
"github.com/video-site/backend/internal/drives/quark"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/drives/wopan"
|
||||
"github.com/video-site/backend/internal/fingerprint"
|
||||
"github.com/video-site/backend/internal/nightly"
|
||||
"github.com/video-site/backend/internal/preview"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
@@ -39,6 +41,8 @@ import (
|
||||
"github.com/video-site/backend/internal/spider91migrate"
|
||||
)
|
||||
|
||||
const fingerprintReconcileInterval = time.Minute
|
||||
|
||||
func main() {
|
||||
cfgPath := "./config.yaml"
|
||||
if v := os.Getenv("VIDEO_CONFIG"); v != "" {
|
||||
@@ -63,12 +67,13 @@ func main() {
|
||||
defer cat.Close()
|
||||
|
||||
app := &App{
|
||||
cfg: cfg,
|
||||
cat: cat,
|
||||
registry: proxy.NewRegistry(),
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
spider91Crawlers: make(map[string]*spider91.Crawler),
|
||||
cfg: cfg,
|
||||
cat: cat,
|
||||
registry: proxy.NewRegistry(),
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
fingerprintWorkers: make(map[string]*fingerprint.Worker),
|
||||
spider91Crawlers: make(map[string]*spider91.Crawler),
|
||||
}
|
||||
app.proxy = proxy.New(app.registry)
|
||||
app.spider91Migrator = spider91migrate.New(spider91migrate.Config{
|
||||
@@ -77,7 +82,8 @@ func main() {
|
||||
GetTargetDriveID: func() string { return app.Spider91UploadDriveID() },
|
||||
})
|
||||
|
||||
// 初始化现有 drives
|
||||
// 初始化本地内置盘;外部云盘放到 HTTP 服务启动后异步挂载,避免上游
|
||||
// 登录态校验拖慢端口监听。
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
@@ -86,16 +92,7 @@ func main() {
|
||||
if err := app.attachLocalUpload(ctx); err != nil {
|
||||
log.Printf("[local-upload] attach failed: %v", err)
|
||||
}
|
||||
|
||||
existing, err := cat.ListDrives(ctx)
|
||||
if err != nil {
|
||||
log.Fatalf("list drives: %v", err)
|
||||
}
|
||||
for _, d := range existing {
|
||||
if err := app.attachDrive(ctx, d); err != nil {
|
||||
log.Printf("[drive %s] attach failed: %v", d.ID, err)
|
||||
}
|
||||
}
|
||||
go app.runFingerprintReconciler(ctx)
|
||||
|
||||
authr := &auth.Authenticator{
|
||||
Username: cfg.Server.Admin.Username,
|
||||
@@ -128,6 +125,7 @@ func main() {
|
||||
Catalog: cat,
|
||||
Auth: authr,
|
||||
VersionFilePath: versionFilePath,
|
||||
ImageVersion: strings.TrimSpace(os.Getenv("VIDEO_IMAGE_VERSION")),
|
||||
GitHubRepo: githubRepo,
|
||||
SetupRequired: func() bool {
|
||||
setupMu.Lock()
|
||||
@@ -240,6 +238,7 @@ func main() {
|
||||
RunSpider91Crawl: app.runSpider91Crawl,
|
||||
WaitPreviewQueuesIdle: app.waitAllPreviewQueuesIdle,
|
||||
RunMigration: app.spider91Migrator.RunOnce,
|
||||
RunDedupeAssetCleanup: app.cleanupDuplicateVideoAssets,
|
||||
})
|
||||
go app.nightlyRunner.Run(ctx)
|
||||
|
||||
@@ -253,6 +252,7 @@ func main() {
|
||||
log.Fatalf("server error: %v", err)
|
||||
}
|
||||
}()
|
||||
go app.attachExistingDrives(ctx)
|
||||
|
||||
// 等待退出信号
|
||||
sigs := make(chan os.Signal, 1)
|
||||
@@ -272,20 +272,25 @@ type App struct {
|
||||
registry *proxy.Registry
|
||||
proxy *proxy.Proxy
|
||||
|
||||
mu sync.Mutex
|
||||
workers map[string]*preview.Worker
|
||||
thumbWorkers map[string]*preview.ThumbWorker
|
||||
cancels map[string]context.CancelFunc
|
||||
mu sync.Mutex
|
||||
workers map[string]*preview.Worker
|
||||
thumbWorkers map[string]*preview.ThumbWorker
|
||||
fingerprintWorkers map[string]*fingerprint.Worker
|
||||
cancels map[string]context.CancelFunc
|
||||
// spider91Crawlers 按 driveID 索引,每个 spider91 drive 独立一个 Crawler
|
||||
spider91Crawlers map[string]*spider91.Crawler
|
||||
|
||||
// driveAttachMu 串行化云盘挂载/重挂载。挂载会访问上游服务,可能较慢;
|
||||
// 串行化可以避免启动后台挂载和手动扫盘按需挂载同一个 drive 时重复创建 worker。
|
||||
driveAttachMu sync.Mutex
|
||||
|
||||
// 全站主题("dark" | "pink"),从 DB 读
|
||||
theme string
|
||||
// 显式指定的 spider91 上传目标 drive ID。
|
||||
// 空字符串表示本地保存不上传,不再自动挑选 pikpak/p115 drive。
|
||||
// 空字符串表示本地保存不上传,不再自动挑选 pikpak/p115/onedrive drive。
|
||||
spider91UploadDriveID string
|
||||
|
||||
// spider91Migrator 周期把 spider91 视频上传到目标 drive(PikPak 或 115)。
|
||||
// spider91Migrator 周期把 spider91 视频上传到目标 drive(PikPak、115 或 OneDrive)。
|
||||
spider91Migrator *spider91migrate.Migrator
|
||||
|
||||
// nightlyRunner 是凌晨流水线调度器:每天 cron_hour 串行跑扫盘 → 91 爬虫 → 迁移。
|
||||
@@ -307,6 +312,11 @@ type App struct {
|
||||
// scanQueued 跟踪哪些 driveID 已经排队或正在跑,去重后续重复点击。
|
||||
// 一个 drive 在 scheduleScan 入队时被加入,在 runScan goroutine 结束时被移除。
|
||||
scanQueued map[string]bool
|
||||
|
||||
// fingerprintQueueing 去重每个 drive 的 pending 指纹补队列任务,避免定时
|
||||
// reconcile 和扫盘结束同时为同一批 pending 视频启动多个长时间入队 goroutine。
|
||||
fingerprintQueueMu sync.Mutex
|
||||
fingerprintQueueing map[string]bool
|
||||
}
|
||||
|
||||
// teaserEnabledForDrive 查询某个 drive 当前的 per-drive teaser 开关。
|
||||
@@ -371,7 +381,7 @@ func (a *App) loadTheme(ctx context.Context) {
|
||||
}
|
||||
|
||||
// Spider91UploadDriveID 返回当前配置的 spider91 上传目标 drive ID。
|
||||
// 空字符串表示本地保存不上传;只有管理员显式选择 pikpak/p115 drive 时才迁移上传。
|
||||
// 空字符串表示本地保存不上传;只有管理员显式选择 pikpak/p115/onedrive drive 时才迁移上传。
|
||||
func (a *App) Spider91UploadDriveID() string {
|
||||
a.mu.Lock()
|
||||
explicit := a.spider91UploadDriveID
|
||||
@@ -388,7 +398,7 @@ func (a *App) Spider91UploadDriveID() string {
|
||||
|
||||
// SetSpider91UploadDriveID 设置 spider91 上传目标 drive ID 并持久化。
|
||||
// 接受空字符串(本地保存不上传)。
|
||||
// 设置一个不存在或 kind 不是 pikpak / p115 的 drive 会返回错误。
|
||||
// 设置一个不存在或 kind 不是 pikpak / p115 / onedrive 的 drive 会返回错误。
|
||||
func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) error {
|
||||
driveID = strings.TrimSpace(driveID)
|
||||
if driveID != "" {
|
||||
@@ -397,7 +407,7 @@ func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) erro
|
||||
return fmt.Errorf("drive %q not found", driveID)
|
||||
}
|
||||
if !isSpider91UploadKind(d.Kind()) {
|
||||
return fmt.Errorf("drive %q kind=%s, only pikpak or p115 can be spider91 upload target", driveID, d.Kind())
|
||||
return fmt.Errorf("drive %q kind=%s, only pikpak, p115 or onedrive can be spider91 upload target", driveID, d.Kind())
|
||||
}
|
||||
}
|
||||
a.mu.Lock()
|
||||
@@ -409,7 +419,7 @@ func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) erro
|
||||
// isSpider91UploadKind 是 spider91 迁移目标盘的 allowlist。
|
||||
// 与 spider91migrate.adaptUploadTarget 的支持范围保持一致。
|
||||
func isSpider91UploadKind(kind string) bool {
|
||||
return kind == "pikpak" || kind == "p115"
|
||||
return kind == "pikpak" || kind == "p115" || kind == "onedrive"
|
||||
}
|
||||
|
||||
// loadSpider91UploadDriveID 从 DB 读上传目标 drive ID 设置;不存在时使用空串。
|
||||
@@ -434,9 +444,13 @@ func (a *App) driveGenerationStatuses() map[string]api.DriveGenerationStatuses {
|
||||
for id, worker := range a.thumbWorkers {
|
||||
thumbWorkers[id] = worker
|
||||
}
|
||||
fingerprintWorkers := make(map[string]*fingerprint.Worker, len(a.fingerprintWorkers))
|
||||
for id, worker := range a.fingerprintWorkers {
|
||||
fingerprintWorkers[id] = worker
|
||||
}
|
||||
a.mu.Unlock()
|
||||
|
||||
out := make(map[string]api.DriveGenerationStatuses, len(previewWorkers)+len(thumbWorkers))
|
||||
out := make(map[string]api.DriveGenerationStatuses, len(previewWorkers)+len(thumbWorkers)+len(fingerprintWorkers))
|
||||
for id, worker := range previewWorkers {
|
||||
status := out[id]
|
||||
status.Preview = generationStatusFromPreview(worker.Status())
|
||||
@@ -447,7 +461,7 @@ func (a *App) driveGenerationStatuses() map[string]api.DriveGenerationStatuses {
|
||||
status.Thumbnail = generationStatusFromPreview(worker.Status())
|
||||
missing, err := a.cat.CountVideosNeedingThumbnail(context.Background(), id)
|
||||
if err != nil {
|
||||
log.Printf("[thumb] count missing thumbnails %s: %v", id, err)
|
||||
log.Printf("[thumb] count thumbnail work %s: %v", id, err)
|
||||
} else {
|
||||
status.Thumbnail.QueueLength = missing
|
||||
if missing > 0 && status.Thumbnail.State == "idle" {
|
||||
@@ -456,6 +470,20 @@ func (a *App) driveGenerationStatuses() map[string]api.DriveGenerationStatuses {
|
||||
}
|
||||
out[id] = status
|
||||
}
|
||||
for id, worker := range fingerprintWorkers {
|
||||
status := out[id]
|
||||
status.Fingerprint = generationStatusFromFingerprint(worker.Status())
|
||||
pending, err := a.cat.CountVideosNeedingFingerprint(context.Background(), id)
|
||||
if err != nil {
|
||||
log.Printf("[fingerprint] count pending fingerprints %s: %v", id, err)
|
||||
} else {
|
||||
status.Fingerprint.QueueLength = pending
|
||||
if pending > 0 && status.Fingerprint.State == "idle" {
|
||||
status.Fingerprint.State = "queued"
|
||||
}
|
||||
}
|
||||
out[id] = status
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -475,7 +503,67 @@ func generationStatusFromPreview(status preview.TaskStatus) api.GenerationStatus
|
||||
return out
|
||||
}
|
||||
|
||||
func generationStatusFromFingerprint(status fingerprint.TaskStatus) api.GenerationStatus {
|
||||
state := status.State
|
||||
if state == "" {
|
||||
state = "idle"
|
||||
}
|
||||
out := api.GenerationStatus{
|
||||
State: state,
|
||||
CurrentTitle: status.CurrentTitle,
|
||||
QueueLength: status.QueueLength,
|
||||
}
|
||||
if !status.CooldownUntil.IsZero() {
|
||||
out.CooldownUntil = status.CooldownUntil.Format(time.RFC3339)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *App) attachDrive(ctx context.Context, d *catalog.Drive) error {
|
||||
a.driveAttachMu.Lock()
|
||||
defer a.driveAttachMu.Unlock()
|
||||
return a.attachDriveUnlocked(ctx, d)
|
||||
}
|
||||
|
||||
func (a *App) ensureDriveAttached(ctx context.Context, driveID string) error {
|
||||
if _, ok := a.registry.Get(driveID); ok {
|
||||
return nil
|
||||
}
|
||||
a.driveAttachMu.Lock()
|
||||
defer a.driveAttachMu.Unlock()
|
||||
if _, ok := a.registry.Get(driveID); ok {
|
||||
return nil
|
||||
}
|
||||
d, err := a.cat.GetDrive(ctx, driveID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return a.attachDriveUnlocked(ctx, d)
|
||||
}
|
||||
|
||||
func (a *App) attachExistingDrives(ctx context.Context) {
|
||||
existing, err := a.cat.ListDrives(ctx)
|
||||
if err != nil {
|
||||
log.Printf("[drive] list existing drives: %v", err)
|
||||
return
|
||||
}
|
||||
log.Printf("[drive] attaching %d configured drive(s) in background", len(existing))
|
||||
for _, d := range existing {
|
||||
if err := ctx.Err(); err != nil {
|
||||
log.Printf("[drive] background attach stopped: %v", err)
|
||||
return
|
||||
}
|
||||
if err := a.attachDrive(ctx, d); err != nil {
|
||||
log.Printf("[drive %s] attach failed: %v", d.ID, err)
|
||||
}
|
||||
}
|
||||
log.Printf("[drive] background attach complete")
|
||||
}
|
||||
|
||||
func (a *App) attachDriveUnlocked(ctx context.Context, d *catalog.Drive) error {
|
||||
if d == nil {
|
||||
return errors.New("nil drive")
|
||||
}
|
||||
var drv drives.Drive
|
||||
switch d.Kind {
|
||||
case "quark":
|
||||
@@ -546,6 +634,11 @@ func (a *App) attachDrive(ctx context.Context, d *catalog.Drive) error {
|
||||
_ = a.cat.UpsertDrive(ctx, d)
|
||||
},
|
||||
})
|
||||
case localstorage.Kind:
|
||||
drv = localstorage.New(localstorage.Config{
|
||||
ID: d.ID,
|
||||
RootPath: d.Credentials["path"],
|
||||
})
|
||||
case spider91.Kind:
|
||||
drv = spider91.New(spider91.Config{
|
||||
ID: d.ID,
|
||||
@@ -579,12 +672,14 @@ func (a *App) attachDrive(ctx context.Context, d *catalog.Drive) error {
|
||||
})
|
||||
worker := preview.NewWorker(gen, a.cat, drv)
|
||||
thumbWorker := preview.NewThumbWorker(gen, a.cat, drv)
|
||||
fingerprintWorker := fingerprint.NewWorker(a.cat, drv, fingerprintConfigForDrive(drv))
|
||||
|
||||
workerCtx, cancel := context.WithCancel(ctx)
|
||||
go worker.Run(workerCtx)
|
||||
go thumbWorker.Run(workerCtx)
|
||||
go fingerprintWorker.Run(workerCtx)
|
||||
|
||||
a.registerPreviewWorkers(ctx, d.ID, worker, thumbWorker, cancel)
|
||||
a.registerPreviewWorkers(ctx, d.ID, worker, thumbWorker, fingerprintWorker, cancel)
|
||||
|
||||
// spider91 driver 还需要一个 crawler,挂在专用 map 里供 crawlerLoop 调用
|
||||
if sd, ok := drv.(*spider91.Driver); ok {
|
||||
@@ -611,12 +706,14 @@ func (a *App) attachLocalUpload(ctx context.Context) error {
|
||||
})
|
||||
worker := preview.NewWorker(gen, a.cat, drv)
|
||||
thumbWorker := preview.NewThumbWorker(gen, a.cat, drv)
|
||||
fingerprintWorker := fingerprint.NewWorker(a.cat, drv, fingerprintConfigForDrive(drv))
|
||||
|
||||
workerCtx, cancel := context.WithCancel(ctx)
|
||||
go worker.Run(workerCtx)
|
||||
go thumbWorker.Run(workerCtx)
|
||||
go fingerprintWorker.Run(workerCtx)
|
||||
|
||||
a.registerPreviewWorkers(ctx, drv.ID(), worker, thumbWorker, cancel)
|
||||
a.registerPreviewWorkers(ctx, drv.ID(), worker, thumbWorker, fingerprintWorker, cancel)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -624,6 +721,20 @@ func (a *App) localUploadDir() string {
|
||||
return filepath.Join(filepath.Dir(a.cfg.Storage.LocalPreviewDir), "uploads")
|
||||
}
|
||||
|
||||
func fingerprintConfigForDrive(drv drives.Drive) fingerprint.Config {
|
||||
cfg := fingerprint.Config{RateLimitCooldown: 5 * time.Minute}
|
||||
if drv == nil {
|
||||
return cfg
|
||||
}
|
||||
switch strings.ToLower(drv.Kind()) {
|
||||
case "p115", "onedrive":
|
||||
cfg.RateLimitCooldown = 10 * time.Minute
|
||||
case "pikpak":
|
||||
cfg.RateLimitCooldown = 5 * time.Minute
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// spider91RootDir 是所有 spider91 drive 共享的根目录。
|
||||
func (a *App) spider91RootDir() string {
|
||||
return filepath.Join(filepath.Dir(a.cfg.Storage.LocalPreviewDir), "spider91")
|
||||
@@ -708,7 +819,7 @@ func (a *App) attachSpider91Crawler(d *catalog.Drive, drv *spider91.Driver) {
|
||||
}()
|
||||
}
|
||||
|
||||
func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker *preview.Worker, thumbWorker *preview.ThumbWorker, cancel context.CancelFunc) {
|
||||
func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker *preview.Worker, thumbWorker *preview.ThumbWorker, fingerprintWorker *fingerprint.Worker, cancel context.CancelFunc) {
|
||||
a.mu.Lock()
|
||||
if a.cancels == nil {
|
||||
a.cancels = make(map[string]context.CancelFunc)
|
||||
@@ -719,6 +830,9 @@ func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker
|
||||
if a.thumbWorkers == nil {
|
||||
a.thumbWorkers = make(map[string]*preview.ThumbWorker)
|
||||
}
|
||||
if a.fingerprintWorkers == nil {
|
||||
a.fingerprintWorkers = make(map[string]*fingerprint.Worker)
|
||||
}
|
||||
if old, ok := a.cancels[driveID]; ok && old != nil {
|
||||
old()
|
||||
}
|
||||
@@ -732,6 +846,11 @@ func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker
|
||||
} else {
|
||||
delete(a.thumbWorkers, driveID)
|
||||
}
|
||||
if fingerprintWorker != nil {
|
||||
a.fingerprintWorkers[driveID] = fingerprintWorker
|
||||
} else {
|
||||
delete(a.fingerprintWorkers, driveID)
|
||||
}
|
||||
if cancel != nil {
|
||||
a.cancels[driveID] = cancel
|
||||
} else {
|
||||
@@ -739,17 +858,10 @@ func (a *App) registerPreviewWorkers(ctx context.Context, driveID string, worker
|
||||
}
|
||||
a.mu.Unlock()
|
||||
|
||||
if worker != nil {
|
||||
if thumbWorker != nil {
|
||||
worker.BeforeTask = func(taskCtx context.Context) bool {
|
||||
return a.waitForThumbnailsBeforePreview(taskCtx, driveID)
|
||||
}
|
||||
} else {
|
||||
worker.BeforeTask = nil
|
||||
}
|
||||
}
|
||||
|
||||
go a.enqueueDriveGeneration(ctx, driveID, worker, thumbWorker)
|
||||
if fingerprintWorker != nil {
|
||||
a.scheduleFingerprintBackfill(ctx, driveID, fingerprintWorker)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) enqueuePending(ctx context.Context, driveID string, w *preview.Worker) {
|
||||
@@ -773,45 +885,16 @@ func (a *App) enqueuePending(ctx context.Context, driveID string, w *preview.Wor
|
||||
func (a *App) enqueueDriveGeneration(ctx context.Context, driveID string, worker *preview.Worker, thumbWorker *preview.ThumbWorker) {
|
||||
// 封面 worker 始终入队(与早期"全局 preview.enabled=false 时仍然生成封面"
|
||||
// 的行为一致);teaser worker 仅在该 drive 的 TeaserEnabled 为 true 时入队。
|
||||
// 两条队列互不等待,避免封面批量生成拖住预览视频生成。
|
||||
if thumbWorker != nil {
|
||||
a.enqueueThumbnails(ctx, driveID, thumbWorker)
|
||||
}
|
||||
if worker == nil || !a.teaserEnabledForDrive(ctx, driveID) {
|
||||
return
|
||||
}
|
||||
if thumbWorker != nil && !a.waitForThumbnailsBeforePreview(ctx, driveID) {
|
||||
return
|
||||
}
|
||||
a.enqueuePending(ctx, driveID, worker)
|
||||
}
|
||||
|
||||
func (a *App) waitForThumbnailsBeforePreview(ctx context.Context, driveID string) bool {
|
||||
const pollInterval = time.Second
|
||||
var lastLog time.Time
|
||||
for {
|
||||
missing, err := a.cat.CountVideosNeedingThumbnail(ctx, driveID)
|
||||
if err != nil {
|
||||
log.Printf("[preview] count missing thumbnails drive=%s: %v", driveID, err)
|
||||
return false
|
||||
}
|
||||
if missing == 0 {
|
||||
return true
|
||||
}
|
||||
now := time.Now()
|
||||
if lastLog.IsZero() || now.Sub(lastLog) >= time.Minute {
|
||||
log.Printf("[preview] drive=%s waiting for %d thumbnails before teaser generation", driveID, missing)
|
||||
lastLog = now
|
||||
}
|
||||
timer := time.NewTimer(pollInterval)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return false
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) enqueueThumbnails(ctx context.Context, driveID string, w *preview.ThumbWorker) {
|
||||
pending, err := a.cat.ListVideosNeedingThumbnail(ctx, driveID, 0)
|
||||
if err != nil {
|
||||
@@ -821,10 +904,81 @@ func (a *App) enqueueThumbnails(ctx context.Context, driveID string, w *preview.
|
||||
if len(pending) == 0 {
|
||||
return
|
||||
}
|
||||
log.Printf("[thumb] enqueue %d missing thumbnails for drive=%s", len(pending), driveID)
|
||||
log.Printf("[thumb] enqueue %d thumbnail/duration tasks for drive=%s", len(pending), driveID)
|
||||
for _, v := range pending {
|
||||
if !w.EnqueueBlocking(ctx, v) {
|
||||
log.Printf("[thumb] enqueue missing thumbnails canceled for drive=%s", driveID)
|
||||
log.Printf("[thumb] enqueue thumbnail/duration tasks canceled for drive=%s", driveID)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) runFingerprintReconciler(ctx context.Context) {
|
||||
ticker := time.NewTicker(fingerprintReconcileInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
a.enqueueAllPendingFingerprints(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) enqueueAllPendingFingerprints(ctx context.Context) {
|
||||
a.mu.Lock()
|
||||
workers := make(map[string]*fingerprint.Worker, len(a.fingerprintWorkers))
|
||||
for id, worker := range a.fingerprintWorkers {
|
||||
workers[id] = worker
|
||||
}
|
||||
a.mu.Unlock()
|
||||
for driveID, worker := range workers {
|
||||
a.scheduleFingerprintBackfill(ctx, driveID, worker)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) scheduleFingerprintBackfill(ctx context.Context, driveID string, w *fingerprint.Worker) {
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
a.fingerprintQueueMu.Lock()
|
||||
if a.fingerprintQueueing == nil {
|
||||
a.fingerprintQueueing = make(map[string]bool)
|
||||
}
|
||||
if a.fingerprintQueueing[driveID] {
|
||||
a.fingerprintQueueMu.Unlock()
|
||||
return
|
||||
}
|
||||
a.fingerprintQueueing[driveID] = true
|
||||
a.fingerprintQueueMu.Unlock()
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
a.fingerprintQueueMu.Lock()
|
||||
delete(a.fingerprintQueueing, driveID)
|
||||
a.fingerprintQueueMu.Unlock()
|
||||
}()
|
||||
a.enqueueFingerprints(ctx, driveID, w)
|
||||
}()
|
||||
}
|
||||
|
||||
func (a *App) enqueueFingerprints(ctx context.Context, driveID string, w *fingerprint.Worker) {
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
pending, err := a.cat.ListVideosNeedingFingerprint(ctx, driveID, 0)
|
||||
if err != nil {
|
||||
log.Printf("[fingerprint] list pending %s: %v", driveID, err)
|
||||
return
|
||||
}
|
||||
if len(pending) == 0 {
|
||||
return
|
||||
}
|
||||
log.Printf("[fingerprint] enqueue %d videos for drive=%s", len(pending), driveID)
|
||||
for _, v := range pending {
|
||||
if !w.EnqueueBlocking(ctx, v) {
|
||||
log.Printf("[fingerprint] enqueue canceled for drive=%s", driveID)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -839,6 +993,7 @@ func (a *App) detachDrive(id string) {
|
||||
}
|
||||
delete(a.workers, id)
|
||||
delete(a.thumbWorkers, id)
|
||||
delete(a.fingerprintWorkers, id)
|
||||
delete(a.spider91Crawlers, id)
|
||||
a.mu.Unlock()
|
||||
}
|
||||
@@ -933,6 +1088,10 @@ func (a *App) runScan(ctx context.Context, driveID string) {
|
||||
a.scanGlobalMu.Lock()
|
||||
defer a.scanGlobalMu.Unlock()
|
||||
|
||||
if err := a.ensureDriveAttached(ctx, driveID); err != nil {
|
||||
log.Printf("[scan] drive %s attach failed: %v", driveID, err)
|
||||
return
|
||||
}
|
||||
drv, ok := a.registry.Get(driveID)
|
||||
if !ok {
|
||||
log.Printf("[scan] drive %s not attached", driveID)
|
||||
@@ -942,14 +1101,15 @@ func (a *App) runScan(ctx context.Context, driveID string) {
|
||||
a.mu.Lock()
|
||||
worker := a.workers[driveID]
|
||||
thumbWorker := a.thumbWorkers[driveID]
|
||||
fingerprintWorker := a.fingerprintWorkers[driveID]
|
||||
a.mu.Unlock()
|
||||
|
||||
var onNew func(v *catalog.Video)
|
||||
if thumbWorker != nil {
|
||||
onNew = func(v *catalog.Video) {
|
||||
if thumbWorker != nil && v.ThumbnailURL == "" {
|
||||
thumbWorker.Enqueue(v)
|
||||
}
|
||||
onNew := func(v *catalog.Video) {
|
||||
if thumbWorker != nil && v.ThumbnailURL == "" {
|
||||
thumbWorker.Enqueue(v)
|
||||
}
|
||||
if fingerprintWorker != nil {
|
||||
fingerprintWorker.Enqueue(v)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -993,6 +1153,7 @@ func (a *App) runScan(ctx context.Context, driveID string) {
|
||||
}
|
||||
}
|
||||
}
|
||||
a.scheduleFingerprintBackfill(ctx, driveID, fingerprintWorker)
|
||||
a.enqueueDriveGeneration(ctx, driveID, worker, thumbWorker)
|
||||
}
|
||||
|
||||
@@ -1063,6 +1224,126 @@ func removeLocalVideoAssets(localDir string, v *catalog.Video) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type duplicateAssetCleanupStats struct {
|
||||
Candidates int
|
||||
VideosUpdated int
|
||||
PreviewFiles int
|
||||
ThumbnailFiles int
|
||||
MissingFiles int
|
||||
SkippedUnsafeRef int
|
||||
}
|
||||
|
||||
func (a *App) cleanupDuplicateVideoAssets(ctx context.Context) error {
|
||||
if a == nil || a.cat == nil {
|
||||
return nil
|
||||
}
|
||||
localDir := ""
|
||||
if a.cfg != nil {
|
||||
localDir = a.cfg.Storage.LocalPreviewDir
|
||||
}
|
||||
if strings.TrimSpace(localDir) == "" {
|
||||
return nil
|
||||
}
|
||||
items, err := a.cat.ListDuplicateAssetCleanupCandidates(ctx, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(items) == 0 {
|
||||
log.Printf("[dedupe-cleanup] no duplicate local assets to clean")
|
||||
return nil
|
||||
}
|
||||
|
||||
stats := duplicateAssetCleanupStats{Candidates: len(items)}
|
||||
for _, item := range items {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
clearPreview, removedPreview, missingPreview, skippedPreview, err := cleanupDuplicatePreviewAsset(localDir, item.PreviewLocal)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cleanup duplicate preview video=%s canonical=%s: %w", item.VideoID, item.CanonicalID, err)
|
||||
}
|
||||
clearThumb, removedThumb, missingThumb, err := cleanupDuplicateThumbnailAsset(localDir, item.VideoID, item.ThumbnailURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cleanup duplicate thumbnail video=%s canonical=%s: %w", item.VideoID, item.CanonicalID, err)
|
||||
}
|
||||
if skippedPreview {
|
||||
stats.SkippedUnsafeRef++
|
||||
}
|
||||
if removedPreview {
|
||||
stats.PreviewFiles++
|
||||
}
|
||||
if removedThumb {
|
||||
stats.ThumbnailFiles++
|
||||
}
|
||||
if missingPreview {
|
||||
stats.MissingFiles++
|
||||
}
|
||||
if missingThumb {
|
||||
stats.MissingFiles++
|
||||
}
|
||||
if !clearPreview && !clearThumb {
|
||||
continue
|
||||
}
|
||||
if err := a.cat.ClearGeneratedAssets(ctx, item.VideoID, clearPreview, clearThumb); err != nil {
|
||||
return fmt.Errorf("mark duplicate assets cleaned video=%s canonical=%s: %w", item.VideoID, item.CanonicalID, err)
|
||||
}
|
||||
stats.VideosUpdated++
|
||||
}
|
||||
log.Printf("[dedupe-cleanup] candidates=%d updated=%d preview_files=%d thumbnail_files=%d missing=%d skipped_unsafe_refs=%d",
|
||||
stats.Candidates, stats.VideosUpdated, stats.PreviewFiles, stats.ThumbnailFiles, stats.MissingFiles, stats.SkippedUnsafeRef)
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanupDuplicatePreviewAsset(localDir, previewLocal string) (clear bool, removed bool, missing bool, skippedUnsafe bool, err error) {
|
||||
clean, ok := localPathWithin(localDir, previewLocal)
|
||||
if !ok {
|
||||
if strings.TrimSpace(previewLocal) != "" {
|
||||
return false, false, false, true, nil
|
||||
}
|
||||
return false, false, false, false, nil
|
||||
}
|
||||
removed, missing, err = removeRegularFileIfExists(clean)
|
||||
if err != nil {
|
||||
return false, false, false, false, err
|
||||
}
|
||||
return true, removed, missing, false, nil
|
||||
}
|
||||
|
||||
func cleanupDuplicateThumbnailAsset(localDir, videoID, thumbnailURL string) (clear bool, removed bool, missing bool, err error) {
|
||||
if thumbnailURL != "/p/thumb/"+videoID {
|
||||
return false, false, false, nil
|
||||
}
|
||||
clean, ok := localPathWithin(localDir, filepath.Join(localDir, "thumbs", videoID+".jpg"))
|
||||
if !ok {
|
||||
return false, false, false, nil
|
||||
}
|
||||
removed, missing, err = removeRegularFileIfExists(clean)
|
||||
if err != nil {
|
||||
return false, false, false, err
|
||||
}
|
||||
return true, removed, missing, nil
|
||||
}
|
||||
|
||||
func removeRegularFileIfExists(path string) (removed bool, missing bool, err error) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, true, nil
|
||||
}
|
||||
return false, false, err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return false, false, nil
|
||||
}
|
||||
if err := os.Remove(path); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, true, nil
|
||||
}
|
||||
return false, false, err
|
||||
}
|
||||
return true, false, nil
|
||||
}
|
||||
|
||||
func localPathWithin(root, path string) (string, bool) {
|
||||
if strings.TrimSpace(root) == "" || strings.TrimSpace(path) == "" {
|
||||
return "", false
|
||||
@@ -1089,6 +1370,7 @@ func (a *App) enqueueUploadedVideo(ctx context.Context, v *catalog.Video) {
|
||||
a.mu.Lock()
|
||||
worker := a.workers[v.DriveID]
|
||||
thumbWorker := a.thumbWorkers[v.DriveID]
|
||||
fingerprintWorker := a.fingerprintWorkers[v.DriveID]
|
||||
a.mu.Unlock()
|
||||
|
||||
if thumbWorker != nil && v.ThumbnailURL == "" {
|
||||
@@ -1097,6 +1379,9 @@ func (a *App) enqueueUploadedVideo(ctx context.Context, v *catalog.Video) {
|
||||
if worker != nil && a.teaserEnabledForDrive(ctx, v.DriveID) {
|
||||
worker.Enqueue(v)
|
||||
}
|
||||
if fingerprintWorker != nil {
|
||||
fingerprintWorker.Enqueue(v)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) regenPreview(ctx context.Context, videoID string) {
|
||||
@@ -1223,26 +1508,36 @@ func (a *App) regenFailedThumbnails(ctx context.Context, driveID string) {
|
||||
}
|
||||
|
||||
// listScanTargetIDs 返回 nightly Phase 1 应扫描的所有 drive ID
|
||||
// (非 spider91、非 localupload)。顺序按 registry.All 给的稳定顺序。
|
||||
func (a *App) listScanTargetIDs(_ context.Context) []string {
|
||||
all := a.registry.All()
|
||||
// (非 spider91、非 localupload)。它直接读 catalog,而不是 registry,这样
|
||||
// 进程刚启动、云盘还在后台挂载时,nightly 也不会漏掉配置过的 drive。
|
||||
func (a *App) listScanTargetIDs(ctx context.Context) []string {
|
||||
all, err := a.cat.ListDrives(ctx)
|
||||
if err != nil {
|
||||
log.Printf("[nightly] list scan target drives: %v", err)
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(all))
|
||||
for _, d := range all {
|
||||
if !shouldScanDrive(d) {
|
||||
if d == nil || d.ID == localupload.DriveID || d.Kind == spider91.Kind {
|
||||
continue
|
||||
}
|
||||
out = append(out, d.ID())
|
||||
out = append(out, d.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// listSpider91DriveIDs 返回 nightly Phase 2 应触发爬取的 spider91 drive ID 列表。
|
||||
func (a *App) listSpider91DriveIDs(_ context.Context) []string {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
out := make([]string, 0, len(a.spider91Crawlers))
|
||||
for id := range a.spider91Crawlers {
|
||||
out = append(out, id)
|
||||
func (a *App) listSpider91DriveIDs(ctx context.Context) []string {
|
||||
all, err := a.cat.ListDrives(ctx)
|
||||
if err != nil {
|
||||
log.Printf("[nightly] list spider91 drives: %v", err)
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(all))
|
||||
for _, d := range all {
|
||||
if d != nil && d.Kind == spider91.Kind {
|
||||
out = append(out, d.ID)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1250,8 +1545,8 @@ func (a *App) listSpider91DriveIDs(_ context.Context) []string {
|
||||
// waitAllPreviewQueuesIdle 阻塞直到所有 drive 的封面 worker 和 teaser worker
|
||||
// 队列都为空且无 in-flight 任务。
|
||||
//
|
||||
// 顺序:先等所有 thumb worker(因为 enqueueDriveGeneration 内部已经先等当前
|
||||
// drive 的封面再入队 teaser,但这里是跨 drive 的全局同步),再等所有 teaser。
|
||||
// 顺序:先等所有 thumb worker,再等所有 teaser。两个队列生成时互不等待;
|
||||
// nightly 只在 phase 边界统一等待它们都 drain。
|
||||
// 若 ctx 在等待中被取消(软超时 / shutdown),立即返回 ctx.Err。
|
||||
func (a *App) waitAllPreviewQueuesIdle(ctx context.Context) error {
|
||||
a.mu.Lock()
|
||||
@@ -1301,7 +1596,17 @@ func (a *App) runSpider91Crawl(ctx context.Context, driveID string) {
|
||||
c := a.spider91Crawlers[driveID]
|
||||
a.mu.Unlock()
|
||||
if c == nil {
|
||||
return
|
||||
if err := a.ensureDriveAttached(ctx, driveID); err != nil {
|
||||
log.Printf("[spider91] drive=%s attach failed: %v", driveID, err)
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
c = a.spider91Crawlers[driveID]
|
||||
a.mu.Unlock()
|
||||
if c == nil {
|
||||
log.Printf("[spider91] drive=%s crawler not attached", driveID)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
d, err := a.cat.GetDrive(ctx, driveID)
|
||||
@@ -1348,7 +1653,9 @@ func (a *App) runSpider91Crawl(ctx context.Context, driveID string) {
|
||||
a.mu.Lock()
|
||||
worker := a.workers[driveID]
|
||||
thumbWorker := a.thumbWorkers[driveID]
|
||||
fingerprintWorker := a.fingerprintWorkers[driveID]
|
||||
a.mu.Unlock()
|
||||
a.scheduleFingerprintBackfill(ctx, driveID, fingerprintWorker)
|
||||
a.enqueueDriveGeneration(ctx, driveID, worker, thumbWorker)
|
||||
}
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@ func TestSpider91IntCredFallbacks(t *testing.T) {
|
||||
func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
|
||||
reg := proxy.NewRegistry()
|
||||
reg.Set("p115-one", &spider91UploadTargetFakeDrive{id: "p115-one", kind: "p115"})
|
||||
reg.Set("onedrive-one", &spider91UploadTargetFakeDrive{id: "onedrive-one", kind: "onedrive"})
|
||||
|
||||
app := &App{registry: reg}
|
||||
if got := app.Spider91UploadDriveID(); got != "" {
|
||||
@@ -49,6 +50,11 @@ func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
|
||||
t.Fatalf("explicit upload target = %q, want p115-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "onedrive-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "onedrive-one" {
|
||||
t.Fatalf("explicit onedrive upload target = %q, want onedrive-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "missing"
|
||||
if got := app.Spider91UploadDriveID(); got != "" {
|
||||
t.Fatalf("missing upload target = %q, want empty", got)
|
||||
|
||||
+368
-37
@@ -13,7 +13,9 @@ import (
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/config"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/fingerprint"
|
||||
"github.com/video-site/backend/internal/preview"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
)
|
||||
|
||||
func TestRegisterPreviewWorkerBackfillsPendingWhenDriveTeaserEnabled(t *testing.T) {
|
||||
@@ -53,7 +55,7 @@ func TestRegisterPreviewWorkerBackfillsPendingWhenDriveTeaserEnabled(t *testing.
|
||||
worker := preview.NewWorker(&serverFakeTeaserGenerator{}, cat, &serverFakeDrive{})
|
||||
go worker.Run(ctx)
|
||||
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, nil, func() {})
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, nil, nil, func() {})
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
@@ -77,7 +79,7 @@ func TestRegisterPreviewWorkerBackfillsPendingWhenDriveTeaserEnabled(t *testing.
|
||||
t.Fatalf("preview status = %q, want ready", got.PreviewStatus)
|
||||
}
|
||||
|
||||
func TestRegisterPreviewWorkersGenerateThumbnailsBeforePreviews(t *testing.T) {
|
||||
func TestRegisterPreviewWorkersRunThumbnailsAndPreviewsIndependently(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
@@ -93,16 +95,18 @@ func TestRegisterPreviewWorkersGenerateThumbnailsBeforePreviews(t *testing.T) {
|
||||
|
||||
seedDriveWithTeaser(t, cat, "drive-id", true)
|
||||
now := time.Now()
|
||||
for _, v := range []*catalog.Video{
|
||||
{ID: "video-1", DriveID: "drive-id", FileID: "file-1", Title: "Clip 1", PreviewStatus: "pending"},
|
||||
{ID: "video-2", DriveID: "drive-id", FileID: "file-2", Title: "Clip 2", PreviewStatus: "pending"},
|
||||
} {
|
||||
v.PublishedAt = now
|
||||
v.CreatedAt = now
|
||||
v.UpdatedAt = now
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed video %s: %v", v.ID, err)
|
||||
}
|
||||
video := &catalog.Video{
|
||||
ID: "video-1",
|
||||
DriveID: "drive-id",
|
||||
FileID: "file-1",
|
||||
Title: "Clip 1",
|
||||
PreviewStatus: "pending",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if err := cat.UpsertVideo(ctx, video); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
app := &App{
|
||||
@@ -110,47 +114,234 @@ func TestRegisterPreviewWorkersGenerateThumbnailsBeforePreviews(t *testing.T) {
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
}
|
||||
gen := &serverFakeTeaserGenerator{}
|
||||
gen := &serverBlockingThumbGenerator{
|
||||
started: make(chan string, 1),
|
||||
release: make(chan struct{}),
|
||||
}
|
||||
drv := &serverFakeDrive{}
|
||||
worker := preview.NewWorker(gen, cat, drv)
|
||||
thumbWorker := preview.NewThumbWorker(gen, cat, drv)
|
||||
go worker.Run(ctx)
|
||||
go thumbWorker.Run(ctx)
|
||||
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, func() {})
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, nil, func() {})
|
||||
|
||||
select {
|
||||
case got := <-gen.started:
|
||||
if got != video.ID {
|
||||
t.Fatalf("thumbnail started for %q, want %q", got, video.ID)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("thumbnail generation did not start")
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
first, err := cat.GetVideo(ctx, "video-1")
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video-1: %v", err)
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
second, err := cat.GetVideo(ctx, "video-2")
|
||||
if err != nil {
|
||||
t.Fatalf("get video-2: %v", err)
|
||||
}
|
||||
if first.ThumbnailURL != "" && second.ThumbnailURL != "" &&
|
||||
first.PreviewStatus == "ready" && second.PreviewStatus == "ready" {
|
||||
events := gen.Events()
|
||||
if len(events) != 4 {
|
||||
t.Fatalf("events = %#v, want 4 generation events", events)
|
||||
}
|
||||
for i, event := range events[:2] {
|
||||
if event[:6] != "thumb:" {
|
||||
t.Fatalf("event %d = %q, want thumbnail before previews; all events=%#v", i, event, events)
|
||||
}
|
||||
}
|
||||
for i, event := range events[2:] {
|
||||
if event[:8] != "preview:" {
|
||||
t.Fatalf("event %d = %q, want previews after thumbnails; all events=%#v", i+2, event, events)
|
||||
}
|
||||
if got.PreviewStatus == "ready" {
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail url = %q, want preview ready while thumbnail is still blocked", got.ThumbnailURL)
|
||||
}
|
||||
close(gen.release)
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
t.Fatalf("generation did not finish, events=%#v", gen.Events())
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video after timeout: %v", err)
|
||||
}
|
||||
t.Fatalf("preview status=%q thumbnail=%q, want preview ready before thumbnail finishes", got.PreviewStatus, got.ThumbnailURL)
|
||||
}
|
||||
|
||||
func TestRegisterPreviewWorkersBackfillsHistoricalFingerprints(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
dataPath := filepath.Join(t.TempDir(), "video.mp4")
|
||||
data := []byte("historical video content for fingerprint")
|
||||
if err := os.WriteFile(dataPath, data, 0o644); err != nil {
|
||||
t.Fatalf("write video data: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
video := &catalog.Video{
|
||||
ID: "historical-video",
|
||||
DriveID: "drive-id",
|
||||
FileID: "file-id",
|
||||
Title: "Historical",
|
||||
Size: int64(len(data)),
|
||||
FingerprintStatus: "pending",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if err := cat.UpsertVideo(ctx, video); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
app := &App{
|
||||
cat: cat,
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
fingerprintWorkers: make(map[string]*fingerprint.Worker),
|
||||
}
|
||||
drv := &serverFingerprintFakeDrive{path: dataPath}
|
||||
fingerprintWorker := fingerprint.NewWorker(cat, drv, fingerprint.Config{})
|
||||
go fingerprintWorker.Run(ctx)
|
||||
|
||||
app.registerPreviewWorkers(ctx, "drive-id", nil, nil, fingerprintWorker, func() {})
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.SampledSHA256 != "" && got.FingerprintStatus == "ready" {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video after timeout: %v", err)
|
||||
}
|
||||
t.Fatalf("fingerprint status=%q sampled=%q, want ready with hash", got.FingerprintStatus, got.SampledSHA256)
|
||||
}
|
||||
|
||||
func TestRunScanStartsFingerprintBeforeThumbnailAndPreviewDrain(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
seedDriveWithTeaser(t, cat, "drive-id", true)
|
||||
|
||||
dataPath := filepath.Join(t.TempDir(), "scan-video.mp4")
|
||||
data := []byte("scan video content for independent fingerprint")
|
||||
if err := os.WriteFile(dataPath, data, 0o644); err != nil {
|
||||
t.Fatalf("write video data: %v", err)
|
||||
}
|
||||
|
||||
drv := &serverScanFingerprintFakeDrive{
|
||||
serverFingerprintFakeDrive: serverFingerprintFakeDrive{path: dataPath},
|
||||
entries: []drives.Entry{{
|
||||
ID: "file-id",
|
||||
Name: "scan-video.mp4",
|
||||
Size: int64(len(data)),
|
||||
ParentID: "root",
|
||||
}},
|
||||
}
|
||||
registry := proxy.NewRegistry()
|
||||
registry.Set("drive-id", drv)
|
||||
|
||||
gen := &serverFakeTeaserGenerator{}
|
||||
worker := preview.NewWorker(gen, cat, drv)
|
||||
thumbWorker := preview.NewThumbWorker(gen, cat, drv)
|
||||
fingerprintWorker := fingerprint.NewWorker(cat, drv, fingerprint.Config{})
|
||||
go fingerprintWorker.Run(ctx)
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{
|
||||
Scanner: config.Scanner{VideoExtensions: []string{".mp4"}},
|
||||
},
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
workers: map[string]*preview.Worker{"drive-id": worker},
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{"drive-id": thumbWorker},
|
||||
fingerprintWorkers: map[string]*fingerprint.Worker{"drive-id": fingerprintWorker},
|
||||
}
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
app.runScan(ctx, "drive-id")
|
||||
}()
|
||||
|
||||
videoID := "fake-drive-id-file-id"
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
got, err := cat.GetVideo(ctx, videoID)
|
||||
if err == nil && got.SampledSHA256 != "" && got.FingerprintStatus == "ready" {
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("scan did not stop after context cancel")
|
||||
}
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail url = %q, want fingerprint before thumbnail generation", got.ThumbnailURL)
|
||||
}
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("scan did not stop after context cancel")
|
||||
}
|
||||
got, err := cat.GetVideo(context.Background(), videoID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video after timeout: %v", err)
|
||||
}
|
||||
t.Fatalf("fingerprint status=%q sampled=%q, want ready before thumbnail/preview drain", got.FingerprintStatus, got.SampledSHA256)
|
||||
}
|
||||
|
||||
func TestNightlyTargetsComeFromCatalogBeforeDriveAttach(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, d := range []*catalog.Drive{
|
||||
{ID: "115", Kind: "p115", Name: "115", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "pikpak", Kind: "pikpak", Name: "PikPak", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "91-spider", Kind: "spider91", Name: "91 Spider", RootID: "0", TeaserEnabled: true},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
t.Fatalf("seed drive %s: %v", d.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
app := &App{cat: cat}
|
||||
scanIDs := app.listScanTargetIDs(ctx)
|
||||
if len(scanIDs) != 2 || scanIDs[0] != "115" || scanIDs[1] != "pikpak" {
|
||||
t.Fatalf("scan target ids = %#v, want 115 and pikpak from catalog", scanIDs)
|
||||
}
|
||||
spiderIDs := app.listSpider91DriveIDs(ctx)
|
||||
if len(spiderIDs) != 1 || spiderIDs[0] != "91-spider" {
|
||||
t.Fatalf("spider91 ids = %#v, want catalog spider drive", spiderIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedThumbnailsDoNotBlockPreviewGeneration(t *testing.T) {
|
||||
@@ -205,7 +396,7 @@ func TestFailedThumbnailsDoNotBlockPreviewGeneration(t *testing.T) {
|
||||
go worker.Run(ctx)
|
||||
go thumbWorker.Run(ctx)
|
||||
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, func() {})
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, nil, func() {})
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
@@ -480,6 +671,106 @@ func TestCleanupMissingPikPakVideosRemovesDatabaseRowsAndLocalAssets(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupDuplicateVideoAssetsRemovesOnlyDuplicateLocalAssets(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
localDir := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
canonicalPreview := filepath.Join(localDir, "canonical.mp4")
|
||||
duplicatePreview := filepath.Join(localDir, "duplicate.mp4")
|
||||
canonicalThumb := filepath.Join(localDir, "thumbs", "canonical-video.jpg")
|
||||
duplicateThumb := filepath.Join(localDir, "thumbs", "duplicate-video.jpg")
|
||||
for _, path := range []string{canonicalPreview, duplicatePreview, canonicalThumb, duplicateThumb} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("asset"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
|
||||
for _, v := range []*catalog.Video{
|
||||
{
|
||||
ID: "canonical-video",
|
||||
DriveID: "115",
|
||||
FileID: "file-a",
|
||||
Title: "Canonical",
|
||||
Size: 2048,
|
||||
ThumbnailURL: "/p/thumb/canonical-video",
|
||||
PreviewLocal: canonicalPreview,
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "duplicate-video",
|
||||
DriveID: "onedrive",
|
||||
FileID: "file-b",
|
||||
Title: "Duplicate",
|
||||
Size: 2048,
|
||||
ThumbnailURL: "/p/thumb/duplicate-video",
|
||||
PreviewLocal: duplicatePreview,
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, v.ID, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
}
|
||||
if err := app.cleanupDuplicateVideoAssets(ctx); err != nil {
|
||||
t.Fatalf("cleanup duplicate video assets: %v", err)
|
||||
}
|
||||
|
||||
for _, path := range []string{canonicalPreview, canonicalThumb} {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("canonical asset %s missing: %v", path, err)
|
||||
}
|
||||
}
|
||||
for _, path := range []string{duplicatePreview, duplicateThumb} {
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("duplicate asset %s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
dup, err := cat.GetVideo(ctx, "duplicate-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get duplicate: %v", err)
|
||||
}
|
||||
if dup.PreviewLocal != "" || dup.PreviewStatus != "pending" {
|
||||
t.Fatalf("duplicate preview local=%q status=%q, want empty pending", dup.PreviewLocal, dup.PreviewStatus)
|
||||
}
|
||||
if dup.ThumbnailURL != "" {
|
||||
t.Fatalf("duplicate thumbnail url = %q, want empty", dup.ThumbnailURL)
|
||||
}
|
||||
canon, err := cat.GetVideo(ctx, "canonical-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get canonical: %v", err)
|
||||
}
|
||||
if canon.PreviewLocal != canonicalPreview || canon.ThumbnailURL != "/p/thumb/canonical-video" {
|
||||
t.Fatalf("canonical changed: preview=%q thumb=%q", canon.PreviewLocal, canon.ThumbnailURL)
|
||||
}
|
||||
}
|
||||
|
||||
type serverFakeTeaserGenerator struct {
|
||||
mu sync.Mutex
|
||||
events []string
|
||||
@@ -520,6 +811,28 @@ func (g *serverFakeTeaserGenerator) GenerateThumbnail(_ context.Context, _ *driv
|
||||
return "/tmp/" + videoID + ".jpg", nil
|
||||
}
|
||||
|
||||
type serverBlockingThumbGenerator struct {
|
||||
serverFakeTeaserGenerator
|
||||
started chan string
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (g *serverBlockingThumbGenerator) GenerateThumbnail(ctx context.Context, _ *drives.StreamLink, videoID string, _ float64) (string, error) {
|
||||
g.record("thumb:" + videoID)
|
||||
if g.started != nil {
|
||||
select {
|
||||
case g.started <- videoID:
|
||||
default:
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-g.release:
|
||||
return "/tmp/" + videoID + ".jpg", nil
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
type serverFakeDrive struct{}
|
||||
|
||||
func (d *serverFakeDrive) Kind() string { return "fake" }
|
||||
@@ -544,6 +857,24 @@ func (d *serverFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
}
|
||||
func (d *serverFakeDrive) RootID() string { return "root" }
|
||||
|
||||
type serverFingerprintFakeDrive struct {
|
||||
serverFakeDrive
|
||||
path string
|
||||
}
|
||||
|
||||
func (d *serverFingerprintFakeDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
|
||||
return &drives.StreamLink{URL: d.path}, nil
|
||||
}
|
||||
|
||||
type serverScanFingerprintFakeDrive struct {
|
||||
serverFingerprintFakeDrive
|
||||
entries []drives.Entry
|
||||
}
|
||||
|
||||
func (d *serverScanFingerprintFakeDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return d.entries, nil
|
||||
}
|
||||
|
||||
type serverLocalUploadFakeDrive struct {
|
||||
serverFakeDrive
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ preview:
|
||||
width: 480
|
||||
|
||||
# 盘列表。上线后请通过管理后台添加,本文件可留空。
|
||||
# kind 支持 quark / p115 / pikpak / wopan / onedrive。
|
||||
# kind 支持 quark / p115 / pikpak / wopan / onedrive / localstorage。
|
||||
# OneDrive 示例:
|
||||
# - id: "my-onedrive"
|
||||
# kind: "onedrive"
|
||||
@@ -67,6 +67,12 @@ preview:
|
||||
# root_id: "root"
|
||||
# params:
|
||||
# refresh_token: "..."
|
||||
# api_url_address: "https://api.oplist.org/onedrive/renewapi"
|
||||
# region: "global"
|
||||
# 本地存储示例:
|
||||
# - id: "local-media"
|
||||
# kind: "localstorage"
|
||||
# name: "本地视频目录"
|
||||
# root_id: "/"
|
||||
# scan_root_id: "/"
|
||||
# params:
|
||||
# path: "/mnt/videos"
|
||||
drives: []
|
||||
|
||||
@@ -23,6 +23,10 @@ type AdminServer struct {
|
||||
Auth *auth.Authenticator
|
||||
// VersionFilePath points to the installer-written .version file.
|
||||
VersionFilePath string
|
||||
// ImageVersion is the Docker image version injected at build/runtime.
|
||||
// It takes precedence over VersionFilePath because Docker data volumes can
|
||||
// keep an older .version file across image upgrades.
|
||||
ImageVersion string
|
||||
// GitHubRepo is the owner/name repo used for update checks.
|
||||
GitHubRepo string
|
||||
// ReleaseAPIURL and HTTPClient are injectable for tests. Production code leaves them empty.
|
||||
@@ -78,8 +82,9 @@ type GenerationStatus struct {
|
||||
}
|
||||
|
||||
type DriveGenerationStatuses struct {
|
||||
Thumbnail GenerationStatus `json:"thumbnail"`
|
||||
Preview GenerationStatus `json:"preview"`
|
||||
Thumbnail GenerationStatus `json:"thumbnail"`
|
||||
Preview GenerationStatus `json:"preview"`
|
||||
Fingerprint GenerationStatus `json:"fingerprint"`
|
||||
}
|
||||
|
||||
func (a *AdminServer) Register(r chi.Router) {
|
||||
@@ -278,6 +283,9 @@ func (a *AdminServer) checkUpdate(ctx context.Context) (updateCheckDTO, error) {
|
||||
}
|
||||
|
||||
func (a *AdminServer) installedVersion() string {
|
||||
if version := strings.TrimSpace(a.ImageVersion); version != "" {
|
||||
return version
|
||||
}
|
||||
path := strings.TrimSpace(a.VersionFilePath)
|
||||
if path == "" {
|
||||
path = ".version"
|
||||
@@ -346,6 +354,11 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
fingerprintCounts, err := a.Catalog.CountFingerprintsByDrive(r.Context())
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
generationStatuses := map[string]DriveGenerationStatuses{}
|
||||
if a.GetDriveGenerationStatuses != nil {
|
||||
generationStatuses = a.GetDriveGenerationStatuses()
|
||||
@@ -368,20 +381,25 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
SkipDirIDs []string `json:"skipDirIds"`
|
||||
// LastCrawlAt 是 spider91 上次成功爬取的 unix 秒(来自 credentials.last_crawl_at)。
|
||||
// 其它 kind 留 0;前端用它显示"上次抓取: N 小时前"。
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
FingerprintReadyCount int `json:"fingerprintReadyCount"`
|
||||
FingerprintPendingCount int `json:"fingerprintPendingCount"`
|
||||
FingerprintFailedCount int `json:"fingerprintFailedCount"`
|
||||
}
|
||||
list := make([]out, 0, len(drives))
|
||||
for _, d := range drives {
|
||||
counts := teaserCounts[d.ID]
|
||||
thumbCounts := thumbnailCounts[d.ID]
|
||||
fingerprintCount := fingerprintCounts[d.ID]
|
||||
generation := generationStatuses[d.ID]
|
||||
if generation.Thumbnail.State == "" {
|
||||
generation.Thumbnail.State = "idle"
|
||||
@@ -389,6 +407,9 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
if generation.Preview.State == "" {
|
||||
generation.Preview.State = "idle"
|
||||
}
|
||||
if generation.Fingerprint.State == "" {
|
||||
generation.Fingerprint.State = "idle"
|
||||
}
|
||||
// spider91 没有用户凭证概念;只要存在 drive 行就视为"已配置"。
|
||||
// last_crawl_at 是后端自动写入的运行状态字段,不计入 hasCredential 判定。
|
||||
hasCred := false
|
||||
@@ -414,18 +435,22 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
ID: d.ID, Kind: d.Kind, Name: d.Name,
|
||||
RootID: d.RootID, ScanRootID: d.ScanRootID,
|
||||
Status: d.Status, LastError: d.LastError,
|
||||
HasCredential: hasCred,
|
||||
TeaserEnabled: d.TeaserEnabled,
|
||||
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
|
||||
LastCrawlAt: lastCrawlAt,
|
||||
ThumbnailGenerationStatus: generation.Thumbnail,
|
||||
PreviewGenerationStatus: generation.Preview,
|
||||
ThumbnailReadyCount: thumbCounts.Ready,
|
||||
ThumbnailPendingCount: thumbCounts.Pending,
|
||||
ThumbnailFailedCount: thumbCounts.Failed,
|
||||
TeaserReadyCount: counts.Ready,
|
||||
TeaserPendingCount: counts.Pending,
|
||||
TeaserFailedCount: counts.Failed,
|
||||
HasCredential: hasCred,
|
||||
TeaserEnabled: d.TeaserEnabled,
|
||||
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
|
||||
LastCrawlAt: lastCrawlAt,
|
||||
ThumbnailGenerationStatus: generation.Thumbnail,
|
||||
PreviewGenerationStatus: generation.Preview,
|
||||
FingerprintGenerationStatus: generation.Fingerprint,
|
||||
ThumbnailReadyCount: thumbCounts.Ready,
|
||||
ThumbnailPendingCount: thumbCounts.Pending,
|
||||
ThumbnailFailedCount: thumbCounts.Failed,
|
||||
TeaserReadyCount: counts.Ready,
|
||||
TeaserPendingCount: counts.Pending,
|
||||
TeaserFailedCount: counts.Failed,
|
||||
FingerprintReadyCount: fingerprintCount.Ready,
|
||||
FingerprintPendingCount: fingerprintCount.Pending,
|
||||
FingerprintFailedCount: fingerprintCount.Failed,
|
||||
})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, list)
|
||||
|
||||
@@ -194,6 +194,54 @@ func TestHandleCheckUpdateReportsUpToDate(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleCheckUpdateUsesDockerImageVersion(t *testing.T) {
|
||||
releaseServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"tag_name": "v0.2.0",
|
||||
"html_url": "https://github.com/nianzhibai/91/releases/tag/v0.2.0",
|
||||
})
|
||||
}))
|
||||
t.Cleanup(releaseServer.Close)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/update/check", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{
|
||||
ImageVersion: "v0.1.0",
|
||||
ReleaseAPIURL: releaseServer.URL,
|
||||
}).handleCheckUpdate(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got updateCheckDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.CurrentVersion != "v0.1.0" {
|
||||
t.Fatalf("currentVersion = %q, want v0.1.0", got.CurrentVersion)
|
||||
}
|
||||
if !got.HasUpdate {
|
||||
t.Fatalf("hasUpdate = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstalledVersionPrefersDockerImageVersionOverVersionFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
versionFile := filepath.Join(dir, ".version")
|
||||
if err := os.WriteFile(versionFile, []byte("v0.1.0\n"), 0o644); err != nil {
|
||||
t.Fatalf("write version file: %v", err)
|
||||
}
|
||||
|
||||
got := (&AdminServer{
|
||||
VersionFilePath: versionFile,
|
||||
ImageVersion: "v0.2.0",
|
||||
}).installedVersion()
|
||||
|
||||
if got != "v0.2.0" {
|
||||
t.Fatalf("installedVersion = %q, want v0.2.0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertDrivePreservesExistingCredentialsWhenRequestCredentialsEmpty(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -323,11 +371,11 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
|
||||
now := time.Now()
|
||||
videos := []*catalog.Video{
|
||||
{ID: "od-ready-1", DriveID: "OneDrive", FileID: "od-file-1", Title: "OD Ready 1", ThumbnailURL: "/p/thumb/od-ready-1", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-ready-2", DriveID: "OneDrive", FileID: "od-file-2", Title: "OD Ready 2", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-pending", DriveID: "OneDrive", FileID: "od-file-3", Title: "OD Pending", PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "pp-pending", DriveID: "PikPak", FileID: "pp-file-1", Title: "PP Pending", PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "pp-failed", DriveID: "PikPak", FileID: "pp-file-2", Title: "PP Failed", ThumbnailURL: "/p/thumb/pp-failed", PreviewStatus: "failed", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-ready-1", DriveID: "OneDrive", FileID: "od-file-1", Title: "OD Ready 1", Size: 100, ThumbnailURL: "/p/thumb/od-ready-1", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-ready-2", DriveID: "OneDrive", FileID: "od-file-2", Title: "OD Ready 2", Size: 100, PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-pending", DriveID: "OneDrive", FileID: "od-file-3", Title: "OD Pending", Size: 100, PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "pp-pending", DriveID: "PikPak", FileID: "pp-file-1", Title: "PP Pending", Size: 100, PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "pp-failed", DriveID: "PikPak", FileID: "pp-file-2", Title: "PP Failed", Size: 100, ThumbnailURL: "/p/thumb/pp-failed", PreviewStatus: "failed", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
}
|
||||
for _, v := range videos {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
@@ -337,6 +385,12 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
if err := cat.UpdateVideoMeta(ctx, "od-ready-2", catalog.VideoMetaPatch{ThumbnailStatus: "failed"}); err != nil {
|
||||
t.Fatalf("mark thumbnail failed: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "od-ready-1", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "ready", ""); err != nil {
|
||||
t.Fatalf("mark fingerprint ready: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "od-ready-2", "", "failed", "sample failed"); err != nil {
|
||||
t.Fatalf("mark fingerprint failed: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/drives", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
@@ -345,8 +399,9 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
GetDriveGenerationStatuses: func() map[string]DriveGenerationStatuses {
|
||||
return map[string]DriveGenerationStatuses{
|
||||
"OneDrive": {
|
||||
Thumbnail: GenerationStatus{State: "cooling", QueueLength: 3, CooldownUntil: "2026-05-16T21:00:00+08:00"},
|
||||
Preview: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
|
||||
Thumbnail: GenerationStatus{State: "cooling", QueueLength: 3, CooldownUntil: "2026-05-16T21:00:00+08:00"},
|
||||
Preview: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
|
||||
Fingerprint: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
|
||||
},
|
||||
}
|
||||
},
|
||||
@@ -356,48 +411,64 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got []struct {
|
||||
ID string `json:"id"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
ID string `json:"id"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
FingerprintReadyCount int `json:"fingerprintReadyCount"`
|
||||
FingerprintPendingCount int `json:"fingerprintPendingCount"`
|
||||
FingerprintFailedCount int `json:"fingerprintFailedCount"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
byID := map[string]struct {
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
FingerprintReady int
|
||||
FingerprintPending int
|
||||
FingerprintFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
Fingerprint GenerationStatus
|
||||
}{}
|
||||
for _, d := range got {
|
||||
byID[d.ID] = struct {
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
FingerprintReady int
|
||||
FingerprintPending int
|
||||
FingerprintFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
Fingerprint GenerationStatus
|
||||
}{
|
||||
TeaserReady: d.TeaserReadyCount,
|
||||
TeaserPending: d.TeaserPendingCount,
|
||||
TeaserFailed: d.TeaserFailedCount,
|
||||
ThumbnailReady: d.ThumbnailReadyCount,
|
||||
ThumbnailPending: d.ThumbnailPendingCount,
|
||||
ThumbnailFailed: d.ThumbnailFailedCount,
|
||||
Thumbnail: d.ThumbnailGenerationStatus,
|
||||
Preview: d.PreviewGenerationStatus,
|
||||
TeaserReady: d.TeaserReadyCount,
|
||||
TeaserPending: d.TeaserPendingCount,
|
||||
TeaserFailed: d.TeaserFailedCount,
|
||||
ThumbnailReady: d.ThumbnailReadyCount,
|
||||
ThumbnailPending: d.ThumbnailPendingCount,
|
||||
ThumbnailFailed: d.ThumbnailFailedCount,
|
||||
FingerprintReady: d.FingerprintReadyCount,
|
||||
FingerprintPending: d.FingerprintPendingCount,
|
||||
FingerprintFailed: d.FingerprintFailedCount,
|
||||
Thumbnail: d.ThumbnailGenerationStatus,
|
||||
Preview: d.PreviewGenerationStatus,
|
||||
Fingerprint: d.FingerprintGenerationStatus,
|
||||
}
|
||||
}
|
||||
if byID["OneDrive"].TeaserReady != 2 || byID["OneDrive"].TeaserPending != 1 || byID["OneDrive"].TeaserFailed != 0 {
|
||||
@@ -409,13 +480,22 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
if byID["OneDrive"].Thumbnail.State != "cooling" || byID["OneDrive"].Preview.State != "generating" {
|
||||
t.Fatalf("OneDrive generation statuses = %#v, want thumbnail cooling and preview generating", byID["OneDrive"])
|
||||
}
|
||||
if byID["OneDrive"].FingerprintReady != 1 || byID["OneDrive"].FingerprintPending != 1 || byID["OneDrive"].FingerprintFailed != 1 {
|
||||
t.Fatalf("OneDrive fingerprint counts = %#v, want ready=1 pending=1 failed=1", byID["OneDrive"])
|
||||
}
|
||||
if byID["OneDrive"].Fingerprint.State != "generating" {
|
||||
t.Fatalf("OneDrive fingerprint status = %#v, want generating", byID["OneDrive"].Fingerprint)
|
||||
}
|
||||
if byID["PikPak"].TeaserReady != 0 || byID["PikPak"].TeaserPending != 1 || byID["PikPak"].TeaserFailed != 1 {
|
||||
t.Fatalf("PikPak counts = %#v, want ready=0 pending=1 failed=1", byID["PikPak"])
|
||||
}
|
||||
if byID["PikPak"].ThumbnailReady != 1 || byID["PikPak"].ThumbnailPending != 1 || byID["PikPak"].ThumbnailFailed != 0 {
|
||||
t.Fatalf("PikPak thumbnail counts = %#v, want ready=1 pending=1 failed=0", byID["PikPak"])
|
||||
}
|
||||
if byID["PikPak"].Thumbnail.State != "idle" || byID["PikPak"].Preview.State != "idle" {
|
||||
if byID["PikPak"].FingerprintPending != 2 {
|
||||
t.Fatalf("PikPak fingerprint counts = %#v, want pending=2", byID["PikPak"])
|
||||
}
|
||||
if byID["PikPak"].Thumbnail.State != "idle" || byID["PikPak"].Preview.State != "idle" || byID["PikPak"].Fingerprint.State != "idle" {
|
||||
t.Fatalf("PikPak generation statuses = %#v, want idle defaults", byID["PikPak"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/auth"
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives/localstorage"
|
||||
"github.com/video-site/backend/internal/drives/localupload"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
@@ -899,6 +900,8 @@ func driveKindLabel(kind string) string {
|
||||
return "联通沃盘"
|
||||
case "onedrive":
|
||||
return "OneDrive"
|
||||
case localstorage.Kind:
|
||||
return "本地存储"
|
||||
case spider91.Kind:
|
||||
return "91 爬虫"
|
||||
default:
|
||||
|
||||
@@ -41,35 +41,38 @@ func (c *Catalog) Close() error { return c.db.Close() }
|
||||
// ---------- Video ----------
|
||||
|
||||
type Video struct {
|
||||
ID string `json:"id"`
|
||||
DriveID string `json:"driveId"`
|
||||
FileID string `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
ContentHash string `json:"contentHash"`
|
||||
ParentID string `json:"parentId"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Tags []string `json:"tags"`
|
||||
DurationSeconds int `json:"durationSeconds"`
|
||||
Size int64 `json:"size"`
|
||||
Ext string `json:"ext"`
|
||||
Quality string `json:"quality"`
|
||||
ThumbnailURL string `json:"thumbnailUrl"`
|
||||
PreviewFileID string `json:"previewFileId"`
|
||||
PreviewLocal string `json:"previewLocal"`
|
||||
PreviewStatus string `json:"previewStatus"`
|
||||
Views int `json:"views"`
|
||||
Favorites int `json:"favorites"`
|
||||
Comments int `json:"comments"`
|
||||
Likes int `json:"likes"`
|
||||
Dislikes int `json:"dislikes"`
|
||||
Category string `json:"category"`
|
||||
Hidden bool `json:"hidden"`
|
||||
Badges []string `json:"badges"`
|
||||
Description string `json:"description"`
|
||||
PublishedAt time.Time `json:"publishedAt"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
ID string `json:"id"`
|
||||
DriveID string `json:"driveId"`
|
||||
FileID string `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
ContentHash string `json:"contentHash"`
|
||||
SampledSHA256 string `json:"sampledSha256"`
|
||||
FingerprintStatus string `json:"fingerprintStatus"`
|
||||
FingerprintError string `json:"fingerprintError"`
|
||||
ParentID string `json:"parentId"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Tags []string `json:"tags"`
|
||||
DurationSeconds int `json:"durationSeconds"`
|
||||
Size int64 `json:"size"`
|
||||
Ext string `json:"ext"`
|
||||
Quality string `json:"quality"`
|
||||
ThumbnailURL string `json:"thumbnailUrl"`
|
||||
PreviewFileID string `json:"previewFileId"`
|
||||
PreviewLocal string `json:"previewLocal"`
|
||||
PreviewStatus string `json:"previewStatus"`
|
||||
Views int `json:"views"`
|
||||
Favorites int `json:"favorites"`
|
||||
Comments int `json:"comments"`
|
||||
Likes int `json:"likes"`
|
||||
Dislikes int `json:"dislikes"`
|
||||
Category string `json:"category"`
|
||||
Hidden bool `json:"hidden"`
|
||||
Badges []string `json:"badges"`
|
||||
Description string `json:"description"`
|
||||
PublishedAt time.Time `json:"publishedAt"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
func (c *Catalog) UpsertVideo(ctx context.Context, v *Video) error {
|
||||
@@ -109,6 +112,18 @@ ON CONFLICT(id) DO UPDATE SET
|
||||
WHEN excluded.content_hash != '' THEN excluded.content_hash
|
||||
ELSE videos.content_hash
|
||||
END,
|
||||
sampled_sha256 = CASE
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN ''
|
||||
ELSE videos.sampled_sha256
|
||||
END,
|
||||
fingerprint_status = CASE
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN 'pending'
|
||||
ELSE COALESCE(videos.fingerprint_status, 'pending')
|
||||
END,
|
||||
fingerprint_error = CASE
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN ''
|
||||
ELSE COALESCE(videos.fingerprint_error, '')
|
||||
END,
|
||||
duration_seconds= excluded.duration_seconds,
|
||||
size_bytes = excluded.size_bytes,
|
||||
ext = excluded.ext,
|
||||
@@ -486,8 +501,14 @@ func (c *Catalog) ListVideosByThumbnailStatus(ctx context.Context, driveID, stat
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ListVideosNeedingThumbnail returns videos that still need a thumbnail attempt.
|
||||
// ListVideosNeedingThumbnail returns videos that still need thumbnail-worker work.
|
||||
// Besides missing thumbnails, this includes videos with an existing thumbnail but
|
||||
// missing duration metadata, because the thumbnail worker probes duration while
|
||||
// it already has a stream link.
|
||||
// Failed thumbnails are reported separately and should not block teaser generation.
|
||||
// Videos whose local assets were cleared because they are fingerprint duplicates
|
||||
// stay pending in the DB, but uniqueVideoWhereSQL keeps them out of this queue
|
||||
// while their canonical sibling still exists.
|
||||
func (c *Catalog) ListVideosNeedingThumbnail(ctx context.Context, driveID string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10000
|
||||
@@ -495,8 +516,11 @@ func (c *Catalog) ListVideosNeedingThumbnail(ctx context.Context, driveID string
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND COALESCE(thumbnail_url, '') = ''
|
||||
AND COALESCE(thumbnail_status, 'pending') != 'failed'
|
||||
AND (
|
||||
COALESCE(thumbnail_url, '') = ''
|
||||
OR COALESCE(duration_seconds, 0) <= 0
|
||||
)
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('failed', 'skipped')
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
AND `+uniqueVideoWhereSQL+`
|
||||
ORDER BY created_at ASC
|
||||
@@ -522,8 +546,11 @@ func (c *Catalog) CountVideosNeedingThumbnail(ctx context.Context, driveID strin
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND COALESCE(thumbnail_url, '') = ''
|
||||
AND COALESCE(thumbnail_status, 'pending') != 'failed'
|
||||
AND (
|
||||
COALESCE(thumbnail_url, '') = ''
|
||||
OR COALESCE(duration_seconds, 0) <= 0
|
||||
)
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('failed', 'skipped')
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
AND `+uniqueVideoWhereSQL,
|
||||
driveID).Scan(&count)
|
||||
@@ -668,6 +695,60 @@ func (c *Catalog) FindVideoByFileSignature(ctx context.Context, fileName string,
|
||||
return scanVideo(row)
|
||||
}
|
||||
|
||||
func (c *Catalog) ListVideosNeedingFingerprint(ctx context.Context, driveID string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10000
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND size_bytes > 0
|
||||
AND COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = 'pending'
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT ?`,
|
||||
driveID, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) UpdateVideoFingerprint(ctx context.Context, id, sampledSHA256, status, errText string) error {
|
||||
sampledSHA256 = normalizeContentHash(sampledSHA256)
|
||||
if status == "" {
|
||||
status = "pending"
|
||||
}
|
||||
if len(errText) > 500 {
|
||||
errText = errText[:500]
|
||||
}
|
||||
res, err := c.db.ExecContext(ctx,
|
||||
`UPDATE videos
|
||||
SET sampled_sha256 = ?,
|
||||
fingerprint_status = ?,
|
||||
fingerprint_error = ?,
|
||||
updated_at = ?
|
||||
WHERE id = ?`,
|
||||
sampledSHA256, status, errText, time.Now().UnixMilli(), id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rows, err := res.RowsAffected(); err == nil && rows == 0 {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type ListParams struct {
|
||||
Keyword string
|
||||
DriveID string
|
||||
@@ -852,6 +933,12 @@ type DriveThumbnailCounts struct {
|
||||
Failed int
|
||||
}
|
||||
|
||||
type DriveFingerprintCounts struct {
|
||||
Ready int
|
||||
Pending int
|
||||
Failed int
|
||||
}
|
||||
|
||||
func (c *Catalog) CountTeasersByDrive(ctx context.Context) (map[string]DriveTeaserCounts, error) {
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT drive_id,
|
||||
@@ -914,6 +1001,52 @@ func (c *Catalog) CountThumbnailsByDrive(ctx context.Context) (map[string]DriveT
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) CountFingerprintsByDrive(ctx context.Context) (map[string]DriveFingerprintCounts, error) {
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT drive_id,
|
||||
COUNT(CASE WHEN COALESCE(sampled_sha256, '') != ''
|
||||
OR COALESCE(fingerprint_status, 'pending') = 'ready' THEN 1 END) AS ready_count,
|
||||
COUNT(CASE WHEN size_bytes > 0
|
||||
AND COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = 'pending' THEN 1 END) AS pending_count,
|
||||
COUNT(CASE WHEN COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = 'failed' THEN 1 END) AS failed_count
|
||||
FROM videos
|
||||
WHERE COALESCE(hidden, 0) = 0
|
||||
GROUP BY drive_id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make(map[string]DriveFingerprintCounts)
|
||||
for rows.Next() {
|
||||
var driveID string
|
||||
var counts DriveFingerprintCounts
|
||||
if err := rows.Scan(&driveID, &counts.Ready, &counts.Pending, &counts.Failed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[driveID] = counts
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) CountVideosNeedingFingerprint(ctx context.Context, driveID string) (int, error) {
|
||||
var count int
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND size_bytes > 0
|
||||
AND COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = 'pending'
|
||||
AND COALESCE(hidden, 0) = 0`,
|
||||
driveID).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
type LocalMediaRef struct {
|
||||
DriveID string
|
||||
VideoID string
|
||||
@@ -943,6 +1076,124 @@ func (c *Catalog) ListLocalMediaRefs(ctx context.Context) ([]LocalMediaRef, erro
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// DuplicateAssetCleanupCandidate points at a non-canonical video in a
|
||||
// size+sampled_sha256 duplicate group that still owns generated local assets.
|
||||
// The cleanup job uses this to remove duplicate thumbnails/teasers without
|
||||
// touching the original cloud file or deleting the catalog row.
|
||||
type DuplicateAssetCleanupCandidate struct {
|
||||
VideoID string
|
||||
DriveID string
|
||||
Title string
|
||||
PreviewLocal string
|
||||
ThumbnailURL string
|
||||
CanonicalID string
|
||||
SampledSHA256 string
|
||||
Size int64
|
||||
}
|
||||
|
||||
// ListDuplicateAssetCleanupCandidates returns duplicate videos whose own local
|
||||
// generated assets can be cleared. A group canonical is the same representative
|
||||
// used by uniqueVideoWhereSQL: earliest created_at, then lexicographically
|
||||
// smallest id.
|
||||
func (c *Catalog) ListDuplicateAssetCleanupCandidates(ctx context.Context, limit int) ([]DuplicateAssetCleanupCandidate, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10000
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
WITH canonical AS (
|
||||
SELECT v.id, v.size_bytes, v.sampled_sha256
|
||||
FROM videos v
|
||||
WHERE v.size_bytes > 0
|
||||
AND COALESCE(v.sampled_sha256, '') != ''
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM videos earlier
|
||||
WHERE earlier.size_bytes = v.size_bytes
|
||||
AND earlier.sampled_sha256 = v.sampled_sha256
|
||||
AND COALESCE(earlier.sampled_sha256, '') != ''
|
||||
AND earlier.size_bytes > 0
|
||||
AND (
|
||||
earlier.created_at < v.created_at
|
||||
OR (earlier.created_at = v.created_at AND earlier.id < v.id)
|
||||
)
|
||||
)
|
||||
)
|
||||
SELECT dup.id,
|
||||
dup.drive_id,
|
||||
dup.title,
|
||||
COALESCE(dup.preview_local, ''),
|
||||
COALESCE(dup.thumbnail_url, ''),
|
||||
canonical.id,
|
||||
dup.sampled_sha256,
|
||||
dup.size_bytes
|
||||
FROM videos dup
|
||||
JOIN canonical
|
||||
ON canonical.size_bytes = dup.size_bytes
|
||||
AND canonical.sampled_sha256 = dup.sampled_sha256
|
||||
WHERE dup.id != canonical.id
|
||||
AND dup.size_bytes > 0
|
||||
AND COALESCE(dup.sampled_sha256, '') != ''
|
||||
AND (
|
||||
COALESCE(dup.preview_local, '') != ''
|
||||
OR COALESCE(dup.thumbnail_url, '') = '/p/thumb/' || dup.id
|
||||
)
|
||||
ORDER BY dup.created_at ASC, dup.id ASC
|
||||
LIMIT ?`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []DuplicateAssetCleanupCandidate
|
||||
for rows.Next() {
|
||||
var item DuplicateAssetCleanupCandidate
|
||||
if err := rows.Scan(
|
||||
&item.VideoID,
|
||||
&item.DriveID,
|
||||
&item.Title,
|
||||
&item.PreviewLocal,
|
||||
&item.ThumbnailURL,
|
||||
&item.CanonicalID,
|
||||
&item.SampledSHA256,
|
||||
&item.Size,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ClearGeneratedAssets clears DB references to generated local assets for a
|
||||
// video. The statuses go back to pending so the video can regenerate assets if
|
||||
// it later becomes the canonical item after its older duplicate is removed.
|
||||
func (c *Catalog) ClearGeneratedAssets(ctx context.Context, videoID string, clearPreview, clearThumbnail bool) error {
|
||||
parts := []string{}
|
||||
args := []any{}
|
||||
if clearPreview {
|
||||
parts = append(parts, "preview_file_id = ''", "preview_local = ''", "preview_status = 'pending'")
|
||||
}
|
||||
if clearThumbnail {
|
||||
parts = append(parts, "thumbnail_url = ''", "thumbnail_status = 'pending'")
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return nil
|
||||
}
|
||||
parts = append(parts, "updated_at = ?")
|
||||
args = append(args, time.Now().UnixMilli(), videoID)
|
||||
res, err := c.db.ExecContext(ctx, `UPDATE videos SET `+strings.Join(parts, ", ")+` WHERE id = ?`, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rows, err := res.RowsAffected(); err == nil && rows == 0 {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------- Drive ----------
|
||||
|
||||
type Drive struct {
|
||||
@@ -1171,7 +1422,9 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.upd
|
||||
// ---------- helpers ----------
|
||||
|
||||
const allVideoCols = `
|
||||
id, drive_id, file_id, COALESCE(file_name, ''), COALESCE(content_hash, ''), COALESCE(parent_id, ''), title, COALESCE(author, ''), COALESCE(tags, '[]'),
|
||||
id, drive_id, file_id, COALESCE(file_name, ''), COALESCE(content_hash, ''),
|
||||
COALESCE(sampled_sha256, ''), COALESCE(fingerprint_status, 'pending'), COALESCE(fingerprint_error, ''),
|
||||
COALESCE(parent_id, ''), title, COALESCE(author, ''), COALESCE(tags, '[]'),
|
||||
duration_seconds, size_bytes, COALESCE(ext, ''), COALESCE(quality, ''), COALESCE(thumbnail_url, ''),
|
||||
COALESCE(preview_file_id, ''), COALESCE(preview_local, ''), COALESCE(preview_status, 'pending'),
|
||||
views, favorites, comments, likes, dislikes,
|
||||
@@ -1190,6 +1443,20 @@ const uniqueVideoWhereSQL = `((COALESCE(videos.content_hash, '') = ''
|
||||
OR (dup.created_at = videos.created_at AND dup.id < videos.id)
|
||||
)
|
||||
))
|
||||
AND (COALESCE(videos.sampled_sha256, '') = ''
|
||||
OR videos.size_bytes <= 0
|
||||
OR NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM videos AS dup
|
||||
WHERE dup.sampled_sha256 = videos.sampled_sha256
|
||||
AND dup.size_bytes = videos.size_bytes
|
||||
AND COALESCE(dup.sampled_sha256, '') != ''
|
||||
AND dup.size_bytes > 0
|
||||
AND (
|
||||
dup.created_at < videos.created_at
|
||||
OR (dup.created_at = videos.created_at AND dup.id < videos.id)
|
||||
)
|
||||
))
|
||||
AND (COALESCE(videos.file_name, '') = ''
|
||||
OR videos.size_bytes <= 0
|
||||
OR NOT EXISTS (
|
||||
@@ -1215,7 +1482,9 @@ func scanVideo(row rowScanner) (*Video, error) {
|
||||
var publishedAt, createdAt, updatedAt int64
|
||||
var hidden int
|
||||
err := row.Scan(
|
||||
&v.ID, &v.DriveID, &v.FileID, &v.FileName, &v.ContentHash, &v.ParentID, &v.Title, &v.Author, &tagsJSON,
|
||||
&v.ID, &v.DriveID, &v.FileID, &v.FileName, &v.ContentHash,
|
||||
&v.SampledSHA256, &v.FingerprintStatus, &v.FingerprintError,
|
||||
&v.ParentID, &v.Title, &v.Author, &tagsJSON,
|
||||
&v.DurationSeconds, &v.Size, &v.Ext, &v.Quality, &v.ThumbnailURL,
|
||||
&v.PreviewFileID, &v.PreviewLocal, &v.PreviewStatus,
|
||||
&v.Views, &v.Favorites, &v.Comments, &v.Likes, &v.Dislikes,
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestListVideosDeduplicatesBySampledSHA256(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{
|
||||
ID: "drive-a-file-a",
|
||||
DriveID: "drive-a",
|
||||
FileID: "file-a",
|
||||
FileName: "first-name.mp4",
|
||||
Title: "First",
|
||||
Size: 1234,
|
||||
PublishedAt: now.Add(-time.Minute),
|
||||
CreatedAt: now.Add(-time.Minute),
|
||||
UpdatedAt: now.Add(-time.Minute),
|
||||
},
|
||||
{
|
||||
ID: "drive-b-file-b",
|
||||
DriveID: "drive-b",
|
||||
FileID: "file-b",
|
||||
FileName: "second-name.mp4",
|
||||
Title: "Second",
|
||||
Size: 1234,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("upsert %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list before fingerprint: %v", err)
|
||||
}
|
||||
if total != 2 || len(items) != 2 {
|
||||
t.Fatalf("before fingerprint total=%d len=%d, want 2", total, len(items))
|
||||
}
|
||||
|
||||
const sampled = "abc123"
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "drive-a-file-a", sampled, "ready", ""); err != nil {
|
||||
t.Fatalf("update a fingerprint: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "drive-b-file-b", sampled, "ready", ""); err != nil {
|
||||
t.Fatalf("update b fingerprint: %v", err)
|
||||
}
|
||||
|
||||
items, total, err = cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list after fingerprint: %v", err)
|
||||
}
|
||||
if total != 1 || len(items) != 1 {
|
||||
t.Fatalf("after fingerprint total=%d len=%d, want 1", total, len(items))
|
||||
}
|
||||
if items[0].ID != "drive-a-file-a" {
|
||||
t.Fatalf("canonical id = %q, want earliest created video", items[0].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuplicateAssetCleanupCandidates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
base := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
|
||||
videos := []*Video{
|
||||
{
|
||||
ID: "drive-a-canonical",
|
||||
DriveID: "drive-a",
|
||||
FileID: "file-a",
|
||||
FileName: "canonical.mp4",
|
||||
Title: "Canonical",
|
||||
Size: 1234,
|
||||
ThumbnailURL: "/p/thumb/drive-a-canonical",
|
||||
PreviewLocal: "/tmp/previews/canonical.mp4",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: base,
|
||||
CreatedAt: base,
|
||||
UpdatedAt: base,
|
||||
},
|
||||
{
|
||||
ID: "drive-b-duplicate",
|
||||
DriveID: "drive-b",
|
||||
FileID: "file-b",
|
||||
FileName: "duplicate.mp4",
|
||||
Title: "Duplicate",
|
||||
Size: 1234,
|
||||
ThumbnailURL: "/p/thumb/drive-b-duplicate",
|
||||
PreviewLocal: "/tmp/previews/duplicate.mp4",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: base.Add(time.Second),
|
||||
CreatedAt: base.Add(time.Second),
|
||||
UpdatedAt: base.Add(time.Second),
|
||||
},
|
||||
{
|
||||
ID: "drive-c-remote-thumb",
|
||||
DriveID: "drive-c",
|
||||
FileID: "file-c",
|
||||
FileName: "remote-thumb.mp4",
|
||||
Title: "Remote Thumbnail",
|
||||
Size: 1234,
|
||||
ThumbnailURL: "https://thumb.example/file-c.jpg",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: base.Add(2 * time.Second),
|
||||
CreatedAt: base.Add(2 * time.Second),
|
||||
UpdatedAt: base.Add(2 * time.Second),
|
||||
},
|
||||
}
|
||||
for _, v := range videos {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
const sampled = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
for _, v := range videos {
|
||||
if err := cat.UpdateVideoFingerprint(ctx, v.ID, sampled, "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, err := cat.ListDuplicateAssetCleanupCandidates(ctx, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list cleanup candidates: %v", err)
|
||||
}
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("candidates = %#v, want only local duplicate", items)
|
||||
}
|
||||
item := items[0]
|
||||
if item.VideoID != "drive-b-duplicate" || item.CanonicalID != "drive-a-canonical" {
|
||||
t.Fatalf("candidate = %#v, want duplicate with canonical", item)
|
||||
}
|
||||
|
||||
if err := cat.ClearGeneratedAssets(ctx, item.VideoID, true, true); err != nil {
|
||||
t.Fatalf("clear generated assets: %v", err)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, item.VideoID)
|
||||
if err != nil {
|
||||
t.Fatalf("get duplicate: %v", err)
|
||||
}
|
||||
if got.PreviewLocal != "" || got.PreviewStatus != "pending" {
|
||||
t.Fatalf("preview after cleanup local=%q status=%q, want empty pending", got.PreviewLocal, got.PreviewStatus)
|
||||
}
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail after cleanup = %q, want empty", got.ThumbnailURL)
|
||||
}
|
||||
var thumbStatus string
|
||||
if err := cat.db.QueryRowContext(ctx, `SELECT thumbnail_status FROM videos WHERE id = ?`, item.VideoID).Scan(&thumbStatus); err != nil {
|
||||
t.Fatalf("query thumbnail status: %v", err)
|
||||
}
|
||||
if thumbStatus != "pending" {
|
||||
t.Fatalf("thumbnail_status = %q, want pending", thumbStatus)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,9 @@ CREATE TABLE IF NOT EXISTS videos (
|
||||
file_id TEXT NOT NULL,
|
||||
file_name TEXT DEFAULT '', -- 网盘侧原始文件名,用于同名同大小去重
|
||||
content_hash TEXT DEFAULT '',
|
||||
sampled_sha256 TEXT DEFAULT '', -- 跨网盘统一采样指纹(size + sampled bytes)
|
||||
fingerprint_status TEXT DEFAULT 'pending', -- pending / ready / failed
|
||||
fingerprint_error TEXT DEFAULT '',
|
||||
parent_id TEXT,
|
||||
title TEXT NOT NULL,
|
||||
author TEXT,
|
||||
@@ -61,7 +64,7 @@ CREATE INDEX IF NOT EXISTS idx_video_tags_video ON video_tags(video_id);
|
||||
-- 网盘账户
|
||||
CREATE TABLE IF NOT EXISTS drives (
|
||||
id TEXT PRIMARY KEY,
|
||||
kind TEXT NOT NULL, -- quark / p115 / pikpak / wopan / onedrive / spider91
|
||||
kind TEXT NOT NULL, -- quark / p115 / pikpak / wopan / onedrive / localstorage / spider91
|
||||
name TEXT NOT NULL,
|
||||
root_id TEXT NOT NULL DEFAULT '0',
|
||||
scan_root_id TEXT, -- 扫描起点(默认 root_id)
|
||||
|
||||
@@ -43,6 +43,15 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "content_hash", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "sampled_sha256", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "fingerprint_status", "TEXT DEFAULT 'pending'"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "fingerprint_error", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "file_name", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -83,6 +92,9 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_content_hash ON videos(content_hash)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_sampled_sha256 ON videos(size_bytes, sampled_sha256)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_hidden ON videos(hidden)`); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -7,6 +7,90 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestListVideosNeedingThumbnailIncludesExistingThumbnailMissingDuration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
videos := []*Video{
|
||||
{
|
||||
ID: "duration-only",
|
||||
DriveID: "drive",
|
||||
FileID: "file-duration-only",
|
||||
Title: "Duration Only",
|
||||
ThumbnailURL: "/p/thumb/duration-only",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "complete",
|
||||
DriveID: "drive",
|
||||
FileID: "file-complete",
|
||||
Title: "Complete",
|
||||
DurationSeconds: 12,
|
||||
ThumbnailURL: "/p/thumb/complete",
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
{
|
||||
ID: "missing-thumb",
|
||||
DriveID: "drive",
|
||||
FileID: "file-missing-thumb",
|
||||
Title: "Missing Thumb",
|
||||
DurationSeconds: 18,
|
||||
PublishedAt: now.Add(2 * time.Second),
|
||||
CreatedAt: now.Add(2 * time.Second),
|
||||
UpdatedAt: now.Add(2 * time.Second),
|
||||
},
|
||||
{
|
||||
ID: "failed",
|
||||
DriveID: "drive",
|
||||
FileID: "file-failed",
|
||||
Title: "Failed",
|
||||
PublishedAt: now.Add(3 * time.Second),
|
||||
CreatedAt: now.Add(3 * time.Second),
|
||||
UpdatedAt: now.Add(3 * time.Second),
|
||||
},
|
||||
}
|
||||
for _, v := range videos {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
if err := cat.UpdateVideoMeta(ctx, "failed", VideoMetaPatch{ThumbnailStatus: "failed"}); err != nil {
|
||||
t.Fatalf("mark failed thumbnail: %v", err)
|
||||
}
|
||||
|
||||
items, err := cat.ListVideosNeedingThumbnail(ctx, "drive", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list videos needing thumbnail: %v", err)
|
||||
}
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("items = %#v, want duration-only and missing-thumb", items)
|
||||
}
|
||||
if items[0].ID != "duration-only" || items[1].ID != "missing-thumb" {
|
||||
t.Fatalf("item ids = %q, %q; want duration-only, missing-thumb", items[0].ID, items[1].ID)
|
||||
}
|
||||
|
||||
count, err := cat.CountVideosNeedingThumbnail(ctx, "drive")
|
||||
if err != nil {
|
||||
t.Fatalf("count videos needing thumbnail: %v", err)
|
||||
}
|
||||
if count != 2 {
|
||||
t.Fatalf("count = %d, want 2", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateTagAndClassifyAddsTagToMatchingExistingVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
@@ -202,7 +202,7 @@ type Nightly struct {
|
||||
// 这里保留 yaml 中的静态定义,用于启动时预置盘。生产建议只在 DB 里维护。
|
||||
type Drive struct {
|
||||
ID string `yaml:"id"`
|
||||
Kind string `yaml:"kind"` // quark / p115 / pikpak / wopan / onedrive
|
||||
Kind string `yaml:"kind"` // quark / p115 / pikpak / wopan / onedrive / localstorage
|
||||
Name string `yaml:"name"`
|
||||
RootID string `yaml:"root_id"`
|
||||
Params map[string]string `yaml:"params,omitempty"`
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
// Drive 是多家网盘统一抽象。上层不区分盘,只区分 Kind。
|
||||
type Drive interface {
|
||||
// Kind 返回驱动代号:"quark" / "p115" / "pikpak" / "wopan" / "onedrive"
|
||||
// Kind 返回驱动代号:"quark" / "p115" / "pikpak" / "wopan" / "onedrive" / "localstorage"
|
||||
Kind() string
|
||||
|
||||
// ID 返回该盘在 catalog 中的唯一标识
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
// Package localstorage exposes an existing server-side directory as a Drive.
|
||||
package localstorage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
const Kind = "localstorage"
|
||||
|
||||
type Config struct {
|
||||
ID string
|
||||
RootPath string
|
||||
}
|
||||
|
||||
type Driver struct {
|
||||
id string
|
||||
rootPath string
|
||||
}
|
||||
|
||||
func New(c Config) *Driver {
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
rootPath: c.RootPath,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Kind() string { return Kind }
|
||||
|
||||
func (d *Driver) ID() string { return d.id }
|
||||
|
||||
func (d *Driver) RootID() string { return "/" }
|
||||
|
||||
func (d *Driver) Init(context.Context) error {
|
||||
root, err := d.root()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("localstorage: stat root: %w", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("localstorage: root is not a directory: %s", root)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error) {
|
||||
dir, rel, err := d.pathForID(dirID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]drives.Entry, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Symlinks can escape the configured root or create cycles. Keep the
|
||||
// local storage drive predictable by scanning real files/directories only.
|
||||
if entry.Type()&os.ModeSymlink != 0 {
|
||||
continue
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !info.IsDir() && !info.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
childRel := joinRel(rel, entry.Name())
|
||||
out = append(out, drives.Entry{
|
||||
ID: encodeRel(childRel),
|
||||
Name: entry.Name(),
|
||||
Size: sizeForEntry(info),
|
||||
IsDir: info.IsDir(),
|
||||
ParentID: idForRel(rel),
|
||||
ModTime: info.ModTime(),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
|
||||
p, rel, err := d.pathForID(fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &drives.Entry{
|
||||
ID: idForRel(rel),
|
||||
Name: filepath.Base(p),
|
||||
Size: sizeForEntry(info),
|
||||
IsDir: info.IsDir(),
|
||||
ParentID: idForRel(parentRel(rel)),
|
||||
ModTime: info.ModTime(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
p, _, err := d.pathForID(fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.IsDir() || !info.Mode().IsRegular() || info.Size() <= 0 {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return &drives.StreamLink{
|
||||
URL: p,
|
||||
Expires: time.Now().Add(24 * time.Hour),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) root() (string, error) {
|
||||
raw := strings.TrimSpace(d.rootPath)
|
||||
if raw == "" {
|
||||
return "", errors.New("localstorage: empty path")
|
||||
}
|
||||
raw = os.ExpandEnv(raw)
|
||||
if strings.HasPrefix(raw, "~") {
|
||||
if home, err := os.UserHomeDir(); err == nil && home != "" {
|
||||
switch {
|
||||
case raw == "~":
|
||||
raw = home
|
||||
case strings.HasPrefix(raw, "~/") || strings.HasPrefix(raw, `~\`):
|
||||
raw = filepath.Join(home, raw[2:])
|
||||
}
|
||||
}
|
||||
}
|
||||
return filepath.Abs(raw)
|
||||
}
|
||||
|
||||
func (d *Driver) pathForID(id string) (string, string, error) {
|
||||
root, err := d.root()
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
rel, err := decodeRel(id)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if rel == "" {
|
||||
return root, "", nil
|
||||
}
|
||||
p, err := filepath.Abs(filepath.Join(root, filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if p != root && !strings.HasPrefix(p, root+string(os.PathSeparator)) {
|
||||
return "", "", errors.New("localstorage: path escapes root")
|
||||
}
|
||||
return p, rel, nil
|
||||
}
|
||||
|
||||
func decodeRel(id string) (string, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" || id == "/" {
|
||||
return "", nil
|
||||
}
|
||||
raw, err := base64.RawURLEncoding.DecodeString(id)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("localstorage: invalid file id: %w", err)
|
||||
}
|
||||
rel := filepath.ToSlash(filepath.Clean(filepath.FromSlash(string(raw))))
|
||||
if rel == "." {
|
||||
return "", nil
|
||||
}
|
||||
if strings.HasPrefix(rel, "../") || rel == ".." || strings.HasPrefix(rel, "/") {
|
||||
return "", errors.New("localstorage: invalid relative path")
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
|
||||
func encodeRel(rel string) string {
|
||||
rel = filepath.ToSlash(filepath.Clean(filepath.FromSlash(rel)))
|
||||
if rel == "." || rel == "" {
|
||||
return "/"
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(rel))
|
||||
}
|
||||
|
||||
func idForRel(rel string) string {
|
||||
if rel == "" {
|
||||
return "/"
|
||||
}
|
||||
return encodeRel(rel)
|
||||
}
|
||||
|
||||
func joinRel(parent, name string) string {
|
||||
if parent == "" {
|
||||
return filepath.ToSlash(name)
|
||||
}
|
||||
return filepath.ToSlash(filepath.Join(filepath.FromSlash(parent), name))
|
||||
}
|
||||
|
||||
func parentRel(rel string) string {
|
||||
if rel == "" {
|
||||
return ""
|
||||
}
|
||||
parent := filepath.ToSlash(filepath.Dir(filepath.FromSlash(rel)))
|
||||
if parent == "." {
|
||||
return ""
|
||||
}
|
||||
return parent
|
||||
}
|
||||
|
||||
func sizeForEntry(info os.FileInfo) int64 {
|
||||
if info == nil || info.IsDir() {
|
||||
return 0
|
||||
}
|
||||
return info.Size()
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
@@ -0,0 +1,119 @@
|
||||
package localstorage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/scanner"
|
||||
)
|
||||
|
||||
func TestListEncodesRelativePathsAndStreamURLResolvesFile(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
sub := filepath.Join(root, "clips")
|
||||
if err := os.MkdirAll(sub, 0o755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
videoPath := filepath.Join(sub, "sample.mp4")
|
||||
if err := os.WriteFile(videoPath, []byte("video"), 0o644); err != nil {
|
||||
t.Fatalf("write video: %v", err)
|
||||
}
|
||||
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
if err := drv.Init(context.Background()); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
rootEntries, err := drv.List(context.Background(), drv.RootID())
|
||||
if err != nil {
|
||||
t.Fatalf("list root: %v", err)
|
||||
}
|
||||
if len(rootEntries) != 1 || !rootEntries[0].IsDir {
|
||||
t.Fatalf("root entries = %#v, want one directory", rootEntries)
|
||||
}
|
||||
if strings.Contains(rootEntries[0].ID, "/") {
|
||||
t.Fatalf("encoded dir id contains slash: %q", rootEntries[0].ID)
|
||||
}
|
||||
|
||||
fileEntries, err := drv.List(context.Background(), rootEntries[0].ID)
|
||||
if err != nil {
|
||||
t.Fatalf("list subdir: %v", err)
|
||||
}
|
||||
if len(fileEntries) != 1 || fileEntries[0].Name != "sample.mp4" {
|
||||
t.Fatalf("file entries = %#v, want sample.mp4", fileEntries)
|
||||
}
|
||||
if strings.Contains(fileEntries[0].ID, "/") {
|
||||
t.Fatalf("encoded file id contains slash: %q", fileEntries[0].ID)
|
||||
}
|
||||
|
||||
link, err := drv.StreamURL(context.Background(), fileEntries[0].ID)
|
||||
if err != nil {
|
||||
t.Fatalf("stream url: %v", err)
|
||||
}
|
||||
if link.URL != videoPath {
|
||||
t.Fatalf("url = %q, want %q", link.URL, videoPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRejectsEscapingID(t *testing.T) {
|
||||
drv := New(Config{ID: "local", RootPath: t.TempDir()})
|
||||
escaped := base64.RawURLEncoding.EncodeToString([]byte("../secret.mp4"))
|
||||
|
||||
_, err := drv.StreamURL(context.Background(), escaped)
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid relative path") {
|
||||
t.Fatalf("error = %v, want invalid relative path", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitRequiresExistingDirectory(t *testing.T) {
|
||||
drv := New(Config{ID: "local", RootPath: filepath.Join(t.TempDir(), "missing")})
|
||||
|
||||
err := drv.Init(context.Background())
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "stat root") {
|
||||
t.Fatalf("error = %v, want stat root failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerPersistsLocalStorageVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(root, "collection"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir collection: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "collection", "clip.mp4"), []byte("video"), 0o644); err != nil {
|
||||
t.Fatalf("write video: %v", err)
|
||||
}
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
sc := scanner.New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
stats, err := sc.Run(ctx, drv.RootID())
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Added != 1 {
|
||||
t.Fatalf("added = %d, want 1", stats.Added)
|
||||
}
|
||||
|
||||
fileID := encodeRel("collection/clip.mp4")
|
||||
got, err := cat.GetVideo(ctx, Kind+"-local-"+fileID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DriveID != "local" || got.FileID != fileID || got.Category != "collection" {
|
||||
t.Fatalf("video = %#v, want local drive video in collection", got)
|
||||
}
|
||||
}
|
||||
@@ -3,14 +3,19 @@ package onedrive
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-resty/resty/v2"
|
||||
@@ -18,8 +23,17 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
maxSmallUploadSize = 250 * 1024 * 1024
|
||||
defaultRenewAPIURL = "https://api.oplist.org/onedrive/renewapi"
|
||||
maxSmallUploadSize = 250 * 1024 * 1024
|
||||
defaultUploadSessionChunk = 10 * 1024 * 1024
|
||||
uploadSessionRetryAttempts = 3
|
||||
defaultRenewAPIURL = "https://api.oplist.org/onedrive/renewapi"
|
||||
onedriveListCooldown = 5 * time.Minute
|
||||
onedriveListInterval = 1 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
smallUploadThreshold = int64(maxSmallUploadSize)
|
||||
uploadSessionChunk = int64(defaultUploadSessionChunk)
|
||||
)
|
||||
|
||||
type Driver struct {
|
||||
@@ -34,6 +48,11 @@ type Driver struct {
|
||||
renewAPIURL string
|
||||
client *resty.Client
|
||||
onTokenUpdate func(access, refresh string)
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
listInterval time.Duration
|
||||
listCooldown time.Duration
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -85,6 +104,8 @@ func New(c Config) *Driver {
|
||||
client: resty.New().
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
listInterval: onedriveListInterval,
|
||||
listCooldown: onedriveListCooldown,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,10 +127,16 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
|
||||
if dirID == "" {
|
||||
dirID = d.rootID
|
||||
}
|
||||
d.listMu.Lock()
|
||||
defer d.listMu.Unlock()
|
||||
|
||||
nextLink := d.childrenURL(dirID)
|
||||
first := true
|
||||
out := make([]drives.Entry, 0)
|
||||
for nextLink != "" {
|
||||
if err := d.waitForListSlotLocked(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var resp filesResp
|
||||
err := d.request(ctx, nextLink, http.MethodGet, func(req *resty.Request) {
|
||||
if first {
|
||||
@@ -120,6 +147,19 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
|
||||
}
|
||||
}, &resp)
|
||||
if err != nil {
|
||||
if wait, ok := drives.RateLimitRetryAfter(err); ok {
|
||||
if wait <= 0 {
|
||||
wait = d.listCooldown
|
||||
if wait <= 0 {
|
||||
wait = onedriveListCooldown
|
||||
}
|
||||
}
|
||||
log.Printf("[onedrive] list cooling down drive=%s dir=%s cooldown=%s err=%v", d.id, dirID, wait, err)
|
||||
if err := sleepContext(ctx, wait); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("onedrive list: %w", err)
|
||||
}
|
||||
for _, item := range resp.Value {
|
||||
@@ -131,6 +171,36 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *Driver) waitForListSlotLocked(ctx context.Context) error {
|
||||
if d.listInterval <= 0 || d.lastListAt.IsZero() {
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
next := d.lastListAt.Add(d.listInterval)
|
||||
now := time.Now()
|
||||
if now.Before(next) {
|
||||
if err := sleepContext(ctx, next.Sub(now)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, d time.Duration) error {
|
||||
if d <= 0 {
|
||||
return ctx.Err()
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
|
||||
var item graphItem
|
||||
if err := d.request(ctx, d.itemURL(fileID), http.MethodGet, nil, &item); err != nil {
|
||||
@@ -156,15 +226,49 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
res, err := d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return res.FileID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
parentID, name, err := d.normalizeUploadArgs(parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
threshold := smallUploadThreshold
|
||||
if threshold <= 0 {
|
||||
threshold = maxSmallUploadSize
|
||||
}
|
||||
if size <= threshold {
|
||||
return d.uploadSmallAndReportHash(ctx, parentID, name, r, size, threshold)
|
||||
}
|
||||
return d.uploadSessionAndReportHash(ctx, parentID, name, r, size)
|
||||
}
|
||||
|
||||
func (d *Driver) normalizeUploadArgs(parentID, name string, r io.Reader, size int64) (string, string, error) {
|
||||
if r == nil {
|
||||
return "", "", errors.New("onedrive upload: body is required")
|
||||
}
|
||||
if size < 0 {
|
||||
return "", "", fmt.Errorf("onedrive upload: invalid size %d", size)
|
||||
}
|
||||
if parentID == "" {
|
||||
parentID = d.rootID
|
||||
}
|
||||
if size > maxSmallUploadSize {
|
||||
return "", fmt.Errorf("onedrive upload: files over %d bytes require upload session", maxSmallUploadSize)
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return "", "", errors.New("onedrive upload: empty file name")
|
||||
}
|
||||
data, err := readSmallUpload(r)
|
||||
return parentID, name, nil
|
||||
}
|
||||
|
||||
func (d *Driver) uploadSmallAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size, limit int64) (UploadResult, error) {
|
||||
data, hash, actualSize, err := readSmallUpload(r, size, limit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return UploadResult{}, err
|
||||
}
|
||||
u := fmt.Sprintf("%s/items/%s:/%s:/content", d.driveBaseURL(), url.PathEscape(parentID), url.PathEscape(name))
|
||||
var item graphItem
|
||||
@@ -173,26 +277,159 @@ func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader,
|
||||
req.SetContentLength(true)
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("onedrive upload: %w", err)
|
||||
return UploadResult{}, fmt.Errorf("onedrive upload: %w", err)
|
||||
}
|
||||
if item.ID == "" {
|
||||
return "", errors.New("onedrive upload: empty item id")
|
||||
return UploadResult{}, errors.New("onedrive upload: empty item id")
|
||||
}
|
||||
return item.ID, nil
|
||||
return UploadResult{FileID: item.ID, Hash: hash, Size: actualSize}, nil
|
||||
}
|
||||
|
||||
func readSmallUpload(r io.Reader) ([]byte, error) {
|
||||
if r == nil {
|
||||
return nil, errors.New("onedrive upload: body is required")
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(r, maxSmallUploadSize+1))
|
||||
func (d *Driver) uploadSessionAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
session, err := d.createUploadSession(ctx, parentID, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("onedrive upload: read body: %w", err)
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if len(data) > maxSmallUploadSize {
|
||||
return nil, fmt.Errorf("onedrive upload: files over %d bytes require upload session", maxSmallUploadSize)
|
||||
if strings.TrimSpace(session.UploadURL) == "" {
|
||||
return UploadResult{}, errors.New("onedrive upload session: empty upload url")
|
||||
}
|
||||
return data, nil
|
||||
|
||||
chunkSize := uploadSessionChunk
|
||||
if chunkSize <= 0 {
|
||||
chunkSize = defaultUploadSessionChunk
|
||||
}
|
||||
buf := make([]byte, int(chunkSize))
|
||||
hasher := sha1.New()
|
||||
var finalItem graphItem
|
||||
var offset int64
|
||||
for offset < size {
|
||||
partSize := minInt64(chunkSize, size-offset)
|
||||
chunk := buf[:int(partSize)]
|
||||
n, err := io.ReadFull(r, chunk)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return UploadResult{}, fmt.Errorf("onedrive upload: size mismatch: declared %d, copied %d", size, offset+int64(n))
|
||||
}
|
||||
return UploadResult{}, fmt.Errorf("onedrive upload: read body: %w", err)
|
||||
}
|
||||
chunk = chunk[:n]
|
||||
_, _ = hasher.Write(chunk)
|
||||
item, err := d.putUploadSessionChunkWithRetry(ctx, session.UploadURL, offset, size, chunk)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if item != nil {
|
||||
finalItem = *item
|
||||
}
|
||||
offset += int64(n)
|
||||
}
|
||||
if finalItem.ID == "" {
|
||||
return UploadResult{}, errors.New("onedrive upload session: empty item id")
|
||||
}
|
||||
return UploadResult{
|
||||
FileID: finalItem.ID,
|
||||
Hash: hex.EncodeToString(hasher.Sum(nil)),
|
||||
Size: offset,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) createUploadSession(ctx context.Context, parentID, name string) (uploadSessionResp, error) {
|
||||
u := fmt.Sprintf("%s/items/%s:/%s:/createUploadSession", d.driveBaseURL(), url.PathEscape(parentID), url.PathEscape(name))
|
||||
body := map[string]any{
|
||||
"item": map[string]any{
|
||||
"@microsoft.graph.conflictBehavior": "rename",
|
||||
},
|
||||
}
|
||||
var out uploadSessionResp
|
||||
err := d.request(ctx, u, http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(body)
|
||||
}, &out)
|
||||
if err != nil {
|
||||
return uploadSessionResp{}, fmt.Errorf("onedrive upload session: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunkWithRetry(ctx context.Context, uploadURL string, start, total int64, data []byte) (*graphItem, error) {
|
||||
var last error
|
||||
for attempt := 0; attempt < uploadSessionRetryAttempts; attempt++ {
|
||||
if attempt > 0 {
|
||||
if err := sleepContext(ctx, time.Duration(attempt)*time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
item, retryable, err := d.putUploadSessionChunk(ctx, uploadURL, start, total, data)
|
||||
if err == nil {
|
||||
return item, nil
|
||||
}
|
||||
last = err
|
||||
if !retryable {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if last == nil {
|
||||
last = errors.New("onedrive upload session: retry attempts exhausted")
|
||||
}
|
||||
return nil, last
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunk(ctx context.Context, uploadURL string, start, total int64, data []byte) (*graphItem, bool, error) {
|
||||
end := start + int64(len(data)) - 1
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
req.ContentLength = int64(len(data))
|
||||
req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, total))
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK, http.StatusCreated:
|
||||
var item graphItem
|
||||
if err := json.NewDecoder(res.Body).Decode(&item); err != nil {
|
||||
return nil, false, fmt.Errorf("onedrive upload session: decode completed item: %w", err)
|
||||
}
|
||||
return &item, false, nil
|
||||
case http.StatusAccepted:
|
||||
return nil, false, nil
|
||||
default:
|
||||
body, _ := io.ReadAll(io.LimitReader(res.Body, 4096))
|
||||
err := fmt.Errorf("onedrive upload session: status=%d body=%s", res.StatusCode, strings.TrimSpace(string(body)))
|
||||
retryable := res.StatusCode == http.StatusTooManyRequests || (res.StatusCode >= 500 && res.StatusCode <= 504)
|
||||
return nil, retryable, err
|
||||
}
|
||||
}
|
||||
|
||||
func readSmallUpload(r io.Reader, declaredSize, limit int64) ([]byte, string, int64, error) {
|
||||
if r == nil {
|
||||
return nil, "", 0, errors.New("onedrive upload: body is required")
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = maxSmallUploadSize
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(r, limit+1))
|
||||
if err != nil {
|
||||
return nil, "", 0, fmt.Errorf("onedrive upload: read body: %w", err)
|
||||
}
|
||||
if int64(len(data)) > limit {
|
||||
return nil, "", 0, fmt.Errorf("onedrive upload: files over %d bytes require upload session", limit)
|
||||
}
|
||||
if declaredSize >= 0 && int64(len(data)) != declaredSize {
|
||||
return nil, "", 0, fmt.Errorf("onedrive upload: size mismatch: declared %d, copied %d", declaredSize, len(data))
|
||||
}
|
||||
sum := sha1.Sum(data)
|
||||
return data, hex.EncodeToString(sum[:]), int64(len(data)), nil
|
||||
}
|
||||
|
||||
func minInt64(a, b int64) int64 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
@@ -245,6 +482,25 @@ func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, er
|
||||
return item.ID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("onedrive rename: empty file id")
|
||||
}
|
||||
newName = strings.TrimSpace(newName)
|
||||
if newName == "" {
|
||||
return errors.New("onedrive rename: empty new name")
|
||||
}
|
||||
var item graphItem
|
||||
err := d.request(ctx, d.itemURL(fileID), http.MethodPatch, func(req *resty.Request) {
|
||||
req.SetBody(map[string]string{"name": newName})
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return fmt.Errorf("onedrive rename: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) request(ctx context.Context, rawURL, method string, configure func(*resty.Request), out any) error {
|
||||
return d.requestOnce(ctx, rawURL, method, configure, out, true)
|
||||
}
|
||||
@@ -265,7 +521,7 @@ func (d *Driver) requestOnce(ctx context.Context, rawURL, method string, configu
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isRateLimitResponse(res, graphErr.Error.Code) {
|
||||
if isRateLimitResponse(res, graphErr.Error.Code, graphErr.Error.Message) {
|
||||
return onedriveRateLimitError(res, graphErr.Error.Message)
|
||||
}
|
||||
if graphErr.Error.Code != "" {
|
||||
@@ -327,11 +583,54 @@ func (d *Driver) refresh(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func isRateLimitResponse(res *resty.Response, code string) bool {
|
||||
if code == "TooManyRequests" || code == "activityLimitReached" {
|
||||
func isRateLimitResponse(res *resty.Response, code, message string) bool {
|
||||
if isRateLimitCode(code) || isRateLimitMessage(message) {
|
||||
return true
|
||||
}
|
||||
return res != nil && res.StatusCode() == http.StatusTooManyRequests
|
||||
if res == nil {
|
||||
return false
|
||||
}
|
||||
if res.StatusCode() == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if res.Header().Get("Retry-After") == "" {
|
||||
return false
|
||||
}
|
||||
switch res.StatusCode() {
|
||||
case http.StatusServiceUnavailable, http.StatusGatewayTimeout:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isRateLimitCode(code string) bool {
|
||||
normalized := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(code), "_", ""))
|
||||
normalized = strings.ReplaceAll(normalized, "-", "")
|
||||
switch normalized {
|
||||
case "toomanyrequests",
|
||||
"activitylimitreached",
|
||||
"throttledrequest",
|
||||
"requestthrottled",
|
||||
"resourcethrottled",
|
||||
"applicationthrottled",
|
||||
"tenantthrottled":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isRateLimitMessage(message string) bool {
|
||||
text := strings.ToLower(strings.TrimSpace(message))
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "throttl") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "activity limit") ||
|
||||
strings.Contains(text, "temporarily blocked")
|
||||
}
|
||||
|
||||
func onedriveRateLimitError(res *resty.Response, message string) error {
|
||||
|
||||
@@ -2,6 +2,8 @@ package onedrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
@@ -199,7 +201,7 @@ func TestGraph429ReturnsRateLimitErrorWithRetryAfter(t *testing.T) {
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
|
||||
_, err := d.List(context.Background(), "root")
|
||||
_, err := d.StreamURL(context.Background(), "file-id")
|
||||
if err == nil {
|
||||
t.Fatal("list succeeded, want rate limit error")
|
||||
}
|
||||
@@ -212,6 +214,92 @@ func TestGraph429ReturnsRateLimitErrorWithRetryAfter(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphThrottleMessageReturnsRateLimitError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{
|
||||
"error": map[string]any{
|
||||
"code": "generalException",
|
||||
"message": "The request has been throttled. Please try again later.",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("write json: %v", err)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
|
||||
_, err := d.StreamURL(context.Background(), "file-id")
|
||||
if err == nil {
|
||||
t.Fatal("list succeeded, want rate limit error")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListCoolsDownAndRetriesOneDriveRateLimit(t *testing.T) {
|
||||
var calls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1.0/me/drive/items/root/children" {
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
|
||||
}
|
||||
calls++
|
||||
if calls == 1 {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{
|
||||
"error": map[string]any{
|
||||
"code": "TooManyRequests",
|
||||
"message": "throttled",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("write json: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
writeJSON(t, w, map[string]any{
|
||||
"value": []map[string]any{
|
||||
{
|
||||
"id": "file-id",
|
||||
"name": "demo.mp4",
|
||||
"size": 100,
|
||||
"file": map[string]any{"mimeType": "video/mp4"},
|
||||
},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
d.listInterval = 0
|
||||
d.listCooldown = time.Millisecond
|
||||
|
||||
got, err := d.List(context.Background(), "root")
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("calls = %d, want retry after rate limit", calls)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID != "file-id" {
|
||||
t.Fatalf("entries = %#v, want retried file", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatAndStreamURLUseDriveItemMetadata(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
@@ -320,6 +408,36 @@ func TestEnsureDirCreatesMissingFolders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenamePatchesDriveItemName(t *testing.T) {
|
||||
var body map[string]string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPatch || r.URL.EscapedPath() != "/v1.0/me/drive/items/file-id" {
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
|
||||
}
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
t.Fatalf("authorization = %q, want bearer token", got)
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode body: %v", err)
|
||||
}
|
||||
writeJSON(t, w, map[string]any{"id": "file-id", "name": "new name.mp4"})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
if err := d.Rename(context.Background(), "file-id", "new name.mp4"); err != nil {
|
||||
t.Fatalf("rename: %v", err)
|
||||
}
|
||||
if body["name"] != "new name.mp4" {
|
||||
t.Fatalf("rename body = %#v, want new name", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadSmallFileReturnsNewItemID(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
@@ -358,6 +476,86 @@ func TestUploadSmallFileReturnsNewItemID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadLargeFileUsesUploadSessionAndReportsHash(t *testing.T) {
|
||||
oldThreshold := smallUploadThreshold
|
||||
oldChunk := uploadSessionChunk
|
||||
smallUploadThreshold = 8
|
||||
uploadSessionChunk = 4
|
||||
t.Cleanup(func() {
|
||||
smallUploadThreshold = oldThreshold
|
||||
uploadSessionChunk = oldChunk
|
||||
})
|
||||
|
||||
body := "0123456789abc"
|
||||
var ranges []string
|
||||
var chunks []string
|
||||
var createdSession bool
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost && r.URL.EscapedPath() == "/v1.0/me/drive/items/parent-id:/big.mp4:/createUploadSession":
|
||||
createdSession = true
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
t.Fatalf("authorization = %q, want bearer token", got)
|
||||
}
|
||||
writeJSON(t, w, map[string]any{"uploadUrl": srv.URL + "/upload-session"})
|
||||
case r.Method == http.MethodPut && r.URL.Path == "/upload-session":
|
||||
ranges = append(ranges, r.Header.Get("Content-Range"))
|
||||
data, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read chunk: %v", err)
|
||||
}
|
||||
chunks = append(chunks, string(data))
|
||||
if len(ranges) < 4 {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
if _, err := w.Write([]byte(`{"nextExpectedRanges":["0-"]}`)); err != nil {
|
||||
t.Fatalf("write accepted: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{"id": "uploaded-big-id"}); err != nil {
|
||||
t.Fatalf("write final item: %v", err)
|
||||
}
|
||||
default:
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
got, err := d.UploadAndReportHash(context.Background(), "parent-id", "big.mp4", strings.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
if !createdSession {
|
||||
t.Fatal("createUploadSession was not called")
|
||||
}
|
||||
wantRanges := []string{
|
||||
"bytes 0-3/13",
|
||||
"bytes 4-7/13",
|
||||
"bytes 8-11/13",
|
||||
"bytes 12-12/13",
|
||||
}
|
||||
if strings.Join(ranges, "|") != strings.Join(wantRanges, "|") {
|
||||
t.Fatalf("ranges = %#v, want %#v", ranges, wantRanges)
|
||||
}
|
||||
if strings.Join(chunks, "") != body {
|
||||
t.Fatalf("uploaded chunks = %q, want %q", strings.Join(chunks, ""), body)
|
||||
}
|
||||
sum := sha1.Sum([]byte(body))
|
||||
if got.FileID != "uploaded-big-id" || got.Size != int64(len(body)) || got.Hash != hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("upload result = %#v, want file id/hash/size for body", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadRefreshesExpiredTokenAndReplaysBody(t *testing.T) {
|
||||
var uploadAttempts int
|
||||
var tokenRefreshes int
|
||||
|
||||
@@ -82,3 +82,13 @@ type filesResp struct {
|
||||
Value []graphItem `json:"value"`
|
||||
NextLink string `json:"@odata.nextLink"`
|
||||
}
|
||||
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
Hash string
|
||||
Size int64
|
||||
}
|
||||
|
||||
type uploadSessionResp struct {
|
||||
UploadURL string `json:"uploadUrl"`
|
||||
}
|
||||
|
||||
@@ -355,7 +355,60 @@ func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
currentID := d.rootID
|
||||
for _, name := range splitPath(pathFromRoot) {
|
||||
childID, err := d.findChildDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if childID == "" {
|
||||
childID, err = d.makeDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
currentID = childID
|
||||
}
|
||||
return currentID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) findChildDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
entries, err := d.List(ctx, parentID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir && e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
var out file
|
||||
err := d.request(ctx, filesURL, http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(map[string]any{
|
||||
"kind": "drive#folder",
|
||||
"parent_id": parentID,
|
||||
"name": name,
|
||||
})
|
||||
}, &out)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("pikpak mkdir %s: %w", name, err)
|
||||
}
|
||||
if out.ID == "" {
|
||||
return "", fmt.Errorf("pikpak mkdir %s: empty folder id", name)
|
||||
}
|
||||
return out.ID, nil
|
||||
}
|
||||
|
||||
func splitPath(p string) []string {
|
||||
p = strings.Trim(p, "/")
|
||||
if p == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(p, "/")
|
||||
}
|
||||
|
||||
func (d *Driver) getFiles(ctx context.Context, parentID string) ([]file, error) {
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package pikpak
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestNewDefaults(t *testing.T) {
|
||||
@@ -95,11 +97,85 @@ func TestFolderToEntry(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDirStillUnsupported(t *testing.T) {
|
||||
d := New(Config{ID: "pikpak-main"})
|
||||
func TestEnsureDirReusesExistingFolder(t *testing.T) {
|
||||
var postCalled bool
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
if got := r.URL.Query().Get("parent_id"); got != "root-id" {
|
||||
t.Fatalf("parent_id = %q, want root-id", got)
|
||||
}
|
||||
writePikPakJSON(t, w, map[string]any{
|
||||
"files": []map[string]any{{
|
||||
"id": "existing-folder-id",
|
||||
"kind": "drive#folder",
|
||||
"name": "91 Spider",
|
||||
}},
|
||||
})
|
||||
case http.MethodPost:
|
||||
postCalled = true
|
||||
t.Fatalf("existing folder should not be created again")
|
||||
default:
|
||||
t.Fatalf("unexpected method %s", r.Method)
|
||||
}
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
if _, err := d.EnsureDir(nil, "/previews"); err != drives.ErrNotSupported {
|
||||
t.Fatalf("EnsureDir error = %v, want ErrNotSupported", err)
|
||||
d := newTestDriver(t, srv)
|
||||
got, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure dir: %v", err)
|
||||
}
|
||||
if got != "existing-folder-id" {
|
||||
t.Fatalf("dir id = %q, want existing-folder-id", got)
|
||||
}
|
||||
if postCalled {
|
||||
t.Fatal("POST should not be called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDirCreatesMissingFolder(t *testing.T) {
|
||||
var got uploadRequestBody
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
writePikPakJSON(t, w, map[string]any{"files": []map[string]any{}})
|
||||
case http.MethodPost:
|
||||
if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode create folder body: %v", err)
|
||||
}
|
||||
writePikPakJSON(t, w, map[string]any{
|
||||
"id": "new-folder-id",
|
||||
"kind": "drive#folder",
|
||||
"name": "91 Spider",
|
||||
})
|
||||
default:
|
||||
t.Fatalf("unexpected method %s", r.Method)
|
||||
}
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
d := newTestDriver(t, srv)
|
||||
id, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure dir: %v", err)
|
||||
}
|
||||
if id != "new-folder-id" {
|
||||
t.Fatalf("dir id = %q, want new-folder-id", id)
|
||||
}
|
||||
if got.Kind != "drive#folder" || got.ParentID != "root-id" || got.Name != "91 Spider" {
|
||||
t.Fatalf("create folder body = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func writePikPakJSON(t *testing.T, w http.ResponseWriter, body any) {
|
||||
t.Helper()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(body); err != nil {
|
||||
t.Fatalf("write json: %v", err)
|
||||
}
|
||||
// Upload 的真实实现见 upload_test.go。
|
||||
}
|
||||
|
||||
@@ -463,8 +463,7 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
// 网站封面下载失败的视频:spider91 drive 的 thumb worker 按设计不
|
||||
// 处理 spider91 视频(封面应是网站原图直接保存),所以没人接手。
|
||||
// 显式标 'failed' 让 CountVideosNeedingThumbnail 排除(条件 status
|
||||
// != 'failed'),否则 enqueueDriveGeneration → waitForThumbnailsBeforePreview
|
||||
// 会因为 count > 0 把 teaser 入队永远卡在等待循环里。
|
||||
// != 'failed'),避免后续封面补队列一直重复捞到这条视频。
|
||||
_ = c.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
ThumbnailStatus: "failed",
|
||||
})
|
||||
|
||||
@@ -234,13 +234,13 @@ func TestCrawlerRunOnceMissingScript(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestCrawlerThumbDownloadFailureMarksStatusFailed 验证:网站封面下载失败时
|
||||
// crawler 把 thumbnail_status 显式标 'failed',避免 enqueueDriveGeneration 的
|
||||
// waitForThumbnailsBeforePreview 因为 count > 0 把 teaser 卡死等待。
|
||||
// crawler 把 thumbnail_status 显式标 'failed',避免后续封面补队列一直重复
|
||||
// 捞到这条 spider91 视频。
|
||||
//
|
||||
// 历史 bug:之前 thumb 下载失败仅打 log,url=”, status 走 schema DEFAULT 'pending'。
|
||||
// CountVideosNeedingThumbnail 条件是 url=” AND status != 'failed' → count=1。
|
||||
// spider91 drive 的 thumb worker 按设计不处理 spider91 视频 → 没人会改 status。
|
||||
// 结果 teaser 永远卡在 [preview] waiting for 1 thumbnails before teaser generation。
|
||||
// spider91 drive 的 thumb worker 按设计不处理 spider91 视频 → 没人会改 status,
|
||||
// 后续补队列会一直认为它还缺封面。
|
||||
func TestCrawlerThumbDownloadFailureMarksStatusFailed(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell-based fake script only on unix")
|
||||
@@ -317,8 +317,7 @@ func TestCrawlerThumbDownloadFailureMarksStatusFailed(t *testing.T) {
|
||||
|
||||
// 关键断言:CountVideosNeedingThumbnail 应该返回 0。
|
||||
// 该函数的 SQL 条件是 `url = '' AND status != 'failed'`;如果 crawler 没把
|
||||
// status 标 'failed'(schema DEFAULT 'pending'),count 就会是 1,外层
|
||||
// waitForThumbnailsBeforePreview 会因为 count > 0 把 teaser 卡死等待。
|
||||
// status 标 'failed'(schema DEFAULT 'pending'),count 就会是 1。
|
||||
count, err := cat.CountVideosNeedingThumbnail(context.Background(), driveID)
|
||||
if err != nil {
|
||||
t.Fatalf("count: %v", err)
|
||||
|
||||
@@ -0,0 +1,468 @@
|
||||
package fingerprint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultSampleSizeBytes int64 = 512 * 1024
|
||||
defaultFullHashMaxSize int64 = 8 * 1024 * 1024
|
||||
defaultCooldown = 5 * time.Minute
|
||||
defaultWorkerQueueSize = 10000
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
SampleSizeBytes int64
|
||||
FullHashMaxSize int64
|
||||
RateLimitCooldown time.Duration
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
type Worker struct {
|
||||
Catalog *catalog.Catalog
|
||||
Drive drives.Drive
|
||||
Config Config
|
||||
|
||||
ch chan *catalog.Video
|
||||
queue videoQueue
|
||||
activity taskActivity
|
||||
cooldown cooldownState
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
type TaskStatus struct {
|
||||
State string
|
||||
CurrentTitle string
|
||||
QueueLength int
|
||||
CooldownUntil time.Time
|
||||
}
|
||||
|
||||
func NewWorker(cat *catalog.Catalog, drv drives.Drive, cfg Config) *Worker {
|
||||
hc := cfg.HTTPClient
|
||||
if hc == nil {
|
||||
hc = &http.Client{Timeout: 0}
|
||||
}
|
||||
if cfg.SampleSizeBytes <= 0 {
|
||||
cfg.SampleSizeBytes = defaultSampleSizeBytes
|
||||
}
|
||||
if cfg.FullHashMaxSize <= 0 {
|
||||
cfg.FullHashMaxSize = defaultFullHashMaxSize
|
||||
}
|
||||
if cfg.RateLimitCooldown <= 0 {
|
||||
cfg.RateLimitCooldown = defaultCooldown
|
||||
}
|
||||
return &Worker{
|
||||
Catalog: cat,
|
||||
Drive: drv,
|
||||
Config: cfg,
|
||||
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
|
||||
http: hc,
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) Enqueue(v *catalog.Video) bool {
|
||||
if v == nil {
|
||||
return false
|
||||
}
|
||||
if !w.queue.reserve(v.ID) {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case w.ch <- v:
|
||||
return true
|
||||
default:
|
||||
w.queue.release(v.ID)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) EnqueueBlocking(ctx context.Context, v *catalog.Video) bool {
|
||||
if v == nil {
|
||||
return false
|
||||
}
|
||||
if !w.queue.reserve(v.ID) {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case w.ch <- v:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
w.queue.release(v.ID)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) Run(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case v := <-w.ch:
|
||||
w.processQueued(ctx, v)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) Status() TaskStatus {
|
||||
if w == nil {
|
||||
return TaskStatus{State: "idle"}
|
||||
}
|
||||
currentID, currentTitle := w.activity.current()
|
||||
status := TaskStatus{
|
||||
State: "idle",
|
||||
CurrentTitle: currentTitle,
|
||||
QueueLength: w.queue.lengthExcluding(currentID),
|
||||
}
|
||||
if until, ok := w.cooldown.active(time.Now()); ok {
|
||||
status.State = "cooling"
|
||||
status.CooldownUntil = until
|
||||
return status
|
||||
}
|
||||
if currentID != "" {
|
||||
status.State = "generating"
|
||||
return status
|
||||
}
|
||||
if status.QueueLength > 0 {
|
||||
status.State = "queued"
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
defer w.queue.release(v.ID)
|
||||
if w.Catalog == nil || w.Drive == nil || v == nil || v.ID == "" {
|
||||
return
|
||||
}
|
||||
current, err := w.Catalog.GetVideo(ctx, v.ID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if current.SampledSHA256 != "" || current.FingerprintStatus == "ready" || current.Hidden {
|
||||
return
|
||||
}
|
||||
w.activity.start(current)
|
||||
defer w.activity.done()
|
||||
sum, err := Compute(ctx, w.Drive, current, w.Config, w.http)
|
||||
if err != nil {
|
||||
var rl *drives.RateLimitError
|
||||
if errors.As(err, &rl) {
|
||||
wait := rl.RetryAfter
|
||||
if wait <= 0 {
|
||||
wait = w.Config.RateLimitCooldown
|
||||
}
|
||||
until := time.Now().Add(wait)
|
||||
w.cooldown.set(until)
|
||||
log.Printf("[fingerprint] drive=%s rate limited; keep video=%s pending and cool down for %s: %v", w.Drive.ID(), current.ID, wait, err)
|
||||
sleepContext(ctx, wait)
|
||||
w.cooldown.clear(until)
|
||||
return
|
||||
}
|
||||
log.Printf("[fingerprint] video=%s failed: %v", current.ID, err)
|
||||
_ = w.Catalog.UpdateVideoFingerprint(ctx, current.ID, "", "failed", err.Error())
|
||||
return
|
||||
}
|
||||
if err := w.Catalog.UpdateVideoFingerprint(ctx, current.ID, sum, "ready", ""); err != nil {
|
||||
log.Printf("[fingerprint] update video=%s: %v", current.ID, err)
|
||||
return
|
||||
}
|
||||
log.Printf("[fingerprint] video=%s ready sampled_sha256=%s", current.ID, sum)
|
||||
}
|
||||
|
||||
func Compute(ctx context.Context, drv drives.Drive, v *catalog.Video, cfg Config, hc *http.Client) (string, error) {
|
||||
if drv == nil {
|
||||
return "", errors.New("fingerprint: nil drive")
|
||||
}
|
||||
if v == nil {
|
||||
return "", errors.New("fingerprint: nil video")
|
||||
}
|
||||
if v.Size <= 0 {
|
||||
return "", errors.New("fingerprint: video size is empty")
|
||||
}
|
||||
if cfg.SampleSizeBytes <= 0 {
|
||||
cfg.SampleSizeBytes = defaultSampleSizeBytes
|
||||
}
|
||||
if cfg.FullHashMaxSize <= 0 {
|
||||
cfg.FullHashMaxSize = defaultFullHashMaxSize
|
||||
}
|
||||
if hc == nil {
|
||||
hc = &http.Client{Timeout: 0}
|
||||
}
|
||||
link, err := drv.StreamURL(ctx, v.FileID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fingerprint: stream url: %w", err)
|
||||
}
|
||||
if link == nil || strings.TrimSpace(link.URL) == "" {
|
||||
return "", errors.New("fingerprint: empty stream url")
|
||||
}
|
||||
ranges := sampleRanges(v.Size, cfg.SampleSizeBytes, cfg.FullHashMaxSize)
|
||||
h := sha256.New()
|
||||
writeHashHeader(h, v.Size, ranges)
|
||||
for _, r := range ranges {
|
||||
data, err := readRange(ctx, hc, link, r)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if int64(len(data)) != r.length {
|
||||
return "", fmt.Errorf("fingerprint: short sample at %d: got %d want %d", r.start, len(data), r.length)
|
||||
}
|
||||
_, _ = h.Write([]byte(fmt.Sprintf("offset=%d length=%d\n", r.start, r.length)))
|
||||
_, _ = h.Write(data)
|
||||
_, _ = h.Write([]byte("\n"))
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
type byteRange struct {
|
||||
start int64
|
||||
length int64
|
||||
}
|
||||
|
||||
func sampleRanges(size, sampleSize, fullHashMax int64) []byteRange {
|
||||
if size <= fullHashMax {
|
||||
return []byteRange{{start: 0, length: size}}
|
||||
}
|
||||
if sampleSize > size {
|
||||
sampleSize = size
|
||||
}
|
||||
maxStart := size - sampleSize
|
||||
percents := []int64{0, 20, 40, 60, 80}
|
||||
out := make([]byteRange, 0, len(percents))
|
||||
seen := make(map[int64]struct{}, len(percents))
|
||||
for _, pct := range percents {
|
||||
start := maxStart * pct / 100
|
||||
if _, ok := seen[start]; ok {
|
||||
continue
|
||||
}
|
||||
seen[start] = struct{}{}
|
||||
out = append(out, byteRange{start: start, length: sampleSize})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func writeHashHeader(w io.Writer, size int64, ranges []byteRange) {
|
||||
_, _ = fmt.Fprintf(w, "video-site-sampled-sha256-v1\nsize=%d\nsamples=%d\n", size, len(ranges))
|
||||
}
|
||||
|
||||
func readRange(ctx context.Context, hc *http.Client, link *drives.StreamLink, r byteRange) ([]byte, error) {
|
||||
u, err := url.Parse(link.URL)
|
||||
if err == nil && (u.Scheme == "http" || u.Scheme == "https") {
|
||||
return readHTTPRange(ctx, hc, link, r)
|
||||
}
|
||||
path := link.URL
|
||||
if err == nil && u.Scheme == "file" {
|
||||
path = u.Path
|
||||
}
|
||||
return readLocalRange(path, r)
|
||||
}
|
||||
|
||||
func readLocalRange(path string, r byteRange) ([]byte, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fingerprint: open local stream: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
buf := make([]byte, r.length)
|
||||
n, err := f.ReadAt(buf, r.start)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, fmt.Errorf("fingerprint: read local sample: %w", err)
|
||||
}
|
||||
if int64(n) != r.length {
|
||||
return nil, fmt.Errorf("fingerprint: read local sample at %d: got %d want %d", r.start, n, r.length)
|
||||
}
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
func readHTTPRange(ctx context.Context, hc *http.Client, link *drives.StreamLink, r byteRange) ([]byte, error) {
|
||||
end := r.start + r.length - 1
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, link.URL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for k, vs := range link.Headers {
|
||||
for _, v := range vs {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", r.start, end))
|
||||
resp, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fingerprint: read remote sample: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
return nil, &drives.RateLimitError{
|
||||
Provider: "fingerprint",
|
||||
RetryAfter: parseRetryAfter(resp.Header.Get("Retry-After")),
|
||||
Err: fmt.Errorf("remote sample rate limited: status=%d", resp.StatusCode),
|
||||
}
|
||||
}
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
if resp.StatusCode == http.StatusOK && r.start == 0 {
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, r.length+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) == r.length {
|
||||
return data, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("fingerprint: range request got status=%d for bytes=%d-%d", resp.StatusCode, r.start, end)
|
||||
}
|
||||
return io.ReadAll(io.LimitReader(resp.Body, r.length))
|
||||
}
|
||||
|
||||
func parseRetryAfter(raw string) time.Duration {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return 0
|
||||
}
|
||||
if seconds, err := strconv.Atoi(raw); err == nil && seconds > 0 {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
if when, err := http.ParseTime(raw); err == nil {
|
||||
d := time.Until(when)
|
||||
if d > 0 {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, d time.Duration) bool {
|
||||
if d <= 0 {
|
||||
return true
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-timer.C:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
type taskActivity struct {
|
||||
mu sync.Mutex
|
||||
currentID string
|
||||
currentTitle string
|
||||
}
|
||||
|
||||
func (a *taskActivity) start(v *catalog.Video) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if v == nil {
|
||||
a.currentID = ""
|
||||
a.currentTitle = ""
|
||||
return
|
||||
}
|
||||
a.currentID = v.ID
|
||||
a.currentTitle = v.Title
|
||||
}
|
||||
|
||||
func (a *taskActivity) done() {
|
||||
a.mu.Lock()
|
||||
a.currentID = ""
|
||||
a.currentTitle = ""
|
||||
a.mu.Unlock()
|
||||
}
|
||||
|
||||
func (a *taskActivity) current() (string, string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.currentID, a.currentTitle
|
||||
}
|
||||
|
||||
type cooldownState struct {
|
||||
mu sync.Mutex
|
||||
until time.Time
|
||||
}
|
||||
|
||||
func (s *cooldownState) set(until time.Time) {
|
||||
s.mu.Lock()
|
||||
s.until = until
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *cooldownState) clear(until time.Time) {
|
||||
s.mu.Lock()
|
||||
if s.until.Equal(until) {
|
||||
s.until = time.Time{}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *cooldownState) active(now time.Time) (time.Time, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.until.IsZero() || !s.until.After(now) {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return s.until, true
|
||||
}
|
||||
|
||||
type videoQueue struct {
|
||||
mu sync.Mutex
|
||||
ids map[string]struct{}
|
||||
}
|
||||
|
||||
func (q *videoQueue) reserve(id string) bool {
|
||||
if id == "" {
|
||||
return true
|
||||
}
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
if q.ids == nil {
|
||||
q.ids = make(map[string]struct{})
|
||||
}
|
||||
if _, ok := q.ids[id]; ok {
|
||||
return false
|
||||
}
|
||||
q.ids[id] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
func (q *videoQueue) release(id string) {
|
||||
if id == "" {
|
||||
return
|
||||
}
|
||||
q.mu.Lock()
|
||||
delete(q.ids, id)
|
||||
q.mu.Unlock()
|
||||
}
|
||||
|
||||
func (q *videoQueue) lengthExcluding(currentID string) int {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
n := len(q.ids)
|
||||
if currentID != "" {
|
||||
if _, ok := q.ids[currentID]; ok {
|
||||
n--
|
||||
}
|
||||
}
|
||||
if n < 0 {
|
||||
return 0
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package fingerprint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestComputeLocalFilesWithSameContentMatch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
body := []byte("same video bytes")
|
||||
a := filepath.Join(dir, "a.mp4")
|
||||
b := filepath.Join(dir, "b.mp4")
|
||||
if err := os.WriteFile(a, body, 0o644); err != nil {
|
||||
t.Fatalf("write a: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(b, body, 0o644); err != nil {
|
||||
t.Fatalf("write b: %v", err)
|
||||
}
|
||||
|
||||
sumA, err := Compute(ctx, &fakeDrive{paths: map[string]string{"a": a}}, &catalog.Video{ID: "a", FileID: "a", Size: int64(len(body))}, Config{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("compute a: %v", err)
|
||||
}
|
||||
sumB, err := Compute(ctx, &fakeDrive{paths: map[string]string{"b": b}}, &catalog.Video{ID: "b", FileID: "b", Size: int64(len(body))}, Config{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("compute b: %v", err)
|
||||
}
|
||||
if sumA == "" || sumA != sumB {
|
||||
t.Fatalf("fingerprints = %q / %q, want same non-empty", sumA, sumB)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeRemoteUsesRangeSamples(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
data := make([]byte, 10*1024*1024)
|
||||
for i := range data {
|
||||
data[i] = byte(i % 251)
|
||||
}
|
||||
var ranges []string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
rawRange := r.Header.Get("Range")
|
||||
ranges = append(ranges, rawRange)
|
||||
var start, end int
|
||||
if _, err := fmt.Sscanf(rawRange, "bytes=%d-%d", &start, &end); err != nil {
|
||||
t.Fatalf("bad range %q: %v", rawRange, err)
|
||||
}
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write(data[start : end+1])
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
drv := &fakeDrive{paths: map[string]string{"remote": srv.URL + "/video.mp4"}}
|
||||
sum, err := Compute(ctx, drv, &catalog.Video{ID: "remote", FileID: "remote", Size: int64(len(data))}, Config{
|
||||
SampleSizeBytes: 4,
|
||||
FullHashMaxSize: 8,
|
||||
HTTPClient: srv.Client(),
|
||||
}, srv.Client())
|
||||
if err != nil {
|
||||
t.Fatalf("compute remote: %v", err)
|
||||
}
|
||||
if sum == "" {
|
||||
t.Fatal("fingerprint should not be empty")
|
||||
}
|
||||
want := []string{
|
||||
"bytes=0-3",
|
||||
"bytes=2097151-2097154",
|
||||
"bytes=4194302-4194305",
|
||||
"bytes=6291453-6291456",
|
||||
"bytes=8388604-8388607",
|
||||
}
|
||||
if fmt.Sprint(ranges) != fmt.Sprint(want) {
|
||||
t.Fatalf("ranges = %#v, want %#v", ranges, want)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeDrive struct {
|
||||
paths map[string]string
|
||||
}
|
||||
|
||||
func (d *fakeDrive) Kind() string { return "fake" }
|
||||
func (d *fakeDrive) ID() string { return "fake" }
|
||||
func (d *fakeDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *fakeDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeDrive) StreamURL(_ context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
return &drives.StreamLink{URL: d.paths[fileID], Expires: time.Now().Add(time.Minute)}, nil
|
||||
}
|
||||
func (d *fakeDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeDrive) RootID() string { return "root" }
|
||||
@@ -12,6 +12,8 @@
|
||||
// wait until teaser queues are idle
|
||||
// Phase 3: spider91 → cloud migration (single sweep, captcha cooldown still
|
||||
// honored within this call)
|
||||
// Phase 4: cleanup duplicate local preview/thumbnail assets after sampled
|
||||
// fingerprints have identified canonical videos
|
||||
//
|
||||
// A 6h soft deadline guards each pipeline run; phases check deadline at their
|
||||
// boundaries and exit cleanly if exceeded (no in-flight ffmpeg / upload is
|
||||
@@ -85,6 +87,11 @@ type Config struct {
|
||||
// RunMigration runs spider91migrate.Migrator.RunOnce for Phase 3.
|
||||
RunMigration func(ctx context.Context) error
|
||||
|
||||
// RunDedupeAssetCleanup removes generated local assets from non-canonical
|
||||
// videos in size+sampled_sha256 duplicate groups. It must not delete cloud
|
||||
// files or catalog rows.
|
||||
RunDedupeAssetCleanup func(ctx context.Context) error
|
||||
|
||||
// Now is injected for tests; nil → time.Now.
|
||||
Now func() time.Time
|
||||
}
|
||||
@@ -241,6 +248,7 @@ func (r *Runner) runPipeline(ctx context.Context) {
|
||||
}
|
||||
if len(spiderIDs) == 0 {
|
||||
log.Printf("[nightly] phase 2/3 skipped: no spider91 drive configured")
|
||||
r.runDedupeAssetCleanupPhase(ctx)
|
||||
return
|
||||
}
|
||||
log.Printf("[nightly] phase 2: crawling %d spider91 drive(s)", len(spiderIDs))
|
||||
@@ -267,6 +275,8 @@ func (r *Runner) runPipeline(ctx context.Context) {
|
||||
log.Printf("[nightly] phase 3 migration: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
r.runDedupeAssetCleanupPhase(ctx)
|
||||
}
|
||||
|
||||
// checkDeadline returns true when ctx is already done (runner shutting down or
|
||||
@@ -292,6 +302,19 @@ func (r *Runner) waitIdle(ctx context.Context, phase string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runner) runDedupeAssetCleanupPhase(ctx context.Context) {
|
||||
if r.checkDeadline(ctx, "phase 4") {
|
||||
return
|
||||
}
|
||||
if r.cfg.RunDedupeAssetCleanup == nil {
|
||||
return
|
||||
}
|
||||
log.Printf("[nightly] phase 4: duplicate asset cleanup")
|
||||
if err := r.cfg.RunDedupeAssetCleanup(ctx); err != nil {
|
||||
log.Printf("[nightly] phase 4 duplicate asset cleanup: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// readLastRunDate reads the persisted last_run_date or returns "" when unset.
|
||||
func (r *Runner) readLastRunDate(ctx context.Context) (string, error) {
|
||||
if r.cfg.Settings == nil {
|
||||
|
||||
@@ -114,6 +114,10 @@ func TestRunPipelineHonoursPhaseOrder(t *testing.T) {
|
||||
rec.push("migrate")
|
||||
return nil
|
||||
},
|
||||
RunDedupeAssetCleanup: func(context.Context) error {
|
||||
rec.push("dedupe-cleanup")
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
r.runPipeline(context.Background())
|
||||
@@ -128,6 +132,7 @@ func TestRunPipelineHonoursPhaseOrder(t *testing.T) {
|
||||
"crawl:sp-1",
|
||||
"wait-idle", // after phase 2
|
||||
"migrate",
|
||||
"dedupe-cleanup",
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("call sequence len = %d, want %d; got=%v", len(got), len(want), got)
|
||||
@@ -156,6 +161,10 @@ func TestRunPipelineSkipsMigrationWhenNoSpider91(t *testing.T) {
|
||||
rec.push("migrate")
|
||||
return nil
|
||||
},
|
||||
RunDedupeAssetCleanup: func(context.Context) error {
|
||||
rec.push("dedupe-cleanup")
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
r.runPipeline(context.Background())
|
||||
@@ -165,6 +174,15 @@ func TestRunPipelineSkipsMigrationWhenNoSpider91(t *testing.T) {
|
||||
t.Fatalf("phase 2/3 should be skipped when no spider91 drive, got call %q", c)
|
||||
}
|
||||
}
|
||||
foundCleanup := false
|
||||
for _, c := range rec.snapshot() {
|
||||
if c == "dedupe-cleanup" {
|
||||
foundCleanup = true
|
||||
}
|
||||
}
|
||||
if !foundCleanup {
|
||||
t.Fatalf("dedupe cleanup should still run when spider91 is absent; calls=%v", rec.snapshot())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
|
||||
@@ -186,6 +204,7 @@ func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
|
||||
RunSpider91Crawl: func(context.Context, string) { rec.push("crawl") },
|
||||
WaitPreviewQueuesIdle: func(context.Context) error { rec.push("wait-idle"); return nil },
|
||||
RunMigration: func(context.Context) error { rec.push("migrate"); return nil },
|
||||
RunDedupeAssetCleanup: func(context.Context) error { rec.push("dedupe-cleanup"); return nil },
|
||||
})
|
||||
|
||||
r.runPipeline(ctx)
|
||||
@@ -200,6 +219,9 @@ func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
|
||||
if c == "crawl" || c == "migrate" {
|
||||
t.Fatalf("subsequent phase should not run after cancel, got call %q", c)
|
||||
}
|
||||
if c == "dedupe-cleanup" {
|
||||
t.Fatalf("dedupe cleanup should not run after cancel, got call %q", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -929,6 +929,7 @@ func ffmpegOutputLooksRateLimited(output []byte) bool {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "throttl") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "rate-limit") ||
|
||||
strings.Contains(text, "server returned 429")
|
||||
@@ -974,7 +975,6 @@ type Worker struct {
|
||||
queue videoQueue
|
||||
|
||||
RateLimitCooldown time.Duration
|
||||
BeforeTask func(context.Context) bool
|
||||
rateLimit rateLimitState
|
||||
activity taskActivity
|
||||
}
|
||||
@@ -984,7 +984,7 @@ func NewWorker(gen TeaserGenerator, cat *catalog.Catalog, drv drives.Drive) *Wor
|
||||
Gen: gen,
|
||||
Catalog: cat,
|
||||
Drive: drv,
|
||||
ch: make(chan *catalog.Video, 4096),
|
||||
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1035,6 +1035,7 @@ type ThumbWorker struct {
|
||||
const (
|
||||
defaultTransientMediaCooldown = 5 * time.Minute
|
||||
defaultGenerationRateLimitCooldown = 5 * time.Minute
|
||||
defaultWorkerQueueSize = 10000
|
||||
maxPreviewTeaserSizeBytes int64 = 5 * 1024 * 1024 * 1024
|
||||
previewStatusSkipped = "skipped"
|
||||
)
|
||||
@@ -1174,7 +1175,7 @@ func NewThumbWorker(gen ThumbnailGenerator, cat *catalog.Catalog, drv drives.Dri
|
||||
Gen: gen,
|
||||
Catalog: cat,
|
||||
Drive: drv,
|
||||
ch: make(chan *catalog.Video, 4096),
|
||||
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1329,10 +1330,6 @@ func (w *ThumbWorker) Run(ctx context.Context) {
|
||||
|
||||
func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
defer w.queue.release(v)
|
||||
if w.BeforeTask != nil && !w.BeforeTask(ctx) {
|
||||
return
|
||||
}
|
||||
|
||||
w.activity.start(v)
|
||||
defer w.activity.done()
|
||||
if !waitForRateLimitCooldown(ctx, &w.rateLimit, "preview", w.Drive) {
|
||||
@@ -1488,29 +1485,53 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
|
||||
if w.skipIfRateLimited(v) {
|
||||
return
|
||||
}
|
||||
if current, err := w.Catalog.GetVideo(ctx, v.ID); err == nil {
|
||||
if current.ThumbnailURL != "" {
|
||||
queued := v
|
||||
current := v
|
||||
if loaded, err := w.Catalog.GetVideo(ctx, v.ID); err == nil {
|
||||
if loaded.PreviewLocal == "" {
|
||||
loaded.PreviewLocal = queued.PreviewLocal
|
||||
}
|
||||
current = loaded
|
||||
v = loaded
|
||||
if loaded.ThumbnailURL != "" && loaded.DurationSeconds > 0 {
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
|
||||
return
|
||||
}
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "pending"})
|
||||
link, err := w.Drive.StreamURL(ctx, v.FileID)
|
||||
if err != nil {
|
||||
if localLink, ok := localPreviewLink(v); ok {
|
||||
link = localLink
|
||||
} else {
|
||||
if w.pauseForRecoverableError(err, "streamURL", v.Title) {
|
||||
if current.ThumbnailURL != "" {
|
||||
if current.DurationSeconds <= 0 {
|
||||
link, err := w.streamLink(ctx, current)
|
||||
if err != nil {
|
||||
if w.pauseForRecoverableError(err, "streamURL", current.Title) {
|
||||
return
|
||||
}
|
||||
log.Printf("[thumb] probe streamURL %s: %v", current.Title, err)
|
||||
} else if w.probeDuration(ctx, current, link) {
|
||||
return
|
||||
}
|
||||
log.Printf("[thumb] streamURL %s: %v", v.Title, err)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, current.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
|
||||
return
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "pending"})
|
||||
link, err := w.streamLink(ctx, v)
|
||||
if err != nil {
|
||||
if w.pauseForRecoverableError(err, "streamURL", v.Title) {
|
||||
return
|
||||
}
|
||||
log.Printf("[thumb] streamURL %s: %v", v.Title, err)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
|
||||
return
|
||||
}
|
||||
if w.probeDuration(ctx, v, link) {
|
||||
return
|
||||
}
|
||||
|
||||
if err := w.generateThumbnailFromLink(ctx, v, link); err != nil {
|
||||
if localLink, ok := localPreviewLink(v); ok && link.URL != localLink.URL {
|
||||
if w.probeDuration(ctx, v, localLink) {
|
||||
return
|
||||
}
|
||||
if localErr := w.generateThumbnailFromLink(ctx, v, localLink); localErr == nil {
|
||||
return
|
||||
}
|
||||
@@ -1524,6 +1545,38 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
|
||||
}
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) streamLink(ctx context.Context, v *catalog.Video) (*drives.StreamLink, error) {
|
||||
link, err := w.Drive.StreamURL(ctx, v.FileID)
|
||||
if err == nil {
|
||||
return link, nil
|
||||
}
|
||||
if localLink, ok := localPreviewLink(v); ok {
|
||||
return localLink, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) probeDuration(ctx context.Context, v *catalog.Video, link *drives.StreamLink) bool {
|
||||
if v.DurationSeconds > 0 {
|
||||
return false
|
||||
}
|
||||
dur, err := w.Gen.Probe(ctx, link)
|
||||
if err == nil {
|
||||
if dur > 0 {
|
||||
v.DurationSeconds = int(dur)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
DurationSeconds: int(dur),
|
||||
})
|
||||
}
|
||||
return false
|
||||
}
|
||||
if w.pauseForRecoverableError(err, "probe", v.Title) {
|
||||
return true
|
||||
}
|
||||
log.Printf("[thumb] probe %s: %v", v.Title, err)
|
||||
return false
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) generateThumbnailFromLink(ctx context.Context, v *catalog.Video, link *drives.StreamLink) error {
|
||||
if _, err := w.Gen.GenerateThumbnail(ctx, link, v.ID, 0); err != nil {
|
||||
return err
|
||||
|
||||
@@ -13,11 +13,11 @@ import (
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
|
||||
func TestThumbWorkerUpdatesThumbnailAndDurationWithoutChangingPreviewStatus(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-video")
|
||||
|
||||
gen := &fakeThumbGenerator{}
|
||||
gen := &fakeThumbGenerator{probeDuration: 42}
|
||||
drv := &previewFakeDrive{}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
@@ -33,8 +33,8 @@ func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
|
||||
if got.PreviewStatus != "pending" {
|
||||
t.Fatalf("preview status = %q, want pending", got.PreviewStatus)
|
||||
}
|
||||
if got.DurationSeconds != 0 {
|
||||
t.Fatalf("duration = %d, want unchanged", got.DurationSeconds)
|
||||
if got.DurationSeconds != 42 {
|
||||
t.Fatalf("duration = %d, want probed duration", got.DurationSeconds)
|
||||
}
|
||||
if gen.thumbnailVideoID != video.ID {
|
||||
t.Fatalf("thumbnail video id = %q, want %q", gen.thumbnailVideoID, video.ID)
|
||||
@@ -42,14 +42,53 @@ func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
|
||||
if gen.thumbnailDuration != 0 {
|
||||
t.Fatalf("thumbnail duration = %.1f, want fixed-offset thumbnail generation", gen.thumbnailDuration)
|
||||
}
|
||||
if gen.probeCalls != 0 {
|
||||
t.Fatalf("probe calls = %d, want 0 for thumbnail generation", gen.probeCalls)
|
||||
if gen.probeCalls != 1 {
|
||||
t.Fatalf("probe calls = %d, want 1 for thumbnail generation", gen.probeCalls)
|
||||
}
|
||||
if drv.streamFileID != video.FileID {
|
||||
t.Fatalf("stream file id = %q, want %q", drv.streamFileID, video.FileID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerBackfillsDurationWhenThumbnailAlreadyExists(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-existing-thumbnail")
|
||||
video.ThumbnailURL = "/p/thumb/" + video.ID
|
||||
if err := cat.UpsertVideo(ctx, video); err != nil {
|
||||
t.Fatalf("update video: %v", err)
|
||||
}
|
||||
|
||||
gen := &fakeThumbGenerator{probeDuration: 19}
|
||||
drv := &previewFakeDrive{}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
worker.process(ctx, video)
|
||||
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DurationSeconds != 19 {
|
||||
t.Fatalf("duration = %d, want probed duration", got.DurationSeconds)
|
||||
}
|
||||
if got.ThumbnailURL != "/p/thumb/"+video.ID {
|
||||
t.Fatalf("thumbnail = %q, want unchanged existing thumbnail", got.ThumbnailURL)
|
||||
}
|
||||
ready, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "ready", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list ready thumbnails: %v", err)
|
||||
}
|
||||
if len(ready) != 1 || ready[0].ID != video.ID {
|
||||
t.Fatalf("ready thumbnails = %#v, want only %s", ready, video.ID)
|
||||
}
|
||||
if gen.probeCalls != 1 {
|
||||
t.Fatalf("probe calls = %d, want 1", gen.probeCalls)
|
||||
}
|
||||
if gen.thumbnailVideoID != "" {
|
||||
t.Fatalf("thumbnail generation video id = %q, want no regeneration", gen.thumbnailVideoID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerFallsBackToLocalPreviewWhenDriveStreamFails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-local-preview")
|
||||
@@ -469,12 +508,17 @@ type fakeThumbGenerator struct {
|
||||
thumbnailDuration float64
|
||||
thumbnailURL string
|
||||
probeCalls int
|
||||
probeDuration float64
|
||||
probeErr error
|
||||
generateErr error
|
||||
}
|
||||
|
||||
func (g *fakeThumbGenerator) Probe(context.Context, *drives.StreamLink) (float64, error) {
|
||||
g.probeCalls++
|
||||
return 42, nil
|
||||
if g.probeErr != nil {
|
||||
return 0, g.probeErr
|
||||
}
|
||||
return g.probeDuration, nil
|
||||
}
|
||||
|
||||
func (g *fakeThumbGenerator) GenerateThumbnail(_ context.Context, link *drives.StreamLink, videoID string, duration float64) (string, error) {
|
||||
@@ -568,7 +612,6 @@ func (d *previewFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
}
|
||||
func (d *previewFakeDrive) RootID() string { return "root" }
|
||||
|
||||
|
||||
func TestWorkerWaitIdleReturnsImmediatelyWhenQueueEmpty(t *testing.T) {
|
||||
worker := NewWorker(&fakeTeaserGenerator{}, nil, &previewFakeDrive{})
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -144,13 +145,15 @@ func (p *Proxy) ServeStream(w http.ResponseWriter, r *http.Request, driveID, fil
|
||||
// 302 之后浏览器用自己的 UA 直连,CDN 仍然认签名
|
||||
// - pikpak:与 OpenList 一致,WebContentLink / media link 都是自签 URL,
|
||||
// CDN 不校验请求头,直连可获得最佳带宽并避免占用 backend 出站
|
||||
// - onedrive:Microsoft Graph 返回的 @microsoft.graph.downloadUrl 是短期
|
||||
// 免鉴权下载 URL,不需要后端继续代传视频字节
|
||||
//
|
||||
// 其余网盘(如 OneDrive / 沃盘 / 夸克等)仍走反代,因为它们的下载
|
||||
// 其余网盘(如沃盘 / 夸克等)仍走反代,因为它们的下载
|
||||
// 链接通常需要随请求带上后端持有的 Cookie / Authorization / Range
|
||||
// 的特殊处理,浏览器拿不到这些上下文。
|
||||
func shouldRedirect(d drives.Drive) bool {
|
||||
switch d.Kind() {
|
||||
case "p115", "pikpak":
|
||||
case "p115", "pikpak", "onedrive":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -169,6 +172,11 @@ func (p *Proxy) serve(w http.ResponseWriter, r *http.Request, link *drives.Strea
|
||||
http.Error(w, "bad upstream url", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
if localPath, ok := localFilePath(u, link.URL); ok {
|
||||
w.Header().Set("Cache-Control", "private, max-age=300")
|
||||
http.ServeFile(w, r, localPath)
|
||||
return
|
||||
}
|
||||
req, err := http.NewRequestWithContext(r.Context(), r.Method, u.String(), nil)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
@@ -211,6 +219,19 @@ func (p *Proxy) ServeLocal(w http.ResponseWriter, r *http.Request, path string)
|
||||
http.ServeFile(w, r, path)
|
||||
}
|
||||
|
||||
func localFilePath(u *url.URL, raw string) (string, bool) {
|
||||
if u == nil {
|
||||
return "", false
|
||||
}
|
||||
if u.Scheme == "file" && u.Path != "" {
|
||||
return u.Path, true
|
||||
}
|
||||
if u.Scheme == "" && u.Host == "" && filepath.IsAbs(raw) {
|
||||
return raw, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
var errDriveNotFound = &httpError{Code: http.StatusNotFound, Msg: "drive not found"}
|
||||
|
||||
type httpError struct {
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -149,6 +151,61 @@ func TestServeStreamPikPakSetsRedirectHeaders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamRedirectsOneDrive(t *testing.T) {
|
||||
reg := NewRegistry()
|
||||
drv := &proxyFakeSimpleDrive{
|
||||
kind: "onedrive",
|
||||
url: "https://public.onedrive.example/video.mp4",
|
||||
}
|
||||
reg.Set("onedrive", drv)
|
||||
|
||||
p := New(reg)
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/onedrive/file-1", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
p.ServeStream(rr, req, "onedrive", "file-1")
|
||||
|
||||
if rr.Code != http.StatusFound {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusFound)
|
||||
}
|
||||
if got := rr.Header().Get("Location"); got != "https://public.onedrive.example/video.mp4" {
|
||||
t.Fatalf("Location = %q", got)
|
||||
}
|
||||
if drv.calls != 1 {
|
||||
t.Fatalf("link calls = %d, want 1", drv.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamServesLocalFilePath(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "video.mp4")
|
||||
if err := os.WriteFile(path, []byte("0123456789"), 0o644); err != nil {
|
||||
t.Fatalf("write local file: %v", err)
|
||||
}
|
||||
reg := NewRegistry()
|
||||
drv := &proxyFakeSimpleDrive{
|
||||
kind: "localstorage",
|
||||
url: path,
|
||||
}
|
||||
reg.Set("local", drv)
|
||||
|
||||
p := New(reg)
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/local/file-1", nil)
|
||||
req.Header.Set("Range", "bytes=2-5")
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
p.ServeStream(rr, req, "local", "file-1")
|
||||
|
||||
if rr.Code != http.StatusPartialContent {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusPartialContent)
|
||||
}
|
||||
if got := rr.Body.String(); got != "2345" {
|
||||
t.Fatalf("body = %q, want range bytes", got)
|
||||
}
|
||||
if drv.calls != 1 {
|
||||
t.Fatalf("link calls = %d, want 1", drv.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func requestPikPak(t *testing.T, p *Proxy, driveID, fileID, ua string) {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/"+driveID+"/"+fileID, nil)
|
||||
@@ -192,3 +249,36 @@ func (d *proxyFakePikPakDrive) EnsureDir(context.Context, string) (string, error
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakePikPakDrive) RootID() string { return "0" }
|
||||
|
||||
type proxyFakeSimpleDrive struct {
|
||||
kind string
|
||||
url string
|
||||
calls int
|
||||
}
|
||||
|
||||
func (d *proxyFakeSimpleDrive) Kind() string { return d.kind }
|
||||
func (d *proxyFakeSimpleDrive) ID() string { return d.kind }
|
||||
func (d *proxyFakeSimpleDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
|
||||
d.calls++
|
||||
return &drives.StreamLink{
|
||||
URL: d.url,
|
||||
Headers: http.Header{},
|
||||
Expires: time.Now().Add(10 * time.Minute),
|
||||
}, nil
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) RootID() string { return "0" }
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Package spider91migrate 周期性把 spider91 drive 下载到本地的视频
|
||||
// 上传到一个指定的目标 drive 目录(PikPak 或 115),上传成功后:
|
||||
// 上传到一个指定的目标 drive 目录(PikPak、115 或 OneDrive),上传成功后:
|
||||
//
|
||||
// - 改写 catalog 行:drive_id / file_id / content_hash 改成目标盘的;
|
||||
// 视频自身的 id 不变(仍是 spider91-<driveID>-<viewkey>),video_tags、
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/onedrive"
|
||||
"github.com/video-site/backend/internal/drives/p115"
|
||||
"github.com/video-site/backend/internal/drives/pikpak"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
@@ -39,12 +40,14 @@ import (
|
||||
// 这一层抽象把"迁移调用方"和"具体盘的 SDK 协议"解耦:
|
||||
// - PikPak 走 GCID + OSS PutObject(pikpak.UploadResult)
|
||||
// - 115 走 SHA1 + 秒传 / OSS / 分片(p115.UploadResult)
|
||||
// - OneDrive 走 SHA1 + 小文件 PUT / 大文件 upload session
|
||||
//
|
||||
// 两个返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
|
||||
// 各家返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
|
||||
type uploadTarget interface {
|
||||
ID() string
|
||||
Kind() string
|
||||
RootID() string
|
||||
EnsureDir(ctx context.Context, pathFromRoot string) (string, error)
|
||||
UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error)
|
||||
Rename(ctx context.Context, fileID, newName string) error
|
||||
}
|
||||
@@ -52,7 +55,7 @@ type uploadTarget interface {
|
||||
// UploadResult 是 uploadTarget.UploadAndReportHash 的归一返回。
|
||||
//
|
||||
// FileID 目标盘上的新文件 ID;
|
||||
// Hash GCID(PikPak)或 SHA1 HEX 大写(115),写入 catalog.content_hash 用于跨盘去重;
|
||||
// Hash GCID(PikPak)或 SHA1 HEX(115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;
|
||||
// Size 实际上传字节数。
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
@@ -60,7 +63,9 @@ type UploadResult struct {
|
||||
Size int64
|
||||
}
|
||||
|
||||
// pikpakAdapter / p115Adapter 把具体 driver 包装成 uploadTarget。
|
||||
const spider91UploadDirName = "91 Spider"
|
||||
|
||||
// pikpakAdapter / p115Adapter / onedriveAdapter 把具体 driver 包装成 uploadTarget。
|
||||
//
|
||||
// 之所以不让 driver 直接实现 uploadTarget:
|
||||
//
|
||||
@@ -74,6 +79,9 @@ type pikpakAdapter struct {
|
||||
func (a *pikpakAdapter) ID() string { return a.d.ID() }
|
||||
func (a *pikpakAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *pikpakAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *pikpakAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *pikpakAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
@@ -92,6 +100,9 @@ type p115Adapter struct {
|
||||
func (a *p115Adapter) ID() string { return a.d.ID() }
|
||||
func (a *p115Adapter) Kind() string { return a.d.Kind() }
|
||||
func (a *p115Adapter) RootID() string { return a.d.RootID() }
|
||||
func (a *p115Adapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *p115Adapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportSha1(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
@@ -103,6 +114,27 @@ func (a *p115Adapter) Rename(ctx context.Context, fileID, newName string) error
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
type onedriveAdapter struct {
|
||||
d *onedrive.Driver
|
||||
}
|
||||
|
||||
func (a *onedriveAdapter) ID() string { return a.d.ID() }
|
||||
func (a *onedriveAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *onedriveAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *onedriveAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *onedriveAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
return UploadResult{FileID: res.FileID, Hash: res.Hash, Size: res.Size}, nil
|
||||
}
|
||||
func (a *onedriveAdapter) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
// adaptUploadTarget 把通用 drive 包装成 uploadTarget。
|
||||
// 不支持的盘 kind 返回 error;调用方静默跳过。
|
||||
func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
@@ -111,6 +143,8 @@ func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
return &pikpakAdapter{d: v}, nil
|
||||
case *p115.Driver:
|
||||
return &p115Adapter{d: v}, nil
|
||||
case *onedrive.Driver:
|
||||
return &onedriveAdapter{d: v}, nil
|
||||
case uploadTarget:
|
||||
// 测试或自定义实现可以直接传入;优先使用具体类型分支以拿到适配器。
|
||||
return v, nil
|
||||
@@ -511,15 +545,19 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// 上传到目标盘的根目录(用户配置的目标 drive 的 rootID)。
|
||||
// 上传到目标盘 rootID 下的固定 "91 Spider" 子目录。若用户把目标盘 rootID
|
||||
// 配成某个自定义目录,这里会在该自定义目录下查找/创建 "91 Spider"。
|
||||
// 上传名走 desiredPikPakName 算出来的方案 B 格式:
|
||||
//
|
||||
// <sanitized title>-<viewkey 后 8 位>.<ext>
|
||||
//
|
||||
// 这样网盘 Web 端列出来的文件名能直接看出是哪个视频,
|
||||
// 又用 viewkey 后 8 位避免同标题撞名。两个目标盘(PikPak / 115)共用同一格式,
|
||||
// 又用 viewkey 后 8 位避免同标题撞名。所有目标盘共用同一格式,
|
||||
// 简化前端 / catalog 的认知。
|
||||
parent := pp.RootID()
|
||||
parent, err := pp.EnsureDir(ctx, spider91UploadDirName)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s ensure %q dir: %w", pp.Kind(), spider91UploadDirName, err)
|
||||
}
|
||||
uploadName := desiredPikPakName(v.Title, extractViewKey(v.ID), v.Ext)
|
||||
res, err := pp.UploadAndReportHash(ctx, parent, uploadName, f, info.Size())
|
||||
if err != nil {
|
||||
@@ -639,7 +677,7 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driv
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// backfillFileNames 扫描目标 drive(PikPak 或 115)下所有 spider91-* 起始 ID 的视频,
|
||||
// backfillFileNames 扫描目标 drive(PikPak、115 或 OneDrive)下所有 spider91-* 起始 ID 的视频,
|
||||
// 对文件名不是 desiredPikPakName(...) 期望格式的,调 target.Rename 修正,
|
||||
// 并把 catalog.file_name 同步到新名字。
|
||||
//
|
||||
|
||||
@@ -53,6 +53,8 @@ type fakePikPak struct {
|
||||
uploadFunc func(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error)
|
||||
mu sync.Mutex
|
||||
gotBodies map[string][]byte
|
||||
gotParents map[string]string
|
||||
ensureCalls []string
|
||||
// renameCalls 记录每次 Rename 的 fileID->newName 历史,用于 backfill 测试断言。
|
||||
renameCalls map[string]string
|
||||
}
|
||||
@@ -62,6 +64,7 @@ func newFakePikPak(id, rootID string) *fakePikPak {
|
||||
id: id,
|
||||
rootID: rootID,
|
||||
gotBodies: make(map[string][]byte),
|
||||
gotParents: make(map[string]string),
|
||||
renameCalls: make(map[string]string),
|
||||
}
|
||||
}
|
||||
@@ -80,8 +83,11 @@ func (d *fakePikPak) StreamURL(context.Context, string) (*drives.StreamLink, err
|
||||
func (d *fakePikPak) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakePikPak) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
func (d *fakePikPak) EnsureDir(_ context.Context, pathFromRoot string) (string, error) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.ensureCalls = append(d.ensureCalls, pathFromRoot)
|
||||
return d.rootID + "/" + pathFromRoot, nil
|
||||
}
|
||||
func (d *fakePikPak) Rename(_ context.Context, fileID, newName string) error {
|
||||
d.mu.Lock()
|
||||
@@ -99,6 +105,7 @@ func (d *fakePikPak) UploadAndReportHash(ctx context.Context, parentID, name str
|
||||
body, _ := io.ReadAll(r)
|
||||
d.mu.Lock()
|
||||
d.gotBodies[name] = body
|
||||
d.gotParents[name] = parentID
|
||||
d.mu.Unlock()
|
||||
return UploadResult{
|
||||
FileID: "remote-" + name,
|
||||
@@ -127,6 +134,19 @@ func (d *fakeP115) Kind() string { return "p115" }
|
||||
var _ drives.Drive = (*fakeP115)(nil)
|
||||
var _ uploadTarget = (*fakeP115)(nil)
|
||||
|
||||
type fakeOneDrive struct {
|
||||
*fakePikPak
|
||||
}
|
||||
|
||||
func newFakeOneDrive(id, rootID string) *fakeOneDrive {
|
||||
return &fakeOneDrive{fakePikPak: newFakePikPak(id, rootID)}
|
||||
}
|
||||
|
||||
func (d *fakeOneDrive) Kind() string { return "onedrive" }
|
||||
|
||||
var _ drives.Drive = (*fakeOneDrive)(nil)
|
||||
var _ uploadTarget = (*fakeOneDrive)(nil)
|
||||
|
||||
// TestBackfillFileNamesRenamesOnlyMismatchedSpider91Videos 验证回填逻辑:
|
||||
//
|
||||
// - 已经是期望格式的不会再调 Rename(幂等)
|
||||
@@ -347,6 +367,12 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
|
||||
if _, ok := pp.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("PikPak did not receive expected upload name %q (got names: %v)", wantName, keysOf(pp.gotBodies))
|
||||
}
|
||||
if gotParent := pp.gotParents[wantName]; gotParent != "pikpak-root-id/"+spider91UploadDirName {
|
||||
t.Fatalf("upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(pp.ensureCalls) != 1 || pp.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("ensure calls = %#v, want %q", pp.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
@@ -884,6 +910,12 @@ func TestRunOnceMigratesToP115Target(t *testing.T) {
|
||||
if _, ok := target.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("p115 did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
|
||||
}
|
||||
if gotParent := target.gotParents[wantName]; gotParent != "p115-root-cid/"+spider91UploadDirName {
|
||||
t.Fatalf("p115 upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("p115 ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
@@ -905,7 +937,67 @@ func TestRunOnceMigratesToP115Target(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak 也不是 115 时,
|
||||
func TestRunOnceMigratesToOneDriveTarget(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src, _ := setupSpider91(t)
|
||||
target := newFakeOneDrive("onedrive-target", "onedrive-root")
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(target)
|
||||
|
||||
now := time.Now()
|
||||
id := writeSpider91Video(t, cat, src, "vk-od-001", ".mp4", []byte("video bytes onedrive"), now)
|
||||
|
||||
m := New(Config{
|
||||
Catalog: cat,
|
||||
Registry: reg,
|
||||
GetTargetDriveID: func() string { return target.ID() },
|
||||
KeepLatestN: -1,
|
||||
})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
if target.uploadCalls != 1 {
|
||||
t.Fatalf("onedrive upload calls = %d, want 1", target.uploadCalls)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DriveID != target.ID() {
|
||||
t.Fatalf("drive_id = %q, want %q", got.DriveID, target.ID())
|
||||
}
|
||||
wantName := "Sample vk-od-001-001.mp4"
|
||||
if _, ok := target.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("onedrive did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
|
||||
}
|
||||
if gotParent := target.gotParents[wantName]; gotParent != "onedrive-root/"+spider91UploadDirName {
|
||||
t.Fatalf("onedrive upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("onedrive ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
if got.FileName != wantName {
|
||||
t.Fatalf("file_name = %q, want %q", got.FileName, wantName)
|
||||
}
|
||||
if got.ContentHash == "" {
|
||||
t.Fatal("content_hash should be set after onedrive migration")
|
||||
}
|
||||
|
||||
videoPath, _ := src.VideoPath("vk-od-001.mp4")
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local mp4 still exists after onedrive migration or stat error: %v", err)
|
||||
}
|
||||
thumbPath, _ := src.ThumbPath("vk-od-001.jpg")
|
||||
if _, err := os.Stat(thumbPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local thumb still exists after onedrive migration or stat error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115 也不是 OneDrive 时,
|
||||
// resolveTarget 拒绝并返回 error,让 runOnce 静默跳过(不会做破坏性变更)。
|
||||
func TestResolveTargetRejectsUnsupportedKind(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
services:
|
||||
video-site-91:
|
||||
image: ghcr.io/nianzhibai/91:stable
|
||||
container_name: video-site-91
|
||||
ports:
|
||||
- "9191:9191"
|
||||
volumes:
|
||||
- ./data:/opt/video-site-91/data
|
||||
restart: unless-stopped
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
APP_DIR="/opt/video-site-91"
|
||||
DATA_DIR="${VIDEO_DATA_DIR:-$APP_DIR/data}"
|
||||
CONFIG="${VIDEO_CONFIG:-$DATA_DIR/config.yaml}"
|
||||
EXAMPLE="$APP_DIR/config.example.yaml"
|
||||
PORT="${VIDEO_LISTEN_PORT:-9191}"
|
||||
|
||||
mkdir -p "$DATA_DIR" "$DATA_DIR/previews" "$DATA_DIR/uploads" "$DATA_DIR/spider91"
|
||||
|
||||
if [ ! -f "$CONFIG" ]; then
|
||||
if [ ! -f "$EXAMPLE" ]; then
|
||||
echo "[entrypoint] missing config template: $EXAMPLE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$(dirname "$CONFIG")"
|
||||
cp "$EXAMPLE" "$CONFIG"
|
||||
|
||||
SECRET="$(openssl rand -hex 32)"
|
||||
sed -i -E "s#^([[:space:]]*listen:[[:space:]]*).*\$#\1\"0.0.0.0:${PORT}\"#" "$CONFIG"
|
||||
sed -i -E "s#^([[:space:]]*session_secret:[[:space:]]*).*\$#\1\"${SECRET}\"#" "$CONFIG"
|
||||
sed -i -E "s#^([[:space:]]*db_path:[[:space:]]*).*\$#\1\"${DATA_DIR}/video-site.db\"#" "$CONFIG"
|
||||
sed -i -E "s#^([[:space:]]*local_preview_dir:[[:space:]]*).*\$#\1\"${DATA_DIR}/previews\"#" "$CONFIG"
|
||||
chmod 600 "$CONFIG"
|
||||
|
||||
echo "[entrypoint] generated $CONFIG"
|
||||
else
|
||||
echo "[entrypoint] using existing $CONFIG"
|
||||
fi
|
||||
|
||||
if [ -n "${VIDEO_VERSION_FILE:-}" ] && [ -n "${VIDEO_IMAGE_VERSION:-}" ]; then
|
||||
mkdir -p "$(dirname "$VIDEO_VERSION_FILE")"
|
||||
printf '%s\n' "$VIDEO_IMAGE_VERSION" > "$VIDEO_VERSION_FILE"
|
||||
fi
|
||||
|
||||
exec "$@"
|
||||
+61
-72
@@ -26,6 +26,7 @@ const kindLabel: Record<string, string> = {
|
||||
pikpak: "PikPak",
|
||||
wopan: "联通沃盘",
|
||||
onedrive: "OneDrive",
|
||||
localstorage: "本地存储",
|
||||
spider91: "91 爬虫",
|
||||
};
|
||||
|
||||
@@ -80,10 +81,10 @@ export function DrivesPage() {
|
||||
const [selectedDriveId, setSelectedDriveId] = useState<string | null>(null);
|
||||
const { show } = useToast();
|
||||
|
||||
// 当前系统中可作为 spider91 上传目标的 drive 列表(pikpak ∪ p115)。
|
||||
// 当前系统中可作为 spider91 上传目标的 drive 列表(pikpak ∪ p115 ∪ onedrive)。
|
||||
// 用户保存 spider91 drive 时从这里挑一个;空表示本地保存不上传。
|
||||
const uploadTargets = useMemo(
|
||||
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115"),
|
||||
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115" || d.kind === "onedrive"),
|
||||
[list]
|
||||
);
|
||||
|
||||
@@ -427,13 +428,13 @@ export function DrivesPage() {
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* 右栏:Teaser / 封面 与 缓存占用 */}
|
||||
{/* 右栏:Teaser / 封面 / 指纹 与 缓存占用 */}
|
||||
<div>
|
||||
<div className="admin-detail-card">
|
||||
<header className="admin-detail-card__title">
|
||||
<div className="admin-detail-card__title-left">
|
||||
<PlayCircle size={16} />
|
||||
<span>Teaser 预览与封面</span>
|
||||
<span>生成状态</span>
|
||||
</div>
|
||||
<div className="admin-detail-actions-inline">
|
||||
<button
|
||||
@@ -481,6 +482,22 @@ export function DrivesPage() {
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">指纹状态</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationStatusLine label="指纹" status={d.fingerprintGenerationStatus} />
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">指纹数量</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationCounts
|
||||
ready={d.fingerprintReadyCount}
|
||||
pending={d.fingerprintPendingCount}
|
||||
failed={d.fingerprintFailedCount}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="admin-detail-actions">
|
||||
@@ -586,7 +603,7 @@ export function DrivesPage() {
|
||||
<div className="admin-empty">加载中...</div>
|
||||
) : list.length === 0 ? (
|
||||
<div className="admin-card admin-empty">
|
||||
还没有配置任何网盘。点击右上角「新建」,选择夸克 / 115 / PikPak / 沃盘 / OneDrive,填入凭证即可。
|
||||
还没有配置任何网盘。点击右上角「新建」,选择夸克 / 115 / PikPak / 沃盘 / OneDrive / 本地存储,填入凭证或路径即可。
|
||||
</div>
|
||||
) : (
|
||||
<div className="admin-drives-grid">
|
||||
@@ -625,6 +642,15 @@ export function DrivesPage() {
|
||||
</span>
|
||||
</strong>
|
||||
</div>
|
||||
<div className="admin-drive-card__metric">
|
||||
<span>指纹数 (就绪/失败)</span>
|
||||
<strong>
|
||||
{d.fingerprintReadyCount ?? 0}
|
||||
<span style={{ fontSize: "11px", fontWeight: "normal", color: "var(--text-faint)" }}>
|
||||
{" "}/ {d.fingerprintFailedCount ?? 0}
|
||||
</span>
|
||||
</strong>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="admin-drive-card__footer">
|
||||
@@ -837,6 +863,11 @@ function DriveForm({
|
||||
}) {
|
||||
const fields = useMemo(() => credentialFields(form.kind), [form.kind]);
|
||||
const help = credentialHelp(form.kind, isEdit);
|
||||
const showDirectoryFields =
|
||||
form.kind !== "spider91" &&
|
||||
form.kind !== "onedrive" &&
|
||||
form.kind !== "localstorage" &&
|
||||
form.kind !== "pikpak";
|
||||
|
||||
function set<K extends keyof FormState>(k: K, v: FormState[K]) {
|
||||
onChange({ ...form, [k]: v });
|
||||
@@ -873,13 +904,14 @@ function DriveForm({
|
||||
>
|
||||
<option value="p115">115 网盘</option>
|
||||
<option value="pikpak">PikPak</option>
|
||||
<option value="spider91">91 爬虫</option>
|
||||
<option value="onedrive">OneDrive</option>
|
||||
<option value="localstorage">本地存储</option>
|
||||
<option value="spider91">91 Spider</option>
|
||||
<option value="quark">夸克网盘</option>
|
||||
<option value="wopan">联通沃盘</option>
|
||||
<option value="onedrive">OneDrive</option>
|
||||
</select>
|
||||
</div>
|
||||
{form.kind !== "spider91" && (
|
||||
{showDirectoryFields && (
|
||||
<>
|
||||
<div className="admin-form__row">
|
||||
<label>根目录 ID</label>
|
||||
@@ -953,9 +985,9 @@ function DriveForm({
|
||||
* Spider91UploadTargetField 是 spider91 drive 表单专属的"上传目标"下拉。
|
||||
*
|
||||
* 行为:
|
||||
* - 选项 = "本地保存,不上传" + 系统中所有 pikpak/p115 drive
|
||||
* - 选项 = "本地保存,不上传" + 系统中所有 pikpak/p115/onedrive drive
|
||||
* - value="" 时后端不迁移上传,视频保存在服务器本地
|
||||
* - 没有任何 pikpak/p115 drive 时仍允许选择本地保存
|
||||
* - 没有任何 pikpak/p115/onedrive drive 时仍允许选择本地保存
|
||||
* - 该字段写入的是全局 setting `spider91.upload_drive_id`,不是 drive 自己的
|
||||
* credentials —— 所有 spider91 drive 共享同一个上传目标
|
||||
*/
|
||||
@@ -980,7 +1012,7 @@ function Spider91UploadTargetField({
|
||||
))}
|
||||
</select>
|
||||
<div className="admin-form__help">
|
||||
选择本地保存时,爬取视频只保存在服务器本地;选择 115 网盘或 PikPak 后,较早的视频会上传到该云盘根目录。该设置全局生效。
|
||||
选择本地保存时,爬取视频只保存在服务器本地;选择 115 网盘、PikPak 或 OneDrive 后,较早的视频会上传到该云盘根目录下的 91 Spider 文件夹。该设置全局生效。
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
@@ -994,13 +1026,15 @@ function credentialHelp(kind: Kind, isEdit: boolean): string {
|
||||
case "p115":
|
||||
return `登录 115.com 后复制 Cookie,形如 "UID=...; CID=...; SEID=...; KID=..."。${note}`;
|
||||
case "pikpak":
|
||||
return `参考 OpenList 的 PikPak 登录方式。可填用户名和密码首次登录,也可填 refresh_token;如返回验证码链接,打开验证后把 captcha_token 粘贴回来。${note}`;
|
||||
return `填写 PikPak 账号和密码即可。平台、设备 ID、验证码 token 和 refresh token 会由服务端自动处理并保存。${note}`;
|
||||
case "wopan":
|
||||
return `需要 access_token 和 refresh_token。后续会加扫码/短信登录入口,第一版只能手工粘贴。${note}`;
|
||||
case "onedrive":
|
||||
return `按 OpenList 默认方式,通过 api.oplist.org 在线刷新 token。只需要 refresh_token;保存后会自动回写新的 access_token / refresh_token。${note}`;
|
||||
return `按 OpenList 默认应用在线挂载,只需要 refresh_token;保存时会自动刷新并保存 token。${note}`;
|
||||
case "localstorage":
|
||||
return `把服务器上的一个已有目录作为视频来源扫描。填写绝对路径,例如 /mnt/videos;系统会读取该目录及子目录中的视频,并生成封面、Teaser 和指纹。${note}`;
|
||||
case "spider91":
|
||||
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;它不是外部网盘,不需要填写 Cookie 或目录 ID。后续流水线会把较早的视频上传到你选择的 115 / PikPak 目标盘。";
|
||||
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;它不是外部网盘,不需要填写 Cookie 或目录 ID。后续流水线会把较早的视频上传到你选择的 115 / PikPak / OneDrive 目标盘。";
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
@@ -1039,42 +1073,15 @@ function credentialFields(kind: Kind): Array<{
|
||||
return [
|
||||
{
|
||||
key: "username",
|
||||
label: "用户名 / 邮箱(无 refresh_token 时必填)",
|
||||
label: "用户名 / 邮箱",
|
||||
placeholder: "user@example.com",
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
key: "password",
|
||||
label: "密码(无 refresh_token 时必填)",
|
||||
label: "密码",
|
||||
placeholder: "PikPak 密码",
|
||||
},
|
||||
{
|
||||
key: "platform",
|
||||
label: "platform",
|
||||
placeholder: "web(可选:android / web / pc)",
|
||||
help: "默认 web;如果登录或直链异常,可尝试 android 或 pc。",
|
||||
},
|
||||
{
|
||||
key: "refresh_token",
|
||||
label: "refresh_token(可选)",
|
||||
placeholder: "已有 token 时可直接粘贴",
|
||||
multiline: true,
|
||||
},
|
||||
{
|
||||
key: "captcha_token",
|
||||
label: "captcha_token(可选)",
|
||||
placeholder: "遇到验证码校验时粘贴",
|
||||
multiline: true,
|
||||
},
|
||||
{
|
||||
key: "device_id",
|
||||
label: "device_id(可选)",
|
||||
placeholder: "留空自动生成并保存",
|
||||
},
|
||||
{
|
||||
key: "disable_media_link",
|
||||
label: "disable_media_link",
|
||||
placeholder: "true",
|
||||
help: "默认 true,使用原始下载链接;填 false 可尝试使用媒体缓存链接。",
|
||||
required: true,
|
||||
},
|
||||
];
|
||||
case "wopan":
|
||||
@@ -1106,34 +1113,15 @@ function credentialFields(kind: Kind): Array<{
|
||||
multiline: true,
|
||||
required: true,
|
||||
},
|
||||
];
|
||||
case "localstorage":
|
||||
return [
|
||||
{
|
||||
key: "access_token",
|
||||
label: "access_token(可选)",
|
||||
placeholder: "留空也可以,保存时会在线刷新",
|
||||
multiline: true,
|
||||
},
|
||||
{
|
||||
key: "api_url_address",
|
||||
label: "api_url_address(可选)",
|
||||
placeholder: "https://api.oplist.org/onedrive/renewapi",
|
||||
help: "默认使用 OpenList 的在线刷新 API;除非你有自建兼容服务,否则留空。",
|
||||
},
|
||||
{
|
||||
key: "region",
|
||||
label: "region(可选)",
|
||||
placeholder: "global(可选:global / cn / us / de)",
|
||||
help: "默认 global;世纪互联填 cn,美国政府云填 us,德国云填 de。",
|
||||
},
|
||||
{
|
||||
key: "is_sharepoint",
|
||||
label: "is_sharepoint(可选)",
|
||||
placeholder: "false",
|
||||
help: "普通 OneDrive 留空或 false;SharePoint 文档库填 true,并填写 site_id。",
|
||||
},
|
||||
{
|
||||
key: "site_id",
|
||||
label: "site_id(SharePoint 必填)",
|
||||
placeholder: "SharePoint site id",
|
||||
key: "path",
|
||||
label: "本地目录路径",
|
||||
placeholder: "/mnt/videos",
|
||||
required: true,
|
||||
help: "路径必须是后端服务器上的已有目录;保存后可手动重扫,系统会递归扫描支持的视频格式。",
|
||||
},
|
||||
];
|
||||
case "spider91":
|
||||
@@ -1144,6 +1132,7 @@ function credentialFields(kind: Kind): Array<{
|
||||
function defaultRootId(kind: Kind): string {
|
||||
if (kind === "pikpak") return "";
|
||||
if (kind === "onedrive") return "root";
|
||||
if (kind === "localstorage") return "/";
|
||||
if (kind === "spider91") return "/";
|
||||
return "0";
|
||||
}
|
||||
|
||||
+8
-4
@@ -77,7 +77,7 @@ export function checkUpdate() {
|
||||
|
||||
export type AdminDrive = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "spider91";
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "localstorage" | "spider91";
|
||||
name: string;
|
||||
rootId: string;
|
||||
scanRootId: string;
|
||||
@@ -96,12 +96,16 @@ export type AdminDrive = {
|
||||
lastCrawlAt?: number;
|
||||
thumbnailGenerationStatus?: DriveGenerationStatus;
|
||||
previewGenerationStatus?: DriveGenerationStatus;
|
||||
fingerprintGenerationStatus?: DriveGenerationStatus;
|
||||
thumbnailReadyCount: number;
|
||||
thumbnailPendingCount: number;
|
||||
thumbnailFailedCount: number;
|
||||
teaserReadyCount: number;
|
||||
teaserPendingCount: number;
|
||||
teaserFailedCount: number;
|
||||
fingerprintReadyCount: number;
|
||||
fingerprintPendingCount: number;
|
||||
fingerprintFailedCount: number;
|
||||
};
|
||||
|
||||
export type DriveGenerationStatus = {
|
||||
@@ -133,7 +137,7 @@ export function getDriveStorage() {
|
||||
|
||||
export type UpsertDriveInput = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "spider91";
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "localstorage" | "spider91";
|
||||
name: string;
|
||||
rootId: string;
|
||||
scanRootId: string;
|
||||
@@ -336,9 +340,9 @@ export type Theme = "dark" | "pink";
|
||||
export type Settings = {
|
||||
theme: Theme;
|
||||
/**
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak 或 p115 drive)。
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115 或 onedrive drive)。
|
||||
* - 空字符串:本地保存,不上传到云盘。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115}。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, onedrive}。
|
||||
*/
|
||||
spider91UploadDriveId: string;
|
||||
};
|
||||
|
||||
@@ -294,5 +294,6 @@ function sourceKindFromLabel(label: string): string {
|
||||
if (value.includes("pikpak")) return "pikpak";
|
||||
if (value.includes("沃盘") || value.includes("wopan") || value.includes("联通")) return "wopan";
|
||||
if (value.includes("onedrive") || value.includes("one drive")) return "onedrive";
|
||||
if (value.includes("本地") || value.includes("localstorage") || value.includes("local storage")) return "localstorage";
|
||||
return "";
|
||||
}
|
||||
|
||||
@@ -72,5 +72,7 @@ function sourceKindFromLabel(label: string): string {
|
||||
if (value.includes("沃盘") || value.includes("wopan") || value.includes("联通"))
|
||||
return "wopan";
|
||||
if (value.includes("onedrive") || value.includes("one drive")) return "onedrive";
|
||||
if (value.includes("本地") || value.includes("localstorage") || value.includes("local storage"))
|
||||
return "localstorage";
|
||||
return "";
|
||||
}
|
||||
|
||||
@@ -1287,6 +1287,7 @@ function getDriveShortName(source: string): string {
|
||||
if (s.includes("quark") || s.includes("夸克")) return "Quak";
|
||||
if (s.includes("onedrive")) return "OneDrive";
|
||||
if (s.includes("wopan") || s.includes("沃盘")) return "沃盘";
|
||||
if (s.includes("localstorage") || s.includes("本地")) return "本地";
|
||||
if (s.includes("spider") || s.includes("爬虫")) return "爬虫";
|
||||
return source.substring(0, 4);
|
||||
}
|
||||
|
||||
@@ -1825,6 +1825,7 @@
|
||||
.admin-drive-card__brand-icon[data-kind="pikpak"] { background: var(--drive-pikpak); }
|
||||
.admin-drive-card__brand-icon[data-kind="wopan"] { background: var(--drive-wopan); }
|
||||
.admin-drive-card__brand-icon[data-kind="onedrive"] { background: var(--drive-onedrive); }
|
||||
.admin-drive-card__brand-icon[data-kind="localstorage"] { background: var(--drive-localstorage); }
|
||||
.admin-drive-card__brand-icon[data-kind="spider91"] { background: var(--accent); }
|
||||
|
||||
.admin-drive-card__info {
|
||||
|
||||
@@ -134,6 +134,7 @@
|
||||
--drive-pikpak: #8a6dff;
|
||||
--drive-wopan: #ff8a3c;
|
||||
--drive-onedrive: #4cabea;
|
||||
--drive-localstorage: #35b88f;
|
||||
|
||||
/* ----- 阴影 ----- */
|
||||
--shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.4);
|
||||
@@ -213,6 +214,7 @@
|
||||
--drive-pikpak: #8466e6;
|
||||
--drive-wopan: #e57a36;
|
||||
--drive-onedrive: #2f95cf;
|
||||
--drive-localstorage: #239978;
|
||||
|
||||
/* ----- 阴影(粉色柔投影 + 一层中性) ----- */
|
||||
--shadow-sm: 0 1px 2px rgba(180, 90, 120, 0.08);
|
||||
|
||||
@@ -306,6 +306,14 @@
|
||||
--drive-shadow: rgba(76, 171, 234, 0.15);
|
||||
--drive-shadow-strong: rgba(76, 171, 234, 0.4);
|
||||
}
|
||||
.source-badge[data-kind="localstorage"] {
|
||||
--drive-color: var(--drive-localstorage);
|
||||
--drive-bg: rgba(53, 184, 143, 0.12);
|
||||
--drive-text: #a8efdc;
|
||||
--drive-border: rgba(53, 184, 143, 0.35);
|
||||
--drive-shadow: rgba(53, 184, 143, 0.15);
|
||||
--drive-shadow-strong: rgba(53, 184, 143, 0.4);
|
||||
}
|
||||
|
||||
/* ----- Preview overlay ----- */
|
||||
.preview-loader {
|
||||
@@ -441,6 +449,7 @@
|
||||
.video-card__source[data-kind="pikpak"] { --source-color: var(--drive-pikpak); }
|
||||
.video-card__source[data-kind="wopan"] { --source-color: var(--drive-wopan); }
|
||||
.video-card__source[data-kind="onedrive"] { --source-color: var(--drive-onedrive); }
|
||||
.video-card__source[data-kind="localstorage"] { --source-color: var(--drive-localstorage); }
|
||||
|
||||
/* =========================================================
|
||||
* Empty / Loading / Skeleton
|
||||
|
||||
@@ -351,6 +351,12 @@
|
||||
color: var(--drive-onedrive);
|
||||
}
|
||||
|
||||
.vd-meta__chip[data-tone="localstorage"] {
|
||||
background: rgba(53, 184, 143, 0.14);
|
||||
border-color: rgba(53, 184, 143, 0.3);
|
||||
color: var(--drive-localstorage);
|
||||
}
|
||||
|
||||
/* ---------- Actions toolbar ---------- */
|
||||
.vd-actions {
|
||||
display: flex;
|
||||
|
||||
@@ -16,6 +16,81 @@ test("spider91 drive form does not expose advanced crawler credentials", () => {
|
||||
|
||||
test("spider91 upload target uses explicit local-save option instead of auto target", () => {
|
||||
assert.match(drivesPageSource, /本地保存,不上传/);
|
||||
assert.match(
|
||||
drivesPageSource,
|
||||
/d\.kind === "pikpak" \|\| d\.kind === "p115" \|\| d\.kind === "onedrive"/
|
||||
);
|
||||
assert.doesNotMatch(drivesPageSource, /自动:唯一/);
|
||||
assert.doesNotMatch(drivesPageSource, /自动模式/);
|
||||
});
|
||||
|
||||
test("onedrive drive form only exposes required default-app fields", () => {
|
||||
assert.match(
|
||||
drivesPageSource,
|
||||
/form\.kind !== "spider91" &&\s*form\.kind !== "onedrive" &&\s*form\.kind !== "localstorage" &&\s*form\.kind !== "pikpak"/
|
||||
);
|
||||
|
||||
const match =
|
||||
/function credentialFields[\s\S]*?case "onedrive":\s*return \[([\s\S]*?)\];\s*case "spider91":/.exec(
|
||||
drivesPageSource
|
||||
);
|
||||
assert.ok(match, "onedrive credential field block should be present");
|
||||
const fields = match[1];
|
||||
|
||||
assert.match(fields, /key: "refresh_token"/);
|
||||
assert.doesNotMatch(fields, /key: "access_token"/);
|
||||
assert.doesNotMatch(fields, /key: "api_url_address"/);
|
||||
assert.doesNotMatch(fields, /key: "region"/);
|
||||
assert.doesNotMatch(fields, /key: "is_sharepoint"/);
|
||||
assert.doesNotMatch(fields, /key: "site_id"/);
|
||||
});
|
||||
|
||||
test("pikpak drive form only exposes account login fields", () => {
|
||||
const match =
|
||||
/case "pikpak":\s*return \[([\s\S]*?)\];\s*case "wopan":/.exec(
|
||||
drivesPageSource
|
||||
);
|
||||
assert.ok(match, "pikpak credential field block should be present");
|
||||
const fields = match[1];
|
||||
|
||||
assert.match(fields, /key: "username"/);
|
||||
assert.match(fields, /key: "password"/);
|
||||
assert.doesNotMatch(fields, /key: "platform"/);
|
||||
assert.doesNotMatch(fields, /key: "refresh_token"/);
|
||||
assert.doesNotMatch(fields, /key: "captcha_token"/);
|
||||
assert.doesNotMatch(fields, /key: "device_id"/);
|
||||
assert.doesNotMatch(fields, /key: "disable_media_link"/);
|
||||
});
|
||||
|
||||
test("localstorage drive form asks for a server directory path", () => {
|
||||
assert.match(drivesPageSource, /<option value="localstorage">本地存储<\/option>/);
|
||||
|
||||
const match =
|
||||
/case "localstorage":\s*return \[([\s\S]*?)\];\s*case "spider91":/.exec(
|
||||
drivesPageSource
|
||||
);
|
||||
assert.ok(match, "localstorage credential field block should be present");
|
||||
const fields = match[1];
|
||||
|
||||
assert.match(fields, /key: "path"/);
|
||||
assert.match(fields, /label: "本地目录路径"/);
|
||||
assert.match(drivesPageSource, /if \(kind === "localstorage"\) return "\/"/);
|
||||
});
|
||||
|
||||
test("drive type selector keeps primary source order", () => {
|
||||
const options = Array.from(
|
||||
drivesPageSource.matchAll(/<option value="([^"]+)">([^<]+)<\/option>/g),
|
||||
(match) => ({ value: match[1], label: match[2] })
|
||||
);
|
||||
const driveOptions = options.slice(0, 7);
|
||||
|
||||
assert.deepEqual(driveOptions, [
|
||||
{ value: "p115", label: "115 网盘" },
|
||||
{ value: "pikpak", label: "PikPak" },
|
||||
{ value: "onedrive", label: "OneDrive" },
|
||||
{ value: "localstorage", label: "本地存储" },
|
||||
{ value: "spider91", label: "91 Spider" },
|
||||
{ value: "quark", label: "夸克网盘" },
|
||||
{ value: "wopan", label: "联通沃盘" },
|
||||
]);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user