mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-15 16:55:42 +08:00
Compare commits
91 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c87208117e | |||
| a770b3af6b | |||
| e1b8f0eae7 | |||
| 2d907da07d | |||
| 78cfb0a9e5 | |||
| fa7823ef3e | |||
| 5b0afcfc6c | |||
| 76ae3cea7d | |||
| abe335cea0 | |||
| 8dff0f07b9 | |||
| 5080203b7c | |||
| df6f0ebbbf | |||
| 8f0d52aec4 | |||
| 53327c9b8e | |||
| 57ed546b83 | |||
| 869c0d5f78 | |||
| 397823bb8d | |||
| 9e1acd4e56 | |||
| 2cd365acd4 | |||
| 48808ec568 | |||
| 5dc00e486d | |||
| 4ec1097496 | |||
| 95e46d8fbb | |||
| fdfc4771df | |||
| c8c6812ae6 | |||
| b938ff1221 | |||
| 7d63a6d265 | |||
| a8de7d2f6b | |||
| d4fcff896e | |||
| cada336e96 | |||
| 5bb93bd95b | |||
| b6be7d021c | |||
| e36a17f99d | |||
| e01b7cc3b7 | |||
| c78f22aedb | |||
| cf9de5b40a | |||
| be19f81e82 | |||
| 4d679ef64f | |||
| 4ba964b7e2 | |||
| cd3b3c6976 | |||
| 91c03947d1 | |||
| 7f1c1a51a3 | |||
| 077c2e2c38 | |||
| 30a62f265a | |||
| 38e62c6a2f | |||
| 6345cf74e0 | |||
| f004b14d20 | |||
| a407312dfa | |||
| a165605b0f | |||
| 0ac1a5b13e | |||
| a83449b129 | |||
| c68891e6f0 | |||
| 9892599412 | |||
| 0cb2a7a1c2 | |||
| 87d197496b | |||
| 0e3a5bd5cd | |||
| d72bfee10f | |||
| 389dd981a8 | |||
| 44d622d49c | |||
| d7ff0c98af | |||
| 66adf444ba | |||
| 8f8037b838 | |||
| 215d9596fd | |||
| e57058db79 | |||
| 6ec61833f2 | |||
| 6e87f88d53 | |||
| e78fa9d978 | |||
| afbff9eb55 | |||
| 039ec2a988 | |||
| da0683344e | |||
| 1a1282382e | |||
| 34b6fa8ea9 | |||
| 08e38bc4ca | |||
| c93d193efe | |||
| 08568c3951 | |||
| 7e394e2971 | |||
| d16e3168f9 | |||
| 81f348b246 | |||
| 1e71c1fb72 | |||
| d5122d289e | |||
| c146ad50ed | |||
| f5c20f9594 | |||
| 62e69d4c06 | |||
| 51725ba82f | |||
| c06db836dd | |||
| b8717da4fd | |||
| 2d57545e87 | |||
| 6518d772c0 | |||
| f2c0e7f854 | |||
| 3c7219ecd6 | |||
| 94669fd35e |
@@ -0,0 +1,21 @@
|
||||
.git
|
||||
.github
|
||||
.gitattributes
|
||||
.gitignore
|
||||
|
||||
node_modules
|
||||
dist
|
||||
release
|
||||
data
|
||||
backend/data
|
||||
backend/config.yaml
|
||||
config.yaml
|
||||
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
*.log
|
||||
*.tmp
|
||||
|
||||
tests
|
||||
video-site-implementation-plan.md
|
||||
@@ -0,0 +1,82 @@
|
||||
name: Docker
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
tags:
|
||||
- "v*"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract Docker metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=tag
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=sha,prefix=sha-
|
||||
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
|
||||
- name: Determine image version
|
||||
id: version
|
||||
shell: bash
|
||||
run: |
|
||||
if [[ "$GITHUB_REF" == refs/tags/v* ]]; then
|
||||
version="$GITHUB_REF_NAME"
|
||||
else
|
||||
version="$(git describe --tags --always --dirty 2>/dev/null || git rev-parse --short=12 HEAD)"
|
||||
fi
|
||||
echo "version=$version" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
VERSION=${{ steps.version.outputs.version }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
@@ -15,6 +15,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
@@ -36,8 +38,11 @@ jobs:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ github.ref_name }}
|
||||
run: |
|
||||
if gh release view "$TAG" >/dev/null 2>&1; then
|
||||
gh release upload "$TAG" release/*.tar.gz --clobber
|
||||
else
|
||||
gh release create "$TAG" release/*.tar.gz --title "$TAG" --notes "Prebuilt Linux release packages."
|
||||
git tag -d "$TAG" >/dev/null 2>&1 || true
|
||||
git fetch --force origin "refs/tags/$TAG:refs/tags/$TAG"
|
||||
NOTES="$(git tag -l "$TAG" --format='%(contents)')"
|
||||
if [ -z "$NOTES" ]; then
|
||||
NOTES="Prebuilt Linux release packages."
|
||||
fi
|
||||
gh release delete "$TAG" --yes >/dev/null 2>&1 || true
|
||||
gh release create "$TAG" release/*.tar.gz --title "$TAG" --notes "$NOTES" --verify-tag
|
||||
|
||||
@@ -23,8 +23,10 @@ tools/
|
||||
|
||||
# 编译产物
|
||||
backend/server
|
||||
backend/server.*
|
||||
release/
|
||||
tsconfig.tsbuildinfo
|
||||
tmp/
|
||||
|
||||
# 91 爬虫脚本独立运行时的默认输出文件(backend 跑时会显式 --output 到 backend/data/spider91/,所以不会落在这里)
|
||||
91porn_videos.json
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
- 视频直链 (MP4)
|
||||
|
||||
依赖安装:
|
||||
pip install requests beautifulsoup4 lxml
|
||||
pip install requests beautifulsoup4 lxml PySocks
|
||||
|
||||
使用方法:
|
||||
# 全量爬取(默认行为,从 page=1 一直爬到末尾,写到 OUTPUT_FILE)
|
||||
@@ -68,6 +68,7 @@ import time
|
||||
import random
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import html
|
||||
from urllib.parse import urljoin, unquote, urlparse
|
||||
@@ -80,6 +81,28 @@ except ImportError:
|
||||
print("请运行: pip install beautifulsoup4 lxml")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def prefer_ipv4_for_plain_socks5_proxy():
|
||||
"""PySocks may pick IPv6 first for socks5://; some SOCKS5 servers only accept IPv4."""
|
||||
proxy_envs = (
|
||||
os.environ.get("HTTPS_PROXY", ""),
|
||||
os.environ.get("HTTP_PROXY", ""),
|
||||
os.environ.get("https_proxy", ""),
|
||||
os.environ.get("http_proxy", ""),
|
||||
)
|
||||
uses_plain_socks5 = any(v.strip().lower().startswith("socks5://") for v in proxy_envs)
|
||||
if not uses_plain_socks5 or getattr(socket, "_spider91_ipv4_first", False):
|
||||
return
|
||||
|
||||
original_getaddrinfo = socket.getaddrinfo
|
||||
|
||||
def getaddrinfo_ipv4_first(*args, **kwargs):
|
||||
infos = original_getaddrinfo(*args, **kwargs)
|
||||
return sorted(infos, key=lambda info: 0 if info[0] == socket.AF_INET else 1)
|
||||
|
||||
socket.getaddrinfo = getaddrinfo_ipv4_first
|
||||
socket._spider91_ipv4_first = True
|
||||
|
||||
# ===================== 配置区域 =====================
|
||||
BASE_URL = "https://www.91porn.com/v.php"
|
||||
LIST_PARAMS = {
|
||||
@@ -706,7 +729,7 @@ def print_help():
|
||||
- 视频直链 (MP4)
|
||||
|
||||
依赖安装:
|
||||
pip install requests beautifulsoup4 lxml
|
||||
pip install requests beautifulsoup4 lxml PySocks
|
||||
|
||||
使用方法:
|
||||
python spider_91porn.py
|
||||
@@ -757,13 +780,15 @@ def main():
|
||||
"日志改走 stderr。配合 backend 边读边下载使用。")
|
||||
|
||||
args, _ = parser.parse_known_args()
|
||||
cli_out = sys.stderr if args.stream_output else sys.stdout
|
||||
prefer_ipv4_for_plain_socks5_proxy()
|
||||
|
||||
print("""
|
||||
================================================
|
||||
91porn 视频爬虫启动中...
|
||||
================================================
|
||||
按 Ctrl+C 可随时中断并保存进度
|
||||
""")
|
||||
""", file=cli_out)
|
||||
|
||||
# 加载已知 ID(来自 backend 的 catalog 已入库列表;兼容旧参数名)
|
||||
seen_viewkeys = []
|
||||
@@ -775,9 +800,9 @@ def main():
|
||||
if line:
|
||||
seen_viewkeys.append(line)
|
||||
except FileNotFoundError:
|
||||
print(f"警告: --seen-viewkeys-file 不存在: {args.seen_viewkeys_file}")
|
||||
print(f"警告: --seen-viewkeys-file 不存在: {args.seen_viewkeys_file}", file=cli_out)
|
||||
except Exception as e:
|
||||
print(f"警告: 读取 --seen-viewkeys-file 失败: {e}")
|
||||
print(f"警告: 读取 --seen-viewkeys-file 失败: {e}", file=cli_out)
|
||||
|
||||
# 决定运行模式
|
||||
if args.target_new is not None:
|
||||
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
# ---- Stage 1: Build frontend ----
|
||||
FROM node:20-slim AS frontend
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tsconfig.json vite.config.ts index.html ./
|
||||
COPY public/ public/
|
||||
COPY src/ src/
|
||||
RUN npm run build
|
||||
|
||||
# ---- Stage 2: Build backend ----
|
||||
FROM golang:1.23-bookworm AS backend
|
||||
|
||||
WORKDIR /app/backend
|
||||
|
||||
COPY backend/go.mod backend/go.sum ./
|
||||
COPY backend/vendor/ vendor/
|
||||
COPY backend/cmd/ cmd/
|
||||
COPY backend/internal/ internal/
|
||||
RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server
|
||||
|
||||
# ---- Stage 3: Runtime ----
|
||||
FROM debian:bookworm-slim AS runtime
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
ffmpeg \
|
||||
openssl \
|
||||
python3 \
|
||||
python3-bs4 \
|
||||
python3-lxml \
|
||||
python3-requests \
|
||||
python3-socks \
|
||||
tar \
|
||||
tzdata \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN python3 -c "import requests, bs4, lxml, socks"
|
||||
|
||||
WORKDIR /opt/video-site-91
|
||||
|
||||
COPY --from=backend /out/server ./server
|
||||
COPY --from=frontend /app/dist ./dist
|
||||
COPY backend/config.example.yaml ./config.example.yaml
|
||||
COPY 91VideoSpider/ ./91VideoSpider/
|
||||
COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
ENV VIDEO_CONFIG=/opt/video-site-91/data/config.yaml \
|
||||
VIDEO_FRONTEND_DIR=/opt/video-site-91/dist \
|
||||
VIDEO_GITHUB_REPO=nianzhibai/91 \
|
||||
VIDEO_IMAGE_VERSION=${VERSION} \
|
||||
VIDEO_LISTEN_PORT=9191 \
|
||||
VIDEO_VERSION_FILE=/opt/video-site-91/data/.version
|
||||
|
||||
RUN chmod +x ./server /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
VOLUME ["/opt/video-site-91/data"]
|
||||
EXPOSE 9191
|
||||
|
||||
ENTRYPOINT ["docker-entrypoint.sh"]
|
||||
CMD ["./server"]
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 nianzhibai
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -1,171 +1,209 @@
|
||||
# 视频聚合站
|
||||
# 91
|
||||
|
||||
把散落在不同网盘里的视频,整理成一个可以自己登录、自己浏览、自己管理的私人视频站。
|
||||
<p align="center">
|
||||
<img width="120" height="120" alt="91" src="https://github.com/user-attachments/assets/5b323c94-bbd3-4dce-bbc8-adc86935b7de" />
|
||||
</p>
|
||||
|
||||
网盘适合存东西,却不适合慢慢看东西。文件多了以后,你很难记住它们在哪里、叫什么、有没有看过、还能不能快速预览。这个项目做的是中间那一层:文件仍然留在原来的网盘里,但你可以用一个更像视频站的界面去搜索、筛选、预览和管理它们。
|
||||
<p align="center">
|
||||
😄 个人私有视频站 😄
|
||||
</p>
|
||||
|
||||
它不是另一个网盘客户端,也不是内容平台。它更像是给你自己的视频收藏做一个入口:安静、集中、可控。
|
||||
<p align="center">
|
||||
<a href="#快速开始">快速开始</a> ·
|
||||
<a href="#功能特性">功能特性</a> ·
|
||||
<a href="#预览图">预览图</a> ·
|
||||
<a href="#数据存放位置">数据目录</a> ·
|
||||
<a href="#许可证">许可证</a>
|
||||
</p>
|
||||
|
||||
## 它能做什么
|
||||
---
|
||||
|
||||
- **统一入口**:把 115、PikPak、夸克、联通沃盘、OneDrive、本地上传和可选的 91 爬虫源放在同一个站里浏览。
|
||||
- **像视频站一样浏览**:首页推荐、最新视频、列表页、搜索、标签筛选、详情播放和相关推荐都已经接好。
|
||||
- **自动生成预览**:后端会用 ffmpeg 在本地生成封面和短 teaser,扫到新视频后不用一条条手动整理。
|
||||
- **保留网盘本身**:视频文件不需要搬家,播放时由后端按来源取链和代理。
|
||||
- **后台可管理**:在管理后台添加网盘、扫描所有网盘、编辑视频信息、维护标签、切换主题。
|
||||
- **首次部署更直接**:第一次访问时会要求设置管理员用户名和密码,设置后保存到本地配置文件。
|
||||
- **适合长期运行**:扫描、预览、隐藏视频、标签归类这些重复工作,都尽量交给系统处理。
|
||||
## 功能特性
|
||||
|
||||
## 适合谁
|
||||
- **多后端支持** — 兼容 115 云盘、PikPak 云盘、123云盘、OneDrive、Google Drive 和本地存储
|
||||
- **低带宽播放** — 115 云盘、PikPak 云盘、123云盘、OneDrive 都支持302模式,在线播放视频时,不占用服务器带宽,播放体验不受服务器带宽影响;Google Drive 不支持302模式,走服务器中转,观看体验会受服务器带宽影响
|
||||
- **封面 & 预览片段** — 自动为每个视频生成封面图和预览片段,首页快速选片
|
||||
- **91 爬虫** — 内置爬虫,支持抓取 91 本月最热视频
|
||||
- **双主题** — 黑黄经典主题 / 粉白清新主题,随时切换
|
||||
- **短视频模式** — 一键切换抖音风格,沉浸刷片
|
||||
- **低资源占用** — 2C2G 服务器稳定运行,主要性能消耗就是封面图和预览视频的生成
|
||||
|
||||
如果你有一批视频散落在多个网盘里,想把它们整理成一个自己的私有站点,这个项目会比较合适。
|
||||
---
|
||||
|
||||
如果你只是想临时播放单个文件,直接用网盘客户端更简单;如果你想做公开视频网站,这个项目也不是为那个场景设计的。它的重点是个人部署、个人管理、个人观看。
|
||||
## 预览图
|
||||
|
||||
## 支持的来源
|
||||
### 电脑端
|
||||
|
||||
- 115 网盘
|
||||
- PikPak
|
||||
- 91 爬虫源
|
||||
- 夸克网盘
|
||||
- 联通沃盘
|
||||
- OneDrive
|
||||
- 本地上传
|
||||
<p>
|
||||
<img width="49%" alt="首页" src="https://github.com/user-attachments/assets/9808fceb-760b-4dd5-b7d2-8622b95b90d5" />
|
||||
<img width="49%" alt="播放页" src="https://github.com/user-attachments/assets/859db4aa-1fba-44f2-bb46-1db07c2f964f" />
|
||||
</p>
|
||||
|
||||
91 爬虫源是一种特殊存储来源,用来把爬虫抓到的视频和封面接入站内目录。它不是必须项;如果你只想管理自己的网盘,可以完全不启用。
|
||||
<p>
|
||||
<img width="49%" alt="主题切换" src="https://github.com/user-attachments/assets/96bea37a-8764-413e-9b70-1856b4ae0cd2" />
|
||||
<img width="49%" alt="管理页" src="https://github.com/user-attachments/assets/29c1e27a-7651-4dfc-93dd-556331844214" />
|
||||
</p>
|
||||
|
||||
### 手机端
|
||||
|
||||
<p align="center">
|
||||
<img width="1284" height="1134" alt="手机端" src="https://github.com/user-attachments/assets/bdb7a86c-a4e5-483e-a307-e02c0bb34dac" />
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
## 快速开始
|
||||
|
||||
需要先准备:
|
||||
|
||||
- Node.js 18+
|
||||
- Go 1.23+
|
||||
- ffmpeg 和 ffprobe
|
||||
|
||||
启动项目:
|
||||
### 方式一:一键安装脚本(推荐)
|
||||
|
||||
```bash
|
||||
npm install
|
||||
./start.sh
|
||||
```
|
||||
|
||||
默认访问地址:
|
||||
|
||||
- 前台:`http://127.0.0.1:9191/`
|
||||
- 后台:`http://127.0.0.1:9191/admin`
|
||||
- 后端:`127.0.0.1:9192`
|
||||
|
||||
第一次打开时,如果还没有设置管理员账号,页面会引导你创建用户名和密码。保存后会写入本地的 `backend/config.yaml`。
|
||||
|
||||
常用命令:
|
||||
|
||||
```bash
|
||||
./start.sh --status
|
||||
./start.sh --restart
|
||||
./start.sh --stop
|
||||
```
|
||||
|
||||
需要前端热更新时:
|
||||
|
||||
```bash
|
||||
FRONTEND_MODE=dev ./start.sh --restart
|
||||
```
|
||||
|
||||
## 新服务器一键安装
|
||||
|
||||
如果你只是想在一台 Ubuntu / Debian 服务器上尽快跑起来,推荐使用预编译安装脚本。普通用户不需要安装 Go、Node.js,也不需要自己编译;脚本会按服务器 CPU 架构下载 GitHub Release 里的预编译包,安装运行依赖,写入 systemd 服务并启动。
|
||||
|
||||
```bash
|
||||
sudo apt update
|
||||
sudo apt install -y curl ca-certificates
|
||||
sudo apt update && sudo apt install -y curl ca-certificates
|
||||
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/install.sh -o install.sh
|
||||
sudo bash install.sh
|
||||
```
|
||||
|
||||
部署完成后访问:
|
||||
|
||||
- 前台:`http://服务器IP:9191/`
|
||||
- 后台:`http://服务器IP:9191/admin`
|
||||
| 地址 | 说明 |
|
||||
|------|------|
|
||||
| `http://服务器IP:9191/` | 前台 |
|
||||
| `http://服务器IP:9191/admin` | 后台管理 |
|
||||
|
||||
第一次打开后台会要求设置管理员用户名和密码。常用维护命令:
|
||||
**注意:如果首次访问,显示502,可以运行 `91 restart` 重启一下服务**
|
||||
|
||||
安装后自动注册 `91` 管理命令:
|
||||
|
||||
```bash
|
||||
sudo bash install.sh status
|
||||
sudo bash install.sh logs
|
||||
sudo bash install.sh update
|
||||
sudo bash install.sh restart
|
||||
sudo bash install.sh stop
|
||||
91 # 打开管理菜单
|
||||
91 status # 查看运行状态
|
||||
91 logs # 查看日志
|
||||
91 update # 更新到最新版本
|
||||
91 restart # 重启服务
|
||||
91 stop # 停止服务
|
||||
```
|
||||
|
||||
安装后会自动创建 `91` 指令,和 OpenList 的管理指令类似:
|
||||
> `video-site-91` 为等效别名,两者可互换使用。
|
||||
|
||||
```bash
|
||||
91 # 打开管理菜单
|
||||
91 status # 查看状态
|
||||
91 logs # 查看日志
|
||||
91 update # 更新
|
||||
91 restart # 重启
|
||||
91 stop # 停止
|
||||
```
|
||||
|
||||
同时也保留 `video-site-91` 作为同等别名。
|
||||
|
||||
想换端口:
|
||||
**自定义端口:**
|
||||
|
||||
```bash
|
||||
FRONTEND_PORT=8080 sudo -E bash install.sh
|
||||
```
|
||||
|
||||
如果服务器还有云厂商安全组,请记得放行对应端口,默认是 `9191/tcp`。
|
||||
**旧版本升级(v0.0.2 之前):**
|
||||
|
||||
如果你是项目维护者,要预先编译发布包:
|
||||
旧版脚本直接执行 `91 update` 可能失败,先执行以下修复命令:
|
||||
|
||||
```bash
|
||||
scripts/build-release.sh
|
||||
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/install.sh -o /tmp/install-91.sh
|
||||
sudo bash /tmp/install-91.sh update
|
||||
```
|
||||
|
||||
它会生成:
|
||||
---
|
||||
|
||||
- `release/video-site-91-linux-amd64.tar.gz`
|
||||
- `release/video-site-91-linux-arm64.tar.gz`
|
||||
### 方式二:Docker Compose 部署
|
||||
|
||||
把这两个文件上传到 GitHub Release 后,`install.sh` 就能自动下载。仓库也带了 GitHub Actions:推送 `v*` 标签时会自动构建并上传这两个 Release 包。
|
||||
**1. 准备目录**
|
||||
|
||||
源码部署仍然保留在 `deploy.sh`,适合你想在服务器上直接 clone、编译和调试时使用。
|
||||
```bash
|
||||
mkdir -p video-site-91 && cd video-site-91
|
||||
```
|
||||
|
||||
## 第一次使用
|
||||
**2. 创建 `docker-compose.yml`**
|
||||
|
||||
1. 打开 `http://127.0.0.1:9191/`,先完成管理员账号设置。
|
||||
2. 进入 `/admin`,在网盘管理里新建一个来源。
|
||||
3. 填入名称和对应凭证,保存。
|
||||
4. 点击“扫描所有网盘”,等待视频入库。
|
||||
5. 回到前台,用首页、搜索、标签和详情页浏览内容。
|
||||
```yaml
|
||||
services:
|
||||
video-site-91:
|
||||
image: ghcr.io/nianzhibai/91:stable
|
||||
container_name: video-site-91
|
||||
ports:
|
||||
- "9191:9191"
|
||||
volumes:
|
||||
- ./data:/opt/video-site-91/data
|
||||
restart: unless-stopped
|
||||
```
|
||||
创建yml文件后运行下面指令
|
||||
```bash
|
||||
docker compose pull
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
## 数据放在哪里
|
||||
如果想固定某个 Release 版本,可以改成明确的 tag,例如:
|
||||
|
||||
项目会把运行数据保存在本地:
|
||||
```yaml
|
||||
image: ghcr.io/nianzhibai/91:v0.0.6
|
||||
```
|
||||
|
||||
- `backend/config.yaml`:本地配置、管理员账号、网盘凭证。
|
||||
- `backend/data/video-site.db`:SQLite 数据库。
|
||||
- `backend/data/previews/`:本地生成的封面和 teaser。
|
||||
或直接拉取仓库内置配置:
|
||||
|
||||
这些文件不应该提交到公开仓库。仓库里的 `backend/config.example.yaml` 只是模板,不应该放真实账号、Cookie、Token 或密码。
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/docker-compose.yml -o docker-compose.yml
|
||||
```
|
||||
|
||||
**3. 启动**
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
**常用命令:**
|
||||
|
||||
```bash
|
||||
docker compose logs -f # 查看日志
|
||||
docker compose pull # 拉取最新正式版 stable 镜像
|
||||
docker compose up -d # 更新并重启
|
||||
```
|
||||
|
||||
> 所有配置、数据库、封面、预览及上传文件均保存在 `./data/` 目录下。
|
||||
|
||||
---
|
||||
|
||||
## 数据存放位置
|
||||
|
||||
### 一键脚本部署
|
||||
|
||||
| 路径 | 内容 |
|
||||
|------|------|
|
||||
| `/opt/video-site-91/config.yaml` | 配置文件、管理员账号、网盘凭证 |
|
||||
| `/opt/video-site-91/data/video-site.db` | SQLite 数据库 |
|
||||
| `/opt/video-site-91/data/previews/` | 封面图和预览片段 |
|
||||
|
||||
### Docker Compose 部署
|
||||
|
||||
| 路径 | 内容 |
|
||||
|------|------|
|
||||
| `./data/config.yaml` | 配置文件、管理员账号、网盘凭证 |
|
||||
| `./data/video-site.db` | SQLite 数据库 |
|
||||
| `./data/previews/` | 封面图和预览片段 |
|
||||
| `./data/uploads/` | 本地上传的视频文件 |
|
||||
| `./data/spider91/` | 91 爬虫抓取的视频文件 |
|
||||
|
||||
---
|
||||
|
||||
## 更多文档
|
||||
|
||||
根目录 README 只保留项目介绍和最短上手路径。更细的实现、接口、网盘字段和部署方式可以看:
|
||||
| 文档 | 内容 |
|
||||
|------|------|
|
||||
| [backend/README.md](backend/README.md) | 后端实现、接口说明、网盘字段 |
|
||||
|
||||
- [backend/README.md](backend/README.md)
|
||||
- [video-site-implementation-plan.md](video-site-implementation-plan.md)
|
||||
---
|
||||
|
||||
## 开发验证
|
||||
## 使用须知
|
||||
|
||||
```bash
|
||||
npm run lint
|
||||
npm test
|
||||
cd backend && go test ./... -count=1
|
||||
```
|
||||
本项目面向**个人私有部署**,请仅接入你有权访问和管理的内容,并遵守对应网盘、站点的服务条款及所在地法律法规。
|
||||
|
||||
## 使用边界
|
||||
> 不对外传播,仅限个人使用。
|
||||
|
||||
这个项目面向个人私有部署。请只接入你有权访问和管理的内容,并遵守对应网盘、站点服务条款以及所在地法律法规。
|
||||
---
|
||||
|
||||
## 许可证
|
||||
|
||||
本项目基于 [MIT License](LICENSE) 开源。
|
||||
|
||||
---
|
||||
|
||||
## 致谢
|
||||
|
||||
- [OpenList](https://github.com/OpenListTeam/OpenList) — 优秀的开源项目
|
||||
- [LinuxDo](https://linux.do/) — 学 AI 上 L 站
|
||||
- [NodeSeek](https://nodeseek.com/) — MJJ 上 N 站
|
||||
|
||||
+41
-18
@@ -2,8 +2,8 @@
|
||||
|
||||
视频聚合站的 Go 后端。提供三件事:
|
||||
|
||||
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive)
|
||||
2. 视频元数据目录(SQLite)+ 扫描 + teaser 预生成
|
||||
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive / Google Drive / 本地存储)
|
||||
2. 视频元数据目录(SQLite)+ 扫描 + 预览视频预生成
|
||||
3. REST API(前台)+ 管理后台 + 直链代理
|
||||
4. 标签池、视频隐藏、按网盘统计和详情页来源网盘类型展示能力
|
||||
|
||||
@@ -21,8 +21,10 @@ internal/
|
||||
pikpak/ PikPak(自己实现,参考 OpenList pikpak)
|
||||
wopan/ 联通沃盘(壳子 + OpenListTeam/wopan-sdk-go)
|
||||
onedrive/ OneDrive(OpenList 在线续期 + Microsoft Graph 文件接口)
|
||||
googledrive/ Google Drive(OpenList 在线续期 + Google Drive API;播放走后端代理)
|
||||
localstorage/ 本地目录扫描(服务器已有视频目录)
|
||||
scanner/ 扫目录 → 落库
|
||||
preview/ ffmpeg 抽封面和生成多段 teaser
|
||||
preview/ ffmpeg 抽封面和生成多段预览视频
|
||||
proxy/ /p/stream/*、/p/preview/* 代理
|
||||
auth/ 管理员 session
|
||||
api/ REST 路由
|
||||
@@ -79,7 +81,7 @@ go run ./cmd/server 后端 9192
|
||||
|
||||
## 添加一个盘
|
||||
|
||||
推荐在前端管理后台 `/admin/drives` 新增网盘。保存后会立即挂载并触发扫描;视频结果可在 `/admin/videos` 按网盘查看,每页 100 条,页面会同时显示各网盘 Teaser 已生成、待生成、失败数量。
|
||||
推荐在前端管理后台 `/admin/drives` 新增网盘。保存后会立即挂载并触发扫描;视频结果可在 `/admin/videos` 按网盘查看,每页 100 条,页面会同时显示各网盘预览视频已生成、待生成、失败数量。
|
||||
|
||||
也可以直接调用后端接口:
|
||||
|
||||
@@ -91,7 +93,6 @@ go run ./cmd/server 后端 9192
|
||||
"kind": "quark",
|
||||
"name": "我的夸克盘",
|
||||
"rootId": "0",
|
||||
"scanRootId": "0",
|
||||
"credentials": {
|
||||
"cookie": "粘贴浏览器 F12 复制的 pan.quark.cn Cookie"
|
||||
}
|
||||
@@ -105,9 +106,11 @@ go run ./cmd/server 后端 9192
|
||||
|--------|---------------------------------------------------------------|
|
||||
| quark | `cookie` |
|
||||
| p115 | `cookie`(形如 `UID=...; CID=...; SEID=...; KID=...`) |
|
||||
| pikpak | `username`、`password`,可选 `refresh_token`、`captcha_token`、`device_id`、`platform`、`disable_media_link` |
|
||||
| pikpak | `username`、`password`(token、验证码和设备 ID 由服务端自动处理并保存) |
|
||||
| wopan | `access_token`、`refresh_token`,可选 `family_id` |
|
||||
| onedrive | `refresh_token`,可选 `access_token`、`api_url_address`、`region`、`is_sharepoint`、`site_id` |
|
||||
| onedrive | `refresh_token` |
|
||||
| googledrive | `refresh_token` |
|
||||
| localstorage | `path`(服务器上的已有视频目录,如 `/mnt/videos`) |
|
||||
|
||||
### PikPak 速度说明
|
||||
|
||||
@@ -115,29 +118,49 @@ go run ./cmd/server 后端 9192
|
||||
|
||||
当前服务器同时存在 sing-box TUN 透明代理,PikPak 默认出站会被 `tun0` 接管;但强制直连物理网卡并没有更快,慢速的主要差异来自 PikPak 取链方式。media/cache CDN 节点仍有波动,偶尔可能遇到慢节点;如果播放变慢,可重新获取直链或重新挂载 PikPak 后再测。
|
||||
|
||||
OneDrive 按 OpenList 默认方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。OpenList 代刷得到的 refresh token 可以直接填到本项目。普通 OneDrive 的 `rootId` / `scanRootId` 可填 `root`;SharePoint 文档库需要额外设置 `is_sharepoint=true` 和 `site_id`。
|
||||
OneDrive 按 OpenList 默认应用方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。后台新建 OneDrive 时只需要填 OpenList 代刷得到的 `refresh_token`;服务端会默认挂载根目录并自动回写新 token。
|
||||
|
||||
Google Drive 按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/renewapi` 刷新 token。后台新建 Google Drive 时只需要填 OpenList Google Drive 获取到的 `refresh_token`。Google Drive 下载地址必须携带 `Authorization` 头,浏览器不能直接 302 使用,所以本站会由后端代理 `/p/stream` 播放,不加入零带宽 302 白名单。
|
||||
|
||||
## 文件名约定
|
||||
|
||||
扫描器按以下顺序解析文件名:
|
||||
扫描器按以下顺序解析文件名,用于提取标题和作者:
|
||||
|
||||
1. `[tag1,tag2] 标题 - 作者.mp4`
|
||||
2. `[tag1,tag2] 标题.mp4`
|
||||
1. `[前缀] 标题 - 作者.mp4`
|
||||
2. `[前缀] 标题.mp4`
|
||||
3. `标题 - 作者.mp4`
|
||||
4. `标题.mp4`
|
||||
|
||||
标签分隔符支持 `, , 、` 和空格。解析结果会和系统标签池匹配,常见番号类噪声会归并到 `AV` 等系统标签,避免把每个番号都变成独立标签。解析结果可在管理后台覆盖。
|
||||
开头的 `[前缀]` 只会从标题里剥离,不会按分隔符作为任意标签入库。视频标签来自三类规则:
|
||||
|
||||
1. 文件名、作者和目录名命中系统标签或已有标签的标签名 / 别名。
|
||||
2. 符合条件的目录名会自动创建 `collection` 合集标签,并给同目录视频打上该标签。
|
||||
3. 常见番号类噪声会统一归并到 `AV`,避免把每个番号都变成独立标签。
|
||||
|
||||
当前内置系统标签为:`后入`、`奶子`、`口交`、`臀`、`人妻`、`女大`、`AV`。解析结果可在管理后台覆盖;手动保存后,该视频会标记为人工标签,后续扫描不会再自动覆盖。
|
||||
|
||||
## 视频去重
|
||||
|
||||
项目有三层去重:
|
||||
|
||||
1. 同一网盘同一文件按 `(drive_id, file_id)` 形成稳定视频 ID,重复扫描只更新同一行。
|
||||
2. 扫描时优先按网盘侧 `content_hash` 去重;没有 hash 时退化为 `file_name + size_bytes`。
|
||||
3. 扫描、爬虫、本地上传或服务启动挂载网盘后,后台指纹 worker 会异步读取视频的少量 Range 片段,生成 `sampled_sha256`。前台列表、首页、搜索、推荐会按 `size_bytes + sampled_sha256` 只展示最早入库的 canonical 视频。
|
||||
|
||||
`sampled_sha256` 是文件级去重:适合识别同一个视频文件被复制到 115 / PikPak / OneDrive 等不同网盘的情况。它不会删除任何网盘文件,也不用于识别转码、裁剪、加水印后的同源视频。
|
||||
|
||||
封面和预览视频仍然优先生成,不等待指纹完成。夜间流水线最后会做一次重复资产清理:对 `size_bytes + sampled_sha256` 命中的非 canonical 视频,只删除本机生成的重复封面和预览视频,并把对应字段重置为 `pending`。网盘原文件和视频元数据记录不会被删除;如果 canonical 视频以后被移除,这些重复项会重新进入生成队列。
|
||||
|
||||
## 管理能力
|
||||
|
||||
- `/admin/drives`:新增、编辑、删除网盘,触发扫描。
|
||||
- `/admin/videos`:按网盘筛选视频,每页 100 条分页,查看各网盘 Teaser 统计,编辑标题/作者/分类/标签,单条或全量重生 teaser。
|
||||
- `/admin/tags`:新增标签并用内置规则自动匹配已有视频。
|
||||
- `/admin/videos`:按网盘筛选视频,每页 100 条分页,查看各网盘预览视频统计,编辑标题/作者/分类/标签,单条或全量重生预览视频。
|
||||
- `/admin/tags`:新增标签并用内置规则自动匹配已有视频;删除非系统标签时会从所有视频上同步移除该标签。
|
||||
- 播放页视频信息会展示来源网盘类型;同时提供“不再展示”,点击后会把视频标记为全局隐藏。隐藏视频不会再出现在首页、列表、搜索、相关推荐和详情接口中。目前没有管理后台恢复入口,如需恢复可把数据库里对应视频的 `hidden` 字段改回 `0`。
|
||||
|
||||
## Teaser 生成
|
||||
## 预览视频生成
|
||||
|
||||
scanner 扫到新视频会把 `(driveID, videoID)` 丢进 worker 队列。worker 会先用 `ffprobe` 探测时长,再用 `ffmpeg` 抽封面和生成无声 teaser:
|
||||
scanner 扫到新视频会把 `(driveID, videoID)` 丢进 worker 队列。worker 会先用 `ffprobe` 探测时长,再用 `ffmpeg` 抽封面和生成无声预览视频:
|
||||
|
||||
```
|
||||
ffmpeg -ss <起点> -headers "UA/Cookie/Referer" -i <直链> \
|
||||
@@ -145,9 +168,9 @@ ffmpeg -ss <起点> -headers "UA/Cookie/Referer" -i <直链> \
|
||||
-movflags +faststart -y <local>.mp4
|
||||
```
|
||||
|
||||
当前策略是每段固定 3 秒;30 秒以下最多 3 段,30 秒及以上固定 4 段;长视频在 20% 到 80% 区间均匀取段。生成的 teaser 和封面都只保存在本地 `data/previews/`,不会回写到网盘;旧数据中的 `preview_file_id` 会被忽略。
|
||||
当前策略是每段固定 3 秒;30 秒以下最多 3 段,30 秒及以上固定 4 段;长视频在 20% 到 80% 区间均匀取段。生成的预览视频和封面都只保存在本地 `data/previews/`,不会回写到网盘;旧数据中的 `preview_file_id` 会被忽略。
|
||||
|
||||
服务启动或网盘重新挂载时,如果 Teaser 开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 直链生成 teaser 时可能触发 Microsoft 429 限流;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。
|
||||
服务启动或网盘重新挂载时,如果预览视频开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 扫盘和直链生成预览视频 / 封面时可能触发 Microsoft Graph 429、`TooManyRequests`、`activityLimitReached` 或 throttled 文本;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。扫盘阶段会按 `Retry-After` 或默认冷却时间等待后继续当前目录。
|
||||
|
||||
前端卡片的 `previewSrc` 统一指向 `/p/preview/<videoID>`,后端只从本地 `preview_local` 文件读取。
|
||||
|
||||
|
||||
@@ -67,3 +67,40 @@ func TestFrontendHandlerDoesNotSwallowBackendRoutes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveFrontendDirFallsBackToParentDist(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
backendDir := filepath.Join(workspace, "backend")
|
||||
distDir := filepath.Join(workspace, "dist")
|
||||
if err := os.MkdirAll(backendDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir backend: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(distDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir dist: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(distDir, "index.html"), []byte("<html>app</html>"), 0o644); err != nil {
|
||||
t.Fatalf("write index: %v", err)
|
||||
}
|
||||
|
||||
oldWD, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("getwd: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := os.Chdir(oldWD); err != nil {
|
||||
t.Fatalf("restore wd: %v", err)
|
||||
}
|
||||
})
|
||||
t.Setenv("VIDEO_FRONTEND_DIR", "")
|
||||
if err := os.Chdir(backendDir); err != nil {
|
||||
t.Fatalf("chdir backend: %v", err)
|
||||
}
|
||||
|
||||
got, ok := resolveFrontendDir()
|
||||
if !ok {
|
||||
t.Fatal("resolveFrontendDir ok = false, want true")
|
||||
}
|
||||
if got != "../dist" {
|
||||
t.Fatalf("frontend dir = %q, want ../dist", got)
|
||||
}
|
||||
}
|
||||
|
||||
+1302
-227
File diff suppressed because it is too large
Load Diff
@@ -1,9 +1,13 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
)
|
||||
|
||||
func TestSpider91IntCredFallbacks(t *testing.T) {
|
||||
@@ -30,3 +34,62 @@ func TestSpider91IntCredFallbacks(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
|
||||
reg := proxy.NewRegistry()
|
||||
reg.Set("p115-one", &spider91UploadTargetFakeDrive{id: "p115-one", kind: "p115"})
|
||||
reg.Set("p123-one", &spider91UploadTargetFakeDrive{id: "p123-one", kind: "p123"})
|
||||
reg.Set("onedrive-one", &spider91UploadTargetFakeDrive{id: "onedrive-one", kind: "onedrive"})
|
||||
|
||||
app := &App{registry: reg}
|
||||
if got := app.Spider91UploadDriveID(); got != "" {
|
||||
t.Fatalf("empty upload target selected %q, want local-only empty target", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "p115-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "p115-one" {
|
||||
t.Fatalf("explicit upload target = %q, want p115-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "p123-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "p123-one" {
|
||||
t.Fatalf("explicit p123 upload target = %q, want p123-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "onedrive-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "onedrive-one" {
|
||||
t.Fatalf("explicit onedrive upload target = %q, want onedrive-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "missing"
|
||||
if got := app.Spider91UploadDriveID(); got != "" {
|
||||
t.Fatalf("missing upload target = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
type spider91UploadTargetFakeDrive struct {
|
||||
id string
|
||||
kind string
|
||||
}
|
||||
|
||||
func (d *spider91UploadTargetFakeDrive) Kind() string { return d.kind }
|
||||
func (d *spider91UploadTargetFakeDrive) ID() string { return d.id }
|
||||
func (d *spider91UploadTargetFakeDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) RootID() string { return "root" }
|
||||
|
||||
+1007
-37
File diff suppressed because it is too large
Load Diff
+24
-10
@@ -22,7 +22,7 @@ server:
|
||||
storage:
|
||||
# SQLite 数据库文件路径
|
||||
db_path: "./data/video-site.db"
|
||||
# 本地 teaser 和封面目录
|
||||
# 本地预览视频和封面目录
|
||||
local_preview_dir: "./data/previews"
|
||||
|
||||
scanner:
|
||||
@@ -33,33 +33,33 @@ scanner:
|
||||
# 单次扫描每家网盘目录递归层数上限
|
||||
max_depth: 5
|
||||
# 被扫描的扩展名
|
||||
video_extensions: [".mp4", ".mkv", ".mov", ".webm", ".avi"]
|
||||
video_extensions: [".mp4", ".mkv", ".mov", ".webm", ".avi", ".strm"]
|
||||
|
||||
nightly:
|
||||
# 凌晨流水线触发整点(0-23),默认 1 即每天 01:00。流程:
|
||||
# Phase 1 扫所有非 spider91 / 非 localupload 网盘 → 检测新增 / 删除
|
||||
# → 入队封面和 teaser → 等所有队列 idle
|
||||
# Phase 2 spider91 爬虫(如配置)→ 入队 teaser → 等队列 idle
|
||||
# → 入队封面和预览视频 → 等所有队列 idle
|
||||
# Phase 2 spider91 爬虫(如配置)→ 入队预览视频 → 等队列 idle
|
||||
# Phase 3 spider91 → 云盘迁移(一次性 sweep)
|
||||
cron_hour: 1
|
||||
# 单次流水线总耗时上限(软超时);超过后当前 phase 跑完不启动后续 phase。
|
||||
max_duration: 6h
|
||||
|
||||
preview:
|
||||
# 是否启用 ffmpeg 抽帧生成 teaser
|
||||
# 是否启用 ffmpeg 抽帧生成预览视频
|
||||
enabled: true
|
||||
# ffmpeg / ffprobe 可执行文件名或绝对路径
|
||||
ffmpeg_path: "ffmpeg"
|
||||
ffprobe_path: "ffprobe"
|
||||
# teaser 每段时长(秒),实际生成时每段最多 3 秒
|
||||
# 预览视频每段时长(秒),实际生成时每段最多 3 秒
|
||||
duration_seconds: 3
|
||||
# 兼容旧配置;当前 30 秒以下最多 3 段,30 秒及以上固定 4 段
|
||||
segments: 3
|
||||
# teaser 视频宽度
|
||||
# 预览视频宽度
|
||||
width: 480
|
||||
|
||||
# 盘列表。上线后请通过管理后台添加,本文件可留空。
|
||||
# kind 支持 quark / p115 / pikpak / wopan / onedrive。
|
||||
# kind 支持 quark / p115 / p123 / pikpak / wopan / onedrive / googledrive / localstorage。
|
||||
# OneDrive 示例:
|
||||
# - id: "my-onedrive"
|
||||
# kind: "onedrive"
|
||||
@@ -67,6 +67,20 @@ preview:
|
||||
# root_id: "root"
|
||||
# params:
|
||||
# refresh_token: "..."
|
||||
# api_url_address: "https://api.oplist.org/onedrive/renewapi"
|
||||
# region: "global"
|
||||
# Google Drive 示例:
|
||||
# - id: "my-google"
|
||||
# kind: "googledrive"
|
||||
# name: "我的 Google Drive"
|
||||
# root_id: "root"
|
||||
# params:
|
||||
# refresh_token: "..."
|
||||
# 本地存储示例:
|
||||
# - id: "local-media"
|
||||
# kind: "localstorage"
|
||||
# name: "本地视频目录"
|
||||
# root_id: "/"
|
||||
# params:
|
||||
# # Docker 部署时这里和 .strm 里的绝对路径都必须使用容器内路径。
|
||||
# # 例如宿主机 /mnt/videos 挂载为 /media,就填写 /media。
|
||||
# path: "/mnt/videos"
|
||||
drives: []
|
||||
|
||||
+4
-4
@@ -7,15 +7,18 @@ toolchain go1.23.4
|
||||
require (
|
||||
github.com/OpenListTeam/wopan-sdk-go v0.2.0
|
||||
github.com/SheltonZhu/115driver v1.3.2
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
|
||||
github.com/go-chi/chi/v5 v5.1.0
|
||||
github.com/go-resty/resty/v2 v2.14.0
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
|
||||
golang.org/x/net v0.27.0
|
||||
golang.org/x/sys v0.30.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
modernc.org/sqlite v1.33.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aead/ecdh v0.2.0 // indirect
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect
|
||||
github.com/andreburgaud/crypt2go v1.1.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
@@ -26,10 +29,7 @@ require (
|
||||
github.com/pierrec/lz4/v4 v4.1.17 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
|
||||
golang.org/x/crypto v0.25.0 // indirect
|
||||
golang.org/x/net v0.27.0 // indirect
|
||||
golang.org/x/sys v0.30.0 // indirect
|
||||
golang.org/x/time v0.8.0 // indirect
|
||||
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
|
||||
modernc.org/libc v1.55.3 // indirect
|
||||
|
||||
+443
-46
@@ -5,53 +5,80 @@ import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/video-site/backend/internal/auth"
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives/p123"
|
||||
)
|
||||
|
||||
type AdminServer struct {
|
||||
Catalog *catalog.Catalog
|
||||
Auth *auth.Authenticator
|
||||
// VersionFilePath points to the installer-written .version file.
|
||||
VersionFilePath string
|
||||
// ImageVersion is the Docker image version injected at build/runtime.
|
||||
// It takes precedence over VersionFilePath because Docker data volumes can
|
||||
// keep an older .version file across image upgrades.
|
||||
ImageVersion string
|
||||
// GitHubRepo is the owner/name repo used for update checks.
|
||||
GitHubRepo string
|
||||
// ReleaseAPIURL and HTTPClient are injectable for tests. Production code leaves them empty.
|
||||
ReleaseAPIURL string
|
||||
HTTPClient *http.Client
|
||||
// SetupRequired 表示当前是否仍处于首次部署初始化状态。
|
||||
SetupRequired func() bool
|
||||
// OnSetup 持久化首次部署时设置的管理员账号密码,并更新运行中认证器。
|
||||
OnSetup func(username, password string) error
|
||||
// LocalPreviewDir is the local directory that stores generated teasers and thumbs.
|
||||
// LocalPreviewDir is the local directory that stores generated preview videos and thumbs.
|
||||
LocalPreviewDir string
|
||||
// Hooks:外层注入实际执行者
|
||||
OnDriveSaved func(driveID string) error
|
||||
OnDriveDeleteCleanup func(ctx context.Context, driveID string) (int, error)
|
||||
OnDriveRemoved func(driveID string)
|
||||
OnScanRequested func(driveID string)
|
||||
OnStopDriveTasks func(driveID string) bool
|
||||
OnStopAllTasks func() int
|
||||
OnRegenPreview func(videoID string)
|
||||
OnRegenAllPreviews func()
|
||||
OnRegenFailedPreviews func(driveID string)
|
||||
OnRegenFailedThumbnails func(driveID string)
|
||||
OnRegenFailedFingerprints func(driveID string)
|
||||
OnDeleteVideo func(ctx context.Context, videoID string) (DeleteVideoResult, error)
|
||||
GetDriveGenerationStatuses func() map[string]DriveGenerationStatuses
|
||||
// OnTeaserEnabledChanged 在 per-drive teaser 开关被切换后调用。
|
||||
// enabled=true 时上层应该重新把 pending teaser 入队(类似旧的全局开关从关到开);
|
||||
// OnTeaserEnabledChanged 在 per-drive 预览视频开关被切换后调用。
|
||||
// enabled=true 时上层应该重新把 pending 预览视频入队(类似旧的全局开关从关到开);
|
||||
// enabled=false 时通常不用做事 —— worker 入队前会再次查 catalog,自然停止。
|
||||
OnTeaserEnabledChanged func(driveID string, enabled bool)
|
||||
// Theme 读写("dark" | "pink")
|
||||
GetTheme func() string
|
||||
SetTheme func(theme string) error
|
||||
// Spider91 → PikPak 上传目标 drive ID 读写
|
||||
// Spider91 → 115/123/PikPak/OneDrive 上传目标 drive ID 读写
|
||||
GetSpider91UploadDriveID func() string
|
||||
SetSpider91UploadDriveID func(driveID string) error
|
||||
// OnRunNightlyJob 触发一次完整的凌晨流水线(Phase1 扫盘 + Phase2 91 爬虫 +
|
||||
// Phase3 迁移)。立即返回 —— 实际任务在后台跑,admin 在日志或下次状态查询里
|
||||
// 看进度。重复点击会被 Runner.TryLock 丢弃。
|
||||
OnRunNightlyJob func()
|
||||
// 看进度。若流水线正在跑或已排队,Runner 会拒绝重复触发。
|
||||
OnRunNightlyJob func() bool
|
||||
// GetNightlyJobStatus 返回凌晨流水线当前状态,用于前端禁用重复触发按钮。
|
||||
GetNightlyJobStatus func() NightlyJobStatus
|
||||
// ListDriveDirChildren 列出某个 drive 在 parentID 目录下的直接子目录。
|
||||
// parentID 为空时使用 drive 的 RootID。返回 (子目录列表, error)。
|
||||
// 用于"设置跳过目录"弹窗按需展开浏览网盘目录树;只返回目录条目,文件忽略。
|
||||
// 调用方应当处理 error 并以 5xx 返回前端。
|
||||
ListDriveDirChildren func(ctx context.Context, driveID, parentID string) ([]DriveDirEntry, error)
|
||||
// 123 云盘扫码登录接口测试注入;生产留空走官方 user.123pan.cn。
|
||||
P123UserAPIBaseURL string
|
||||
P123HTTPClient *http.Client
|
||||
}
|
||||
|
||||
// DriveDirEntry 是 dirtree 接口的一条返回项:网盘上的一个目录节点。
|
||||
@@ -68,8 +95,22 @@ type GenerationStatus struct {
|
||||
}
|
||||
|
||||
type DriveGenerationStatuses struct {
|
||||
Thumbnail GenerationStatus `json:"thumbnail"`
|
||||
Preview GenerationStatus `json:"preview"`
|
||||
Thumbnail GenerationStatus `json:"thumbnail"`
|
||||
Preview GenerationStatus `json:"preview"`
|
||||
Fingerprint GenerationStatus `json:"fingerprint"`
|
||||
}
|
||||
|
||||
type NightlyJobStatus struct {
|
||||
State string `json:"state"`
|
||||
Running bool `json:"running"`
|
||||
Queued bool `json:"queued"`
|
||||
StartedAt string `json:"startedAt,omitempty"`
|
||||
LastFinishedAt string `json:"lastFinishedAt,omitempty"`
|
||||
}
|
||||
|
||||
type DeleteVideoResult struct {
|
||||
OK bool `json:"ok"`
|
||||
DeletedSource bool `json:"deletedSource"`
|
||||
}
|
||||
|
||||
func (a *AdminServer) Register(r chi.Router) {
|
||||
@@ -89,34 +130,56 @@ func (a *AdminServer) Register(r chi.Router) {
|
||||
r.Get("/drives", a.handleListDrives)
|
||||
r.Get("/drives/storage", a.handleDriveStorage)
|
||||
r.Post("/drives", a.handleUpsertDrive)
|
||||
r.Post("/drives/p123/qr", a.handleP123QRStart)
|
||||
r.Get("/drives/p123/qr/{uniID}", a.handleP123QRStatus)
|
||||
r.Delete("/drives/{id}", a.handleDeleteDrive)
|
||||
r.Post("/drives/{id}/rescan", a.handleRescan)
|
||||
r.Post("/drives/{id}/tasks/stop", a.handleStopDriveTasks)
|
||||
r.Post("/drives/{id}/teaser-enabled", a.handleSetDriveTeaserEnabled)
|
||||
r.Post("/drives/{id}/skip-dirs", a.handleSetDriveSkipDirs)
|
||||
r.Get("/drives/{id}/dirtree", a.handleListDriveDirTree)
|
||||
r.Post("/drives/{id}/previews/failed/regenerate", a.handleRegenFailedPreviews)
|
||||
r.Post("/drives/{id}/thumbnails/failed/regenerate", a.handleRegenFailedThumbnails)
|
||||
r.Post("/drives/{id}/fingerprints/failed/regenerate", a.handleRegenFailedFingerprints)
|
||||
|
||||
// 视频
|
||||
r.Get("/videos", a.handleAdminListVideos)
|
||||
r.Put("/videos/{id}", a.handleUpdateVideo)
|
||||
r.Delete("/videos/{id}", a.handleDeleteVideo)
|
||||
r.Post("/videos/regen-preview", a.handleRegenAllPreviews)
|
||||
r.Post("/videos/{id}/regen-preview", a.handleRegenPreview)
|
||||
|
||||
// 标签
|
||||
r.Get("/tags", a.handleListTags)
|
||||
r.Post("/tags", a.handleCreateTag)
|
||||
r.Delete("/tags/{id}", a.handleDeleteTag)
|
||||
|
||||
// 运行时设置
|
||||
r.Get("/settings", a.handleGetSettings)
|
||||
r.Put("/settings", a.handlePutSettings)
|
||||
|
||||
// 运维任务
|
||||
r.Get("/update/check", a.handleCheckUpdate)
|
||||
r.Get("/jobs/nightly/status", a.handleNightlyJobStatus)
|
||||
r.Post("/jobs/nightly/run", a.handleRunNightlyJob)
|
||||
r.Post("/tasks/stop", a.handleStopAllTasks)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
type updateCheckDTO struct {
|
||||
CurrentVersion string `json:"currentVersion"`
|
||||
LatestVersion string `json:"latestVersion"`
|
||||
HasUpdate bool `json:"hasUpdate"`
|
||||
ReleaseURL string `json:"releaseUrl,omitempty"`
|
||||
CheckedAt string `json:"checkedAt"`
|
||||
}
|
||||
|
||||
type githubReleaseDTO struct {
|
||||
TagName string `json:"tag_name"`
|
||||
HTMLURL string `json:"html_url"`
|
||||
}
|
||||
|
||||
type loginReq struct {
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
@@ -221,6 +284,94 @@ func (a *AdminServer) handleMe(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"authenticated": ok})
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleCheckUpdate(w http.ResponseWriter, r *http.Request) {
|
||||
info, err := a.checkUpdate(r.Context())
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusBadGateway, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, http.StatusOK, info)
|
||||
}
|
||||
|
||||
func (a *AdminServer) checkUpdate(ctx context.Context) (updateCheckDTO, error) {
|
||||
current := a.installedVersion()
|
||||
if current == "" {
|
||||
current = "unknown"
|
||||
}
|
||||
release, err := a.latestRelease(ctx)
|
||||
if err != nil {
|
||||
return updateCheckDTO{
|
||||
CurrentVersion: current,
|
||||
CheckedAt: time.Now().Format(time.RFC3339),
|
||||
}, err
|
||||
}
|
||||
latest := strings.TrimSpace(release.TagName)
|
||||
return updateCheckDTO{
|
||||
CurrentVersion: current,
|
||||
LatestVersion: latest,
|
||||
HasUpdate: current != "unknown" && latest != "" && current != latest,
|
||||
ReleaseURL: release.HTMLURL,
|
||||
CheckedAt: time.Now().Format(time.RFC3339),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *AdminServer) installedVersion() string {
|
||||
if version := strings.TrimSpace(a.ImageVersion); version != "" {
|
||||
return version
|
||||
}
|
||||
path := strings.TrimSpace(a.VersionFilePath)
|
||||
if path == "" {
|
||||
path = ".version"
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
lines := strings.Split(strings.ReplaceAll(string(data), "\r\n", "\n"), "\n")
|
||||
if len(lines) == 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(lines[0])
|
||||
}
|
||||
|
||||
func (a *AdminServer) latestRelease(ctx context.Context) (githubReleaseDTO, error) {
|
||||
url := strings.TrimSpace(a.ReleaseAPIURL)
|
||||
if url == "" {
|
||||
repo := strings.TrimSpace(a.GitHubRepo)
|
||||
if repo == "" {
|
||||
repo = "nianzhibai/91"
|
||||
}
|
||||
url = "https://api.github.com/repos/" + repo + "/releases/latest"
|
||||
}
|
||||
client := a.HTTPClient
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 8 * time.Second}
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return githubReleaseDTO{}, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/vnd.github+json")
|
||||
req.Header.Set("User-Agent", "video-site-91")
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return githubReleaseDTO{}, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
return githubReleaseDTO{}, fmt.Errorf("github release check failed: HTTP %d", res.StatusCode)
|
||||
}
|
||||
var release githubReleaseDTO
|
||||
if err := json.NewDecoder(res.Body).Decode(&release); err != nil {
|
||||
return githubReleaseDTO{}, err
|
||||
}
|
||||
if strings.TrimSpace(release.TagName) == "" {
|
||||
return githubReleaseDTO{}, errors.New("github release check returned empty tag")
|
||||
}
|
||||
return release, nil
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
drives, err := a.Catalog.ListDrives(r.Context())
|
||||
if err != nil {
|
||||
@@ -237,6 +388,11 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
fingerprintCounts, err := a.Catalog.CountFingerprintsByDrive(r.Context())
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
generationStatuses := map[string]DriveGenerationStatuses{}
|
||||
if a.GetDriveGenerationStatuses != nil {
|
||||
generationStatuses = a.GetDriveGenerationStatuses()
|
||||
@@ -251,7 +407,7 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
Status string `json:"status"`
|
||||
LastError string `json:"lastError,omitempty"`
|
||||
HasCredential bool `json:"hasCredential"`
|
||||
// TeaserEnabled 控制是否给本盘生成 teaser/封面。前端用它在网盘列表/编辑表单展示开关状态。
|
||||
// TeaserEnabled 控制是否给本盘生成预览视频/封面。前端用它在网盘列表/编辑表单展示开关状态。
|
||||
TeaserEnabled bool `json:"teaserEnabled"`
|
||||
// SkipDirIDs 是用户在 admin 配置的"扫描跳过目录"集合(drive 侧目录 fileID)。
|
||||
// 前端用它在"设置跳过目录"弹窗里回显已选项;JSON 字段名 camelCase 与
|
||||
@@ -259,20 +415,27 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
SkipDirIDs []string `json:"skipDirIds"`
|
||||
// LastCrawlAt 是 spider91 上次成功爬取的 unix 秒(来自 credentials.last_crawl_at)。
|
||||
// 其它 kind 留 0;前端用它显示"上次抓取: N 小时前"。
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
Spider91Proxy string `json:"spider91Proxy,omitempty"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
ThumbnailDurationPendingCount int `json:"thumbnailDurationPendingCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
FingerprintReadyCount int `json:"fingerprintReadyCount"`
|
||||
FingerprintPendingCount int `json:"fingerprintPendingCount"`
|
||||
FingerprintFailedCount int `json:"fingerprintFailedCount"`
|
||||
}
|
||||
list := make([]out, 0, len(drives))
|
||||
for _, d := range drives {
|
||||
counts := teaserCounts[d.ID]
|
||||
thumbCounts := thumbnailCounts[d.ID]
|
||||
fingerprintCount := fingerprintCounts[d.ID]
|
||||
generation := generationStatuses[d.ID]
|
||||
if generation.Thumbnail.State == "" {
|
||||
generation.Thumbnail.State = "idle"
|
||||
@@ -280,6 +443,9 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
if generation.Preview.State == "" {
|
||||
generation.Preview.State = "idle"
|
||||
}
|
||||
if generation.Fingerprint.State == "" {
|
||||
generation.Fingerprint.State = "idle"
|
||||
}
|
||||
// spider91 没有用户凭证概念;只要存在 drive 行就视为"已配置"。
|
||||
// last_crawl_at 是后端自动写入的运行状态字段,不计入 hasCredential 判定。
|
||||
hasCred := false
|
||||
@@ -305,31 +471,38 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
ID: d.ID, Kind: d.Kind, Name: d.Name,
|
||||
RootID: d.RootID, ScanRootID: d.ScanRootID,
|
||||
Status: d.Status, LastError: d.LastError,
|
||||
HasCredential: hasCred,
|
||||
TeaserEnabled: d.TeaserEnabled,
|
||||
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
|
||||
LastCrawlAt: lastCrawlAt,
|
||||
ThumbnailGenerationStatus: generation.Thumbnail,
|
||||
PreviewGenerationStatus: generation.Preview,
|
||||
ThumbnailReadyCount: thumbCounts.Ready,
|
||||
ThumbnailPendingCount: thumbCounts.Pending,
|
||||
ThumbnailFailedCount: thumbCounts.Failed,
|
||||
TeaserReadyCount: counts.Ready,
|
||||
TeaserPendingCount: counts.Pending,
|
||||
TeaserFailedCount: counts.Failed,
|
||||
HasCredential: hasCred,
|
||||
TeaserEnabled: d.TeaserEnabled,
|
||||
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
|
||||
Spider91Proxy: spider91ProxyForDrive(d),
|
||||
LastCrawlAt: lastCrawlAt,
|
||||
ThumbnailGenerationStatus: generation.Thumbnail,
|
||||
PreviewGenerationStatus: generation.Preview,
|
||||
FingerprintGenerationStatus: generation.Fingerprint,
|
||||
ThumbnailReadyCount: thumbCounts.Ready,
|
||||
ThumbnailPendingCount: thumbCounts.Pending,
|
||||
ThumbnailFailedCount: thumbCounts.Failed,
|
||||
ThumbnailDurationPendingCount: thumbCounts.DurationPending,
|
||||
TeaserReadyCount: counts.Ready,
|
||||
TeaserPendingCount: counts.Pending,
|
||||
TeaserFailedCount: counts.Failed,
|
||||
FingerprintReadyCount: fingerprintCount.Ready,
|
||||
FingerprintPendingCount: fingerprintCount.Pending,
|
||||
FingerprintFailedCount: fingerprintCount.Failed,
|
||||
})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, list)
|
||||
}
|
||||
|
||||
type upsertDriveReq struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
RootID string `json:"rootId"`
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
RootID string `json:"rootId"`
|
||||
// Deprecated: 扫描起点已固定为 rootId;保留字段只为兼容旧客户端请求体。
|
||||
ScanRootID string `json:"scanRootId"`
|
||||
Credentials map[string]string `json:"credentials"`
|
||||
// TeaserEnabled 是 per-drive teaser/封面生成开关。
|
||||
// TeaserEnabled 是 per-drive 预览视频/封面生成开关。
|
||||
// 用 *bool 区分 "未传" / "传了 false":未传时表示客户端不打算改这个字段,
|
||||
// 沿用 catalog 现有值;新建时未传一律默认开启(true)。
|
||||
TeaserEnabled *bool `json:"teaserEnabled,omitempty"`
|
||||
@@ -354,7 +527,14 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
|
||||
if existingDrive, err := a.Catalog.GetDrive(r.Context(), body.ID); err == nil {
|
||||
existing = existingDrive
|
||||
}
|
||||
if len(body.Credentials) == 0 && existing != nil && len(existing.Credentials) > 0 {
|
||||
if body.Kind == "spider91" {
|
||||
credentials, err := mergeSpider91Credentials(existing, body.Credentials)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
body.Credentials = credentials
|
||||
} else if len(body.Credentials) == 0 && existing != nil && len(existing.Credentials) > 0 {
|
||||
body.Credentials = existing.Credentials
|
||||
}
|
||||
|
||||
@@ -384,7 +564,7 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
d := &catalog.Drive{
|
||||
ID: body.ID, Kind: body.Kind, Name: body.Name,
|
||||
RootID: body.RootID, ScanRootID: body.ScanRootID,
|
||||
RootID: body.RootID,
|
||||
Credentials: body.Credentials,
|
||||
Status: "disconnected",
|
||||
TeaserEnabled: teaserEnabled,
|
||||
@@ -403,8 +583,82 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
|
||||
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
func spider91ProxyForDrive(d *catalog.Drive) string {
|
||||
if d == nil || d.Kind != "spider91" || d.Credentials == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(d.Credentials["proxy"])
|
||||
}
|
||||
|
||||
func mergeSpider91Credentials(existing *catalog.Drive, incoming map[string]string) (map[string]string, error) {
|
||||
merged := map[string]string{}
|
||||
if existing != nil {
|
||||
for k, v := range existing.Credentials {
|
||||
merged[k] = v
|
||||
}
|
||||
}
|
||||
for k, v := range incoming {
|
||||
if strings.TrimSpace(k) == "" {
|
||||
continue
|
||||
}
|
||||
if k == "proxy" {
|
||||
proxy, err := normalizeSpider91ProxyURL(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if proxy == "" {
|
||||
delete(merged, "proxy")
|
||||
} else {
|
||||
merged["proxy"] = proxy
|
||||
}
|
||||
continue
|
||||
}
|
||||
merged[k] = v
|
||||
}
|
||||
return merged, nil
|
||||
}
|
||||
|
||||
func normalizeSpider91ProxyURL(raw string) (string, error) {
|
||||
proxy := strings.TrimSpace(raw)
|
||||
if proxy == "" {
|
||||
return "", nil
|
||||
}
|
||||
u, err := url.Parse(proxy)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return "", fmt.Errorf("91Spider 代理地址格式无效,请填写类似 http://127.0.0.1:7890 的地址")
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https", "socks5", "socks5h":
|
||||
return proxy, nil
|
||||
default:
|
||||
return "", fmt.Errorf("91Spider 代理地址仅支持 http://、https://、socks5:// 或 socks5h://")
|
||||
}
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleDeleteDrive(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
var body deleteDriveReq
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
|
||||
writeErr(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
if !body.DeleteVideos {
|
||||
http.Error(w, "deleteVideos=true is required when deleting a drive", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
deletedVideos := 0
|
||||
if a.OnDriveDeleteCleanup == nil {
|
||||
http.Error(w, "drive video cleanup is not available", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
removed, err := a.OnDriveDeleteCleanup(r.Context(), id)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
deletedVideos = removed
|
||||
|
||||
if err := a.Catalog.DeleteDrive(r.Context(), id); err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
@@ -412,7 +666,11 @@ func (a *AdminServer) handleDeleteDrive(w http.ResponseWriter, r *http.Request)
|
||||
if a.OnDriveRemoved != nil {
|
||||
a.OnDriveRemoved(id)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
|
||||
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "deletedVideos": deletedVideos})
|
||||
}
|
||||
|
||||
type deleteDriveReq struct {
|
||||
DeleteVideos bool `json:"deleteVideos"`
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleRescan(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -423,14 +681,91 @@ func (a *AdminServer) handleRescan(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleStopDriveTasks(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
stopped := false
|
||||
if a.OnStopDriveTasks != nil {
|
||||
stopped = a.OnStopDriveTasks(id)
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"ok": true,
|
||||
"stopped": stopped,
|
||||
})
|
||||
}
|
||||
|
||||
func (a *AdminServer) p123QRClient() *p123.QRClient {
|
||||
return p123.NewQRClient(p123.QRConfig{
|
||||
UserAPIBaseURL: a.P123UserAPIBaseURL,
|
||||
HTTPClient: a.P123HTTPClient,
|
||||
})
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleP123QRStart(w http.ResponseWriter, r *http.Request) {
|
||||
session, err := a.p123QRClient().Generate(r.Context())
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusBadGateway, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, http.StatusOK, session)
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleP123QRStatus(w http.ResponseWriter, r *http.Request) {
|
||||
uniID := chi.URLParam(r, "uniID")
|
||||
loginUUID := r.URL.Query().Get("loginUuid")
|
||||
if strings.TrimSpace(uniID) == "" || strings.TrimSpace(loginUUID) == "" {
|
||||
http.Error(w, "uniID and loginUuid are required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
status, err := a.p123QRClient().Poll(r.Context(), loginUUID, uniID)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusBadGateway, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, http.StatusOK, status)
|
||||
}
|
||||
|
||||
// handleRunNightlyJob 触发一次完整的凌晨流水线(不论当前时间,不论今日是否已跑)。
|
||||
// 立即返回 202;进度通过 backend 日志和下次 GET /admin/api/drives 的状态变化观察。
|
||||
// 流水线已在跑时 Runner 会丢弃此次触发并记日志。
|
||||
// 流水线已在跑或已排队时,Runner 会拒绝重复触发。
|
||||
func (a *AdminServer) handleRunNightlyJob(w http.ResponseWriter, r *http.Request) {
|
||||
accepted := false
|
||||
if a.OnRunNightlyJob != nil {
|
||||
a.OnRunNightlyJob()
|
||||
accepted = a.OnRunNightlyJob()
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"ok": true,
|
||||
"accepted": accepted,
|
||||
"status": a.nightlyJobStatus(),
|
||||
})
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleNightlyJobStatus(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.nightlyJobStatus())
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleStopAllTasks(w http.ResponseWriter, r *http.Request) {
|
||||
stoppedDrives := 0
|
||||
if a.OnStopAllTasks != nil {
|
||||
stoppedDrives = a.OnStopAllTasks()
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"ok": true,
|
||||
"stoppedDrives": stoppedDrives,
|
||||
"status": a.nightlyJobStatus(),
|
||||
})
|
||||
}
|
||||
|
||||
func (a *AdminServer) nightlyJobStatus() NightlyJobStatus {
|
||||
if a.GetNightlyJobStatus == nil {
|
||||
return NightlyJobStatus{State: "idle"}
|
||||
}
|
||||
status := a.GetNightlyJobStatus()
|
||||
if status.State == "" {
|
||||
status.State = "idle"
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
// teaserEnabledReq 是 POST /admin/api/drives/{id}/teaser-enabled 的入参。
|
||||
@@ -438,11 +773,11 @@ type teaserEnabledReq struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
// handleSetDriveTeaserEnabled 切换某盘的 teaser 生成开关。
|
||||
// handleSetDriveTeaserEnabled 切换某盘的预览视频生成开关。
|
||||
//
|
||||
// 行为:
|
||||
// - 写 catalog.drives.teaser_enabled
|
||||
// - 调 OnTeaserEnabledChanged(main 注入;从关到开时会重新入队 pending teaser)
|
||||
// - 调 OnTeaserEnabledChanged(main 注入;从关到开时会重新入队 pending 预览视频)
|
||||
// - 返回切换后的新值,方便前端乐观更新但又能以服务端为准
|
||||
//
|
||||
// 与 upsertDrive 的区别:那条接口要重传 kind / name / rootId 等,开关切换不该
|
||||
@@ -564,6 +899,7 @@ func (a *AdminServer) handleAdminListVideos(w http.ResponseWriter, r *http.Reque
|
||||
size = 100
|
||||
}
|
||||
items, total, err := a.Catalog.ListVideos(r.Context(), catalog.ListParams{
|
||||
Keyword: q.Get("keyword"),
|
||||
DriveID: q.Get("driveId"),
|
||||
Page: page,
|
||||
PageSize: size,
|
||||
@@ -611,6 +947,27 @@ func (a *AdminServer) handleCreateTag(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleDeleteTag(w http.ResponseWriter, r *http.Request) {
|
||||
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
||||
if err != nil || id <= 0 {
|
||||
writeErr(w, http.StatusBadRequest, errors.New("invalid tag id"))
|
||||
return
|
||||
}
|
||||
removedVideos, err := a.Catalog.DeleteTag(r.Context(), id)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
writeErr(w, http.StatusNotFound, err)
|
||||
case errors.Is(err, catalog.ErrSystemTag):
|
||||
writeErr(w, http.StatusBadRequest, err)
|
||||
default:
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "removedVideos": removedVideos})
|
||||
}
|
||||
|
||||
type updateVideoReq struct {
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
@@ -681,6 +1038,36 @@ func (a *AdminServer) handleUpdateVideo(w http.ResponseWriter, r *http.Request)
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleDeleteVideo(w http.ResponseWriter, r *http.Request) {
|
||||
id := strings.TrimSpace(chi.URLParam(r, "id"))
|
||||
if id == "" {
|
||||
writeErr(w, http.StatusBadRequest, errors.New("invalid video id"))
|
||||
return
|
||||
}
|
||||
var (
|
||||
result DeleteVideoResult
|
||||
err error
|
||||
)
|
||||
if a.OnDeleteVideo != nil {
|
||||
result, err = a.OnDeleteVideo(r.Context(), id)
|
||||
} else {
|
||||
err = a.Catalog.DeleteVideoWithTombstone(r.Context(), id)
|
||||
result = DeleteVideoResult{OK: err == nil}
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
writeErr(w, http.StatusNotFound, err)
|
||||
return
|
||||
}
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
if !result.OK {
|
||||
result.OK = true
|
||||
}
|
||||
writeJSON(w, http.StatusOK, result)
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleRegenPreview(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
if a.OnRegenPreview != nil {
|
||||
@@ -705,7 +1092,7 @@ func (a *AdminServer) handleRegenFailedPreviews(w http.ResponseWriter, r *http.R
|
||||
}
|
||||
|
||||
// handleRegenFailedThumbnails 触发某 drive 下所有 thumbnail_status=failed 的封面
|
||||
// 重新入队生成。和 handleRegenFailedPreviews 行为对称(一个管 teaser,一个管封面)。
|
||||
// 重新入队生成。和 handleRegenFailedPreviews 行为对称(一个管预览视频,一个管封面)。
|
||||
//
|
||||
// 立即返回 202;实际执行在后台 goroutine 跑,状态可在下次 GET /admin/api/drives
|
||||
// 的 thumbnailFailedCount / thumbnailGenerationStatus 看变化。
|
||||
@@ -717,12 +1104,22 @@ func (a *AdminServer) handleRegenFailedThumbnails(w http.ResponseWriter, r *http
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
// handleRegenFailedFingerprints triggers regeneration for all failed sampled
|
||||
// fingerprints on a drive. It mirrors the failed preview-video/thumbnail retry endpoints.
|
||||
func (a *AdminServer) handleRegenFailedFingerprints(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
if a.OnRegenFailedFingerprints != nil {
|
||||
a.OnRegenFailedFingerprints(id)
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
// ---------- Settings ----------
|
||||
|
||||
// settingsDTO 是 GET/PUT /admin/api/settings 的入参/出参。
|
||||
//
|
||||
// 注意:早期的全局 previewEnabled 字段已经下沉为每盘 teaser_enabled,
|
||||
// 不再出现在这里;前端要切换某个盘的 teaser 生成请用 POST /admin/api/drives 上传
|
||||
// 不再出现在这里;前端要切换某个盘的预览视频生成请用 POST /admin/api/drives 上传
|
||||
// teaserEnabled 字段。保留 settings 用作主题、spider91 上传目标这类全局配置。
|
||||
type settingsDTO struct {
|
||||
Theme string `json:"theme"`
|
||||
@@ -748,7 +1145,7 @@ func (a *AdminServer) handleGetSettings(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
func (a *AdminServer) handlePutSettings(w http.ResponseWriter, r *http.Request) {
|
||||
// 用 map 区分"没传"和"传了空字符串"两种语义;空 spider91 上传 ID 表示
|
||||
// 清除显式设置(回退到自动模式)。
|
||||
// 本地保存不上传。
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.NewDecoder(r.Body).Decode(&raw); err != nil {
|
||||
writeErr(w, http.StatusBadRequest, err)
|
||||
|
||||
@@ -3,11 +3,13 @@ package api
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -115,6 +117,254 @@ func TestHandleSetupStoresCredentialsAndCreatesSession(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleCheckUpdateReportsNewRelease(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
versionFile := filepath.Join(dir, ".version")
|
||||
if err := os.WriteFile(versionFile, []byte("v0.1.0\n2026-05-29 12:00:00\n"), 0o644); err != nil {
|
||||
t.Fatalf("write version file: %v", err)
|
||||
}
|
||||
releaseServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("User-Agent") == "" {
|
||||
http.Error(w, "missing user agent", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"tag_name": "v0.2.0",
|
||||
"html_url": "https://github.com/nianzhibai/91/releases/tag/v0.2.0",
|
||||
})
|
||||
}))
|
||||
t.Cleanup(releaseServer.Close)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/update/check", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{
|
||||
VersionFilePath: versionFile,
|
||||
ReleaseAPIURL: releaseServer.URL,
|
||||
}).handleCheckUpdate(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got updateCheckDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.CurrentVersion != "v0.1.0" {
|
||||
t.Fatalf("currentVersion = %q, want v0.1.0", got.CurrentVersion)
|
||||
}
|
||||
if got.LatestVersion != "v0.2.0" {
|
||||
t.Fatalf("latestVersion = %q, want v0.2.0", got.LatestVersion)
|
||||
}
|
||||
if !got.HasUpdate {
|
||||
t.Fatalf("hasUpdate = false, want true")
|
||||
}
|
||||
if got.ReleaseURL == "" {
|
||||
t.Fatalf("releaseUrl is empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleCheckUpdateReportsUpToDate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
versionFile := filepath.Join(dir, ".version")
|
||||
if err := os.WriteFile(versionFile, []byte("v0.2.0\n"), 0o644); err != nil {
|
||||
t.Fatalf("write version file: %v", err)
|
||||
}
|
||||
releaseServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"tag_name": "v0.2.0",
|
||||
"html_url": "https://github.com/nianzhibai/91/releases/tag/v0.2.0",
|
||||
})
|
||||
}))
|
||||
t.Cleanup(releaseServer.Close)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/update/check", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{
|
||||
VersionFilePath: versionFile,
|
||||
ReleaseAPIURL: releaseServer.URL,
|
||||
}).handleCheckUpdate(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got updateCheckDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.HasUpdate {
|
||||
t.Fatalf("hasUpdate = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleCheckUpdateUsesDockerImageVersion(t *testing.T) {
|
||||
releaseServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"tag_name": "v0.2.0",
|
||||
"html_url": "https://github.com/nianzhibai/91/releases/tag/v0.2.0",
|
||||
})
|
||||
}))
|
||||
t.Cleanup(releaseServer.Close)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/update/check", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{
|
||||
ImageVersion: "v0.1.0",
|
||||
ReleaseAPIURL: releaseServer.URL,
|
||||
}).handleCheckUpdate(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got updateCheckDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.CurrentVersion != "v0.1.0" {
|
||||
t.Fatalf("currentVersion = %q, want v0.1.0", got.CurrentVersion)
|
||||
}
|
||||
if !got.HasUpdate {
|
||||
t.Fatalf("hasUpdate = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstalledVersionPrefersDockerImageVersionOverVersionFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
versionFile := filepath.Join(dir, ".version")
|
||||
if err := os.WriteFile(versionFile, []byte("v0.1.0\n"), 0o644); err != nil {
|
||||
t.Fatalf("write version file: %v", err)
|
||||
}
|
||||
|
||||
got := (&AdminServer{
|
||||
VersionFilePath: versionFile,
|
||||
ImageVersion: "v0.2.0",
|
||||
}).installedVersion()
|
||||
|
||||
if got != "v0.2.0" {
|
||||
t.Fatalf("installedVersion = %q, want v0.2.0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRunNightlyJobReturnsAcceptedStatus(t *testing.T) {
|
||||
called := false
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/jobs/nightly/run", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
OnRunNightlyJob: func() bool {
|
||||
called = true
|
||||
return true
|
||||
},
|
||||
GetNightlyJobStatus: func() NightlyJobStatus {
|
||||
return NightlyJobStatus{State: "queued", Queued: true}
|
||||
},
|
||||
}).handleRunNightlyJob(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, want 202; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !called {
|
||||
t.Fatal("OnRunNightlyJob was not called")
|
||||
}
|
||||
var got struct {
|
||||
OK bool `json:"ok"`
|
||||
Accepted bool `json:"accepted"`
|
||||
Status NightlyJobStatus `json:"status"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if !got.OK || !got.Accepted || got.Status.State != "queued" || !got.Status.Queued {
|
||||
t.Fatalf("response = %#v, want accepted queued status", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleNightlyJobStatusDefaultsToIdle(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/jobs/nightly/status", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{}).handleNightlyJobStatus(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got NightlyJobStatus
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.State != "idle" || got.Running || got.Queued {
|
||||
t.Fatalf("status = %#v, want idle", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStopDriveTasksInvokesHookWithDriveID(t *testing.T) {
|
||||
calledWith := ""
|
||||
server := &AdminServer{
|
||||
OnStopDriveTasks: func(driveID string) bool {
|
||||
calledWith = driveID
|
||||
return true
|
||||
},
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives/PikPak/tasks/stop", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "PikPak")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
server.handleStopDriveTasks(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if calledWith != "PikPak" {
|
||||
t.Fatalf("hook called with %q, want PikPak", calledWith)
|
||||
}
|
||||
var got struct {
|
||||
OK bool `json:"ok"`
|
||||
Stopped bool `json:"stopped"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if !got.OK || !got.Stopped {
|
||||
t.Fatalf("response = %#v, want stopped", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStopAllTasksInvokesHookAndReturnsStatus(t *testing.T) {
|
||||
called := false
|
||||
server := &AdminServer{
|
||||
OnStopAllTasks: func() int {
|
||||
called = true
|
||||
return 2
|
||||
},
|
||||
GetNightlyJobStatus: func() NightlyJobStatus {
|
||||
return NightlyJobStatus{State: "running", Running: true}
|
||||
},
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/tasks/stop", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
server.handleStopAllTasks(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !called {
|
||||
t.Fatal("OnStopAllTasks was not called")
|
||||
}
|
||||
var got struct {
|
||||
OK bool `json:"ok"`
|
||||
StoppedDrives int `json:"stoppedDrives"`
|
||||
Status NightlyJobStatus `json:"status"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if !got.OK || got.StoppedDrives != 2 || got.Status.State != "running" || !got.Status.Running {
|
||||
t.Fatalf("response = %#v, want stopped drives and status", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertDrivePreservesExistingCredentialsWhenRequestCredentialsEmpty(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -163,14 +413,52 @@ func TestHandleUpsertDrivePreservesExistingCredentialsWhenRequestCredentialsEmpt
|
||||
if got.Name != "New name" {
|
||||
t.Fatalf("name = %q, want New name", got.Name)
|
||||
}
|
||||
if got.ScanRootID != "scan-root" {
|
||||
t.Fatalf("scanRootId = %q, want scan-root", got.ScanRootID)
|
||||
if got.ScanRootID != "0" {
|
||||
t.Fatalf("scanRootId = %q, want rootId 0", got.ScanRootID)
|
||||
}
|
||||
if got.Credentials["cookie"] != "existing-cookie" {
|
||||
t.Fatalf("cookie credential = %q, want existing-cookie", got.Credentials["cookie"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertDriveDefaultsEmptyRootID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "onedrive-main",
|
||||
"kind": "onedrive",
|
||||
"name": "OneDrive",
|
||||
"rootId": "",
|
||||
"credentials": {"refresh_token": "token"}
|
||||
}`))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, "onedrive-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.RootID != "root" {
|
||||
t.Fatalf("rootId = %q, want root", got.RootID)
|
||||
}
|
||||
if got.ScanRootID != got.RootID {
|
||||
t.Fatalf("scanRootId = %q, want rootId %q", got.ScanRootID, got.RootID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertDriveReplacesExistingCredentialsWhenProvided(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -221,6 +509,300 @@ func TestHandleUpsertDriveReplacesExistingCredentialsWhenProvided(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertSpider91ProxyPreservesRuntimeCredentials(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "spider91-main",
|
||||
Kind: "spider91",
|
||||
Name: "91 Spider",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"last_crawl_at": "1800000000",
|
||||
"proxy": "http://old-proxy.local:7890",
|
||||
"script_path": "/opt/video-site-91/91VideoSpider/spider_91porn.py",
|
||||
},
|
||||
Status: "ok",
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"kind": "spider91",
|
||||
"name": "91 Spider",
|
||||
"rootId": "/",
|
||||
"credentials": {"proxy": " socks5h://proxy-user:proxy-pass@127.0.0.1:7891 "}
|
||||
}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
got, err := cat.GetDrive(ctx, "spider91-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.Credentials["proxy"] != "socks5h://proxy-user:proxy-pass@127.0.0.1:7891" {
|
||||
t.Fatalf("proxy = %q, want trimmed new proxy", got.Credentials["proxy"])
|
||||
}
|
||||
if got.Credentials["last_crawl_at"] != "1800000000" {
|
||||
t.Fatalf("last_crawl_at = %q, want preserved", got.Credentials["last_crawl_at"])
|
||||
}
|
||||
if got.Credentials["script_path"] == "" {
|
||||
t.Fatalf("script_path should be preserved")
|
||||
}
|
||||
|
||||
req = httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"kind": "spider91",
|
||||
"name": "91 Spider",
|
||||
"rootId": "/",
|
||||
"credentials": {"proxy": " "}
|
||||
}`))
|
||||
rr = httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("clear status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
got, err = cat.GetDrive(ctx, "spider91-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get cleared drive: %v", err)
|
||||
}
|
||||
if _, ok := got.Credentials["proxy"]; ok {
|
||||
t.Fatalf("proxy should be removed after empty save, got %q", got.Credentials["proxy"])
|
||||
}
|
||||
if got.Credentials["last_crawl_at"] != "1800000000" {
|
||||
t.Fatalf("last_crawl_at after clear = %q, want preserved", got.Credentials["last_crawl_at"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertSpider91RejectsUnsupportedProxyScheme(t *testing.T) {
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"kind": "spider91",
|
||||
"name": "91 Spider",
|
||||
"rootId": "/",
|
||||
"credentials": {"proxy": "ftp://127.0.0.1:21"}
|
||||
}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), "socks5:// 或 socks5h://") {
|
||||
t.Fatalf("body = %q, want supported schemes message", rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleDeleteDriveRunsRequestedCleanupBeforeDeletingDrive(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "drive-one",
|
||||
Kind: "pikpak",
|
||||
Name: "Drive One",
|
||||
RootID: "root",
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
cleanupCalled := ""
|
||||
removedCalled := ""
|
||||
req := httptest.NewRequest(http.MethodDelete, "/admin/api/drives/drive-one", strings.NewReader(`{"deleteVideos":true}`))
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "drive-one")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
Catalog: cat,
|
||||
OnDriveDeleteCleanup: func(cleanupCtx context.Context, driveID string) (int, error) {
|
||||
cleanupCalled = driveID
|
||||
if _, err := cat.GetDrive(cleanupCtx, driveID); err != nil {
|
||||
t.Fatalf("drive should still exist during cleanup: %v", err)
|
||||
}
|
||||
return 3, nil
|
||||
},
|
||||
OnDriveRemoved: func(driveID string) {
|
||||
removedCalled = driveID
|
||||
},
|
||||
}).handleDeleteDrive(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if cleanupCalled != "drive-one" {
|
||||
t.Fatalf("cleanup called with %q, want drive-one", cleanupCalled)
|
||||
}
|
||||
if removedCalled != "drive-one" {
|
||||
t.Fatalf("removed hook called with %q, want drive-one", removedCalled)
|
||||
}
|
||||
if _, err := cat.GetDrive(ctx, "drive-one"); err != sql.ErrNoRows {
|
||||
t.Fatalf("drive lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
var got struct {
|
||||
OK bool `json:"ok"`
|
||||
DeletedVideos int `json:"deletedVideos"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if !got.OK || got.DeletedVideos != 3 {
|
||||
t.Fatalf("response = %#v, want ok with deletedVideos=3", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleDeleteDriveRequiresCleanupConfirmation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "drive-one",
|
||||
Kind: "pikpak",
|
||||
Name: "Drive One",
|
||||
RootID: "root",
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodDelete, "/admin/api/drives/drive-one", strings.NewReader(`{"deleteVideos":false}`))
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "drive-one")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
Catalog: cat,
|
||||
OnDriveDeleteCleanup: func(context.Context, string) (int, error) {
|
||||
t.Fatal("cleanup hook should not be called without confirmation")
|
||||
return 0, nil
|
||||
},
|
||||
}).handleDeleteDrive(rr, req)
|
||||
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if _, err := cat.GetDrive(ctx, "drive-one"); err != nil {
|
||||
t.Fatalf("drive should remain after rejected delete: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListDrivesIncludesSpider91Proxy(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, d := range []*catalog.Drive{
|
||||
{
|
||||
ID: "spider91-main",
|
||||
Kind: "spider91",
|
||||
Name: "91 Spider",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"last_crawl_at": "1800000000",
|
||||
"proxy": " http://127.0.0.1:7890 ",
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
{
|
||||
ID: "onedrive-main",
|
||||
Kind: "onedrive",
|
||||
Name: "OneDrive",
|
||||
RootID: "root",
|
||||
Credentials: map[string]string{
|
||||
"proxy": "http://should-not-leak.local:7890",
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
t.Fatalf("seed drive %s: %v", d.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/drives", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleListDrives(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
var got []struct {
|
||||
ID string `json:"id"`
|
||||
Spider91Proxy string `json:"spider91Proxy"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
byID := map[string]struct {
|
||||
Spider91Proxy string
|
||||
LastCrawlAt int64
|
||||
}{}
|
||||
for _, d := range got {
|
||||
byID[d.ID] = struct {
|
||||
Spider91Proxy string
|
||||
LastCrawlAt int64
|
||||
}{Spider91Proxy: d.Spider91Proxy, LastCrawlAt: d.LastCrawlAt}
|
||||
}
|
||||
if byID["spider91-main"].Spider91Proxy != "http://127.0.0.1:7890" {
|
||||
t.Fatalf("spider91 proxy = %q, want trimmed proxy", byID["spider91-main"].Spider91Proxy)
|
||||
}
|
||||
if byID["spider91-main"].LastCrawlAt != 1800000000 {
|
||||
t.Fatalf("lastCrawlAt = %d, want 1800000000", byID["spider91-main"].LastCrawlAt)
|
||||
}
|
||||
if byID["onedrive-main"].Spider91Proxy != "" {
|
||||
t.Fatalf("onedrive spider91Proxy = %q, want empty", byID["onedrive-main"].Spider91Proxy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -244,11 +826,11 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
|
||||
now := time.Now()
|
||||
videos := []*catalog.Video{
|
||||
{ID: "od-ready-1", DriveID: "OneDrive", FileID: "od-file-1", Title: "OD Ready 1", ThumbnailURL: "/p/thumb/od-ready-1", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-ready-2", DriveID: "OneDrive", FileID: "od-file-2", Title: "OD Ready 2", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-pending", DriveID: "OneDrive", FileID: "od-file-3", Title: "OD Pending", PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "pp-pending", DriveID: "PikPak", FileID: "pp-file-1", Title: "PP Pending", PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "pp-failed", DriveID: "PikPak", FileID: "pp-file-2", Title: "PP Failed", ThumbnailURL: "/p/thumb/pp-failed", PreviewStatus: "failed", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-ready-1", DriveID: "OneDrive", FileID: "od-file-1", Title: "OD Ready 1", Size: 100, ThumbnailURL: "/p/thumb/od-ready-1", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-ready-2", DriveID: "OneDrive", FileID: "od-file-2", Title: "OD Ready 2", Size: 100, PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-pending", DriveID: "OneDrive", FileID: "od-file-3", Title: "OD Pending", Size: 100, PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "pp-pending", DriveID: "PikPak", FileID: "pp-file-1", Title: "PP Pending", Size: 100, PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "pp-failed", DriveID: "PikPak", FileID: "pp-file-2", Title: "PP Failed", Size: 100, ThumbnailURL: "/p/thumb/pp-failed", PreviewStatus: "failed", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
}
|
||||
for _, v := range videos {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
@@ -258,6 +840,12 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
if err := cat.UpdateVideoMeta(ctx, "od-ready-2", catalog.VideoMetaPatch{ThumbnailStatus: "failed"}); err != nil {
|
||||
t.Fatalf("mark thumbnail failed: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "od-ready-1", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "ready", ""); err != nil {
|
||||
t.Fatalf("mark fingerprint ready: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "od-ready-2", "", "failed", "sample failed"); err != nil {
|
||||
t.Fatalf("mark fingerprint failed: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/drives", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
@@ -266,8 +854,9 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
GetDriveGenerationStatuses: func() map[string]DriveGenerationStatuses {
|
||||
return map[string]DriveGenerationStatuses{
|
||||
"OneDrive": {
|
||||
Thumbnail: GenerationStatus{State: "cooling", QueueLength: 3, CooldownUntil: "2026-05-16T21:00:00+08:00"},
|
||||
Preview: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
|
||||
Thumbnail: GenerationStatus{State: "cooling", QueueLength: 3, CooldownUntil: "2026-05-16T21:00:00+08:00"},
|
||||
Preview: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
|
||||
Fingerprint: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
|
||||
},
|
||||
}
|
||||
},
|
||||
@@ -277,48 +866,68 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got []struct {
|
||||
ID string `json:"id"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
ID string `json:"id"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
ThumbnailDurationPendingCount int `json:"thumbnailDurationPendingCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
FingerprintReadyCount int `json:"fingerprintReadyCount"`
|
||||
FingerprintPendingCount int `json:"fingerprintPendingCount"`
|
||||
FingerprintFailedCount int `json:"fingerprintFailedCount"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
byID := map[string]struct {
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
ThumbnailDurationPending int
|
||||
FingerprintReady int
|
||||
FingerprintPending int
|
||||
FingerprintFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
Fingerprint GenerationStatus
|
||||
}{}
|
||||
for _, d := range got {
|
||||
byID[d.ID] = struct {
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
ThumbnailDurationPending int
|
||||
FingerprintReady int
|
||||
FingerprintPending int
|
||||
FingerprintFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
Fingerprint GenerationStatus
|
||||
}{
|
||||
TeaserReady: d.TeaserReadyCount,
|
||||
TeaserPending: d.TeaserPendingCount,
|
||||
TeaserFailed: d.TeaserFailedCount,
|
||||
ThumbnailReady: d.ThumbnailReadyCount,
|
||||
ThumbnailPending: d.ThumbnailPendingCount,
|
||||
ThumbnailFailed: d.ThumbnailFailedCount,
|
||||
Thumbnail: d.ThumbnailGenerationStatus,
|
||||
Preview: d.PreviewGenerationStatus,
|
||||
TeaserReady: d.TeaserReadyCount,
|
||||
TeaserPending: d.TeaserPendingCount,
|
||||
TeaserFailed: d.TeaserFailedCount,
|
||||
ThumbnailReady: d.ThumbnailReadyCount,
|
||||
ThumbnailPending: d.ThumbnailPendingCount,
|
||||
ThumbnailFailed: d.ThumbnailFailedCount,
|
||||
ThumbnailDurationPending: d.ThumbnailDurationPendingCount,
|
||||
FingerprintReady: d.FingerprintReadyCount,
|
||||
FingerprintPending: d.FingerprintPendingCount,
|
||||
FingerprintFailed: d.FingerprintFailedCount,
|
||||
Thumbnail: d.ThumbnailGenerationStatus,
|
||||
Preview: d.PreviewGenerationStatus,
|
||||
Fingerprint: d.FingerprintGenerationStatus,
|
||||
}
|
||||
}
|
||||
if byID["OneDrive"].TeaserReady != 2 || byID["OneDrive"].TeaserPending != 1 || byID["OneDrive"].TeaserFailed != 0 {
|
||||
@@ -327,20 +936,54 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
if byID["OneDrive"].ThumbnailReady != 1 || byID["OneDrive"].ThumbnailPending != 1 || byID["OneDrive"].ThumbnailFailed != 1 {
|
||||
t.Fatalf("OneDrive thumbnail counts = %#v, want ready=1 pending=1 failed=1", byID["OneDrive"])
|
||||
}
|
||||
if byID["OneDrive"].ThumbnailDurationPending != 1 {
|
||||
t.Fatalf("OneDrive thumbnail duration pending = %#v, want 1", byID["OneDrive"])
|
||||
}
|
||||
if byID["OneDrive"].Thumbnail.State != "cooling" || byID["OneDrive"].Preview.State != "generating" {
|
||||
t.Fatalf("OneDrive generation statuses = %#v, want thumbnail cooling and preview generating", byID["OneDrive"])
|
||||
}
|
||||
if byID["OneDrive"].FingerprintReady != 1 || byID["OneDrive"].FingerprintPending != 1 || byID["OneDrive"].FingerprintFailed != 1 {
|
||||
t.Fatalf("OneDrive fingerprint counts = %#v, want ready=1 pending=1 failed=1", byID["OneDrive"])
|
||||
}
|
||||
if byID["OneDrive"].Fingerprint.State != "generating" {
|
||||
t.Fatalf("OneDrive fingerprint status = %#v, want generating", byID["OneDrive"].Fingerprint)
|
||||
}
|
||||
if byID["PikPak"].TeaserReady != 0 || byID["PikPak"].TeaserPending != 1 || byID["PikPak"].TeaserFailed != 1 {
|
||||
t.Fatalf("PikPak counts = %#v, want ready=0 pending=1 failed=1", byID["PikPak"])
|
||||
}
|
||||
if byID["PikPak"].ThumbnailReady != 1 || byID["PikPak"].ThumbnailPending != 1 || byID["PikPak"].ThumbnailFailed != 0 {
|
||||
t.Fatalf("PikPak thumbnail counts = %#v, want ready=1 pending=1 failed=0", byID["PikPak"])
|
||||
}
|
||||
if byID["PikPak"].Thumbnail.State != "idle" || byID["PikPak"].Preview.State != "idle" {
|
||||
if byID["PikPak"].FingerprintPending != 2 {
|
||||
t.Fatalf("PikPak fingerprint counts = %#v, want pending=2", byID["PikPak"])
|
||||
}
|
||||
if byID["PikPak"].Thumbnail.State != "idle" || byID["PikPak"].Preview.State != "idle" || byID["PikPak"].Fingerprint.State != "idle" {
|
||||
t.Fatalf("PikPak generation statuses = %#v, want idle defaults", byID["PikPak"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRegenFailedFingerprintsInvokesHook(t *testing.T) {
|
||||
called := ""
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives/drive-one/fingerprints/failed/regenerate", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "drive-one")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
OnRegenFailedFingerprints: func(driveID string) {
|
||||
called = driveID
|
||||
},
|
||||
}).handleRegenFailedFingerprints(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if called != "drive-one" {
|
||||
t.Fatalf("called drive = %q, want drive-one", called)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleDriveStorageReportsLocalMediaUsage(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
@@ -504,6 +1147,64 @@ func TestHandleCreateTagClassifiesExistingVideos(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleDeleteTagRemovesTagFromVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "video-1",
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
Title: "清纯短发",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
if _, err := cat.CreateTagAndClassify(ctx, "清纯", nil, "user"); err != nil {
|
||||
t.Fatalf("create tag: %v", err)
|
||||
}
|
||||
tags, err := cat.ListTags(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list tags: %v", err)
|
||||
}
|
||||
var tagID int64
|
||||
for _, tag := range tags {
|
||||
if tag.Label == "清纯" {
|
||||
tagID = tag.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
if tagID == 0 {
|
||||
t.Fatal("created tag not found")
|
||||
}
|
||||
|
||||
req := requestWithRouteParam(http.MethodDelete, "/admin/api/tags/1", "id", strconv.FormatInt(tagID, 10), strings.NewReader(``))
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleDeleteTag(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
video, err := cat.GetVideo(ctx, "video-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if len(video.Tags) != 0 {
|
||||
t.Fatalf("video tags = %#v, want none", video.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleAdminListVideosFiltersByDriveID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
+304
-95
@@ -15,14 +15,17 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/video-site/backend/internal/auth"
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives/localstorage"
|
||||
"github.com/video-site/backend/internal/drives/localupload"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
)
|
||||
|
||||
@@ -39,7 +42,7 @@ var allowedUploadExtensions = map[string]struct{}{
|
||||
var allowedUploadTags = map[string]struct{}{
|
||||
"奶子": {},
|
||||
"臀": {},
|
||||
"口角": {},
|
||||
"口交": {},
|
||||
"女大": {},
|
||||
"人妻": {},
|
||||
"AV": {},
|
||||
@@ -52,6 +55,10 @@ type Server struct {
|
||||
UploadDir string
|
||||
OnVideoUploaded func(*catalog.Video)
|
||||
|
||||
tagCacheMu sync.Mutex
|
||||
tagCacheUntil time.Time
|
||||
tagCache []TagDTO
|
||||
|
||||
// GetTheme 返回当前生效的主题("dark" | "pink")。前台 /api/settings/theme 用,
|
||||
// 不需要登录。无注入时返回 "dark"。
|
||||
GetTheme func() string
|
||||
@@ -85,6 +92,12 @@ type VideoDTO struct {
|
||||
Category string `json:"category,omitempty"`
|
||||
}
|
||||
|
||||
type TagDTO struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
type VideoDetailDTO struct {
|
||||
VideoDTO
|
||||
VideoSrc string `json:"videoSrc"`
|
||||
@@ -155,26 +168,117 @@ func (s *Server) handleGetTheme(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) {
|
||||
// 拉一批候选(按发布时间倒序,覆盖最近 200 个),然后随机洗牌取前 homePageSize 个。
|
||||
// 如果库内不足 200 个会自动按实际数量返回,最后裁剪到 homePageSize。
|
||||
const candidatePool = 200
|
||||
items, _, err := s.Catalog.ListVideos(r.Context(), catalog.ListParams{
|
||||
Sort: "latest", Page: 1, PageSize: candidatePool,
|
||||
})
|
||||
// 首页优先从全量已有封面的视频里随机抽取,避免只在最近一小段候选中反复出现。
|
||||
excludeIDs := parseVideoIDQuery(r, "exclude", 120)
|
||||
items, err := s.Catalog.RandomVideosWithReadyThumbnailsExcluding(r.Context(), excludeIDs, homePageSize)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
rand.Shuffle(len(items), func(i, j int) {
|
||||
items[i], items[j] = items[j], items[i]
|
||||
})
|
||||
if len(items) > homePageSize {
|
||||
items = items[:homePageSize]
|
||||
if len(items) < homePageSize {
|
||||
fallbackExclude := append([]string{}, excludeIDs...)
|
||||
for _, item := range items {
|
||||
if item != nil {
|
||||
fallbackExclude = append(fallbackExclude, item.ID)
|
||||
}
|
||||
}
|
||||
fallback, err := s.Catalog.RandomVideosExcluding(r.Context(), fallbackExclude, homePageSize-len(items))
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
items = appendUniqueVideos(items, fallback, homePageSize)
|
||||
}
|
||||
if len(items) < homePageSize && len(excludeIDs) > 0 {
|
||||
// The browser keeps a recent-video exclude list so normal refreshes do not
|
||||
// repeat too quickly. On small libraries that list can cover every visible
|
||||
// video; when that happens, start a new random round instead of returning
|
||||
// an empty home section.
|
||||
roundExclude := videoIDs(items)
|
||||
fallback, err := s.Catalog.RandomVideosWithReadyThumbnailsExcluding(r.Context(), roundExclude, homePageSize-len(items))
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
items = appendUniqueVideos(items, fallback, homePageSize)
|
||||
}
|
||||
if len(items) < homePageSize && len(excludeIDs) > 0 {
|
||||
fallback, err := s.Catalog.RandomVideosExcluding(r.Context(), videoIDs(items), homePageSize-len(items))
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
items = appendUniqueVideos(items, fallback, homePageSize)
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, http.StatusOK, mapVideos(items))
|
||||
}
|
||||
|
||||
func parseVideoIDQuery(r *http.Request, key string, limit int) []string {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
values := r.URL.Query()[key]
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
out := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
for _, id := range strings.Split(value, ",") {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
out = append(out, id)
|
||||
if limit > 0 && len(out) >= limit {
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func appendUniqueVideos(dst []*catalog.Video, candidates []*catalog.Video, limit int) []*catalog.Video {
|
||||
if len(dst) >= limit {
|
||||
return dst[:limit]
|
||||
}
|
||||
seen := make(map[string]struct{}, len(dst))
|
||||
for _, v := range dst {
|
||||
if v != nil {
|
||||
seen[v.ID] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, v := range candidates {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
dst = append(dst, v)
|
||||
seen[v.ID] = struct{}{}
|
||||
if len(dst) >= limit {
|
||||
return dst
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
func videoIDs(items []*catalog.Video) []string {
|
||||
out := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
if item != nil && item.ID != "" {
|
||||
out = append(out, item.ID)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Server) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
page, _ := strconv.Atoi(q.Get("page"))
|
||||
@@ -182,13 +286,18 @@ func (s *Server) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
if size <= 0 {
|
||||
size = 24
|
||||
}
|
||||
sort := q.Get("sort")
|
||||
params := catalog.ListParams{
|
||||
Keyword: q.Get("q"),
|
||||
Tag: q.Get("tag"),
|
||||
Category: q.Get("cat"),
|
||||
Sort: q.Get("sort"),
|
||||
Page: page,
|
||||
PageSize: size,
|
||||
Keyword: q.Get("q"),
|
||||
Tag: q.Get("tag"),
|
||||
Category: q.Get("cat"),
|
||||
Sort: sort,
|
||||
Page: page,
|
||||
PageSize: size,
|
||||
SkipTotal: strings.EqualFold(q.Get("count"), "false"),
|
||||
}
|
||||
if sort == "" || sort == "latest" {
|
||||
params.PreferReadyThumbnails = true
|
||||
}
|
||||
items, total, err := s.Catalog.ListVideos(r.Context(), params)
|
||||
if err != nil {
|
||||
@@ -214,6 +323,15 @@ func (s *Server) handleVideoDetail(w http.ResponseWriter, r *http.Request) {
|
||||
writeErr(w, http.StatusNotFound, sql.ErrNoRows)
|
||||
return
|
||||
}
|
||||
if v.DriveID != localUploadDriveID {
|
||||
if _, err := s.Catalog.GetDrive(r.Context(), v.DriveID); err != nil {
|
||||
drives, listErr := s.Catalog.ListDrives(r.Context())
|
||||
if listErr != nil || len(drives) > 0 {
|
||||
writeErr(w, http.StatusNotFound, sql.ErrNoRows)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
related := s.pickRelatedVideos(r.Context(), v, 6)
|
||||
dto := mapVideo(v)
|
||||
if d, err := s.Catalog.GetDrive(r.Context(), v.DriveID); err == nil {
|
||||
@@ -241,7 +359,8 @@ func (s *Server) handleVideoDetail(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// pickRelatedVideos 选 total 个推荐视频。
|
||||
// 一半(向上取整)来自同标签命中,剩下用全库随机补齐;不会重复,也不会包含当前视频。
|
||||
// 一半来自同标签命中,剩下用全库随机补齐;两段都优先取已有封面的视频,
|
||||
// 不够时再回退到未生成封面的候选。结果不会重复,也不会包含当前视频。
|
||||
func (s *Server) pickRelatedVideos(ctx context.Context, current *catalog.Video, total int) []*catalog.Video {
|
||||
if total <= 0 || current == nil {
|
||||
return nil
|
||||
@@ -254,93 +373,163 @@ func (s *Server) pickRelatedVideos(ctx context.Context, current *catalog.Video,
|
||||
picked := make([]*catalog.Video, 0, total)
|
||||
seen := map[string]struct{}{current.ID: {}}
|
||||
|
||||
// 1) 同标签候选:对每个 tag 取一批,合并去重,洗牌后取 tagQuota 个
|
||||
// 1) 同标签候选:先取已有封面的候选,数量不够再从全部候选里补。
|
||||
if tagQuota > 0 && len(current.Tags) > 0 {
|
||||
var tagPool []*catalog.Video
|
||||
for _, tag := range current.Tags {
|
||||
if tag == "" {
|
||||
continue
|
||||
}
|
||||
items, _, err := s.Catalog.ListVideos(ctx, catalog.ListParams{
|
||||
Tag: tag, Sort: "latest", Page: 1, PageSize: 30,
|
||||
})
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, v := range items {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[v.ID] = struct{}{}
|
||||
tagPool = append(tagPool, v)
|
||||
}
|
||||
picked = appendRandomRelated(
|
||||
picked,
|
||||
s.relatedTagPool(ctx, current.Tags, seen, true),
|
||||
tagQuota,
|
||||
seen,
|
||||
)
|
||||
if len(picked) < tagQuota {
|
||||
picked = appendRandomRelated(
|
||||
picked,
|
||||
s.relatedTagPool(ctx, current.Tags, seen, false),
|
||||
tagQuota,
|
||||
seen,
|
||||
)
|
||||
}
|
||||
rand.Shuffle(len(tagPool), func(i, j int) {
|
||||
tagPool[i], tagPool[j] = tagPool[j], tagPool[i]
|
||||
})
|
||||
if len(tagPool) > tagQuota {
|
||||
tagPool = tagPool[:tagQuota]
|
||||
}
|
||||
picked = append(picked, tagPool...)
|
||||
}
|
||||
|
||||
// 2) 随机补齐:从全库取一批(避开已选 ID),洗牌后取剩下的名额
|
||||
remaining := total - len(picked)
|
||||
if remaining > 0 {
|
||||
items, _, err := s.Catalog.ListVideos(ctx, catalog.ListParams{
|
||||
Sort: "latest", Page: 1, PageSize: 200,
|
||||
})
|
||||
if err == nil {
|
||||
var randomPool []*catalog.Video
|
||||
for _, v := range items {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[v.ID] = struct{}{}
|
||||
randomPool = append(randomPool, v)
|
||||
}
|
||||
rand.Shuffle(len(randomPool), func(i, j int) {
|
||||
randomPool[i], randomPool[j] = randomPool[j], randomPool[i]
|
||||
})
|
||||
if len(randomPool) > remaining {
|
||||
randomPool = randomPool[:remaining]
|
||||
}
|
||||
picked = append(picked, randomPool...)
|
||||
}
|
||||
// 2) 随机补齐:同样优先已有封面的全库候选,不够再回退。
|
||||
if len(picked) < total {
|
||||
picked = appendRandomRelated(
|
||||
picked,
|
||||
s.relatedListPool(ctx, seen, true, 200),
|
||||
total,
|
||||
seen,
|
||||
)
|
||||
}
|
||||
if len(picked) < total {
|
||||
picked = appendRandomRelated(
|
||||
picked,
|
||||
s.relatedListPool(ctx, seen, false, 200),
|
||||
total,
|
||||
seen,
|
||||
)
|
||||
}
|
||||
|
||||
return picked
|
||||
}
|
||||
|
||||
func (s *Server) relatedTagPool(ctx context.Context, tags []string, seen map[string]struct{}, readyOnly bool) []*catalog.Video {
|
||||
var pool []*catalog.Video
|
||||
poolSeen := make(map[string]struct{})
|
||||
for _, tag := range tags {
|
||||
if tag == "" {
|
||||
continue
|
||||
}
|
||||
items, _, err := s.Catalog.ListVideos(ctx, catalog.ListParams{
|
||||
Tag: tag,
|
||||
Sort: "latest",
|
||||
Page: 1,
|
||||
PageSize: 30,
|
||||
ThumbnailReadyOnly: readyOnly,
|
||||
PreferReadyThumbnails: !readyOnly,
|
||||
})
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, v := range items {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
if _, ok := poolSeen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
poolSeen[v.ID] = struct{}{}
|
||||
pool = append(pool, v)
|
||||
}
|
||||
}
|
||||
return pool
|
||||
}
|
||||
|
||||
func (s *Server) relatedListPool(ctx context.Context, seen map[string]struct{}, readyOnly bool, pageSize int) []*catalog.Video {
|
||||
items, _, err := s.Catalog.ListVideos(ctx, catalog.ListParams{
|
||||
Sort: "latest",
|
||||
Page: 1,
|
||||
PageSize: pageSize,
|
||||
ThumbnailReadyOnly: readyOnly,
|
||||
PreferReadyThumbnails: !readyOnly,
|
||||
})
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
pool := make([]*catalog.Video, 0, len(items))
|
||||
for _, v := range items {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
pool = append(pool, v)
|
||||
}
|
||||
return pool
|
||||
}
|
||||
|
||||
func appendRandomRelated(picked []*catalog.Video, pool []*catalog.Video, targetLen int, seen map[string]struct{}) []*catalog.Video {
|
||||
if len(picked) >= targetLen || len(pool) == 0 {
|
||||
return picked
|
||||
}
|
||||
rand.Shuffle(len(pool), func(i, j int) {
|
||||
pool[i], pool[j] = pool[j], pool[i]
|
||||
})
|
||||
for _, v := range pool {
|
||||
if len(picked) >= targetLen {
|
||||
break
|
||||
}
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[v.ID] = struct{}{}
|
||||
picked = append(picked, v)
|
||||
}
|
||||
return picked
|
||||
}
|
||||
|
||||
func (s *Server) handleTags(w http.ResponseWriter, r *http.Request) {
|
||||
now := time.Now()
|
||||
s.tagCacheMu.Lock()
|
||||
if s.tagCache != nil && now.Before(s.tagCacheUntil) {
|
||||
out := append([]TagDTO(nil), s.tagCache...)
|
||||
s.tagCacheMu.Unlock()
|
||||
w.Header().Set("Cache-Control", "private, max-age=15")
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
return
|
||||
}
|
||||
s.tagCacheMu.Unlock()
|
||||
|
||||
stats, err := s.Catalog.ListTags(r.Context())
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
type tag struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
out := make([]tag, 0, len(stats))
|
||||
out := make([]TagDTO, 0, len(stats))
|
||||
for _, stat := range stats {
|
||||
out = append(out, tag{ID: stat.Label, Label: stat.Label, Count: stat.Count})
|
||||
out = append(out, TagDTO{ID: stat.Label, Label: stat.Label, Count: stat.Count})
|
||||
}
|
||||
s.tagCacheMu.Lock()
|
||||
s.tagCache = append([]TagDTO(nil), out...)
|
||||
s.tagCacheUntil = now.Add(30 * time.Second)
|
||||
s.tagCacheMu.Unlock()
|
||||
|
||||
w.Header().Set("Cache-Control", "private, max-age=15")
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// shortsNextReq 客户端把当前轮已看过的 video id 列表传上来,
|
||||
// 服务器从未在列表中的视频里随机抽 count 个返回。
|
||||
// shortsNextReq 客户端把当前轮已看过的 video id 列表传上来。
|
||||
// PreferredFromVideoID 来自短视频页最近一次点赞成功的视频,用于优先推荐相似标签。
|
||||
type shortsNextReq struct {
|
||||
SeenIDs []string `json:"seenIds"`
|
||||
Count int `json:"count"`
|
||||
SeenIDs []string `json:"seenIds"`
|
||||
Count int `json:"count"`
|
||||
PreferredFromVideoID string `json:"preferredFromVideoId"`
|
||||
}
|
||||
|
||||
// ShortsItemDTO 是短视频流单条的精简结构。比 VideoDTO 多 videoSrc / poster,
|
||||
@@ -386,7 +575,12 @@ func (s *Server) handleShortsNext(w http.ResponseWriter, r *http.Request) {
|
||||
exclude = nil
|
||||
}
|
||||
|
||||
items, err := s.Catalog.RandomVideosExcluding(r.Context(), exclude, count)
|
||||
var items []*catalog.Video
|
||||
if strings.TrimSpace(body.PreferredFromVideoID) != "" {
|
||||
items, err = s.Catalog.RandomVideosForPreferredVideoExcluding(r.Context(), body.PreferredFromVideoID, exclude, count)
|
||||
} else {
|
||||
items, err = s.Catalog.RandomVideosExcluding(r.Context(), exclude, count)
|
||||
}
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
@@ -698,14 +892,19 @@ func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
func (s *Server) handleThumb(w http.ResponseWriter, r *http.Request) {
|
||||
videoID := chi.URLParam(r, "videoID")
|
||||
// 直接读本地 thumbs 目录中 <videoID>.jpg
|
||||
path := filepath.Join(s.LocalDir, "thumbs", videoID+".jpg")
|
||||
clean := filepath.Clean(path)
|
||||
if !strings.HasPrefix(clean, filepath.Clean(s.LocalDir)) {
|
||||
http.Error(w, "invalid path", http.StatusForbidden)
|
||||
return
|
||||
var clean string
|
||||
for _, path := range mediaasset.ThumbnailPathCandidates(s.LocalDir, videoID) {
|
||||
candidate := filepath.Clean(path)
|
||||
if !strings.HasPrefix(candidate, filepath.Clean(s.LocalDir)) {
|
||||
http.Error(w, "invalid path", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if _, err := os.Stat(candidate); err == nil {
|
||||
clean = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(clean); err != nil {
|
||||
if clean == "" {
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
@@ -757,10 +956,14 @@ func previewURL(v *catalog.Video) string {
|
||||
}
|
||||
|
||||
func thumbnailURL(v *catalog.Video) string {
|
||||
base := "/p/thumb/" + v.ID
|
||||
if v.ThumbnailURL != "" {
|
||||
return v.ThumbnailURL
|
||||
base = v.ThumbnailURL
|
||||
}
|
||||
return "/p/thumb/" + v.ID
|
||||
if !strings.HasPrefix(base, "/p/thumb/") || v.UpdatedAt.IsZero() {
|
||||
return base
|
||||
}
|
||||
return base + "?v=" + strconv.FormatInt(v.UpdatedAt.UnixMilli(), 10)
|
||||
}
|
||||
|
||||
func (s *Server) videoSource(v *catalog.Video) string {
|
||||
@@ -790,12 +993,18 @@ func driveKindLabel(kind string) string {
|
||||
return "夸克网盘"
|
||||
case "p115":
|
||||
return "115 网盘"
|
||||
case "p123":
|
||||
return "123 云盘"
|
||||
case "pikpak":
|
||||
return "PikPak"
|
||||
case "wopan":
|
||||
return "联通沃盘"
|
||||
case "onedrive":
|
||||
return "OneDrive"
|
||||
case "googledrive":
|
||||
return "Google Drive"
|
||||
case localstorage.Kind:
|
||||
return "本地存储"
|
||||
case spider91.Kind:
|
||||
return "91 爬虫"
|
||||
default:
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -16,6 +17,7 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
)
|
||||
|
||||
@@ -98,6 +100,297 @@ func TestPreviewURLFallsBackWithoutUpdatedAt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailURLVersionsLocalGeneratedThumbnails(t *testing.T) {
|
||||
got := thumbnailURL(&catalog.Video{
|
||||
ID: "video-1",
|
||||
ThumbnailURL: "/p/thumb/video-1",
|
||||
UpdatedAt: time.UnixMilli(1778863000123),
|
||||
})
|
||||
if got != "/p/thumb/video-1?v=1778863000123" {
|
||||
t.Fatalf("thumbnail URL = %q, want versioned local URL", got)
|
||||
}
|
||||
|
||||
remote := "https://thumb.example/video-1.jpg"
|
||||
got = thumbnailURL(&catalog.Video{
|
||||
ID: "video-1",
|
||||
ThumbnailURL: remote,
|
||||
UpdatedAt: time.UnixMilli(1778863000123),
|
||||
})
|
||||
if got != remote {
|
||||
t.Fatalf("remote thumbnail URL = %q, want unchanged %q", got, remote)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleHomePrioritizesVideosWithReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < 20; i++ {
|
||||
id := "pending-video-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
PublishedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
CreatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
UpdatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed pending video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < homePageSize+2; i++ {
|
||||
id := "ready-video-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
|
||||
PublishedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
CreatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
UpdatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/home", nil)
|
||||
(&Server{Catalog: cat}).handleHome(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got []VideoDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(got) != homePageSize {
|
||||
t.Fatalf("home items = %d, want %d", len(got), homePageSize)
|
||||
}
|
||||
for _, item := range got {
|
||||
if !strings.HasPrefix(item.ID, "ready-video-") {
|
||||
t.Fatalf("home returned %q without a ready thumbnail; items=%#v", item.ID, got)
|
||||
}
|
||||
if !strings.HasPrefix(item.Thumbnail, "https://thumb.example/") {
|
||||
t.Fatalf("thumbnail for %q = %q, want ready thumbnail URL", item.ID, item.Thumbnail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleHomeExcludesRecentlyShownVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < homePageSize+4; i++ {
|
||||
id := "ready-video-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
|
||||
PublishedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
CreatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
UpdatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/home?exclude=ready-video-0&exclude=ready-video-1", nil)
|
||||
(&Server{Catalog: cat}).handleHome(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got []VideoDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(got) != homePageSize {
|
||||
t.Fatalf("home items = %d, want %d", len(got), homePageSize)
|
||||
}
|
||||
for _, item := range got {
|
||||
if item.ID == "ready-video-0" || item.ID == "ready-video-1" {
|
||||
t.Fatalf("home returned excluded video %q; items=%#v", item.ID, got)
|
||||
}
|
||||
if !strings.HasPrefix(item.ID, "ready-video-") {
|
||||
t.Fatalf("home returned %q without a ready thumbnail; items=%#v", item.ID, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleHomeStartsNewRoundWhenRecentExcludesAllVisibleVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
excludes := make([]string, 0, homePageSize+2)
|
||||
for i := 0; i < homePageSize+2; i++ {
|
||||
id := "ready-video-" + strconv.Itoa(i)
|
||||
excludes = append(excludes, "exclude="+id)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
|
||||
PublishedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
CreatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
UpdatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/home?"+strings.Join(excludes, "&"), nil)
|
||||
(&Server{Catalog: cat}).handleHome(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got []VideoDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(got) != homePageSize {
|
||||
t.Fatalf("home items = %d, want %d; body=%s", len(got), homePageSize, rr.Body.String())
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, item := range got {
|
||||
if seen[item.ID] {
|
||||
t.Fatalf("home returned duplicate video %q; items=%#v", item.ID, got)
|
||||
}
|
||||
seen[item.ID] = true
|
||||
if !strings.HasPrefix(item.ID, "ready-video-") {
|
||||
t.Fatalf("home returned unexpected video %q; items=%#v", item.ID, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListLatestPrefersReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < 20; i++ {
|
||||
id := "pending-latest-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
PublishedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
CreatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
UpdatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed pending video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < 12; i++ {
|
||||
id := "ready-latest-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
|
||||
PublishedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
CreatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
UpdatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/list?page=1&size=12&sort=latest", nil)
|
||||
(&Server{Catalog: cat}).handleList(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
Items []VideoDTO `json:"items"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if got.Total != 32 {
|
||||
t.Fatalf("total = %d, want all matching videos included", got.Total)
|
||||
}
|
||||
if len(got.Items) != 12 {
|
||||
t.Fatalf("items = %d, want 12", len(got.Items))
|
||||
}
|
||||
for _, item := range got.Items {
|
||||
if !strings.HasPrefix(item.ID, "ready-latest-") {
|
||||
t.Fatalf("latest list returned %q before ready thumbnails; items=%#v", item.ID, got.Items)
|
||||
}
|
||||
if !strings.HasPrefix(item.Thumbnail, "https://thumb.example/") {
|
||||
t.Fatalf("thumbnail for %q = %q, want ready thumbnail URL", item.ID, item.Thumbnail)
|
||||
}
|
||||
}
|
||||
|
||||
rr = httptest.NewRecorder()
|
||||
req = httptest.NewRequest(http.MethodGet, "/api/list?page=1&size=12&sort=latest&count=false", nil)
|
||||
(&Server{Catalog: cat}).handleList(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("count=false status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
got = struct {
|
||||
Items []VideoDTO `json:"items"`
|
||||
Total int `json:"total"`
|
||||
}{}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode count=false response: %v", err)
|
||||
}
|
||||
if got.Total != 0 {
|
||||
t.Fatalf("count=false total = %d, want 0", got.Total)
|
||||
}
|
||||
if len(got.Items) != 12 {
|
||||
t.Fatalf("count=false items = %d, want 12", len(got.Items))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUploadVideoSavesFileVideoTagsAndQueuesPreview(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -120,7 +413,7 @@ func TestHandleUploadVideoSavesFileVideoTagsAndQueuesPreview(t *testing.T) {
|
||||
}
|
||||
req := multipartUploadRequest(t, map[string]string{
|
||||
"title": "用户上传标题",
|
||||
"tags": "奶子,AV,女大",
|
||||
"tags": "奶子,口交,AV,女大",
|
||||
}, "clip.mp4", "video-bytes")
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
@@ -146,7 +439,7 @@ func TestHandleUploadVideoSavesFileVideoTagsAndQueuesPreview(t *testing.T) {
|
||||
if got.Title != "用户上传标题" {
|
||||
t.Fatalf("title = %q, want submitted title", got.Title)
|
||||
}
|
||||
if !sameStringSet(got.Tags, []string{"奶子", "AV", "女大"}) {
|
||||
if !sameStringSet(got.Tags, []string{"奶子", "口交", "AV", "女大"}) {
|
||||
t.Fatalf("tags = %#v, want selected tags", got.Tags)
|
||||
}
|
||||
if got.PreviewStatus != "pending" {
|
||||
@@ -317,6 +610,34 @@ func TestHandlePreviewIgnoresRemotePreviewFileIDAndServesLocalFile(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleThumbServesHashedPathForLongVideoID(t *testing.T) {
|
||||
localDir := t.TempDir()
|
||||
longID := "localstorage-" + strings.Repeat("x", 240)
|
||||
thumbPath := mediaasset.ThumbnailPath(localDir, longID)
|
||||
if err := os.MkdirAll(filepath.Dir(thumbPath), 0o755); err != nil {
|
||||
t.Fatalf("mkdir thumb dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(thumbPath, []byte("thumb-bytes"), 0o644); err != nil {
|
||||
t.Fatalf("write thumb: %v", err)
|
||||
}
|
||||
|
||||
server := &Server{
|
||||
LocalDir: localDir,
|
||||
Proxy: proxy.New(proxy.NewRegistry()),
|
||||
}
|
||||
req := requestWithRouteParam(http.MethodGet, "/p/thumb/"+longID, "videoID", longID, strings.NewReader(``))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
server.handleThumb(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if rr.Body.String() != "thumb-bytes" {
|
||||
t.Fatalf("body = %q, want thumb bytes", rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleTagsReturnsUnifiedTagPool(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -382,6 +703,66 @@ func TestHandleTagsReturnsUnifiedTagPool(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleShortsNextUsesPreferredVideoLeastPopulatedTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*catalog.Video{
|
||||
{ID: "current", DriveID: "drive", FileID: "f-current", Title: "current", Tags: []string{"common", "rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-1", DriveID: "drive", FileID: "f-common-1", Title: "common 1", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-2", DriveID: "drive", FileID: "f-common-2", Title: "common 2", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "rare-1", DriveID: "drive", FileID: "f-rare-1", Title: "rare 1", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/shorts/next", strings.NewReader(`{"seenIds":["current"],"count":3,"preferredFromVideoId":"current"}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&Server{Catalog: cat}).handleShortsNext(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
Items []ShortsItemDTO `json:"items"`
|
||||
Total int `json:"total"`
|
||||
RoundComplete bool `json:"roundComplete"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
ids := make([]string, 0, len(got.Items))
|
||||
for _, item := range got.Items {
|
||||
ids = append(ids, item.ID)
|
||||
}
|
||||
if got.Total != 4 {
|
||||
t.Fatalf("total = %d, want 4", got.Total)
|
||||
}
|
||||
if got.RoundComplete {
|
||||
t.Fatalf("roundComplete = true, want false with fallback-filled batch")
|
||||
}
|
||||
if !containsString(ids, "rare-1") {
|
||||
t.Fatalf("ids = %#v, want rare-1 from least populated tag", ids)
|
||||
}
|
||||
if containsString(ids, "current") {
|
||||
t.Fatalf("ids = %#v, should exclude current", ids)
|
||||
}
|
||||
if len(ids) != 3 {
|
||||
t.Fatalf("ids = %#v, want 3 items", ids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpdateVideoTagsRejectsUnknownTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -509,6 +890,88 @@ func TestHandleVideoDetailIncludesDriveKindLabel(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleVideoDetailRecommendationsPreferReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "current-video",
|
||||
DriveID: "drive",
|
||||
FileID: "current-video",
|
||||
Title: "Current",
|
||||
Tags: []string{"same-tag"},
|
||||
ThumbnailURL: "https://thumb.example/current-video.jpg",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed current video: %v", err)
|
||||
}
|
||||
for i := 0; i < 20; i++ {
|
||||
id := "pending-related-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
Tags: []string{"same-tag"},
|
||||
PublishedAt: now.Add(time.Duration(i+1) * time.Minute),
|
||||
CreatedAt: now.Add(time.Duration(i+1) * time.Minute),
|
||||
UpdatedAt: now.Add(time.Duration(i+1) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed pending related video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < 8; i++ {
|
||||
id := "ready-related-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
Tags: []string{"same-tag"},
|
||||
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
|
||||
PublishedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
CreatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
UpdatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready related video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
req := requestWithVideoID(http.MethodGet, "/api/video/current-video", "current-video", strings.NewReader(``))
|
||||
rr := httptest.NewRecorder()
|
||||
(&Server{Catalog: cat}).handleVideoDetail(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got VideoDetailDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(got.RelatedVideos) != 6 {
|
||||
t.Fatalf("related videos = %d, want 6; items=%#v", len(got.RelatedVideos), got.RelatedVideos)
|
||||
}
|
||||
for _, item := range got.RelatedVideos {
|
||||
if !strings.HasPrefix(item.ID, "ready-related-") {
|
||||
t.Fatalf("related returned %q before ready thumbnails; items=%#v", item.ID, got.RelatedVideos)
|
||||
}
|
||||
if !strings.HasPrefix(item.Thumbnail, "https://thumb.example/") {
|
||||
t.Fatalf("thumbnail for %q = %q, want ready thumbnail URL", item.ID, item.Thumbnail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleHideVideoRemovesVideoFromPublicListAndDetail(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,126 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUpsertDriveUsesRootIDAsScanRootID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: "drive",
|
||||
Kind: "p115",
|
||||
Name: "115",
|
||||
RootID: "root-folder",
|
||||
ScanRootID: "ignored-scan-root",
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert drive: %v", err)
|
||||
}
|
||||
|
||||
got, err := cat.GetDrive(ctx, "drive")
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.RootID != "root-folder" {
|
||||
t.Fatalf("rootId = %q, want root-folder", got.RootID)
|
||||
}
|
||||
if got.ScanRootID != "root-folder" {
|
||||
t.Fatalf("scanRootId = %q, want root-folder", got.ScanRootID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertDriveDefaultsRootIDByKind(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
cases := []struct {
|
||||
id string
|
||||
kind string
|
||||
want string
|
||||
}{
|
||||
{id: "p115", kind: "p115", want: "0"},
|
||||
{id: "pikpak", kind: "pikpak", want: ""},
|
||||
{id: "onedrive", kind: "onedrive", want: "root"},
|
||||
{id: "googledrive", kind: "googledrive", want: "root"},
|
||||
{id: "localstorage", kind: "localstorage", want: "/"},
|
||||
{id: "spider91", kind: "spider91", want: "/"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: tc.id,
|
||||
Kind: tc.kind,
|
||||
Name: tc.kind,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert %s: %v", tc.kind, err)
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, tc.id)
|
||||
if err != nil {
|
||||
t.Fatalf("get %s: %v", tc.kind, err)
|
||||
}
|
||||
if got.RootID != tc.want {
|
||||
t.Fatalf("%s rootId = %q, want %q", tc.kind, got.RootID, tc.want)
|
||||
}
|
||||
if got.ScanRootID != tc.want {
|
||||
t.Fatalf("%s scanRootId = %q, want %q", tc.kind, got.ScanRootID, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertDriveIgnoresRootIDForLocalStorageAndSpider91(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, tc := range []struct {
|
||||
id string
|
||||
kind string
|
||||
}{
|
||||
{id: "localstorage", kind: "localstorage"},
|
||||
{id: "spider91", kind: "spider91"},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: tc.id,
|
||||
Kind: tc.kind,
|
||||
Name: tc.kind,
|
||||
RootID: "manual-root",
|
||||
ScanRootID: "manual-scan-root",
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert %s: %v", tc.kind, err)
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, tc.id)
|
||||
if err != nil {
|
||||
t.Fatalf("get %s: %v", tc.kind, err)
|
||||
}
|
||||
if got.RootID != "/" {
|
||||
t.Fatalf("%s rootId = %q, want /", tc.kind, got.RootID)
|
||||
}
|
||||
if got.ScanRootID != "/" {
|
||||
t.Fatalf("%s scanRootId = %q, want /", tc.kind, got.ScanRootID)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -126,3 +127,50 @@ func TestListSpider91ViewkeysFindsMigratedVideos(t *testing.T) {
|
||||
t.Fatalf("non-existent drive: got %v, want empty", other)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteVideoWithTombstonePreventsReimport(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: "spider91-91Spider-vk004",
|
||||
DriveID: "91Spider",
|
||||
FileID: "vk004.mp4",
|
||||
FileName: "vk004.mp4",
|
||||
ContentHash: "ABCDEF",
|
||||
Title: "Deleted Spider",
|
||||
Size: 2048,
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert: %v", err)
|
||||
}
|
||||
|
||||
if err := cat.DeleteVideoWithTombstone(ctx, "spider91-91Spider-vk004"); err != nil {
|
||||
t.Fatalf("delete with tombstone: %v", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "spider91-91Spider-vk004"); err != sql.ErrNoRows {
|
||||
t.Fatalf("get deleted video error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
deleted, err := cat.IsDeletedVideoCandidate(ctx, "spider91-91Spider-vk004", "91Spider", "vk004.mp4", "abcdef", "vk004.mp4", 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("check deleted candidate: %v", err)
|
||||
}
|
||||
if !deleted {
|
||||
t.Fatal("deleted candidate was not recognized")
|
||||
}
|
||||
viewkeys, err := cat.ListSpider91Viewkeys(ctx, "91Spider")
|
||||
if err != nil {
|
||||
t.Fatalf("ListSpider91Viewkeys: %v", err)
|
||||
}
|
||||
if len(viewkeys) != 1 || viewkeys[0] != "vk004" {
|
||||
t.Fatalf("viewkeys = %#v, want [vk004]", viewkeys)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestListVideosDeduplicatesBySampledSHA256(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{
|
||||
ID: "drive-a-file-a",
|
||||
DriveID: "drive-a",
|
||||
FileID: "file-a",
|
||||
FileName: "first-name.mp4",
|
||||
Title: "First",
|
||||
Size: 1234,
|
||||
PublishedAt: now.Add(-time.Minute),
|
||||
CreatedAt: now.Add(-time.Minute),
|
||||
UpdatedAt: now.Add(-time.Minute),
|
||||
},
|
||||
{
|
||||
ID: "drive-b-file-b",
|
||||
DriveID: "drive-b",
|
||||
FileID: "file-b",
|
||||
FileName: "second-name.mp4",
|
||||
Title: "Second",
|
||||
Size: 1234,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("upsert %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list before fingerprint: %v", err)
|
||||
}
|
||||
if total != 2 || len(items) != 2 {
|
||||
t.Fatalf("before fingerprint total=%d len=%d, want 2", total, len(items))
|
||||
}
|
||||
|
||||
const sampled = "abc123"
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "drive-a-file-a", sampled, "ready", ""); err != nil {
|
||||
t.Fatalf("update a fingerprint: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "drive-b-file-b", sampled, "ready", ""); err != nil {
|
||||
t.Fatalf("update b fingerprint: %v", err)
|
||||
}
|
||||
|
||||
items, total, err = cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list after fingerprint: %v", err)
|
||||
}
|
||||
if total != 1 || len(items) != 1 {
|
||||
t.Fatalf("after fingerprint total=%d len=%d, want 1", total, len(items))
|
||||
}
|
||||
if items[0].ID != "drive-a-file-a" {
|
||||
t.Fatalf("canonical id = %q, want earliest created video", items[0].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuplicateAssetCleanupCandidates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
base := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
|
||||
videos := []*Video{
|
||||
{
|
||||
ID: "drive-a-canonical",
|
||||
DriveID: "drive-a",
|
||||
FileID: "file-a",
|
||||
FileName: "canonical.mp4",
|
||||
Title: "Canonical",
|
||||
Size: 1234,
|
||||
ThumbnailURL: "/p/thumb/drive-a-canonical",
|
||||
PreviewLocal: "/tmp/previews/canonical.mp4",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: base,
|
||||
CreatedAt: base,
|
||||
UpdatedAt: base,
|
||||
},
|
||||
{
|
||||
ID: "drive-b-duplicate",
|
||||
DriveID: "drive-b",
|
||||
FileID: "file-b",
|
||||
FileName: "duplicate.mp4",
|
||||
Title: "Duplicate",
|
||||
Size: 1234,
|
||||
ThumbnailURL: "/p/thumb/drive-b-duplicate",
|
||||
PreviewLocal: "/tmp/previews/duplicate.mp4",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: base.Add(time.Second),
|
||||
CreatedAt: base.Add(time.Second),
|
||||
UpdatedAt: base.Add(time.Second),
|
||||
},
|
||||
{
|
||||
ID: "drive-c-remote-thumb",
|
||||
DriveID: "drive-c",
|
||||
FileID: "file-c",
|
||||
FileName: "remote-thumb.mp4",
|
||||
Title: "Remote Thumbnail",
|
||||
Size: 1234,
|
||||
ThumbnailURL: "https://thumb.example/file-c.jpg",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: base.Add(2 * time.Second),
|
||||
CreatedAt: base.Add(2 * time.Second),
|
||||
UpdatedAt: base.Add(2 * time.Second),
|
||||
},
|
||||
}
|
||||
for _, v := range videos {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
const sampled = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
for _, v := range videos {
|
||||
if err := cat.UpdateVideoFingerprint(ctx, v.ID, sampled, "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, err := cat.ListDuplicateAssetCleanupCandidates(ctx, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list cleanup candidates: %v", err)
|
||||
}
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("candidates = %#v, want only local duplicate", items)
|
||||
}
|
||||
item := items[0]
|
||||
if item.VideoID != "drive-b-duplicate" || item.CanonicalID != "drive-a-canonical" {
|
||||
t.Fatalf("candidate = %#v, want duplicate with canonical", item)
|
||||
}
|
||||
|
||||
if err := cat.ClearGeneratedAssets(ctx, item.VideoID, true, true); err != nil {
|
||||
t.Fatalf("clear generated assets: %v", err)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, item.VideoID)
|
||||
if err != nil {
|
||||
t.Fatalf("get duplicate: %v", err)
|
||||
}
|
||||
if got.PreviewLocal != "" || got.PreviewStatus != "pending" {
|
||||
t.Fatalf("preview after cleanup local=%q status=%q, want empty pending", got.PreviewLocal, got.PreviewStatus)
|
||||
}
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail after cleanup = %q, want empty", got.ThumbnailURL)
|
||||
}
|
||||
var thumbStatus string
|
||||
if err := cat.db.QueryRowContext(ctx, `SELECT thumbnail_status FROM videos WHERE id = ?`, item.VideoID).Scan(&thumbStatus); err != nil {
|
||||
t.Fatalf("query thumbnail status: %v", err)
|
||||
}
|
||||
if thumbStatus != "pending" {
|
||||
t.Fatalf("thumbnail_status = %q, want pending", thumbStatus)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestListVideosHidesMissingDriveVideosWhenDrivesExist(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: "active-drive",
|
||||
Kind: "pikpak",
|
||||
Name: "Active",
|
||||
RootID: "root",
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{
|
||||
ID: "visible-video",
|
||||
DriveID: "active-drive",
|
||||
FileID: "visible-file",
|
||||
Title: "Visible",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "orphan-video",
|
||||
DriveID: "deleted-drive",
|
||||
FileID: "orphan-file",
|
||||
Title: "Orphan",
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed video %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 10, Sort: "latest"})
|
||||
if err != nil {
|
||||
t.Fatalf("list videos: %v", err)
|
||||
}
|
||||
if total != 1 || len(items) != 1 || items[0].ID != "visible-video" {
|
||||
t.Fatalf("items total=%d items=%v, want only visible-video", total, items)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,9 @@ CREATE TABLE IF NOT EXISTS videos (
|
||||
file_id TEXT NOT NULL,
|
||||
file_name TEXT DEFAULT '', -- 网盘侧原始文件名,用于同名同大小去重
|
||||
content_hash TEXT DEFAULT '',
|
||||
sampled_sha256 TEXT DEFAULT '', -- 跨网盘统一采样指纹(size + sampled bytes)
|
||||
fingerprint_status TEXT DEFAULT 'pending', -- pending / ready / failed
|
||||
fingerprint_error TEXT DEFAULT '',
|
||||
parent_id TEXT,
|
||||
title TEXT NOT NULL,
|
||||
author TEXT,
|
||||
@@ -14,9 +17,10 @@ CREATE TABLE IF NOT EXISTS videos (
|
||||
ext TEXT,
|
||||
quality TEXT, -- HD / SD
|
||||
thumbnail_url TEXT,
|
||||
thumbnail_status TEXT DEFAULT 'pending', -- pending / ready / failed
|
||||
preview_file_id TEXT, -- deprecated: 旧版回写网盘后的 teaser file id
|
||||
preview_local TEXT, -- 本地 teaser 路径(兜底)
|
||||
thumbnail_status TEXT DEFAULT 'pending', -- pending / ready / failed / skipped
|
||||
thumbnail_failures INTEGER DEFAULT 0, -- consecutive transient thumbnail generation failures
|
||||
preview_file_id TEXT, -- deprecated: 旧版回写网盘后的预览视频 file id
|
||||
preview_local TEXT, -- 本地预览视频路径(兜底)
|
||||
preview_status TEXT DEFAULT 'pending', -- pending / ready / failed
|
||||
views INTEGER DEFAULT 0,
|
||||
favorites INTEGER DEFAULT 0,
|
||||
@@ -58,17 +62,44 @@ CREATE TABLE IF NOT EXISTS video_tags (
|
||||
CREATE INDEX IF NOT EXISTS idx_video_tags_tag ON video_tags(tag_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_video_tags_video ON video_tags(video_id);
|
||||
|
||||
-- 用户手动删除过的非系统标签。自动扫描/迁移不再重新创建同名标签;
|
||||
-- 管理员手动新建同名标签时会移除这里的记录。
|
||||
CREATE TABLE IF NOT EXISTS deleted_tags (
|
||||
label TEXT PRIMARY KEY COLLATE NOCASE,
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
deleted_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- 管理员显式删除过的视频。用于防止后续扫描 / spider91 爬虫把同一个源文件
|
||||
-- 再次入库;不代表原始云盘文件已被删除。
|
||||
CREATE TABLE IF NOT EXISTS deleted_videos (
|
||||
id TEXT PRIMARY KEY,
|
||||
drive_id TEXT NOT NULL DEFAULT '',
|
||||
file_id TEXT NOT NULL DEFAULT '',
|
||||
content_hash TEXT NOT NULL DEFAULT '',
|
||||
file_name TEXT NOT NULL DEFAULT '',
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
deleted_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_deleted_videos_drive_file
|
||||
ON deleted_videos(drive_id, file_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_deleted_videos_drive_hash
|
||||
ON deleted_videos(drive_id, content_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_deleted_videos_drive_signature
|
||||
ON deleted_videos(drive_id, file_name, size_bytes);
|
||||
|
||||
-- 网盘账户
|
||||
CREATE TABLE IF NOT EXISTS drives (
|
||||
id TEXT PRIMARY KEY,
|
||||
kind TEXT NOT NULL, -- quark / p115 / pikpak / wopan / onedrive / spider91
|
||||
kind TEXT NOT NULL, -- quark / p115 / p123 / pikpak / wopan / onedrive / googledrive / localstorage / spider91
|
||||
name TEXT NOT NULL,
|
||||
root_id TEXT NOT NULL DEFAULT '0',
|
||||
scan_root_id TEXT, -- 扫描起点(默认 root_id)
|
||||
scan_root_id TEXT, -- deprecated: 扫描起点固定等于 root_id
|
||||
credentials TEXT, -- JSON: cookie / refresh_token 等
|
||||
status TEXT DEFAULT 'disconnected', -- disconnected / ok / error
|
||||
last_error TEXT,
|
||||
-- 是否给该盘生成 teaser/封面:1 开 / 0 关。
|
||||
-- 是否给该盘生成预览视频/封面:1 开 / 0 关。
|
||||
-- 替代了早期的全局 preview.enabled 设置(保留旧 setting 行不再读)。
|
||||
teaser_enabled INTEGER NOT NULL DEFAULT 1,
|
||||
-- 扫描时要跳过的目录 ID 集合(JSON array of string)。命中其中任意一个的目录及其
|
||||
|
||||
@@ -109,3 +109,227 @@ func TestRandomVideosExcluding(t *testing.T) {
|
||||
t.Fatalf("limit 0 should return nil, got %v", got4)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosWithReadyThumbnailsExcluding(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < 4; i++ {
|
||||
id := "ready-" + string(rune('a'+i))
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: "f-" + id,
|
||||
Title: id,
|
||||
ThumbnailURL: "/p/thumb/" + id,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
id := "pending-" + string(rune('a'+i))
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: "f-" + id,
|
||||
Title: id,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosWithReadyThumbnailsExcluding(ctx, []string{"ready-a"}, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("random ready excluding: %v", err)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("ready random count = %d, want 3", len(got))
|
||||
}
|
||||
for _, v := range got {
|
||||
if v.ID == "ready-a" {
|
||||
t.Fatal("excluded ready video was returned")
|
||||
}
|
||||
if v.ThumbnailURL == "" {
|
||||
t.Fatalf("pending video %q was returned", v.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoChoosesLeastPopulatedTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "current", DriveID: "drive", FileID: "f-current", Title: "current", Tags: []string{"common", "rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-1", DriveID: "drive", FileID: "f-common-1", Title: "common 1", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-2", DriveID: "drive", FileID: "f-common-2", Title: "common 2", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "rare-1", DriveID: "drive", FileID: "f-rare-1", Title: "rare 1", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
tag, err := cat.LeastPopulatedVisibleUniqueTag(ctx, []string{"common", "rare"})
|
||||
if err != nil {
|
||||
t.Fatalf("least populated tag: %v", err)
|
||||
}
|
||||
if tag != "rare" {
|
||||
t.Fatalf("least populated tag = %q, want rare", tag)
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosForPreferredVideoExcluding(ctx, "current", []string{"current"}, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("random preferred: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID != "rare-1" {
|
||||
t.Fatalf("preferred result = %#v, want rare-1", videoIDs(got))
|
||||
}
|
||||
|
||||
got, err = cat.RandomVideosForPreferredVideoExcluding(ctx, "current", nil, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("random preferred without explicit exclude: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID == "current" {
|
||||
t.Fatalf("preferred result without explicit exclude = %#v, should not return current", videoIDs(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoFallsBackToFillBatch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "current", DriveID: "drive", FileID: "f-current", Title: "current", Tags: []string{"common", "rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-1", DriveID: "drive", FileID: "f-common-1", Title: "common 1", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-2", DriveID: "drive", FileID: "f-common-2", Title: "common 2", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "rare-1", DriveID: "drive", FileID: "f-rare-1", Title: "rare 1", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "hidden-rare", DriveID: "drive", FileID: "f-hidden-rare", Title: "hidden rare", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
if err := cat.HideVideo(ctx, "hidden-rare"); err != nil {
|
||||
t.Fatalf("hide hidden-rare: %v", err)
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosForPreferredVideoExcluding(ctx, "current", []string{"current"}, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("random preferred: %v", err)
|
||||
}
|
||||
ids := videoIDs(got)
|
||||
if len(ids) != 3 {
|
||||
t.Fatalf("result ids = %#v, want 3 items", ids)
|
||||
}
|
||||
for _, excluded := range []string{"current", "hidden-rare"} {
|
||||
if hasVideoID(ids, excluded) {
|
||||
t.Fatalf("result ids = %#v, should not include %s", ids, excluded)
|
||||
}
|
||||
}
|
||||
if !hasVideoID(ids, "rare-1") {
|
||||
t.Fatalf("result ids = %#v, want rare-1 from least populated tag", ids)
|
||||
}
|
||||
if len(uniqueVideoIDs(ids)) != len(ids) {
|
||||
t.Fatalf("result ids = %#v, want no duplicates", ids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoFallbacksWhenPreferenceUnavailable(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "untagged", DriveID: "drive", FileID: "f-untagged", Title: "untagged", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "visible-1", DriveID: "drive", FileID: "f-visible-1", Title: "visible 1", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "visible-2", DriveID: "drive", FileID: "f-visible-2", Title: "visible 2", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosForPreferredVideoExcluding(ctx, "missing", []string{"untagged"}, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("random missing preferred: %v", err)
|
||||
}
|
||||
if !sameVideoIDSet(videoIDs(got), []string{"visible-1", "visible-2"}) {
|
||||
t.Fatalf("missing preferred ids = %#v, want visible fallback videos", videoIDs(got))
|
||||
}
|
||||
|
||||
got, err = cat.RandomVideosForPreferredVideoExcluding(ctx, "untagged", []string{"untagged"}, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("random untagged preferred: %v", err)
|
||||
}
|
||||
if !sameVideoIDSet(videoIDs(got), []string{"visible-1", "visible-2"}) {
|
||||
t.Fatalf("untagged preferred ids = %#v, want visible fallback videos", videoIDs(got))
|
||||
}
|
||||
}
|
||||
|
||||
func videoIDs(videos []*Video) []string {
|
||||
ids := make([]string, 0, len(videos))
|
||||
for _, v := range videos {
|
||||
ids = append(ids, v.ID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func hasVideoID(ids []string, want string) bool {
|
||||
for _, id := range ids {
|
||||
if id == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func uniqueVideoIDs(ids []string) map[string]struct{} {
|
||||
seen := make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
seen[id] = struct{}{}
|
||||
}
|
||||
return seen
|
||||
}
|
||||
|
||||
func sameVideoIDSet(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
seen := make(map[string]int, len(a))
|
||||
for _, value := range a {
|
||||
seen[value]++
|
||||
}
|
||||
for _, value := range b {
|
||||
if seen[value] == 0 {
|
||||
return false
|
||||
}
|
||||
seen[value]--
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -17,6 +17,8 @@ import (
|
||||
)
|
||||
|
||||
var ErrUnknownTag = errors.New("unknown tag")
|
||||
var ErrSystemTag = errors.New("system tag cannot be deleted")
|
||||
var ErrDeletedTag = errors.New("tag was previously deleted")
|
||||
|
||||
const avTagLabel = "AV"
|
||||
|
||||
@@ -43,6 +45,15 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "content_hash", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "sampled_sha256", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "fingerprint_status", "TEXT DEFAULT 'pending'"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "fingerprint_error", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "file_name", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -52,10 +63,13 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "thumbnail_status", "TEXT DEFAULT 'pending'"); err != nil {
|
||||
return err
|
||||
}
|
||||
// drives.teaser_enabled:每盘 teaser 开关,替代旧的全局 preview.enabled。
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "thumbnail_failures", "INTEGER DEFAULT 0"); err != nil {
|
||||
return err
|
||||
}
|
||||
// drives.teaser_enabled:每盘预览视频开关,替代旧的全局 preview.enabled。
|
||||
// 升级路径:直接让 ALTER TABLE 的 DEFAULT 1 兜底 —— 每个现存 drive 都默认开启,
|
||||
// 不读旧的 settings.preview.enabled 字段。这样老用户即便之前关过全局开关,
|
||||
// 升级后所有盘也都恢复"默认生成 teaser",跟新建保持一致。
|
||||
// 升级后所有盘也都恢复"默认生成预览视频",跟新建保持一致。
|
||||
if _, err := c.addColumnIfMissingReportNew(ctx, "drives", "teaser_enabled", "INTEGER NOT NULL DEFAULT 1"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -65,6 +79,21 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "drives", "skip_dir_ids", "TEXT NOT NULL DEFAULT '[]'"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS deleted_videos (
|
||||
id TEXT PRIMARY KEY,
|
||||
drive_id TEXT NOT NULL DEFAULT '',
|
||||
file_id TEXT NOT NULL DEFAULT '',
|
||||
content_hash TEXT NOT NULL DEFAULT '',
|
||||
file_name TEXT NOT NULL DEFAULT '',
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
deleted_at INTEGER NOT NULL
|
||||
)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.syncDriveScanRootIDToRootID(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
// 一次性修正:早期版本(短暂存在过)会把现存 drive 的 teaser_enabled 同步成
|
||||
// 旧的全局 preview.enabled 值,导致升级后所有 drive 都是关。"默认开启"约定下,
|
||||
// 这里一次性把所有 drive 强制重置为 1,并用 marker setting 记号,避免之后
|
||||
@@ -83,12 +112,36 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_content_hash ON videos(content_hash)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_content_hash_created ON videos(content_hash, created_at, id)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_sampled_sha256 ON videos(size_bytes, sampled_sha256)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_sampled_sha256_created ON videos(size_bytes, sampled_sha256, created_at, id)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_hidden ON videos(hidden)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_visible_pub ON videos(COALESCE(hidden, 0), published_at DESC)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_file_name_size ON videos(file_name, size_bytes)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_file_name_size_created ON videos(file_name, size_bytes, created_at, id)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_deleted_videos_drive_file ON deleted_videos(drive_id, file_id)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_deleted_videos_drive_hash ON deleted_videos(drive_id, content_hash)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_deleted_videos_drive_signature ON deleted_videos(drive_id, file_name, size_bytes)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.seedSystemTags(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -107,6 +160,12 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.clearVolatileOneDriveThumbnails(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.clearRemoteP123ThumbnailsOnce(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.clearRemoteNonSpider91Thumbnails(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.hideZeroSizeVideosFromKnownDrives(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -155,7 +214,7 @@ func (c *Catalog) addColumnIfMissingReportNew(ctx context.Context, table, column
|
||||
// 设为 1(开启),但仅在历史上没跑过这条迁移时执行(用 marker setting 记号)。
|
||||
//
|
||||
// 为什么需要:早期短暂存在过的版本会从旧的全局 preview.enabled = "0" 同步到
|
||||
// 所有 drive 的 teaser_enabled = 0;用户报告升级后页面全显示"Teaser 关"。新版
|
||||
// 所有 drive 的 teaser_enabled = 0;用户报告升级后页面全显示"预览视频关"。新版
|
||||
// 约定 per-drive 默认开启,所以这里跑一次性修正。
|
||||
//
|
||||
// 幂等保证:marker setting 设过了就不再跑,确保用户在 UI 里把某盘关了不会被
|
||||
@@ -189,8 +248,9 @@ func (c *Catalog) resetDriveTeaserEnabledToDefaultOnce(ctx context.Context) erro
|
||||
// - 管理员凭直觉认知字段名时会被误导
|
||||
//
|
||||
// 修正策略:
|
||||
// - thumbnail_url 非空 + status 非 'ready' + status 非 'failed' → 改成 'ready'
|
||||
// - thumbnail_url 非空 + status 非 'ready' + status 非 'failed' + status 非 'skipped' → 改成 'ready'
|
||||
// - status='failed' 不动(这是 worker 显式标的失败,要保留以便管理员手动重生)
|
||||
// - status='skipped' 不动(已有封面但时长探测不可用,避免重启后重复排队)
|
||||
//
|
||||
// 幂等保证:marker setting 写过就不再跑,避免每次重启都 update 一遍。
|
||||
func (c *Catalog) reconcileThumbnailStatusOnce(ctx context.Context) error {
|
||||
@@ -207,7 +267,7 @@ UPDATE videos
|
||||
SET thumbnail_status = 'ready',
|
||||
updated_at = ?
|
||||
WHERE COALESCE(thumbnail_url, '') != ''
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('ready', 'failed')
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('ready', 'failed', 'skipped')
|
||||
`, time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
return fmt.Errorf("reconcile thumbnail_status: %w", err)
|
||||
@@ -236,6 +296,85 @@ UPDATE videos
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) clearRemoteP123ThumbnailsOnce(ctx context.Context) error {
|
||||
// 123 云盘列表返回的缩略图尺寸和稳定性都不适合作为站内封面;清空历史写入的
|
||||
// 远程 URL,让封面 worker 统一从视频直链抽帧生成本地 /p/thumb/<id>。
|
||||
const markerKey = "videos.p123.remote_thumbnails_cleared"
|
||||
marker, err := c.GetSetting(ctx, markerKey, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read %s marker: %w", markerKey, err)
|
||||
}
|
||||
if strings.TrimSpace(marker) == "1" {
|
||||
return nil
|
||||
}
|
||||
|
||||
var p123Drives int
|
||||
if err := c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM drives WHERE kind = 'p123'`).Scan(&p123Drives); err != nil {
|
||||
return fmt.Errorf("count p123 drives: %w", err)
|
||||
}
|
||||
if p123Drives == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
res, err := c.db.ExecContext(ctx, `
|
||||
UPDATE videos
|
||||
SET thumbnail_url = '',
|
||||
thumbnail_status = 'pending',
|
||||
thumbnail_failures = 0,
|
||||
updated_at = ?
|
||||
WHERE EXISTS (
|
||||
SELECT 1
|
||||
FROM drives
|
||||
WHERE drives.id = videos.drive_id
|
||||
AND drives.kind = 'p123'
|
||||
)
|
||||
AND (
|
||||
lower(COALESCE(thumbnail_url, '')) LIKE 'http://%'
|
||||
OR lower(COALESCE(thumbnail_url, '')) LIKE 'https://%'
|
||||
)
|
||||
`, time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected > 0 {
|
||||
log.Printf("[catalog] cleared %d remote 123pan thumbnail(s) for local regeneration", affected)
|
||||
}
|
||||
if err := c.SetSetting(ctx, markerKey, "1"); err != nil {
|
||||
return fmt.Errorf("write %s marker: %w", markerKey, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Catalog) clearRemoteNonSpider91Thumbnails(ctx context.Context) error {
|
||||
// 非 91Spider 视频不再使用网盘侧返回的远程缩略图。清空历史 http/https
|
||||
// thumbnail_url 后,封面 worker 会重新从视频中间帧生成本地 /p/thumb/<id>。
|
||||
// 91Spider 的封面是爬虫下载后保存到本地 /p/thumb/<id>,不受这条规则影响。
|
||||
res, err := c.db.ExecContext(ctx, `
|
||||
UPDATE videos
|
||||
SET thumbnail_url = '',
|
||||
thumbnail_status = 'pending',
|
||||
thumbnail_failures = 0,
|
||||
updated_at = ?
|
||||
WHERE (
|
||||
lower(COALESCE(thumbnail_url, '')) LIKE 'http://%'
|
||||
OR lower(COALESCE(thumbnail_url, '')) LIKE 'https://%'
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM drives
|
||||
WHERE drives.id = videos.drive_id
|
||||
AND drives.kind = 'spider91'
|
||||
)
|
||||
`, time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected > 0 {
|
||||
log.Printf("[catalog] cleared %d remote non-91Spider thumbnail(s) for local regeneration", affected)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Catalog) hideZeroSizeVideosFromKnownDrives(ctx context.Context) error {
|
||||
_, err := c.db.ExecContext(ctx, `
|
||||
UPDATE videos
|
||||
@@ -281,7 +420,15 @@ func (c *Catalog) classifySystemTags(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (c *Catalog) backfillVideoTags(ctx context.Context) error {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT id, COALESCE(tags, '[]') FROM videos`)
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT id, COALESCE(tags, '[]')
|
||||
FROM videos
|
||||
WHERE COALESCE(tags, '') NOT IN ('', '[]', 'null')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
WHERE vt.video_id = videos.id
|
||||
)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -298,11 +445,14 @@ func (c *Catalog) backfillVideoTags(ctx context.Context) error {
|
||||
if len(labels) == 0 {
|
||||
continue
|
||||
}
|
||||
if err := c.addVideoTags(ctx, videoID, labels, "legacy", true); err != nil {
|
||||
added, err := c.addVideoTags(ctx, videoID, labels, "legacy", true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return err
|
||||
if added {
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -350,6 +500,9 @@ GROUP BY category`)
|
||||
if !LooksLikeCollectionTag(stat.category) {
|
||||
continue
|
||||
}
|
||||
if c.tagDeleted(ctx, stat.category) {
|
||||
continue
|
||||
}
|
||||
if _, err := c.ensureTag(ctx, stat.category, nil, "collection"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -368,12 +521,178 @@ func (c *Catalog) CreateTagAndClassify(ctx context.Context, label string, aliase
|
||||
return c.classifyTag(ctx, tag)
|
||||
}
|
||||
|
||||
func (c *Catalog) EnsureTagForVideoIDPrefix(ctx context.Context, prefix, label string, aliases []string, source string) (int, error) {
|
||||
prefix = strings.TrimSpace(prefix)
|
||||
if prefix == "" {
|
||||
return 0, errors.New("video id prefix is required")
|
||||
}
|
||||
tag, err := c.ensureTag(ctx, label, aliases, source)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT v.id
|
||||
FROM videos v
|
||||
WHERE v.id LIKE ? || '%'
|
||||
AND COALESCE(v.tags_manual, 0) = 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
WHERE vt.video_id = v.id
|
||||
AND vt.tag_id = ?
|
||||
)
|
||||
ORDER BY v.id ASC`, prefix, tag.ID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var videoIDs []string
|
||||
for rows.Next() {
|
||||
var videoID string
|
||||
if err := rows.Scan(&videoID); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
videoIDs = append(videoIDs, videoID)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, videoID := range videoIDs {
|
||||
if err := c.insertVideoTag(ctx, videoID, tag.ID, "auto"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return len(videoIDs), nil
|
||||
}
|
||||
|
||||
func (c *Catalog) DeleteTag(ctx context.Context, tagID int64) (int, error) {
|
||||
tx, err := c.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
tag, err := c.getTagByIDTx(ctx, tx, tagID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if tag.Source == "system" {
|
||||
return 0, ErrSystemTag
|
||||
}
|
||||
|
||||
rows, err := tx.QueryContext(ctx, `SELECT video_id FROM video_tags WHERE tag_id = ?`, tagID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var videoIDs []string
|
||||
for rows.Next() {
|
||||
var videoID string
|
||||
if err := rows.Scan(&videoID); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
videoIDs = append(videoIDs, videoID)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM video_tags WHERE tag_id = ?`, tagID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM tags WHERE id = ?`, tagID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := markDeletedTagTx(ctx, tx, tag); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
for _, videoID := range videoIDs {
|
||||
manual := hasManualTagsTx(ctx, tx, videoID)
|
||||
if err := syncVideoTagsJSONTx(ctx, tx, videoID, manual); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(videoIDs), nil
|
||||
}
|
||||
|
||||
func (c *Catalog) ListTags(ctx context.Context) ([]Tag, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT t.id, t.label, t.aliases, t.source, COUNT(v.id) AS cnt
|
||||
WITH tagged_tags AS (
|
||||
SELECT vt.tag_id,
|
||||
tagged.id,
|
||||
COALESCE(tagged.content_hash, '') AS content_hash,
|
||||
COALESCE(tagged.sampled_sha256, '') AS sampled_sha256,
|
||||
tagged.size_bytes,
|
||||
COALESCE(tagged.file_name, '') AS file_name
|
||||
FROM video_tags vt
|
||||
JOIN videos tagged ON tagged.id = vt.video_id
|
||||
WHERE COALESCE(tagged.hidden, 0) = 0
|
||||
),
|
||||
tag_candidates AS (
|
||||
SELECT tag_id, id AS video_id
|
||||
FROM tagged_tags
|
||||
UNION ALL
|
||||
SELECT tag_id,
|
||||
(SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_tags.content_hash != ''
|
||||
AND canonical.content_hash = tagged_tags.content_hash
|
||||
AND COALESCE(canonical.content_hash, '') != ''
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_tags
|
||||
WHERE content_hash != ''
|
||||
UNION ALL
|
||||
SELECT tag_id,
|
||||
(SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_tags.sampled_sha256 != ''
|
||||
AND tagged_tags.size_bytes > 0
|
||||
AND canonical.sampled_sha256 = tagged_tags.sampled_sha256
|
||||
AND canonical.size_bytes = tagged_tags.size_bytes
|
||||
AND COALESCE(canonical.sampled_sha256, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_tags
|
||||
WHERE sampled_sha256 != '' AND size_bytes > 0
|
||||
UNION ALL
|
||||
SELECT tag_id,
|
||||
(SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_tags.file_name != ''
|
||||
AND tagged_tags.size_bytes > 0
|
||||
AND canonical.file_name = tagged_tags.file_name
|
||||
AND canonical.size_bytes = tagged_tags.size_bytes
|
||||
AND COALESCE(canonical.file_name, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_tags
|
||||
WHERE file_name != '' AND size_bytes > 0
|
||||
)
|
||||
SELECT t.id, t.label, t.aliases, t.source, COUNT(DISTINCT videos.id) AS cnt
|
||||
FROM tags t
|
||||
LEFT JOIN video_tags vt ON vt.tag_id = t.id
|
||||
LEFT JOIN videos v ON v.id = vt.video_id AND COALESCE(v.hidden, 0) = 0
|
||||
LEFT JOIN tag_candidates tc ON tc.tag_id = t.id AND tc.video_id IS NOT NULL
|
||||
LEFT JOIN videos ON videos.id = tc.video_id
|
||||
AND COALESCE(videos.hidden, 0) = 0
|
||||
AND `+uniqueVideoWhereSQL+`
|
||||
GROUP BY t.id, t.label, t.aliases, t.source
|
||||
ORDER BY cnt DESC, t.label ASC`)
|
||||
if err != nil {
|
||||
@@ -391,6 +710,66 @@ ORDER BY cnt DESC, t.label ASC`)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func videoMatchesTagLabelSQL(videoAlias string) string {
|
||||
return fmt.Sprintf(`%s.id IN (
|
||||
WITH tagged_videos AS (
|
||||
SELECT tagged.id,
|
||||
COALESCE(tagged.content_hash, '') AS content_hash,
|
||||
COALESCE(tagged.sampled_sha256, '') AS sampled_sha256,
|
||||
tagged.size_bytes,
|
||||
COALESCE(tagged.file_name, '') AS file_name
|
||||
FROM video_tags vt
|
||||
JOIN tags tag_filter ON tag_filter.id = vt.tag_id
|
||||
JOIN videos tagged ON tagged.id = vt.video_id
|
||||
WHERE tag_filter.label = ? COLLATE NOCASE
|
||||
AND COALESCE(tagged.hidden, 0) = 0
|
||||
),
|
||||
tag_candidates AS (
|
||||
SELECT id AS video_id
|
||||
FROM tagged_videos
|
||||
UNION ALL
|
||||
SELECT (SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_videos.content_hash != ''
|
||||
AND canonical.content_hash = tagged_videos.content_hash
|
||||
AND COALESCE(canonical.content_hash, '') != ''
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_videos
|
||||
WHERE content_hash != ''
|
||||
UNION ALL
|
||||
SELECT (SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_videos.sampled_sha256 != ''
|
||||
AND tagged_videos.size_bytes > 0
|
||||
AND canonical.sampled_sha256 = tagged_videos.sampled_sha256
|
||||
AND canonical.size_bytes = tagged_videos.size_bytes
|
||||
AND COALESCE(canonical.sampled_sha256, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_videos
|
||||
WHERE sampled_sha256 != '' AND size_bytes > 0
|
||||
UNION ALL
|
||||
SELECT (SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_videos.file_name != ''
|
||||
AND tagged_videos.size_bytes > 0
|
||||
AND canonical.file_name = tagged_videos.file_name
|
||||
AND canonical.size_bytes = tagged_videos.size_bytes
|
||||
AND COALESCE(canonical.file_name, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_videos
|
||||
WHERE file_name != '' AND size_bytes > 0
|
||||
)
|
||||
SELECT video_id
|
||||
FROM tag_candidates
|
||||
WHERE video_id IS NOT NULL
|
||||
)`, videoAlias)
|
||||
}
|
||||
|
||||
func (c *Catalog) SetManualVideoTags(ctx context.Context, videoID string, labels []string) error {
|
||||
if _, err := c.GetVideo(ctx, videoID); err != nil {
|
||||
return err
|
||||
@@ -441,6 +820,9 @@ func (c *Catalog) EnsureCollectionTag(ctx context.Context, label string) (string
|
||||
if !LooksLikeCollectionTag(label) {
|
||||
return "", false, nil
|
||||
}
|
||||
if c.tagDeleted(ctx, label) {
|
||||
return "", false, nil
|
||||
}
|
||||
if !c.tagExists(ctx, label) {
|
||||
count, err := c.categoryVideoCount(ctx, label)
|
||||
if err != nil {
|
||||
@@ -472,6 +854,14 @@ func (c *Catalog) ensureTag(ctx context.Context, label string, aliases []string,
|
||||
if source == "" {
|
||||
source = "user"
|
||||
}
|
||||
if source != "system" && source != "user" && c.tagDeleted(ctx, label) {
|
||||
return Tag{}, ErrDeletedTag
|
||||
}
|
||||
if source == "system" || source == "user" {
|
||||
if err := c.restoreDeletedTag(ctx, label); err != nil {
|
||||
return Tag{}, err
|
||||
}
|
||||
}
|
||||
aliases = cleanAliases(aliases, label)
|
||||
aliasesJSON, _ := json.Marshal(aliases)
|
||||
now := time.Now().UnixMilli()
|
||||
@@ -498,6 +888,10 @@ func (c *Catalog) getTagByLabel(ctx context.Context, label string) (Tag, error)
|
||||
}
|
||||
|
||||
func (c *Catalog) classifyTag(ctx context.Context, tag Tag) (int, error) {
|
||||
existingIDs, err := c.videoIDSetForTagID(ctx, tag.ID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT id, title, COALESCE(author, ''), COALESCE(category, ''), COALESCE(tags_manual, 0)
|
||||
FROM videos`)
|
||||
@@ -529,13 +923,14 @@ FROM videos`)
|
||||
continue
|
||||
}
|
||||
}
|
||||
added, err := c.addVideoTag(ctx, videoID, tag.Label, "auto", false)
|
||||
if err != nil {
|
||||
if existingIDs[videoID] {
|
||||
continue
|
||||
}
|
||||
if err := c.insertVideoTag(ctx, videoID, tag.ID, "auto"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if added {
|
||||
classified++
|
||||
}
|
||||
existingIDs[videoID] = true
|
||||
classified++
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -545,9 +940,15 @@ FROM videos`)
|
||||
|
||||
func (c *Catalog) replaceVideoTags(ctx context.Context, videoID string, labels []string, source string, manual bool, createMissing bool) error {
|
||||
labels = uniqueStrings(cleanLabels(labels))
|
||||
if source != "manual" {
|
||||
labels = c.filterDeletedTagLabels(ctx, labels)
|
||||
}
|
||||
if createMissing {
|
||||
for _, label := range labels {
|
||||
if _, err := c.ensureTag(ctx, label, nil, "legacy"); err != nil {
|
||||
if errors.Is(err, ErrDeletedTag) {
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -589,18 +990,33 @@ func (c *Catalog) replaceVideoTags(ctx context.Context, videoID string, labels [
|
||||
return c.syncVideoTagsJSON(ctx, videoID, manual)
|
||||
}
|
||||
|
||||
func (c *Catalog) addVideoTags(ctx context.Context, videoID string, labels []string, source string, createMissing bool) error {
|
||||
for _, label := range uniqueStrings(cleanLabels(labels)) {
|
||||
if _, err := c.addVideoTag(ctx, videoID, label, source, createMissing); err != nil {
|
||||
return err
|
||||
func (c *Catalog) addVideoTags(ctx context.Context, videoID string, labels []string, source string, createMissing bool) (bool, error) {
|
||||
labels = uniqueStrings(cleanLabels(labels))
|
||||
if source != "manual" {
|
||||
labels = c.filterDeletedTagLabels(ctx, labels)
|
||||
}
|
||||
changed := false
|
||||
for _, label := range labels {
|
||||
added, err := c.addVideoTag(ctx, videoID, label, source, createMissing)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if added {
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) addVideoTag(ctx context.Context, videoID, label, source string, createMissing bool) (bool, error) {
|
||||
if source != "manual" && c.tagDeleted(ctx, label) {
|
||||
return false, nil
|
||||
}
|
||||
if createMissing {
|
||||
if _, err := c.ensureTag(ctx, label, nil, "legacy"); err != nil {
|
||||
if errors.Is(err, ErrDeletedTag) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
@@ -619,12 +1035,33 @@ func (c *Catalog) addVideoTag(ctx context.Context, videoID, label, source string
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) insertVideoTag(ctx context.Context, videoID string, tagID int64, source string) error {
|
||||
_, err := c.db.ExecContext(ctx,
|
||||
`INSERT OR IGNORE INTO video_tags (video_id, tag_id, source, created_at) VALUES (?, ?, ?, ?)`,
|
||||
videoID, tagID, source, time.Now().UnixMilli())
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) addCollectionTagToVideos(ctx context.Context, category string) error {
|
||||
return c.addTagToVideosByCategory(ctx, category, category, "auto")
|
||||
}
|
||||
|
||||
func (c *Catalog) addTagToVideosByCategory(ctx context.Context, category, label, source string) error {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT id FROM videos WHERE category = ? AND COALESCE(tags_manual, 0) = 0`, category)
|
||||
tag, err := c.getTagByLabel(ctx, label)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT v.id
|
||||
FROM videos v
|
||||
WHERE v.category = ?
|
||||
AND COALESCE(v.tags_manual, 0) = 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
WHERE vt.video_id = v.id
|
||||
AND vt.tag_id = ?
|
||||
)`, category, tag.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -643,7 +1080,7 @@ func (c *Catalog) addTagToVideosByCategory(ctx context.Context, category, label,
|
||||
return err
|
||||
}
|
||||
for _, videoID := range videoIDs {
|
||||
if _, err := c.addVideoTag(ctx, videoID, label, source, false); err != nil {
|
||||
if err := c.insertVideoTag(ctx, videoID, tag.ID, source); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
@@ -727,6 +1164,23 @@ func (c *Catalog) videoIDsForTagID(ctx context.Context, tagID int64) ([]string,
|
||||
return videoIDs, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) videoIDSetForTagID(ctx context.Context, tagID int64) (map[string]bool, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT video_id FROM video_tags WHERE tag_id = ?`, tagID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var videoID string
|
||||
if err := rows.Scan(&videoID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[videoID] = true
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) validateTagsExist(ctx context.Context, labels []string) error {
|
||||
for _, label := range labels {
|
||||
if _, err := c.getTagByLabel(ctx, label); err != nil {
|
||||
@@ -792,6 +1246,39 @@ func (c *Catalog) tagExists(ctx context.Context, label string) bool {
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (c *Catalog) tagDeleted(ctx context.Context, label string) bool {
|
||||
label = cleanTagLabel(label)
|
||||
if label == "" {
|
||||
return false
|
||||
}
|
||||
var exists int
|
||||
err := c.db.QueryRowContext(ctx, `SELECT 1 FROM deleted_tags WHERE label = ? COLLATE NOCASE`, label).Scan(&exists)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (c *Catalog) filterDeletedTagLabels(ctx context.Context, labels []string) []string {
|
||||
if len(labels) == 0 {
|
||||
return labels
|
||||
}
|
||||
out := labels[:0]
|
||||
for _, label := range labels {
|
||||
if c.tagDeleted(ctx, label) {
|
||||
continue
|
||||
}
|
||||
out = append(out, label)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (c *Catalog) restoreDeletedTag(ctx context.Context, label string) error {
|
||||
label = cleanTagLabel(label)
|
||||
if label == "" {
|
||||
return nil
|
||||
}
|
||||
_, err := c.db.ExecContext(ctx, `DELETE FROM deleted_tags WHERE label = ? COLLATE NOCASE`, label)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) categoryVideoCount(ctx context.Context, category string) (int, error) {
|
||||
var count int
|
||||
err := c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM videos WHERE category = ?`, category).Scan(&count)
|
||||
@@ -805,6 +1292,71 @@ func (c *Catalog) getTagByLabelTx(ctx context.Context, tx *sql.Tx, label string)
|
||||
return scanTag(row)
|
||||
}
|
||||
|
||||
func (c *Catalog) getTagByIDTx(ctx context.Context, tx *sql.Tx, id int64) (Tag, error) {
|
||||
row := tx.QueryRowContext(ctx,
|
||||
`SELECT id, label, aliases, source, 0 FROM tags WHERE id = ?`,
|
||||
id)
|
||||
return scanTag(row)
|
||||
}
|
||||
|
||||
func hasManualTagsTx(ctx context.Context, tx *sql.Tx, videoID string) bool {
|
||||
var manual int
|
||||
err := tx.QueryRowContext(ctx, `SELECT COALESCE(tags_manual, 0) FROM videos WHERE id = ?`, videoID).Scan(&manual)
|
||||
return err == nil && manual == 1
|
||||
}
|
||||
|
||||
func markDeletedTagTx(ctx context.Context, tx *sql.Tx, tag Tag) error {
|
||||
label := cleanTagLabel(tag.Label)
|
||||
if label == "" {
|
||||
return nil
|
||||
}
|
||||
now := time.Now().UnixMilli()
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO deleted_tags (label, source, deleted_at)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(label) DO UPDATE SET
|
||||
source = excluded.source,
|
||||
deleted_at = excluded.deleted_at`, label, tag.Source, now)
|
||||
return err
|
||||
}
|
||||
|
||||
func syncVideoTagsJSONTx(ctx context.Context, tx *sql.Tx, videoID string, manual bool) error {
|
||||
rows, err := tx.QueryContext(ctx, `
|
||||
SELECT t.label
|
||||
FROM video_tags vt
|
||||
JOIN tags t ON t.id = vt.tag_id
|
||||
WHERE vt.video_id = ?
|
||||
ORDER BY t.id ASC`, videoID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var labels []string
|
||||
for rows.Next() {
|
||||
var label string
|
||||
if err := rows.Scan(&label); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
labels = append(labels, label)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
labelsJSON, _ := json.Marshal(labels)
|
||||
manualValue := 0
|
||||
if manual {
|
||||
manualValue = 1
|
||||
}
|
||||
_, err = tx.ExecContext(ctx,
|
||||
`UPDATE videos SET tags = ?, tags_manual = ?, updated_at = ? WHERE id = ?`,
|
||||
string(labelsJSON), manualValue, time.Now().UnixMilli(), videoID)
|
||||
return err
|
||||
}
|
||||
|
||||
type tagRowScanner interface {
|
||||
Scan(dest ...any) error
|
||||
}
|
||||
|
||||
@@ -3,10 +3,121 @@ package catalog
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestListVideosNeedingThumbnailIncludesExistingThumbnailMissingDuration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
videos := []*Video{
|
||||
{
|
||||
ID: "duration-only",
|
||||
DriveID: "drive",
|
||||
FileID: "file-duration-only",
|
||||
Title: "Duration Only",
|
||||
ThumbnailURL: "/p/thumb/duration-only",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "complete",
|
||||
DriveID: "drive",
|
||||
FileID: "file-complete",
|
||||
Title: "Complete",
|
||||
DurationSeconds: 12,
|
||||
ThumbnailURL: "/p/thumb/complete",
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
{
|
||||
ID: "missing-thumb",
|
||||
DriveID: "drive",
|
||||
FileID: "file-missing-thumb",
|
||||
Title: "Missing Thumb",
|
||||
DurationSeconds: 18,
|
||||
PublishedAt: now.Add(2 * time.Second),
|
||||
CreatedAt: now.Add(2 * time.Second),
|
||||
UpdatedAt: now.Add(2 * time.Second),
|
||||
},
|
||||
{
|
||||
ID: "failed",
|
||||
DriveID: "drive",
|
||||
FileID: "file-failed",
|
||||
Title: "Failed",
|
||||
PublishedAt: now.Add(3 * time.Second),
|
||||
CreatedAt: now.Add(3 * time.Second),
|
||||
UpdatedAt: now.Add(3 * time.Second),
|
||||
},
|
||||
}
|
||||
for _, v := range videos {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
if err := cat.UpdateVideoMeta(ctx, "failed", VideoMetaPatch{ThumbnailStatus: "failed"}); err != nil {
|
||||
t.Fatalf("mark failed thumbnail: %v", err)
|
||||
}
|
||||
|
||||
items, err := cat.ListVideosNeedingThumbnail(ctx, "drive", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list videos needing thumbnail: %v", err)
|
||||
}
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("items = %#v, want duration-only and missing-thumb", items)
|
||||
}
|
||||
if items[0].ID != "duration-only" || items[1].ID != "missing-thumb" {
|
||||
t.Fatalf("item ids = %q, %q; want duration-only, missing-thumb", items[0].ID, items[1].ID)
|
||||
}
|
||||
|
||||
count, err := cat.CountVideosNeedingThumbnail(ctx, "drive")
|
||||
if err != nil {
|
||||
t.Fatalf("count videos needing thumbnail: %v", err)
|
||||
}
|
||||
if count != 2 {
|
||||
t.Fatalf("count = %d, want 2", count)
|
||||
}
|
||||
|
||||
counts, err := cat.CountThumbnailsByDrive(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("count thumbnails by drive: %v", err)
|
||||
}
|
||||
if got := counts["drive"]; got.Ready != 2 || got.Pending != 1 || got.Failed != 1 || got.DurationPending != 1 {
|
||||
t.Fatalf("thumbnail counts = %#v, want ready=2 pending=1 failed=1 durationPending=1", got)
|
||||
}
|
||||
|
||||
if err := cat.UpdateVideoMeta(ctx, "duration-only", VideoMetaPatch{ThumbnailStatus: "skipped"}); err != nil {
|
||||
t.Fatalf("mark duration-only skipped: %v", err)
|
||||
}
|
||||
count, err = cat.CountVideosNeedingThumbnail(ctx, "drive")
|
||||
if err != nil {
|
||||
t.Fatalf("count videos needing thumbnail after skip: %v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("count after skip = %d, want 1", count)
|
||||
}
|
||||
counts, err = cat.CountThumbnailsByDrive(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("count thumbnails by drive after skip: %v", err)
|
||||
}
|
||||
if got := counts["drive"]; got.Ready != 2 || got.Pending != 1 || got.Failed != 1 || got.DurationPending != 0 {
|
||||
t.Fatalf("thumbnail counts after skip = %#v, want ready=2 pending=1 failed=1 durationPending=0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateTagAndClassifyAddsTagToMatchingExistingVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
@@ -70,6 +181,242 @@ func TestCreateTagAndClassifyAddsTagToMatchingExistingVideos(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteTagRemovesTagFromVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: "video-1",
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
Title: "清纯短发",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
if _, err := cat.CreateTagAndClassify(ctx, "清纯", nil, "user"); err != nil {
|
||||
t.Fatalf("create tag: %v", err)
|
||||
}
|
||||
|
||||
tag := mustTagByLabel(t, ctx, cat, "清纯")
|
||||
removed, err := cat.DeleteTag(ctx, tag.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("delete tag: %v", err)
|
||||
}
|
||||
if removed != 1 {
|
||||
t.Fatalf("removed = %d, want 1", removed)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(ctx, "video-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if len(got.Tags) != 0 {
|
||||
t.Fatalf("video tags = %#v, want none", got.Tags)
|
||||
}
|
||||
for _, tag := range mustListTags(t, ctx, cat) {
|
||||
if tag.Label == "清纯" {
|
||||
t.Fatal("deleted tag still appears in ListTags")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteTagSuppressesAutomaticCollectionRecreation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, id := range []string{"video-1", "video-2"} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: "合集视频",
|
||||
Category: "sunny",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
if label, ok, err := cat.EnsureCollectionTag(ctx, "sunny"); err != nil || !ok || label != "sunny" {
|
||||
t.Fatalf("ensure collection = %q, %v, %v; want sunny true nil", label, ok, err)
|
||||
}
|
||||
tag := mustTagByLabel(t, ctx, cat, "sunny")
|
||||
if _, err := cat.DeleteTag(ctx, tag.ID); err != nil {
|
||||
t.Fatalf("delete tag: %v", err)
|
||||
}
|
||||
|
||||
if label, ok, err := cat.EnsureCollectionTag(ctx, "sunny"); err != nil || ok || label != "" {
|
||||
t.Fatalf("ensure deleted collection = %q, %v, %v; want empty false nil", label, ok, err)
|
||||
}
|
||||
for _, tag := range mustListTags(t, ctx, cat) {
|
||||
if tag.Label == "sunny" {
|
||||
t.Fatal("deleted collection tag was recreated automatically")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateTagAndClassifyRestoresDeletedTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: "video-1",
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
Title: "清纯短发",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
if _, err := cat.CreateTagAndClassify(ctx, "清纯", nil, "user"); err != nil {
|
||||
t.Fatalf("create tag: %v", err)
|
||||
}
|
||||
tag := mustTagByLabel(t, ctx, cat, "清纯")
|
||||
if _, err := cat.DeleteTag(ctx, tag.ID); err != nil {
|
||||
t.Fatalf("delete tag: %v", err)
|
||||
}
|
||||
|
||||
classified, err := cat.CreateTagAndClassify(ctx, "清纯", nil, "user")
|
||||
if err != nil {
|
||||
t.Fatalf("recreate tag: %v", err)
|
||||
}
|
||||
if classified != 1 {
|
||||
t.Fatalf("classified = %d, want 1", classified)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, "video-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if !sameStrings(got.Tags, []string{"清纯"}) {
|
||||
t.Fatalf("video tags = %#v, want 清纯", got.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureTagForVideoIDPrefixBackfillsSourceTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, seed := range []struct {
|
||||
id string
|
||||
manual bool
|
||||
}{
|
||||
{id: "spider91-91-spider-1200001"},
|
||||
{id: "spider91-91-spider-1200002", manual: true},
|
||||
{id: "spider91-other-1200003"},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: seed.id,
|
||||
DriveID: "91-spider",
|
||||
FileID: seed.id + ".mp4",
|
||||
Title: "legacy title without source text",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", seed.id, err)
|
||||
}
|
||||
if seed.manual {
|
||||
if err := cat.SetManualVideoTags(ctx, seed.id, nil); err != nil {
|
||||
t.Fatalf("mark %s manual: %v", seed.id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
added, err := cat.EnsureTagForVideoIDPrefix(ctx, "spider91-91-spider-", "91porn", nil, "system")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure prefix tag: %v", err)
|
||||
}
|
||||
if added != 1 {
|
||||
t.Fatalf("added = %d, want 1", added)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, "spider91-91-spider-1200001")
|
||||
if err != nil {
|
||||
t.Fatalf("get tagged video: %v", err)
|
||||
}
|
||||
if !sameStrings(got.Tags, []string{"91porn"}) {
|
||||
t.Fatalf("tagged video tags = %#v, want 91porn", got.Tags)
|
||||
}
|
||||
manual, err := cat.GetVideo(ctx, "spider91-91-spider-1200002")
|
||||
if err != nil {
|
||||
t.Fatalf("get manual video: %v", err)
|
||||
}
|
||||
if len(manual.Tags) != 0 {
|
||||
t.Fatalf("manual video tags = %#v, want unchanged", manual.Tags)
|
||||
}
|
||||
other, err := cat.GetVideo(ctx, "spider91-other-1200003")
|
||||
if err != nil {
|
||||
t.Fatalf("get other prefix video: %v", err)
|
||||
}
|
||||
if len(other.Tags) != 0 {
|
||||
t.Fatalf("other prefix video tags = %#v, want unchanged", other.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteTagRejectsSystemTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
tag := mustTagByLabel(t, ctx, cat, "AV")
|
||||
if _, err := cat.DeleteTag(ctx, tag.ID); !errors.Is(err, ErrSystemTag) {
|
||||
t.Fatalf("delete system tag err = %v, want ErrSystemTag", err)
|
||||
}
|
||||
|
||||
if tag := mustTagByLabel(t, ctx, cat, "AV"); tag.Source != "system" {
|
||||
t.Fatalf("AV source = %q, want system", tag.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenClassifiesSystemTagsForExistingVideos(t *testing.T) {
|
||||
path := t.TempDir() + "/catalog.db"
|
||||
db, err := sql.Open("sqlite", path)
|
||||
@@ -120,6 +467,84 @@ VALUES
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateDoesNotRewriteAlreadySyncedVideoTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, id := range []string{"video-1", "video-2", "video-3"} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: "巨乳后入合集",
|
||||
Category: "Better Call Saul S03",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
if err := cat.migrate(ctx); err != nil {
|
||||
t.Fatalf("first migrate: %v", err)
|
||||
}
|
||||
|
||||
before := videoUpdatedAtByID(t, ctx, cat, "video-1", "video-2", "video-3")
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
if err := cat.migrate(ctx); err != nil {
|
||||
t.Fatalf("second migrate: %v", err)
|
||||
}
|
||||
after := videoUpdatedAtByID(t, ctx, cat, "video-1", "video-2", "video-3")
|
||||
for id, want := range before {
|
||||
if got := after[id]; got != want {
|
||||
t.Fatalf("%s updated_at changed on no-op migrate: got %d, want %d", id, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateBackfillsLegacyTagsWithoutRelations(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
if _, err := cat.db.ExecContext(ctx, `
|
||||
INSERT INTO videos (id, drive_id, file_id, title, tags, tags_manual, published_at, created_at, updated_at)
|
||||
VALUES ('legacy-video', 'drive', 'file-legacy', 'legacy title', '["legacy-tag"]', 0, ?, ?, ?)`,
|
||||
now, now, now); err != nil {
|
||||
t.Fatalf("seed legacy video: %v", err)
|
||||
}
|
||||
if err := cat.migrate(ctx); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
|
||||
tag := mustTagByLabel(t, ctx, cat, "legacy-tag")
|
||||
var count int
|
||||
if err := cat.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM video_tags WHERE video_id = 'legacy-video' AND tag_id = ?`, tag.ID).Scan(&count); err != nil {
|
||||
t.Fatalf("count video tag: %v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("legacy video tag relation count = %d, want 1", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenMigratesLegacyVideosWithoutFileName(t *testing.T) {
|
||||
path := t.TempDir() + "/catalog.db"
|
||||
db, err := sql.Open("sqlite", path)
|
||||
@@ -379,7 +804,7 @@ func TestMigrateCollapsesAVCodeTagsIntoAV(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateClearsVolatileOneDriveThumbnailURLs(t *testing.T) {
|
||||
func TestMigrateClearsRemoteNonSpiderThumbnailURLs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -402,6 +827,36 @@ func TestMigrateClearsVolatileOneDriveThumbnailURLs(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatalf("seed onedrive: %v", err)
|
||||
}
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: "p123-main",
|
||||
Kind: "p123",
|
||||
Name: "123Pan",
|
||||
RootID: "root",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed p123: %v", err)
|
||||
}
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: "pikpak-main",
|
||||
Kind: "pikpak",
|
||||
Name: "PikPak",
|
||||
RootID: "root",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed pikpak: %v", err)
|
||||
}
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: "spider91-main",
|
||||
Kind: "spider91",
|
||||
Name: "91Spider",
|
||||
RootID: "root",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed spider91: %v", err)
|
||||
}
|
||||
|
||||
videos := []*Video{
|
||||
{
|
||||
@@ -425,6 +880,27 @@ func TestMigrateClearsVolatileOneDriveThumbnailURLs(t *testing.T) {
|
||||
Title: "PikPak",
|
||||
ThumbnailURL: "https://sg-thumbnail-drive.mypikpak.net/v0/screenshot-thumbnails/demo",
|
||||
},
|
||||
{
|
||||
ID: "p123-remote-thumb-video",
|
||||
DriveID: "p123-main",
|
||||
FileID: "file-4",
|
||||
Title: "123Pan remote thumb",
|
||||
ThumbnailURL: "https://download.123pan.com/thumb/file_70_70?w=70&h=70",
|
||||
},
|
||||
{
|
||||
ID: "p123-local-thumb-video",
|
||||
DriveID: "p123-main",
|
||||
FileID: "file-5",
|
||||
Title: "123Pan local thumb",
|
||||
ThumbnailURL: "/p/thumb/p123-local-thumb-video",
|
||||
},
|
||||
{
|
||||
ID: "spider91-local-thumb-video",
|
||||
DriveID: "spider91-main",
|
||||
FileID: "file-6",
|
||||
Title: "91Spider local thumb",
|
||||
ThumbnailURL: "/p/thumb/spider91-local-thumb-video",
|
||||
},
|
||||
}
|
||||
for _, v := range videos {
|
||||
v.PublishedAt = now
|
||||
@@ -459,8 +935,39 @@ func TestMigrateClearsVolatileOneDriveThumbnailURLs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("get pikpak video: %v", err)
|
||||
}
|
||||
if pikpak.ThumbnailURL == "" {
|
||||
t.Fatal("pikpak thumbnail was cleared")
|
||||
if pikpak.ThumbnailURL != "" {
|
||||
t.Fatalf("pikpak thumbnail = %q, want cleared", pikpak.ThumbnailURL)
|
||||
}
|
||||
|
||||
p123Remote, err := cat.GetVideo(ctx, "p123-remote-thumb-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get p123 remote thumb video: %v", err)
|
||||
}
|
||||
if p123Remote.ThumbnailURL != "" {
|
||||
t.Fatalf("p123 remote thumbnail = %q, want cleared", p123Remote.ThumbnailURL)
|
||||
}
|
||||
var p123Status string
|
||||
if err := cat.db.QueryRowContext(ctx, `SELECT thumbnail_status FROM videos WHERE id = ?`, "p123-remote-thumb-video").Scan(&p123Status); err != nil {
|
||||
t.Fatalf("read p123 thumbnail status: %v", err)
|
||||
}
|
||||
if p123Status != "pending" {
|
||||
t.Fatalf("p123 remote thumbnail_status = %q, want pending", p123Status)
|
||||
}
|
||||
|
||||
p123Local, err := cat.GetVideo(ctx, "p123-local-thumb-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get p123 local thumb video: %v", err)
|
||||
}
|
||||
if p123Local.ThumbnailURL != "/p/thumb/p123-local-thumb-video" {
|
||||
t.Fatalf("p123 local thumbnail = %q, want preserved", p123Local.ThumbnailURL)
|
||||
}
|
||||
|
||||
spider91Local, err := cat.GetVideo(ctx, "spider91-local-thumb-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get spider91 local thumb video: %v", err)
|
||||
}
|
||||
if spider91Local.ThumbnailURL != "/p/thumb/spider91-local-thumb-video" {
|
||||
t.Fatalf("spider91 local thumbnail = %q, want preserved", spider91Local.ThumbnailURL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -581,6 +1088,151 @@ func TestListVideosHidesDuplicateContentHashes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTagFilterMatchesCanonicalDuplicateVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{
|
||||
ID: "pikpak-canonical",
|
||||
DriveID: "pikpak",
|
||||
FileID: "canonical.mp4",
|
||||
Title: "Canonical",
|
||||
Size: 1024,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "spider91-dup-1",
|
||||
DriveID: "91-spider",
|
||||
FileID: "dup-1.mp4",
|
||||
Title: "Spider duplicate 1",
|
||||
Tags: []string{"91porn"},
|
||||
Size: 1024,
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
{
|
||||
ID: "spider91-dup-2",
|
||||
DriveID: "91-spider",
|
||||
FileID: "dup-2.mp4",
|
||||
Title: "Spider duplicate 2",
|
||||
Tags: []string{"91porn"},
|
||||
Size: 1024,
|
||||
PublishedAt: now.Add(2 * time.Second),
|
||||
CreatedAt: now.Add(2 * time.Second),
|
||||
UpdatedAt: now.Add(2 * time.Second),
|
||||
},
|
||||
{
|
||||
ID: "spider91-visible",
|
||||
DriveID: "91-spider",
|
||||
FileID: "visible.mp4",
|
||||
Title: "Spider visible",
|
||||
Tags: []string{"91porn"},
|
||||
Size: 2048,
|
||||
PublishedAt: now.Add(3 * time.Second),
|
||||
CreatedAt: now.Add(3 * time.Second),
|
||||
UpdatedAt: now.Add(3 * time.Second),
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"pikpak-canonical", "spider91-dup-1", "spider91-dup-2"} {
|
||||
if err := cat.UpdateVideoFingerprint(ctx, id, "same-sampled-sha256", "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "spider91-visible", "unique-sampled-sha256", "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint visible: %v", err)
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{Tag: "91porn", Page: 1, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list videos by tag: %v", err)
|
||||
}
|
||||
if total != 2 || len(items) != 2 {
|
||||
t.Fatalf("tagged videos total=%d len=%d, want 2", total, len(items))
|
||||
}
|
||||
gotIDs := map[string]bool{}
|
||||
for _, item := range items {
|
||||
gotIDs[item.ID] = true
|
||||
}
|
||||
for _, want := range []string{"pikpak-canonical", "spider91-visible"} {
|
||||
if !gotIDs[want] {
|
||||
t.Fatalf("tagged video ids = %#v, want %s", gotIDs, want)
|
||||
}
|
||||
}
|
||||
if got := mustTagByLabel(t, ctx, cat, "91porn").Count; got != 2 {
|
||||
t.Fatalf("91porn count = %d, want 2 visible canonical videos", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListVideosCanFilterReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{
|
||||
ID: "ready-video",
|
||||
DriveID: "drive",
|
||||
FileID: "file-ready",
|
||||
Title: "Ready",
|
||||
ThumbnailURL: "/p/thumb/ready-video",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "pending-video",
|
||||
DriveID: "drive",
|
||||
FileID: "file-pending",
|
||||
Title: "Pending",
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed video %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{
|
||||
Page: 1, PageSize: 10, ThumbnailReadyOnly: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list videos: %v", err)
|
||||
}
|
||||
if total != 1 || len(items) != 1 {
|
||||
t.Fatalf("ready videos total=%d len=%d, want 1", total, len(items))
|
||||
}
|
||||
if items[0].ID != "ready-video" {
|
||||
t.Fatalf("ready video id = %q, want ready-video", items[0].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func sameStrings(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
@@ -593,6 +1245,39 @@ func sameStrings(a, b []string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func mustListTags(t *testing.T, ctx context.Context, cat *Catalog) []Tag {
|
||||
t.Helper()
|
||||
tags, err := cat.ListTags(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list tags: %v", err)
|
||||
}
|
||||
return tags
|
||||
}
|
||||
|
||||
func mustTagByLabel(t *testing.T, ctx context.Context, cat *Catalog, label string) Tag {
|
||||
t.Helper()
|
||||
for _, tag := range mustListTags(t, ctx, cat) {
|
||||
if tag.Label == label {
|
||||
return tag
|
||||
}
|
||||
}
|
||||
t.Fatalf("tag %q not found", label)
|
||||
return Tag{}
|
||||
}
|
||||
|
||||
func videoUpdatedAtByID(t *testing.T, ctx context.Context, cat *Catalog, ids ...string) map[string]int64 {
|
||||
t.Helper()
|
||||
out := make(map[string]int64, len(ids))
|
||||
for _, id := range ids {
|
||||
var updatedAt int64
|
||||
if err := cat.db.QueryRowContext(ctx, `SELECT updated_at FROM videos WHERE id = ?`, id).Scan(&updatedAt); err != nil {
|
||||
t.Fatalf("read updated_at for %s: %v", id, err)
|
||||
}
|
||||
out[id] = updatedAt
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// 删除 collection 标签的最后一个引用视频后,标签应当自动从 tags 表里消失。
|
||||
// user/system 标签不受影响:用户/系统标签的语义由人维护,孤儿状态保留。
|
||||
func TestDeleteVideoPrunesOrphanCollectionTag(t *testing.T) {
|
||||
@@ -786,11 +1471,12 @@ func TestReconcileThumbnailStatusOnce(t *testing.T) {
|
||||
id, url, status string
|
||||
wantStatus string
|
||||
}{
|
||||
{"v-pending-url", "/p/thumb/v-pending-url", "pending", "ready"}, // 主要修复目标
|
||||
{"v-empty-url-pending", "", "pending", "pending"}, // 没 url 不动
|
||||
{"v-failed-with-url", "/p/thumb/v-failed-with-url", "failed", "failed"}, // 显式失败保留
|
||||
{"v-empty-url-failed", "", "failed", "failed"}, // 失败 + 没 url 也保留
|
||||
{"v-already-ready", "/p/thumb/v-already-ready", "ready", "ready"}, // 幂等
|
||||
{"v-pending-url", "/p/thumb/v-pending-url", "pending", "ready"}, // 主要修复目标
|
||||
{"v-empty-url-pending", "", "pending", "pending"}, // 没 url 不动
|
||||
{"v-failed-with-url", "/p/thumb/v-failed-with-url", "failed", "failed"}, // 显式失败保留
|
||||
{"v-empty-url-failed", "", "failed", "failed"}, // 失败 + 没 url 也保留
|
||||
{"v-skipped-with-url", "/p/thumb/v-skipped-with-url", "skipped", "skipped"}, // 已跳过的时长补全保留
|
||||
{"v-already-ready", "/p/thumb/v-already-ready", "ready", "ready"}, // 幂等
|
||||
}
|
||||
for _, c := range cases {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
|
||||
@@ -16,6 +16,11 @@ const (
|
||||
DefaultAdminPassword = "admin123"
|
||||
)
|
||||
|
||||
var (
|
||||
legacyDefaultVideoExtensions = []string{".mp4", ".mkv", ".mov", ".webm", ".avi"}
|
||||
defaultVideoExtensions = []string{".mp4", ".mkv", ".mov", ".webm", ".avi", ".strm"}
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Server Server `yaml:"server"`
|
||||
Storage Storage `yaml:"storage"`
|
||||
@@ -202,7 +207,7 @@ type Nightly struct {
|
||||
// 这里保留 yaml 中的静态定义,用于启动时预置盘。生产建议只在 DB 里维护。
|
||||
type Drive struct {
|
||||
ID string `yaml:"id"`
|
||||
Kind string `yaml:"kind"` // quark / p115 / pikpak / wopan / onedrive
|
||||
Kind string `yaml:"kind"` // quark / p115 / p123 / pikpak / wopan / onedrive / googledrive / localstorage
|
||||
Name string `yaml:"name"`
|
||||
RootID string `yaml:"root_id"`
|
||||
Params map[string]string `yaml:"params,omitempty"`
|
||||
@@ -247,7 +252,9 @@ func (c *Config) applyDefaults() {
|
||||
c.Scanner.MaxDepth = 5
|
||||
}
|
||||
if len(c.Scanner.VideoExtensions) == 0 {
|
||||
c.Scanner.VideoExtensions = []string{".mp4", ".mkv", ".mov", ".webm", ".avi"}
|
||||
c.Scanner.VideoExtensions = append([]string{}, defaultVideoExtensions...)
|
||||
} else if isLegacyDefaultVideoExtensions(c.Scanner.VideoExtensions) {
|
||||
c.Scanner.VideoExtensions = append(c.Scanner.VideoExtensions, ".strm")
|
||||
}
|
||||
if c.Preview.FFmpegPath == "" {
|
||||
c.Preview.FFmpegPath = "ffmpeg"
|
||||
@@ -276,3 +283,19 @@ func (c *Config) applyDefaults() {
|
||||
c.Nightly.CronHour = 1
|
||||
}
|
||||
}
|
||||
|
||||
func isLegacyDefaultVideoExtensions(exts []string) bool {
|
||||
if len(exts) != len(legacyDefaultVideoExtensions) {
|
||||
return false
|
||||
}
|
||||
seen := make(map[string]struct{}, len(exts))
|
||||
for _, ext := range exts {
|
||||
seen[strings.ToLower(strings.TrimSpace(ext))] = struct{}{}
|
||||
}
|
||||
for _, ext := range legacyDefaultVideoExtensions {
|
||||
if _, ok := seen[ext]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package config
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -50,3 +51,64 @@ storage:
|
||||
t.Fatalf("db path = %q, want preserved value", cfg.Storage.DBPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadDefaultScannerVideoExtensionsIncludeSTRM(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(`{}`), 0o644); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
if !hasVideoExtension(cfg.Scanner.VideoExtensions, ".strm") {
|
||||
t.Fatalf("video extensions = %#v, want .strm", cfg.Scanner.VideoExtensions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadLegacyDefaultScannerVideoExtensionsIncludeSTRM(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(`
|
||||
scanner:
|
||||
video_extensions: [".mp4", ".mkv", ".mov", ".webm", ".avi"]
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
if !hasVideoExtension(cfg.Scanner.VideoExtensions, ".strm") {
|
||||
t.Fatalf("video extensions = %#v, want .strm appended for legacy default list", cfg.Scanner.VideoExtensions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCustomScannerVideoExtensionsArePreserved(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(`
|
||||
scanner:
|
||||
video_extensions: [".mp4"]
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
if len(cfg.Scanner.VideoExtensions) != 1 || cfg.Scanner.VideoExtensions[0] != ".mp4" {
|
||||
t.Fatalf("video extensions = %#v, want custom list preserved", cfg.Scanner.VideoExtensions)
|
||||
}
|
||||
}
|
||||
|
||||
func hasVideoExtension(exts []string, want string) bool {
|
||||
want = strings.ToLower(strings.TrimSpace(want))
|
||||
for _, ext := range exts {
|
||||
if strings.ToLower(strings.TrimSpace(ext)) == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,505 @@
|
||||
package googledrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-resty/resty/v2"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
const (
|
||||
Kind = "googledrive"
|
||||
defaultAPIBaseURL = "https://www.googleapis.com/drive/v3"
|
||||
defaultOAuthURL = "https://www.googleapis.com/oauth2/v4/token"
|
||||
defaultRenewAPIURL = "https://api.oplist.org/googleui/renewapi"
|
||||
defaultListInterval = 1 * time.Second
|
||||
defaultListCooldown = 5 * time.Minute
|
||||
|
||||
filesListFields = "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken"
|
||||
fileInfoFields = "id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum"
|
||||
)
|
||||
|
||||
type Driver struct {
|
||||
id string
|
||||
rootID string
|
||||
refreshToken string
|
||||
accessToken string
|
||||
clientID string
|
||||
clientSecret string
|
||||
useOnlineAPI bool
|
||||
renewAPIURL string
|
||||
oauthURL string
|
||||
apiBaseURL string
|
||||
client *resty.Client
|
||||
onTokenUpdate func(access, refresh string)
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
listInterval time.Duration
|
||||
listCooldown time.Duration
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ID string
|
||||
RootID string
|
||||
RefreshToken string
|
||||
AccessToken string
|
||||
ClientID string
|
||||
ClientSecret string
|
||||
UseOnlineAPI bool
|
||||
RenewAPIURL string
|
||||
OAuthURL string
|
||||
APIBaseURL string
|
||||
|
||||
OnTokenUpdate func(access, refresh string)
|
||||
}
|
||||
|
||||
func New(c Config) *Driver {
|
||||
rootID := strings.TrimSpace(c.RootID)
|
||||
if rootID == "" {
|
||||
rootID = "root"
|
||||
}
|
||||
renewAPIURL := strings.TrimSpace(c.RenewAPIURL)
|
||||
if renewAPIURL == "" {
|
||||
renewAPIURL = defaultRenewAPIURL
|
||||
}
|
||||
oauthURL := strings.TrimSpace(c.OAuthURL)
|
||||
if oauthURL == "" {
|
||||
oauthURL = defaultOAuthURL
|
||||
}
|
||||
apiBaseURL := strings.TrimRight(strings.TrimSpace(c.APIBaseURL), "/")
|
||||
if apiBaseURL == "" {
|
||||
apiBaseURL = defaultAPIBaseURL
|
||||
}
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
rootID: rootID,
|
||||
refreshToken: strings.TrimSpace(c.RefreshToken),
|
||||
accessToken: strings.TrimSpace(c.AccessToken),
|
||||
clientID: strings.TrimSpace(c.ClientID),
|
||||
clientSecret: strings.TrimSpace(c.ClientSecret),
|
||||
useOnlineAPI: c.UseOnlineAPI,
|
||||
renewAPIURL: renewAPIURL,
|
||||
oauthURL: oauthURL,
|
||||
apiBaseURL: apiBaseURL,
|
||||
onTokenUpdate: c.OnTokenUpdate,
|
||||
client: resty.New().
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
listInterval: defaultListInterval,
|
||||
listCooldown: defaultListCooldown,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Kind() string { return Kind }
|
||||
func (d *Driver) ID() string { return d.id }
|
||||
func (d *Driver) RootID() string { return d.rootID }
|
||||
|
||||
func (d *Driver) Init(ctx context.Context) error {
|
||||
if d.refreshToken == "" {
|
||||
return errors.New("googledrive init: refresh_token is required")
|
||||
}
|
||||
return d.refresh(ctx)
|
||||
}
|
||||
|
||||
func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error) {
|
||||
if dirID == "" {
|
||||
dirID = d.rootID
|
||||
}
|
||||
d.listMu.Lock()
|
||||
defer d.listMu.Unlock()
|
||||
|
||||
pageToken := ""
|
||||
out := make([]drives.Entry, 0)
|
||||
for {
|
||||
if err := d.waitForListSlotLocked(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var resp filesResp
|
||||
err := d.request(ctx, d.filesURL(), http.MethodGet, func(req *resty.Request) {
|
||||
params := map[string]string{
|
||||
"fields": filesListFields,
|
||||
"pageSize": "1000",
|
||||
"q": fmt.Sprintf("'%s' in parents and trashed = false", strings.ReplaceAll(dirID, "'", "\\'")),
|
||||
"orderBy": "folder,name,modifiedTime desc",
|
||||
}
|
||||
if pageToken != "" {
|
||||
params["pageToken"] = pageToken
|
||||
}
|
||||
req.SetQueryParams(params)
|
||||
}, &resp)
|
||||
if err != nil {
|
||||
if wait, ok := drives.RateLimitRetryAfter(err); ok {
|
||||
if wait <= 0 {
|
||||
wait = d.listCooldown
|
||||
}
|
||||
if sleepErr := sleepContext(ctx, wait); sleepErr != nil {
|
||||
return nil, sleepErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("googledrive list: %w", err)
|
||||
}
|
||||
if err := d.fillShortcutFileMetadata(ctx, resp.Files); err != nil {
|
||||
return nil, fmt.Errorf("googledrive shortcut metadata: %w", err)
|
||||
}
|
||||
for _, f := range resp.Files {
|
||||
out = append(out, fileToEntry(f, dirID))
|
||||
}
|
||||
pageToken = resp.NextPageToken
|
||||
if pageToken == "" {
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) waitForListSlotLocked(ctx context.Context) error {
|
||||
if d.listInterval <= 0 || d.lastListAt.IsZero() {
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
next := d.lastListAt.Add(d.listInterval)
|
||||
now := time.Now()
|
||||
if now.Before(next) {
|
||||
if err := sleepContext(ctx, next.Sub(now)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, d time.Duration) error {
|
||||
if d <= 0 {
|
||||
return ctx.Err()
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
|
||||
var f driveFile
|
||||
if err := d.request(ctx, d.fileURL(fileID), http.MethodGet, func(req *resty.Request) {
|
||||
req.SetQueryParam("fields", fileInfoFields)
|
||||
}, &f); err != nil {
|
||||
return nil, fmt.Errorf("googledrive stat: %w", err)
|
||||
}
|
||||
e := fileToEntry(f, "")
|
||||
return &e, nil
|
||||
}
|
||||
|
||||
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
if fileID == "" {
|
||||
return nil, errors.New("googledrive stream: empty file id")
|
||||
}
|
||||
if _, err := d.Stat(ctx, fileID); err != nil {
|
||||
return nil, fmt.Errorf("googledrive stream: %w", err)
|
||||
}
|
||||
u := d.fileURL(fileID) + "?alt=media&acknowledgeAbuse=true&supportsAllDrives=true"
|
||||
return &drives.StreamLink{
|
||||
URL: u,
|
||||
Headers: http.Header{
|
||||
"Authorization": []string{"Bearer " + d.accessToken},
|
||||
},
|
||||
Expires: time.Now().Add(30 * time.Minute),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) refresh(ctx context.Context) error {
|
||||
if d.useOnlineAPI && d.renewAPIURL != "" {
|
||||
var out tokenResp
|
||||
res, err := d.client.R().
|
||||
SetContext(ctx).
|
||||
SetQueryParams(map[string]string{
|
||||
"refresh_ui": d.refreshToken,
|
||||
"server_use": "true",
|
||||
"driver_txt": "googleui_go",
|
||||
}).
|
||||
SetResult(&out).
|
||||
SetError(&out).
|
||||
Get(d.renewAPIURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("googledrive refresh token: %w", err)
|
||||
}
|
||||
if err := tokenResponseError("googledrive refresh token", res, out, true); err != nil {
|
||||
return err
|
||||
}
|
||||
d.applyToken(out)
|
||||
return nil
|
||||
}
|
||||
if d.clientID == "" || d.clientSecret == "" {
|
||||
return errors.New("googledrive refresh token: client_id and client_secret are required when online API is disabled")
|
||||
}
|
||||
var out tokenResp
|
||||
res, err := d.client.R().
|
||||
SetContext(ctx).
|
||||
SetFormData(map[string]string{
|
||||
"client_id": d.clientID,
|
||||
"client_secret": d.clientSecret,
|
||||
"refresh_token": d.refreshToken,
|
||||
"grant_type": "refresh_token",
|
||||
}).
|
||||
SetResult(&out).
|
||||
SetError(&out).
|
||||
Post(d.oauthURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("googledrive refresh token: %w", err)
|
||||
}
|
||||
if err := tokenResponseError("googledrive refresh token", res, out, false); err != nil {
|
||||
return err
|
||||
}
|
||||
d.applyToken(out)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) applyToken(out tokenResp) {
|
||||
d.accessToken = out.AccessToken
|
||||
if strings.TrimSpace(out.RefreshToken) != "" {
|
||||
d.refreshToken = out.RefreshToken
|
||||
}
|
||||
if d.onTokenUpdate != nil {
|
||||
d.onTokenUpdate(d.accessToken, d.refreshToken)
|
||||
}
|
||||
}
|
||||
|
||||
func tokenResponseError(prefix string, res *resty.Response, out tokenResp, requireRefresh bool) error {
|
||||
if out.Text != "" {
|
||||
return fmt.Errorf("%s: %s", prefix, out.Text)
|
||||
}
|
||||
if out.Error != "" {
|
||||
if out.ErrorDescription != "" {
|
||||
return fmt.Errorf("%s: %s", prefix, out.ErrorDescription)
|
||||
}
|
||||
return fmt.Errorf("%s: %s", prefix, out.Error)
|
||||
}
|
||||
if res != nil && res.IsError() {
|
||||
return fmt.Errorf("%s: status=%d body=%s", prefix, res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
if out.AccessToken == "" || (requireRefresh && out.RefreshToken == "") {
|
||||
return fmt.Errorf("%s: empty token", prefix)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) request(ctx context.Context, rawURL, method string, configure func(*resty.Request), out any) error {
|
||||
return d.requestOnce(ctx, rawURL, method, configure, out, true)
|
||||
}
|
||||
|
||||
func (d *Driver) requestOnce(ctx context.Context, rawURL, method string, configure func(*resty.Request), out any, retry bool) error {
|
||||
req := d.client.R().
|
||||
SetContext(ctx).
|
||||
SetHeader("Authorization", "Bearer "+d.accessToken).
|
||||
SetQueryParam("includeItemsFromAllDrives", "true").
|
||||
SetQueryParam("supportsAllDrives", "true")
|
||||
if configure != nil {
|
||||
configure(req)
|
||||
}
|
||||
if out != nil {
|
||||
req.SetResult(out)
|
||||
}
|
||||
var apiErr apiErrorResp
|
||||
req.SetError(&apiErr)
|
||||
res, err := req.Execute(method, rawURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isGoogleRateLimit(res, apiErr.Error) {
|
||||
return googleRateLimitError(res, apiErr.Error.Message)
|
||||
}
|
||||
if apiErr.Error.Code != 0 {
|
||||
if apiErr.Error.Code == http.StatusUnauthorized && retry {
|
||||
if err := d.refresh(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return d.requestOnce(ctx, rawURL, method, configure, out, false)
|
||||
}
|
||||
return googleAPIError(apiErr.Error)
|
||||
}
|
||||
if res.IsError() {
|
||||
return fmt.Errorf("google drive api error: status=%d body=%s", res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) fillShortcutFileMetadata(ctx context.Context, files []driveFile) error {
|
||||
for i := range files {
|
||||
f := &files[i]
|
||||
if f.MimeType != "application/vnd.google-apps.shortcut" ||
|
||||
f.Shortcut.TargetID == "" ||
|
||||
f.Shortcut.TargetMimeType == "application/vnd.google-apps.folder" {
|
||||
continue
|
||||
}
|
||||
var target driveFile
|
||||
if err := d.request(ctx, d.fileURL(f.Shortcut.TargetID), http.MethodGet, func(req *resty.Request) {
|
||||
req.SetQueryParam("fields", fileInfoFields)
|
||||
}, &target); err != nil {
|
||||
return err
|
||||
}
|
||||
if target.Size != "" {
|
||||
f.Size = target.Size
|
||||
}
|
||||
if target.MD5Checksum != "" {
|
||||
f.MD5Checksum = target.MD5Checksum
|
||||
}
|
||||
if target.SHA1Checksum != "" {
|
||||
f.SHA1Checksum = target.SHA1Checksum
|
||||
}
|
||||
if target.SHA256Checksum != "" {
|
||||
f.SHA256Checksum = target.SHA256Checksum
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) filesURL() string {
|
||||
return d.apiBaseURL + "/files"
|
||||
}
|
||||
|
||||
func (d *Driver) fileURL(fileID string) string {
|
||||
return d.filesURL() + "/" + url.PathEscape(fileID)
|
||||
}
|
||||
|
||||
func fileToEntry(f driveFile, fallbackParentID string) drives.Entry {
|
||||
id := f.ID
|
||||
isDir := f.MimeType == "application/vnd.google-apps.folder"
|
||||
if f.MimeType == "application/vnd.google-apps.shortcut" && f.Shortcut.TargetID != "" {
|
||||
id = f.Shortcut.TargetID
|
||||
isDir = f.Shortcut.TargetMimeType == "application/vnd.google-apps.folder"
|
||||
}
|
||||
size, _ := strconv.ParseInt(f.Size, 10, 64)
|
||||
hash := f.MD5Checksum
|
||||
if hash == "" {
|
||||
hash = f.SHA1Checksum
|
||||
}
|
||||
if hash == "" {
|
||||
hash = f.SHA256Checksum
|
||||
}
|
||||
return drives.Entry{
|
||||
ID: id,
|
||||
Name: f.Name,
|
||||
Size: size,
|
||||
Hash: hash,
|
||||
IsDir: isDir,
|
||||
ParentID: fallbackParentID,
|
||||
MimeType: mimeType(f),
|
||||
ModTime: f.ModifiedTime,
|
||||
ThumbnailURL: f.ThumbnailLink,
|
||||
}
|
||||
}
|
||||
|
||||
func mimeType(f driveFile) string {
|
||||
if f.MimeType != "" && f.MimeType != "application/vnd.google-apps.shortcut" {
|
||||
return f.MimeType
|
||||
}
|
||||
if f.Shortcut.TargetMimeType != "" {
|
||||
return f.Shortcut.TargetMimeType
|
||||
}
|
||||
ext := strings.ToLower(path.Ext(f.Name))
|
||||
switch ext {
|
||||
case ".mp4":
|
||||
return "video/mp4"
|
||||
case ".mkv":
|
||||
return "video/x-matroska"
|
||||
case ".mov":
|
||||
return "video/quicktime"
|
||||
case ".webm":
|
||||
return "video/webm"
|
||||
case ".avi":
|
||||
return "video/x-msvideo"
|
||||
case ".jpg", ".jpeg":
|
||||
return "image/jpeg"
|
||||
case ".png":
|
||||
return "image/png"
|
||||
default:
|
||||
return "application/octet-stream"
|
||||
}
|
||||
}
|
||||
|
||||
func isGoogleRateLimit(res *resty.Response, body apiErrorBody) bool {
|
||||
if res != nil && res.StatusCode() == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if body.Code == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
for _, e := range body.Errors {
|
||||
reason := strings.ToLower(strings.TrimSpace(e.Reason))
|
||||
switch reason {
|
||||
case "ratelimitexceeded", "userratelimitexceeded", "downloadquotaexceeded", "sharingratelimitexceeded":
|
||||
return true
|
||||
}
|
||||
}
|
||||
msg := strings.ToLower(body.Message)
|
||||
return strings.Contains(msg, "rate limit") || strings.Contains(msg, "too many requests") || strings.Contains(msg, "quota exceeded")
|
||||
}
|
||||
|
||||
func googleRateLimitError(res *resty.Response, message string) error {
|
||||
if strings.TrimSpace(message) == "" {
|
||||
message = "google drive rate limited"
|
||||
}
|
||||
if res != nil && strings.TrimSpace(res.String()) != "" {
|
||||
message = fmt.Sprintf("%s: status=%d body=%s", message, res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: parseRetryAfter(res),
|
||||
Err: errors.New(message),
|
||||
}
|
||||
}
|
||||
|
||||
func googleAPIError(body apiErrorBody) error {
|
||||
if body.Message != "" {
|
||||
return errors.New(body.Message)
|
||||
}
|
||||
if body.Code != 0 {
|
||||
return fmt.Errorf("google drive api error: code=%d", body.Code)
|
||||
}
|
||||
return errors.New("google drive api error")
|
||||
}
|
||||
|
||||
func parseRetryAfter(res *resty.Response) time.Duration {
|
||||
if res == nil {
|
||||
return 0
|
||||
}
|
||||
raw := strings.TrimSpace(res.Header().Get("Retry-After"))
|
||||
if raw == "" {
|
||||
return 0
|
||||
}
|
||||
if seconds, err := strconv.Atoi(raw); err == nil && seconds > 0 {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
if when, err := http.ParseTime(raw); err == nil {
|
||||
d := time.Until(when)
|
||||
if d > 0 {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
@@ -0,0 +1,190 @@
|
||||
package googledrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestInitUsesOnlineRenewAPI(t *testing.T) {
|
||||
var savedAccess, savedRefresh string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/renew" {
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
if got := r.URL.Query().Get("refresh_ui"); got != "old-refresh" {
|
||||
t.Fatalf("refresh_ui = %q", got)
|
||||
}
|
||||
if got := r.URL.Query().Get("server_use"); got != "true" {
|
||||
t.Fatalf("server_use = %q", got)
|
||||
}
|
||||
if got := r.URL.Query().Get("driver_txt"); got != "googleui_go" {
|
||||
t.Fatalf("driver_txt = %q", got)
|
||||
}
|
||||
writeTestJSON(w, tokenResp{
|
||||
AccessToken: "new-access",
|
||||
RefreshToken: "new-refresh",
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "g",
|
||||
RefreshToken: "old-refresh",
|
||||
UseOnlineAPI: true,
|
||||
RenewAPIURL: srv.URL + "/renew",
|
||||
OnTokenUpdate: func(access, refresh string) {
|
||||
savedAccess = access
|
||||
savedRefresh = refresh
|
||||
},
|
||||
})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("Init() error = %v", err)
|
||||
}
|
||||
if d.accessToken != "new-access" || d.refreshToken != "new-refresh" {
|
||||
t.Fatalf("tokens not applied: access=%q refresh=%q", d.accessToken, d.refreshToken)
|
||||
}
|
||||
if savedAccess != "new-access" || savedRefresh != "new-refresh" {
|
||||
t.Fatalf("tokens not persisted: access=%q refresh=%q", savedAccess, savedRefresh)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListMapsGoogleDriveFiles(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("Authorization = %q", got)
|
||||
}
|
||||
if r.URL.Path != "/drive/v3/files" {
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
if !strings.Contains(r.URL.Query().Get("q"), "'root' in parents") {
|
||||
t.Fatalf("unexpected q = %q", r.URL.Query().Get("q"))
|
||||
}
|
||||
writeTestJSON(w, filesResp{Files: []driveFile{
|
||||
{ID: "folder-1", Name: "Movies", MimeType: "application/vnd.google-apps.folder"},
|
||||
{
|
||||
ID: "file-1",
|
||||
Name: "clip.mp4",
|
||||
MimeType: "video/mp4",
|
||||
Size: "1234",
|
||||
MD5Checksum: "abc",
|
||||
ThumbnailLink: "https://thumb.example/1",
|
||||
},
|
||||
}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", RootID: "root", APIBaseURL: srv.URL + "/drive/v3"})
|
||||
d.accessToken = "access"
|
||||
d.listInterval = -1
|
||||
|
||||
entries, err := d.List(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("List() error = %v", err)
|
||||
}
|
||||
if len(entries) != 2 {
|
||||
t.Fatalf("len(entries) = %d", len(entries))
|
||||
}
|
||||
if !entries[0].IsDir || entries[0].ID != "folder-1" {
|
||||
t.Fatalf("folder entry = %+v", entries[0])
|
||||
}
|
||||
if entries[1].ID != "file-1" || entries[1].Size != 1234 || entries[1].Hash != "abc" || entries[1].ThumbnailURL == "" {
|
||||
t.Fatalf("file entry = %+v", entries[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLReturnsAuthenticatedMediaLinkWithoutRedirectRequirement(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("Authorization = %q", got)
|
||||
}
|
||||
if r.URL.Path != "/drive/v3/files/file-1" {
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
writeTestJSON(w, driveFile{
|
||||
ID: "file-1",
|
||||
Name: "clip.mp4",
|
||||
MimeType: "video/mp4",
|
||||
Size: "1234",
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", APIBaseURL: srv.URL + "/drive/v3"})
|
||||
d.accessToken = "access"
|
||||
|
||||
link, err := d.StreamURL(context.Background(), "file-1")
|
||||
if err != nil {
|
||||
t.Fatalf("StreamURL() error = %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(link.URL, srv.URL+"/drive/v3/files/file-1?") {
|
||||
t.Fatalf("link URL = %q", link.URL)
|
||||
}
|
||||
if !strings.Contains(link.URL, "alt=media") {
|
||||
t.Fatalf("link URL missing alt=media: %q", link.URL)
|
||||
}
|
||||
if got := link.Headers.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("link Authorization = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestRefreshesOnUnauthorized(t *testing.T) {
|
||||
var fileCalls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/renew":
|
||||
writeTestJSON(w, tokenResp{
|
||||
AccessToken: "new-access",
|
||||
RefreshToken: "new-refresh",
|
||||
})
|
||||
case "/drive/v3/files/file-1":
|
||||
fileCalls++
|
||||
if fileCalls == 1 {
|
||||
writeTestJSONStatus(w, http.StatusUnauthorized, apiErrorResp{Error: apiErrorBody{
|
||||
Code: http.StatusUnauthorized,
|
||||
Message: "Invalid Credentials",
|
||||
}})
|
||||
return
|
||||
}
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer new-access" {
|
||||
t.Fatalf("Authorization after refresh = %q", got)
|
||||
}
|
||||
writeTestJSON(w, driveFile{ID: "file-1", Name: "clip.mp4", Size: "1"})
|
||||
default:
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "g",
|
||||
RefreshToken: "old-refresh",
|
||||
UseOnlineAPI: true,
|
||||
RenewAPIURL: srv.URL + "/renew",
|
||||
APIBaseURL: srv.URL + "/drive/v3",
|
||||
})
|
||||
d.accessToken = "old-access"
|
||||
|
||||
if _, err := d.Stat(context.Background(), "file-1"); err != nil {
|
||||
t.Fatalf("Stat() error = %v", err)
|
||||
}
|
||||
if fileCalls != 2 {
|
||||
t.Fatalf("fileCalls = %d", fileCalls)
|
||||
}
|
||||
if d.accessToken != "new-access" || d.refreshToken != "new-refresh" {
|
||||
t.Fatalf("tokens not refreshed: access=%q refresh=%q", d.accessToken, d.refreshToken)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestJSON(w http.ResponseWriter, v any) {
|
||||
writeTestJSONStatus(w, http.StatusOK, v)
|
||||
}
|
||||
|
||||
func writeTestJSONStatus(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package googledrive
|
||||
|
||||
import "time"
|
||||
|
||||
type tokenResp struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
RefreshToken string `json:"refresh_token"`
|
||||
ExpiresIn int64 `json:"expires_in"`
|
||||
Error string `json:"error"`
|
||||
ErrorDescription string `json:"error_description"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
type filesResp struct {
|
||||
NextPageToken string `json:"nextPageToken"`
|
||||
Files []driveFile `json:"files"`
|
||||
Error apiErrorBody `json:"error"`
|
||||
}
|
||||
|
||||
type driveFile struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
MimeType string `json:"mimeType"`
|
||||
ModifiedTime time.Time `json:"modifiedTime"`
|
||||
CreatedTime time.Time `json:"createdTime"`
|
||||
Size string `json:"size"`
|
||||
ThumbnailLink string `json:"thumbnailLink"`
|
||||
MD5Checksum string `json:"md5Checksum"`
|
||||
SHA1Checksum string `json:"sha1Checksum"`
|
||||
SHA256Checksum string `json:"sha256Checksum"`
|
||||
Shortcut struct {
|
||||
TargetID string `json:"targetId"`
|
||||
TargetMimeType string `json:"targetMimeType"`
|
||||
} `json:"shortcutDetails"`
|
||||
}
|
||||
|
||||
type apiErrorResp struct {
|
||||
Error apiErrorBody `json:"error"`
|
||||
}
|
||||
|
||||
type apiErrorBody struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Errors []struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
} `json:"errors"`
|
||||
}
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
// Drive 是多家网盘统一抽象。上层不区分盘,只区分 Kind。
|
||||
type Drive interface {
|
||||
// Kind 返回驱动代号:"quark" / "p115" / "pikpak" / "wopan" / "onedrive"
|
||||
// Kind 返回驱动代号:"quark" / "p115" / "p123" / "pikpak" / "wopan" / "onedrive" / "googledrive" / "localstorage"
|
||||
Kind() string
|
||||
|
||||
// ID 返回该盘在 catalog 中的唯一标识
|
||||
@@ -30,7 +30,7 @@ type Drive interface {
|
||||
StreamURL(ctx context.Context, fileID string) (*StreamLink, error)
|
||||
|
||||
// Upload 把本地流写入指定目录,返回新文件 fileID。
|
||||
// 当前 teaser 和封面只保存在本地,不再通过该方法写回网盘。
|
||||
// 当前预览视频和封面只保存在本地,不再通过该方法写回网盘。
|
||||
Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error)
|
||||
|
||||
// EnsureDir 保证指定路径存在(相对根目录),返回最终目录 fileID。
|
||||
|
||||
@@ -0,0 +1,411 @@
|
||||
// Package localstorage exposes an existing server-side directory as a Drive.
|
||||
package localstorage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
const Kind = "localstorage"
|
||||
|
||||
const maxSTRMBytes = 64 * 1024
|
||||
|
||||
type Config struct {
|
||||
ID string
|
||||
RootPath string
|
||||
}
|
||||
|
||||
type Driver struct {
|
||||
id string
|
||||
rootPath string
|
||||
}
|
||||
|
||||
func New(c Config) *Driver {
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
rootPath: c.RootPath,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Kind() string { return Kind }
|
||||
|
||||
func (d *Driver) ID() string { return d.id }
|
||||
|
||||
func (d *Driver) RootID() string { return "/" }
|
||||
|
||||
func (d *Driver) Init(context.Context) error {
|
||||
root, err := d.root()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("localstorage: stat root %q: %w%s", root, err, localStoragePathHint(d.rootPath))
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("localstorage: root is not a directory: %s", root)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error) {
|
||||
dir, rel, err := d.pathForID(dirID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]drives.Entry, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Symlinks can escape the configured root or create cycles. Keep the
|
||||
// local storage drive predictable by scanning real files/directories only.
|
||||
if entry.Type()&os.ModeSymlink != 0 {
|
||||
continue
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !info.IsDir() && !info.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
childRel := joinRel(rel, entry.Name())
|
||||
out = append(out, drives.Entry{
|
||||
ID: encodeRel(childRel),
|
||||
Name: entry.Name(),
|
||||
Size: sizeForEntry(info),
|
||||
IsDir: info.IsDir(),
|
||||
ParentID: idForRel(rel),
|
||||
ModTime: info.ModTime(),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
|
||||
p, rel, err := d.pathForID(fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &drives.Entry{
|
||||
ID: idForRel(rel),
|
||||
Name: filepath.Base(p),
|
||||
Size: sizeForEntry(info),
|
||||
IsDir: info.IsDir(),
|
||||
ParentID: idForRel(parentRel(rel)),
|
||||
ModTime: info.ModTime(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
p, _, err := d.pathForID(fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.IsDir() || !info.Mode().IsRegular() {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
if strings.EqualFold(filepath.Ext(p), ".strm") {
|
||||
return d.streamURLFromSTRM(ctx, p)
|
||||
}
|
||||
if info.Size() <= 0 {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return &drives.StreamLink{
|
||||
URL: p,
|
||||
Expires: time.Now().Add(24 * time.Hour),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) streamURLFromSTRM(ctx context.Context, strmPath string) (*drives.StreamLink, error) {
|
||||
target, err := readSTRMTarget(strmPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if filepath.IsAbs(target) {
|
||||
return d.localSTRMLink(strmPath, target)
|
||||
}
|
||||
u, err := url.Parse(target)
|
||||
if err == nil {
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https":
|
||||
if u.Host == "" {
|
||||
return nil, fmt.Errorf("localstorage: invalid strm url %q", target)
|
||||
}
|
||||
return &drives.StreamLink{
|
||||
URL: target,
|
||||
Expires: time.Now().Add(24 * time.Hour),
|
||||
}, nil
|
||||
case "file":
|
||||
if u.Host != "" && !strings.EqualFold(u.Host, "localhost") {
|
||||
return nil, fmt.Errorf("localstorage: unsupported strm file url host %q", u.Host)
|
||||
}
|
||||
return d.localSTRMLink(strmPath, u.Path)
|
||||
case "":
|
||||
// Local path below.
|
||||
default:
|
||||
return nil, fmt.Errorf("localstorage: unsupported strm target scheme %q", u.Scheme)
|
||||
}
|
||||
} else if strings.Contains(target, "://") {
|
||||
return nil, fmt.Errorf("localstorage: invalid strm url %q: %w", target, err)
|
||||
}
|
||||
return d.localSTRMLink(strmPath, target)
|
||||
}
|
||||
|
||||
func readSTRMTarget(path string) (string, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(f, maxSTRMBytes+1))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(data) > maxSTRMBytes {
|
||||
return "", errors.New("localstorage: strm file is too large")
|
||||
}
|
||||
lines := strings.Split(string(data), "\n")
|
||||
for i, line := range lines {
|
||||
if i == 0 {
|
||||
line = strings.TrimPrefix(line, "\ufeff")
|
||||
}
|
||||
line = strings.TrimSpace(line)
|
||||
if line != "" {
|
||||
return line, nil
|
||||
}
|
||||
}
|
||||
return "", errors.New("localstorage: empty strm target")
|
||||
}
|
||||
|
||||
func (d *Driver) localSTRMLink(strmPath, target string) (*drives.StreamLink, error) {
|
||||
target = strings.TrimSpace(target)
|
||||
if target == "" {
|
||||
return nil, errors.New("localstorage: empty strm target")
|
||||
}
|
||||
|
||||
var p string
|
||||
if filepath.IsAbs(target) {
|
||||
p = filepath.Clean(target)
|
||||
} else {
|
||||
p = filepath.Join(filepath.Dir(strmPath), filepath.FromSlash(target))
|
||||
}
|
||||
p, err := filepath.Abs(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root, err := d.root()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
realPath, within, err := realPathWithinRoot(root, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !within {
|
||||
return nil, errors.New("localstorage: strm target escapes root")
|
||||
}
|
||||
if strings.EqualFold(filepath.Ext(p), ".strm") || strings.EqualFold(filepath.Ext(realPath), ".strm") {
|
||||
return nil, errors.New("localstorage: nested strm target is not supported")
|
||||
}
|
||||
info, err := os.Stat(realPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.IsDir() || !info.Mode().IsRegular() || info.Size() <= 0 {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return &drives.StreamLink{
|
||||
URL: realPath,
|
||||
Expires: time.Now().Add(24 * time.Hour),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) root() (string, error) {
|
||||
raw := strings.TrimSpace(d.rootPath)
|
||||
if raw == "" {
|
||||
return "", errors.New("localstorage: empty path")
|
||||
}
|
||||
raw = os.ExpandEnv(raw)
|
||||
if strings.HasPrefix(raw, "~") {
|
||||
if home, err := os.UserHomeDir(); err == nil && home != "" {
|
||||
switch {
|
||||
case raw == "~":
|
||||
raw = home
|
||||
case strings.HasPrefix(raw, "~/") || strings.HasPrefix(raw, `~\`):
|
||||
raw = filepath.Join(home, raw[2:])
|
||||
}
|
||||
}
|
||||
}
|
||||
return filepath.Abs(raw)
|
||||
}
|
||||
|
||||
func (d *Driver) pathForID(id string) (string, string, error) {
|
||||
root, err := d.root()
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
rel, err := decodeRel(id)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if rel == "" {
|
||||
return root, "", nil
|
||||
}
|
||||
p, err := filepath.Abs(filepath.Join(root, filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if !pathWithinRoot(root, p) {
|
||||
return "", "", errors.New("localstorage: path escapes root")
|
||||
}
|
||||
if _, within, err := realPathWithinRoot(root, p); err != nil {
|
||||
return "", "", err
|
||||
} else if !within {
|
||||
return "", "", errors.New("localstorage: path escapes root")
|
||||
}
|
||||
return p, rel, nil
|
||||
}
|
||||
|
||||
func pathWithinRoot(root, path string) bool {
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator)))
|
||||
}
|
||||
|
||||
func realPathWithinRoot(root, path string) (string, bool, error) {
|
||||
realRoot, err := filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
realRoot, err = filepath.Abs(realRoot)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
realPath, err := filepath.EvalSymlinks(path)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
realPath, err = filepath.Abs(realPath)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return realPath, pathWithinRoot(realRoot, realPath), nil
|
||||
}
|
||||
|
||||
func localStoragePathHint(configured string) string {
|
||||
cwd, _ := os.Getwd()
|
||||
parts := []string{}
|
||||
if strings.TrimSpace(configured) != "" {
|
||||
parts = append(parts, fmt.Sprintf("configured=%q", strings.TrimSpace(configured)))
|
||||
}
|
||||
if cwd != "" {
|
||||
parts = append(parts, fmt.Sprintf("cwd=%q", cwd))
|
||||
}
|
||||
if _, err := os.Stat("/.dockerenv"); err == nil {
|
||||
parts = append(parts, "docker=host paths must be bind-mounted into the container")
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
return " (" + strings.Join(parts, ", ") + ")"
|
||||
}
|
||||
|
||||
func decodeRel(id string) (string, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" || id == "/" {
|
||||
return "", nil
|
||||
}
|
||||
raw, err := base64.RawURLEncoding.DecodeString(id)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("localstorage: invalid file id: %w", err)
|
||||
}
|
||||
rel := filepath.ToSlash(filepath.Clean(filepath.FromSlash(string(raw))))
|
||||
if rel == "." {
|
||||
return "", nil
|
||||
}
|
||||
if strings.HasPrefix(rel, "../") || rel == ".." || strings.HasPrefix(rel, "/") {
|
||||
return "", errors.New("localstorage: invalid relative path")
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
|
||||
func encodeRel(rel string) string {
|
||||
rel = filepath.ToSlash(filepath.Clean(filepath.FromSlash(rel)))
|
||||
if rel == "." || rel == "" {
|
||||
return "/"
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(rel))
|
||||
}
|
||||
|
||||
func idForRel(rel string) string {
|
||||
if rel == "" {
|
||||
return "/"
|
||||
}
|
||||
return encodeRel(rel)
|
||||
}
|
||||
|
||||
func joinRel(parent, name string) string {
|
||||
if parent == "" {
|
||||
return filepath.ToSlash(name)
|
||||
}
|
||||
return filepath.ToSlash(filepath.Join(filepath.FromSlash(parent), name))
|
||||
}
|
||||
|
||||
func parentRel(rel string) string {
|
||||
if rel == "" {
|
||||
return ""
|
||||
}
|
||||
parent := filepath.ToSlash(filepath.Dir(filepath.FromSlash(rel)))
|
||||
if parent == "." {
|
||||
return ""
|
||||
}
|
||||
return parent
|
||||
}
|
||||
|
||||
func sizeForEntry(info os.FileInfo) int64 {
|
||||
if info == nil || info.IsDir() {
|
||||
return 0
|
||||
}
|
||||
return info.Size()
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
@@ -0,0 +1,339 @@
|
||||
package localstorage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/scanner"
|
||||
)
|
||||
|
||||
func TestListEncodesRelativePathsAndStreamURLResolvesFile(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
sub := filepath.Join(root, "clips")
|
||||
if err := os.MkdirAll(sub, 0o755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
videoPath := filepath.Join(sub, "sample.mp4")
|
||||
if err := os.WriteFile(videoPath, []byte("video"), 0o644); err != nil {
|
||||
t.Fatalf("write video: %v", err)
|
||||
}
|
||||
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
if err := drv.Init(context.Background()); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
rootEntries, err := drv.List(context.Background(), drv.RootID())
|
||||
if err != nil {
|
||||
t.Fatalf("list root: %v", err)
|
||||
}
|
||||
if len(rootEntries) != 1 || !rootEntries[0].IsDir {
|
||||
t.Fatalf("root entries = %#v, want one directory", rootEntries)
|
||||
}
|
||||
if strings.Contains(rootEntries[0].ID, "/") {
|
||||
t.Fatalf("encoded dir id contains slash: %q", rootEntries[0].ID)
|
||||
}
|
||||
|
||||
fileEntries, err := drv.List(context.Background(), rootEntries[0].ID)
|
||||
if err != nil {
|
||||
t.Fatalf("list subdir: %v", err)
|
||||
}
|
||||
if len(fileEntries) != 1 || fileEntries[0].Name != "sample.mp4" {
|
||||
t.Fatalf("file entries = %#v, want sample.mp4", fileEntries)
|
||||
}
|
||||
if strings.Contains(fileEntries[0].ID, "/") {
|
||||
t.Fatalf("encoded file id contains slash: %q", fileEntries[0].ID)
|
||||
}
|
||||
|
||||
link, err := drv.StreamURL(context.Background(), fileEntries[0].ID)
|
||||
if err != nil {
|
||||
t.Fatalf("stream url: %v", err)
|
||||
}
|
||||
if link.URL != videoPath {
|
||||
t.Fatalf("url = %q, want %q", link.URL, videoPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLResolvesHTTPSTRM(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
strmPath := filepath.Join(root, "movie.strm")
|
||||
target := "https://media.example/clip.mp4?token=abc"
|
||||
if err := os.WriteFile(strmPath, []byte("\ufeff\n "+target+"\n"), 0o644); err != nil {
|
||||
t.Fatalf("write strm: %v", err)
|
||||
}
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
|
||||
link, err := drv.StreamURL(context.Background(), encodeRel("movie.strm"))
|
||||
if err != nil {
|
||||
t.Fatalf("stream url: %v", err)
|
||||
}
|
||||
if link.URL != target {
|
||||
t.Fatalf("url = %q, want %q", link.URL, target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLResolvesRelativeLocalSTRM(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(root, "links"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir links: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(root, "media"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir media: %v", err)
|
||||
}
|
||||
videoPath := filepath.Join(root, "media", "clip.mp4")
|
||||
if err := os.WriteFile(videoPath, []byte("video"), 0o644); err != nil {
|
||||
t.Fatalf("write video: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "links", "movie.strm"), []byte("../media/clip.mp4\n"), 0o644); err != nil {
|
||||
t.Fatalf("write strm: %v", err)
|
||||
}
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
|
||||
link, err := drv.StreamURL(context.Background(), encodeRel("links/movie.strm"))
|
||||
if err != nil {
|
||||
t.Fatalf("stream url: %v", err)
|
||||
}
|
||||
if link.URL != videoPath {
|
||||
t.Fatalf("url = %q, want %q", link.URL, videoPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRejectsInvalidSTRMTargets(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(t *testing.T, root string) string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
setup: func(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "empty.strm"), []byte("\n \r\n"))
|
||||
return "empty.strm"
|
||||
},
|
||||
want: "empty strm target",
|
||||
},
|
||||
{
|
||||
name: "escapes root",
|
||||
setup: func(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
writeLocalStorageTestFile(t, filepath.Join(filepath.Dir(root), "outside.mp4"), []byte("video"))
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "escape.strm"), []byte("../outside.mp4\n"))
|
||||
return "escape.strm"
|
||||
},
|
||||
want: "escapes root",
|
||||
},
|
||||
{
|
||||
name: "nested",
|
||||
setup: func(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "nested.strm"), []byte("https://media.example/clip.mp4\n"))
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "outer.strm"), []byte("nested.strm\n"))
|
||||
return "outer.strm"
|
||||
},
|
||||
want: "nested strm target",
|
||||
},
|
||||
{
|
||||
name: "unsupported scheme",
|
||||
setup: func(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "ftp.strm"), []byte("ftp://media.example/clip.mp4\n"))
|
||||
return "ftp.strm"
|
||||
},
|
||||
want: "unsupported strm target scheme",
|
||||
},
|
||||
{
|
||||
name: "too large",
|
||||
setup: func(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "large.strm"), []byte(strings.Repeat("x", maxSTRMBytes+1)))
|
||||
return "large.strm"
|
||||
},
|
||||
want: "strm file is too large",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
rel := tt.setup(t, root)
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
|
||||
_, err := drv.StreamURL(context.Background(), encodeRel(rel))
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("error = %v, want contain %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRejectsSTRMTargetEscapingRootThroughSymlink(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
writeLocalStorageTestFile(t, filepath.Join(outside, "secret.mp4"), []byte("secret"))
|
||||
if err := os.MkdirAll(filepath.Join(root, "links"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir links: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(root, "real"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir real: %v", err)
|
||||
}
|
||||
if err := os.Symlink(outside, filepath.Join(root, "real", "outside")); err != nil {
|
||||
t.Fatalf("symlink: %v", err)
|
||||
}
|
||||
writeLocalStorageTestFile(t, filepath.Join(root, "links", "movie.strm"), []byte("../real/outside/secret.mp4\n"))
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
|
||||
_, err := drv.StreamURL(context.Background(), encodeRel("links/movie.strm"))
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "strm target escapes root") {
|
||||
t.Fatalf("error = %v, want strm target escapes root", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRejectsSymlinkFileIDEscapingRoot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
writeLocalStorageTestFile(t, filepath.Join(outside, "secret.mp4"), []byte("secret"))
|
||||
if err := os.Symlink(filepath.Join(outside, "secret.mp4"), filepath.Join(root, "link.mp4")); err != nil {
|
||||
t.Fatalf("symlink: %v", err)
|
||||
}
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
|
||||
_, err := drv.StreamURL(context.Background(), encodeRel("link.mp4"))
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "path escapes root") {
|
||||
t.Fatalf("error = %v, want path escapes root", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRejectsEscapingID(t *testing.T) {
|
||||
drv := New(Config{ID: "local", RootPath: t.TempDir()})
|
||||
escaped := base64.RawURLEncoding.EncodeToString([]byte("../secret.mp4"))
|
||||
|
||||
_, err := drv.StreamURL(context.Background(), escaped)
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid relative path") {
|
||||
t.Fatalf("error = %v, want invalid relative path", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitRequiresExistingDirectory(t *testing.T) {
|
||||
missing := filepath.Join(t.TempDir(), "missing")
|
||||
drv := New(Config{ID: "local", RootPath: missing})
|
||||
|
||||
err := drv.Init(context.Background())
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "stat root") {
|
||||
t.Fatalf("error = %v, want stat root failure", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), missing) || !strings.Contains(err.Error(), "configured=") {
|
||||
t.Fatalf("error = %v, want diagnostic path details", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathForIDAllowsRootPathSlash(t *testing.T) {
|
||||
drv := New(Config{ID: "local", RootPath: string(os.PathSeparator)})
|
||||
childID := encodeRel("tmp")
|
||||
|
||||
path, rel, err := drv.pathForID(childID)
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("pathForID: %v", err)
|
||||
}
|
||||
if rel != "tmp" {
|
||||
t.Fatalf("rel = %q, want tmp", rel)
|
||||
}
|
||||
if path != filepath.Join(string(os.PathSeparator), "tmp") {
|
||||
t.Fatalf("path = %q, want /tmp", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerPersistsLocalStorageSTRM(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(root, "collection"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir collection: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "collection", "clip.strm"), []byte("https://media.example/clip.mp4\n"), 0o644); err != nil {
|
||||
t.Fatalf("write strm: %v", err)
|
||||
}
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
sc := scanner.New(cat, drv, []string{".strm"}, nil, nil)
|
||||
stats, err := sc.Run(ctx, drv.RootID())
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Added != 1 {
|
||||
t.Fatalf("added = %d, want 1", stats.Added)
|
||||
}
|
||||
|
||||
fileID := encodeRel("collection/clip.strm")
|
||||
got, err := cat.GetVideo(ctx, Kind+"-local-"+fileID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.Ext != "strm" || got.FileID != fileID || got.Category != "collection" {
|
||||
t.Fatalf("video = %#v, want local strm video in collection", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerPersistsLocalStorageVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(root, "collection"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir collection: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "collection", "clip.mp4"), []byte("video"), 0o644); err != nil {
|
||||
t.Fatalf("write video: %v", err)
|
||||
}
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
sc := scanner.New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
stats, err := sc.Run(ctx, drv.RootID())
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Added != 1 {
|
||||
t.Fatalf("added = %d, want 1", stats.Added)
|
||||
}
|
||||
|
||||
fileID := encodeRel("collection/clip.mp4")
|
||||
got, err := cat.GetVideo(ctx, Kind+"-local-"+fileID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DriveID != "local" || got.FileID != fileID || got.Category != "collection" {
|
||||
t.Fatalf("video = %#v, want local drive video in collection", got)
|
||||
}
|
||||
}
|
||||
|
||||
func writeLocalStorageTestFile(t *testing.T, path string, data []byte) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
@@ -3,14 +3,19 @@ package onedrive
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-resty/resty/v2"
|
||||
@@ -18,8 +23,17 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
maxSmallUploadSize = 250 * 1024 * 1024
|
||||
defaultRenewAPIURL = "https://api.oplist.org/onedrive/renewapi"
|
||||
maxSmallUploadSize = 250 * 1024 * 1024
|
||||
defaultUploadSessionChunk = 10 * 1024 * 1024
|
||||
uploadSessionRetryAttempts = 3
|
||||
defaultRenewAPIURL = "https://api.oplist.org/onedrive/renewapi"
|
||||
onedriveListCooldown = 5 * time.Minute
|
||||
onedriveListInterval = 1 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
smallUploadThreshold = int64(maxSmallUploadSize)
|
||||
uploadSessionChunk = int64(defaultUploadSessionChunk)
|
||||
)
|
||||
|
||||
type Driver struct {
|
||||
@@ -34,6 +48,11 @@ type Driver struct {
|
||||
renewAPIURL string
|
||||
client *resty.Client
|
||||
onTokenUpdate func(access, refresh string)
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
listInterval time.Duration
|
||||
listCooldown time.Duration
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -85,6 +104,8 @@ func New(c Config) *Driver {
|
||||
client: resty.New().
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
listInterval: onedriveListInterval,
|
||||
listCooldown: onedriveListCooldown,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,10 +127,16 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
|
||||
if dirID == "" {
|
||||
dirID = d.rootID
|
||||
}
|
||||
d.listMu.Lock()
|
||||
defer d.listMu.Unlock()
|
||||
|
||||
nextLink := d.childrenURL(dirID)
|
||||
first := true
|
||||
out := make([]drives.Entry, 0)
|
||||
for nextLink != "" {
|
||||
if err := d.waitForListSlotLocked(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var resp filesResp
|
||||
err := d.request(ctx, nextLink, http.MethodGet, func(req *resty.Request) {
|
||||
if first {
|
||||
@@ -120,6 +147,19 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
|
||||
}
|
||||
}, &resp)
|
||||
if err != nil {
|
||||
if wait, ok := drives.RateLimitRetryAfter(err); ok {
|
||||
if wait <= 0 {
|
||||
wait = d.listCooldown
|
||||
if wait <= 0 {
|
||||
wait = onedriveListCooldown
|
||||
}
|
||||
}
|
||||
log.Printf("[onedrive] list cooling down drive=%s dir=%s cooldown=%s err=%v", d.id, dirID, wait, err)
|
||||
if err := sleepContext(ctx, wait); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("onedrive list: %w", err)
|
||||
}
|
||||
for _, item := range resp.Value {
|
||||
@@ -131,6 +171,36 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *Driver) waitForListSlotLocked(ctx context.Context) error {
|
||||
if d.listInterval <= 0 || d.lastListAt.IsZero() {
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
next := d.lastListAt.Add(d.listInterval)
|
||||
now := time.Now()
|
||||
if now.Before(next) {
|
||||
if err := sleepContext(ctx, next.Sub(now)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, d time.Duration) error {
|
||||
if d <= 0 {
|
||||
return ctx.Err()
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
|
||||
var item graphItem
|
||||
if err := d.request(ctx, d.itemURL(fileID), http.MethodGet, nil, &item); err != nil {
|
||||
@@ -156,15 +226,49 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
res, err := d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return res.FileID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
parentID, name, err := d.normalizeUploadArgs(parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
threshold := smallUploadThreshold
|
||||
if threshold <= 0 {
|
||||
threshold = maxSmallUploadSize
|
||||
}
|
||||
if size <= threshold {
|
||||
return d.uploadSmallAndReportHash(ctx, parentID, name, r, size, threshold)
|
||||
}
|
||||
return d.uploadSessionAndReportHash(ctx, parentID, name, r, size)
|
||||
}
|
||||
|
||||
func (d *Driver) normalizeUploadArgs(parentID, name string, r io.Reader, size int64) (string, string, error) {
|
||||
if r == nil {
|
||||
return "", "", errors.New("onedrive upload: body is required")
|
||||
}
|
||||
if size < 0 {
|
||||
return "", "", fmt.Errorf("onedrive upload: invalid size %d", size)
|
||||
}
|
||||
if parentID == "" {
|
||||
parentID = d.rootID
|
||||
}
|
||||
if size > maxSmallUploadSize {
|
||||
return "", fmt.Errorf("onedrive upload: files over %d bytes require upload session", maxSmallUploadSize)
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return "", "", errors.New("onedrive upload: empty file name")
|
||||
}
|
||||
data, err := readSmallUpload(r)
|
||||
return parentID, name, nil
|
||||
}
|
||||
|
||||
func (d *Driver) uploadSmallAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size, limit int64) (UploadResult, error) {
|
||||
data, hash, actualSize, err := readSmallUpload(r, size, limit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return UploadResult{}, err
|
||||
}
|
||||
u := fmt.Sprintf("%s/items/%s:/%s:/content", d.driveBaseURL(), url.PathEscape(parentID), url.PathEscape(name))
|
||||
var item graphItem
|
||||
@@ -173,26 +277,159 @@ func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader,
|
||||
req.SetContentLength(true)
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("onedrive upload: %w", err)
|
||||
return UploadResult{}, fmt.Errorf("onedrive upload: %w", err)
|
||||
}
|
||||
if item.ID == "" {
|
||||
return "", errors.New("onedrive upload: empty item id")
|
||||
return UploadResult{}, errors.New("onedrive upload: empty item id")
|
||||
}
|
||||
return item.ID, nil
|
||||
return UploadResult{FileID: item.ID, Hash: hash, Size: actualSize}, nil
|
||||
}
|
||||
|
||||
func readSmallUpload(r io.Reader) ([]byte, error) {
|
||||
if r == nil {
|
||||
return nil, errors.New("onedrive upload: body is required")
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(r, maxSmallUploadSize+1))
|
||||
func (d *Driver) uploadSessionAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
session, err := d.createUploadSession(ctx, parentID, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("onedrive upload: read body: %w", err)
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if len(data) > maxSmallUploadSize {
|
||||
return nil, fmt.Errorf("onedrive upload: files over %d bytes require upload session", maxSmallUploadSize)
|
||||
if strings.TrimSpace(session.UploadURL) == "" {
|
||||
return UploadResult{}, errors.New("onedrive upload session: empty upload url")
|
||||
}
|
||||
return data, nil
|
||||
|
||||
chunkSize := uploadSessionChunk
|
||||
if chunkSize <= 0 {
|
||||
chunkSize = defaultUploadSessionChunk
|
||||
}
|
||||
buf := make([]byte, int(chunkSize))
|
||||
hasher := sha1.New()
|
||||
var finalItem graphItem
|
||||
var offset int64
|
||||
for offset < size {
|
||||
partSize := minInt64(chunkSize, size-offset)
|
||||
chunk := buf[:int(partSize)]
|
||||
n, err := io.ReadFull(r, chunk)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return UploadResult{}, fmt.Errorf("onedrive upload: size mismatch: declared %d, copied %d", size, offset+int64(n))
|
||||
}
|
||||
return UploadResult{}, fmt.Errorf("onedrive upload: read body: %w", err)
|
||||
}
|
||||
chunk = chunk[:n]
|
||||
_, _ = hasher.Write(chunk)
|
||||
item, err := d.putUploadSessionChunkWithRetry(ctx, session.UploadURL, offset, size, chunk)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if item != nil {
|
||||
finalItem = *item
|
||||
}
|
||||
offset += int64(n)
|
||||
}
|
||||
if finalItem.ID == "" {
|
||||
return UploadResult{}, errors.New("onedrive upload session: empty item id")
|
||||
}
|
||||
return UploadResult{
|
||||
FileID: finalItem.ID,
|
||||
Hash: hex.EncodeToString(hasher.Sum(nil)),
|
||||
Size: offset,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) createUploadSession(ctx context.Context, parentID, name string) (uploadSessionResp, error) {
|
||||
u := fmt.Sprintf("%s/items/%s:/%s:/createUploadSession", d.driveBaseURL(), url.PathEscape(parentID), url.PathEscape(name))
|
||||
body := map[string]any{
|
||||
"item": map[string]any{
|
||||
"@microsoft.graph.conflictBehavior": "rename",
|
||||
},
|
||||
}
|
||||
var out uploadSessionResp
|
||||
err := d.request(ctx, u, http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(body)
|
||||
}, &out)
|
||||
if err != nil {
|
||||
return uploadSessionResp{}, fmt.Errorf("onedrive upload session: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunkWithRetry(ctx context.Context, uploadURL string, start, total int64, data []byte) (*graphItem, error) {
|
||||
var last error
|
||||
for attempt := 0; attempt < uploadSessionRetryAttempts; attempt++ {
|
||||
if attempt > 0 {
|
||||
if err := sleepContext(ctx, time.Duration(attempt)*time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
item, retryable, err := d.putUploadSessionChunk(ctx, uploadURL, start, total, data)
|
||||
if err == nil {
|
||||
return item, nil
|
||||
}
|
||||
last = err
|
||||
if !retryable {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if last == nil {
|
||||
last = errors.New("onedrive upload session: retry attempts exhausted")
|
||||
}
|
||||
return nil, last
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunk(ctx context.Context, uploadURL string, start, total int64, data []byte) (*graphItem, bool, error) {
|
||||
end := start + int64(len(data)) - 1
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
req.ContentLength = int64(len(data))
|
||||
req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, total))
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK, http.StatusCreated:
|
||||
var item graphItem
|
||||
if err := json.NewDecoder(res.Body).Decode(&item); err != nil {
|
||||
return nil, false, fmt.Errorf("onedrive upload session: decode completed item: %w", err)
|
||||
}
|
||||
return &item, false, nil
|
||||
case http.StatusAccepted:
|
||||
return nil, false, nil
|
||||
default:
|
||||
body, _ := io.ReadAll(io.LimitReader(res.Body, 4096))
|
||||
err := fmt.Errorf("onedrive upload session: status=%d body=%s", res.StatusCode, strings.TrimSpace(string(body)))
|
||||
retryable := res.StatusCode == http.StatusTooManyRequests || (res.StatusCode >= 500 && res.StatusCode <= 504)
|
||||
return nil, retryable, err
|
||||
}
|
||||
}
|
||||
|
||||
func readSmallUpload(r io.Reader, declaredSize, limit int64) ([]byte, string, int64, error) {
|
||||
if r == nil {
|
||||
return nil, "", 0, errors.New("onedrive upload: body is required")
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = maxSmallUploadSize
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(r, limit+1))
|
||||
if err != nil {
|
||||
return nil, "", 0, fmt.Errorf("onedrive upload: read body: %w", err)
|
||||
}
|
||||
if int64(len(data)) > limit {
|
||||
return nil, "", 0, fmt.Errorf("onedrive upload: files over %d bytes require upload session", limit)
|
||||
}
|
||||
if declaredSize >= 0 && int64(len(data)) != declaredSize {
|
||||
return nil, "", 0, fmt.Errorf("onedrive upload: size mismatch: declared %d, copied %d", declaredSize, len(data))
|
||||
}
|
||||
sum := sha1.Sum(data)
|
||||
return data, hex.EncodeToString(sum[:]), int64(len(data)), nil
|
||||
}
|
||||
|
||||
func minInt64(a, b int64) int64 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
@@ -245,6 +482,25 @@ func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, er
|
||||
return item.ID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("onedrive rename: empty file id")
|
||||
}
|
||||
newName = strings.TrimSpace(newName)
|
||||
if newName == "" {
|
||||
return errors.New("onedrive rename: empty new name")
|
||||
}
|
||||
var item graphItem
|
||||
err := d.request(ctx, d.itemURL(fileID), http.MethodPatch, func(req *resty.Request) {
|
||||
req.SetBody(map[string]string{"name": newName})
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return fmt.Errorf("onedrive rename: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) request(ctx context.Context, rawURL, method string, configure func(*resty.Request), out any) error {
|
||||
return d.requestOnce(ctx, rawURL, method, configure, out, true)
|
||||
}
|
||||
@@ -265,7 +521,7 @@ func (d *Driver) requestOnce(ctx context.Context, rawURL, method string, configu
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isRateLimitResponse(res, graphErr.Error.Code) {
|
||||
if isRateLimitResponse(res, graphErr.Error.Code, graphErr.Error.Message) {
|
||||
return onedriveRateLimitError(res, graphErr.Error.Message)
|
||||
}
|
||||
if graphErr.Error.Code != "" {
|
||||
@@ -327,11 +583,54 @@ func (d *Driver) refresh(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func isRateLimitResponse(res *resty.Response, code string) bool {
|
||||
if code == "TooManyRequests" || code == "activityLimitReached" {
|
||||
func isRateLimitResponse(res *resty.Response, code, message string) bool {
|
||||
if isRateLimitCode(code) || isRateLimitMessage(message) {
|
||||
return true
|
||||
}
|
||||
return res != nil && res.StatusCode() == http.StatusTooManyRequests
|
||||
if res == nil {
|
||||
return false
|
||||
}
|
||||
if res.StatusCode() == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if res.Header().Get("Retry-After") == "" {
|
||||
return false
|
||||
}
|
||||
switch res.StatusCode() {
|
||||
case http.StatusServiceUnavailable, http.StatusGatewayTimeout:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isRateLimitCode(code string) bool {
|
||||
normalized := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(code), "_", ""))
|
||||
normalized = strings.ReplaceAll(normalized, "-", "")
|
||||
switch normalized {
|
||||
case "toomanyrequests",
|
||||
"activitylimitreached",
|
||||
"throttledrequest",
|
||||
"requestthrottled",
|
||||
"resourcethrottled",
|
||||
"applicationthrottled",
|
||||
"tenantthrottled":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isRateLimitMessage(message string) bool {
|
||||
text := strings.ToLower(strings.TrimSpace(message))
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "throttl") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "activity limit") ||
|
||||
strings.Contains(text, "temporarily blocked")
|
||||
}
|
||||
|
||||
func onedriveRateLimitError(res *resty.Response, message string) error {
|
||||
|
||||
@@ -2,6 +2,8 @@ package onedrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
@@ -199,7 +201,7 @@ func TestGraph429ReturnsRateLimitErrorWithRetryAfter(t *testing.T) {
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
|
||||
_, err := d.List(context.Background(), "root")
|
||||
_, err := d.StreamURL(context.Background(), "file-id")
|
||||
if err == nil {
|
||||
t.Fatal("list succeeded, want rate limit error")
|
||||
}
|
||||
@@ -212,6 +214,92 @@ func TestGraph429ReturnsRateLimitErrorWithRetryAfter(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphThrottleMessageReturnsRateLimitError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{
|
||||
"error": map[string]any{
|
||||
"code": "generalException",
|
||||
"message": "The request has been throttled. Please try again later.",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("write json: %v", err)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
|
||||
_, err := d.StreamURL(context.Background(), "file-id")
|
||||
if err == nil {
|
||||
t.Fatal("list succeeded, want rate limit error")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListCoolsDownAndRetriesOneDriveRateLimit(t *testing.T) {
|
||||
var calls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1.0/me/drive/items/root/children" {
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
|
||||
}
|
||||
calls++
|
||||
if calls == 1 {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{
|
||||
"error": map[string]any{
|
||||
"code": "TooManyRequests",
|
||||
"message": "throttled",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("write json: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
writeJSON(t, w, map[string]any{
|
||||
"value": []map[string]any{
|
||||
{
|
||||
"id": "file-id",
|
||||
"name": "demo.mp4",
|
||||
"size": 100,
|
||||
"file": map[string]any{"mimeType": "video/mp4"},
|
||||
},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
d.listInterval = 0
|
||||
d.listCooldown = time.Millisecond
|
||||
|
||||
got, err := d.List(context.Background(), "root")
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("calls = %d, want retry after rate limit", calls)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID != "file-id" {
|
||||
t.Fatalf("entries = %#v, want retried file", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatAndStreamURLUseDriveItemMetadata(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
@@ -320,6 +408,36 @@ func TestEnsureDirCreatesMissingFolders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenamePatchesDriveItemName(t *testing.T) {
|
||||
var body map[string]string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPatch || r.URL.EscapedPath() != "/v1.0/me/drive/items/file-id" {
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
|
||||
}
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
t.Fatalf("authorization = %q, want bearer token", got)
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode body: %v", err)
|
||||
}
|
||||
writeJSON(t, w, map[string]any{"id": "file-id", "name": "new name.mp4"})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
if err := d.Rename(context.Background(), "file-id", "new name.mp4"); err != nil {
|
||||
t.Fatalf("rename: %v", err)
|
||||
}
|
||||
if body["name"] != "new name.mp4" {
|
||||
t.Fatalf("rename body = %#v, want new name", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadSmallFileReturnsNewItemID(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
@@ -358,6 +476,86 @@ func TestUploadSmallFileReturnsNewItemID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadLargeFileUsesUploadSessionAndReportsHash(t *testing.T) {
|
||||
oldThreshold := smallUploadThreshold
|
||||
oldChunk := uploadSessionChunk
|
||||
smallUploadThreshold = 8
|
||||
uploadSessionChunk = 4
|
||||
t.Cleanup(func() {
|
||||
smallUploadThreshold = oldThreshold
|
||||
uploadSessionChunk = oldChunk
|
||||
})
|
||||
|
||||
body := "0123456789abc"
|
||||
var ranges []string
|
||||
var chunks []string
|
||||
var createdSession bool
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost && r.URL.EscapedPath() == "/v1.0/me/drive/items/parent-id:/big.mp4:/createUploadSession":
|
||||
createdSession = true
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
t.Fatalf("authorization = %q, want bearer token", got)
|
||||
}
|
||||
writeJSON(t, w, map[string]any{"uploadUrl": srv.URL + "/upload-session"})
|
||||
case r.Method == http.MethodPut && r.URL.Path == "/upload-session":
|
||||
ranges = append(ranges, r.Header.Get("Content-Range"))
|
||||
data, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read chunk: %v", err)
|
||||
}
|
||||
chunks = append(chunks, string(data))
|
||||
if len(ranges) < 4 {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
if _, err := w.Write([]byte(`{"nextExpectedRanges":["0-"]}`)); err != nil {
|
||||
t.Fatalf("write accepted: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{"id": "uploaded-big-id"}); err != nil {
|
||||
t.Fatalf("write final item: %v", err)
|
||||
}
|
||||
default:
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
got, err := d.UploadAndReportHash(context.Background(), "parent-id", "big.mp4", strings.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
if !createdSession {
|
||||
t.Fatal("createUploadSession was not called")
|
||||
}
|
||||
wantRanges := []string{
|
||||
"bytes 0-3/13",
|
||||
"bytes 4-7/13",
|
||||
"bytes 8-11/13",
|
||||
"bytes 12-12/13",
|
||||
}
|
||||
if strings.Join(ranges, "|") != strings.Join(wantRanges, "|") {
|
||||
t.Fatalf("ranges = %#v, want %#v", ranges, wantRanges)
|
||||
}
|
||||
if strings.Join(chunks, "") != body {
|
||||
t.Fatalf("uploaded chunks = %q, want %q", strings.Join(chunks, ""), body)
|
||||
}
|
||||
sum := sha1.Sum([]byte(body))
|
||||
if got.FileID != "uploaded-big-id" || got.Size != int64(len(body)) || got.Hash != hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("upload result = %#v, want file id/hash/size for body", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadRefreshesExpiredTokenAndReplaysBody(t *testing.T) {
|
||||
var uploadAttempts int
|
||||
var tokenRefreshes int
|
||||
|
||||
@@ -82,3 +82,13 @@ type filesResp struct {
|
||||
Value []graphItem `json:"value"`
|
||||
NextLink string `json:"@odata.nextLink"`
|
||||
}
|
||||
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
Hash string
|
||||
Size int64
|
||||
}
|
||||
|
||||
type uploadSessionResp struct {
|
||||
UploadURL string `json:"uploadUrl"`
|
||||
}
|
||||
|
||||
@@ -149,6 +149,10 @@ func sleepContext(ctx context.Context, d time.Duration) error {
|
||||
}
|
||||
|
||||
func isTransient115ListError(err error) bool {
|
||||
return isTransient115UpstreamError(err)
|
||||
}
|
||||
|
||||
func isTransient115UpstreamError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
@@ -248,11 +252,11 @@ func (d *Driver) streamURLWithUA(ctx context.Context, fileID string, ua string)
|
||||
// 需要先拿到 pickCode
|
||||
f, err := d.client.GetFile(fileID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("115 get file: %w", err)
|
||||
return nil, wrap115StreamTransientError("115 get file", err)
|
||||
}
|
||||
info, ua, err := d.downloadInfo(f.PickCode, ua)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("115 download url: %w", err)
|
||||
return nil, wrap115StreamTransientError("115 download url", err)
|
||||
}
|
||||
if info == nil || info.Url.Url == "" {
|
||||
return nil, errors.New("115 download url: empty")
|
||||
@@ -288,6 +292,18 @@ func (d *Driver) downloadInfo(pickCode string, ua string) (*sdk.DownloadInfo, st
|
||||
return info, ua, nil
|
||||
}
|
||||
|
||||
func wrap115StreamTransientError(op string, err error) error {
|
||||
wrapped := fmt.Errorf("%s: %w", op, err)
|
||||
if !isTransient115UpstreamError(err) {
|
||||
return wrapped
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: "p115",
|
||||
RetryAfter: p115ListCooldown,
|
||||
Err: wrapped,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
res, err := d.UploadAndReportSha1(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
|
||||
@@ -10,6 +10,9 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestIsTransient115ListError(t *testing.T) {
|
||||
@@ -34,6 +37,42 @@ func TestIsTransient115ListError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrap115StreamTransientError(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
wantRateLimit bool
|
||||
}{
|
||||
{name: "unexpected", err: errors.New("unexpected error"), wantRateLimit: true},
|
||||
{name: "405 blocked", err: errors.New("405 request has been blocked"), wantRateLimit: true},
|
||||
{name: "429", err: errors.New("429 too many requests"), wantRateLimit: true},
|
||||
{name: "blocked", err: errors.New("blocked by waf"), wantRateLimit: true},
|
||||
{name: "auth", err: errors.New("invalid credential"), wantRateLimit: false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := wrap115StreamTransientError("115 get file", tc.err)
|
||||
var rateLimit *drives.RateLimitError
|
||||
isRateLimit := errors.As(got, &rateLimit)
|
||||
if isRateLimit != tc.wantRateLimit {
|
||||
t.Fatalf("rate limit = %v, want %v; err=%v", isRateLimit, tc.wantRateLimit, got)
|
||||
}
|
||||
if !strings.Contains(got.Error(), "115 get file") {
|
||||
t.Fatalf("err = %v, want operation prefix", got)
|
||||
}
|
||||
if tc.wantRateLimit {
|
||||
if rateLimit.Provider != "p115" {
|
||||
t.Fatalf("provider = %q, want p115", rateLimit.Provider)
|
||||
}
|
||||
if rateLimit.RetryAfter != 10*time.Minute {
|
||||
t.Fatalf("retry after = %s, want 10m", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBufferAndHashSha1 验证 bufferAndHashSha1:
|
||||
//
|
||||
// - 把 reader 的全部字节落到 tmp 文件
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,487 @@
|
||||
package p123
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestStreamURLResolvesDownloadInfoRedirect(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var downloadReferer string
|
||||
var download *httptest.Server
|
||||
download = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/resolve":
|
||||
downloadReferer = r.Header.Get("Referer")
|
||||
http.Redirect(w, r, download.URL+"/cdn/video.mp4", http.StatusFound)
|
||||
case "/cdn/video.mp4":
|
||||
t.Fatalf("driver followed redirect unexpectedly")
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer download.Close()
|
||||
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/api/user/sign_in":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 200,
|
||||
"data": map[string]string{"token": "token-1"},
|
||||
})
|
||||
case "/b/api/user/info":
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer token-1" {
|
||||
t.Fatalf("Authorization = %q, want bearer token", got)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{}})
|
||||
case "/b/api/file/list/new":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"Next": "-1",
|
||||
"Total": 1,
|
||||
"InfoList": []map[string]any{
|
||||
{
|
||||
"FileName": "video.mp4",
|
||||
"Size": 1234,
|
||||
"UpdateAt": "2026-01-02 03:04:05",
|
||||
"FileId": 100,
|
||||
"Type": 0,
|
||||
"Etag": "ABCDEF",
|
||||
"S3KeyFlag": "flag-1",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
case "/b/api/file/download_info":
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode download_info body: %v", err)
|
||||
}
|
||||
if got := body["fileName"]; got != "video.mp4" {
|
||||
t.Fatalf("fileName = %#v, want cached file metadata", got)
|
||||
}
|
||||
if got := body["etag"]; got != "ABCDEF" {
|
||||
t.Fatalf("etag = %#v, want cached etag", got)
|
||||
}
|
||||
entryURL := download.URL + "/entry?params=" + base64.StdEncoding.EncodeToString([]byte(download.URL+"/resolve"))
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]string{"DownloadUrl": entryURL},
|
||||
})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
var savedToken string
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
Username: "user@example.com",
|
||||
Password: "secret",
|
||||
MainAPIBaseURL: api.URL + "/b/api",
|
||||
LoginAPIBaseURL: api.URL + "/api",
|
||||
OnTokenUpdate: func(access string) {
|
||||
savedToken = access
|
||||
},
|
||||
})
|
||||
if err := d.Init(ctx); err != nil {
|
||||
t.Fatalf("Init() error = %v", err)
|
||||
}
|
||||
if savedToken != "token-1" {
|
||||
t.Fatalf("saved token = %q, want token-1", savedToken)
|
||||
}
|
||||
if _, err := d.List(ctx, d.RootID()); err != nil {
|
||||
t.Fatalf("List() error = %v", err)
|
||||
}
|
||||
|
||||
link, err := d.StreamURL(ctx, "100")
|
||||
if err != nil {
|
||||
t.Fatalf("StreamURL() error = %v", err)
|
||||
}
|
||||
if got := link.URL; got != download.URL+"/cdn/video.mp4" {
|
||||
t.Fatalf("URL = %q, want final CDN URL", got)
|
||||
}
|
||||
if got := link.Headers.Get("Referer"); !strings.HasPrefix(got, download.URL) {
|
||||
t.Fatalf("Referer = %q, want original download host", got)
|
||||
}
|
||||
if downloadReferer != defaultReferer {
|
||||
t.Fatalf("resolve Referer = %q, want %q", downloadReferer, defaultReferer)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitUsesAccessTokenWithoutLogin(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/api/user/sign_in":
|
||||
t.Fatalf("driver should not password-login when access_token is configured")
|
||||
case "/b/api/user/info":
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer token-1" {
|
||||
t.Fatalf("Authorization = %q, want bearer token", got)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{}})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "Bearer token-1",
|
||||
MainAPIBaseURL: api.URL + "/b/api",
|
||||
LoginAPIBaseURL: api.URL + "/api",
|
||||
})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("Init() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginRiskErrorSuggestsAccessToken(t *testing.T) {
|
||||
err := loginError("当前账号存在境外登录风险,请使用短信验证码或者微信进行登录。")
|
||||
if err == nil || !strings.Contains(err.Error(), "access_token") {
|
||||
t.Fatalf("loginError() = %v, want access_token guidance", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestCode429ReturnsRateLimitError(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Retry-After", "2")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 429,
|
||||
"message": "请求太频繁",
|
||||
})
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "token-1",
|
||||
MainAPIBaseURL: api.URL,
|
||||
})
|
||||
_, err := d.request(context.Background(), endpointFileList, http.MethodGet, nil, nil)
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != 2*time.Second {
|
||||
t.Fatalf("RetryAfter = %s, want 2s", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListCoolsDownAndRetriesRateLimit(t *testing.T) {
|
||||
var listCalls int
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/file/list/new" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
listCalls++
|
||||
if listCalls == 1 {
|
||||
w.Header().Set("Retry-After", "1")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 429,
|
||||
"message": "请求太频繁",
|
||||
})
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"Next": "-1",
|
||||
"Total": 1,
|
||||
"InfoList": []map[string]any{
|
||||
{
|
||||
"FileName": "video.mp4",
|
||||
"Size": 1234,
|
||||
"UpdateAt": "2026-01-02 03:04:05",
|
||||
"FileId": 100,
|
||||
"Type": 0,
|
||||
"Etag": "ABCDEF",
|
||||
"S3KeyFlag": "flag-1",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "token-1",
|
||||
MainAPIBaseURL: api.URL,
|
||||
})
|
||||
entries, err := d.List(context.Background(), d.RootID())
|
||||
if err != nil {
|
||||
t.Fatalf("List() error = %v", err)
|
||||
}
|
||||
if listCalls != 2 {
|
||||
t.Fatalf("list calls = %d, want 2", listCalls)
|
||||
}
|
||||
if len(entries) != 1 || entries[0].ID != "100" {
|
||||
t.Fatalf("entries = %#v, want one file", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDownloadURL429ReturnsRateLimitError(t *testing.T) {
|
||||
download := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Retry-After", "3")
|
||||
http.Error(w, "too many requests", http.StatusTooManyRequests)
|
||||
}))
|
||||
defer download.Close()
|
||||
|
||||
d := New(Config{ID: "123-main"})
|
||||
_, err := d.resolveDownloadURL(context.Background(), download.URL)
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != 3*time.Second {
|
||||
t.Fatalf("RetryAfter = %s, want 3s", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAndReportHashUsesPresignedPUTAndComplete(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
body := []byte("video bytes for 123 upload")
|
||||
wantMD5 := fmt.Sprintf("%x", md5.Sum(body))
|
||||
|
||||
var putBody []byte
|
||||
upload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPut {
|
||||
t.Fatalf("upload method = %s, want PUT", r.Method)
|
||||
}
|
||||
if r.ContentLength != int64(len(body)) {
|
||||
t.Fatalf("ContentLength = %d, want %d", r.ContentLength, len(body))
|
||||
}
|
||||
got, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read upload body: %v", err)
|
||||
}
|
||||
putBody = got
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer upload.Close()
|
||||
|
||||
var uploadRequest map[string]any
|
||||
var uploadURLRequest map[string]any
|
||||
var completeRequest map[string]any
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/file/upload_request":
|
||||
if err := json.NewDecoder(r.Body).Decode(&uploadRequest); err != nil {
|
||||
t.Fatalf("decode upload_request: %v", err)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"FileId": 9001,
|
||||
"Bucket": "bucket-1",
|
||||
"Key": "key-1",
|
||||
"StorageNode": "node-1",
|
||||
"UploadId": "upload-1",
|
||||
},
|
||||
})
|
||||
case "/file/s3_upload_object/auth":
|
||||
if err := json.NewDecoder(r.Body).Decode(&uploadURLRequest); err != nil {
|
||||
t.Fatalf("decode s3 auth: %v", err)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"presignedUrls": map[string]string{
|
||||
"1": upload.URL + "/part-1",
|
||||
},
|
||||
},
|
||||
})
|
||||
case "/file/upload_complete/v2":
|
||||
if err := json.NewDecoder(r.Body).Decode(&completeRequest); err != nil {
|
||||
t.Fatalf("decode complete: %v", err)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{}})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "token-1",
|
||||
MainAPIBaseURL: api.URL,
|
||||
})
|
||||
res, err := d.UploadAndReportHash(ctx, "parent-1", "video.mp4", bytes.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("UploadAndReportHash() error = %v", err)
|
||||
}
|
||||
if res.FileID != "9001" {
|
||||
t.Fatalf("FileID = %q, want 9001", res.FileID)
|
||||
}
|
||||
if res.Hash != wantMD5 {
|
||||
t.Fatalf("Hash = %q, want %q", res.Hash, wantMD5)
|
||||
}
|
||||
if res.Size != int64(len(body)) {
|
||||
t.Fatalf("Size = %d, want %d", res.Size, len(body))
|
||||
}
|
||||
if !bytes.Equal(putBody, body) {
|
||||
t.Fatalf("PUT body = %q, want %q", putBody, body)
|
||||
}
|
||||
if uploadRequest["etag"] != wantMD5 {
|
||||
t.Fatalf("upload etag = %#v, want %q", uploadRequest["etag"], wantMD5)
|
||||
}
|
||||
if uploadRequest["fileName"] != "video.mp4" || uploadRequest["parentFileId"] != "parent-1" {
|
||||
t.Fatalf("upload request = %#v, want fileName and parentFileId", uploadRequest)
|
||||
}
|
||||
if uploadURLRequest["partNumberStart"].(float64) != 1 || uploadURLRequest["partNumberEnd"].(float64) != 2 {
|
||||
t.Fatalf("s3 auth request = %#v, want part range 1..2", uploadURLRequest)
|
||||
}
|
||||
if completeRequest["fileId"].(float64) != 9001 || completeRequest["fileSize"].(float64) != float64(len(body)) {
|
||||
t.Fatalf("complete request = %#v, want file id and size", completeRequest)
|
||||
}
|
||||
if completeRequest["isMultipart"].(bool) {
|
||||
t.Fatalf("complete isMultipart = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAndReportHashReuseSkipsPUTAndComplete(t *testing.T) {
|
||||
body := []byte("reused body")
|
||||
var presignedCalled bool
|
||||
var completeCalled bool
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/file/upload_request":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"FileId": 7001,
|
||||
"Reuse": true,
|
||||
},
|
||||
})
|
||||
case "/file/s3_upload_object/auth", "/file/s3_repare_upload_parts_batch":
|
||||
presignedCalled = true
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0})
|
||||
case "/file/upload_complete/v2":
|
||||
completeCalled = true
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "token-1",
|
||||
MainAPIBaseURL: api.URL,
|
||||
})
|
||||
res, err := d.UploadAndReportHash(context.Background(), "parent-1", "reused.mp4", bytes.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("UploadAndReportHash() error = %v", err)
|
||||
}
|
||||
if res.FileID != "7001" {
|
||||
t.Fatalf("FileID = %q, want 7001", res.FileID)
|
||||
}
|
||||
if presignedCalled {
|
||||
t.Fatal("reuse upload should not request presigned URLs")
|
||||
}
|
||||
if completeCalled {
|
||||
t.Fatal("reuse upload should not call upload_complete")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadPresignedPUT429ReturnsRateLimitError(t *testing.T) {
|
||||
upload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Retry-After", "4")
|
||||
http.Error(w, "too many requests", http.StatusTooManyRequests)
|
||||
}))
|
||||
defer upload.Close()
|
||||
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/file/upload_request":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"FileId": 9001,
|
||||
"Bucket": "bucket-1",
|
||||
"Key": "key-1",
|
||||
"StorageNode": "node-1",
|
||||
"UploadId": "upload-1",
|
||||
},
|
||||
})
|
||||
case "/file/s3_upload_object/auth":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"presignedUrls": map[string]string{"1": upload.URL},
|
||||
},
|
||||
})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "token-1",
|
||||
MainAPIBaseURL: api.URL,
|
||||
})
|
||||
_, err := d.UploadAndReportHash(context.Background(), "parent-1", "limited.mp4", strings.NewReader("limited"), int64(len("limited")))
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != 4*time.Second {
|
||||
t.Fatalf("RetryAfter = %s, want 4s", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameSendsExpectedBody(t *testing.T) {
|
||||
var renameRequest map[string]any
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/file/rename" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&renameRequest); err != nil {
|
||||
t.Fatalf("decode rename: %v", err)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{}})
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "token-1",
|
||||
MainAPIBaseURL: api.URL,
|
||||
})
|
||||
if err := d.Rename(context.Background(), "9001", "new name.mp4"); err != nil {
|
||||
t.Fatalf("Rename() error = %v", err)
|
||||
}
|
||||
if renameRequest["driveId"].(float64) != 0 || renameRequest["fileId"] != "9001" || renameRequest["fileName"] != "new name.mp4" {
|
||||
t.Fatalf("rename request = %#v, want driveId/fileId/fileName", renameRequest)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
package p123
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-resty/resty/v2"
|
||||
"github.com/skip2/go-qrcode"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultUserAPIBase = "https://user.123pan.cn/api"
|
||||
defaultQRLoginPage = "https://www.123pan.com/wx-app-login.html"
|
||||
defaultQRReferer = "https://user.123pan.com/centerlogin"
|
||||
defaultQROrigin = "https://user.123pan.com"
|
||||
defaultQRUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36"
|
||||
|
||||
endpointQRCodeGenerate = "/user/qr-code/generate"
|
||||
endpointQRCodeResult = "/user/qr-code/result"
|
||||
endpointQRCodeWXCode = "/user/qr-code/wx_code"
|
||||
)
|
||||
|
||||
type QRConfig struct {
|
||||
UserAPIBaseURL string
|
||||
HTTPClient *http.Client
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type QRClient struct {
|
||||
userAPIBase string
|
||||
client *resty.Client
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type QRCodeSession struct {
|
||||
LoginUUID string `json:"loginUuid"`
|
||||
UniID string `json:"uniID"`
|
||||
QRCodeURL string `json:"qrCodeUrl"`
|
||||
QRImageDataURL string `json:"qrImageDataUrl"`
|
||||
ExpiresAt string `json:"expiresAt,omitempty"`
|
||||
}
|
||||
|
||||
type QRCodeStatus struct {
|
||||
LoginStatus int `json:"loginStatus"`
|
||||
StatusText string `json:"statusText"`
|
||||
ScanPlatform int `json:"scanPlatform,omitempty"`
|
||||
PlatformText string `json:"platformText,omitempty"`
|
||||
AccessToken string `json:"accessToken,omitempty"`
|
||||
}
|
||||
|
||||
func NewQRClient(c QRConfig) *QRClient {
|
||||
userAPIBase := strings.TrimRight(strings.TrimSpace(c.UserAPIBaseURL), "/")
|
||||
if userAPIBase == "" {
|
||||
userAPIBase = defaultUserAPIBase
|
||||
}
|
||||
httpClient := c.HTTPClient
|
||||
if httpClient == nil {
|
||||
httpClient = &http.Client{Timeout: 20 * time.Second}
|
||||
}
|
||||
now := c.Now
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
return &QRClient{
|
||||
userAPIBase: userAPIBase,
|
||||
client: resty.NewWithClient(httpClient).
|
||||
SetTimeout(20*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
now: now,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *QRClient) Generate(ctx context.Context) (QRCodeSession, error) {
|
||||
loginUUID, err := newLoginUUID()
|
||||
if err != nil {
|
||||
return QRCodeSession{}, err
|
||||
}
|
||||
var resp qrGenerateResp
|
||||
res, err := c.request(ctx, loginUUID).
|
||||
SetResult(&resp).
|
||||
Get(c.userAPIBase + endpointQRCodeGenerate)
|
||||
if err != nil {
|
||||
return QRCodeSession{}, err
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return QRCodeSession{}, qrAPIError(resp.Message, res.StatusCode(), resp.Code)
|
||||
}
|
||||
uniID := strings.TrimSpace(resp.Data.UniID)
|
||||
if uniID == "" {
|
||||
return QRCodeSession{}, errors.New("123pan qr: empty uniID")
|
||||
}
|
||||
qrURL := buildQRLoginURL(resp.Data.URL, uniID)
|
||||
png, err := qrcode.Encode(qrURL, qrcode.Medium, 220)
|
||||
if err != nil {
|
||||
return QRCodeSession{}, err
|
||||
}
|
||||
return QRCodeSession{
|
||||
LoginUUID: loginUUID,
|
||||
UniID: uniID,
|
||||
QRCodeURL: qrURL,
|
||||
QRImageDataURL: "data:image/png;base64," + base64.StdEncoding.EncodeToString(png),
|
||||
ExpiresAt: c.now().Add(5 * time.Minute).Format(time.RFC3339),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *QRClient) Poll(ctx context.Context, loginUUID, uniID string) (QRCodeStatus, error) {
|
||||
loginUUID = strings.TrimSpace(loginUUID)
|
||||
uniID = strings.TrimSpace(uniID)
|
||||
if loginUUID == "" {
|
||||
return QRCodeStatus{}, errors.New("loginUuid is required")
|
||||
}
|
||||
if uniID == "" {
|
||||
return QRCodeStatus{}, errors.New("uniID is required")
|
||||
}
|
||||
var resp qrResultResp
|
||||
res, err := c.request(ctx, loginUUID).
|
||||
SetQueryParam("uniID", uniID).
|
||||
SetResult(&resp).
|
||||
Get(c.userAPIBase + endpointQRCodeResult)
|
||||
if err != nil {
|
||||
return QRCodeStatus{}, err
|
||||
}
|
||||
if resp.Code != 0 && resp.Code != 200 {
|
||||
return QRCodeStatus{}, qrAPIError(resp.Message, res.StatusCode(), resp.Code)
|
||||
}
|
||||
if resp.Code == 200 {
|
||||
resp.Data.LoginStatus = 3
|
||||
if resp.Data.ScanPlatform == 0 {
|
||||
resp.Data.ScanPlatform = resp.Data.LoginType
|
||||
}
|
||||
}
|
||||
status := QRCodeStatus{
|
||||
LoginStatus: resp.Data.LoginStatus,
|
||||
StatusText: qrLoginStatusText(resp.Data.LoginStatus),
|
||||
ScanPlatform: resp.Data.ScanPlatform,
|
||||
PlatformText: qrScanPlatformText(resp.Data.ScanPlatform),
|
||||
}
|
||||
if status.LoginStatus != 3 {
|
||||
return status, nil
|
||||
}
|
||||
if token := resp.TokenValue(); token != "" {
|
||||
status.AccessToken = normalizeAccessToken(token)
|
||||
return status, nil
|
||||
}
|
||||
if resp.Data.ScanPlatform == 4 {
|
||||
token, err := c.finishWechatLogin(ctx, loginUUID, uniID)
|
||||
if err != nil {
|
||||
return QRCodeStatus{}, err
|
||||
}
|
||||
status.AccessToken = normalizeAccessToken(token)
|
||||
return status, nil
|
||||
}
|
||||
return QRCodeStatus{}, errors.New("123pan qr: confirmed login returned empty token")
|
||||
}
|
||||
|
||||
func (c *QRClient) finishWechatLogin(ctx context.Context, loginUUID, uniID string) (string, error) {
|
||||
var wxResp qrWXCodeResp
|
||||
res, err := c.request(ctx, loginUUID).
|
||||
SetBody(map[string]string{"uniID": uniID}).
|
||||
SetResult(&wxResp).
|
||||
Post(c.userAPIBase + endpointQRCodeWXCode)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if wxResp.Code != 0 {
|
||||
return "", qrAPIError(wxResp.Message, res.StatusCode(), wxResp.Code)
|
||||
}
|
||||
wxCode := strings.TrimSpace(wxResp.WXCode())
|
||||
if wxCode == "" {
|
||||
return "", errors.New("123pan qr: empty wechat code")
|
||||
}
|
||||
var signIn loginResp
|
||||
res, err = c.request(ctx, loginUUID).
|
||||
SetBody(map[string]any{
|
||||
"from": "web",
|
||||
"wechat_code": wxCode,
|
||||
"type": 4,
|
||||
}).
|
||||
SetResult(&signIn).
|
||||
Post(c.userAPIBase + endpointSignIn)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if signIn.Code != 200 && signIn.Code != 0 {
|
||||
return "", qrAPIError(signIn.Message, res.StatusCode(), signIn.Code)
|
||||
}
|
||||
token := strings.TrimSpace(signIn.Data.Token)
|
||||
if token == "" {
|
||||
return "", errors.New("123pan qr: empty token")
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func (c *QRClient) request(ctx context.Context, loginUUID string) *resty.Request {
|
||||
return c.client.R().
|
||||
SetContext(ctx).
|
||||
SetHeaders(map[string]string{
|
||||
"Content-Type": "application/json;charset=UTF-8",
|
||||
"platform": defaultPlatform,
|
||||
"App-Version": defaultAppVersion,
|
||||
"LoginUuid": loginUUID,
|
||||
"Referer": defaultQRReferer,
|
||||
"Origin": defaultQROrigin,
|
||||
"User-Agent": defaultQRUserAgent,
|
||||
})
|
||||
}
|
||||
|
||||
func buildQRLoginURL(raw, uniID string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
raw = defaultQRLoginPage
|
||||
}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return defaultQRLoginPage + "?env=production&uniID=" + url.QueryEscape(uniID) + "&source=123pan&type=login"
|
||||
}
|
||||
q := u.Query()
|
||||
q.Set("env", "production")
|
||||
q.Set("uniID", uniID)
|
||||
q.Set("source", "123pan")
|
||||
q.Set("type", "login")
|
||||
u.RawQuery = q.Encode()
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func newLoginUUID() (string, error) {
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
b[6] = (b[6] & 0x0f) | 0x40
|
||||
b[8] = (b[8] & 0x3f) | 0x80
|
||||
parts := []string{
|
||||
hex.EncodeToString(b[0:4]),
|
||||
hex.EncodeToString(b[4:6]),
|
||||
hex.EncodeToString(b[6:8]),
|
||||
hex.EncodeToString(b[8:10]),
|
||||
hex.EncodeToString(b[10:16]),
|
||||
}
|
||||
return strings.Join(parts, "-"), nil
|
||||
}
|
||||
|
||||
func qrAPIError(message string, httpStatus, apiCode int) error {
|
||||
message = strings.TrimSpace(message)
|
||||
if message == "" {
|
||||
message = fmt.Sprintf("HTTP %d code=%d", httpStatus, apiCode)
|
||||
}
|
||||
return errors.New(message)
|
||||
}
|
||||
|
||||
func qrLoginStatusText(status int) string {
|
||||
switch status {
|
||||
case 0:
|
||||
return "等待扫码"
|
||||
case 1:
|
||||
return "已扫码,等待确认"
|
||||
case 2:
|
||||
return "已拒绝"
|
||||
case 3:
|
||||
return "已确认"
|
||||
case 4:
|
||||
return "已过期"
|
||||
default:
|
||||
return "未知状态"
|
||||
}
|
||||
}
|
||||
|
||||
func qrScanPlatformText(platform int) string {
|
||||
switch platform {
|
||||
case 4:
|
||||
return "微信"
|
||||
case 7:
|
||||
return "123 云盘 App"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
package p123
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestQRCodeGenerateBuildsImage(t *testing.T) {
|
||||
var seenLoginUUID string
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/api/user/qr-code/generate" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
seenLoginUUID = r.Header.Get("LoginUuid")
|
||||
if seenLoginUUID == "" {
|
||||
t.Fatalf("missing LoginUuid header")
|
||||
}
|
||||
if r.Header.Get("platform") != defaultPlatform {
|
||||
t.Fatalf("platform header = %q, want %q", r.Header.Get("platform"), defaultPlatform)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"message": "ok",
|
||||
"data": map[string]string{
|
||||
"uniID": "uni-1",
|
||||
"url": "https://www.123pan.com/wx-app-login.html",
|
||||
},
|
||||
})
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{UserAPIBaseURL: api.URL + "/api"}).Generate(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
if got.LoginUUID != seenLoginUUID {
|
||||
t.Fatalf("loginUuid = %q, want header %q", got.LoginUUID, seenLoginUUID)
|
||||
}
|
||||
if got.UniID != "uni-1" {
|
||||
t.Fatalf("uniID = %q, want uni-1", got.UniID)
|
||||
}
|
||||
if !strings.Contains(got.QRCodeURL, "uniID=uni-1") || !strings.Contains(got.QRCodeURL, "type=login") {
|
||||
t.Fatalf("qrCodeUrl = %q, want login params", got.QRCodeURL)
|
||||
}
|
||||
if !strings.HasPrefix(got.QRImageDataURL, "data:image/png;base64,") {
|
||||
t.Fatalf("qrImageDataUrl missing png data url prefix")
|
||||
}
|
||||
if got.ExpiresAt == "" {
|
||||
t.Fatalf("expiresAt is empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQRCodePollCompletesWechatLogin(t *testing.T) {
|
||||
var wxCodeRequested bool
|
||||
var signInRequested bool
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.Header.Get("LoginUuid") != "login-1" {
|
||||
t.Fatalf("LoginUuid = %q, want login-1", r.Header.Get("LoginUuid"))
|
||||
}
|
||||
switch r.URL.Path {
|
||||
case "/api/user/qr-code/result":
|
||||
if r.URL.Query().Get("uniID") != "uni-1" {
|
||||
t.Fatalf("uniID = %q, want uni-1", r.URL.Query().Get("uniID"))
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"loginStatus": 3,
|
||||
"scanPlatform": 4,
|
||||
},
|
||||
})
|
||||
case "/api/user/qr-code/wx_code":
|
||||
wxCodeRequested = true
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode wx_code body: %v", err)
|
||||
}
|
||||
if body["uniID"] != "uni-1" {
|
||||
t.Fatalf("wx_code uniID = %q, want uni-1", body["uniID"])
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]string{"wxCode": "wx-code-1"},
|
||||
})
|
||||
case "/api/user/sign_in":
|
||||
signInRequested = true
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode sign_in body: %v", err)
|
||||
}
|
||||
if body["wechat_code"] != "wx-code-1" {
|
||||
t.Fatalf("wechat_code = %#v, want wx-code-1", body["wechat_code"])
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 200,
|
||||
"data": map[string]string{"token": "Bearer token-1"},
|
||||
})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{UserAPIBaseURL: api.URL + "/api"}).Poll(context.Background(), "login-1", "uni-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Poll() error = %v", err)
|
||||
}
|
||||
if !wxCodeRequested || !signInRequested {
|
||||
t.Fatalf("wechat completion calls wx=%v signIn=%v, want both", wxCodeRequested, signInRequested)
|
||||
}
|
||||
if got.LoginStatus != 3 || got.AccessToken != "token-1" || got.PlatformText != "微信" {
|
||||
t.Fatalf("status = %#v, want confirmed wechat token", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQRCodePollUsesAppToken(t *testing.T) {
|
||||
var wxCodeRequested bool
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/api/user/qr-code/result":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"loginStatus": 3,
|
||||
"scanPlatform": 7,
|
||||
"token": "app-token",
|
||||
},
|
||||
})
|
||||
case "/api/user/qr-code/wx_code":
|
||||
wxCodeRequested = true
|
||||
http.Error(w, "unexpected wx_code", http.StatusInternalServerError)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{UserAPIBaseURL: api.URL + "/api"}).Poll(context.Background(), "login-1", "uni-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Poll() error = %v", err)
|
||||
}
|
||||
if wxCodeRequested {
|
||||
t.Fatalf("wx_code should not be called when app token is already returned")
|
||||
}
|
||||
if got.AccessToken != "app-token" || got.PlatformText != "123 云盘 App" {
|
||||
t.Fatalf("status = %#v, want app token", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQRCodePollUsesOfficialAppSuccessCode(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/api/user/qr-code/result" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 200,
|
||||
"data": map[string]any{
|
||||
"login_type": 7,
|
||||
"token": "app-token",
|
||||
},
|
||||
})
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{UserAPIBaseURL: api.URL + "/api"}).Poll(context.Background(), "login-1", "uni-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Poll() error = %v", err)
|
||||
}
|
||||
if got.LoginStatus != 3 || got.ScanPlatform != 7 || got.AccessToken != "app-token" {
|
||||
t.Fatalf("status = %#v, want official app success token", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package p123
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type apiEnvelope struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
type loginResp struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
Token string `json:"token"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type qrGenerateResp struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
UniID string `json:"uniID"`
|
||||
URL string `json:"url"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type qrResultResp struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
LoginStatus int `json:"loginStatus"`
|
||||
ScanPlatform int `json:"scanPlatform"`
|
||||
LoginType int `json:"login_type"`
|
||||
Token string `json:"token"`
|
||||
AccessToken string `json:"accessToken"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func (r qrResultResp) TokenValue() string {
|
||||
if strings.TrimSpace(r.Data.Token) != "" {
|
||||
return r.Data.Token
|
||||
}
|
||||
return r.Data.AccessToken
|
||||
}
|
||||
|
||||
type qrWXCodeResp struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
WXCodeLower string `json:"wxCode"`
|
||||
WXCodeTitle string `json:"WxCode"`
|
||||
Code string `json:"code"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func (r qrWXCodeResp) WXCode() string {
|
||||
if r.Data.WXCodeLower != "" {
|
||||
return r.Data.WXCodeLower
|
||||
}
|
||||
if r.Data.WXCodeTitle != "" {
|
||||
return r.Data.WXCodeTitle
|
||||
}
|
||||
return r.Data.Code
|
||||
}
|
||||
|
||||
type fileListResp struct {
|
||||
Data struct {
|
||||
Next string `json:"Next"`
|
||||
Total int `json:"Total"`
|
||||
InfoList []panFile `json:"InfoList"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type panFile struct {
|
||||
FileName string `json:"FileName"`
|
||||
Size int64 `json:"Size"`
|
||||
UpdateAt flexibleTime `json:"UpdateAt"`
|
||||
FileID int64 `json:"FileId"`
|
||||
Type int `json:"Type"`
|
||||
Etag string `json:"Etag"`
|
||||
S3KeyFlag string `json:"S3KeyFlag"`
|
||||
}
|
||||
|
||||
type cachedFile struct {
|
||||
file panFile
|
||||
parentID string
|
||||
}
|
||||
|
||||
type downloadInfoResp struct {
|
||||
Data struct {
|
||||
DownloadURL string `json:"DownloadUrl"`
|
||||
DownloadURLLower string `json:"downloadUrl"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func (r downloadInfoResp) URL() string {
|
||||
if r.Data.DownloadURL != "" {
|
||||
return r.Data.DownloadURL
|
||||
}
|
||||
return r.Data.DownloadURLLower
|
||||
}
|
||||
|
||||
type redirectResp struct {
|
||||
Data struct {
|
||||
RedirectURL string `json:"redirect_url"`
|
||||
RedirectURLCamel string `json:"redirectUrl"`
|
||||
RedirectURLTitle string `json:"RedirectUrl"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func (r redirectResp) URL() string {
|
||||
if r.Data.RedirectURL != "" {
|
||||
return r.Data.RedirectURL
|
||||
}
|
||||
if r.Data.RedirectURLCamel != "" {
|
||||
return r.Data.RedirectURLCamel
|
||||
}
|
||||
return r.Data.RedirectURLTitle
|
||||
}
|
||||
|
||||
type mkdirResp struct {
|
||||
Data struct {
|
||||
FileID int64 `json:"FileId"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type uploadResp struct {
|
||||
Data struct {
|
||||
AccessKeyID string `json:"AccessKeyId"`
|
||||
Bucket string `json:"Bucket"`
|
||||
Key string `json:"Key"`
|
||||
SecretAccessKey string `json:"SecretAccessKey"`
|
||||
SessionToken string `json:"SessionToken"`
|
||||
FileID int64 `json:"FileId"`
|
||||
Reuse bool `json:"Reuse"`
|
||||
EndPoint string `json:"EndPoint"`
|
||||
StorageNode string `json:"StorageNode"`
|
||||
UploadID string `json:"UploadId"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type s3PreSignedURLsResp struct {
|
||||
Data struct {
|
||||
PreSignedURLs map[string]string `json:"presignedUrls"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type flexibleTime struct {
|
||||
t time.Time
|
||||
}
|
||||
|
||||
func (t *flexibleTime) UnmarshalJSON(data []byte) error {
|
||||
if string(data) == "null" || string(data) == `""` {
|
||||
return nil
|
||||
}
|
||||
var s string
|
||||
if err := json.Unmarshal(data, &s); err == nil {
|
||||
t.t = parseTimeString(s)
|
||||
return nil
|
||||
}
|
||||
var n int64
|
||||
if err := json.Unmarshal(data, &n); err == nil {
|
||||
if n > 1_000_000_000_000 {
|
||||
t.t = time.UnixMilli(n)
|
||||
} else {
|
||||
t.t = time.Unix(n, 0)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t flexibleTime) Time() time.Time {
|
||||
return t.t
|
||||
}
|
||||
|
||||
func parseTimeString(s string) time.Time {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
for _, layout := range []string{
|
||||
time.RFC3339Nano,
|
||||
time.RFC3339,
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02T15:04:05",
|
||||
} {
|
||||
if parsed, err := time.ParseInLocation(layout, s, time.FixedZone("UTC+8", 8*3600)); err == nil {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
if n, err := strconv.ParseInt(s, 10, 64); err == nil {
|
||||
if n > 1_000_000_000_000 {
|
||||
return time.UnixMilli(n)
|
||||
}
|
||||
return time.Unix(n, 0)
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
@@ -199,9 +199,8 @@ func (d *Driver) refreshCaptchaToken(ctx context.Context, action string, meta ma
|
||||
|
||||
// refreshCaptchaTokenOnce 调 /v1/shield/captcha/init 申请新 captcha token。
|
||||
//
|
||||
// 如果 retry=true 且服务端返回 4002(captcha_token expired,意味着 body 里
|
||||
// 携带的 d.captchaToken 已经过期),就清空缓存的 captcha_token 后再调一次;
|
||||
// 这次 body 里 captcha_token 为空,服务端会直接发一个新的。这覆盖
|
||||
// 如果 retry=true 且服务端返回 captcha 失效错误(4002 或 9),就清空缓存的
|
||||
// captcha_token 后再调一次;这次 body 里 captcha_token 为空,服务端会直接发一个新的。这覆盖
|
||||
// driver 重启后 Init() 用持久化的旧 captcha_token 调 captcha init 失败的
|
||||
// 场景。
|
||||
func (d *Driver) refreshCaptchaTokenOnce(ctx context.Context, action string, meta map[string]string, retry bool) error {
|
||||
@@ -230,7 +229,7 @@ func (d *Driver) refreshCaptchaTokenOnce(ctx context.Context, action string, met
|
||||
return err
|
||||
}
|
||||
if e.isError() {
|
||||
if retry && e.ErrorCode == 4002 && d.captchaToken != "" {
|
||||
if retry && isCaptchaTokenRejectedCode(e.ErrorCode) && d.captchaToken != "" {
|
||||
d.captchaToken = ""
|
||||
return d.refreshCaptchaTokenOnce(ctx, action, meta, false)
|
||||
}
|
||||
|
||||
@@ -96,6 +96,65 @@ func TestRefreshCaptchaTokenRecoversFrom4002(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefreshCaptchaTokenRecoversFrom9 覆盖 PikPak 返回 error_code=9
|
||||
// captcha_invalid 的路径。这个错误和 4002 一样表示当前 captcha_token 已被拒绝;
|
||||
// 重试 captcha/init 前必须先清空旧 token,否则服务端会继续拒绝。
|
||||
func TestRefreshCaptchaTokenRecoversFrom9(t *testing.T) {
|
||||
var calls int32
|
||||
type bodyShape struct {
|
||||
CaptchaToken string `json:"captcha_token"`
|
||||
}
|
||||
var (
|
||||
firstBody bodyShape
|
||||
secondBody bodyShape
|
||||
)
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v1/shield/captcha/init", func(w http.ResponseWriter, r *http.Request) {
|
||||
n := atomic.AddInt32(&calls, 1)
|
||||
switch n {
|
||||
case 1:
|
||||
_ = json.NewDecoder(r.Body).Decode(&firstBody)
|
||||
writeErrorJSON(w, `{
|
||||
"error_code": 9,
|
||||
"error": "captcha_invalid",
|
||||
"error_description": "Verification code is invalid"
|
||||
}`)
|
||||
case 2:
|
||||
_ = json.NewDecoder(r.Body).Decode(&secondBody)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"captcha_token": "fresh-captcha",
|
||||
"expires_in": 300
|
||||
}`))
|
||||
default:
|
||||
t.Errorf("unexpected captcha init call #%d", n)
|
||||
}
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
d.captchaToken = "expired-captcha"
|
||||
|
||||
if err := d.refreshCaptchaTokenAtLogin(context.Background(), "GET:/drive/v1/files", "user-1"); err != nil {
|
||||
t.Fatalf("refreshCaptchaTokenAtLogin: %v", err)
|
||||
}
|
||||
|
||||
if got := atomic.LoadInt32(&calls); got != 2 {
|
||||
t.Fatalf("captcha init called %d times, want 2", got)
|
||||
}
|
||||
if firstBody.CaptchaToken != "expired-captcha" {
|
||||
t.Errorf("first body captcha_token = %q, want \"expired-captcha\"", firstBody.CaptchaToken)
|
||||
}
|
||||
if secondBody.CaptchaToken != "" {
|
||||
t.Errorf("second body captcha_token = %q, want empty (cleared after error_code=9)", secondBody.CaptchaToken)
|
||||
}
|
||||
if d.captchaToken != "fresh-captcha" {
|
||||
t.Errorf("d.captchaToken = %q, want \"fresh-captcha\"", d.captchaToken)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefreshCaptchaTokenDoesNotLoopOn4002WithEmptyToken 防止退化成无限重试:
|
||||
// 如果调用方一开始 captchaToken 就是空,又遇上 4002,不应该再清空一次重试
|
||||
// (清空后还是空,再发会拿到同样的错误),应该直接返回错误让上层处理。
|
||||
@@ -121,6 +180,141 @@ func TestRefreshCaptchaTokenDoesNotLoopOn4002WithEmptyToken(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitWithRefreshTokenDoesNotSendPersistedCaptchaToken(t *testing.T) {
|
||||
var captchaCalls int32
|
||||
var captchaBody struct {
|
||||
CaptchaToken string `json:"captcha_token"`
|
||||
}
|
||||
var persisted struct {
|
||||
access, refresh, captcha string
|
||||
calls int
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v1/auth/token", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"access_token": "fresh-access",
|
||||
"refresh_token": "fresh-refresh",
|
||||
"sub": "user-1"
|
||||
}`))
|
||||
})
|
||||
mux.HandleFunc("/v1/shield/captcha/init", func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&captchaCalls, 1)
|
||||
_ = json.NewDecoder(r.Body).Decode(&captchaBody)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"captcha_token": "fresh-captcha",
|
||||
"expires_in": 300
|
||||
}`))
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
d.captchaToken = "persisted-stale-captcha"
|
||||
d.onTokenUpdate = func(access, refresh, captcha, deviceID string) {
|
||||
persisted.access = access
|
||||
persisted.refresh = refresh
|
||||
persisted.captcha = captcha
|
||||
persisted.calls++
|
||||
}
|
||||
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
|
||||
if got := atomic.LoadInt32(&captchaCalls); got != 1 {
|
||||
t.Fatalf("captcha init calls = %d, want 1", got)
|
||||
}
|
||||
if captchaBody.CaptchaToken != "" {
|
||||
t.Errorf("captcha init body captcha_token = %q, want empty", captchaBody.CaptchaToken)
|
||||
}
|
||||
if d.captchaToken != "fresh-captcha" {
|
||||
t.Errorf("d.captchaToken = %q, want \"fresh-captcha\"", d.captchaToken)
|
||||
}
|
||||
if persisted.access != "fresh-access" || persisted.refresh != "fresh-refresh" || persisted.captcha != "fresh-captcha" {
|
||||
t.Errorf("persisted tokens = (%q, %q, %q), want fresh values", persisted.access, persisted.refresh, persisted.captcha)
|
||||
}
|
||||
if persisted.calls < 2 {
|
||||
t.Errorf("persist callback calls = %d, want at least 2 (clear stale + persist fresh)", persisted.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitFallsBackToLoginWhenRefreshReturnsCaptchaInvalid(t *testing.T) {
|
||||
var (
|
||||
tokenCalls int32
|
||||
captchaCalls int32
|
||||
signinCalls int32
|
||||
)
|
||||
var signinBody struct {
|
||||
CaptchaToken string `json:"captcha_token"`
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v1/auth/token", func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&tokenCalls, 1)
|
||||
writeErrorJSON(w, `{
|
||||
"error_code": 4002,
|
||||
"error": "captcha_invalid",
|
||||
"error_description": "Code(4002) - captcha_token expired"
|
||||
}`)
|
||||
})
|
||||
mux.HandleFunc("/v1/shield/captcha/init", func(w http.ResponseWriter, r *http.Request) {
|
||||
n := atomic.AddInt32(&captchaCalls, 1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch n {
|
||||
case 1:
|
||||
_, _ = w.Write([]byte(`{
|
||||
"captcha_token": "login-captcha",
|
||||
"expires_in": 300
|
||||
}`))
|
||||
case 2:
|
||||
_, _ = w.Write([]byte(`{
|
||||
"captcha_token": "files-captcha",
|
||||
"expires_in": 300
|
||||
}`))
|
||||
default:
|
||||
t.Errorf("unexpected captcha init call #%d", n)
|
||||
}
|
||||
})
|
||||
mux.HandleFunc("/v1/auth/signin", func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&signinCalls, 1)
|
||||
_ = json.NewDecoder(r.Body).Decode(&signinBody)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"access_token": "login-access",
|
||||
"refresh_token": "login-refresh",
|
||||
"sub": "user-1"
|
||||
}`))
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
d.captchaToken = "persisted-stale-captcha"
|
||||
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
|
||||
if got := atomic.LoadInt32(&tokenCalls); got != 1 {
|
||||
t.Fatalf("token refresh calls = %d, want 1", got)
|
||||
}
|
||||
if got := atomic.LoadInt32(&signinCalls); got != 1 {
|
||||
t.Fatalf("signin calls = %d, want 1", got)
|
||||
}
|
||||
if got := atomic.LoadInt32(&captchaCalls); got != 2 {
|
||||
t.Fatalf("captcha init calls = %d, want 2 (login + post-login files action)", got)
|
||||
}
|
||||
if signinBody.CaptchaToken != "login-captcha" {
|
||||
t.Errorf("signin captcha_token = %q, want \"login-captcha\"", signinBody.CaptchaToken)
|
||||
}
|
||||
if d.accessToken != "login-access" || d.refreshToken != "login-refresh" || d.captchaToken != "files-captcha" {
|
||||
t.Errorf("driver tokens = (%q, %q, %q), want login/files tokens", d.accessToken, d.refreshToken, d.captchaToken)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRequestOnceRecoversFrom4002OnAPICall 验证一个普通 API 调用收到 4002
|
||||
// 时,requestOnce 会先清空 captchaToken、再走 captcha 刷新,最后用新 token
|
||||
// 重试请求,最终成功返回。
|
||||
@@ -196,6 +390,76 @@ func TestRequestOnceRecoversFrom4002OnAPICall(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRequestOnceRecoversFrom9OnAPICall 验证普通 API 调用收到 error_code=9
|
||||
// 时,会先清空旧 captchaToken,再刷新 captcha 并重试原请求。
|
||||
func TestRequestOnceRecoversFrom9OnAPICall(t *testing.T) {
|
||||
var (
|
||||
filesCalls int32
|
||||
captchaCalls int32
|
||||
)
|
||||
type capturedFiles struct {
|
||||
captchaHeader string
|
||||
}
|
||||
var firstFiles, secondFiles capturedFiles
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
n := atomic.AddInt32(&filesCalls, 1)
|
||||
switch n {
|
||||
case 1:
|
||||
firstFiles.captchaHeader = r.Header.Get("X-Captcha-Token")
|
||||
writeErrorJSON(w, `{
|
||||
"error_code": 9,
|
||||
"error": "captcha_invalid",
|
||||
"error_description": "Verification code is invalid"
|
||||
}`)
|
||||
case 2:
|
||||
secondFiles.captchaHeader = r.Header.Get("X-Captcha-Token")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"files": [], "next_page_token": ""}`))
|
||||
default:
|
||||
t.Errorf("unexpected /drive/v1/files call #%d", n)
|
||||
}
|
||||
})
|
||||
mux.HandleFunc("/v1/shield/captcha/init", func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&captchaCalls, 1)
|
||||
var body struct {
|
||||
CaptchaToken string `json:"captcha_token"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
if body.CaptchaToken != "" {
|
||||
t.Errorf("captcha init body captcha_token = %q, want empty (error_code=9 path should clear cache)", body.CaptchaToken)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"captcha_token": "fresh-captcha", "expires_in": 300}`))
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
d.captchaToken = "expired-captcha"
|
||||
|
||||
if _, err := d.List(context.Background(), "any-parent"); err != nil {
|
||||
t.Fatalf("List: %v", err)
|
||||
}
|
||||
|
||||
if got := atomic.LoadInt32(&filesCalls); got != 2 {
|
||||
t.Fatalf("/drive/v1/files calls = %d, want 2 (initial + retry)", got)
|
||||
}
|
||||
if got := atomic.LoadInt32(&captchaCalls); got != 1 {
|
||||
t.Fatalf("captcha init calls = %d, want 1", got)
|
||||
}
|
||||
if firstFiles.captchaHeader != "expired-captcha" {
|
||||
t.Errorf("first request X-Captcha-Token = %q, want \"expired-captcha\"", firstFiles.captchaHeader)
|
||||
}
|
||||
if secondFiles.captchaHeader != "fresh-captcha" {
|
||||
t.Errorf("retry X-Captcha-Token = %q, want \"fresh-captcha\"", secondFiles.captchaHeader)
|
||||
}
|
||||
if d.captchaToken != "fresh-captcha" {
|
||||
t.Errorf("d.captchaToken after recovery = %q, want \"fresh-captcha\"", d.captchaToken)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRequestOnceDoesNotRetryTwiceOn4002 验证 4002 恢复路径只重试一次;
|
||||
// 如果重试请求依然失败(哪怕是再来一个 4002),也不会再次进入恢复逻辑,
|
||||
// 而是把错误返回出去,避免无限循环。
|
||||
|
||||
@@ -121,9 +121,28 @@ func (d *Driver) ID() string { return d.id }
|
||||
func (d *Driver) RootID() string { return d.rootID }
|
||||
|
||||
func (d *Driver) Init(ctx context.Context) error {
|
||||
clearPersistedCaptcha := func() {
|
||||
if d.captchaToken == "" {
|
||||
return
|
||||
}
|
||||
d.captchaToken = ""
|
||||
d.persistTokens()
|
||||
}
|
||||
|
||||
if d.refreshToken != "" {
|
||||
if err := d.refresh(ctx, d.refreshToken); err != nil {
|
||||
return err
|
||||
if !IsCaptchaError(err) || d.username == "" || d.password == "" {
|
||||
return err
|
||||
}
|
||||
clearPersistedCaptcha()
|
||||
if err := d.login(ctx); err != nil {
|
||||
return fmt.Errorf("pikpak refresh captcha recovery login: %w", err)
|
||||
}
|
||||
} else {
|
||||
// Persisted captcha tokens are short-lived. With a refresh token we can
|
||||
// safely request a fresh captcha token after auth, and avoiding the
|
||||
// stored value prevents known-stale tokens from poisoning startup.
|
||||
clearPersistedCaptcha()
|
||||
}
|
||||
} else {
|
||||
if err := d.login(ctx); err != nil {
|
||||
@@ -336,7 +355,60 @@ func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
currentID := d.rootID
|
||||
for _, name := range splitPath(pathFromRoot) {
|
||||
childID, err := d.findChildDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if childID == "" {
|
||||
childID, err = d.makeDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
currentID = childID
|
||||
}
|
||||
return currentID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) findChildDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
entries, err := d.List(ctx, parentID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir && e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
var out file
|
||||
err := d.request(ctx, filesURL, http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(map[string]any{
|
||||
"kind": "drive#folder",
|
||||
"parent_id": parentID,
|
||||
"name": name,
|
||||
})
|
||||
}, &out)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("pikpak mkdir %s: %w", name, err)
|
||||
}
|
||||
if out.ID == "" {
|
||||
return "", fmt.Errorf("pikpak mkdir %s: empty folder id", name)
|
||||
}
|
||||
return out.ID, nil
|
||||
}
|
||||
|
||||
func splitPath(p string) []string {
|
||||
p = strings.Trim(p, "/")
|
||||
if p == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(p, "/")
|
||||
}
|
||||
|
||||
func (d *Driver) getFiles(ctx context.Context, parentID string) ([]file, error) {
|
||||
@@ -408,14 +480,15 @@ func (d *Driver) requestOnce(ctx context.Context, url, method string, configure
|
||||
// serialized. Once we hold the lock, if d.captchaToken has
|
||||
// already moved past staleToken, another goroutine has refreshed
|
||||
// it for us — we skip the refresh and just retry. Otherwise we
|
||||
// clear the cached token (4002 means "the value in the body is
|
||||
// expired"; sending it again will keep returning 4002) and ask
|
||||
// /v1/shield/captcha/init for a fresh one.
|
||||
// clear the cached token before asking /v1/shield/captcha/init
|
||||
// for a fresh one. PikPak may report stale captcha as either
|
||||
// 4002 or 9, and sending the rejected token into captcha init can
|
||||
// keep returning captcha_invalid.
|
||||
staleToken := d.captchaToken
|
||||
d.captchaMu.Lock()
|
||||
var refreshErr error
|
||||
if d.captchaToken == staleToken {
|
||||
if e.ErrorCode == 4002 {
|
||||
if d.captchaToken != "" {
|
||||
d.captchaToken = ""
|
||||
}
|
||||
refreshErr = d.refreshCaptchaTokenAtLogin(ctx, getAction(method, url), d.userID)
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package pikpak
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestNewDefaults(t *testing.T) {
|
||||
@@ -95,11 +97,85 @@ func TestFolderToEntry(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDirStillUnsupported(t *testing.T) {
|
||||
d := New(Config{ID: "pikpak-main"})
|
||||
func TestEnsureDirReusesExistingFolder(t *testing.T) {
|
||||
var postCalled bool
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
if got := r.URL.Query().Get("parent_id"); got != "root-id" {
|
||||
t.Fatalf("parent_id = %q, want root-id", got)
|
||||
}
|
||||
writePikPakJSON(t, w, map[string]any{
|
||||
"files": []map[string]any{{
|
||||
"id": "existing-folder-id",
|
||||
"kind": "drive#folder",
|
||||
"name": "91 Spider",
|
||||
}},
|
||||
})
|
||||
case http.MethodPost:
|
||||
postCalled = true
|
||||
t.Fatalf("existing folder should not be created again")
|
||||
default:
|
||||
t.Fatalf("unexpected method %s", r.Method)
|
||||
}
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
if _, err := d.EnsureDir(nil, "/previews"); err != drives.ErrNotSupported {
|
||||
t.Fatalf("EnsureDir error = %v, want ErrNotSupported", err)
|
||||
d := newTestDriver(t, srv)
|
||||
got, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure dir: %v", err)
|
||||
}
|
||||
if got != "existing-folder-id" {
|
||||
t.Fatalf("dir id = %q, want existing-folder-id", got)
|
||||
}
|
||||
if postCalled {
|
||||
t.Fatal("POST should not be called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDirCreatesMissingFolder(t *testing.T) {
|
||||
var got uploadRequestBody
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
writePikPakJSON(t, w, map[string]any{"files": []map[string]any{}})
|
||||
case http.MethodPost:
|
||||
if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode create folder body: %v", err)
|
||||
}
|
||||
writePikPakJSON(t, w, map[string]any{
|
||||
"id": "new-folder-id",
|
||||
"kind": "drive#folder",
|
||||
"name": "91 Spider",
|
||||
})
|
||||
default:
|
||||
t.Fatalf("unexpected method %s", r.Method)
|
||||
}
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
d := newTestDriver(t, srv)
|
||||
id, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure dir: %v", err)
|
||||
}
|
||||
if id != "new-folder-id" {
|
||||
t.Fatalf("dir id = %q, want new-folder-id", id)
|
||||
}
|
||||
if got.Kind != "drive#folder" || got.ParentID != "root-id" || got.Name != "91 Spider" {
|
||||
t.Fatalf("create folder body = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func writePikPakJSON(t *testing.T, w http.ResponseWriter, body any) {
|
||||
t.Helper()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(body); err != nil {
|
||||
t.Fatalf("write json: %v", err)
|
||||
}
|
||||
// Upload 的真实实现见 upload_test.go。
|
||||
}
|
||||
|
||||
@@ -59,6 +59,10 @@ func (e *errResp) Error() string {
|
||||
return fmt.Sprintf("pikpak error_code=%d error=%s description=%s", e.ErrorCode, e.ErrorMsg, e.ErrorDescription)
|
||||
}
|
||||
|
||||
func isCaptchaTokenRejectedCode(code int64) bool {
|
||||
return code == 9 || code == 4002
|
||||
}
|
||||
|
||||
// APIError is the public alias for the PikPak API error response. Callers
|
||||
// outside this package (e.g. the spider91→PikPak migrator, tests) can either
|
||||
// construct it for fakes or unwrap it via errors.As. Prefer IsCaptchaError
|
||||
@@ -76,7 +80,7 @@ func IsCaptchaError(err error) bool {
|
||||
}
|
||||
var e *errResp
|
||||
if errors.As(err, &e) {
|
||||
return e != nil && (e.ErrorCode == 4002 || e.ErrorCode == 9)
|
||||
return e != nil && isCaptchaTokenRejectedCode(e.ErrorCode)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -263,7 +263,7 @@ func (d *Driver) findChildDir(ctx context.Context, parent, name string) (string,
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// ---------- 上传(第一版不实现,走本地 teaser 兜底) ----------
|
||||
// ---------- 上传(第一版不实现,走本地预览视频兜底) ----------
|
||||
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -20,6 +21,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
"golang.org/x/net/proxy"
|
||||
)
|
||||
|
||||
// 默认 author/tag 标签,便于在前端筛选 spider91 来源的视频。
|
||||
@@ -59,7 +62,7 @@ type CrawlerConfig struct {
|
||||
// DownloadTimeout 限制单条视频/封面下载的耗时。
|
||||
DownloadTimeout time.Duration
|
||||
|
||||
// OnNewVideo 是新视频成功入库后的回调,用于触发 teaser worker。
|
||||
// OnNewVideo 是新视频成功入库后的回调,用于触发预览视频 worker。
|
||||
OnNewVideo func(v *catalog.Video)
|
||||
}
|
||||
|
||||
@@ -79,29 +82,123 @@ func NewCrawler(cfg CrawlerConfig) *Crawler {
|
||||
cfg.DownloadTimeout = 30 * time.Minute
|
||||
}
|
||||
if cfg.HTTPClient == nil {
|
||||
// 选 proxy 函数:显式 ProxyURL > 环境变量 > 直连
|
||||
proxyFn := http.ProxyFromEnvironment
|
||||
if strings.TrimSpace(cfg.ProxyURL) != "" {
|
||||
if u, err := url.Parse(cfg.ProxyURL); err == nil {
|
||||
proxyFn = http.ProxyURL(u)
|
||||
} else {
|
||||
log.Printf("[spider91] invalid proxy URL %q, falling back to env: %v", cfg.ProxyURL, err)
|
||||
}
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
ResponseHeaderTimeout: 60 * time.Second,
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
}
|
||||
if err := configureExplicitProxy(transport, cfg.ProxyURL); err != nil {
|
||||
log.Printf("[spider91] invalid configured proxy URL, falling back to env: %v", err)
|
||||
}
|
||||
cfg.HTTPClient = &http.Client{
|
||||
// 不限制总下载时长,靠 ctx 控制;只挡 dial / handshake / header
|
||||
Timeout: 0,
|
||||
Transport: &http.Transport{
|
||||
Proxy: proxyFn,
|
||||
ResponseHeaderTimeout: 60 * time.Second,
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
},
|
||||
Timeout: 0,
|
||||
Transport: transport,
|
||||
}
|
||||
}
|
||||
return &Crawler{cfg: cfg}
|
||||
}
|
||||
|
||||
func configureExplicitProxy(transport *http.Transport, raw string) error {
|
||||
proxyURL := strings.TrimSpace(raw)
|
||||
if proxyURL == "" {
|
||||
return nil
|
||||
}
|
||||
u, err := url.Parse(proxyURL)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return fmt.Errorf("invalid proxy URL")
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https":
|
||||
transport.Proxy = http.ProxyURL(u)
|
||||
transport.DialContext = nil
|
||||
return nil
|
||||
case "socks5", "socks5h":
|
||||
dialContext, err := socksProxyDialContext(u)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
transport.Proxy = nil
|
||||
transport.DialContext = dialContext
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("unsupported proxy scheme %q", u.Scheme)
|
||||
}
|
||||
}
|
||||
|
||||
func socksProxyDialContext(proxyURL *url.URL) (func(context.Context, string, string) (net.Conn, error), error) {
|
||||
var auth *proxy.Auth
|
||||
if proxyURL.User != nil {
|
||||
username := proxyURL.User.Username()
|
||||
password, _ := proxyURL.User.Password()
|
||||
auth = &proxy.Auth{User: username, Password: password}
|
||||
}
|
||||
dialer, err := proxy.SOCKS5("tcp", proxyURL.Host, auth, &net.Dialer{Timeout: 60 * time.Second})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remoteDNS := strings.EqualFold(proxyURL.Scheme, "socks5h")
|
||||
return func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
target := addr
|
||||
if !remoteDNS {
|
||||
resolved, err := resolveSocksTarget(ctx, addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target = resolved
|
||||
}
|
||||
if ctxDialer, ok := dialer.(proxy.ContextDialer); ok {
|
||||
return ctxDialer.DialContext(ctx, network, target)
|
||||
}
|
||||
type result struct {
|
||||
conn net.Conn
|
||||
err error
|
||||
}
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
conn, err := dialer.Dial(network, target)
|
||||
ch <- result{conn: conn, err: err}
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case res := <-ch:
|
||||
return res.conn, res.err
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveSocksTarget(ctx context.Context, addr string) (string, error) {
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil || net.ParseIP(host) != nil {
|
||||
return addr, nil
|
||||
}
|
||||
ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
ip := selectSocksTargetIP(ips)
|
||||
if ip == nil {
|
||||
return "", fmt.Errorf("resolve %s: no address", host)
|
||||
}
|
||||
return net.JoinHostPort(ip.String(), port), nil
|
||||
}
|
||||
|
||||
func selectSocksTargetIP(ips []net.IPAddr) net.IP {
|
||||
for _, addr := range ips {
|
||||
if ip4 := addr.IP.To4(); ip4 != nil {
|
||||
return ip4
|
||||
}
|
||||
}
|
||||
for _, addr := range ips {
|
||||
if addr.IP != nil {
|
||||
return addr.IP
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CrawlResult 汇总一次 RunOnce 的结果。
|
||||
type CrawlResult struct {
|
||||
// TargetNew 是本次 RunOnce 的目标新增数(来自 drive.Credentials.target_new)。
|
||||
@@ -139,7 +236,7 @@ type spiderVideoEntry struct {
|
||||
// 3. Go 端 bufio.Scanner 按行读:每行立即下载视频和封面、入库。
|
||||
// 这样 "Python 翻页找下一个" 与 "Go 下载当前一个" 在时间上重叠,缩短整轮耗时;
|
||||
// 更重要的是不会让前几个下载耽误后面签名链接 e= 过期。
|
||||
// 4. 全部消费完 + 子进程退出 → 返回 CrawlResult。teaser 不在此处入队,
|
||||
// 4. 全部消费完 + 子进程退出 → 返回 CrawlResult。预览视频不在此处入队,
|
||||
// 由调用方 (App.runSpider91Crawl) 在 RunOnce 后统一调 enqueueDriveGeneration。
|
||||
//
|
||||
// targetNew <= 0 会被规范化成 spider91DefaultTargetNew(15)。
|
||||
@@ -234,6 +331,16 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
break
|
||||
}
|
||||
videoID := buildVideoID(c.cfg.Driver.ID(), sourceID)
|
||||
deleted, err := c.cfg.Catalog.IsVideoDeleted(ctx, videoID)
|
||||
if err != nil {
|
||||
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s check deleted: %v", c.cfg.Driver.ID(), item.Viewkey, sourceID, err)
|
||||
result.Failed++
|
||||
continue
|
||||
}
|
||||
if deleted {
|
||||
result.Skipped++
|
||||
continue
|
||||
}
|
||||
if existing, _ := c.cfg.Catalog.GetVideo(ctx, videoID); existing != nil {
|
||||
result.Skipped++
|
||||
continue
|
||||
@@ -324,6 +431,16 @@ func (c *Crawler) startSpiderTargetNew(ctx context.Context, targetNew int, seenP
|
||||
if c.cfg.WorkDir != "" {
|
||||
cmd.Dir = c.cfg.WorkDir
|
||||
}
|
||||
if proxyURL := strings.TrimSpace(c.cfg.ProxyURL); proxyURL != "" {
|
||||
cmd.Env = append(os.Environ(),
|
||||
"HTTP_PROXY="+proxyURL,
|
||||
"HTTPS_PROXY="+proxyURL,
|
||||
"http_proxy="+proxyURL,
|
||||
"https_proxy="+proxyURL,
|
||||
"NO_PROXY=",
|
||||
"no_proxy=",
|
||||
)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("stdout pipe: %w", err)
|
||||
@@ -419,7 +536,7 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
log.Printf("[spider91] drive=%s mkdir common thumbs: %v", c.cfg.Driver.ID(), err)
|
||||
thumbReady = false
|
||||
} else {
|
||||
dst := filepath.Join(c.cfg.CommonThumbDir, videoID+".jpg")
|
||||
dst := mediaasset.ThumbnailPathInDir(c.cfg.CommonThumbDir, videoID)
|
||||
if err := copyFileAtomic(thumbPath, dst); err != nil {
|
||||
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s copy thumb to common dir: %v", c.cfg.Driver.ID(), viewkey, sourceID, err)
|
||||
thumbReady = false
|
||||
@@ -427,6 +544,17 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
}
|
||||
}
|
||||
|
||||
title := strings.TrimSpace(item.Title)
|
||||
if title == "" {
|
||||
title = sourceID
|
||||
}
|
||||
tags := []string{DefaultTag}
|
||||
if matched, err := c.cfg.Catalog.MatchTags(ctx, title+" "+DefaultAuthor); err == nil {
|
||||
tags = mergeCatalogTags(tags, matched)
|
||||
} else {
|
||||
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s match tags: %v", c.cfg.Driver.ID(), viewkey, sourceID, err)
|
||||
}
|
||||
|
||||
// 入库
|
||||
now := time.Now()
|
||||
v := &catalog.Video{
|
||||
@@ -434,9 +562,9 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
DriveID: c.cfg.Driver.ID(),
|
||||
FileID: videoFile,
|
||||
FileName: videoFile,
|
||||
Title: strings.TrimSpace(item.Title),
|
||||
Title: title,
|
||||
Author: DefaultAuthor,
|
||||
Tags: []string{DefaultTag},
|
||||
Tags: tags,
|
||||
Ext: strings.TrimPrefix(videoExt, "."),
|
||||
Quality: "HD",
|
||||
Size: videoSize,
|
||||
@@ -445,9 +573,6 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if v.Title == "" {
|
||||
v.Title = sourceID
|
||||
}
|
||||
if thumbReady {
|
||||
// 设了 ThumbnailURL 后 thumb worker 会跳过这条视频,
|
||||
// 不再尝试用 ffmpeg 抽帧(封面已经是网站原图)。
|
||||
@@ -463,8 +588,7 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
// 网站封面下载失败的视频:spider91 drive 的 thumb worker 按设计不
|
||||
// 处理 spider91 视频(封面应是网站原图直接保存),所以没人接手。
|
||||
// 显式标 'failed' 让 CountVideosNeedingThumbnail 排除(条件 status
|
||||
// != 'failed'),否则 enqueueDriveGeneration → waitForThumbnailsBeforePreview
|
||||
// 会因为 count > 0 把 teaser 入队永远卡在等待循环里。
|
||||
// != 'failed'),避免后续封面补队列一直重复捞到这条视频。
|
||||
_ = c.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
ThumbnailStatus: "failed",
|
||||
})
|
||||
@@ -889,6 +1013,26 @@ func copyFileAtomic(src, dst string) error {
|
||||
return os.Rename(tmp, dst)
|
||||
}
|
||||
|
||||
func mergeCatalogTags(lists ...[]string) []string {
|
||||
out := []string{}
|
||||
seen := map[string]bool{}
|
||||
for _, list := range lists {
|
||||
for _, tag := range list {
|
||||
tag = strings.TrimSpace(tag)
|
||||
if tag == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(tag)
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, tag)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BuildVideoID 给定 driveID + 91 源视频 ID,按统一规则生成 catalog 中 videos.id。
|
||||
// 与 scanner 用法一致:<kind>-<driveID>-<fileID>。
|
||||
func BuildVideoID(driveID, sourceID string) string {
|
||||
|
||||
@@ -3,6 +3,8 @@ package spider91
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
@@ -53,7 +55,7 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
|
||||
// 同时仍写 --output 文件作归档。
|
||||
videoEntries := []map[string]string{
|
||||
{
|
||||
"title": "Video One",
|
||||
"title": "Video One 口交",
|
||||
"thumb_url": srv.URL + "/thumb/not-120001.jpg",
|
||||
"video_url": srv.URL + "/videos/120001.mp4",
|
||||
"viewkey": "vk-001",
|
||||
@@ -94,6 +96,9 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert drive: %v", err)
|
||||
}
|
||||
if _, err := cat.CreateTagAndClassify(context.Background(), "Video One", nil, "user"); err != nil {
|
||||
t.Fatalf("create user tag: %v", err)
|
||||
}
|
||||
|
||||
var newVideos []*catalog.Video
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
@@ -188,6 +193,17 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
|
||||
if !hasDefaultTag {
|
||||
t.Fatalf("video %s tags = %v, want contain %q", videoID, v.Tags, DefaultTag)
|
||||
}
|
||||
if sourceID == "120001" {
|
||||
if !containsString(v.Tags, "口交") {
|
||||
t.Fatalf("video %s tags = %v, want contain built-in tag 口交", videoID, v.Tags)
|
||||
}
|
||||
if !containsString(v.Tags, "Video One") {
|
||||
t.Fatalf("video %s tags = %v, want contain user tag Video One", videoID, v.Tags)
|
||||
}
|
||||
}
|
||||
if sourceID == "120002" && (containsString(v.Tags, "口交") || containsString(v.Tags, "Video One")) {
|
||||
t.Fatalf("video %s tags = %v, should not inherit tags from other spider91 videos", videoID, v.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
// 7. 第二次 RunOnce:源视频 ID 已存在 → 全部 skipped,无新文件下载
|
||||
@@ -233,14 +249,116 @@ func TestCrawlerRunOnceMissingScript(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerPassesProxyToSpiderProcess(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell-based fake script only on unix")
|
||||
}
|
||||
|
||||
tmp := t.TempDir()
|
||||
scriptPath := filepath.Join(tmp, "print_proxy_env.sh")
|
||||
script := `#!/bin/sh
|
||||
printf 'HTTP_PROXY=%s\n' "$HTTP_PROXY"
|
||||
printf 'HTTPS_PROXY=%s\n' "$HTTPS_PROXY"
|
||||
printf 'http_proxy=%s\n' "$http_proxy"
|
||||
printf 'https_proxy=%s\n' "$https_proxy"
|
||||
printf 'NO_PROXY=%s\n' "$NO_PROXY"
|
||||
printf 'no_proxy=%s\n' "$no_proxy"
|
||||
`
|
||||
if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
|
||||
proxyURL := "socks5h://proxy.local:1080"
|
||||
drv := New(Config{ID: "proxy-drive", RootDir: filepath.Join(tmp, "proxy-drive")})
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
PythonPath: "sh",
|
||||
ScriptPath: scriptPath,
|
||||
ProxyURL: proxyURL,
|
||||
})
|
||||
cmd, stdout, err := c.startSpiderTargetNew(
|
||||
context.Background(),
|
||||
1,
|
||||
filepath.Join(tmp, "seen.txt"),
|
||||
filepath.Join(tmp, "out.json"),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("startSpiderTargetNew: %v", err)
|
||||
}
|
||||
raw, err := io.ReadAll(stdout)
|
||||
if err != nil {
|
||||
t.Fatalf("read stdout: %v", err)
|
||||
}
|
||||
if err := cmd.Wait(); err != nil {
|
||||
t.Fatalf("wait: %v", err)
|
||||
}
|
||||
|
||||
want := strings.Join([]string{
|
||||
"HTTP_PROXY=" + proxyURL,
|
||||
"HTTPS_PROXY=" + proxyURL,
|
||||
"http_proxy=" + proxyURL,
|
||||
"https_proxy=" + proxyURL,
|
||||
"NO_PROXY=",
|
||||
"no_proxy=",
|
||||
}, "\n") + "\n"
|
||||
if string(raw) != want {
|
||||
t.Fatalf("proxy env = %q, want %q", string(raw), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigureExplicitProxySupportsSocksSchemes(t *testing.T) {
|
||||
for _, raw := range []string{
|
||||
"socks5://127.0.0.1:1080",
|
||||
"socks5h://proxy-user:proxy-pass@127.0.0.1:1080",
|
||||
} {
|
||||
t.Run(raw, func(t *testing.T) {
|
||||
transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
|
||||
if err := configureExplicitProxy(transport, raw); err != nil {
|
||||
t.Fatalf("configureExplicitProxy: %v", err)
|
||||
}
|
||||
if transport.Proxy != nil {
|
||||
t.Fatalf("Transport.Proxy should be nil for SOCKS proxy")
|
||||
}
|
||||
if transport.DialContext == nil {
|
||||
t.Fatalf("Transport.DialContext should be set for SOCKS proxy")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
|
||||
if err := configureExplicitProxy(transport, "http://127.0.0.1:7890"); err != nil {
|
||||
t.Fatalf("configureExplicitProxy http: %v", err)
|
||||
}
|
||||
if transport.Proxy == nil {
|
||||
t.Fatalf("Transport.Proxy should be set for HTTP proxy")
|
||||
}
|
||||
if transport.DialContext != nil {
|
||||
t.Fatalf("Transport.DialContext should not be set for HTTP proxy")
|
||||
}
|
||||
|
||||
if err := configureExplicitProxy(&http.Transport{}, "ftp://127.0.0.1:21"); err == nil {
|
||||
t.Fatalf("expected unsupported proxy scheme error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSocksTargetIPPrefersIPv4(t *testing.T) {
|
||||
got := selectSocksTargetIP([]net.IPAddr{
|
||||
{IP: net.ParseIP("2606:4700:20::681a:229")},
|
||||
{IP: net.ParseIP("104.26.3.41")},
|
||||
})
|
||||
if got == nil || got.String() != "104.26.3.41" {
|
||||
t.Fatalf("selectSocksTargetIP = %v, want IPv4 104.26.3.41", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCrawlerThumbDownloadFailureMarksStatusFailed 验证:网站封面下载失败时
|
||||
// crawler 把 thumbnail_status 显式标 'failed',避免 enqueueDriveGeneration 的
|
||||
// waitForThumbnailsBeforePreview 因为 count > 0 把 teaser 卡死等待。
|
||||
// crawler 把 thumbnail_status 显式标 'failed',避免后续封面补队列一直重复
|
||||
// 捞到这条 spider91 视频。
|
||||
//
|
||||
// 历史 bug:之前 thumb 下载失败仅打 log,url=”, status 走 schema DEFAULT 'pending'。
|
||||
// CountVideosNeedingThumbnail 条件是 url=” AND status != 'failed' → count=1。
|
||||
// spider91 drive 的 thumb worker 按设计不处理 spider91 视频 → 没人会改 status。
|
||||
// 结果 teaser 永远卡在 [preview] waiting for 1 thumbnails before teaser generation。
|
||||
// spider91 drive 的 thumb worker 按设计不处理 spider91 视频 → 没人会改 status,
|
||||
// 后续补队列会一直认为它还缺封面。
|
||||
func TestCrawlerThumbDownloadFailureMarksStatusFailed(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell-based fake script only on unix")
|
||||
@@ -317,8 +435,7 @@ func TestCrawlerThumbDownloadFailureMarksStatusFailed(t *testing.T) {
|
||||
|
||||
// 关键断言:CountVideosNeedingThumbnail 应该返回 0。
|
||||
// 该函数的 SQL 条件是 `url = '' AND status != 'failed'`;如果 crawler 没把
|
||||
// status 标 'failed'(schema DEFAULT 'pending'),count 就会是 1,外层
|
||||
// waitForThumbnailsBeforePreview 会因为 count > 0 把 teaser 卡死等待。
|
||||
// status 标 'failed'(schema DEFAULT 'pending'),count 就会是 1。
|
||||
count, err := cat.CountVideosNeedingThumbnail(context.Background(), driveID)
|
||||
if err != nil {
|
||||
t.Fatalf("count: %v", err)
|
||||
@@ -659,3 +776,12 @@ func buildFakeSpiderScript(entries []map[string]string) string {
|
||||
sb.WriteString("fi\n")
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func containsString(values []string, want string) bool {
|
||||
for _, value := range values {
|
||||
if value == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -138,7 +138,7 @@ func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error)
|
||||
|
||||
// StreamURL 返回本地视频文件路径,给 ffmpeg / 上层服务使用。
|
||||
// 注意:proxy.serve 不能直接处理本地路径,回放要走 api.handleSpider91Video。
|
||||
// teaser/封面 worker 通过 localPreviewLink 兜底走本地文件,刚好兼容 path 形式的 URL。
|
||||
// 预览视频/封面 worker 通过 localPreviewLink 兜底走本地文件,刚好兼容 path 形式的 URL。
|
||||
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
path, err := d.VideoPath(fileID)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,468 @@
|
||||
package fingerprint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultSampleSizeBytes int64 = 512 * 1024
|
||||
defaultFullHashMaxSize int64 = 8 * 1024 * 1024
|
||||
defaultCooldown = 5 * time.Minute
|
||||
defaultWorkerQueueSize = 10000
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
SampleSizeBytes int64
|
||||
FullHashMaxSize int64
|
||||
RateLimitCooldown time.Duration
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
type Worker struct {
|
||||
Catalog *catalog.Catalog
|
||||
Drive drives.Drive
|
||||
Config Config
|
||||
|
||||
ch chan *catalog.Video
|
||||
queue videoQueue
|
||||
activity taskActivity
|
||||
cooldown cooldownState
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
type TaskStatus struct {
|
||||
State string
|
||||
CurrentTitle string
|
||||
QueueLength int
|
||||
CooldownUntil time.Time
|
||||
}
|
||||
|
||||
func NewWorker(cat *catalog.Catalog, drv drives.Drive, cfg Config) *Worker {
|
||||
hc := cfg.HTTPClient
|
||||
if hc == nil {
|
||||
hc = &http.Client{Timeout: 0}
|
||||
}
|
||||
if cfg.SampleSizeBytes <= 0 {
|
||||
cfg.SampleSizeBytes = defaultSampleSizeBytes
|
||||
}
|
||||
if cfg.FullHashMaxSize <= 0 {
|
||||
cfg.FullHashMaxSize = defaultFullHashMaxSize
|
||||
}
|
||||
if cfg.RateLimitCooldown <= 0 {
|
||||
cfg.RateLimitCooldown = defaultCooldown
|
||||
}
|
||||
return &Worker{
|
||||
Catalog: cat,
|
||||
Drive: drv,
|
||||
Config: cfg,
|
||||
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
|
||||
http: hc,
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) Enqueue(v *catalog.Video) bool {
|
||||
if v == nil {
|
||||
return false
|
||||
}
|
||||
if !w.queue.reserve(v.ID) {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case w.ch <- v:
|
||||
return true
|
||||
default:
|
||||
w.queue.release(v.ID)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) EnqueueBlocking(ctx context.Context, v *catalog.Video) bool {
|
||||
if v == nil {
|
||||
return false
|
||||
}
|
||||
if !w.queue.reserve(v.ID) {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case w.ch <- v:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
w.queue.release(v.ID)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) Run(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case v := <-w.ch:
|
||||
w.processQueued(ctx, v)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) Status() TaskStatus {
|
||||
if w == nil {
|
||||
return TaskStatus{State: "idle"}
|
||||
}
|
||||
currentID, currentTitle := w.activity.current()
|
||||
status := TaskStatus{
|
||||
State: "idle",
|
||||
CurrentTitle: currentTitle,
|
||||
QueueLength: w.queue.lengthExcluding(currentID),
|
||||
}
|
||||
if until, ok := w.cooldown.active(time.Now()); ok {
|
||||
status.State = "cooling"
|
||||
status.CooldownUntil = until
|
||||
return status
|
||||
}
|
||||
if currentID != "" {
|
||||
status.State = "generating"
|
||||
return status
|
||||
}
|
||||
if status.QueueLength > 0 {
|
||||
status.State = "queued"
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
defer w.queue.release(v.ID)
|
||||
if w.Catalog == nil || w.Drive == nil || v == nil || v.ID == "" {
|
||||
return
|
||||
}
|
||||
current, err := w.Catalog.GetVideo(ctx, v.ID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if current.SampledSHA256 != "" || current.FingerprintStatus == "ready" || current.Hidden {
|
||||
return
|
||||
}
|
||||
w.activity.start(current)
|
||||
defer w.activity.done()
|
||||
sum, err := Compute(ctx, w.Drive, current, w.Config, w.http)
|
||||
if err != nil {
|
||||
var rl *drives.RateLimitError
|
||||
if errors.As(err, &rl) {
|
||||
wait := rl.RetryAfter
|
||||
if wait <= 0 {
|
||||
wait = w.Config.RateLimitCooldown
|
||||
}
|
||||
until := time.Now().Add(wait)
|
||||
w.cooldown.set(until)
|
||||
log.Printf("[fingerprint] drive=%s rate limited; keep video=%s pending and cool down for %s: %v", w.Drive.ID(), current.ID, wait, err)
|
||||
sleepContext(ctx, wait)
|
||||
w.cooldown.clear(until)
|
||||
return
|
||||
}
|
||||
log.Printf("[fingerprint] video=%s failed: %v", current.ID, err)
|
||||
_ = w.Catalog.UpdateVideoFingerprint(ctx, current.ID, "", "failed", err.Error())
|
||||
return
|
||||
}
|
||||
if err := w.Catalog.UpdateVideoFingerprint(ctx, current.ID, sum, "ready", ""); err != nil {
|
||||
log.Printf("[fingerprint] update video=%s: %v", current.ID, err)
|
||||
return
|
||||
}
|
||||
log.Printf("[fingerprint] video=%s ready sampled_sha256=%s", current.ID, sum)
|
||||
}
|
||||
|
||||
func Compute(ctx context.Context, drv drives.Drive, v *catalog.Video, cfg Config, hc *http.Client) (string, error) {
|
||||
if drv == nil {
|
||||
return "", errors.New("fingerprint: nil drive")
|
||||
}
|
||||
if v == nil {
|
||||
return "", errors.New("fingerprint: nil video")
|
||||
}
|
||||
if v.Size <= 0 {
|
||||
return "", errors.New("fingerprint: video size is empty")
|
||||
}
|
||||
if cfg.SampleSizeBytes <= 0 {
|
||||
cfg.SampleSizeBytes = defaultSampleSizeBytes
|
||||
}
|
||||
if cfg.FullHashMaxSize <= 0 {
|
||||
cfg.FullHashMaxSize = defaultFullHashMaxSize
|
||||
}
|
||||
if hc == nil {
|
||||
hc = &http.Client{Timeout: 0}
|
||||
}
|
||||
link, err := drv.StreamURL(ctx, v.FileID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fingerprint: stream url: %w", err)
|
||||
}
|
||||
if link == nil || strings.TrimSpace(link.URL) == "" {
|
||||
return "", errors.New("fingerprint: empty stream url")
|
||||
}
|
||||
ranges := sampleRanges(v.Size, cfg.SampleSizeBytes, cfg.FullHashMaxSize)
|
||||
h := sha256.New()
|
||||
writeHashHeader(h, v.Size, ranges)
|
||||
for _, r := range ranges {
|
||||
data, err := readRange(ctx, hc, link, r)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if int64(len(data)) != r.length {
|
||||
return "", fmt.Errorf("fingerprint: short sample at %d: got %d want %d", r.start, len(data), r.length)
|
||||
}
|
||||
_, _ = h.Write([]byte(fmt.Sprintf("offset=%d length=%d\n", r.start, r.length)))
|
||||
_, _ = h.Write(data)
|
||||
_, _ = h.Write([]byte("\n"))
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
type byteRange struct {
|
||||
start int64
|
||||
length int64
|
||||
}
|
||||
|
||||
func sampleRanges(size, sampleSize, fullHashMax int64) []byteRange {
|
||||
if size <= fullHashMax {
|
||||
return []byteRange{{start: 0, length: size}}
|
||||
}
|
||||
if sampleSize > size {
|
||||
sampleSize = size
|
||||
}
|
||||
maxStart := size - sampleSize
|
||||
percents := []int64{0, 20, 40, 60, 80}
|
||||
out := make([]byteRange, 0, len(percents))
|
||||
seen := make(map[int64]struct{}, len(percents))
|
||||
for _, pct := range percents {
|
||||
start := maxStart * pct / 100
|
||||
if _, ok := seen[start]; ok {
|
||||
continue
|
||||
}
|
||||
seen[start] = struct{}{}
|
||||
out = append(out, byteRange{start: start, length: sampleSize})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func writeHashHeader(w io.Writer, size int64, ranges []byteRange) {
|
||||
_, _ = fmt.Fprintf(w, "video-site-sampled-sha256-v1\nsize=%d\nsamples=%d\n", size, len(ranges))
|
||||
}
|
||||
|
||||
func readRange(ctx context.Context, hc *http.Client, link *drives.StreamLink, r byteRange) ([]byte, error) {
|
||||
u, err := url.Parse(link.URL)
|
||||
if err == nil && (u.Scheme == "http" || u.Scheme == "https") {
|
||||
return readHTTPRange(ctx, hc, link, r)
|
||||
}
|
||||
path := link.URL
|
||||
if err == nil && u.Scheme == "file" {
|
||||
path = u.Path
|
||||
}
|
||||
return readLocalRange(path, r)
|
||||
}
|
||||
|
||||
func readLocalRange(path string, r byteRange) ([]byte, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fingerprint: open local stream: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
buf := make([]byte, r.length)
|
||||
n, err := f.ReadAt(buf, r.start)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, fmt.Errorf("fingerprint: read local sample: %w", err)
|
||||
}
|
||||
if int64(n) != r.length {
|
||||
return nil, fmt.Errorf("fingerprint: read local sample at %d: got %d want %d", r.start, n, r.length)
|
||||
}
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
func readHTTPRange(ctx context.Context, hc *http.Client, link *drives.StreamLink, r byteRange) ([]byte, error) {
|
||||
end := r.start + r.length - 1
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, link.URL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for k, vs := range link.Headers {
|
||||
for _, v := range vs {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", r.start, end))
|
||||
resp, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fingerprint: read remote sample: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
return nil, &drives.RateLimitError{
|
||||
Provider: "fingerprint",
|
||||
RetryAfter: parseRetryAfter(resp.Header.Get("Retry-After")),
|
||||
Err: fmt.Errorf("remote sample rate limited: status=%d", resp.StatusCode),
|
||||
}
|
||||
}
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
if resp.StatusCode == http.StatusOK && r.start == 0 {
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, r.length+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) == r.length {
|
||||
return data, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("fingerprint: range request got status=%d for bytes=%d-%d", resp.StatusCode, r.start, end)
|
||||
}
|
||||
return io.ReadAll(io.LimitReader(resp.Body, r.length))
|
||||
}
|
||||
|
||||
func parseRetryAfter(raw string) time.Duration {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return 0
|
||||
}
|
||||
if seconds, err := strconv.Atoi(raw); err == nil && seconds > 0 {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
if when, err := http.ParseTime(raw); err == nil {
|
||||
d := time.Until(when)
|
||||
if d > 0 {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, d time.Duration) bool {
|
||||
if d <= 0 {
|
||||
return true
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-timer.C:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
type taskActivity struct {
|
||||
mu sync.Mutex
|
||||
currentID string
|
||||
currentTitle string
|
||||
}
|
||||
|
||||
func (a *taskActivity) start(v *catalog.Video) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if v == nil {
|
||||
a.currentID = ""
|
||||
a.currentTitle = ""
|
||||
return
|
||||
}
|
||||
a.currentID = v.ID
|
||||
a.currentTitle = v.Title
|
||||
}
|
||||
|
||||
func (a *taskActivity) done() {
|
||||
a.mu.Lock()
|
||||
a.currentID = ""
|
||||
a.currentTitle = ""
|
||||
a.mu.Unlock()
|
||||
}
|
||||
|
||||
func (a *taskActivity) current() (string, string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.currentID, a.currentTitle
|
||||
}
|
||||
|
||||
type cooldownState struct {
|
||||
mu sync.Mutex
|
||||
until time.Time
|
||||
}
|
||||
|
||||
func (s *cooldownState) set(until time.Time) {
|
||||
s.mu.Lock()
|
||||
s.until = until
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *cooldownState) clear(until time.Time) {
|
||||
s.mu.Lock()
|
||||
if s.until.Equal(until) {
|
||||
s.until = time.Time{}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *cooldownState) active(now time.Time) (time.Time, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.until.IsZero() || !s.until.After(now) {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return s.until, true
|
||||
}
|
||||
|
||||
type videoQueue struct {
|
||||
mu sync.Mutex
|
||||
ids map[string]struct{}
|
||||
}
|
||||
|
||||
func (q *videoQueue) reserve(id string) bool {
|
||||
if id == "" {
|
||||
return true
|
||||
}
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
if q.ids == nil {
|
||||
q.ids = make(map[string]struct{})
|
||||
}
|
||||
if _, ok := q.ids[id]; ok {
|
||||
return false
|
||||
}
|
||||
q.ids[id] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
func (q *videoQueue) release(id string) {
|
||||
if id == "" {
|
||||
return
|
||||
}
|
||||
q.mu.Lock()
|
||||
delete(q.ids, id)
|
||||
q.mu.Unlock()
|
||||
}
|
||||
|
||||
func (q *videoQueue) lengthExcluding(currentID string) int {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
n := len(q.ids)
|
||||
if currentID != "" {
|
||||
if _, ok := q.ids[currentID]; ok {
|
||||
n--
|
||||
}
|
||||
}
|
||||
if n < 0 {
|
||||
return 0
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package fingerprint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestComputeLocalFilesWithSameContentMatch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
body := []byte("same video bytes")
|
||||
a := filepath.Join(dir, "a.mp4")
|
||||
b := filepath.Join(dir, "b.mp4")
|
||||
if err := os.WriteFile(a, body, 0o644); err != nil {
|
||||
t.Fatalf("write a: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(b, body, 0o644); err != nil {
|
||||
t.Fatalf("write b: %v", err)
|
||||
}
|
||||
|
||||
sumA, err := Compute(ctx, &fakeDrive{paths: map[string]string{"a": a}}, &catalog.Video{ID: "a", FileID: "a", Size: int64(len(body))}, Config{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("compute a: %v", err)
|
||||
}
|
||||
sumB, err := Compute(ctx, &fakeDrive{paths: map[string]string{"b": b}}, &catalog.Video{ID: "b", FileID: "b", Size: int64(len(body))}, Config{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("compute b: %v", err)
|
||||
}
|
||||
if sumA == "" || sumA != sumB {
|
||||
t.Fatalf("fingerprints = %q / %q, want same non-empty", sumA, sumB)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeRemoteUsesRangeSamples(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
data := make([]byte, 10*1024*1024)
|
||||
for i := range data {
|
||||
data[i] = byte(i % 251)
|
||||
}
|
||||
var ranges []string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
rawRange := r.Header.Get("Range")
|
||||
ranges = append(ranges, rawRange)
|
||||
var start, end int
|
||||
if _, err := fmt.Sscanf(rawRange, "bytes=%d-%d", &start, &end); err != nil {
|
||||
t.Fatalf("bad range %q: %v", rawRange, err)
|
||||
}
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write(data[start : end+1])
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
drv := &fakeDrive{paths: map[string]string{"remote": srv.URL + "/video.mp4"}}
|
||||
sum, err := Compute(ctx, drv, &catalog.Video{ID: "remote", FileID: "remote", Size: int64(len(data))}, Config{
|
||||
SampleSizeBytes: 4,
|
||||
FullHashMaxSize: 8,
|
||||
HTTPClient: srv.Client(),
|
||||
}, srv.Client())
|
||||
if err != nil {
|
||||
t.Fatalf("compute remote: %v", err)
|
||||
}
|
||||
if sum == "" {
|
||||
t.Fatal("fingerprint should not be empty")
|
||||
}
|
||||
want := []string{
|
||||
"bytes=0-3",
|
||||
"bytes=2097151-2097154",
|
||||
"bytes=4194302-4194305",
|
||||
"bytes=6291453-6291456",
|
||||
"bytes=8388604-8388607",
|
||||
}
|
||||
if fmt.Sprint(ranges) != fmt.Sprint(want) {
|
||||
t.Fatalf("ranges = %#v, want %#v", ranges, want)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeDrive struct {
|
||||
paths map[string]string
|
||||
}
|
||||
|
||||
func (d *fakeDrive) Kind() string { return "fake" }
|
||||
func (d *fakeDrive) ID() string { return "fake" }
|
||||
func (d *fakeDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *fakeDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeDrive) StreamURL(_ context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
return &drives.StreamLink{URL: d.paths[fileID], Expires: time.Now().Add(time.Minute)}, nil
|
||||
}
|
||||
func (d *fakeDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeDrive) RootID() string { return "root" }
|
||||
@@ -0,0 +1,69 @@
|
||||
package mediaasset
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const maxPlainStemBytes = 180
|
||||
const maxLegacyFilenameBytes = 255
|
||||
|
||||
func PreviewPath(localDir, videoID string) string {
|
||||
return filepath.Join(localDir, PreviewFilename(videoID))
|
||||
}
|
||||
|
||||
func ThumbnailPath(localDir, videoID string) string {
|
||||
return ThumbnailPathInDir(filepath.Join(localDir, "thumbs"), videoID)
|
||||
}
|
||||
|
||||
func ThumbnailPathInDir(thumbDir, videoID string) string {
|
||||
return filepath.Join(thumbDir, ThumbnailFilename(videoID))
|
||||
}
|
||||
|
||||
func PreviewPathCandidates(localDir, videoID string) []string {
|
||||
return pathCandidates(localDir, videoID, ".mp4", "")
|
||||
}
|
||||
|
||||
func ThumbnailPathCandidates(localDir, videoID string) []string {
|
||||
return pathCandidates(localDir, videoID, ".jpg", "thumbs")
|
||||
}
|
||||
|
||||
func PreviewFilename(videoID string) string {
|
||||
return safeFilename(videoID, ".mp4")
|
||||
}
|
||||
|
||||
func ThumbnailFilename(videoID string) string {
|
||||
return safeFilename(videoID, ".jpg")
|
||||
}
|
||||
|
||||
func pathCandidates(localDir, videoID, ext, subdir string) []string {
|
||||
safe := safeFilename(videoID, ext)
|
||||
legacy := videoID + ext
|
||||
base := localDir
|
||||
if subdir != "" {
|
||||
base = filepath.Join(base, subdir)
|
||||
}
|
||||
out := []string{filepath.Join(base, safe)}
|
||||
if legacy != safe && isPlainSafeStem(videoID) && len([]byte(legacy)) <= maxLegacyFilenameBytes {
|
||||
out = append(out, filepath.Join(base, legacy))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func safeFilename(videoID, ext string) string {
|
||||
if isPlainSafeStem(videoID) && len([]byte(videoID))+len(ext) <= maxPlainStemBytes {
|
||||
return videoID + ext
|
||||
}
|
||||
sum := sha256.Sum256([]byte(videoID))
|
||||
return "v-" + hex.EncodeToString(sum[:]) + ext
|
||||
}
|
||||
|
||||
func isPlainSafeStem(value string) bool {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" || value == "." || value == ".." {
|
||||
return false
|
||||
}
|
||||
return !strings.ContainsAny(value, `/\`+"\x00")
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package mediaasset
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFilenamesKeepShortSafeIDs(t *testing.T) {
|
||||
if got := ThumbnailFilename("video-1"); got != "video-1.jpg" {
|
||||
t.Fatalf("thumbnail filename = %q, want video-1.jpg", got)
|
||||
}
|
||||
if got := PreviewFilename("video-1"); got != "video-1.mp4" {
|
||||
t.Fatalf("preview filename = %q, want video-1.mp4", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilenamesHashLongOrUnsafeIDs(t *testing.T) {
|
||||
longID := "localstorage-" + strings.Repeat("x", 240)
|
||||
got := ThumbnailFilename(longID)
|
||||
if !strings.HasPrefix(got, "v-") || !strings.HasSuffix(got, ".jpg") {
|
||||
t.Fatalf("thumbnail filename = %q, want hashed jpg", got)
|
||||
}
|
||||
if len([]byte(got)) >= len([]byte(longID+".jpg")) {
|
||||
t.Fatalf("thumbnail filename = %q should be shorter than original id", got)
|
||||
}
|
||||
|
||||
unsafe := ThumbnailFilename("dir/video")
|
||||
if unsafe == "dir/video.jpg" || strings.ContainsAny(unsafe, `/\`) {
|
||||
t.Fatalf("unsafe thumbnail filename = %q, want hashed single filename", unsafe)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailPathCandidatesIncludeLegacyForHashedIDs(t *testing.T) {
|
||||
localDir := t.TempDir()
|
||||
mediumID := "localstorage-" + strings.Repeat("x", 190)
|
||||
got := ThumbnailPathCandidates(localDir, mediumID)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("candidates = %#v, want hashed and legacy paths", got)
|
||||
}
|
||||
if got[0] != ThumbnailPath(localDir, mediumID) {
|
||||
t.Fatalf("first candidate = %q, want safe path %q", got[0], ThumbnailPath(localDir, mediumID))
|
||||
}
|
||||
if filepath.Base(got[1]) != mediumID+".jpg" {
|
||||
t.Fatalf("legacy candidate = %q, want original id jpg", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailPathCandidatesSkipOverlongLegacy(t *testing.T) {
|
||||
localDir := t.TempDir()
|
||||
longID := "localstorage-" + strings.Repeat("x", 240)
|
||||
got := ThumbnailPathCandidates(localDir, longID)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("candidates = %#v, want only hashed path for overlong id", got)
|
||||
}
|
||||
}
|
||||
@@ -5,13 +5,15 @@
|
||||
// "扫描所有网盘"):
|
||||
//
|
||||
// Phase 1: for each non-spider91 cloud drive
|
||||
// scan + delete-detection + enqueue thumb + enqueue teaser
|
||||
// wait until all thumb / teaser queues are idle
|
||||
// scan + delete-detection + enqueue thumb + enqueue preview video
|
||||
// wait until all thumb / preview-video queues are idle
|
||||
// Phase 2: if any spider91 drive configured
|
||||
// crawl + enqueue teaser for new videos
|
||||
// wait until teaser queues are idle
|
||||
// crawl + enqueue preview video for new videos
|
||||
// wait until preview-video queues are idle
|
||||
// Phase 3: spider91 → cloud migration (single sweep, captcha cooldown still
|
||||
// honored within this call)
|
||||
// Phase 4: cleanup duplicate local preview/thumbnail assets after sampled
|
||||
// fingerprints have identified canonical videos
|
||||
//
|
||||
// A 6h soft deadline guards each pipeline run; phases check deadline at their
|
||||
// boundaries and exit cleanly if exceeded (no in-flight ffmpeg / upload is
|
||||
@@ -74,10 +76,10 @@ type Config struct {
|
||||
ListSpider91Drives func(ctx context.Context) []string
|
||||
|
||||
// RunSpider91Crawl synchronously runs one crawl cycle (downloads + thumbs +
|
||||
// teaser enqueue) for a single spider91 drive.
|
||||
// preview-video enqueue) for a single spider91 drive.
|
||||
RunSpider91Crawl func(ctx context.Context, driveID string)
|
||||
|
||||
// WaitPreviewQueuesIdle blocks until both the thumbnail and teaser queues
|
||||
// WaitPreviewQueuesIdle blocks until both the thumbnail and preview-video queues
|
||||
// across all drives are drained (queue empty + no in-flight task). It must
|
||||
// honor ctx cancellation.
|
||||
WaitPreviewQueuesIdle func(ctx context.Context) error
|
||||
@@ -85,15 +87,35 @@ type Config struct {
|
||||
// RunMigration runs spider91migrate.Migrator.RunOnce for Phase 3.
|
||||
RunMigration func(ctx context.Context) error
|
||||
|
||||
// RunDedupeAssetCleanup removes generated local assets from non-canonical
|
||||
// videos in size+sampled_sha256 duplicate groups. It must not delete cloud
|
||||
// files or catalog rows.
|
||||
RunDedupeAssetCleanup func(ctx context.Context) error
|
||||
|
||||
// Now is injected for tests; nil → time.Now.
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type Status struct {
|
||||
State string
|
||||
Running bool
|
||||
Queued bool
|
||||
StartedAt time.Time
|
||||
LastFinishedAt time.Time
|
||||
}
|
||||
|
||||
// Runner drives the nightly pipeline.
|
||||
type Runner struct {
|
||||
cfg Config
|
||||
trigger chan struct{} // buffered(1); manual "run now"
|
||||
runMu sync.Mutex // prevents overlapping pipeline runs
|
||||
|
||||
stateMu sync.Mutex
|
||||
running bool
|
||||
queued bool
|
||||
startedAt time.Time
|
||||
lastFinishedAt time.Time
|
||||
currentCancel context.CancelFunc
|
||||
}
|
||||
|
||||
// New constructs a Runner. cfg is shallow-copied; defaults are applied.
|
||||
@@ -131,13 +153,75 @@ func (r *Runner) Run(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// TriggerNow asks the running loop to fire a pipeline ASAP. If a pipeline is
|
||||
// already in progress (or another trigger is already pending), the request
|
||||
// is dropped — the in-progress run will absorb the intent.
|
||||
func (r *Runner) TriggerNow() {
|
||||
// TriggerNow asks the running loop to fire a pipeline ASAP. Only one manual
|
||||
// trigger can be active at a time: if a pipeline is already running or waiting
|
||||
// in the trigger channel, the request is ignored and returns false.
|
||||
func (r *Runner) TriggerNow() bool {
|
||||
r.stateMu.Lock()
|
||||
if r.running || r.queued {
|
||||
r.stateMu.Unlock()
|
||||
return false
|
||||
}
|
||||
r.queued = true
|
||||
r.stateMu.Unlock()
|
||||
|
||||
select {
|
||||
case r.trigger <- struct{}{}:
|
||||
return true
|
||||
default:
|
||||
r.stateMu.Lock()
|
||||
r.queued = false
|
||||
r.stateMu.Unlock()
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// StopCurrent cancels the currently running pipeline and drops one queued
|
||||
// manual trigger, if present. It returns true when there was something to stop.
|
||||
func (r *Runner) StopCurrent() bool {
|
||||
r.stateMu.Lock()
|
||||
wasRunning := r.running
|
||||
wasQueued := r.queued
|
||||
cancel := r.currentCancel
|
||||
r.queued = false
|
||||
r.stateMu.Unlock()
|
||||
|
||||
if wasQueued {
|
||||
select {
|
||||
case <-r.trigger:
|
||||
default:
|
||||
}
|
||||
}
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
return wasRunning || wasQueued || cancel != nil
|
||||
}
|
||||
|
||||
func (r *Runner) Status() Status {
|
||||
r.stateMu.Lock()
|
||||
running := r.running
|
||||
queued := r.queued
|
||||
startedAt := r.startedAt
|
||||
lastFinishedAt := r.lastFinishedAt
|
||||
r.stateMu.Unlock()
|
||||
|
||||
state := "idle"
|
||||
switch {
|
||||
case running && queued:
|
||||
state = "running_queued"
|
||||
case running:
|
||||
state = "running"
|
||||
case queued:
|
||||
state = "queued"
|
||||
}
|
||||
|
||||
return Status{
|
||||
State: state,
|
||||
Running: running,
|
||||
Queued: queued,
|
||||
StartedAt: startedAt,
|
||||
LastFinishedAt: lastFinishedAt,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,13 +255,28 @@ func shouldRun(now time.Time, lastRunDate string) bool {
|
||||
//
|
||||
// 流水线没有总耗时上限:一直跑到 ctx 取消(进程退出)或所有 phase 完成。
|
||||
func (r *Runner) runPipelineLocked(ctx context.Context, manual bool) {
|
||||
if manual {
|
||||
r.stateMu.Lock()
|
||||
queued := r.queued
|
||||
r.stateMu.Unlock()
|
||||
if !queued {
|
||||
log.Printf("[nightly] manual trigger was canceled before start")
|
||||
return
|
||||
}
|
||||
}
|
||||
if !r.runMu.TryLock() {
|
||||
log.Printf("[nightly] another pipeline is already running, skipping this trigger")
|
||||
return
|
||||
}
|
||||
defer r.runMu.Unlock()
|
||||
|
||||
started := r.cfg.Now()
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
r.markStarted(started, cancel)
|
||||
defer func() {
|
||||
cancel()
|
||||
r.markFinished(r.cfg.Now())
|
||||
r.runMu.Unlock()
|
||||
}()
|
||||
|
||||
mode := "scheduled"
|
||||
if manual {
|
||||
@@ -185,7 +284,7 @@ func (r *Runner) runPipelineLocked(ctx context.Context, manual bool) {
|
||||
}
|
||||
log.Printf("[nightly] pipeline (%s) start", mode)
|
||||
|
||||
r.runPipeline(ctx)
|
||||
r.runPipeline(runCtx)
|
||||
|
||||
finished := r.cfg.Now()
|
||||
log.Printf("[nightly] pipeline (%s) finish; took=%s", mode, finished.Sub(started).Round(time.Second))
|
||||
@@ -199,6 +298,24 @@ func (r *Runner) runPipelineLocked(ctx context.Context, manual bool) {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) markStarted(started time.Time, cancel context.CancelFunc) {
|
||||
r.stateMu.Lock()
|
||||
defer r.stateMu.Unlock()
|
||||
r.running = true
|
||||
r.queued = false
|
||||
r.startedAt = started
|
||||
r.currentCancel = cancel
|
||||
}
|
||||
|
||||
func (r *Runner) markFinished(finished time.Time) {
|
||||
r.stateMu.Lock()
|
||||
defer r.stateMu.Unlock()
|
||||
r.running = false
|
||||
r.startedAt = time.Time{}
|
||||
r.lastFinishedAt = finished
|
||||
r.currentCancel = nil
|
||||
}
|
||||
|
||||
// runPipeline executes the three phases. It returns when the pipeline finishes
|
||||
// OR ctx is done (deadline / cancel). Errors are logged but not propagated —
|
||||
// each phase is best-effort; downstream phases still attempt to run unless ctx
|
||||
@@ -240,6 +357,7 @@ func (r *Runner) runPipeline(ctx context.Context) {
|
||||
}
|
||||
if len(spiderIDs) == 0 {
|
||||
log.Printf("[nightly] phase 2/3 skipped: no spider91 drive configured")
|
||||
r.runDedupeAssetCleanupPhase(ctx)
|
||||
return
|
||||
}
|
||||
log.Printf("[nightly] phase 2: crawling %d spider91 drive(s)", len(spiderIDs))
|
||||
@@ -266,6 +384,8 @@ func (r *Runner) runPipeline(ctx context.Context) {
|
||||
log.Printf("[nightly] phase 3 migration: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
r.runDedupeAssetCleanupPhase(ctx)
|
||||
}
|
||||
|
||||
// checkDeadline returns true when ctx is already done (runner shutting down or
|
||||
@@ -291,6 +411,19 @@ func (r *Runner) waitIdle(ctx context.Context, phase string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runner) runDedupeAssetCleanupPhase(ctx context.Context) {
|
||||
if r.checkDeadline(ctx, "phase 4") {
|
||||
return
|
||||
}
|
||||
if r.cfg.RunDedupeAssetCleanup == nil {
|
||||
return
|
||||
}
|
||||
log.Printf("[nightly] phase 4: duplicate asset cleanup")
|
||||
if err := r.cfg.RunDedupeAssetCleanup(ctx); err != nil {
|
||||
log.Printf("[nightly] phase 4 duplicate asset cleanup: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// readLastRunDate reads the persisted last_run_date or returns "" when unset.
|
||||
func (r *Runner) readLastRunDate(ctx context.Context) (string, error) {
|
||||
if r.cfg.Settings == nil {
|
||||
|
||||
@@ -114,6 +114,10 @@ func TestRunPipelineHonoursPhaseOrder(t *testing.T) {
|
||||
rec.push("migrate")
|
||||
return nil
|
||||
},
|
||||
RunDedupeAssetCleanup: func(context.Context) error {
|
||||
rec.push("dedupe-cleanup")
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
r.runPipeline(context.Background())
|
||||
@@ -128,6 +132,7 @@ func TestRunPipelineHonoursPhaseOrder(t *testing.T) {
|
||||
"crawl:sp-1",
|
||||
"wait-idle", // after phase 2
|
||||
"migrate",
|
||||
"dedupe-cleanup",
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("call sequence len = %d, want %d; got=%v", len(got), len(want), got)
|
||||
@@ -156,6 +161,10 @@ func TestRunPipelineSkipsMigrationWhenNoSpider91(t *testing.T) {
|
||||
rec.push("migrate")
|
||||
return nil
|
||||
},
|
||||
RunDedupeAssetCleanup: func(context.Context) error {
|
||||
rec.push("dedupe-cleanup")
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
r.runPipeline(context.Background())
|
||||
@@ -165,6 +174,15 @@ func TestRunPipelineSkipsMigrationWhenNoSpider91(t *testing.T) {
|
||||
t.Fatalf("phase 2/3 should be skipped when no spider91 drive, got call %q", c)
|
||||
}
|
||||
}
|
||||
foundCleanup := false
|
||||
for _, c := range rec.snapshot() {
|
||||
if c == "dedupe-cleanup" {
|
||||
foundCleanup = true
|
||||
}
|
||||
}
|
||||
if !foundCleanup {
|
||||
t.Fatalf("dedupe cleanup should still run when spider91 is absent; calls=%v", rec.snapshot())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
|
||||
@@ -186,6 +204,7 @@ func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
|
||||
RunSpider91Crawl: func(context.Context, string) { rec.push("crawl") },
|
||||
WaitPreviewQueuesIdle: func(context.Context) error { rec.push("wait-idle"); return nil },
|
||||
RunMigration: func(context.Context) error { rec.push("migrate"); return nil },
|
||||
RunDedupeAssetCleanup: func(context.Context) error { rec.push("dedupe-cleanup"); return nil },
|
||||
})
|
||||
|
||||
r.runPipeline(ctx)
|
||||
@@ -200,6 +219,9 @@ func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
|
||||
if c == "crawl" || c == "migrate" {
|
||||
t.Fatalf("subsequent phase should not run after cancel, got call %q", c)
|
||||
}
|
||||
if c == "dedupe-cleanup" {
|
||||
t.Fatalf("dedupe cleanup should not run after cancel, got call %q", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -290,11 +312,14 @@ func TestCtxCancelPreventsLaterPhases(t *testing.T) {
|
||||
func TestTriggerNowIsNonBlocking(t *testing.T) {
|
||||
r := New(Config{Settings: newStubSettings()})
|
||||
// fill the trigger channel
|
||||
r.TriggerNow()
|
||||
if !r.TriggerNow() {
|
||||
t.Fatal("first TriggerNow should be accepted")
|
||||
}
|
||||
// Second call must not block
|
||||
done := make(chan struct{})
|
||||
var accepted bool
|
||||
go func() {
|
||||
r.TriggerNow()
|
||||
accepted = r.TriggerNow()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
@@ -302,4 +327,153 @@ func TestTriggerNowIsNonBlocking(t *testing.T) {
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
t.Fatal("TriggerNow blocked when channel is full")
|
||||
}
|
||||
if accepted {
|
||||
t.Fatal("second TriggerNow should be ignored when trigger channel is full")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusTracksQueuedRunningAndFinished(t *testing.T) {
|
||||
blockScan := make(chan struct{})
|
||||
scanStarted := make(chan struct{})
|
||||
var startedOnce sync.Once
|
||||
r := New(Config{
|
||||
Settings: newStubSettings(),
|
||||
ListScanTargets: func(context.Context) []string {
|
||||
return []string{"drive"}
|
||||
},
|
||||
RunScan: func(context.Context, string) {
|
||||
startedOnce.Do(func() { close(scanStarted) })
|
||||
<-blockScan
|
||||
},
|
||||
})
|
||||
|
||||
if got := r.Status(); got.State != "idle" || got.Running || got.Queued {
|
||||
t.Fatalf("initial status = %#v, want idle", got)
|
||||
}
|
||||
|
||||
if !r.TriggerNow() {
|
||||
t.Fatal("TriggerNow should queue a manual run")
|
||||
}
|
||||
if got := r.Status(); got.State != "queued" || got.Running || !got.Queued {
|
||||
t.Fatalf("queued status = %#v, want queued", got)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go r.Run(ctx)
|
||||
|
||||
select {
|
||||
case <-scanStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("pipeline did not start")
|
||||
}
|
||||
|
||||
if got := r.Status(); got.State != "running" || !got.Running || got.Queued || got.StartedAt.IsZero() {
|
||||
t.Fatalf("running status = %#v, want running with startedAt", got)
|
||||
}
|
||||
|
||||
if r.TriggerNow() {
|
||||
t.Fatal("TriggerNow during a run should be ignored")
|
||||
}
|
||||
if got := r.Status(); got.State != "running" || !got.Running || got.Queued {
|
||||
t.Fatalf("status after ignored trigger = %#v, want running", got)
|
||||
}
|
||||
|
||||
close(blockScan)
|
||||
deadline := time.After(time.Second)
|
||||
for {
|
||||
got := r.Status()
|
||||
if !got.Running && !got.Queued && !got.LastFinishedAt.IsZero() {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatalf("status did not finish; got=%#v", got)
|
||||
default:
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopCurrentCancelsRunningPipeline(t *testing.T) {
|
||||
scanStarted := make(chan struct{})
|
||||
scanCanceled := make(chan struct{})
|
||||
var startedOnce sync.Once
|
||||
r := New(Config{
|
||||
Settings: newStubSettings(),
|
||||
ListScanTargets: func(context.Context) []string {
|
||||
return []string{"drive"}
|
||||
},
|
||||
RunScan: func(ctx context.Context, _ string) {
|
||||
startedOnce.Do(func() { close(scanStarted) })
|
||||
<-ctx.Done()
|
||||
close(scanCanceled)
|
||||
},
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go r.Run(ctx)
|
||||
|
||||
if !r.TriggerNow() {
|
||||
t.Fatal("TriggerNow should queue a manual run")
|
||||
}
|
||||
select {
|
||||
case <-scanStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("pipeline did not start")
|
||||
}
|
||||
|
||||
if !r.StopCurrent() {
|
||||
t.Fatal("StopCurrent should report a running pipeline")
|
||||
}
|
||||
select {
|
||||
case <-scanCanceled:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("StopCurrent did not cancel pipeline context")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopCurrentDropsQueuedTrigger(t *testing.T) {
|
||||
r := New(Config{Settings: newStubSettings()})
|
||||
if !r.TriggerNow() {
|
||||
t.Fatal("TriggerNow should queue a manual run")
|
||||
}
|
||||
if !r.StopCurrent() {
|
||||
t.Fatal("StopCurrent should report a queued pipeline")
|
||||
}
|
||||
if got := r.Status(); got.State != "idle" || got.Running || got.Queued {
|
||||
t.Fatalf("status = %#v, want idle after dropping queued trigger", got)
|
||||
}
|
||||
if !r.TriggerNow() {
|
||||
t.Fatal("TriggerNow should accept a new request after queued stop")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTriggerNowAcceptsOnlyOneConcurrentRequest(t *testing.T) {
|
||||
r := New(Config{Settings: newStubSettings()})
|
||||
|
||||
const callers = 16
|
||||
start := make(chan struct{})
|
||||
results := make(chan bool, callers)
|
||||
for i := 0; i < callers; i++ {
|
||||
go func() {
|
||||
<-start
|
||||
results <- r.TriggerNow()
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
|
||||
accepted := 0
|
||||
for i := 0; i < callers; i++ {
|
||||
if <-results {
|
||||
accepted++
|
||||
}
|
||||
}
|
||||
if accepted != 1 {
|
||||
t.Fatalf("accepted triggers = %d, want 1", accepted)
|
||||
}
|
||||
if got := r.Status(); got.State != "queued" || got.Running || !got.Queued {
|
||||
t.Fatalf("status = %#v, want one queued trigger", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package preview
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -20,15 +21,16 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
FFmpegPath string
|
||||
FFprobePath string
|
||||
DurationSeconds int // 兼容旧配置;当前 teaser 每段固定 3 秒
|
||||
DurationSeconds int // 兼容旧配置;当前预览视频每段固定 3 秒
|
||||
Width int
|
||||
Segments int // 兼容旧配置;当前 30 秒及以上视频固定使用 4 段
|
||||
LocalDir string // 本地 teaser 和封面目录
|
||||
LocalDir string // 本地预览视频和封面目录
|
||||
}
|
||||
|
||||
type Generator struct {
|
||||
@@ -235,23 +237,43 @@ func appendUniqueStart(starts []float64, start, eachSec float64) []float64 {
|
||||
return append(starts, start)
|
||||
}
|
||||
|
||||
// thumbnailOffsets 选封面抽帧的时间点(秒)。独立于 teaser。
|
||||
func thumbnailOffsets() []float64 {
|
||||
return []float64{5, 1, 0}
|
||||
// thumbnailOffsets 选封面抽帧的时间点(秒)。独立于预览视频。
|
||||
// 默认取视频中间帧;时长未知时退回早期帧。
|
||||
func thumbnailOffsets(duration float64) []float64 {
|
||||
if duration <= 0 {
|
||||
return []float64{5, 1, 0}
|
||||
}
|
||||
mid := duration / 2
|
||||
out := []float64{mid}
|
||||
for _, fallback := range []float64{5, 1, 0} {
|
||||
if !containsOffset(out, fallback) {
|
||||
out = append(out, fallback)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func containsOffset(offsets []float64, target float64) bool {
|
||||
for _, offset := range offsets {
|
||||
if math.Abs(offset-target) < 0.01 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// --- 封面 ---
|
||||
|
||||
// GenerateThumbnail 抽一张 jpg 封面。默认从第 5 秒抽帧,失败时回退到更早时间点。
|
||||
// GenerateThumbnail 抽一张 jpg 封面。默认从视频中间抽帧,失败时回退到更早时间点。
|
||||
func (g *Generator) GenerateThumbnail(ctx context.Context, link *drives.StreamLink, videoID string, duration float64) (string, error) {
|
||||
dir := filepath.Join(g.cfg.LocalDir, "thumbs")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
dst := filepath.Join(dir, videoID+".jpg")
|
||||
dst := mediaasset.ThumbnailPath(g.cfg.LocalDir, videoID)
|
||||
|
||||
var lastErr error
|
||||
offsets := thumbnailOffsets()
|
||||
offsets := thumbnailOffsets(duration)
|
||||
for i, offset := range offsets {
|
||||
if i > 0 {
|
||||
_ = os.Remove(dst)
|
||||
@@ -289,7 +311,7 @@ func (g *Generator) generateThumbnailAtOffset(ctx context.Context, link *drives.
|
||||
args = append(args,
|
||||
"-i", ffmpegLink.URL,
|
||||
"-frames:v", "1",
|
||||
"-vf", fmt.Sprintf("scale=%d:-2", g.cfg.Width),
|
||||
"-vf", thumbnailVideoFilter(g.cfg.Width),
|
||||
"-q:v", "3",
|
||||
"-y", dst,
|
||||
)
|
||||
@@ -307,6 +329,12 @@ func (g *Generator) generateThumbnailAtOffset(ctx context.Context, link *drives.
|
||||
return nil
|
||||
}
|
||||
|
||||
func thumbnailVideoFilter(width int) string {
|
||||
// FFmpeg 7 rejects non-full-range YUV for MJPEG/JPEG output. Force the
|
||||
// scaled frame into a JPEG-friendly full-range pixel format before encode.
|
||||
return fmt.Sprintf("scale=%d:-2:out_range=pc,format=yuvj420p", width)
|
||||
}
|
||||
|
||||
func thumbnailOffsetFallbackAllowed(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
@@ -339,9 +367,15 @@ func (g *Generator) Probe(ctx context.Context, link *drives.StreamLink) (float64
|
||||
args = append(args, ffmpegLink.URL)
|
||||
|
||||
cmd := exec.CommandContext(ctx2, g.cfg.FFprobePath, args...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return 0, ffmpegCommandError("ffprobe", err, out)
|
||||
errOut := stderr.Bytes()
|
||||
if len(errOut) == 0 {
|
||||
errOut = out
|
||||
}
|
||||
return 0, ffmpegCommandError("ffprobe", err, errOut)
|
||||
}
|
||||
raw := strings.TrimSpace(string(out))
|
||||
if raw == "" || raw == "N/A" {
|
||||
@@ -350,9 +384,9 @@ func (g *Generator) Probe(ctx context.Context, link *drives.StreamLink) (float64
|
||||
return strconv.ParseFloat(raw, 64)
|
||||
}
|
||||
|
||||
// --- Teaser ---
|
||||
// --- 预览视频 ---
|
||||
|
||||
// Generate 拉取 teaser 到本地临时文件,返回路径。
|
||||
// Generate 拉取预览视频到本地临时文件,返回路径。
|
||||
// 根据 Config.Segments 和视频时长决定是单段还是多段拼接。
|
||||
func (g *Generator) Generate(ctx context.Context, link *drives.StreamLink, duration float64) (string, error) {
|
||||
return g.generate(ctx, duration, func(int) (*drives.StreamLink, error) {
|
||||
@@ -923,6 +957,7 @@ func ffmpegOutputLooksRateLimited(output []byte) bool {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "throttl") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "rate-limit") ||
|
||||
strings.Contains(text, "server returned 429")
|
||||
@@ -932,7 +967,10 @@ func ffmpegOutputLooksRateLimited(output []byte) bool {
|
||||
|
||||
// MoveToLocal 把临时文件改名到稳定位置,返回最终路径
|
||||
func (g *Generator) MoveToLocal(tmpPath, videoID string) (string, error) {
|
||||
dst := filepath.Join(g.cfg.LocalDir, videoID+".mp4")
|
||||
if err := os.MkdirAll(g.cfg.LocalDir, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
dst := mediaasset.PreviewPath(g.cfg.LocalDir, videoID)
|
||||
if err := os.Rename(tmpPath, dst); err != nil {
|
||||
// 跨盘 rename 可能失败,fallback 到 copy
|
||||
if cerr := copyFile(tmpPath, dst); cerr != nil {
|
||||
@@ -968,7 +1006,6 @@ type Worker struct {
|
||||
queue videoQueue
|
||||
|
||||
RateLimitCooldown time.Duration
|
||||
BeforeTask func(context.Context) bool
|
||||
rateLimit rateLimitState
|
||||
activity taskActivity
|
||||
}
|
||||
@@ -978,7 +1015,7 @@ func NewWorker(gen TeaserGenerator, cat *catalog.Catalog, drv drives.Drive) *Wor
|
||||
Gen: gen,
|
||||
Catalog: cat,
|
||||
Drive: drv,
|
||||
ch: make(chan *catalog.Video, 4096),
|
||||
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1027,10 +1064,12 @@ type ThumbWorker struct {
|
||||
}
|
||||
|
||||
const (
|
||||
defaultTransientMediaCooldown = 5 * time.Minute
|
||||
defaultGenerationRateLimitCooldown = 5 * time.Minute
|
||||
maxPreviewTeaserSizeBytes int64 = 5 * 1024 * 1024 * 1024
|
||||
previewStatusSkipped = "skipped"
|
||||
defaultTransientMediaCooldown = 5 * time.Minute
|
||||
defaultGenerationRateLimitCooldown = 5 * time.Minute
|
||||
defaultThumbTransientMediaMaxFailures = 3
|
||||
defaultWorkerQueueSize = 10000
|
||||
maxPreviewTeaserSizeBytes int64 = 5 * 1024 * 1024 * 1024
|
||||
previewStatusSkipped = "skipped"
|
||||
)
|
||||
|
||||
type rateLimitState struct {
|
||||
@@ -1168,7 +1207,7 @@ func NewThumbWorker(gen ThumbnailGenerator, cat *catalog.Catalog, drv drives.Dri
|
||||
Gen: gen,
|
||||
Catalog: cat,
|
||||
Drive: drv,
|
||||
ch: make(chan *catalog.Video, 4096),
|
||||
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1323,26 +1362,32 @@ func (w *ThumbWorker) Run(ctx context.Context) {
|
||||
|
||||
func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
defer w.queue.release(v)
|
||||
if w.BeforeTask != nil && !w.BeforeTask(ctx) {
|
||||
if w.Catalog == nil || v == nil || v.ID == "" {
|
||||
return
|
||||
}
|
||||
|
||||
w.activity.start(v)
|
||||
current, err := w.Catalog.GetVideo(ctx, v.ID)
|
||||
if err != nil || current.Hidden {
|
||||
return
|
||||
}
|
||||
w.activity.start(current)
|
||||
defer w.activity.done()
|
||||
if !waitForRateLimitCooldown(ctx, &w.rateLimit, "preview", w.Drive) {
|
||||
return
|
||||
}
|
||||
w.process(ctx, v)
|
||||
w.process(ctx, current)
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
defer w.queue.release(v)
|
||||
w.activity.start(v)
|
||||
defer w.activity.done()
|
||||
if !waitForRateLimitCooldown(ctx, &w.rateLimit, "thumb", w.Drive) {
|
||||
return
|
||||
retry := false
|
||||
if waitForRateLimitCooldown(ctx, &w.rateLimit, "thumb", w.Drive) {
|
||||
retry = w.process(ctx, v)
|
||||
}
|
||||
w.activity.done()
|
||||
w.queue.release(v)
|
||||
if retry && ctx.Err() == nil {
|
||||
w.EnqueueBlocking(ctx, v)
|
||||
}
|
||||
w.process(ctx, v)
|
||||
}
|
||||
|
||||
func waitForRateLimitCooldown(ctx context.Context, state *rateLimitState, label string, drive drives.Drive) bool {
|
||||
@@ -1424,15 +1469,34 @@ func (w *ThumbWorker) pauseForRateLimit(err error, step, title string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) pauseForRecoverableError(err error, step, title string) bool {
|
||||
func (w *ThumbWorker) pauseForRecoverableError(ctx context.Context, v *catalog.Video, err error, step string) bool {
|
||||
title := ""
|
||||
videoID := ""
|
||||
if v != nil {
|
||||
title = v.Title
|
||||
videoID = v.ID
|
||||
}
|
||||
if w.pauseForRateLimit(err, step, title) {
|
||||
return true
|
||||
}
|
||||
if !driveErrorShouldCooldown(w.Drive, err) {
|
||||
return false
|
||||
}
|
||||
failures := 1
|
||||
if w.Catalog != nil && videoID != "" {
|
||||
count, countErr := w.Catalog.IncrementThumbnailFailures(ctx, videoID)
|
||||
if countErr != nil {
|
||||
log.Printf("[thumb] drive=%s transient media source error count failed step=%s video=%s: %v", w.Drive.ID(), step, title, countErr)
|
||||
} else {
|
||||
failures = count
|
||||
}
|
||||
}
|
||||
if failures >= defaultThumbTransientMediaMaxFailures {
|
||||
log.Printf("[thumb] drive=%s transient media source error reached retry limit failures=%d/%d step=%s video=%s: %v", w.Drive.ID(), failures, defaultThumbTransientMediaMaxFailures, step, title, err)
|
||||
return false
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), w.RateLimitCooldown)
|
||||
log.Printf("[thumb] drive=%s transient media source error until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err)
|
||||
log.Printf("[thumb] drive=%s transient media source error until=%s failures=%d/%d step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), failures, defaultThumbTransientMediaMaxFailures, step, title, err)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -1453,7 +1517,7 @@ func driveErrorShouldCooldown(d drives.Drive, err error) bool {
|
||||
strings.Contains(text, "request has been blocked") ||
|
||||
strings.Contains(text, "访问被阻断")
|
||||
case "pikpak":
|
||||
// PikPak 在 teaser / 封面生成阶段(取链或拉直链字节)可能命中:
|
||||
// PikPak 在预览视频 / 封面生成阶段(取链或拉直链字节)可能命中:
|
||||
// - error_code=10 操作频繁
|
||||
// - HTTP 429 / 5xx / 509 限流和服务端不可用
|
||||
// - 通用文本:rate limit / too many requests / blocked
|
||||
@@ -1471,65 +1535,171 @@ func driveErrorShouldCooldown(d drives.Drive, err error) bool {
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "blocked") ||
|
||||
strings.Contains(text, "moov atom not found") ||
|
||||
strings.Contains(text, "partial file") ||
|
||||
strings.Contains(text, "service unavailable")
|
||||
case "p123":
|
||||
// 123 云盘直链解析 / ffmpeg 读取阶段可能返回 429、5xx,或 WAF 类
|
||||
// blocked / 访问阻断文本。命中时冷却,避免封面和预览视频生成连续打接口。
|
||||
text := strings.ToLower(err.Error())
|
||||
return strings.Contains(text, "请求太频繁") ||
|
||||
strings.Contains(text, "请求过于频繁") ||
|
||||
strings.Contains(text, "请求频繁") ||
|
||||
strings.Contains(text, "操作频繁") ||
|
||||
strings.Contains(text, "频率限制") ||
|
||||
strings.Contains(text, "请求次数过多") ||
|
||||
strings.Contains(text, "429") ||
|
||||
strings.Contains(text, "http 500") ||
|
||||
strings.Contains(text, "http 502") ||
|
||||
strings.Contains(text, "http 503") ||
|
||||
strings.Contains(text, "http 504") ||
|
||||
strings.Contains(text, "server returned 403") ||
|
||||
strings.Contains(text, "403 forbidden") ||
|
||||
strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "blocked") ||
|
||||
strings.Contains(text, "访问被阻断") ||
|
||||
strings.Contains(text, "service unavailable")
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
|
||||
func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) bool {
|
||||
if w.skipIfRateLimited(v) {
|
||||
return
|
||||
return false
|
||||
}
|
||||
if current, err := w.Catalog.GetVideo(ctx, v.ID); err == nil {
|
||||
if current.ThumbnailURL != "" {
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
|
||||
return
|
||||
if w.Catalog == nil || v == nil || v.ID == "" {
|
||||
return false
|
||||
}
|
||||
queued := v
|
||||
loaded, err := w.Catalog.GetVideo(ctx, v.ID)
|
||||
if err != nil || loaded.Hidden {
|
||||
return false
|
||||
}
|
||||
if loaded.PreviewLocal == "" {
|
||||
loaded.PreviewLocal = queued.PreviewLocal
|
||||
}
|
||||
current := loaded
|
||||
v = loaded
|
||||
if loaded.ThumbnailURL != "" && loaded.DurationSeconds > 0 {
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
|
||||
return false
|
||||
}
|
||||
if current.ThumbnailURL != "" {
|
||||
durationBackfillFailed := false
|
||||
if current.DurationSeconds <= 0 {
|
||||
link, err := w.streamLink(ctx, current)
|
||||
if err != nil {
|
||||
if w.pauseForRecoverableError(ctx, current, err, "streamURL") {
|
||||
return true
|
||||
}
|
||||
log.Printf("[thumb] probe streamURL %s: %v", current.Title, err)
|
||||
durationBackfillFailed = true
|
||||
} else if w.probeDuration(ctx, current, link) {
|
||||
return true
|
||||
} else if current.DurationSeconds <= 0 {
|
||||
durationBackfillFailed = true
|
||||
}
|
||||
}
|
||||
if durationBackfillFailed {
|
||||
log.Printf("[thumb] skip duration backfill %s: thumbnail already exists but duration could not be probed", current.Title)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, current.ID, catalog.VideoMetaPatch{ThumbnailStatus: "skipped"})
|
||||
return false
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, current.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
|
||||
return false
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "pending"})
|
||||
link, err := w.Drive.StreamURL(ctx, v.FileID)
|
||||
if isSpider91OriginVideo(v) {
|
||||
log.Printf("[thumb] skip %s: spider91-origin video must use crawled thumbnail", v.Title)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
|
||||
return false
|
||||
}
|
||||
link, err := w.streamLink(ctx, v)
|
||||
if err != nil {
|
||||
if localLink, ok := localPreviewLink(v); ok {
|
||||
link = localLink
|
||||
} else {
|
||||
if w.pauseForRecoverableError(err, "streamURL", v.Title) {
|
||||
return
|
||||
}
|
||||
log.Printf("[thumb] streamURL %s: %v", v.Title, err)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
|
||||
return
|
||||
if w.pauseForRecoverableError(ctx, v, err, "streamURL") {
|
||||
return true
|
||||
}
|
||||
log.Printf("[thumb] streamURL %s: %v", v.Title, err)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
|
||||
return false
|
||||
}
|
||||
if w.probeDuration(ctx, v, link) {
|
||||
return true
|
||||
}
|
||||
|
||||
if err := w.generateThumbnailFromLink(ctx, v, link); err != nil {
|
||||
if localLink, ok := localPreviewLink(v); ok && link.URL != localLink.URL {
|
||||
if w.probeDuration(ctx, v, localLink) {
|
||||
return true
|
||||
}
|
||||
if localErr := w.generateThumbnailFromLink(ctx, v, localLink); localErr == nil {
|
||||
return
|
||||
return false
|
||||
}
|
||||
}
|
||||
if w.pauseForRecoverableError(err, "generate", v.Title) {
|
||||
return
|
||||
if w.pauseForRecoverableError(ctx, v, err, "generate") {
|
||||
return true
|
||||
}
|
||||
log.Printf("[thumb] generate %s: %v", v.Title, err)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
|
||||
return
|
||||
return false
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) streamLink(ctx context.Context, v *catalog.Video) (*drives.StreamLink, error) {
|
||||
link, err := w.Drive.StreamURL(ctx, v.FileID)
|
||||
if err == nil {
|
||||
return link, nil
|
||||
}
|
||||
if localLink, ok := localPreviewLink(v); ok {
|
||||
return localLink, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) probeDuration(ctx context.Context, v *catalog.Video, link *drives.StreamLink) bool {
|
||||
if v.DurationSeconds > 0 {
|
||||
return false
|
||||
}
|
||||
dur, err := w.Gen.Probe(ctx, link)
|
||||
if err == nil {
|
||||
if dur > 0 {
|
||||
v.DurationSeconds = int(dur)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
DurationSeconds: int(dur),
|
||||
})
|
||||
}
|
||||
return false
|
||||
}
|
||||
if w.pauseForRecoverableError(ctx, v, err, "probe") {
|
||||
return true
|
||||
}
|
||||
log.Printf("[thumb] probe %s: %v", v.Title, err)
|
||||
return false
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) generateThumbnailFromLink(ctx context.Context, v *catalog.Video, link *drives.StreamLink) error {
|
||||
if _, err := w.Gen.GenerateThumbnail(ctx, link, v.ID, 0); err != nil {
|
||||
local, err := w.Gen.GenerateThumbnail(ctx, link, v.ID, float64(v.DurationSeconds))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
if err := w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
ThumbnailURL: "/p/thumb/" + v.ID,
|
||||
ThumbnailStatus: "ready",
|
||||
})
|
||||
}); err != nil {
|
||||
_ = os.Remove(local)
|
||||
log.Printf("[thumb] update %s after generate: %v", v.Title, err)
|
||||
return nil
|
||||
}
|
||||
log.Printf("[thumb] ready %s", v.Title)
|
||||
return nil
|
||||
}
|
||||
|
||||
func isSpider91OriginVideo(v *catalog.Video) bool {
|
||||
return v != nil && strings.HasPrefix(v.ID, "spider91-")
|
||||
}
|
||||
|
||||
func localPreviewLink(v *catalog.Video) (*drives.StreamLink, bool) {
|
||||
if v.PreviewLocal == "" {
|
||||
return nil, false
|
||||
@@ -1578,7 +1748,7 @@ func (w *Worker) process(ctx context.Context, v *catalog.Video) {
|
||||
}
|
||||
}
|
||||
|
||||
// 2) teaser
|
||||
// 2) 预览视频
|
||||
tmp, err := w.generateTeaser(ctx, v, link, duration)
|
||||
if err != nil {
|
||||
if w.pauseForRecoverableError(err, "generate", v.Title) {
|
||||
@@ -1596,7 +1766,11 @@ func (w *Worker) process(ctx context.Context, v *catalog.Video) {
|
||||
}
|
||||
|
||||
removePreviousLocalTeaser(v.PreviewLocal, local)
|
||||
w.Catalog.UpdatePreview(ctx, v.ID, local, "ready")
|
||||
if err := w.Catalog.UpdatePreview(ctx, v.ID, local, "ready"); err != nil {
|
||||
removePreviousLocalTeaser(local, "")
|
||||
log.Printf("[preview] update %s after generate: %v", v.Title, err)
|
||||
return
|
||||
}
|
||||
log.Printf("[preview] ready %s (duration=%.1fs)", v.Title, duration)
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"errors"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -95,6 +97,24 @@ func TestTinyVideoPreviewPlanUsesWholeVideoAsSingleSegment(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeIgnoresStderrWarnings(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ffprobePath := filepath.Join(dir, "ffprobe")
|
||||
script := "#!/bin/sh\nprintf '%s\\n' 'h264 warning' >&2\nprintf '%s\\n' '364.800000'\n"
|
||||
if err := os.WriteFile(ffprobePath, []byte(script), 0o755); err != nil {
|
||||
t.Fatalf("write ffprobe stub: %v", err)
|
||||
}
|
||||
|
||||
gen := New(Config{FFprobePath: ffprobePath})
|
||||
got, err := gen.Probe(context.Background(), &drives.StreamLink{URL: filepath.Join(dir, "video.mp4")})
|
||||
if err != nil {
|
||||
t.Fatalf("probe: %v", err)
|
||||
}
|
||||
if got != 364.8 {
|
||||
t.Fatalf("duration = %v, want 364.8", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTeaserCandidateStartsKeepPrimaryAndAddFallbacks(t *testing.T) {
|
||||
primary := []float64{10.2, 64.65, 119.1, 173.55}
|
||||
got := teaserCandidateStarts(204, primary, 3)
|
||||
@@ -148,16 +168,39 @@ func TestMediumAndLongVideosStillRequirePlannedTeaserSegments(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailOffsetsUseFiveSecondsWithEarlyFallbacks(t *testing.T) {
|
||||
got := thumbnailOffsets()
|
||||
want := []float64{5, 1, 0}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("offsets = %#v, want %#v", got, want)
|
||||
func TestThumbnailOffsetsPreferMiddleFrame(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
duration float64
|
||||
want []float64
|
||||
}{
|
||||
{name: "unknown duration", duration: 0, want: []float64{5, 1, 0}},
|
||||
{name: "long video", duration: 2804.9, want: []float64{1402.45, 5, 1, 0}},
|
||||
{name: "short video", duration: 8.9, want: []float64{4.45, 5, 1, 0}},
|
||||
{name: "middle equals fallback", duration: 10, want: []float64{5, 1, 0}},
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("offset[%d] = %.2f, want %.2f", i, got[i], want[i])
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := thumbnailOffsets(tt.duration)
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("offsets = %#v, want %#v", got, tt.want)
|
||||
}
|
||||
for i := range tt.want {
|
||||
if math.Abs(got[i]-tt.want[i]) > 0.001 {
|
||||
t.Fatalf("offset[%d] = %.2f, want %.2f", i, got[i], tt.want[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailVideoFilterUsesFullRangeJPEGPixelFormat(t *testing.T) {
|
||||
got := thumbnailVideoFilter(480)
|
||||
if !strings.Contains(got, "scale=480:-2:out_range=pc") {
|
||||
t.Fatalf("thumbnail filter = %q, want full-range scale output", got)
|
||||
}
|
||||
if !strings.Contains(got, "format=yuvj420p") {
|
||||
t.Fatalf("thumbnail filter = %q, want JPEG-friendly pixel format", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,11 +13,11 @@ import (
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
|
||||
func TestThumbWorkerUpdatesThumbnailAndDurationWithoutChangingPreviewStatus(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-video")
|
||||
|
||||
gen := &fakeThumbGenerator{}
|
||||
gen := &fakeThumbGenerator{probeDuration: 42}
|
||||
drv := &previewFakeDrive{}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
@@ -33,23 +33,131 @@ func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
|
||||
if got.PreviewStatus != "pending" {
|
||||
t.Fatalf("preview status = %q, want pending", got.PreviewStatus)
|
||||
}
|
||||
if got.DurationSeconds != 0 {
|
||||
t.Fatalf("duration = %d, want unchanged", got.DurationSeconds)
|
||||
if got.DurationSeconds != 42 {
|
||||
t.Fatalf("duration = %d, want probed duration", got.DurationSeconds)
|
||||
}
|
||||
if gen.thumbnailVideoID != video.ID {
|
||||
t.Fatalf("thumbnail video id = %q, want %q", gen.thumbnailVideoID, video.ID)
|
||||
}
|
||||
if gen.thumbnailDuration != 0 {
|
||||
t.Fatalf("thumbnail duration = %.1f, want fixed-offset thumbnail generation", gen.thumbnailDuration)
|
||||
if gen.thumbnailDuration != 42 {
|
||||
t.Fatalf("thumbnail duration = %.1f, want probed duration", gen.thumbnailDuration)
|
||||
}
|
||||
if gen.probeCalls != 0 {
|
||||
t.Fatalf("probe calls = %d, want 0 for thumbnail generation", gen.probeCalls)
|
||||
if gen.probeCalls != 1 {
|
||||
t.Fatalf("probe calls = %d, want 1 for thumbnail generation", gen.probeCalls)
|
||||
}
|
||||
if drv.streamFileID != video.FileID {
|
||||
t.Fatalf("stream file id = %q, want %q", drv.streamFileID, video.FileID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerBackfillsDurationWhenThumbnailAlreadyExists(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-existing-thumbnail")
|
||||
video.ThumbnailURL = "/p/thumb/" + video.ID
|
||||
if err := cat.UpsertVideo(ctx, video); err != nil {
|
||||
t.Fatalf("update video: %v", err)
|
||||
}
|
||||
|
||||
gen := &fakeThumbGenerator{probeDuration: 19}
|
||||
drv := &previewFakeDrive{}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
worker.process(ctx, video)
|
||||
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DurationSeconds != 19 {
|
||||
t.Fatalf("duration = %d, want probed duration", got.DurationSeconds)
|
||||
}
|
||||
if got.ThumbnailURL != "/p/thumb/"+video.ID {
|
||||
t.Fatalf("thumbnail = %q, want unchanged existing thumbnail", got.ThumbnailURL)
|
||||
}
|
||||
ready, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "ready", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list ready thumbnails: %v", err)
|
||||
}
|
||||
if len(ready) != 1 || ready[0].ID != video.ID {
|
||||
t.Fatalf("ready thumbnails = %#v, want only %s", ready, video.ID)
|
||||
}
|
||||
if gen.probeCalls != 1 {
|
||||
t.Fatalf("probe calls = %d, want 1", gen.probeCalls)
|
||||
}
|
||||
if gen.thumbnailVideoID != "" {
|
||||
t.Fatalf("thumbnail generation video id = %q, want no regeneration", gen.thumbnailVideoID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerDoesNotGenerateThumbnailForSpider91OriginVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "spider91-91-spider-1200001")
|
||||
|
||||
gen := &fakeThumbGenerator{probeDuration: 42}
|
||||
drv := &previewFakeDrive{kind: "pikpak"}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
worker.process(ctx, video)
|
||||
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want empty when crawled spider91 thumbnail is missing", got.ThumbnailURL)
|
||||
}
|
||||
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list failed thumbnails: %v", err)
|
||||
}
|
||||
if len(failed) != 1 || failed[0].ID != video.ID {
|
||||
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
|
||||
}
|
||||
if gen.probeCalls != 0 || gen.generateCalls != 0 {
|
||||
t.Fatalf("generator calls probe=%d generate=%d, want no ffmpeg work for spider91-origin thumbnail", gen.probeCalls, gen.generateCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerSkipsDurationBackfillWhenExistingThumbnailCannotBeProbed(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-existing-thumbnail-probe-fails")
|
||||
video.ThumbnailURL = "/p/thumb/" + video.ID
|
||||
if err := cat.UpsertVideo(ctx, video); err != nil {
|
||||
t.Fatalf("update video: %v", err)
|
||||
}
|
||||
|
||||
gen := &fakeThumbGenerator{probeErr: errors.New("invalid media")}
|
||||
drv := &previewFakeDrive{}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
worker.process(ctx, video)
|
||||
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "/p/thumb/"+video.ID {
|
||||
t.Fatalf("thumbnail = %q, want unchanged existing thumbnail", got.ThumbnailURL)
|
||||
}
|
||||
if got.DurationSeconds != 0 {
|
||||
t.Fatalf("duration = %d, want still unknown", got.DurationSeconds)
|
||||
}
|
||||
skipped, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "skipped", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list skipped thumbnails: %v", err)
|
||||
}
|
||||
if len(skipped) != 1 || skipped[0].ID != video.ID {
|
||||
t.Fatalf("skipped thumbnails = %#v, want only %s", skipped, video.ID)
|
||||
}
|
||||
missing, err := cat.CountVideosNeedingThumbnail(ctx, video.DriveID)
|
||||
if err != nil {
|
||||
t.Fatalf("count videos needing thumbnail: %v", err)
|
||||
}
|
||||
if missing != 0 {
|
||||
t.Fatalf("missing thumbnails = %d, want 0 after duration backfill is skipped", missing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerFallsBackToLocalPreviewWhenDriveStreamFails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-local-preview")
|
||||
@@ -377,6 +485,142 @@ func TestThumbWorkerRateLimitCoolsDownFiveMinutes(t *testing.T) {
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 5*time.Minute)
|
||||
}
|
||||
|
||||
func TestThumbWorkerP115TransientErrorFailsAfterRetryLimit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-p115-transient")
|
||||
|
||||
gen := &fakeThumbGenerator{
|
||||
generateErr: errors.New("ffmpeg thumb: exit status 183, stderr: partial file Cannot determine format of input 0:0 after EOF"),
|
||||
}
|
||||
drv := &previewFakeDrive{kind: "p115"}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
for attempt := 1; attempt <= defaultThumbTransientMediaMaxFailures; attempt++ {
|
||||
worker.rateLimit = rateLimitState{}
|
||||
worker.process(ctx, video)
|
||||
|
||||
if attempt < defaultThumbTransientMediaMaxFailures {
|
||||
pending, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "pending", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list pending thumbnails: %v", err)
|
||||
}
|
||||
if len(pending) != 1 || pending[0].ID != video.ID {
|
||||
t.Fatalf("attempt %d pending thumbnails = %#v, want only %s", attempt, pending, video.ID)
|
||||
}
|
||||
missing, err := cat.CountVideosNeedingThumbnail(ctx, video.DriveID)
|
||||
if err != nil {
|
||||
t.Fatalf("count missing thumbnails: %v", err)
|
||||
}
|
||||
if missing != 1 {
|
||||
t.Fatalf("attempt %d missing thumbnails = %d, want 1 before retry limit", attempt, missing)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list failed thumbnails: %v", err)
|
||||
}
|
||||
if len(failed) != 1 || failed[0].ID != video.ID {
|
||||
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
|
||||
}
|
||||
missing, err := cat.CountVideosNeedingThumbnail(ctx, video.DriveID)
|
||||
if err != nil {
|
||||
t.Fatalf("count missing thumbnails: %v", err)
|
||||
}
|
||||
if missing != 0 {
|
||||
t.Fatalf("missing thumbnails = %d, want 0 after retry limit marks failed", missing)
|
||||
}
|
||||
}
|
||||
|
||||
if gen.generateCalls != defaultThumbTransientMediaMaxFailures {
|
||||
t.Fatalf("generate calls = %d, want %d", gen.generateCalls, defaultThumbTransientMediaMaxFailures)
|
||||
}
|
||||
|
||||
if err := cat.UpdateVideoMeta(ctx, video.ID, catalog.VideoMetaPatch{
|
||||
ThumbnailStatus: "pending",
|
||||
ResetThumbnailFailures: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("reset thumbnail status: %v", err)
|
||||
}
|
||||
worker.rateLimit = rateLimitState{}
|
||||
worker.process(ctx, video)
|
||||
|
||||
pending, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "pending", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list pending thumbnails after reset: %v", err)
|
||||
}
|
||||
if len(pending) != 1 || pending[0].ID != video.ID {
|
||||
t.Fatalf("pending thumbnails after reset = %#v, want only %s", pending, video.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerRequeuesP115TransientErrorBeforeRetryLimit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-p115-requeue")
|
||||
|
||||
gen := &fakeThumbGenerator{
|
||||
generateErr: errors.New("ffmpeg thumb: partial file Cannot determine format of input 0:0 after EOF"),
|
||||
}
|
||||
drv := &previewFakeDrive{kind: "p115"}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
worker.processQueued(ctx, video)
|
||||
|
||||
select {
|
||||
case queued := <-worker.ch:
|
||||
if queued.ID != video.ID {
|
||||
t.Fatalf("requeued video id = %q, want %q", queued.ID, video.ID)
|
||||
}
|
||||
default:
|
||||
t.Fatal("expected transient thumbnail failure to requeue the same video")
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want empty after transient failure", got.ThumbnailURL)
|
||||
}
|
||||
pending, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "pending", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list pending thumbnails: %v", err)
|
||||
}
|
||||
if len(pending) != 1 || pending[0].ID != video.ID {
|
||||
t.Fatalf("pending thumbnails = %#v, want only %s", pending, video.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerPikPakMoovAtomErrorFailsWithoutCooldown(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-pikpak-missing-moov")
|
||||
|
||||
mediaErr := errors.New("ffprobe: exit status 1, stderr: moov atom not found Invalid data found when processing input")
|
||||
gen := &fakeThumbGenerator{
|
||||
probeErr: mediaErr,
|
||||
generateErr: mediaErr,
|
||||
}
|
||||
drv := &previewFakeDrive{kind: "pikpak"}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
worker.process(ctx, video)
|
||||
|
||||
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list failed thumbnails: %v", err)
|
||||
}
|
||||
if len(failed) != 1 || failed[0].ID != video.ID {
|
||||
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
|
||||
}
|
||||
if !worker.Status().CooldownUntil.IsZero() {
|
||||
t.Fatalf("cooldown until = %s, want no cooldown for invalid PikPak MP4", worker.Status().CooldownUntil)
|
||||
}
|
||||
if gen.generateCalls != 1 {
|
||||
t.Fatalf("generate calls = %d, want 1", gen.generateCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewWorkerP115TransientErrorKeepsVideoPending(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "preview-p115-transient")
|
||||
@@ -401,6 +645,22 @@ func TestPreviewWorkerP115TransientErrorKeepsVideoPending(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestP123TransientErrorsShouldCooldown(t *testing.T) {
|
||||
drv := &previewFakeDrive{kind: "p123"}
|
||||
for _, err := range []error{
|
||||
errors.New("Server returned 403 Forbidden"),
|
||||
errors.New("请求太频繁"),
|
||||
errors.New("http 503 service unavailable"),
|
||||
} {
|
||||
if !driveErrorShouldCooldown(drv, err) {
|
||||
t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err)
|
||||
}
|
||||
}
|
||||
if driveErrorShouldCooldown(drv, errors.New("invalid credential")) {
|
||||
t.Fatal("invalid credential should not trigger p123 cooldown")
|
||||
}
|
||||
}
|
||||
|
||||
func assertCooldownAround(t *testing.T, until time.Time, before time.Time, want time.Duration) {
|
||||
t.Helper()
|
||||
if until.IsZero() {
|
||||
@@ -469,15 +729,22 @@ type fakeThumbGenerator struct {
|
||||
thumbnailDuration float64
|
||||
thumbnailURL string
|
||||
probeCalls int
|
||||
generateCalls int
|
||||
probeDuration float64
|
||||
probeErr error
|
||||
generateErr error
|
||||
}
|
||||
|
||||
func (g *fakeThumbGenerator) Probe(context.Context, *drives.StreamLink) (float64, error) {
|
||||
g.probeCalls++
|
||||
return 42, nil
|
||||
if g.probeErr != nil {
|
||||
return 0, g.probeErr
|
||||
}
|
||||
return g.probeDuration, nil
|
||||
}
|
||||
|
||||
func (g *fakeThumbGenerator) GenerateThumbnail(_ context.Context, link *drives.StreamLink, videoID string, duration float64) (string, error) {
|
||||
g.generateCalls++
|
||||
g.thumbnailVideoID = videoID
|
||||
g.thumbnailDuration = duration
|
||||
if link != nil {
|
||||
@@ -568,7 +835,6 @@ func (d *previewFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
}
|
||||
func (d *previewFakeDrive) RootID() string { return "root" }
|
||||
|
||||
|
||||
func TestWorkerWaitIdleReturnsImmediatelyWhenQueueEmpty(t *testing.T) {
|
||||
worker := NewWorker(&fakeTeaserGenerator{}, nil, &previewFakeDrive{})
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -144,13 +145,17 @@ func (p *Proxy) ServeStream(w http.ResponseWriter, r *http.Request, driveID, fil
|
||||
// 302 之后浏览器用自己的 UA 直连,CDN 仍然认签名
|
||||
// - pikpak:与 OpenList 一致,WebContentLink / media link 都是自签 URL,
|
||||
// CDN 不校验请求头,直连可获得最佳带宽并避免占用 backend 出站
|
||||
// - onedrive:Microsoft Graph 返回的 @microsoft.graph.downloadUrl 是短期
|
||||
// 免鉴权下载 URL,不需要后端继续代传视频字节
|
||||
// - p123:123 云盘 download_info 返回的下载页会再跳 CDN;driver 已在后端
|
||||
// 先解出最终 Location,浏览器可直接 302 到该短期地址
|
||||
//
|
||||
// 其余网盘(如 OneDrive / 沃盘 / 夸克等)仍走反代,因为它们的下载
|
||||
// 其余网盘(如沃盘 / 夸克等)仍走反代,因为它们的下载
|
||||
// 链接通常需要随请求带上后端持有的 Cookie / Authorization / Range
|
||||
// 的特殊处理,浏览器拿不到这些上下文。
|
||||
func shouldRedirect(d drives.Drive) bool {
|
||||
switch d.Kind() {
|
||||
case "p115", "pikpak":
|
||||
case "p115", "pikpak", "onedrive", "p123":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -169,6 +174,11 @@ func (p *Proxy) serve(w http.ResponseWriter, r *http.Request, link *drives.Strea
|
||||
http.Error(w, "bad upstream url", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
if localPath, ok := localFilePath(u, link.URL); ok {
|
||||
w.Header().Set("Cache-Control", "private, max-age=300")
|
||||
http.ServeFile(w, r, localPath)
|
||||
return
|
||||
}
|
||||
req, err := http.NewRequestWithContext(r.Context(), r.Method, u.String(), nil)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
@@ -206,11 +216,24 @@ func (p *Proxy) serve(w http.ResponseWriter, r *http.Request, link *drives.Strea
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
// ServeLocal 服务本地 teaser 文件
|
||||
// ServeLocal 服务本地预览视频文件
|
||||
func (p *Proxy) ServeLocal(w http.ResponseWriter, r *http.Request, path string) {
|
||||
http.ServeFile(w, r, path)
|
||||
}
|
||||
|
||||
func localFilePath(u *url.URL, raw string) (string, bool) {
|
||||
if u == nil {
|
||||
return "", false
|
||||
}
|
||||
if u.Scheme == "file" && u.Path != "" {
|
||||
return u.Path, true
|
||||
}
|
||||
if u.Scheme == "" && u.Host == "" && filepath.IsAbs(raw) {
|
||||
return raw, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
var errDriveNotFound = &httpError{Code: http.StatusNotFound, Msg: "drive not found"}
|
||||
|
||||
type httpError struct {
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -149,6 +151,86 @@ func TestServeStreamPikPakSetsRedirectHeaders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamRedirectsOneDrive(t *testing.T) {
|
||||
reg := NewRegistry()
|
||||
drv := &proxyFakeSimpleDrive{
|
||||
kind: "onedrive",
|
||||
url: "https://public.onedrive.example/video.mp4",
|
||||
}
|
||||
reg.Set("onedrive", drv)
|
||||
|
||||
p := New(reg)
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/onedrive/file-1", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
p.ServeStream(rr, req, "onedrive", "file-1")
|
||||
|
||||
if rr.Code != http.StatusFound {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusFound)
|
||||
}
|
||||
if got := rr.Header().Get("Location"); got != "https://public.onedrive.example/video.mp4" {
|
||||
t.Fatalf("Location = %q", got)
|
||||
}
|
||||
if drv.calls != 1 {
|
||||
t.Fatalf("link calls = %d, want 1", drv.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamRedirectsP123(t *testing.T) {
|
||||
reg := NewRegistry()
|
||||
drv := &proxyFakeSimpleDrive{
|
||||
kind: "p123",
|
||||
url: "https://cdn.123pan.example/video.mp4",
|
||||
}
|
||||
reg.Set("p123", drv)
|
||||
|
||||
p := New(reg)
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/p123/file-1", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
p.ServeStream(rr, req, "p123", "file-1")
|
||||
|
||||
if rr.Code != http.StatusFound {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusFound)
|
||||
}
|
||||
if got := rr.Header().Get("Location"); got != "https://cdn.123pan.example/video.mp4" {
|
||||
t.Fatalf("Location = %q", got)
|
||||
}
|
||||
if drv.calls != 1 {
|
||||
t.Fatalf("link calls = %d, want 1", drv.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamServesLocalFilePath(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "video.mp4")
|
||||
if err := os.WriteFile(path, []byte("0123456789"), 0o644); err != nil {
|
||||
t.Fatalf("write local file: %v", err)
|
||||
}
|
||||
reg := NewRegistry()
|
||||
drv := &proxyFakeSimpleDrive{
|
||||
kind: "localstorage",
|
||||
url: path,
|
||||
}
|
||||
reg.Set("local", drv)
|
||||
|
||||
p := New(reg)
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/local/file-1", nil)
|
||||
req.Header.Set("Range", "bytes=2-5")
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
p.ServeStream(rr, req, "local", "file-1")
|
||||
|
||||
if rr.Code != http.StatusPartialContent {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusPartialContent)
|
||||
}
|
||||
if got := rr.Body.String(); got != "2345" {
|
||||
t.Fatalf("body = %q, want range bytes", got)
|
||||
}
|
||||
if drv.calls != 1 {
|
||||
t.Fatalf("link calls = %d, want 1", drv.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func requestPikPak(t *testing.T, p *Proxy, driveID, fileID, ua string) {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/"+driveID+"/"+fileID, nil)
|
||||
@@ -192,3 +274,36 @@ func (d *proxyFakePikPakDrive) EnsureDir(context.Context, string) (string, error
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakePikPakDrive) RootID() string { return "0" }
|
||||
|
||||
type proxyFakeSimpleDrive struct {
|
||||
kind string
|
||||
url string
|
||||
calls int
|
||||
}
|
||||
|
||||
func (d *proxyFakeSimpleDrive) Kind() string { return d.kind }
|
||||
func (d *proxyFakeSimpleDrive) ID() string { return d.kind }
|
||||
func (d *proxyFakeSimpleDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
|
||||
d.calls++
|
||||
return &drives.StreamLink{
|
||||
URL: d.url,
|
||||
Headers: http.Header{},
|
||||
Expires: time.Now().Add(10 * time.Minute),
|
||||
}, nil
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) RootID() string { return "0" }
|
||||
|
||||
@@ -16,11 +16,11 @@ type ParsedName struct {
|
||||
}
|
||||
|
||||
var (
|
||||
reTags = regexp.MustCompile(`^\s*\[([^\]]+)\]\s*`) // [tag1,tag2]
|
||||
reTags = regexp.MustCompile(`^\s*\[([^\]]+)\]\s*`) // [前缀]
|
||||
reAuthor = regexp.MustCompile(`\s*-\s*([^-]+?)\s*$`) // - author
|
||||
)
|
||||
|
||||
// Parse 按约定解析:[tag1,tag2] 标题 - 作者.ext
|
||||
// Parse 按约定解析:[前缀] 标题 - 作者.ext
|
||||
// 任何字段缺失都能降级
|
||||
func Parse(filename string) ParsedName {
|
||||
name := strings.TrimSuffix(filename, path.Ext(filename))
|
||||
|
||||
@@ -23,7 +23,7 @@ type Scanner struct {
|
||||
//
|
||||
// nil / 空集合 → 行为等同于不跳过任何目录。
|
||||
SkipDirIDs map[string]struct{}
|
||||
// 回调:新视频被加入后触发 teaser 生成
|
||||
// 回调:新视频被加入后触发预览视频生成
|
||||
OnNewVideo func(v *catalog.Video)
|
||||
// ProgressInterval 控制扫描内部 heartbeat 的最小输出间隔。
|
||||
// 0 → 默认 30s;< 0 → 关闭 heartbeat(仅留外层 start / done 两行)。
|
||||
@@ -127,8 +127,11 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
}
|
||||
|
||||
for _, e := range entries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if e.IsDir {
|
||||
// 跳过 previews 目录,避免扫到自己生成的 teaser
|
||||
// 跳过 previews 目录,避免扫到自己生成的预览视频
|
||||
if strings.EqualFold(e.Name, "previews") {
|
||||
continue
|
||||
}
|
||||
@@ -137,6 +140,9 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
continue
|
||||
}
|
||||
if err := s.walk(ctx, e.ID, e.Name, stats, progress); err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return ctxErr
|
||||
}
|
||||
stats.Errors++
|
||||
log.Printf("[scanner] walk %s error: %v", e.Name, err)
|
||||
}
|
||||
@@ -154,6 +160,17 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
stats.SeenFileIDs[e.ID] = struct{}{}
|
||||
|
||||
id := s.Drive.Kind() + "-" + s.Drive.ID() + "-" + e.ID
|
||||
if deleted, err := s.Catalog.IsDeletedVideoCandidate(ctx, id, s.Drive.ID(), e.ID, e.Hash, e.Name, e.Size); err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return ctxErr
|
||||
}
|
||||
stats.Errors++
|
||||
log.Printf("[scanner] check deleted video %s error: %v", id, err)
|
||||
continue
|
||||
} else if deleted {
|
||||
continue
|
||||
}
|
||||
|
||||
parsed := Parse(e.Name)
|
||||
if parsed.Title == "" {
|
||||
parsed.Title = strings.TrimSuffix(e.Name, ext)
|
||||
@@ -162,11 +179,20 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
if matched, err := s.Catalog.MatchTags(ctx, e.Name+" "+dirName+" "+parsed.Author); err == nil {
|
||||
tags = mergeTags(tags, matched)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if label, ok, err := s.Catalog.EnsureCollectionTag(ctx, dirName); err == nil && ok {
|
||||
tags = mergeTags(tags, []string{label})
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
existing, _ := s.Catalog.GetVideo(ctx, id)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if existing != nil {
|
||||
patch := catalog.VideoMetaPatch{}
|
||||
if e.Hash != "" && existing.ContentHash == "" {
|
||||
@@ -181,26 +207,33 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
if existing.Category == "" && dirName != "" {
|
||||
patch.Category = dirName
|
||||
}
|
||||
if existing.ThumbnailURL == "" && e.ThumbnailURL != "" {
|
||||
patch.ThumbnailURL = e.ThumbnailURL
|
||||
}
|
||||
if patch.Category != "" || patch.ThumbnailURL != "" || patch.ContentHash != "" || patch.FileName != "" {
|
||||
if patch.Category != "" || patch.ContentHash != "" || patch.FileName != "" {
|
||||
_ = s.Catalog.UpdateVideoMeta(ctx, id, patch)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if dup := s.findDuplicate(ctx, e.Hash, e.Name, e.Size, id); dup != nil {
|
||||
s.backfillDuplicateThumbnail(ctx, dup, e.ThumbnailURL)
|
||||
continue
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !sameTags(existing.Tags, tags) {
|
||||
_ = s.Catalog.SetAutoVideoTags(ctx, id, tags)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if dup := s.findDuplicate(ctx, e.Hash, e.Name, e.Size, id); dup != nil {
|
||||
s.backfillDuplicateThumbnail(ctx, dup, e.ThumbnailURL)
|
||||
continue
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
v := &catalog.Video{
|
||||
@@ -216,7 +249,6 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
Ext: strings.TrimPrefix(ext, "."),
|
||||
Quality: "HD",
|
||||
Size: e.Size,
|
||||
ThumbnailURL: e.ThumbnailURL,
|
||||
PreviewStatus: "pending",
|
||||
Category: dirName,
|
||||
PublishedAt: now,
|
||||
@@ -224,9 +256,15 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if err := s.Catalog.UpsertVideo(ctx, v); err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return ctxErr
|
||||
}
|
||||
log.Printf("[scanner] upsert %s error: %v", v.Title, err)
|
||||
continue
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
stats.Added++
|
||||
if s.OnNewVideo != nil {
|
||||
s.OnNewVideo(v)
|
||||
@@ -268,13 +306,6 @@ func (s *Scanner) findDuplicateByFileSignature(ctx context.Context, fileName str
|
||||
return dup
|
||||
}
|
||||
|
||||
func (s *Scanner) backfillDuplicateThumbnail(ctx context.Context, canonical *catalog.Video, thumbnailURL string) {
|
||||
if canonical.ThumbnailURL != "" || thumbnailURL == "" {
|
||||
return
|
||||
}
|
||||
_ = s.Catalog.UpdateVideoMeta(ctx, canonical.ID, catalog.VideoMetaPatch{ThumbnailURL: thumbnailURL})
|
||||
}
|
||||
|
||||
func sameTags(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
|
||||
@@ -3,6 +3,7 @@ package scanner
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -14,7 +15,7 @@ import (
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestRunPersistsRemoteThumbnailFromDriveEntry(t *testing.T) {
|
||||
func TestRunIgnoresRemoteThumbnailFromDriveEntry(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -50,8 +51,8 @@ func TestRunPersistsRemoteThumbnailFromDriveEntry(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "https://thumbnail.example/clip.jpg" {
|
||||
t.Fatalf("thumbnail = %q, want remote thumbnail", got.ThumbnailURL)
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want empty so local thumbnail worker regenerates it", got.ThumbnailURL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,7 +91,106 @@ func TestRunIgnoresZeroSizeVideoFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBackfillsRemoteThumbnailForExistingVideo(t *testing.T) {
|
||||
func TestRunStopsWhenContextCanceledDuringFileLoop(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := &scannerFakeDrive{
|
||||
entries: []drives.Entry{
|
||||
{ID: "file-1", Name: "one.mp4", Size: 123},
|
||||
{ID: "file-2", Name: "two.mp4", Size: 123},
|
||||
{ID: "file-3", Name: "three.mp4", Size: 123},
|
||||
},
|
||||
}
|
||||
callbacks := 0
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, func(*catalog.Video) {
|
||||
callbacks++
|
||||
cancel()
|
||||
})
|
||||
|
||||
stats, err := sc.Run(ctx, "")
|
||||
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("scan error = %v, want context.Canceled", err)
|
||||
}
|
||||
if stats.Added != 1 || callbacks != 1 {
|
||||
t.Fatalf("added=%d callbacks=%d, want exactly one video before cancellation", stats.Added, callbacks)
|
||||
}
|
||||
if _, err := cat.GetVideo(context.Background(), "fake-drive-file-1"); err != nil {
|
||||
t.Fatalf("first video should be persisted before cancellation: %v", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(context.Background(), "fake-drive-file-2"); err != sql.ErrNoRows {
|
||||
t.Fatalf("second video lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(context.Background(), "fake-drive-file-3"); err != sql.ErrNoRows {
|
||||
t.Fatalf("third video lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSkipsAdminDeletedVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "fake-drive-file-1",
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
FileName: "clip.mp4",
|
||||
ContentHash: "HASH-1",
|
||||
Title: "Deleted Clip",
|
||||
Size: 123,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
if err := cat.DeleteVideoWithTombstone(ctx, "fake-drive-file-1"); err != nil {
|
||||
t.Fatalf("delete with tombstone: %v", err)
|
||||
}
|
||||
|
||||
drv := &scannerFakeDrive{
|
||||
entries: []drives.Entry{{
|
||||
ID: "file-1",
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
Hash: "hash-1",
|
||||
MimeType: "video/mp4",
|
||||
ModTime: now,
|
||||
}},
|
||||
}
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
|
||||
stats, err := sc.Run(ctx, "")
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Added != 0 {
|
||||
t.Fatalf("added = %d, want 0", stats.Added)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "fake-drive-file-1"); err != sql.ErrNoRows {
|
||||
t.Fatalf("deleted video was recreated, get error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDoesNotBackfillRemoteThumbnailForExistingVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -140,8 +240,8 @@ func TestRunBackfillsRemoteThumbnailForExistingVideo(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "https://thumbnail.example/backfilled.jpg" {
|
||||
t.Fatalf("thumbnail = %q, want backfilled remote thumbnail", got.ThumbnailURL)
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want empty so local thumbnail worker regenerates it", got.ThumbnailURL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -254,6 +354,93 @@ func TestRunAddsShortCollectionDirectoryAsTag(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDoesNotRecreateDeletedCollectionDirectoryTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
now := time.Now()
|
||||
for _, id := range []string{"existing-1", "existing-2"} {
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: "Existing",
|
||||
Category: "sunny",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed existing sunny video: %v", err)
|
||||
}
|
||||
}
|
||||
if label, ok, err := cat.EnsureCollectionTag(ctx, "sunny"); err != nil || !ok || label != "sunny" {
|
||||
t.Fatalf("ensure collection = %q, %v, %v; want sunny true nil", label, ok, err)
|
||||
}
|
||||
tags, err := cat.ListTags(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list tags: %v", err)
|
||||
}
|
||||
var tagID int64
|
||||
for _, tag := range tags {
|
||||
if tag.Label == "sunny" {
|
||||
tagID = tag.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
if tagID == 0 {
|
||||
t.Fatal("sunny tag not found before delete")
|
||||
}
|
||||
if _, err := cat.DeleteTag(ctx, tagID); err != nil {
|
||||
t.Fatalf("delete tag: %v", err)
|
||||
}
|
||||
|
||||
drv := &scannerTreeFakeDrive{
|
||||
entries: map[string][]drives.Entry{
|
||||
"root": {{
|
||||
ID: "dir-1",
|
||||
Name: "sunny",
|
||||
IsDir: true,
|
||||
}},
|
||||
"dir-1": {{
|
||||
ID: "file-1",
|
||||
ParentID: "dir-1",
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
ModTime: now,
|
||||
}},
|
||||
},
|
||||
}
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
|
||||
if _, err := sc.Run(ctx, ""); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(ctx, "fake-drive-file-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if len(got.Tags) != 0 {
|
||||
t.Fatalf("tags = %#v, want none", got.Tags)
|
||||
}
|
||||
tags, err = cat.ListTags(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list tags after scan: %v", err)
|
||||
}
|
||||
for _, tag := range tags {
|
||||
if tag.Label == "sunny" {
|
||||
t.Fatal("deleted collection tag was recreated during scan")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMapsAVCodeDirectoryToAVTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
// Package spider91migrate 周期性把 spider91 drive 下载到本地的视频
|
||||
// 上传到一个指定的目标 drive 目录(PikPak 或 115),上传成功后:
|
||||
// 上传到一个指定的目标 drive 目录(PikPak、115、123 或 OneDrive),上传成功后:
|
||||
//
|
||||
// - 改写 catalog 行:drive_id / file_id / content_hash 改成目标盘的;
|
||||
// 视频自身的 id 不变(仍是 spider91-<driveID>-<viewkey>),video_tags、
|
||||
// 收藏、点赞、views 等关联数据全部保留
|
||||
// - 删除本地 mp4(spider91/<id>/videos/<viewkey>.<ext>)和 thumb(spider91/<id>/thumbs/<viewkey>.jpg)
|
||||
// - 删除本地 mp4(spider91/<id>/videos/<viewkey>.<ext>)和源 thumb
|
||||
// (spider91/<id>/thumbs/<viewkey>.jpg);公共 /p/thumb/<videoID> 副本会保留
|
||||
//
|
||||
// 之后回放时,videoSource() 自动落到 /p/stream/<target>/<file_id>,
|
||||
// proxy 层走对应盘的直链 / 302 直连。
|
||||
@@ -28,23 +29,29 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/onedrive"
|
||||
"github.com/video-site/backend/internal/drives/p115"
|
||||
"github.com/video-site/backend/internal/drives/p123"
|
||||
"github.com/video-site/backend/internal/drives/pikpak"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
)
|
||||
|
||||
// uploadTarget 是 migrator 调用目标 drive 的最小接口。任何一种"接收 spider91 上传"的
|
||||
// 网盘都要实现它;当前 PikPak 和 115 各自通过适配器满足。
|
||||
// 网盘都要实现它;当前 PikPak、115、123 和 OneDrive 各自通过适配器满足。
|
||||
//
|
||||
// 这一层抽象把"迁移调用方"和"具体盘的 SDK 协议"解耦:
|
||||
// - PikPak 走 GCID + OSS PutObject(pikpak.UploadResult)
|
||||
// - 115 走 SHA1 + 秒传 / OSS / 分片(p115.UploadResult)
|
||||
// - 123 走 MD5 + 秒传 / S3 预签名分片(p123.UploadResult)
|
||||
// - OneDrive 走 SHA1 + 小文件 PUT / 大文件 upload session
|
||||
//
|
||||
// 两个返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
|
||||
// 各家返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
|
||||
type uploadTarget interface {
|
||||
ID() string
|
||||
Kind() string
|
||||
RootID() string
|
||||
EnsureDir(ctx context.Context, pathFromRoot string) (string, error)
|
||||
UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error)
|
||||
Rename(ctx context.Context, fileID, newName string) error
|
||||
}
|
||||
@@ -52,7 +59,7 @@ type uploadTarget interface {
|
||||
// UploadResult 是 uploadTarget.UploadAndReportHash 的归一返回。
|
||||
//
|
||||
// FileID 目标盘上的新文件 ID;
|
||||
// Hash GCID(PikPak)或 SHA1 HEX 大写(115),写入 catalog.content_hash 用于跨盘去重;
|
||||
// Hash GCID(PikPak)、MD5 HEX(123)或 SHA1 HEX(115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;
|
||||
// Size 实际上传字节数。
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
@@ -60,7 +67,9 @@ type UploadResult struct {
|
||||
Size int64
|
||||
}
|
||||
|
||||
// pikpakAdapter / p115Adapter 把具体 driver 包装成 uploadTarget。
|
||||
const spider91UploadDirName = "91 Spider"
|
||||
|
||||
// pikpakAdapter / p115Adapter / p123Adapter / onedriveAdapter 把具体 driver 包装成 uploadTarget。
|
||||
//
|
||||
// 之所以不让 driver 直接实现 uploadTarget:
|
||||
//
|
||||
@@ -74,6 +83,9 @@ type pikpakAdapter struct {
|
||||
func (a *pikpakAdapter) ID() string { return a.d.ID() }
|
||||
func (a *pikpakAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *pikpakAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *pikpakAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *pikpakAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
@@ -92,6 +104,9 @@ type p115Adapter struct {
|
||||
func (a *p115Adapter) ID() string { return a.d.ID() }
|
||||
func (a *p115Adapter) Kind() string { return a.d.Kind() }
|
||||
func (a *p115Adapter) RootID() string { return a.d.RootID() }
|
||||
func (a *p115Adapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *p115Adapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportSha1(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
@@ -103,6 +118,48 @@ func (a *p115Adapter) Rename(ctx context.Context, fileID, newName string) error
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
type p123Adapter struct {
|
||||
d *p123.Driver
|
||||
}
|
||||
|
||||
func (a *p123Adapter) ID() string { return a.d.ID() }
|
||||
func (a *p123Adapter) Kind() string { return a.d.Kind() }
|
||||
func (a *p123Adapter) RootID() string { return a.d.RootID() }
|
||||
func (a *p123Adapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *p123Adapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
return UploadResult{FileID: res.FileID, Hash: res.Hash, Size: res.Size}, nil
|
||||
}
|
||||
func (a *p123Adapter) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
type onedriveAdapter struct {
|
||||
d *onedrive.Driver
|
||||
}
|
||||
|
||||
func (a *onedriveAdapter) ID() string { return a.d.ID() }
|
||||
func (a *onedriveAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *onedriveAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *onedriveAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *onedriveAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
return UploadResult{FileID: res.FileID, Hash: res.Hash, Size: res.Size}, nil
|
||||
}
|
||||
func (a *onedriveAdapter) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
// adaptUploadTarget 把通用 drive 包装成 uploadTarget。
|
||||
// 不支持的盘 kind 返回 error;调用方静默跳过。
|
||||
func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
@@ -111,6 +168,10 @@ func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
return &pikpakAdapter{d: v}, nil
|
||||
case *p115.Driver:
|
||||
return &p115Adapter{d: v}, nil
|
||||
case *p123.Driver:
|
||||
return &p123Adapter{d: v}, nil
|
||||
case *onedrive.Driver:
|
||||
return &onedriveAdapter{d: v}, nil
|
||||
case uploadTarget:
|
||||
// 测试或自定义实现可以直接传入;优先使用具体类型分支以拿到适配器。
|
||||
return v, nil
|
||||
@@ -141,6 +202,7 @@ type Config struct {
|
||||
// 4002 / 9)后整体进入冷却的时长。冷却期间 runOnce 直接返回,不再发起任何
|
||||
// PikPak API 请求,避免被进一步风控。0 时默认 5 分钟;< 0 关闭冷却(仅用于测试)。
|
||||
CaptchaCooldown time.Duration
|
||||
CommonThumbDir string
|
||||
OnMigrated func(videoID string)
|
||||
}
|
||||
|
||||
@@ -272,7 +334,7 @@ func (m *Migrator) runOnce(ctx context.Context) {
|
||||
|
||||
target, pp, err := m.resolveTarget()
|
||||
if err != nil {
|
||||
// 没目标就静默 —— 用户可能还没配 PikPak drive
|
||||
// 没目标就静默 —— 用户选择了本地保存,或还没配 115/PikPak drive。
|
||||
return
|
||||
}
|
||||
|
||||
@@ -382,7 +444,7 @@ func (m *Migrator) spider91Drives() []*spider91.Driver {
|
||||
// - 列出 spider91 drive 本地 videos/ 目录所有 mp4 文件,按 mtime 降序排
|
||||
// - 跳过最新 KeepLatestN 个:这些是用户希望保留在本地的最新爬取
|
||||
// - 对剩下的(更旧)逐个处理:
|
||||
// - 还没迁移(drive_id 仍是 src.ID())→ 上传到 PikPak + 改 catalog + 删本地
|
||||
// - 还没迁移(drive_id 仍是 src.ID())→ 上传到目标盘 + 改 catalog + 删本地
|
||||
// - 已经迁移过但本地还有残留 → 仅删本地(兜底)
|
||||
//
|
||||
// KeepLatestN < 0 时不保护任何本地文件,全部尝试迁移(旧行为,主要给测试用)。
|
||||
@@ -484,7 +546,7 @@ func (m *Migrator) migrateDrive(ctx context.Context, src *spider91.Driver, targe
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
// migrateOne 把单条 spider91 视频上传到 PikPak 并改写 catalog。
|
||||
// migrateOne 把单条 spider91 视频上传到目标盘并改写 catalog。
|
||||
// 返回 (true, nil) 表示真的迁了一条;(false, nil) 表示跳过(本地文件已不在等);
|
||||
// (false, err) 表示真出错。
|
||||
func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider91.Driver, targetDriveID string, pp uploadTarget) (bool, error) {
|
||||
@@ -511,15 +573,19 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// 上传到目标盘的根目录(用户配置的目标 drive 的 rootID)。
|
||||
// 上传到目标盘 rootID 下的固定 "91 Spider" 子目录。若用户把目标盘 rootID
|
||||
// 配成某个自定义目录,这里会在该自定义目录下查找/创建 "91 Spider"。
|
||||
// 上传名走 desiredPikPakName 算出来的方案 B 格式:
|
||||
//
|
||||
// <sanitized title>-<viewkey 后 8 位>.<ext>
|
||||
//
|
||||
// 这样网盘 Web 端列出来的文件名能直接看出是哪个视频,
|
||||
// 又用 viewkey 后 8 位避免同标题撞名。两个目标盘(PikPak / 115)共用同一格式,
|
||||
// 又用 viewkey 后 8 位避免同标题撞名。所有目标盘共用同一格式,
|
||||
// 简化前端 / catalog 的认知。
|
||||
parent := pp.RootID()
|
||||
parent, err := pp.EnsureDir(ctx, spider91UploadDirName)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s ensure %q dir: %w", pp.Kind(), spider91UploadDirName, err)
|
||||
}
|
||||
uploadName := desiredPikPakName(v.Title, extractViewKey(v.ID), v.Ext)
|
||||
res, err := pp.UploadAndReportHash(ctx, parent, uploadName, f, info.Size())
|
||||
if err != nil {
|
||||
@@ -533,18 +599,98 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
|
||||
if err := m.cfg.Catalog.MigrateVideoToDrive(ctx, v.ID, targetDriveID, res.FileID, res.Hash); err != nil {
|
||||
return false, fmt.Errorf("catalog migrate: %w", err)
|
||||
}
|
||||
m.preserveCrawledThumbnail(ctx, src, v)
|
||||
// 同步 catalog 里的 file_name,让下次目标盘扫盘时 (file_name, size) 也能匹配上
|
||||
if err := m.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{FileName: uploadName}); err != nil {
|
||||
log.Printf("[spider91migrate] %s update file_name after migrate: %v", v.ID, err)
|
||||
}
|
||||
|
||||
// 删除本地 mp4 和 thumb(thumb 在 previews/thumbs/ 还有副本,不影响展示)
|
||||
// 删除本地 mp4 和源 thumb(公共 /p/thumb 副本已在 preserveCrawledThumbnail 中保留)。
|
||||
CleanupSpider91Local(src, v.FileID)
|
||||
|
||||
log.Printf("[spider91migrate] %s migrated to drive=%s(kind=%s) file=%s name=%q", v.ID, targetDriveID, pp.Kind(), res.FileID, uploadName)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *Migrator) preserveCrawledThumbnail(ctx context.Context, src *spider91.Driver, v *catalog.Video) {
|
||||
if m == nil || m.cfg.Catalog == nil || src == nil || v == nil || v.ID == "" || v.FileID == "" {
|
||||
return
|
||||
}
|
||||
commonDir := strings.TrimSpace(m.cfg.CommonThumbDir)
|
||||
if commonDir == "" {
|
||||
return
|
||||
}
|
||||
thumbPath, ok := findSpider91ThumbPath(src, v.FileID)
|
||||
if !ok {
|
||||
if v.ThumbnailURL == "" {
|
||||
log.Printf("[spider91migrate] %s crawled thumbnail missing before migration cleanup", v.ID)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(commonDir, 0o755); err != nil {
|
||||
log.Printf("[spider91migrate] %s mkdir common thumbs: %v", v.ID, err)
|
||||
return
|
||||
}
|
||||
dst := mediaasset.ThumbnailPathInDir(commonDir, v.ID)
|
||||
if _, err := os.Stat(dst); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
log.Printf("[spider91migrate] %s stat common thumb: %v", v.ID, err)
|
||||
return
|
||||
}
|
||||
if err := copyFileAtomic(thumbPath, dst); err != nil {
|
||||
log.Printf("[spider91migrate] %s preserve crawled thumbnail: %v", v.ID, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := m.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
ThumbnailURL: "/p/thumb/" + v.ID,
|
||||
}); err != nil {
|
||||
log.Printf("[spider91migrate] %s update crawled thumbnail url: %v", v.ID, err)
|
||||
return
|
||||
}
|
||||
v.ThumbnailURL = "/p/thumb/" + v.ID
|
||||
}
|
||||
|
||||
func findSpider91ThumbPath(src *spider91.Driver, fileID string) (string, bool) {
|
||||
thumbBase := stripExt(fileID)
|
||||
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
|
||||
thumbPath, err := src.ThumbPath(thumbBase + ext)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
info, statErr := os.Stat(thumbPath)
|
||||
if statErr == nil && info.Mode().IsRegular() && info.Size() > 0 {
|
||||
return thumbPath, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func copyFileAtomic(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
|
||||
tmp := dst + ".part"
|
||||
out, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, copyErr := io.Copy(out, in)
|
||||
closeErr := out.Close()
|
||||
if copyErr != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return copyErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return closeErr
|
||||
}
|
||||
return os.Rename(tmp, dst)
|
||||
}
|
||||
|
||||
// CleanupSpider91Local 删除已迁移视频的本地 mp4 和 thumb。
|
||||
//
|
||||
// thumb 删除是 best-effort —— 找不到就算了(spider91 thumb 文件名带后缀,
|
||||
@@ -639,7 +785,7 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driv
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// backfillFileNames 扫描目标 drive(PikPak 或 115)下所有 spider91-* 起始 ID 的视频,
|
||||
// backfillFileNames 扫描目标 drive(PikPak、115、123 或 OneDrive)下所有 spider91-* 起始 ID 的视频,
|
||||
// 对文件名不是 desiredPikPakName(...) 期望格式的,调 target.Rename 修正,
|
||||
// 并把 catalog.file_name 同步到新名字。
|
||||
//
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/p123"
|
||||
"github.com/video-site/backend/internal/drives/pikpak"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
)
|
||||
@@ -53,6 +54,8 @@ type fakePikPak struct {
|
||||
uploadFunc func(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error)
|
||||
mu sync.Mutex
|
||||
gotBodies map[string][]byte
|
||||
gotParents map[string]string
|
||||
ensureCalls []string
|
||||
// renameCalls 记录每次 Rename 的 fileID->newName 历史,用于 backfill 测试断言。
|
||||
renameCalls map[string]string
|
||||
}
|
||||
@@ -62,6 +65,7 @@ func newFakePikPak(id, rootID string) *fakePikPak {
|
||||
id: id,
|
||||
rootID: rootID,
|
||||
gotBodies: make(map[string][]byte),
|
||||
gotParents: make(map[string]string),
|
||||
renameCalls: make(map[string]string),
|
||||
}
|
||||
}
|
||||
@@ -80,8 +84,11 @@ func (d *fakePikPak) StreamURL(context.Context, string) (*drives.StreamLink, err
|
||||
func (d *fakePikPak) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakePikPak) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
func (d *fakePikPak) EnsureDir(_ context.Context, pathFromRoot string) (string, error) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.ensureCalls = append(d.ensureCalls, pathFromRoot)
|
||||
return d.rootID + "/" + pathFromRoot, nil
|
||||
}
|
||||
func (d *fakePikPak) Rename(_ context.Context, fileID, newName string) error {
|
||||
d.mu.Lock()
|
||||
@@ -99,6 +106,7 @@ func (d *fakePikPak) UploadAndReportHash(ctx context.Context, parentID, name str
|
||||
body, _ := io.ReadAll(r)
|
||||
d.mu.Lock()
|
||||
d.gotBodies[name] = body
|
||||
d.gotParents[name] = parentID
|
||||
d.mu.Unlock()
|
||||
return UploadResult{
|
||||
FileID: "remote-" + name,
|
||||
@@ -127,6 +135,32 @@ func (d *fakeP115) Kind() string { return "p115" }
|
||||
var _ drives.Drive = (*fakeP115)(nil)
|
||||
var _ uploadTarget = (*fakeP115)(nil)
|
||||
|
||||
type fakeP123 struct {
|
||||
*fakePikPak
|
||||
}
|
||||
|
||||
func newFakeP123(id, rootID string) *fakeP123 {
|
||||
return &fakeP123{fakePikPak: newFakePikPak(id, rootID)}
|
||||
}
|
||||
|
||||
func (d *fakeP123) Kind() string { return "p123" }
|
||||
|
||||
var _ drives.Drive = (*fakeP123)(nil)
|
||||
var _ uploadTarget = (*fakeP123)(nil)
|
||||
|
||||
type fakeOneDrive struct {
|
||||
*fakePikPak
|
||||
}
|
||||
|
||||
func newFakeOneDrive(id, rootID string) *fakeOneDrive {
|
||||
return &fakeOneDrive{fakePikPak: newFakePikPak(id, rootID)}
|
||||
}
|
||||
|
||||
func (d *fakeOneDrive) Kind() string { return "onedrive" }
|
||||
|
||||
var _ drives.Drive = (*fakeOneDrive)(nil)
|
||||
var _ uploadTarget = (*fakeOneDrive)(nil)
|
||||
|
||||
// TestBackfillFileNamesRenamesOnlyMismatchedSpider91Videos 验证回填逻辑:
|
||||
//
|
||||
// - 已经是期望格式的不会再调 Rename(幂等)
|
||||
@@ -319,12 +353,14 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
|
||||
|
||||
now := time.Now()
|
||||
id := writeSpider91Video(t, cat, src, "vk001", ".mp4", []byte("video bytes here"), now)
|
||||
commonThumbDir := t.TempDir()
|
||||
|
||||
m := New(Config{
|
||||
Catalog: cat,
|
||||
Registry: reg,
|
||||
GetTargetDriveID: func() string { return pp.ID() },
|
||||
KeepLatestN: -1, // 关闭"保留最新 N 个",让 1 条也能立即上传
|
||||
CommonThumbDir: commonThumbDir,
|
||||
})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
@@ -347,6 +383,12 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
|
||||
if _, ok := pp.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("PikPak did not receive expected upload name %q (got names: %v)", wantName, keysOf(pp.gotBodies))
|
||||
}
|
||||
if gotParent := pp.gotParents[wantName]; gotParent != "pikpak-root-id/"+spider91UploadDirName {
|
||||
t.Fatalf("upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(pp.ensureCalls) != 1 || pp.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("ensure calls = %#v, want %q", pp.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
@@ -356,8 +398,15 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
|
||||
if got.ContentHash == "" {
|
||||
t.Fatalf("content_hash should be set after migration")
|
||||
}
|
||||
if got.ThumbnailURL != "/p/thumb/"+id {
|
||||
t.Fatalf("thumbnail_url = %q, want preserved crawled thumbnail URL", got.ThumbnailURL)
|
||||
}
|
||||
commonThumbPath := filepath.Join(commonThumbDir, id+".jpg")
|
||||
if data, err := os.ReadFile(commonThumbPath); err != nil || string(data) != "thumb" {
|
||||
t.Fatalf("common thumb = %q, %v; want copied crawled thumb", string(data), err)
|
||||
}
|
||||
|
||||
// 3) 本地视频和 thumb 都被删了
|
||||
// 3) 本地视频和源 thumb 都被删了;公共 /p/thumb 副本保留。
|
||||
videoPath, _ := src.VideoPath("vk001.mp4")
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local mp4 still exists or stat error %v", err)
|
||||
@@ -588,7 +637,7 @@ func TestRunOnceKeepsAllLocalWhenWithinKeepWindow(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestRunOnceMigratesOnlyOlderFilesBeyondKeepWindow 验证:本地文件数 > KeepLatestN 时
|
||||
// 按 mtime 降序保留最新 N 个,超出部分(更旧的)才上传到 PikPak。
|
||||
// 按 mtime 降序保留最新 N 个,超出部分(更旧的)才上传到目标盘。
|
||||
func TestRunOnceMigratesOnlyOlderFilesBeyondKeepWindow(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src, _ := setupSpider91(t)
|
||||
@@ -841,7 +890,6 @@ func TestNonCaptchaErrorDoesNotTriggerCooldown(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// TestRunOnceMigratesToP115Target 验证:当目标 drive 是 115(kind="p115")时,
|
||||
// migrator 也能正确把 spider91 视频上传过去并改写 catalog。
|
||||
//
|
||||
@@ -885,6 +933,12 @@ func TestRunOnceMigratesToP115Target(t *testing.T) {
|
||||
if _, ok := target.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("p115 did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
|
||||
}
|
||||
if gotParent := target.gotParents[wantName]; gotParent != "p115-root-cid/"+spider91UploadDirName {
|
||||
t.Fatalf("p115 upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("p115 ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
@@ -906,7 +960,142 @@ func TestRunOnceMigratesToP115Target(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak 也不是 115 时,
|
||||
func TestRunOnceMigratesToP123Target(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src, _ := setupSpider91(t)
|
||||
target := newFakeP123("p123-target", "p123-root-id")
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(target)
|
||||
|
||||
now := time.Now()
|
||||
id := writeSpider91Video(t, cat, src, "vk-123-001", ".mp4", []byte("video bytes 123"), now)
|
||||
|
||||
m := New(Config{
|
||||
Catalog: cat,
|
||||
Registry: reg,
|
||||
GetTargetDriveID: func() string { return target.ID() },
|
||||
KeepLatestN: -1,
|
||||
})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
if target.uploadCalls != 1 {
|
||||
t.Fatalf("p123 upload calls = %d, want 1", target.uploadCalls)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DriveID != target.ID() {
|
||||
t.Fatalf("drive_id = %q, want %q", got.DriveID, target.ID())
|
||||
}
|
||||
wantName := "Sample vk-123-001-001.mp4"
|
||||
if _, ok := target.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("p123 did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
|
||||
}
|
||||
if gotParent := target.gotParents[wantName]; gotParent != "p123-root-id/"+spider91UploadDirName {
|
||||
t.Fatalf("p123 upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("p123 ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
if got.FileName != wantName {
|
||||
t.Fatalf("file_name = %q, want %q", got.FileName, wantName)
|
||||
}
|
||||
if got.ContentHash == "" {
|
||||
t.Fatal("content_hash should be set after p123 migration")
|
||||
}
|
||||
|
||||
videoPath, _ := src.VideoPath("vk-123-001.mp4")
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local mp4 still exists after p123 migration or stat error: %v", err)
|
||||
}
|
||||
thumbPath, _ := src.ThumbPath("vk-123-001.jpg")
|
||||
if _, err := os.Stat(thumbPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local thumb still exists after p123 migration or stat error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceMigratesToOneDriveTarget(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src, _ := setupSpider91(t)
|
||||
target := newFakeOneDrive("onedrive-target", "onedrive-root")
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(target)
|
||||
|
||||
now := time.Now()
|
||||
id := writeSpider91Video(t, cat, src, "vk-od-001", ".mp4", []byte("video bytes onedrive"), now)
|
||||
|
||||
m := New(Config{
|
||||
Catalog: cat,
|
||||
Registry: reg,
|
||||
GetTargetDriveID: func() string { return target.ID() },
|
||||
KeepLatestN: -1,
|
||||
})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
if target.uploadCalls != 1 {
|
||||
t.Fatalf("onedrive upload calls = %d, want 1", target.uploadCalls)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DriveID != target.ID() {
|
||||
t.Fatalf("drive_id = %q, want %q", got.DriveID, target.ID())
|
||||
}
|
||||
wantName := "Sample vk-od-001-001.mp4"
|
||||
if _, ok := target.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("onedrive did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
|
||||
}
|
||||
if gotParent := target.gotParents[wantName]; gotParent != "onedrive-root/"+spider91UploadDirName {
|
||||
t.Fatalf("onedrive upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("onedrive ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
if got.FileName != wantName {
|
||||
t.Fatalf("file_name = %q, want %q", got.FileName, wantName)
|
||||
}
|
||||
if got.ContentHash == "" {
|
||||
t.Fatal("content_hash should be set after onedrive migration")
|
||||
}
|
||||
|
||||
videoPath, _ := src.VideoPath("vk-od-001.mp4")
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local mp4 still exists after onedrive migration or stat error: %v", err)
|
||||
}
|
||||
thumbPath, _ := src.ThumbPath("vk-od-001.jpg")
|
||||
if _, err := os.Stat(thumbPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local thumb still exists after onedrive migration or stat error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdaptUploadTargetSupportsP123Driver(t *testing.T) {
|
||||
d := p123.New(p123.Config{
|
||||
ID: "p123-target",
|
||||
RootID: "root-123",
|
||||
AccessToken: "token-1",
|
||||
})
|
||||
target, err := adaptUploadTarget(d)
|
||||
if err != nil {
|
||||
t.Fatalf("adaptUploadTarget() error = %v", err)
|
||||
}
|
||||
if target.ID() != "p123-target" || target.Kind() != "p123" || target.RootID() != "root-123" {
|
||||
t.Fatalf("target id/kind/root = %q/%q/%q, want p123-target/p123/root-123", target.ID(), target.Kind(), target.RootID())
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115、123 也不是 OneDrive 时,
|
||||
// resolveTarget 拒绝并返回 error,让 runOnce 静默跳过(不会做破坏性变更)。
|
||||
func TestResolveTargetRejectsUnsupportedKind(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
)
|
||||
|
||||
type VideoAssetRef struct {
|
||||
@@ -71,14 +73,15 @@ func Compute(
|
||||
continue
|
||||
}
|
||||
driveUsage := out.Drives[ref.DriveID]
|
||||
thumbPath := filepath.Join(localDir, "thumbs", ref.ID+".jpg")
|
||||
if size, exists, err := regularFileSize(thumbPath); err != nil {
|
||||
return Usage{}, err
|
||||
} else if exists {
|
||||
key := ref.DriveID + "\x00thumb\x00" + thumbPath
|
||||
if !seen[key] {
|
||||
driveUsage.ThumbnailBytes += size
|
||||
seen[key] = true
|
||||
for _, thumbPath := range mediaasset.ThumbnailPathCandidates(localDir, ref.ID) {
|
||||
if size, exists, err := regularFileSize(thumbPath); err != nil {
|
||||
return Usage{}, err
|
||||
} else if exists {
|
||||
key := ref.DriveID + "\x00thumb\x00" + thumbPath
|
||||
if !seen[key] {
|
||||
driveUsage.ThumbnailBytes += size
|
||||
seen[key] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,10 @@ package storageusage
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
)
|
||||
|
||||
func TestComputeCountsLocalThumbnailsAndTeasersByDrive(t *testing.T) {
|
||||
@@ -13,6 +16,8 @@ func TestComputeCountsLocalThumbnailsAndTeasersByDrive(t *testing.T) {
|
||||
}
|
||||
writeSizedFile(t, filepath.Join(localDir, "thumbs", "video-a.jpg"), 3)
|
||||
writeSizedFile(t, filepath.Join(localDir, "thumbs", "video-b.jpg"), 5)
|
||||
longID := "localstorage-" + strings.Repeat("x", 240)
|
||||
writeSizedFile(t, mediaasset.ThumbnailPath(localDir, longID), 13)
|
||||
teaserA := filepath.Join(localDir, "video-a.mp4")
|
||||
teaserB := filepath.Join(localDir, "video-b.mp4")
|
||||
writeSizedFile(t, teaserA, 7)
|
||||
@@ -24,6 +29,7 @@ func TestComputeCountsLocalThumbnailsAndTeasersByDrive(t *testing.T) {
|
||||
{ID: "video-a", DriveID: "drive-a", PreviewLocal: teaserA},
|
||||
{ID: "video-a-copy", DriveID: "drive-a", PreviewLocal: teaserA},
|
||||
{ID: "video-b", DriveID: "drive-b", PreviewLocal: teaserB},
|
||||
{ID: longID, DriveID: "drive-b"},
|
||||
{ID: "outside", DriveID: "drive-b", PreviewLocal: outside},
|
||||
{ID: "unknown-drive-video", DriveID: "missing", PreviewLocal: teaserB},
|
||||
}, []string{"drive-a", "drive-b"}, func(string) (DiskStats, error) {
|
||||
@@ -41,11 +47,11 @@ func TestComputeCountsLocalThumbnailsAndTeasersByDrive(t *testing.T) {
|
||||
t.Fatalf("drive-a usage = %#v, want thumbnails=3 teaser=7 total=10", driveA)
|
||||
}
|
||||
driveB := got.Drives["drive-b"]
|
||||
if driveB.ThumbnailBytes != 5 || driveB.TeaserBytes != 11 || driveB.TotalBytes != 16 {
|
||||
t.Fatalf("drive-b usage = %#v, want thumbnails=5 teaser=11 total=16", driveB)
|
||||
if driveB.ThumbnailBytes != 18 || driveB.TeaserBytes != 11 || driveB.TotalBytes != 29 {
|
||||
t.Fatalf("drive-b usage = %#v, want thumbnails=18 teaser=11 total=29", driveB)
|
||||
}
|
||||
if got.ThumbnailBytes != 8 || got.TeaserBytes != 18 || got.TotalBytes != 26 {
|
||||
t.Fatalf("totals = %#v, want thumbnails=8 teaser=18 total=26", got)
|
||||
if got.ThumbnailBytes != 21 || got.TeaserBytes != 18 || got.TotalBytes != 39 {
|
||||
t.Fatalf("totals = %#v, want thumbnails=21 teaser=18 total=39", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
// Copyright 2018 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package socks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
noDeadline = time.Time{}
|
||||
aLongTimeAgo = time.Unix(1, 0)
|
||||
)
|
||||
|
||||
func (d *Dialer) connect(ctx context.Context, c net.Conn, address string) (_ net.Addr, ctxErr error) {
|
||||
host, port, err := splitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if deadline, ok := ctx.Deadline(); ok && !deadline.IsZero() {
|
||||
c.SetDeadline(deadline)
|
||||
defer c.SetDeadline(noDeadline)
|
||||
}
|
||||
if ctx != context.Background() {
|
||||
errCh := make(chan error, 1)
|
||||
done := make(chan struct{})
|
||||
defer func() {
|
||||
close(done)
|
||||
if ctxErr == nil {
|
||||
ctxErr = <-errCh
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
c.SetDeadline(aLongTimeAgo)
|
||||
errCh <- ctx.Err()
|
||||
case <-done:
|
||||
errCh <- nil
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
b := make([]byte, 0, 6+len(host)) // the size here is just an estimate
|
||||
b = append(b, Version5)
|
||||
if len(d.AuthMethods) == 0 || d.Authenticate == nil {
|
||||
b = append(b, 1, byte(AuthMethodNotRequired))
|
||||
} else {
|
||||
ams := d.AuthMethods
|
||||
if len(ams) > 255 {
|
||||
return nil, errors.New("too many authentication methods")
|
||||
}
|
||||
b = append(b, byte(len(ams)))
|
||||
for _, am := range ams {
|
||||
b = append(b, byte(am))
|
||||
}
|
||||
}
|
||||
if _, ctxErr = c.Write(b); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if _, ctxErr = io.ReadFull(c, b[:2]); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
if b[0] != Version5 {
|
||||
return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0])))
|
||||
}
|
||||
am := AuthMethod(b[1])
|
||||
if am == AuthMethodNoAcceptableMethods {
|
||||
return nil, errors.New("no acceptable authentication methods")
|
||||
}
|
||||
if d.Authenticate != nil {
|
||||
if ctxErr = d.Authenticate(ctx, c, am); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
b = b[:0]
|
||||
b = append(b, Version5, byte(d.cmd), 0)
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
b = append(b, AddrTypeIPv4)
|
||||
b = append(b, ip4...)
|
||||
} else if ip6 := ip.To16(); ip6 != nil {
|
||||
b = append(b, AddrTypeIPv6)
|
||||
b = append(b, ip6...)
|
||||
} else {
|
||||
return nil, errors.New("unknown address type")
|
||||
}
|
||||
} else {
|
||||
if len(host) > 255 {
|
||||
return nil, errors.New("FQDN too long")
|
||||
}
|
||||
b = append(b, AddrTypeFQDN)
|
||||
b = append(b, byte(len(host)))
|
||||
b = append(b, host...)
|
||||
}
|
||||
b = append(b, byte(port>>8), byte(port))
|
||||
if _, ctxErr = c.Write(b); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if _, ctxErr = io.ReadFull(c, b[:4]); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
if b[0] != Version5 {
|
||||
return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0])))
|
||||
}
|
||||
if cmdErr := Reply(b[1]); cmdErr != StatusSucceeded {
|
||||
return nil, errors.New("unknown error " + cmdErr.String())
|
||||
}
|
||||
if b[2] != 0 {
|
||||
return nil, errors.New("non-zero reserved field")
|
||||
}
|
||||
l := 2
|
||||
var a Addr
|
||||
switch b[3] {
|
||||
case AddrTypeIPv4:
|
||||
l += net.IPv4len
|
||||
a.IP = make(net.IP, net.IPv4len)
|
||||
case AddrTypeIPv6:
|
||||
l += net.IPv6len
|
||||
a.IP = make(net.IP, net.IPv6len)
|
||||
case AddrTypeFQDN:
|
||||
if _, err := io.ReadFull(c, b[:1]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l += int(b[0])
|
||||
default:
|
||||
return nil, errors.New("unknown address type " + strconv.Itoa(int(b[3])))
|
||||
}
|
||||
if cap(b) < l {
|
||||
b = make([]byte, l)
|
||||
} else {
|
||||
b = b[:l]
|
||||
}
|
||||
if _, ctxErr = io.ReadFull(c, b); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
if a.IP != nil {
|
||||
copy(a.IP, b)
|
||||
} else {
|
||||
a.Name = string(b[:len(b)-2])
|
||||
}
|
||||
a.Port = int(b[len(b)-2])<<8 | int(b[len(b)-1])
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
func splitHostPort(address string) (string, int, error) {
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
portnum, err := strconv.Atoi(port)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if 1 > portnum || portnum > 0xffff {
|
||||
return "", 0, errors.New("port number out of range " + port)
|
||||
}
|
||||
return host, portnum, nil
|
||||
}
|
||||
+317
@@ -0,0 +1,317 @@
|
||||
// Copyright 2018 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package socks provides a SOCKS version 5 client implementation.
|
||||
//
|
||||
// SOCKS protocol version 5 is defined in RFC 1928.
|
||||
// Username/Password authentication for SOCKS version 5 is defined in
|
||||
// RFC 1929.
|
||||
package socks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// A Command represents a SOCKS command.
|
||||
type Command int
|
||||
|
||||
func (cmd Command) String() string {
|
||||
switch cmd {
|
||||
case CmdConnect:
|
||||
return "socks connect"
|
||||
case cmdBind:
|
||||
return "socks bind"
|
||||
default:
|
||||
return "socks " + strconv.Itoa(int(cmd))
|
||||
}
|
||||
}
|
||||
|
||||
// An AuthMethod represents a SOCKS authentication method.
|
||||
type AuthMethod int
|
||||
|
||||
// A Reply represents a SOCKS command reply code.
|
||||
type Reply int
|
||||
|
||||
func (code Reply) String() string {
|
||||
switch code {
|
||||
case StatusSucceeded:
|
||||
return "succeeded"
|
||||
case 0x01:
|
||||
return "general SOCKS server failure"
|
||||
case 0x02:
|
||||
return "connection not allowed by ruleset"
|
||||
case 0x03:
|
||||
return "network unreachable"
|
||||
case 0x04:
|
||||
return "host unreachable"
|
||||
case 0x05:
|
||||
return "connection refused"
|
||||
case 0x06:
|
||||
return "TTL expired"
|
||||
case 0x07:
|
||||
return "command not supported"
|
||||
case 0x08:
|
||||
return "address type not supported"
|
||||
default:
|
||||
return "unknown code: " + strconv.Itoa(int(code))
|
||||
}
|
||||
}
|
||||
|
||||
// Wire protocol constants.
|
||||
const (
|
||||
Version5 = 0x05
|
||||
|
||||
AddrTypeIPv4 = 0x01
|
||||
AddrTypeFQDN = 0x03
|
||||
AddrTypeIPv6 = 0x04
|
||||
|
||||
CmdConnect Command = 0x01 // establishes an active-open forward proxy connection
|
||||
cmdBind Command = 0x02 // establishes a passive-open forward proxy connection
|
||||
|
||||
AuthMethodNotRequired AuthMethod = 0x00 // no authentication required
|
||||
AuthMethodUsernamePassword AuthMethod = 0x02 // use username/password
|
||||
AuthMethodNoAcceptableMethods AuthMethod = 0xff // no acceptable authentication methods
|
||||
|
||||
StatusSucceeded Reply = 0x00
|
||||
)
|
||||
|
||||
// An Addr represents a SOCKS-specific address.
|
||||
// Either Name or IP is used exclusively.
|
||||
type Addr struct {
|
||||
Name string // fully-qualified domain name
|
||||
IP net.IP
|
||||
Port int
|
||||
}
|
||||
|
||||
func (a *Addr) Network() string { return "socks" }
|
||||
|
||||
func (a *Addr) String() string {
|
||||
if a == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
port := strconv.Itoa(a.Port)
|
||||
if a.IP == nil {
|
||||
return net.JoinHostPort(a.Name, port)
|
||||
}
|
||||
return net.JoinHostPort(a.IP.String(), port)
|
||||
}
|
||||
|
||||
// A Conn represents a forward proxy connection.
|
||||
type Conn struct {
|
||||
net.Conn
|
||||
|
||||
boundAddr net.Addr
|
||||
}
|
||||
|
||||
// BoundAddr returns the address assigned by the proxy server for
|
||||
// connecting to the command target address from the proxy server.
|
||||
func (c *Conn) BoundAddr() net.Addr {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
return c.boundAddr
|
||||
}
|
||||
|
||||
// A Dialer holds SOCKS-specific options.
|
||||
type Dialer struct {
|
||||
cmd Command // either CmdConnect or cmdBind
|
||||
proxyNetwork string // network between a proxy server and a client
|
||||
proxyAddress string // proxy server address
|
||||
|
||||
// ProxyDial specifies the optional dial function for
|
||||
// establishing the transport connection.
|
||||
ProxyDial func(context.Context, string, string) (net.Conn, error)
|
||||
|
||||
// AuthMethods specifies the list of request authentication
|
||||
// methods.
|
||||
// If empty, SOCKS client requests only AuthMethodNotRequired.
|
||||
AuthMethods []AuthMethod
|
||||
|
||||
// Authenticate specifies the optional authentication
|
||||
// function. It must be non-nil when AuthMethods is not empty.
|
||||
// It must return an error when the authentication is failed.
|
||||
Authenticate func(context.Context, io.ReadWriter, AuthMethod) error
|
||||
}
|
||||
|
||||
// DialContext connects to the provided address on the provided
|
||||
// network.
|
||||
//
|
||||
// The returned error value may be a net.OpError. When the Op field of
|
||||
// net.OpError contains "socks", the Source field contains a proxy
|
||||
// server address and the Addr field contains a command target
|
||||
// address.
|
||||
//
|
||||
// See func Dial of the net package of standard library for a
|
||||
// description of the network and address parameters.
|
||||
func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
if err := d.validateTarget(network, address); err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
if ctx == nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")}
|
||||
}
|
||||
var err error
|
||||
var c net.Conn
|
||||
if d.ProxyDial != nil {
|
||||
c, err = d.ProxyDial(ctx, d.proxyNetwork, d.proxyAddress)
|
||||
} else {
|
||||
var dd net.Dialer
|
||||
c, err = dd.DialContext(ctx, d.proxyNetwork, d.proxyAddress)
|
||||
}
|
||||
if err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
a, err := d.connect(ctx, c, address)
|
||||
if err != nil {
|
||||
c.Close()
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
return &Conn{Conn: c, boundAddr: a}, nil
|
||||
}
|
||||
|
||||
// DialWithConn initiates a connection from SOCKS server to the target
|
||||
// network and address using the connection c that is already
|
||||
// connected to the SOCKS server.
|
||||
//
|
||||
// It returns the connection's local address assigned by the SOCKS
|
||||
// server.
|
||||
func (d *Dialer) DialWithConn(ctx context.Context, c net.Conn, network, address string) (net.Addr, error) {
|
||||
if err := d.validateTarget(network, address); err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
if ctx == nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")}
|
||||
}
|
||||
a, err := d.connect(ctx, c, address)
|
||||
if err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// Dial connects to the provided address on the provided network.
|
||||
//
|
||||
// Unlike DialContext, it returns a raw transport connection instead
|
||||
// of a forward proxy connection.
|
||||
//
|
||||
// Deprecated: Use DialContext or DialWithConn instead.
|
||||
func (d *Dialer) Dial(network, address string) (net.Conn, error) {
|
||||
if err := d.validateTarget(network, address); err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
var err error
|
||||
var c net.Conn
|
||||
if d.ProxyDial != nil {
|
||||
c, err = d.ProxyDial(context.Background(), d.proxyNetwork, d.proxyAddress)
|
||||
} else {
|
||||
c, err = net.Dial(d.proxyNetwork, d.proxyAddress)
|
||||
}
|
||||
if err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
if _, err := d.DialWithConn(context.Background(), c, network, address); err != nil {
|
||||
c.Close()
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (d *Dialer) validateTarget(network, address string) error {
|
||||
switch network {
|
||||
case "tcp", "tcp6", "tcp4":
|
||||
default:
|
||||
return errors.New("network not implemented")
|
||||
}
|
||||
switch d.cmd {
|
||||
case CmdConnect, cmdBind:
|
||||
default:
|
||||
return errors.New("command not implemented")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Dialer) pathAddrs(address string) (proxy, dst net.Addr, err error) {
|
||||
for i, s := range []string{d.proxyAddress, address} {
|
||||
host, port, err := splitHostPort(s)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
a := &Addr{Port: port}
|
||||
a.IP = net.ParseIP(host)
|
||||
if a.IP == nil {
|
||||
a.Name = host
|
||||
}
|
||||
if i == 0 {
|
||||
proxy = a
|
||||
} else {
|
||||
dst = a
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// NewDialer returns a new Dialer that dials through the provided
|
||||
// proxy server's network and address.
|
||||
func NewDialer(network, address string) *Dialer {
|
||||
return &Dialer{proxyNetwork: network, proxyAddress: address, cmd: CmdConnect}
|
||||
}
|
||||
|
||||
const (
|
||||
authUsernamePasswordVersion = 0x01
|
||||
authStatusSucceeded = 0x00
|
||||
)
|
||||
|
||||
// UsernamePassword are the credentials for the username/password
|
||||
// authentication method.
|
||||
type UsernamePassword struct {
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
// Authenticate authenticates a pair of username and password with the
|
||||
// proxy server.
|
||||
func (up *UsernamePassword) Authenticate(ctx context.Context, rw io.ReadWriter, auth AuthMethod) error {
|
||||
switch auth {
|
||||
case AuthMethodNotRequired:
|
||||
return nil
|
||||
case AuthMethodUsernamePassword:
|
||||
if len(up.Username) == 0 || len(up.Username) > 255 || len(up.Password) > 255 {
|
||||
return errors.New("invalid username/password")
|
||||
}
|
||||
b := []byte{authUsernamePasswordVersion}
|
||||
b = append(b, byte(len(up.Username)))
|
||||
b = append(b, up.Username...)
|
||||
b = append(b, byte(len(up.Password)))
|
||||
b = append(b, up.Password...)
|
||||
// TODO(mikio): handle IO deadlines and cancelation if
|
||||
// necessary
|
||||
if _, err := rw.Write(b); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.ReadFull(rw, b[:2]); err != nil {
|
||||
return err
|
||||
}
|
||||
if b[0] != authUsernamePasswordVersion {
|
||||
return errors.New("invalid username/password version")
|
||||
}
|
||||
if b[1] != authStatusSucceeded {
|
||||
return errors.New("username/password authentication failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errors.New("unsupported authentication method " + strconv.Itoa(int(auth)))
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
// Copyright 2019 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
)
|
||||
|
||||
// A ContextDialer dials using a context.
|
||||
type ContextDialer interface {
|
||||
DialContext(ctx context.Context, network, address string) (net.Conn, error)
|
||||
}
|
||||
|
||||
// Dial works like DialContext on net.Dialer but using a dialer returned by FromEnvironment.
|
||||
//
|
||||
// The passed ctx is only used for returning the Conn, not the lifetime of the Conn.
|
||||
//
|
||||
// Custom dialers (registered via RegisterDialerType) that do not implement ContextDialer
|
||||
// can leak a goroutine for as long as it takes the underlying Dialer implementation to timeout.
|
||||
//
|
||||
// A Conn returned from a successful Dial after the context has been cancelled will be immediately closed.
|
||||
func Dial(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
d := FromEnvironment()
|
||||
if xd, ok := d.(ContextDialer); ok {
|
||||
return xd.DialContext(ctx, network, address)
|
||||
}
|
||||
return dialContext(ctx, d, network, address)
|
||||
}
|
||||
|
||||
// WARNING: this can leak a goroutine for as long as the underlying Dialer implementation takes to timeout
|
||||
// A Conn returned from a successful Dial after the context has been cancelled will be immediately closed.
|
||||
func dialContext(ctx context.Context, d Dialer, network, address string) (net.Conn, error) {
|
||||
var (
|
||||
conn net.Conn
|
||||
done = make(chan struct{}, 1)
|
||||
err error
|
||||
)
|
||||
go func() {
|
||||
conn, err = d.Dial(network, address)
|
||||
close(done)
|
||||
if conn != nil && ctx.Err() != nil {
|
||||
conn.Close()
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
case <-done:
|
||||
}
|
||||
return conn, err
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
)
|
||||
|
||||
type direct struct{}
|
||||
|
||||
// Direct implements Dialer by making network connections directly using net.Dial or net.DialContext.
|
||||
var Direct = direct{}
|
||||
|
||||
var (
|
||||
_ Dialer = Direct
|
||||
_ ContextDialer = Direct
|
||||
)
|
||||
|
||||
// Dial directly invokes net.Dial with the supplied parameters.
|
||||
func (direct) Dial(network, addr string) (net.Conn, error) {
|
||||
return net.Dial(network, addr)
|
||||
}
|
||||
|
||||
// DialContext instantiates a net.Dialer and invokes its DialContext receiver with the supplied parameters.
|
||||
func (direct) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, network, addr)
|
||||
}
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A PerHost directs connections to a default Dialer unless the host name
|
||||
// requested matches one of a number of exceptions.
|
||||
type PerHost struct {
|
||||
def, bypass Dialer
|
||||
|
||||
bypassNetworks []*net.IPNet
|
||||
bypassIPs []net.IP
|
||||
bypassZones []string
|
||||
bypassHosts []string
|
||||
}
|
||||
|
||||
// NewPerHost returns a PerHost Dialer that directs connections to either
|
||||
// defaultDialer or bypass, depending on whether the connection matches one of
|
||||
// the configured rules.
|
||||
func NewPerHost(defaultDialer, bypass Dialer) *PerHost {
|
||||
return &PerHost{
|
||||
def: defaultDialer,
|
||||
bypass: bypass,
|
||||
}
|
||||
}
|
||||
|
||||
// Dial connects to the address addr on the given network through either
|
||||
// defaultDialer or bypass.
|
||||
func (p *PerHost) Dial(network, addr string) (c net.Conn, err error) {
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return p.dialerForRequest(host).Dial(network, addr)
|
||||
}
|
||||
|
||||
// DialContext connects to the address addr on the given network through either
|
||||
// defaultDialer or bypass.
|
||||
func (p *PerHost) DialContext(ctx context.Context, network, addr string) (c net.Conn, err error) {
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := p.dialerForRequest(host)
|
||||
if x, ok := d.(ContextDialer); ok {
|
||||
return x.DialContext(ctx, network, addr)
|
||||
}
|
||||
return dialContext(ctx, d, network, addr)
|
||||
}
|
||||
|
||||
func (p *PerHost) dialerForRequest(host string) Dialer {
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
for _, net := range p.bypassNetworks {
|
||||
if net.Contains(ip) {
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
for _, bypassIP := range p.bypassIPs {
|
||||
if bypassIP.Equal(ip) {
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
return p.def
|
||||
}
|
||||
|
||||
for _, zone := range p.bypassZones {
|
||||
if strings.HasSuffix(host, zone) {
|
||||
return p.bypass
|
||||
}
|
||||
if host == zone[1:] {
|
||||
// For a zone ".example.com", we match "example.com"
|
||||
// too.
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
for _, bypassHost := range p.bypassHosts {
|
||||
if bypassHost == host {
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
return p.def
|
||||
}
|
||||
|
||||
// AddFromString parses a string that contains comma-separated values
|
||||
// specifying hosts that should use the bypass proxy. Each value is either an
|
||||
// IP address, a CIDR range, a zone (*.example.com) or a host name
|
||||
// (localhost). A best effort is made to parse the string and errors are
|
||||
// ignored.
|
||||
func (p *PerHost) AddFromString(s string) {
|
||||
hosts := strings.Split(s, ",")
|
||||
for _, host := range hosts {
|
||||
host = strings.TrimSpace(host)
|
||||
if len(host) == 0 {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(host, "/") {
|
||||
// We assume that it's a CIDR address like 127.0.0.0/8
|
||||
if _, net, err := net.ParseCIDR(host); err == nil {
|
||||
p.AddNetwork(net)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
p.AddIP(ip)
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(host, "*.") {
|
||||
p.AddZone(host[1:])
|
||||
continue
|
||||
}
|
||||
p.AddHost(host)
|
||||
}
|
||||
}
|
||||
|
||||
// AddIP specifies an IP address that will use the bypass proxy. Note that
|
||||
// this will only take effect if a literal IP address is dialed. A connection
|
||||
// to a named host will never match an IP.
|
||||
func (p *PerHost) AddIP(ip net.IP) {
|
||||
p.bypassIPs = append(p.bypassIPs, ip)
|
||||
}
|
||||
|
||||
// AddNetwork specifies an IP range that will use the bypass proxy. Note that
|
||||
// this will only take effect if a literal IP address is dialed. A connection
|
||||
// to a named host will never match.
|
||||
func (p *PerHost) AddNetwork(net *net.IPNet) {
|
||||
p.bypassNetworks = append(p.bypassNetworks, net)
|
||||
}
|
||||
|
||||
// AddZone specifies a DNS suffix that will use the bypass proxy. A zone of
|
||||
// "example.com" matches "example.com" and all of its subdomains.
|
||||
func (p *PerHost) AddZone(zone string) {
|
||||
zone = strings.TrimSuffix(zone, ".")
|
||||
if !strings.HasPrefix(zone, ".") {
|
||||
zone = "." + zone
|
||||
}
|
||||
p.bypassZones = append(p.bypassZones, zone)
|
||||
}
|
||||
|
||||
// AddHost specifies a host name that will use the bypass proxy.
|
||||
func (p *PerHost) AddHost(host string) {
|
||||
host = strings.TrimSuffix(host, ".")
|
||||
p.bypassHosts = append(p.bypassHosts, host)
|
||||
}
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package proxy provides support for a variety of protocols to proxy network
|
||||
// data.
|
||||
package proxy // import "golang.org/x/net/proxy"
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// A Dialer is a means to establish a connection.
|
||||
// Custom dialers should also implement ContextDialer.
|
||||
type Dialer interface {
|
||||
// Dial connects to the given address via the proxy.
|
||||
Dial(network, addr string) (c net.Conn, err error)
|
||||
}
|
||||
|
||||
// Auth contains authentication parameters that specific Dialers may require.
|
||||
type Auth struct {
|
||||
User, Password string
|
||||
}
|
||||
|
||||
// FromEnvironment returns the dialer specified by the proxy-related
|
||||
// variables in the environment and makes underlying connections
|
||||
// directly.
|
||||
func FromEnvironment() Dialer {
|
||||
return FromEnvironmentUsing(Direct)
|
||||
}
|
||||
|
||||
// FromEnvironmentUsing returns the dialer specify by the proxy-related
|
||||
// variables in the environment and makes underlying connections
|
||||
// using the provided forwarding Dialer (for instance, a *net.Dialer
|
||||
// with desired configuration).
|
||||
func FromEnvironmentUsing(forward Dialer) Dialer {
|
||||
allProxy := allProxyEnv.Get()
|
||||
if len(allProxy) == 0 {
|
||||
return forward
|
||||
}
|
||||
|
||||
proxyURL, err := url.Parse(allProxy)
|
||||
if err != nil {
|
||||
return forward
|
||||
}
|
||||
proxy, err := FromURL(proxyURL, forward)
|
||||
if err != nil {
|
||||
return forward
|
||||
}
|
||||
|
||||
noProxy := noProxyEnv.Get()
|
||||
if len(noProxy) == 0 {
|
||||
return proxy
|
||||
}
|
||||
|
||||
perHost := NewPerHost(proxy, forward)
|
||||
perHost.AddFromString(noProxy)
|
||||
return perHost
|
||||
}
|
||||
|
||||
// proxySchemes is a map from URL schemes to a function that creates a Dialer
|
||||
// from a URL with such a scheme.
|
||||
var proxySchemes map[string]func(*url.URL, Dialer) (Dialer, error)
|
||||
|
||||
// RegisterDialerType takes a URL scheme and a function to generate Dialers from
|
||||
// a URL with that scheme and a forwarding Dialer. Registered schemes are used
|
||||
// by FromURL.
|
||||
func RegisterDialerType(scheme string, f func(*url.URL, Dialer) (Dialer, error)) {
|
||||
if proxySchemes == nil {
|
||||
proxySchemes = make(map[string]func(*url.URL, Dialer) (Dialer, error))
|
||||
}
|
||||
proxySchemes[scheme] = f
|
||||
}
|
||||
|
||||
// FromURL returns a Dialer given a URL specification and an underlying
|
||||
// Dialer for it to make network requests.
|
||||
func FromURL(u *url.URL, forward Dialer) (Dialer, error) {
|
||||
var auth *Auth
|
||||
if u.User != nil {
|
||||
auth = new(Auth)
|
||||
auth.User = u.User.Username()
|
||||
if p, ok := u.User.Password(); ok {
|
||||
auth.Password = p
|
||||
}
|
||||
}
|
||||
|
||||
switch u.Scheme {
|
||||
case "socks5", "socks5h":
|
||||
addr := u.Hostname()
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
port = "1080"
|
||||
}
|
||||
return SOCKS5("tcp", net.JoinHostPort(addr, port), auth, forward)
|
||||
}
|
||||
|
||||
// If the scheme doesn't match any of the built-in schemes, see if it
|
||||
// was registered by another package.
|
||||
if proxySchemes != nil {
|
||||
if f, ok := proxySchemes[u.Scheme]; ok {
|
||||
return f(u, forward)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.New("proxy: unknown scheme: " + u.Scheme)
|
||||
}
|
||||
|
||||
var (
|
||||
allProxyEnv = &envOnce{
|
||||
names: []string{"ALL_PROXY", "all_proxy"},
|
||||
}
|
||||
noProxyEnv = &envOnce{
|
||||
names: []string{"NO_PROXY", "no_proxy"},
|
||||
}
|
||||
)
|
||||
|
||||
// envOnce looks up an environment variable (optionally by multiple
|
||||
// names) once. It mitigates expensive lookups on some platforms
|
||||
// (e.g. Windows).
|
||||
// (Borrowed from net/http/transport.go)
|
||||
type envOnce struct {
|
||||
names []string
|
||||
once sync.Once
|
||||
val string
|
||||
}
|
||||
|
||||
func (e *envOnce) Get() string {
|
||||
e.once.Do(e.init)
|
||||
return e.val
|
||||
}
|
||||
|
||||
func (e *envOnce) init() {
|
||||
for _, n := range e.names {
|
||||
e.val = os.Getenv(n)
|
||||
if e.val != "" {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reset is used by tests
|
||||
func (e *envOnce) reset() {
|
||||
e.once = sync.Once{}
|
||||
e.val = ""
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
|
||||
"golang.org/x/net/internal/socks"
|
||||
)
|
||||
|
||||
// SOCKS5 returns a Dialer that makes SOCKSv5 connections to the given
|
||||
// address with an optional username and password.
|
||||
// See RFC 1928 and RFC 1929.
|
||||
func SOCKS5(network, address string, auth *Auth, forward Dialer) (Dialer, error) {
|
||||
d := socks.NewDialer(network, address)
|
||||
if forward != nil {
|
||||
if f, ok := forward.(ContextDialer); ok {
|
||||
d.ProxyDial = func(ctx context.Context, network string, address string) (net.Conn, error) {
|
||||
return f.DialContext(ctx, network, address)
|
||||
}
|
||||
} else {
|
||||
d.ProxyDial = func(ctx context.Context, network string, address string) (net.Conn, error) {
|
||||
return dialContext(ctx, forward, network, address)
|
||||
}
|
||||
}
|
||||
}
|
||||
if auth != nil {
|
||||
up := socks.UsernamePassword{
|
||||
Username: auth.User,
|
||||
Password: auth.Password,
|
||||
}
|
||||
d.AuthMethods = []socks.AuthMethod{
|
||||
socks.AuthMethodNotRequired,
|
||||
socks.AuthMethodUsernamePassword,
|
||||
}
|
||||
d.Authenticate = up.Authenticate
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
Vendored
+2
@@ -67,6 +67,8 @@ github.com/skip2/go-qrcode/reedsolomon
|
||||
golang.org/x/crypto/curve25519
|
||||
# golang.org/x/net v0.27.0
|
||||
## explicit; go 1.18
|
||||
golang.org/x/net/internal/socks
|
||||
golang.org/x/net/proxy
|
||||
golang.org/x/net/publicsuffix
|
||||
# golang.org/x/sys v0.30.0
|
||||
## explicit; go 1.18
|
||||
|
||||
@@ -49,7 +49,7 @@ Common overrides:
|
||||
FRONTEND_PORT=9191 Public web port
|
||||
FRONTEND_HOST=0.0.0.0 Public web bind address
|
||||
GO_VERSION=1.23.12
|
||||
INSTALL_DEPS=0 Do not install missing Node/Go/ffmpeg
|
||||
INSTALL_DEPS=0 Do not install missing Node/Go/ffmpeg/Python runtime deps
|
||||
CONFIGURE_UFW=0 Do not open UFW port automatically
|
||||
DEPLOY_USER=<user> Service user; defaults to sudo user or root
|
||||
|
||||
@@ -130,7 +130,25 @@ apt_install() {
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
log "installing base packages"
|
||||
apt-get update
|
||||
apt-get install -y ca-certificates curl git ffmpeg openssl iproute2 build-essential
|
||||
apt-get install -y ca-certificates curl git ffmpeg openssl iproute2 build-essential \
|
||||
python3 python3-requests python3-bs4 python3-lxml python3-socks
|
||||
}
|
||||
|
||||
verify_spider91_python_deps() {
|
||||
command -v python3 >/dev/null 2>&1 || die "python3 is required for 91Spider"
|
||||
python3 - <<'PY' || die "missing Python modules for 91Spider: requests, bs4, lxml, socks"
|
||||
import importlib.util
|
||||
import sys
|
||||
|
||||
missing = [
|
||||
name
|
||||
for name in ("requests", "bs4", "lxml", "socks")
|
||||
if importlib.util.find_spec(name) is None
|
||||
]
|
||||
if missing:
|
||||
print("missing Python modules: " + ", ".join(missing), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
install_node() {
|
||||
@@ -182,6 +200,7 @@ install_dependencies() {
|
||||
install_go
|
||||
command -v ffmpeg >/dev/null 2>&1 || die "ffmpeg is required"
|
||||
command -v ffprobe >/dev/null 2>&1 || die "ffprobe is required"
|
||||
verify_spider91_python_deps
|
||||
}
|
||||
|
||||
ensure_ownership() {
|
||||
@@ -315,8 +334,8 @@ EOF
|
||||
}
|
||||
|
||||
open_firewall_port() {
|
||||
[[ "$CONFIGURE_UFW" == "1" ]] || return
|
||||
command -v ufw >/dev/null 2>&1 || return
|
||||
[[ "$CONFIGURE_UFW" == "1" ]] || return 0
|
||||
command -v ufw >/dev/null 2>&1 || return 0
|
||||
if ufw status 2>/dev/null | grep -qi "Status: active"; then
|
||||
log "UFW is active; allowing ${FRONTEND_PORT}/tcp"
|
||||
ufw allow "${FRONTEND_PORT}/tcp"
|
||||
@@ -359,7 +378,9 @@ install_or_update() {
|
||||
open_firewall_port
|
||||
restart_services
|
||||
show_status
|
||||
[[ "$mode" == "install" ]] && show_summary
|
||||
if [[ "$mode" == "install" ]]; then
|
||||
show_summary
|
||||
fi
|
||||
}
|
||||
|
||||
uninstall_services() {
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
services:
|
||||
video-site-91:
|
||||
image: ghcr.io/nianzhibai/91:stable
|
||||
container_name: video-site-91
|
||||
ports:
|
||||
- "9191:9191"
|
||||
volumes:
|
||||
- ./data:/opt/video-site-91/data
|
||||
restart: unless-stopped
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
APP_DIR="/opt/video-site-91"
|
||||
DATA_DIR="${VIDEO_DATA_DIR:-$APP_DIR/data}"
|
||||
CONFIG="${VIDEO_CONFIG:-$DATA_DIR/config.yaml}"
|
||||
EXAMPLE="$APP_DIR/config.example.yaml"
|
||||
PORT="${VIDEO_LISTEN_PORT:-9191}"
|
||||
|
||||
mkdir -p "$DATA_DIR" "$DATA_DIR/previews" "$DATA_DIR/uploads" "$DATA_DIR/spider91"
|
||||
|
||||
if [ ! -f "$CONFIG" ]; then
|
||||
if [ ! -f "$EXAMPLE" ]; then
|
||||
echo "[entrypoint] missing config template: $EXAMPLE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$(dirname "$CONFIG")"
|
||||
cp "$EXAMPLE" "$CONFIG"
|
||||
|
||||
SECRET="$(openssl rand -hex 32)"
|
||||
sed -i -E "s#^([[:space:]]*listen:[[:space:]]*).*\$#\1\"0.0.0.0:${PORT}\"#" "$CONFIG"
|
||||
sed -i -E "s#^([[:space:]]*session_secret:[[:space:]]*).*\$#\1\"${SECRET}\"#" "$CONFIG"
|
||||
sed -i -E "s#^([[:space:]]*db_path:[[:space:]]*).*\$#\1\"${DATA_DIR}/video-site.db\"#" "$CONFIG"
|
||||
sed -i -E "s#^([[:space:]]*local_preview_dir:[[:space:]]*).*\$#\1\"${DATA_DIR}/previews\"#" "$CONFIG"
|
||||
chmod 600 "$CONFIG"
|
||||
|
||||
echo "[entrypoint] generated $CONFIG"
|
||||
else
|
||||
echo "[entrypoint] using existing $CONFIG"
|
||||
fi
|
||||
|
||||
if [ -n "${VIDEO_VERSION_FILE:-}" ] && [ -n "${VIDEO_IMAGE_VERSION:-}" ]; then
|
||||
mkdir -p "$(dirname "$VIDEO_VERSION_FILE")"
|
||||
printf '%s\n' "$VIDEO_IMAGE_VERSION" > "$VIDEO_VERSION_FILE"
|
||||
fi
|
||||
|
||||
exec "$@"
|
||||
+443
-41
@@ -11,6 +11,12 @@ VERSION="${VERSION:-latest}"
|
||||
GH_PROXY="${GH_PROXY:-}"
|
||||
CONFIGURE_UFW="${CONFIGURE_UFW:-1}"
|
||||
INSTALL_DEPS="${INSTALL_DEPS:-1}"
|
||||
SELF_UPDATE="${SELF_UPDATE:-1}"
|
||||
FORCE_UPDATE="${FORCE_UPDATE:-0}"
|
||||
INSTALL_SCRIPT_REF="${INSTALL_SCRIPT_REF:-main}"
|
||||
INSTALL_SCRIPT_URL="${INSTALL_SCRIPT_URL:-${GH_PROXY}https://raw.githubusercontent.com/${GITHUB_REPO}/${INSTALL_SCRIPT_REF}/install.sh}"
|
||||
VIDEO_SITE_SKIP_SELF_UPDATE="${VIDEO_SITE_SKIP_SELF_UPDATE:-0}"
|
||||
SERVICE_READY_TIMEOUT="${SERVICE_READY_TIMEOUT:-90}"
|
||||
VERSION_FILE="$INSTALL_PATH/.version"
|
||||
MANAGER_PATH="/usr/local/sbin/${APP_NAME}-manager"
|
||||
COMMAND_LINK="/usr/local/bin/91"
|
||||
@@ -47,7 +53,7 @@ Default action:
|
||||
|
||||
Actions:
|
||||
install Install to $INSTALL_PATH
|
||||
update Download latest release and replace program files, keeping config/data
|
||||
update Refresh manager script, download latest release, and keep config/data
|
||||
restart Restart service
|
||||
stop Stop service
|
||||
status Show service status
|
||||
@@ -62,6 +68,12 @@ Options via environment:
|
||||
GH_PROXY=$GH_PROXY
|
||||
INSTALL_DEPS=$INSTALL_DEPS
|
||||
CONFIGURE_UFW=$CONFIGURE_UFW
|
||||
SELF_UPDATE=$SELF_UPDATE
|
||||
FORCE_UPDATE=$FORCE_UPDATE
|
||||
UNINSTALL_DELETE_FILES=0 Set to 1 for non-interactive uninstall to delete $INSTALL_PATH
|
||||
INSTALL_SCRIPT_REF=$INSTALL_SCRIPT_REF
|
||||
INSTALL_SCRIPT_URL=$INSTALL_SCRIPT_URL
|
||||
SERVICE_READY_TIMEOUT=$SERVICE_READY_TIMEOUT
|
||||
|
||||
Examples:
|
||||
sudo bash install.sh
|
||||
@@ -110,6 +122,27 @@ asset_name() {
|
||||
printf '%s-linux-%s.tar.gz' "$APP_NAME" "$ARCH"
|
||||
}
|
||||
|
||||
verify_runtime_deps() {
|
||||
local cmd
|
||||
for cmd in curl tar ffmpeg ffprobe openssl python3; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || die "missing command: $cmd"
|
||||
done
|
||||
|
||||
python3 - <<'PY' || die "missing Python modules for 91Spider: requests, bs4, lxml, socks"
|
||||
import importlib.util
|
||||
import sys
|
||||
|
||||
missing = [
|
||||
name
|
||||
for name in ("requests", "bs4", "lxml", "socks")
|
||||
if importlib.util.find_spec(name) is None
|
||||
]
|
||||
if missing:
|
||||
print("missing Python modules: " + ", ".join(missing), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
install_deps() {
|
||||
if [[ "$INSTALL_DEPS" != "1" ]]; then
|
||||
return
|
||||
@@ -118,13 +151,12 @@ install_deps() {
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
log "installing runtime dependencies"
|
||||
apt-get update
|
||||
apt-get install -y ca-certificates curl tar ffmpeg openssl iproute2 python3 python3-requests python3-bs4 python3-lxml
|
||||
apt-get install -y ca-certificates curl tar ffmpeg openssl iproute2 python3 python3-requests python3-bs4 python3-lxml python3-socks
|
||||
verify_runtime_deps
|
||||
return
|
||||
fi
|
||||
|
||||
for cmd in curl tar ffmpeg ffprobe openssl; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || die "missing command: $cmd"
|
||||
done
|
||||
verify_runtime_deps
|
||||
}
|
||||
|
||||
check_system() {
|
||||
@@ -158,6 +190,30 @@ download_file() {
|
||||
return 1
|
||||
}
|
||||
|
||||
backup_install_files() {
|
||||
local backup="$1"
|
||||
mkdir -p "$backup"
|
||||
cp -a "$INSTALL_PATH/server" "$backup/server"
|
||||
for item in dist config.example.yaml 91VideoSpider config.yaml .version; do
|
||||
if [[ -e "$INSTALL_PATH/$item" ]]; then
|
||||
cp -a "$INSTALL_PATH/$item" "$backup/$item"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
restore_install_files() {
|
||||
local backup="$1"
|
||||
mkdir -p "$INSTALL_PATH"
|
||||
cp -a "$backup/server" "$INSTALL_PATH/server"
|
||||
for item in dist config.example.yaml 91VideoSpider config.yaml .version; do
|
||||
rm -rf "${INSTALL_PATH:?}/$item"
|
||||
if [[ -e "$backup/$item" ]]; then
|
||||
cp -a "$backup/$item" "$INSTALL_PATH/$item"
|
||||
fi
|
||||
done
|
||||
chmod +x "$INSTALL_PATH/server"
|
||||
}
|
||||
|
||||
prepare_config() {
|
||||
local cfg="$INSTALL_PATH/config.yaml"
|
||||
local example="$INSTALL_PATH/config.example.yaml"
|
||||
@@ -200,6 +256,8 @@ RestartSec=5
|
||||
TimeoutStopSec=20
|
||||
Environment=VIDEO_CONFIG=${INSTALL_PATH}/config.yaml
|
||||
Environment=VIDEO_FRONTEND_DIR=${INSTALL_PATH}/dist
|
||||
Environment=VIDEO_VERSION_FILE=${VERSION_FILE}
|
||||
Environment=VIDEO_GITHUB_REPO=${GITHUB_REPO}
|
||||
Environment=HOME=/root
|
||||
Environment=PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
|
||||
LimitNOFILE=65536
|
||||
@@ -217,23 +275,297 @@ EOF
|
||||
install_cli() {
|
||||
local src
|
||||
src="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")"
|
||||
if [[ -f "$src" ]]; then
|
||||
cp "$src" "$MANAGER_PATH"
|
||||
chmod 755 "$MANAGER_PATH"
|
||||
ln -sf "$MANAGER_PATH" "$COMMAND_LINK"
|
||||
ln -sf "$MANAGER_PATH" "$APP_COMMAND_LINK"
|
||||
install_cli_from_file "$src"
|
||||
}
|
||||
|
||||
install_cli_from_file() {
|
||||
local src="$1"
|
||||
local tmp
|
||||
[[ -f "$src" ]] || return 0
|
||||
mkdir -p "$(dirname "$MANAGER_PATH")" "$(dirname "$COMMAND_LINK")" "$(dirname "$APP_COMMAND_LINK")"
|
||||
tmp="${MANAGER_PATH}.tmp.$$"
|
||||
cp "$src" "$tmp"
|
||||
chmod 755 "$tmp"
|
||||
mv "$tmp" "$MANAGER_PATH"
|
||||
ln -sfn "$MANAGER_PATH" "$COMMAND_LINK"
|
||||
ln -sfn "$MANAGER_PATH" "$APP_COMMAND_LINK"
|
||||
}
|
||||
|
||||
self_update_manager() {
|
||||
[[ "$SELF_UPDATE" == "1" ]] || return 1
|
||||
[[ "$VIDEO_SITE_SKIP_SELF_UPDATE" != "1" ]] || return 1
|
||||
[[ -n "$INSTALL_SCRIPT_URL" ]] || return 1
|
||||
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
log "checking latest manager script"
|
||||
if ! download_file "$INSTALL_SCRIPT_URL" "$tmp"; then
|
||||
warn "manager self-update skipped: cannot download $INSTALL_SCRIPT_URL"
|
||||
rm -f "$tmp"
|
||||
return 1
|
||||
fi
|
||||
if ! bash -n "$tmp"; then
|
||||
warn "manager self-update skipped: downloaded script has syntax errors"
|
||||
rm -f "$tmp"
|
||||
return 1
|
||||
fi
|
||||
if [[ -f "$MANAGER_PATH" ]] && cmp -s "$tmp" "$MANAGER_PATH"; then
|
||||
rm -f "$tmp"
|
||||
return 1
|
||||
fi
|
||||
|
||||
install_cli_from_file "$tmp"
|
||||
rm -f "$tmp"
|
||||
log "manager script updated"
|
||||
return 0
|
||||
}
|
||||
|
||||
exec_latest_manager_update() {
|
||||
local env_args=(
|
||||
"VIDEO_SITE_SKIP_SELF_UPDATE=1"
|
||||
"APP_NAME=$APP_NAME"
|
||||
"GITHUB_REPO=$GITHUB_REPO"
|
||||
"INSTALL_PATH=$INSTALL_PATH"
|
||||
"SERVICE_NAME=$SERVICE_NAME"
|
||||
"VERSION=$VERSION"
|
||||
"GH_PROXY=$GH_PROXY"
|
||||
"CONFIGURE_UFW=$CONFIGURE_UFW"
|
||||
"INSTALL_DEPS=$INSTALL_DEPS"
|
||||
"SELF_UPDATE=$SELF_UPDATE"
|
||||
"FORCE_UPDATE=$FORCE_UPDATE"
|
||||
"INSTALL_SCRIPT_REF=$INSTALL_SCRIPT_REF"
|
||||
"INSTALL_SCRIPT_URL=$INSTALL_SCRIPT_URL"
|
||||
"SERVICE_READY_TIMEOUT=$SERVICE_READY_TIMEOUT"
|
||||
)
|
||||
if [[ -n "$FRONTEND_PORT_WAS_SET" ]]; then
|
||||
env_args+=("FRONTEND_PORT=$FRONTEND_PORT")
|
||||
fi
|
||||
exec env "${env_args[@]}" bash "$MANAGER_PATH" update
|
||||
}
|
||||
|
||||
open_firewall_port() {
|
||||
[[ "$CONFIGURE_UFW" == "1" ]] || return
|
||||
command -v ufw >/dev/null 2>&1 || return
|
||||
[[ "$CONFIGURE_UFW" == "1" ]] || return 0
|
||||
command -v ufw >/dev/null 2>&1 || return 0
|
||||
if ufw status 2>/dev/null | grep -qi "Status: active"; then
|
||||
log "allowing ${FRONTEND_PORT}/tcp in UFW"
|
||||
ufw allow "${FRONTEND_PORT}/tcp"
|
||||
fi
|
||||
}
|
||||
|
||||
listen_port_from_config() {
|
||||
local cfg="$INSTALL_PATH/config.yaml"
|
||||
local listen="" port
|
||||
if [[ -f "$cfg" ]]; then
|
||||
listen="$(sed -nE 's/^[[:space:]]*listen:[[:space:]]*"?([^" #]+)"?.*/\1/p' "$cfg" | head -n1)"
|
||||
fi
|
||||
port="${listen##*:}"
|
||||
if [[ "$port" =~ ^[0-9]+$ ]]; then
|
||||
printf '%s' "$port"
|
||||
return
|
||||
fi
|
||||
printf '%s' "$FRONTEND_PORT"
|
||||
}
|
||||
|
||||
append_unique() {
|
||||
local value="$1"
|
||||
shift
|
||||
for existing in "$@"; do
|
||||
[[ "$existing" == "$value" ]] && return 1
|
||||
done
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
app_service_names() {
|
||||
local names=()
|
||||
local name
|
||||
for name in "$SERVICE_NAME" "$APP_NAME" video-site-91 video-site-backend video-site-frontend; do
|
||||
[[ -n "$name" ]] || continue
|
||||
if append_unique "$name" "${names[@]}" >/dev/null; then
|
||||
names+=("$name")
|
||||
fi
|
||||
done
|
||||
printf '%s\n' "${names[@]}"
|
||||
}
|
||||
|
||||
stop_app_services() {
|
||||
local name unit
|
||||
while IFS= read -r name; do
|
||||
[[ -n "$name" ]] || continue
|
||||
unit="${name}.service"
|
||||
systemctl disable --now "$unit" 2>/dev/null || systemctl stop "$unit" 2>/dev/null || true
|
||||
rm -f "/etc/systemd/system/$unit"
|
||||
done < <(app_service_names)
|
||||
systemctl daemon-reload
|
||||
}
|
||||
|
||||
remove_app_containers() {
|
||||
command -v docker >/dev/null 2>&1 || return 0
|
||||
|
||||
local names=()
|
||||
local name
|
||||
for name in "$SERVICE_NAME" "$APP_NAME" video-site-91; do
|
||||
[[ -n "$name" ]] || continue
|
||||
if append_unique "$name" "${names[@]}" >/dev/null; then
|
||||
names+=("$name")
|
||||
fi
|
||||
done
|
||||
|
||||
for name in "${names[@]}"; do
|
||||
if docker ps -a --format '{{.Names}}' 2>/dev/null | grep -Fxq "$name"; then
|
||||
log "removing docker container $name"
|
||||
docker rm -f "$name" >/dev/null 2>&1 || true
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
pids_listening_on_port() {
|
||||
local port="$1"
|
||||
[[ "$port" =~ ^[0-9]+$ ]] || return 0
|
||||
command -v ss >/dev/null 2>&1 || return 0
|
||||
|
||||
ss -ltnp 2>/dev/null \
|
||||
| awk -v port="$port" '$4 ~ ":" port "$" {print}' \
|
||||
| grep -oE 'pid=[0-9]+' \
|
||||
| cut -d= -f2 \
|
||||
| sort -u || true
|
||||
}
|
||||
|
||||
process_looks_like_app() {
|
||||
local pid="$1"
|
||||
local exe="" cmd=""
|
||||
exe="$(readlink "/proc/$pid/exe" 2>/dev/null || true)"
|
||||
cmd="$(tr '\0' ' ' <"/proc/$pid/cmdline" 2>/dev/null || true)"
|
||||
|
||||
[[ "$exe" == "$INSTALL_PATH/server" ]] && return 0
|
||||
[[ "$cmd" == *"$INSTALL_PATH"* ]] && return 0
|
||||
[[ "$cmd" == *"VIDEO_FRONTEND_DIR=$INSTALL_PATH/dist"* ]] && return 0
|
||||
[[ "$cmd" == *"VIDEO_CONFIG=$INSTALL_PATH/config.yaml"* ]] && return 0
|
||||
[[ "$cmd" == *"video-site-91"* ]] && return 0
|
||||
[[ "$cmd" == *"91VideoSpider"* ]] && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
stop_lingering_app_processes() {
|
||||
local ports=("$@")
|
||||
local port pid pids=()
|
||||
|
||||
for port in "${ports[@]}"; do
|
||||
[[ "$port" =~ ^[0-9]+$ ]] || continue
|
||||
while IFS= read -r pid; do
|
||||
[[ -n "$pid" ]] || continue
|
||||
process_looks_like_app "$pid" || continue
|
||||
if append_unique "$pid" "${pids[@]}" >/dev/null; then
|
||||
pids+=("$pid")
|
||||
fi
|
||||
done < <(pids_listening_on_port "$port")
|
||||
done
|
||||
|
||||
if (( ${#pids[@]} == 0 )); then
|
||||
return
|
||||
fi
|
||||
|
||||
warn "stopping lingering app process(es): ${pids[*]}"
|
||||
kill "${pids[@]}" 2>/dev/null || true
|
||||
sleep 1
|
||||
|
||||
local alive=()
|
||||
for pid in "${pids[@]}"; do
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
alive+=("$pid")
|
||||
fi
|
||||
done
|
||||
if (( ${#alive[@]} > 0 )); then
|
||||
warn "force killing lingering app process(es): ${alive[*]}"
|
||||
kill -9 "${alive[@]}" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
warn_remaining_listeners() {
|
||||
local ports=("$@")
|
||||
local port pid cmd
|
||||
for port in "${ports[@]}"; do
|
||||
[[ "$port" =~ ^[0-9]+$ ]] || continue
|
||||
while IFS= read -r pid; do
|
||||
[[ -n "$pid" ]] || continue
|
||||
cmd="$(tr '\0' ' ' <"/proc/$pid/cmdline" 2>/dev/null || true)"
|
||||
warn "port $port is still listening after uninstall: pid=$pid ${cmd:-unknown}"
|
||||
done < <(pids_listening_on_port "$port")
|
||||
done
|
||||
}
|
||||
|
||||
has_interactive_tty() {
|
||||
[[ -t 0 ]]
|
||||
}
|
||||
|
||||
confirm_uninstall_app() {
|
||||
if ! has_interactive_tty; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local confirm=""
|
||||
printf '确认卸载 91 吗?这会停止服务、移除管理命令,并可选择是否删除项目文件。[y/N]: ' >/dev/tty
|
||||
IFS= read -r confirm </dev/tty || confirm=""
|
||||
case "$confirm" in
|
||||
[yY]) return 0 ;;
|
||||
*)
|
||||
log "uninstall cancelled"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
delete_install_path_requested() {
|
||||
if [[ "${UNINSTALL_DELETE_FILES:-0}" == "1" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if ! has_interactive_tty; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
local confirm=""
|
||||
printf '删除 %s 里的程序、配置和数据吗?[y/N]: ' "$INSTALL_PATH" >/dev/tty
|
||||
IFS= read -r confirm </dev/tty || confirm=""
|
||||
case "$confirm" in
|
||||
[yY]) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
service_health_url() {
|
||||
printf 'http://127.0.0.1:%s/admin/api/setup' "$(listen_port_from_config)"
|
||||
}
|
||||
|
||||
wait_for_service_ready() {
|
||||
local url deadline
|
||||
url="$(service_health_url)"
|
||||
deadline=$((SECONDS + SERVICE_READY_TIMEOUT))
|
||||
log "waiting for service at $url"
|
||||
while (( SECONDS < deadline )); do
|
||||
if curl -fsS --connect-timeout 2 --max-time 5 "$url" >/dev/null 2>&1; then
|
||||
log "service is ready"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
restart_service_ready() {
|
||||
if systemctl restart "${SERVICE_NAME}.service" && wait_for_service_ready; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
warn "service did not become ready; retrying restart"
|
||||
if systemctl restart "${SERVICE_NAME}.service" && wait_for_service_ready; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
warn "service failed to become ready"
|
||||
systemctl --no-pager --full status "${SERVICE_NAME}.service" || true
|
||||
journalctl -u "${SERVICE_NAME}.service" -n 80 --no-pager || true
|
||||
return 1
|
||||
}
|
||||
|
||||
fetch_and_unpack() {
|
||||
local tmp archive url root
|
||||
tmp="$(mktemp -d)"
|
||||
@@ -271,19 +603,63 @@ fetch_and_unpack() {
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
current_version_from_github() {
|
||||
installed_version() {
|
||||
if [[ -f "$VERSION_FILE" ]]; then
|
||||
head -n1 "$VERSION_FILE" 2>/dev/null | tr -d '\r'
|
||||
fi
|
||||
}
|
||||
|
||||
target_version() {
|
||||
if [[ "$VERSION" != "latest" ]]; then
|
||||
printf '%s' "$VERSION"
|
||||
return
|
||||
fi
|
||||
curl -fsSL "https://api.github.com/repos/${GITHUB_REPO}/releases/latest" \
|
||||
|
||||
local body version effective_url
|
||||
body="$(curl -fsSL \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "User-Agent: video-site-91-installer" \
|
||||
"https://api.github.com/repos/${GITHUB_REPO}/releases/latest" 2>/dev/null || true)"
|
||||
version="$(printf '%s\n' "$body" \
|
||||
| sed -nE 's/.*"tag_name"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/p' \
|
||||
| head -n1)"
|
||||
if [[ -n "$version" ]]; then
|
||||
printf '%s' "$version"
|
||||
return
|
||||
fi
|
||||
|
||||
effective_url="$(curl -fsSLI -o /dev/null -w '%{url_effective}' "$(download_base_url)/$(asset_name)" 2>/dev/null || true)"
|
||||
printf '%s\n' "$effective_url" \
|
||||
| sed -nE 's#.*/releases/download/([^/]+)/.*#\1#p' \
|
||||
| head -n1
|
||||
}
|
||||
|
||||
should_skip_update() {
|
||||
[[ "$FORCE_UPDATE" != "1" ]] || return 1
|
||||
|
||||
local current target
|
||||
current="$(installed_version)"
|
||||
target="$(target_version || true)"
|
||||
|
||||
if [[ -z "$target" ]]; then
|
||||
warn "cannot determine target version; continuing update"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -z "$current" ]]; then
|
||||
log "installed version: unknown"
|
||||
log "target version: $target"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "installed version: $current"
|
||||
log "target version: $target"
|
||||
[[ "$current" == "$target" ]]
|
||||
}
|
||||
|
||||
record_version() {
|
||||
local version
|
||||
version="$(current_version_from_github || true)"
|
||||
version="$(target_version || true)"
|
||||
[[ -n "$version" ]] || version="$VERSION"
|
||||
{
|
||||
echo "$version"
|
||||
@@ -298,7 +674,7 @@ show_success() {
|
||||
version="$(head -n1 "$VERSION_FILE" 2>/dev/null || echo unknown)"
|
||||
|
||||
echo
|
||||
printf "${GREEN}安装完成${RESET}\n"
|
||||
printf '%b安装完成%b\n' "$GREEN" "$RESET"
|
||||
echo "版本:$version"
|
||||
[[ -n "$local_ip" ]] && echo "局域网:http://${local_ip}:${FRONTEND_PORT}/"
|
||||
[[ -n "$public_ip" ]] && echo "公网: http://${public_ip}:${FRONTEND_PORT}/"
|
||||
@@ -319,57 +695,79 @@ install_app() {
|
||||
write_service
|
||||
install_cli
|
||||
open_firewall_port
|
||||
restart_service_ready || die "service failed to start"
|
||||
record_version
|
||||
systemctl restart "${SERVICE_NAME}.service"
|
||||
show_success
|
||||
}
|
||||
|
||||
update_app() {
|
||||
check_system
|
||||
check_disk_space
|
||||
install_deps
|
||||
[[ -f "$INSTALL_PATH/server" ]] || die "not installed at $INSTALL_PATH"
|
||||
|
||||
if self_update_manager; then
|
||||
log "re-running update with latest manager script"
|
||||
exec_latest_manager_update
|
||||
fi
|
||||
|
||||
install_deps
|
||||
|
||||
if should_skip_update; then
|
||||
log "already up to date; skipped app update"
|
||||
return 0
|
||||
fi
|
||||
|
||||
check_disk_space
|
||||
|
||||
local backup
|
||||
backup="$(mktemp -d)"
|
||||
cp "$INSTALL_PATH/server" "$backup/server"
|
||||
[[ -d "$INSTALL_PATH/dist" ]] && cp -R "$INSTALL_PATH/dist" "$backup/dist"
|
||||
backup_install_files "$backup"
|
||||
|
||||
systemctl stop "${SERVICE_NAME}.service" 2>/dev/null || true
|
||||
if ! fetch_and_unpack; then
|
||||
if ! (fetch_and_unpack && prepare_config && write_service && install_cli); then
|
||||
warn "update failed; restoring previous files"
|
||||
cp "$backup/server" "$INSTALL_PATH/server"
|
||||
rm -rf "$INSTALL_PATH/dist"
|
||||
[[ -d "$backup/dist" ]] && cp -R "$backup/dist" "$INSTALL_PATH/dist"
|
||||
restore_install_files "$backup"
|
||||
systemctl start "${SERVICE_NAME}.service" 2>/dev/null || true
|
||||
rm -rf "$backup"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
prepare_config
|
||||
write_service
|
||||
install_cli
|
||||
if ! restart_service_ready; then
|
||||
warn "new version failed to start; restoring previous files"
|
||||
restore_install_files "$backup"
|
||||
restart_service_ready 2>/dev/null || true
|
||||
rm -rf "$backup"
|
||||
exit 1
|
||||
fi
|
||||
record_version
|
||||
systemctl restart "${SERVICE_NAME}.service"
|
||||
rm -rf "$backup"
|
||||
log "updated"
|
||||
}
|
||||
|
||||
uninstall_app() {
|
||||
systemctl disable --now "${SERVICE_NAME}.service" 2>/dev/null || true
|
||||
rm -f "/etc/systemd/system/${SERVICE_NAME}.service"
|
||||
systemctl daemon-reload
|
||||
local listen_port port ports=()
|
||||
confirm_uninstall_app || return 1
|
||||
|
||||
listen_port="$(listen_port_from_config)"
|
||||
for port in "$listen_port" "$FRONTEND_PORT" 9191 9192; do
|
||||
[[ "$port" =~ ^[0-9]+$ ]] || continue
|
||||
if append_unique "$port" "${ports[@]}" >/dev/null; then
|
||||
ports+=("$port")
|
||||
fi
|
||||
done
|
||||
|
||||
stop_app_services
|
||||
remove_app_containers
|
||||
stop_lingering_app_processes "${ports[@]}"
|
||||
rm -f "$COMMAND_LINK" "$APP_COMMAND_LINK" "$MANAGER_PATH"
|
||||
|
||||
if [[ -t 0 ]]; then
|
||||
read -r -p "删除 $INSTALL_PATH 里的程序、配置和数据吗?[y/N]: " confirm
|
||||
case "$confirm" in
|
||||
[yY]) rm -rf "$INSTALL_PATH" ;;
|
||||
*) log "kept $INSTALL_PATH" ;;
|
||||
esac
|
||||
if delete_install_path_requested; then
|
||||
rm -rf "$INSTALL_PATH"
|
||||
log "removed $INSTALL_PATH"
|
||||
else
|
||||
log "removed service; kept $INSTALL_PATH"
|
||||
log "kept $INSTALL_PATH"
|
||||
fi
|
||||
|
||||
warn_remaining_listeners "${ports[@]}"
|
||||
}
|
||||
|
||||
show_menu() {
|
||||
@@ -399,7 +797,11 @@ show_menu() {
|
||||
3) main update ;;
|
||||
4) main restart ;;
|
||||
5) main stop ;;
|
||||
6) main uninstall ;;
|
||||
6)
|
||||
if main uninstall; then
|
||||
exit 0
|
||||
fi
|
||||
;;
|
||||
0) exit 0 ;;
|
||||
*) echo "无效的选项" ;;
|
||||
esac
|
||||
@@ -430,7 +832,7 @@ main() {
|
||||
;;
|
||||
restart)
|
||||
need_root "$@"
|
||||
systemctl restart "${SERVICE_NAME}.service"
|
||||
restart_service_ready || die "service failed to start"
|
||||
;;
|
||||
stop)
|
||||
need_root "$@"
|
||||
|
||||
Generated
+830
-1531
File diff suppressed because it is too large
Load Diff
+5
-4
@@ -1,6 +1,7 @@
|
||||
{
|
||||
"name": "video-site",
|
||||
"private": true,
|
||||
"license": "MIT",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
@@ -15,14 +16,14 @@
|
||||
"lucide-react": "0.453.0",
|
||||
"react": "18.3.1",
|
||||
"react-dom": "18.3.1",
|
||||
"react-router-dom": "6.26.2"
|
||||
"react-router-dom": "^6.30.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "18.3.12",
|
||||
"@types/react-dom": "18.3.1",
|
||||
"@vitejs/plugin-react": "4.3.3",
|
||||
"tsx": "^4.19.2",
|
||||
"@vitejs/plugin-react": "^6.0.2",
|
||||
"tsx": "^4.22.3",
|
||||
"typescript": "5.6.3",
|
||||
"vite": "5.4.10"
|
||||
"vite": "^8.0.14"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,6 +61,7 @@ build_package() {
|
||||
)
|
||||
|
||||
cp "$ROOT_DIR/backend/config.example.yaml" "$work/config.example.yaml"
|
||||
cp "$ROOT_DIR/install.sh" "$work/install.sh"
|
||||
cp -R "$ROOT_DIR/dist" "$work/dist"
|
||||
mkdir -p "$work/91VideoSpider"
|
||||
cp "$ROOT_DIR/91VideoSpider/spider_91porn.py" "$work/91VideoSpider/spider_91porn.py"
|
||||
@@ -69,10 +70,11 @@ build_package() {
|
||||
$APP_NAME $VERSION
|
||||
|
||||
This is a prebuilt release package.
|
||||
Use install.sh from the repository to install it on a Linux server.
|
||||
Use install.sh in this package or from the repository to install it on a Linux server.
|
||||
EOF
|
||||
|
||||
chmod +x "$work/server"
|
||||
chmod +x "$work/install.sh"
|
||||
tar -C "$OUT_DIR/.work" -czf "$OUT_DIR/$artifact.tar.gz" "$artifact"
|
||||
log "wrote $OUT_DIR/$artifact.tar.gz"
|
||||
}
|
||||
|
||||
@@ -1,5 +1,17 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { NavLink, Outlet, useNavigate } from "react-router-dom";
|
||||
import { HardDrive, Film, LogOut, Play, Home, Tags, Palette } from "lucide-react";
|
||||
import {
|
||||
HardDrive,
|
||||
Film,
|
||||
LogOut,
|
||||
Play,
|
||||
Home,
|
||||
Tags,
|
||||
Palette,
|
||||
RefreshCw,
|
||||
MoreVertical,
|
||||
} from "lucide-react";
|
||||
import * as api from "./api";
|
||||
import { useAuth } from "./AuthContext";
|
||||
import { useToast } from "./ToastContext";
|
||||
|
||||
@@ -7,6 +19,43 @@ export function AdminLayout() {
|
||||
const { logout } = useAuth();
|
||||
const navigate = useNavigate();
|
||||
const { show } = useToast();
|
||||
const [checkingUpdate, setCheckingUpdate] = useState(false);
|
||||
const [mobileMenuOpen, setMobileMenuOpen] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!mobileMenuOpen) return;
|
||||
function onKeyDown(e: KeyboardEvent) {
|
||||
if (e.key === "Escape") {
|
||||
setMobileMenuOpen(false);
|
||||
}
|
||||
}
|
||||
document.addEventListener("keydown", onKeyDown);
|
||||
return () => document.removeEventListener("keydown", onKeyDown);
|
||||
}, [mobileMenuOpen]);
|
||||
|
||||
async function handleCheckUpdate() {
|
||||
if (checkingUpdate) return;
|
||||
setCheckingUpdate(true);
|
||||
try {
|
||||
const result = await api.checkUpdate();
|
||||
if (result.hasUpdate) {
|
||||
show(
|
||||
`发现新版本 ${result.latestVersion},当前 ${result.currentVersion}`,
|
||||
"success"
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (result.currentVersion === "unknown") {
|
||||
show(`当前版本未知,GitHub 最新版本为 ${result.latestVersion}`, "info");
|
||||
return;
|
||||
}
|
||||
show(`当前已是最新版本:${result.currentVersion}`, "success");
|
||||
} catch {
|
||||
show("检查更新失败,请稍后重试", "error");
|
||||
} finally {
|
||||
setCheckingUpdate(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleLogout() {
|
||||
try {
|
||||
@@ -65,12 +114,44 @@ export function AdminLayout() {
|
||||
</NavLink>
|
||||
</nav>
|
||||
<div className="admin-sidebar__footer">
|
||||
<button
|
||||
className="admin-sidebar__check-update"
|
||||
onClick={handleCheckUpdate}
|
||||
disabled={checkingUpdate}
|
||||
>
|
||||
<RefreshCw size={14} />
|
||||
{checkingUpdate ? "检查中" : "检查更新"}
|
||||
</button>
|
||||
<button className="admin-sidebar__logout" onClick={handleLogout}>
|
||||
<LogOut size={14} style={{ verticalAlign: -2, marginRight: 4 }} />
|
||||
<LogOut size={14} />
|
||||
退出登录
|
||||
</button>
|
||||
</div>
|
||||
<button
|
||||
className="admin-sidebar__mobile-menu"
|
||||
onClick={() => setMobileMenuOpen((v) => !v)}
|
||||
aria-label="更多操作"
|
||||
>
|
||||
<MoreVertical size={18} />
|
||||
</button>
|
||||
</aside>
|
||||
{mobileMenuOpen && (
|
||||
<div className="admin-sidebar__mobile-overlay" onClick={() => setMobileMenuOpen(false)} />
|
||||
)}
|
||||
<div className={`admin-sidebar__mobile-panel${mobileMenuOpen ? " is-open" : ""}`}>
|
||||
<button
|
||||
className="admin-sidebar__check-update"
|
||||
onClick={() => { handleCheckUpdate(); setMobileMenuOpen(false); }}
|
||||
disabled={checkingUpdate}
|
||||
>
|
||||
<RefreshCw size={14} />
|
||||
{checkingUpdate ? "检查中" : "检查更新"}
|
||||
</button>
|
||||
<button className="admin-sidebar__logout" onClick={() => { handleLogout(); setMobileMenuOpen(false); }}>
|
||||
<LogOut size={14} />
|
||||
退出登录
|
||||
</button>
|
||||
</div>
|
||||
<main className="admin-main">
|
||||
<Outlet />
|
||||
</main>
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import { AlertTriangle } from "lucide-react";
|
||||
import { Modal } from "./Modal";
|
||||
|
||||
type ConfirmModalProps = {
|
||||
open: boolean;
|
||||
title: string;
|
||||
message: string;
|
||||
details?: string[];
|
||||
confirmText?: string;
|
||||
cancelText?: string;
|
||||
danger?: boolean;
|
||||
centerMessage?: boolean;
|
||||
modalClassName?: string;
|
||||
loading?: boolean;
|
||||
onCancel: () => void;
|
||||
onConfirm: () => void;
|
||||
};
|
||||
|
||||
export function ConfirmModal({
|
||||
open,
|
||||
title,
|
||||
message,
|
||||
details,
|
||||
confirmText = "确认",
|
||||
cancelText = "取消",
|
||||
danger = false,
|
||||
centerMessage = false,
|
||||
modalClassName = "",
|
||||
loading = false,
|
||||
onCancel,
|
||||
onConfirm,
|
||||
}: ConfirmModalProps) {
|
||||
return (
|
||||
<Modal
|
||||
open={open}
|
||||
title={title}
|
||||
onClose={onCancel}
|
||||
className={modalClassName}
|
||||
footer={
|
||||
<>
|
||||
<button type="button" className="admin-btn" onClick={onCancel} disabled={loading}>
|
||||
{cancelText}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`admin-btn${danger ? " is-danger" : " is-primary"}`}
|
||||
onClick={onConfirm}
|
||||
disabled={loading}
|
||||
>
|
||||
{loading ? "处理中..." : confirmText}
|
||||
</button>
|
||||
</>
|
||||
}
|
||||
>
|
||||
<div className={`admin-confirm${centerMessage ? " is-message-centered" : ""}`}>
|
||||
<div className={`admin-confirm__icon${danger ? " is-danger" : ""}`} aria-hidden={centerMessage}>
|
||||
<AlertTriangle size={20} />
|
||||
</div>
|
||||
<div className="admin-confirm__content">
|
||||
<p className="admin-confirm__message">{message}</p>
|
||||
{details && details.length > 0 && (
|
||||
<ul className="admin-confirm__list">
|
||||
{details.map((item) => (
|
||||
<li key={item}>{item}</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
+428
-1127
File diff suppressed because it is too large
Load Diff
+16
-9
@@ -16,6 +16,7 @@ export function LoginPage() {
|
||||
const navigate = useNavigate();
|
||||
const location = useLocation();
|
||||
const { show } = useToast();
|
||||
const passwordMismatch = setupRequired === true && p2.length > 0 && p !== p2;
|
||||
|
||||
useEffect(() => {
|
||||
let active = true;
|
||||
@@ -82,14 +83,10 @@ export function LoginPage() {
|
||||
<Play size={18} fill="currentColor" /> {setupRequired ? "首次设置管理员" : "登录"}
|
||||
</h1>
|
||||
<div className="admin-form">
|
||||
{setupRequired && (
|
||||
<div className="admin-form__help admin-form__help--lead">
|
||||
请先设置后台管理员账号。保存后会写入本机配置文件,之后使用该账号登录。
|
||||
</div>
|
||||
)}
|
||||
<div className="admin-form__row">
|
||||
<label>用户名</label>
|
||||
<label htmlFor="admin-login-username">用户名</label>
|
||||
<input
|
||||
id="admin-login-username"
|
||||
autoFocus
|
||||
value={u}
|
||||
onChange={(e) => setU(e.target.value)}
|
||||
@@ -97,8 +94,9 @@ export function LoginPage() {
|
||||
/>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>密码</label>
|
||||
<label htmlFor="admin-login-password">密码</label>
|
||||
<input
|
||||
id="admin-login-password"
|
||||
type="password"
|
||||
value={p}
|
||||
onChange={(e) => setP(e.target.value)}
|
||||
@@ -107,19 +105,28 @@ export function LoginPage() {
|
||||
</div>
|
||||
{setupRequired && (
|
||||
<div className="admin-form__row">
|
||||
<label>确认密码</label>
|
||||
<label htmlFor="admin-login-password-confirm">确认密码</label>
|
||||
<input
|
||||
id="admin-login-password-confirm"
|
||||
type="password"
|
||||
value={p2}
|
||||
onChange={(e) => setP2(e.target.value)}
|
||||
autoComplete="new-password"
|
||||
className={passwordMismatch ? "is-invalid" : undefined}
|
||||
aria-invalid={passwordMismatch ? "true" : undefined}
|
||||
aria-describedby={passwordMismatch ? "admin-login-password-confirm-error" : undefined}
|
||||
/>
|
||||
{passwordMismatch && (
|
||||
<div className="admin-form__error" id="admin-login-password-confirm-error">
|
||||
密码不一致
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
<button
|
||||
className="admin-btn is-primary"
|
||||
type="submit"
|
||||
disabled={loading || !u || !p || (setupRequired && !p2)}
|
||||
disabled={loading || !u || !p || (setupRequired && (!p2 || passwordMismatch))}
|
||||
>
|
||||
{loading
|
||||
? setupRequired
|
||||
|
||||
+97
-4
@@ -1,4 +1,4 @@
|
||||
import { ReactNode } from "react";
|
||||
import { useEffect, useId, useRef, ReactNode } from "react";
|
||||
import { X } from "lucide-react";
|
||||
|
||||
type Props = {
|
||||
@@ -7,9 +7,72 @@ type Props = {
|
||||
onClose: () => void;
|
||||
children: ReactNode;
|
||||
footer?: ReactNode;
|
||||
className?: string;
|
||||
};
|
||||
|
||||
export function Modal({ open, title, onClose, children, footer }: Props) {
|
||||
export function Modal({ open, title, onClose, children, footer, className = "" }: Props) {
|
||||
const dialogRef = useRef<HTMLDivElement>(null);
|
||||
const onCloseRef = useRef(onClose);
|
||||
const titleId = useId();
|
||||
|
||||
useEffect(() => {
|
||||
onCloseRef.current = onClose;
|
||||
}, [onClose]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) return;
|
||||
const previousFocus =
|
||||
document.activeElement instanceof HTMLElement ? document.activeElement : null;
|
||||
|
||||
function onKeyDown(e: KeyboardEvent) {
|
||||
const dialog = dialogRef.current;
|
||||
if (!dialog || !isTopDialog(dialog)) return;
|
||||
|
||||
if (e.key === "Escape") {
|
||||
e.preventDefault();
|
||||
onCloseRef.current();
|
||||
return;
|
||||
}
|
||||
|
||||
if (e.key !== "Tab") return;
|
||||
|
||||
const focusable = getFocusableElements(dialog);
|
||||
if (focusable.length === 0) {
|
||||
e.preventDefault();
|
||||
dialog.focus();
|
||||
return;
|
||||
}
|
||||
|
||||
const first = focusable[0];
|
||||
const last = focusable[focusable.length - 1];
|
||||
const current = document.activeElement;
|
||||
|
||||
if (e.shiftKey && current === first) {
|
||||
e.preventDefault();
|
||||
last.focus();
|
||||
} else if (!e.shiftKey && current === last) {
|
||||
e.preventDefault();
|
||||
first.focus();
|
||||
}
|
||||
}
|
||||
|
||||
const focusTimer = window.setTimeout(() => {
|
||||
const dialog = dialogRef.current;
|
||||
if (!dialog || !isTopDialog(dialog)) return;
|
||||
const first = getFocusableElements(dialog)[0];
|
||||
(first ?? dialog).focus();
|
||||
}, 0);
|
||||
|
||||
document.addEventListener("keydown", onKeyDown);
|
||||
return () => {
|
||||
window.clearTimeout(focusTimer);
|
||||
document.removeEventListener("keydown", onKeyDown);
|
||||
if (previousFocus?.isConnected) {
|
||||
previousFocus.focus();
|
||||
}
|
||||
};
|
||||
}, [open]);
|
||||
|
||||
if (!open) return null;
|
||||
return (
|
||||
<div
|
||||
@@ -18,10 +81,18 @@ export function Modal({ open, title, onClose, children, footer }: Props) {
|
||||
if (e.target === e.currentTarget) onClose();
|
||||
}}
|
||||
>
|
||||
<div className="admin-modal" role="dialog" aria-modal="true">
|
||||
<div
|
||||
ref={dialogRef}
|
||||
className={`admin-modal${className ? ` ${className}` : ""}`}
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
aria-labelledby={titleId}
|
||||
tabIndex={-1}
|
||||
>
|
||||
<div className="admin-modal__header">
|
||||
<span>{title}</span>
|
||||
<span id={titleId}>{title}</span>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={onClose}
|
||||
aria-label="关闭"
|
||||
@@ -36,3 +107,25 @@ export function Modal({ open, title, onClose, children, footer }: Props) {
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function getFocusableElements(root: HTMLElement): HTMLElement[] {
|
||||
const selectors = [
|
||||
"a[href]",
|
||||
"button:not([disabled])",
|
||||
"textarea:not([disabled])",
|
||||
"input:not([disabled])",
|
||||
"select:not([disabled])",
|
||||
"[tabindex]:not([tabindex='-1'])",
|
||||
].join(",");
|
||||
|
||||
return Array.from(root.querySelectorAll<HTMLElement>(selectors)).filter(
|
||||
(el) => !el.hasAttribute("disabled") && el.getAttribute("aria-hidden") !== "true"
|
||||
);
|
||||
}
|
||||
|
||||
function isTopDialog(dialog: HTMLElement): boolean {
|
||||
const dialogs = Array.from(
|
||||
document.querySelectorAll<HTMLElement>('[role="dialog"][aria-modal="true"]')
|
||||
);
|
||||
return dialogs[dialogs.length - 1] === dialog;
|
||||
}
|
||||
|
||||
@@ -8,11 +8,7 @@ export function RequireAuth({ children }: { children: ReactNode }) {
|
||||
const location = useLocation();
|
||||
|
||||
if (status === "loading") {
|
||||
return (
|
||||
<div className="admin-loading-screen">
|
||||
加载中...
|
||||
</div>
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (status === "guest") {
|
||||
|
||||
+335
-48
@@ -1,24 +1,45 @@
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { Film, Plus, RefreshCw, Search, Tags } from "lucide-react";
|
||||
import { CheckSquare, Film, Plus, RefreshCw, Search, Tags, Trash2 } from "lucide-react";
|
||||
import * as api from "./api";
|
||||
import { useToast } from "./ToastContext";
|
||||
import { ConfirmModal } from "./ConfirmModal";
|
||||
|
||||
const DESKTOP_TAGS_PAGE_SIZE = 25;
|
||||
const MOBILE_TAGS_PAGE_SIZE = 8;
|
||||
const TAGS_MOBILE_QUERY = "(max-width: 640px)";
|
||||
|
||||
type DeleteConfirmState =
|
||||
| { kind: "single"; tag: api.AdminTag }
|
||||
| { kind: "bulk"; ids: number[] }
|
||||
| null;
|
||||
|
||||
export function TagsPage() {
|
||||
const [tags, setTags] = useState<api.AdminTag[]>([]);
|
||||
const [label, setLabel] = useState("");
|
||||
const [aliases, setAliases] = useState("");
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [loadError, setLoadError] = useState("");
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [deletingId, setDeletingId] = useState<number | null>(null);
|
||||
const [deleteConfirm, setDeleteConfirm] = useState<DeleteConfirmState>(null);
|
||||
const [searchQuery, setSearchQuery] = useState("");
|
||||
const [filterSource, setFilterSource] = useState<string>("all");
|
||||
const [selectMode, setSelectMode] = useState(false);
|
||||
const [selected, setSelected] = useState<Set<number>>(new Set());
|
||||
const [bulkDeleting, setBulkDeleting] = useState(false);
|
||||
const pageSize = useTagsPageSize();
|
||||
const [page, setPage] = useState(1);
|
||||
const { show } = useToast();
|
||||
|
||||
async function refresh() {
|
||||
setLoading(true);
|
||||
setLoadError("");
|
||||
try {
|
||||
setTags(await api.listTags());
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "加载标签失败", "error");
|
||||
const message = e instanceof Error ? e.message : "加载标签失败";
|
||||
setLoadError(message);
|
||||
show(message, "error");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
@@ -45,6 +66,67 @@ export function TagsPage() {
|
||||
}
|
||||
}
|
||||
|
||||
function handleDelete(tag: api.AdminTag) {
|
||||
if (tag.source === "system") return;
|
||||
setDeleteConfirm({ kind: "single", tag });
|
||||
}
|
||||
|
||||
function toggleSelectMode() {
|
||||
setSelectMode((m) => !m);
|
||||
setSelected(new Set());
|
||||
}
|
||||
|
||||
function toggleSelect(id: number) {
|
||||
setSelected((prev) => {
|
||||
const next = new Set(prev);
|
||||
next.has(id) ? next.delete(id) : next.add(id);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
async function handleBulkDelete() {
|
||||
const ids = [...selected];
|
||||
if (ids.length === 0) return;
|
||||
setDeleteConfirm({ kind: "bulk", ids });
|
||||
}
|
||||
|
||||
async function confirmDelete() {
|
||||
if (!deleteConfirm) return;
|
||||
|
||||
if (deleteConfirm.kind === "single") {
|
||||
const tag = deleteConfirm.tag;
|
||||
setDeletingId(tag.id);
|
||||
try {
|
||||
const r = await api.deleteTag(tag.id);
|
||||
show(`已删除标签,并从 ${r.removedVideos} 个视频移除`, "success");
|
||||
setDeleteConfirm(null);
|
||||
await refresh();
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "删除标签失败", "error");
|
||||
} finally {
|
||||
setDeletingId(null);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const ids = deleteConfirm.ids;
|
||||
setBulkDeleting(true);
|
||||
try {
|
||||
const results = await Promise.allSettled(
|
||||
ids.map((id) => api.deleteTag(id))
|
||||
);
|
||||
const ok = results.filter((r) => r.status === "fulfilled").length;
|
||||
const failed = ids.length - ok;
|
||||
show(failed ? `已删除 ${ok} 个,${failed} 个失败` : `已删除 ${ok} 个标签`, failed ? "error" : "success");
|
||||
setSelected(new Set());
|
||||
setSelectMode(false);
|
||||
setDeleteConfirm(null);
|
||||
await refresh();
|
||||
} finally {
|
||||
setBulkDeleting(false);
|
||||
}
|
||||
}
|
||||
|
||||
const stats = useMemo(() => {
|
||||
let totalVideos = 0;
|
||||
let systemCount = 0;
|
||||
@@ -82,11 +164,46 @@ export function TagsPage() {
|
||||
});
|
||||
}, [tags, searchQuery, filterSource]);
|
||||
|
||||
const totalPages = Math.max(1, Math.ceil(filteredTags.length / pageSize));
|
||||
const currentPage = Math.min(page, totalPages);
|
||||
const pageStartIndex = (currentPage - 1) * pageSize;
|
||||
const pageEndIndex = pageStartIndex + pageSize;
|
||||
const pagedTags = useMemo(
|
||||
() => filteredTags.slice(pageStartIndex, pageEndIndex),
|
||||
[filteredTags, pageStartIndex, pageEndIndex]
|
||||
);
|
||||
const pageStart = filteredTags.length === 0 ? 0 : pageStartIndex + 1;
|
||||
const pageEnd = Math.min(filteredTags.length, pageEndIndex);
|
||||
|
||||
useEffect(() => {
|
||||
setPage(1);
|
||||
}, [searchQuery, filterSource, pageSize]);
|
||||
|
||||
useEffect(() => {
|
||||
setPage((p) => Math.min(Math.max(1, p), totalPages));
|
||||
}, [totalPages]);
|
||||
|
||||
const deletablePageTags = useMemo(
|
||||
() => pagedTags.filter((t) => t.source !== "system"),
|
||||
[pagedTags]
|
||||
);
|
||||
const allSelected =
|
||||
deletablePageTags.length > 0 && deletablePageTags.every((t) => selected.has(t.id));
|
||||
|
||||
function toggleSelectAll() {
|
||||
setSelected((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (allSelected) deletablePageTags.forEach((t) => next.delete(t.id));
|
||||
else deletablePageTags.forEach((t) => next.add(t.id));
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<section>
|
||||
<header className="admin-page__header">
|
||||
<h1 className="admin-page__title">标签管理</h1>
|
||||
<button className="admin-btn" onClick={refresh}>
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 刷新
|
||||
</button>
|
||||
</header>
|
||||
@@ -98,39 +215,44 @@ export function TagsPage() {
|
||||
<div className="admin-card__title">
|
||||
<Plus size={15} /> 新增分类标签
|
||||
</div>
|
||||
<div className="admin-form">
|
||||
<form
|
||||
className="admin-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
handleCreate();
|
||||
}}
|
||||
>
|
||||
<div className="admin-form__row">
|
||||
<label>标签名</label>
|
||||
<label htmlFor="admin-tag-label">标签名</label>
|
||||
<input
|
||||
id="admin-tag-label"
|
||||
value={label}
|
||||
onChange={(e) => setLabel(e.target.value)}
|
||||
placeholder="例如:清纯"
|
||||
/>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>别名</label>
|
||||
<label htmlFor="admin-tag-aliases">别名</label>
|
||||
<input
|
||||
id="admin-tag-aliases"
|
||||
value={aliases}
|
||||
onChange={(e) => setAliases(e.target.value)}
|
||||
placeholder="逗号分隔,例如:纯欲, 清新"
|
||||
/>
|
||||
<div className="admin-form__help">
|
||||
新增后会按别名和标签名匹配已有视频的标题、作者和目录并自动归类。
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
type="submit"
|
||||
className="admin-btn is-primary"
|
||||
onClick={handleCreate}
|
||||
disabled={saving || !label.trim()}
|
||||
>
|
||||
<Plus size={13} /> {saving ? "添加中..." : "添加并自动归类"}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div className="admin-card">
|
||||
<div className="admin-card__title">
|
||||
<Tags size={15} /> 系统标签统计
|
||||
<Tags size={15} /> 标签总览
|
||||
</div>
|
||||
<div className="admin-tag-stats-list">
|
||||
<div className="admin-tag-stat-item">
|
||||
@@ -141,14 +263,6 @@ export function TagsPage() {
|
||||
<span>关联视频次</span>
|
||||
<strong>{stats.totalVideos}</strong>
|
||||
</div>
|
||||
<div className="admin-tag-stat-item">
|
||||
<span>系统提取</span>
|
||||
<strong>{stats.systemCount}</strong>
|
||||
</div>
|
||||
<div className="admin-tag-stat-item">
|
||||
<span>用户创建</span>
|
||||
<strong>{stats.userCount}</strong>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -159,6 +273,7 @@ export function TagsPage() {
|
||||
<div className="admin-tags-search">
|
||||
<Search className="admin-tags-search__icon" size={14} />
|
||||
<input
|
||||
aria-label="搜索标签名或别名"
|
||||
type="text"
|
||||
value={searchQuery}
|
||||
onChange={(e) => setSearchQuery(e.target.value)}
|
||||
@@ -195,53 +310,225 @@ export function TagsPage() {
|
||||
>
|
||||
合集 ({stats.collectionCount})
|
||||
</button>
|
||||
{stats.legacyCount > 0 && (
|
||||
<button
|
||||
type="button"
|
||||
className={`admin-tags-filter-tab ${filterSource === "legacy" ? "is-active" : ""}`}
|
||||
onClick={() => setFilterSource("legacy")}
|
||||
>
|
||||
旧数据 ({stats.legacyCount})
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<button
|
||||
type="button"
|
||||
className={`admin-btn ${selectMode ? "is-primary" : ""}`}
|
||||
onClick={toggleSelectMode}
|
||||
>
|
||||
<CheckSquare size={13} /> {selectMode ? "退出批量" : "批量删除"}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{selectMode && (
|
||||
<div className="admin-tags-bulkbar">
|
||||
<label className="admin-check">
|
||||
<input type="checkbox" checked={allSelected} onChange={toggleSelectAll} />
|
||||
<span>全选本页 ({deletablePageTags.length})</span>
|
||||
</label>
|
||||
<span className="admin-tags-bulkbar__count">已选 {selected.size} 个</span>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setSelected(new Set())}
|
||||
disabled={selected.size === 0}
|
||||
>
|
||||
清空
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn is-danger"
|
||||
onClick={handleBulkDelete}
|
||||
disabled={selected.size === 0 || bulkDeleting}
|
||||
>
|
||||
<Trash2 size={13} /> {bulkDeleting ? "删除中..." : `删除选中 (${selected.size})`}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{loading ? (
|
||||
<div className="admin-empty">加载中...</div>
|
||||
<div className="admin-loading-state">
|
||||
<RefreshCw size={20} className="admin-spin" />
|
||||
<span>加载中...</span>
|
||||
</div>
|
||||
) : loadError ? (
|
||||
<div className="admin-error-state">
|
||||
<strong>标签加载失败</strong>
|
||||
<span>{loadError}</span>
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 重试
|
||||
</button>
|
||||
</div>
|
||||
) : filteredTags.length === 0 ? (
|
||||
<div className="admin-card admin-empty">
|
||||
没有找到匹配的标签。
|
||||
</div>
|
||||
) : (
|
||||
<div className="admin-tags-grid">
|
||||
{filteredTags.map((tag) => (
|
||||
<div key={tag.id} className="admin-tag-card">
|
||||
<div className="admin-tag-card__head">
|
||||
<span className="admin-tag-card__title">{tag.label}</span>
|
||||
<span className="admin-tag-card__source-badge" data-source={tag.source}>
|
||||
{sourceLabel(tag.source)}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{tag.aliases && tag.aliases.length > 0 && (
|
||||
<div className="admin-tag-card__aliases">
|
||||
{tag.aliases.map((alias) => (
|
||||
<span key={alias} className="admin-tag-card__alias-pill">
|
||||
{alias}
|
||||
<>
|
||||
<div className="admin-tags-grid">
|
||||
{pagedTags.map((tag) => {
|
||||
const selectable = selectMode && tag.source !== "system";
|
||||
const isSelected = selected.has(tag.id);
|
||||
const cardClass = `admin-tag-card${selectable ? " is-selectable" : ""}${
|
||||
selectable && isSelected ? " is-selected" : ""
|
||||
}`;
|
||||
const cardContent = (
|
||||
<>
|
||||
<div className="admin-tag-card__head">
|
||||
{selectable && (
|
||||
<input
|
||||
type="checkbox"
|
||||
className="admin-tag-card__check"
|
||||
checked={isSelected}
|
||||
onChange={() => toggleSelect(tag.id)}
|
||||
/>
|
||||
)}
|
||||
<span className="admin-tag-card__title">{tag.label}</span>
|
||||
<span className="admin-tag-card__source-badge" data-source={tag.source}>
|
||||
{sourceLabel(tag.source)}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="admin-tag-card__footer">
|
||||
<span>ID: {tag.id}</span>
|
||||
<span className="admin-tag-card__count">
|
||||
<Film size={11} />
|
||||
<strong>{tag.count} 视频</strong>
|
||||
</span>
|
||||
</div>
|
||||
{tag.aliases && tag.aliases.length > 0 && (
|
||||
<div className="admin-tag-card__aliases">
|
||||
{tag.aliases.map((alias) => (
|
||||
<span key={alias} className="admin-tag-card__alias-pill">
|
||||
{alias}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="admin-tag-card__footer">
|
||||
<span className="admin-tag-card__count">
|
||||
<Film size={13} />
|
||||
<strong>{tag.count}</strong> 视频
|
||||
</span>
|
||||
<div className="admin-tag-card__footer-actions">
|
||||
<span className="admin-tag-card__id">#{tag.id}</span>
|
||||
{!selectMode && tag.source !== "system" && (
|
||||
<button
|
||||
type="button"
|
||||
className="admin-tag-card__delete"
|
||||
onClick={() => handleDelete(tag)}
|
||||
disabled={deletingId === tag.id}
|
||||
aria-label={`删除标签 ${tag.label}`}
|
||||
>
|
||||
<Trash2 size={11} />
|
||||
<span>{deletingId === tag.id ? "删除中" : "删除"}</span>
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
return selectable ? (
|
||||
<label key={tag.id} className={cardClass}>
|
||||
{cardContent}
|
||||
</label>
|
||||
) : (
|
||||
<div key={tag.id} className={cardClass}>
|
||||
{cardContent}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
{totalPages > 1 && (
|
||||
<div className="admin-table-pagination admin-tags-pagination">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage(1)}
|
||||
disabled={currentPage <= 1}
|
||||
>
|
||||
首页
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage((p) => Math.max(1, p - 1))}
|
||||
disabled={currentPage <= 1}
|
||||
>
|
||||
上一页
|
||||
</button>
|
||||
<span className="admin-table-pagination__info">
|
||||
第 {currentPage} / {totalPages} 页,显示 {pageStart}-{pageEnd} / {filteredTags.length},每页 {pageSize} 个
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage((p) => Math.min(totalPages, p + 1))}
|
||||
disabled={currentPage >= totalPages}
|
||||
>
|
||||
下一页
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage(totalPages)}
|
||||
disabled={currentPage >= totalPages}
|
||||
>
|
||||
末页
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<ConfirmModal
|
||||
open={!!deleteConfirm}
|
||||
title={deleteConfirm?.kind === "bulk" ? "删除选中标签" : "删除标签"}
|
||||
message={
|
||||
deleteConfirm?.kind === "bulk"
|
||||
? `确定要删除选中的 ${deleteConfirm.ids.length} 个标签吗?`
|
||||
: `确定要删除标签「${deleteConfirm?.tag.label ?? ""}」吗?`
|
||||
}
|
||||
confirmText="确认删除"
|
||||
danger
|
||||
centerMessage
|
||||
modalClassName="admin-modal--delete-confirm"
|
||||
loading={deletingId !== null || bulkDeleting}
|
||||
onCancel={() => {
|
||||
if (deletingId === null && !bulkDeleting) setDeleteConfirm(null);
|
||||
}}
|
||||
onConfirm={confirmDelete}
|
||||
/>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
function useTagsPageSize() {
|
||||
const [pageSize, setPageSize] = useState(() =>
|
||||
window.matchMedia(TAGS_MOBILE_QUERY).matches
|
||||
? MOBILE_TAGS_PAGE_SIZE
|
||||
: DESKTOP_TAGS_PAGE_SIZE
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const media = window.matchMedia(TAGS_MOBILE_QUERY);
|
||||
const update = () => {
|
||||
setPageSize(media.matches ? MOBILE_TAGS_PAGE_SIZE : DESKTOP_TAGS_PAGE_SIZE);
|
||||
};
|
||||
update();
|
||||
media.addEventListener("change", update);
|
||||
return () => media.removeEventListener("change", update);
|
||||
}, []);
|
||||
|
||||
return pageSize;
|
||||
}
|
||||
|
||||
function splitList(s: string): string[] {
|
||||
return s
|
||||
.split(/[,,、\s]+/)
|
||||
|
||||
@@ -103,9 +103,6 @@ export function ThemePage() {
|
||||
<div className="theme-page">
|
||||
<header className="theme-page__head">
|
||||
<h1 className="theme-page__title">主题外观</h1>
|
||||
<p className="theme-page__sub">
|
||||
切换全站主题。所有访客都会看到这里选定的主题,本设置会写入数据库永久保存。
|
||||
</p>
|
||||
</header>
|
||||
|
||||
<div className="theme-grid">
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
useCallback,
|
||||
useContext,
|
||||
useEffect,
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
|
||||
@@ -18,23 +19,37 @@ const ToastCtx = createContext<Ctx | null>(null);
|
||||
|
||||
export function ToastProvider({ children }: { children: ReactNode }) {
|
||||
const [items, setItems] = useState<Toast[]>([]);
|
||||
const timers = useRef<Record<string, ReturnType<typeof window.setTimeout>>>({});
|
||||
|
||||
// Deduplicate: same text won't stack, just resets the dismiss timer
|
||||
const show = useCallback((text: string, kind: ToastKind = "info") => {
|
||||
// Reset timer if duplicate
|
||||
if (timers.current[text]) {
|
||||
window.clearTimeout(timers.current[text]);
|
||||
timers.current[text] = window.setTimeout(() => {
|
||||
setItems((list) => list.filter((t) => t.text !== text));
|
||||
delete timers.current[text];
|
||||
}, 2600);
|
||||
return;
|
||||
}
|
||||
const id = Date.now() + Math.random();
|
||||
setItems((list) => [...list, { id, kind, text }]);
|
||||
window.setTimeout(() => {
|
||||
timers.current[text] = window.setTimeout(() => {
|
||||
setItems((list) => list.filter((t) => t.id !== id));
|
||||
delete timers.current[text];
|
||||
}, 2600);
|
||||
setItems((list) => [...list, { id, kind, text }]);
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<ToastCtx.Provider value={{ show }}>
|
||||
{children}
|
||||
{items.map((t) => (
|
||||
<div key={t.id} className={`admin-toast is-${t.kind}`}>
|
||||
{t.text}
|
||||
</div>
|
||||
))}
|
||||
<div className="admin-toast-stack" role="status" aria-live="polite">
|
||||
{items.map((t) => (
|
||||
<div key={t.id} className={`admin-toast is-${t.kind}`}>
|
||||
{t.text}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</ToastCtx.Provider>
|
||||
);
|
||||
}
|
||||
|
||||
+395
-139
@@ -1,28 +1,43 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Edit, RefreshCw, Search } from "lucide-react";
|
||||
import { useEffect, useId, useState } from "react";
|
||||
import { ChevronDown, Edit, RefreshCw, Search, CheckSquare, Square, Image, Trash2 } from "lucide-react";
|
||||
import * as api from "./api";
|
||||
import { useToast } from "./ToastContext";
|
||||
import { Modal } from "./Modal";
|
||||
import { ConfirmModal } from "./ConfirmModal";
|
||||
import { formatBytes } from "./storageFormat";
|
||||
|
||||
const PAGE_SIZE = 100;
|
||||
const DESKTOP_VIDEOS_PAGE_SIZE = 50;
|
||||
const MOBILE_VIDEOS_PAGE_SIZE = 20;
|
||||
const VIDEOS_MOBILE_QUERY = "(max-width: 640px)";
|
||||
|
||||
export function VideosPage() {
|
||||
const [list, setList] = useState<api.AdminVideo[]>([]);
|
||||
const [drives, setDrives] = useState<api.AdminDrive[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [loadError, setLoadError] = useState("");
|
||||
const [keyword, setKeyword] = useState("");
|
||||
const [searchKeyword, setSearchKeyword] = useState("");
|
||||
const [driveId, setDriveId] = useState("");
|
||||
const [page, setPage] = useState(1);
|
||||
const [total, setTotal] = useState(0);
|
||||
const [editing, setEditing] = useState<api.AdminVideo | null>(null);
|
||||
const [availableTags, setAvailableTags] = useState<api.AdminTag[]>([]);
|
||||
const [selectedIds, setSelectedIds] = useState<Set<string>>(new Set());
|
||||
const [batchRegenOpen, setBatchRegenOpen] = useState(false);
|
||||
const [batchRegening, setBatchRegening] = useState(false);
|
||||
const [batchDeleteOpen, setBatchDeleteOpen] = useState(false);
|
||||
const [batchDeleting, setBatchDeleting] = useState(false);
|
||||
const [deleteTarget, setDeleteTarget] = useState<api.AdminVideo | null>(null);
|
||||
const [deleting, setDeleting] = useState(false);
|
||||
const pageSize = useVideosPageSize();
|
||||
const { show } = useToast();
|
||||
|
||||
async function refresh() {
|
||||
setLoading(true);
|
||||
setLoadError("");
|
||||
try {
|
||||
const [r, tagList, driveList] = await Promise.all([
|
||||
api.listVideos({ driveId, page, size: PAGE_SIZE }),
|
||||
api.listVideos({ driveId, page, size: pageSize, keyword: searchKeyword }),
|
||||
api.listTags(),
|
||||
api.listDrives(),
|
||||
]);
|
||||
@@ -30,8 +45,11 @@ export function VideosPage() {
|
||||
setTotal(r.total ?? 0);
|
||||
setAvailableTags(tagList);
|
||||
setDrives(driveList ?? []);
|
||||
setSelectedIds(new Set());
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "加载失败", "error");
|
||||
const message = e instanceof Error ? e.message : "加载失败";
|
||||
setLoadError(message);
|
||||
show(message, "error");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
@@ -39,164 +57,314 @@ export function VideosPage() {
|
||||
|
||||
useEffect(() => {
|
||||
refresh();
|
||||
}, [driveId, page]);
|
||||
}, [driveId, page, searchKeyword, pageSize]);
|
||||
|
||||
useEffect(() => {
|
||||
setPage(1);
|
||||
}, [pageSize]);
|
||||
|
||||
useEffect(() => {
|
||||
if (keyword === searchKeyword) return;
|
||||
const timer = window.setTimeout(() => {
|
||||
setSearchKeyword(keyword);
|
||||
setPage(1);
|
||||
}, 300);
|
||||
return () => window.clearTimeout(timer);
|
||||
}, [keyword]);
|
||||
|
||||
const driveNameMap = new Map(
|
||||
drives.map((d) => [d.id, d.name || d.id])
|
||||
);
|
||||
|
||||
const filtered = keyword.trim()
|
||||
? list.filter((v) => {
|
||||
const k = keyword.toLowerCase();
|
||||
return (
|
||||
v.title.toLowerCase().includes(k) ||
|
||||
(v.author ?? "").toLowerCase().includes(k)
|
||||
);
|
||||
})
|
||||
: list;
|
||||
const totalPages = Math.max(1, Math.ceil(total / PAGE_SIZE));
|
||||
const pageStart = total === 0 ? 0 : (page - 1) * PAGE_SIZE + 1;
|
||||
const pageEnd = Math.min(total, page * PAGE_SIZE);
|
||||
const listItems = list;
|
||||
const totalPages = Math.max(1, Math.ceil(total / pageSize));
|
||||
const pageStart = total === 0 ? 0 : (page - 1) * pageSize + 1;
|
||||
const pageEnd = Math.min(total, page * pageSize);
|
||||
const listSummary = driveId
|
||||
? `${driveNameMap.get(driveId) ?? driveId}:共 ${total} 个视频,第 ${page} / ${totalPages} 页,显示 ${pageStart}-${pageEnd}`
|
||||
: `全部网盘:共 ${total} 个视频,第 ${page} / ${totalPages} 页,显示 ${pageStart}-${pageEnd}`;
|
||||
|
||||
async function handleRegen(v: api.AdminVideo) {
|
||||
try {
|
||||
await api.regenPreview(v.id);
|
||||
show("已触发 teaser 重生", "success");
|
||||
show("已触发预览视频重生", "success");
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "触发失败", "error");
|
||||
}
|
||||
}
|
||||
|
||||
async function handleBatchRegen() {
|
||||
if (selectedIds.size === 0) return;
|
||||
setBatchRegenOpen(true);
|
||||
}
|
||||
|
||||
async function handleBatchDelete() {
|
||||
if (selectedIds.size === 0) return;
|
||||
setBatchDeleteOpen(true);
|
||||
}
|
||||
|
||||
async function confirmBatchRegen() {
|
||||
const ids = [...selectedIds];
|
||||
setBatchRegening(true);
|
||||
let success = 0;
|
||||
try {
|
||||
const results = await Promise.allSettled(
|
||||
ids.map((id) => api.regenPreview(id))
|
||||
);
|
||||
for (const r of results) {
|
||||
if (r.status === "fulfilled") success++;
|
||||
}
|
||||
show(`批量触发完成,成功 ${success} / ${ids.length} 个`, success === ids.length ? "success" : "info");
|
||||
setSelectedIds(new Set());
|
||||
setBatchRegenOpen(false);
|
||||
} finally {
|
||||
setBatchRegening(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function confirmDeleteVideo() {
|
||||
if (!deleteTarget) return;
|
||||
const target = deleteTarget;
|
||||
setDeleting(true);
|
||||
try {
|
||||
const result = await api.deleteVideo(target.id);
|
||||
setDeleteTarget(null);
|
||||
setSelectedIds((ids) => {
|
||||
const next = new Set(ids);
|
||||
next.delete(target.id);
|
||||
return next;
|
||||
});
|
||||
show(result.deletedSource ? "已删除视频,并清理 91Spider 源文件" : "已删除视频", "success");
|
||||
if (listItems.length === 1 && page > 1) {
|
||||
setPage((p) => Math.max(1, p - 1));
|
||||
} else {
|
||||
refresh();
|
||||
}
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "删除失败", "error");
|
||||
} finally {
|
||||
setDeleting(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function confirmBatchDelete() {
|
||||
const ids = [...selectedIds];
|
||||
if (ids.length === 0) return;
|
||||
setBatchDeleting(true);
|
||||
try {
|
||||
let success = 0;
|
||||
let deletedSources = 0;
|
||||
for (const id of ids) {
|
||||
try {
|
||||
const result = await api.deleteVideo(id);
|
||||
success++;
|
||||
if (result.deletedSource) deletedSources++;
|
||||
} catch {
|
||||
// Keep deleting the rest of the selected videos; report aggregate failure below.
|
||||
}
|
||||
}
|
||||
const failed = ids.length - success;
|
||||
if (failed === 0) {
|
||||
const extra = deletedSources > 0 ? `,其中 ${deletedSources} 个清理了 91Spider 源文件` : "";
|
||||
show(`批量删除完成,成功 ${success} 个${extra}`, "success");
|
||||
} else {
|
||||
show(`批量删除完成,成功 ${success} / ${ids.length} 个,失败 ${failed} 个`, success > 0 ? "info" : "error");
|
||||
}
|
||||
setSelectedIds(new Set());
|
||||
setBatchDeleteOpen(false);
|
||||
if (success >= listItems.length && page > 1) {
|
||||
setPage((p) => Math.max(1, p - 1));
|
||||
} else {
|
||||
refresh();
|
||||
}
|
||||
} finally {
|
||||
setBatchDeleting(false);
|
||||
}
|
||||
}
|
||||
|
||||
const toggleSelectAll = () => {
|
||||
if (selectedIds.size === listItems.length && listItems.length > 0) {
|
||||
setSelectedIds(new Set());
|
||||
} else {
|
||||
setSelectedIds(new Set(listItems.map(v => v.id)));
|
||||
}
|
||||
};
|
||||
|
||||
const toggleSelect = (id: string) => {
|
||||
const next = new Set(selectedIds);
|
||||
if (next.has(id)) next.delete(id);
|
||||
else next.add(id);
|
||||
setSelectedIds(next);
|
||||
};
|
||||
|
||||
function handleSearchSubmit(e: React.FormEvent) {
|
||||
e.preventDefault();
|
||||
setSearchKeyword(keyword);
|
||||
setPage(1);
|
||||
}
|
||||
|
||||
return (
|
||||
<section>
|
||||
<header className="admin-page__header">
|
||||
<h1 className="admin-page__title">视频管理</h1>
|
||||
<div className="admin-page__actions admin-videos-filter">
|
||||
<select
|
||||
className="admin-videos-filter__select"
|
||||
value={driveId}
|
||||
onChange={(e) => {
|
||||
setDriveId(e.target.value);
|
||||
setPage(1);
|
||||
}}
|
||||
>
|
||||
<option value="">全部网盘</option>
|
||||
{drives.map((d) => (
|
||||
<option key={d.id} value={d.id}>
|
||||
{d.name || d.id}(已生成 {d.teaserReadyCount ?? 0},待生成{" "}
|
||||
{d.teaserPendingCount ?? 0})
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<div className="admin-videos-filter__search">
|
||||
<div className="admin-videos-filter__select-wrap">
|
||||
<select
|
||||
className="admin-videos-filter__select"
|
||||
value={driveId}
|
||||
onChange={(e) => {
|
||||
setDriveId(e.target.value);
|
||||
setPage(1);
|
||||
}}
|
||||
>
|
||||
<option value="">全部网盘</option>
|
||||
{drives.map((d) => (
|
||||
<option key={d.id} value={d.id}>
|
||||
{d.name || d.id}(已生成 {d.teaserReadyCount ?? 0},待生成{" "}
|
||||
{d.teaserPendingCount ?? 0})
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<ChevronDown size={15} className="admin-videos-filter__select-icon" aria-hidden="true" />
|
||||
</div>
|
||||
<form className="admin-videos-filter__search" onSubmit={handleSearchSubmit}>
|
||||
<Search size={14} className="admin-videos-filter__search-icon" />
|
||||
<input
|
||||
aria-label="搜索标题或作者"
|
||||
value={keyword}
|
||||
onChange={(e) => setKeyword(e.target.value)}
|
||||
placeholder="搜索标题 / 作者"
|
||||
/>
|
||||
</div>
|
||||
<button className="admin-btn" onClick={refresh}>
|
||||
</form>
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 刷新
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{drives.length > 0 && (
|
||||
<div className="admin-drive-teasers" aria-label="网盘 Teaser 统计">
|
||||
{drives.map((d) => (
|
||||
<button
|
||||
key={d.id}
|
||||
type="button"
|
||||
className={`admin-drive-teaser${
|
||||
driveId === d.id ? " is-active" : ""
|
||||
}`}
|
||||
onClick={() => {
|
||||
setDriveId(d.id);
|
||||
setPage(1);
|
||||
}}
|
||||
>
|
||||
<span className="admin-drive-teaser__name">{d.name || d.id}</span>
|
||||
<span className="admin-drive-teaser__metric is-ready">
|
||||
已生成 {d.teaserReadyCount ?? 0}
|
||||
</span>
|
||||
<span className="admin-drive-teaser__metric is-pending">
|
||||
待生成 {d.teaserPendingCount ?? 0}
|
||||
</span>
|
||||
{(d.teaserFailedCount ?? 0) > 0 && (
|
||||
<span className="admin-drive-teaser__metric is-failed">
|
||||
失败 {d.teaserFailedCount}
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!loading && (
|
||||
<div className="admin-videos-summary">
|
||||
{driveId
|
||||
? `${driveNameMap.get(driveId) ?? driveId}:共 ${total} 个视频,第 ${page} / ${totalPages} 页,显示 ${pageStart}-${pageEnd}`
|
||||
: `全部网盘:共 ${total} 个视频,第 ${page} / ${totalPages} 页,显示 ${pageStart}-${pageEnd}`}
|
||||
<div className="admin-videos-list-toolbar">
|
||||
<div className="admin-videos-summary">{listSummary}</div>
|
||||
{selectedIds.size > 0 && (
|
||||
<div className="admin-videos-bulk-actions">
|
||||
<span className="admin-videos-bulk-actions__count">
|
||||
已选择 {selectedIds.size} 项
|
||||
</span>
|
||||
<button type="button" className="admin-btn is-primary admin-videos-bulk-actions__btn" onClick={handleBatchRegen}>
|
||||
<RefreshCw size={13} /> 批量重生预览视频
|
||||
</button>
|
||||
<button type="button" className="admin-btn is-danger admin-videos-bulk-actions__btn" onClick={handleBatchDelete}>
|
||||
<Trash2 size={13} /> 批量删除
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{loading ? (
|
||||
<div className="admin-empty">加载中...</div>
|
||||
) : filtered.length === 0 ? (
|
||||
<div className="admin-card admin-empty">
|
||||
{driveId
|
||||
? "这个网盘下还没有可显示的视频。可以在「网盘管理」里触发重扫。"
|
||||
: "还没有视频。先在「网盘管理」里配置好盘并触发扫描。"}
|
||||
<div className="admin-loading-state">
|
||||
<RefreshCw size={20} className="admin-spin" />
|
||||
<span>加载中...</span>
|
||||
</div>
|
||||
) : loadError ? (
|
||||
<div className="admin-error-state">
|
||||
<strong>视频加载失败</strong>
|
||||
<span>{loadError}</span>
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 重试
|
||||
</button>
|
||||
</div>
|
||||
) : listItems.length === 0 ? (
|
||||
<div className="admin-empty-state">
|
||||
<div className="admin-empty-state__icon">
|
||||
<Image size={48} />
|
||||
</div>
|
||||
<div className="admin-empty-state__text">
|
||||
{driveId
|
||||
? "这个网盘下还没有可显示的视频,或未匹配到搜索结果。"
|
||||
: "还没有视频。先在「网盘管理」里配置好盘并触发扫描,或调整搜索词。"}
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<table className="admin-table">
|
||||
<table className="admin-table is-selectable admin-videos-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th className="is-checkbox" style={{ width: '40px' }}>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-table-checkbox-btn"
|
||||
onClick={toggleSelectAll}
|
||||
aria-label={selectedIds.size > 0 && selectedIds.size === listItems.length ? "清空当前页选择" : "选择当前页视频"}
|
||||
>
|
||||
{selectedIds.size > 0 && selectedIds.size === listItems.length ? <CheckSquare size={16} /> : <Square size={16} />}
|
||||
</button>
|
||||
</th>
|
||||
<th>标题</th>
|
||||
<th>作者</th>
|
||||
<th>标签</th>
|
||||
<th>时长</th>
|
||||
<th>Teaser</th>
|
||||
<th>预览视频</th>
|
||||
<th>来源</th>
|
||||
<th className="is-actions">操作</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{filtered.map((v) => (
|
||||
<tr key={v.id}>
|
||||
<td data-label="标题">
|
||||
<div className="admin-video-title">{v.title}</div>
|
||||
{fileMeta(v) && (
|
||||
<div className="admin-video-filemeta">
|
||||
{fileMeta(v)}
|
||||
</div>
|
||||
)}
|
||||
{listItems.map((v) => (
|
||||
<tr key={v.id} className={selectedIds.has(v.id) ? "is-selected" : ""}>
|
||||
<td className="is-checkbox">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-table-checkbox-btn"
|
||||
onClick={() => toggleSelect(v.id)}
|
||||
aria-label={`${selectedIds.has(v.id) ? "取消选择" : "选择"}视频 ${v.title}`}
|
||||
>
|
||||
{selectedIds.has(v.id) ? <CheckSquare size={16} color="var(--accent)" /> : <Square size={16} color="var(--border-strong)" />}
|
||||
</button>
|
||||
</td>
|
||||
<td data-label="作者">{v.author || <span className="admin-text-faint">—</span>}</td>
|
||||
<td data-label="标签">
|
||||
<div className="admin-pills">
|
||||
{(v.tags ?? []).map((t) => (
|
||||
<span key={t} className="admin-pill">
|
||||
{t}
|
||||
</span>
|
||||
))}
|
||||
<td data-label="标题">
|
||||
<div className="admin-video-title-cell">
|
||||
<div className="admin-video-thumb-wrap" aria-hidden="true">
|
||||
{v.thumbnailUrl ? (
|
||||
<img className="admin-video-thumb" src={v.thumbnailUrl} alt="" />
|
||||
) : (
|
||||
<div className="admin-video-thumb-placeholder">
|
||||
<Image size={14} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="admin-video-title-body">
|
||||
<div className="admin-video-title">{v.title}</div>
|
||||
{fileMeta(v) && (
|
||||
<div className="admin-video-filemeta">{fileMeta(v)}</div>
|
||||
)}
|
||||
{(v.tags ?? []).length > 0 && (
|
||||
<div className="admin-pills admin-video-title-tags">
|
||||
{(v.tags ?? []).map((t) => (
|
||||
<span key={t} className="admin-pill">{t}</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<VideoFileMetaPills video={v} />
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
<td data-label="作者">{v.author || <span className="admin-text-faint">—</span>}</td>
|
||||
<td data-label="时长">{formatDur(v.durationSeconds)}</td>
|
||||
<td data-label="Teaser">
|
||||
<td data-label="预览视频">
|
||||
<PreviewStatus s={v.previewStatus} />
|
||||
</td>
|
||||
<td data-label="来源" className="admin-mono-cell">
|
||||
{driveNameMap.get(v.driveId) ?? v.driveId}
|
||||
</td>
|
||||
<td className="is-actions" data-label="操作">
|
||||
<button className="admin-btn" onClick={() => setEditing(v)}>
|
||||
<Edit size={13} /> 编辑
|
||||
<button type="button" className="admin-btn" onClick={() => setEditing(v)} title="编辑视频">
|
||||
<Edit size={13} />
|
||||
</button>{" "}
|
||||
<button className="admin-btn" onClick={() => handleRegen(v)}>
|
||||
<RefreshCw size={13} /> 重生 teaser
|
||||
<button type="button" className="admin-btn" onClick={() => handleRegen(v)} title="重生预览视频">
|
||||
<RefreshCw size={13} />
|
||||
</button>{" "}
|
||||
<button type="button" className="admin-btn is-danger" onClick={() => setDeleteTarget(v)} title="删除视频">
|
||||
<Trash2 size={13} />
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
@@ -205,6 +373,7 @@ export function VideosPage() {
|
||||
</table>
|
||||
<div className="admin-table-pagination">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage(1)}
|
||||
disabled={page <= 1}
|
||||
@@ -212,6 +381,7 @@ export function VideosPage() {
|
||||
首页
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage((p) => Math.max(1, p - 1))}
|
||||
disabled={page <= 1}
|
||||
@@ -219,9 +389,10 @@ export function VideosPage() {
|
||||
上一页
|
||||
</button>
|
||||
<span className="admin-table-pagination__info">
|
||||
第 {page} / {totalPages} 页,每页 {PAGE_SIZE} 个
|
||||
第 {page} / {totalPages} 页,每页 {pageSize} 个
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage((p) => Math.min(totalPages, p + 1))}
|
||||
disabled={page >= totalPages}
|
||||
@@ -229,6 +400,7 @@ export function VideosPage() {
|
||||
下一页
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage(totalPages)}
|
||||
disabled={page >= totalPages}
|
||||
@@ -250,6 +422,45 @@ export function VideosPage() {
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
<ConfirmModal
|
||||
open={batchRegenOpen}
|
||||
title="批量重生预览视频"
|
||||
message={`确定要为当前页选中的 ${selectedIds.size} 个视频重新生成预览视频吗?`}
|
||||
confirmText="确认重生"
|
||||
loading={batchRegening}
|
||||
onCancel={() => {
|
||||
if (!batchRegening) setBatchRegenOpen(false);
|
||||
}}
|
||||
onConfirm={confirmBatchRegen}
|
||||
/>
|
||||
<ConfirmModal
|
||||
open={deleteTarget !== null}
|
||||
title="删除视频"
|
||||
message={deleteTarget ? `确定要删除「${deleteTarget.title}」吗?` : ""}
|
||||
confirmText="删除视频"
|
||||
danger
|
||||
centerMessage
|
||||
modalClassName="admin-modal--delete-confirm"
|
||||
loading={deleting}
|
||||
onCancel={() => {
|
||||
if (!deleting) setDeleteTarget(null);
|
||||
}}
|
||||
onConfirm={confirmDeleteVideo}
|
||||
/>
|
||||
<ConfirmModal
|
||||
open={batchDeleteOpen}
|
||||
title="批量删除视频"
|
||||
message={`确定要删除当前页选中的 ${selectedIds.size} 个视频吗?`}
|
||||
confirmText="批量删除"
|
||||
danger
|
||||
centerMessage
|
||||
modalClassName="admin-modal--delete-confirm"
|
||||
loading={batchDeleting}
|
||||
onCancel={() => {
|
||||
if (!batchDeleting) setBatchDeleteOpen(false);
|
||||
}}
|
||||
onConfirm={confirmBatchDelete}
|
||||
/>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -261,6 +472,27 @@ function PreviewStatus({ s }: { s: string }) {
|
||||
return <span className="admin-status is-pending">待生成</span>;
|
||||
}
|
||||
|
||||
function VideoFileMetaPills({ video }: { video: api.AdminVideo }) {
|
||||
const parts = fileMetaParts(video);
|
||||
const category = (video.category ?? "").trim();
|
||||
if (parts.length === 0 && !category) return null;
|
||||
|
||||
return (
|
||||
<div className="admin-video-filemeta-pills" aria-label="视频文件信息">
|
||||
{parts.map((part, index) => (
|
||||
<span key={`${part}-${index}`} className="admin-video-filemeta-pill">
|
||||
{part}
|
||||
</span>
|
||||
))}
|
||||
{category && (
|
||||
<span className="admin-video-filemeta-pill is-category">
|
||||
{category}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function formatDur(sec: number): string {
|
||||
if (!sec) return "—";
|
||||
const m = Math.floor(sec / 60);
|
||||
@@ -268,6 +500,26 @@ function formatDur(sec: number): string {
|
||||
return `${m.toString().padStart(2, "0")}:${s.toString().padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
function useVideosPageSize() {
|
||||
const [pageSize, setPageSize] = useState(() =>
|
||||
window.matchMedia(VIDEOS_MOBILE_QUERY).matches
|
||||
? MOBILE_VIDEOS_PAGE_SIZE
|
||||
: DESKTOP_VIDEOS_PAGE_SIZE
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const media = window.matchMedia(VIDEOS_MOBILE_QUERY);
|
||||
const update = () => {
|
||||
setPageSize(media.matches ? MOBILE_VIDEOS_PAGE_SIZE : DESKTOP_VIDEOS_PAGE_SIZE);
|
||||
};
|
||||
update();
|
||||
media.addEventListener("change", update);
|
||||
return () => media.removeEventListener("change", update);
|
||||
}, []);
|
||||
|
||||
return pageSize;
|
||||
}
|
||||
|
||||
function EditVideoModal({
|
||||
video,
|
||||
availableTags,
|
||||
@@ -279,6 +531,7 @@ function EditVideoModal({
|
||||
onClose: () => void;
|
||||
onSaved: () => void;
|
||||
}) {
|
||||
const idPrefix = useId();
|
||||
const [title, setTitle] = useState(video.title);
|
||||
const [author, setAuthor] = useState(video.author ?? "");
|
||||
const [selectedTags, setSelectedTags] = useState(video.tags ?? []);
|
||||
@@ -321,10 +574,10 @@ function EditVideoModal({
|
||||
onClose={onClose}
|
||||
footer={
|
||||
<>
|
||||
<button className="admin-btn" onClick={onClose}>
|
||||
<button type="button" className="admin-btn" onClick={onClose}>
|
||||
取消
|
||||
</button>
|
||||
<button className="admin-btn is-primary" onClick={handleSave} disabled={saving}>
|
||||
<button type="button" className="admin-btn is-primary" onClick={handleSave} disabled={saving}>
|
||||
{saving ? "保存中..." : "保存"}
|
||||
</button>
|
||||
</>
|
||||
@@ -332,15 +585,15 @@ function EditVideoModal({
|
||||
>
|
||||
<div className="admin-form">
|
||||
<div className="admin-form__row">
|
||||
<label>标题</label>
|
||||
<input value={title} onChange={(e) => setTitle(e.target.value)} />
|
||||
<label htmlFor={`${idPrefix}-video-title`}>标题</label>
|
||||
<input id={`${idPrefix}-video-title`} value={title} onChange={(e) => setTitle(e.target.value)} />
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>作者</label>
|
||||
<input value={author} onChange={(e) => setAuthor(e.target.value)} />
|
||||
<label htmlFor={`${idPrefix}-video-author`}>作者</label>
|
||||
<input id={`${idPrefix}-video-author`} value={author} onChange={(e) => setAuthor(e.target.value)} />
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>标签</label>
|
||||
<div className="admin-form__label">标签</div>
|
||||
<div className="admin-tag-picker">
|
||||
{availableTags.map((tag) => (
|
||||
<label key={tag.id} className="admin-check">
|
||||
@@ -356,36 +609,49 @@ function EditVideoModal({
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>分类</label>
|
||||
<input value={category} onChange={(e) => setCategory(e.target.value)} />
|
||||
<label htmlFor={`${idPrefix}-video-category`}>分类</label>
|
||||
<input id={`${idPrefix}-video-category`} value={category} onChange={(e) => setCategory(e.target.value)} />
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>徽标(逗号分隔,例如 精选, 原创)</label>
|
||||
<input value={badges} onChange={(e) => setBadges(e.target.value)} />
|
||||
<label htmlFor={`${idPrefix}-video-badges`}>徽标(逗号分隔,例如 精选, 原创)</label>
|
||||
<input id={`${idPrefix}-video-badges`} value={badges} onChange={(e) => setBadges(e.target.value)} />
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>质量</label>
|
||||
<select value={quality} onChange={(e) => setQuality(e.target.value)}>
|
||||
<label htmlFor={`${idPrefix}-video-quality`}>质量</label>
|
||||
<select id={`${idPrefix}-video-quality`} value={quality} onChange={(e) => setQuality(e.target.value)}>
|
||||
<option value="">未设置</option>
|
||||
<option value="HD">HD</option>
|
||||
<option value="SD">SD</option>
|
||||
</select>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>时长(秒)</label>
|
||||
<label htmlFor={`${idPrefix}-video-duration`}>时长(秒)</label>
|
||||
<input
|
||||
id={`${idPrefix}-video-duration`}
|
||||
value={durationSec}
|
||||
onChange={(e) => setDurationSec(e.target.value)}
|
||||
inputMode="numeric"
|
||||
/>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>封面 URL</label>
|
||||
<input value={thumbnail} onChange={(e) => setThumbnail(e.target.value)} />
|
||||
<label htmlFor={`${idPrefix}-video-thumbnail`}>封面 URL</label>
|
||||
<div className="admin-thumbnail-preview">
|
||||
<input id={`${idPrefix}-video-thumbnail`} value={thumbnail} onChange={(e) => setThumbnail(e.target.value)} />
|
||||
{thumbnail && (
|
||||
<img
|
||||
src={thumbnail}
|
||||
alt="封面预览"
|
||||
className="admin-thumbnail-img"
|
||||
onError={(e) => (e.currentTarget.style.display = 'none')}
|
||||
onLoad={(e) => (e.currentTarget.style.display = 'block')}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>描述</label>
|
||||
<label htmlFor={`${idPrefix}-video-description`}>描述</label>
|
||||
<textarea
|
||||
id={`${idPrefix}-video-description`}
|
||||
value={description}
|
||||
onChange={(e) => setDescription(e.target.value)}
|
||||
/>
|
||||
@@ -395,7 +661,7 @@ function EditVideoModal({
|
||||
<dd>{video.driveId}</dd>
|
||||
<dt>文件信息</dt>
|
||||
<dd>{fileMeta(video) || "—"}</dd>
|
||||
<dt>Teaser</dt>
|
||||
<dt>预览视频</dt>
|
||||
<dd>
|
||||
<PreviewStatus s={video.previewStatus} />
|
||||
</dd>
|
||||
@@ -415,12 +681,15 @@ function EditVideoModal({
|
||||
}
|
||||
|
||||
function fileMeta(v: api.AdminVideo): string {
|
||||
const parts = [
|
||||
return fileMetaParts(v).join(" · ");
|
||||
}
|
||||
|
||||
function fileMetaParts(v: api.AdminVideo): string[] {
|
||||
return [
|
||||
normalizeExt(v.ext),
|
||||
v.quality,
|
||||
formatBytes(v.size),
|
||||
v.size > 0 ? formatBytes(v.size) : "",
|
||||
].filter(Boolean);
|
||||
return parts.join(" · ");
|
||||
}
|
||||
|
||||
function normalizeExt(ext: string): string {
|
||||
@@ -428,19 +697,6 @@ function normalizeExt(ext: string): string {
|
||||
return value ? value.toUpperCase() : "";
|
||||
}
|
||||
|
||||
function formatBytes(size: number): string {
|
||||
if (!size || size <= 0) return "";
|
||||
const units = ["B", "KB", "MB", "GB", "TB"];
|
||||
let value = size;
|
||||
let unit = 0;
|
||||
while (value >= 1024 && unit < units.length - 1) {
|
||||
value /= 1024;
|
||||
unit += 1;
|
||||
}
|
||||
const digits = unit === 0 || value >= 100 ? 0 : 1;
|
||||
return `${value.toFixed(digits)} ${units[unit]}`;
|
||||
}
|
||||
|
||||
function splitList(s: string): string[] {
|
||||
return s
|
||||
.split(/[,,、\s]+/)
|
||||
|
||||
+118
-18
@@ -61,18 +61,29 @@ export function me() {
|
||||
return request<{ authenticated: boolean }>("/me");
|
||||
}
|
||||
|
||||
export type UpdateCheck = {
|
||||
currentVersion: string;
|
||||
latestVersion: string;
|
||||
hasUpdate: boolean;
|
||||
releaseUrl?: string;
|
||||
checkedAt: string;
|
||||
};
|
||||
|
||||
export function checkUpdate() {
|
||||
return request<UpdateCheck>("/update/check");
|
||||
}
|
||||
|
||||
// ---------- Drives ----------
|
||||
|
||||
export type AdminDrive = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "spider91";
|
||||
kind: "quark" | "p115" | "p123" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
name: string;
|
||||
rootId: string;
|
||||
scanRootId: string;
|
||||
status: string;
|
||||
lastError?: string;
|
||||
hasCredential: boolean;
|
||||
/** 当前是否给该盘生成 teaser/封面(per-drive 开关,替代旧的全局 preview.enabled)。 */
|
||||
/** 当前是否给该盘生成预览视频/封面(per-drive 开关,替代旧的全局 preview.enabled)。 */
|
||||
teaserEnabled: boolean;
|
||||
/**
|
||||
* 用户在 admin 配置的"扫描跳过目录"集合(drive 侧目录 fileID 列表)。
|
||||
@@ -82,14 +93,21 @@ export type AdminDrive = {
|
||||
skipDirIds: string[];
|
||||
// spider91 上次成功爬取时间(unix 秒);其它 kind 留空。
|
||||
lastCrawlAt?: number;
|
||||
// spider91 专用代理地址;仅后台管理接口返回,用于编辑表单回显。
|
||||
spider91Proxy?: string;
|
||||
thumbnailGenerationStatus?: DriveGenerationStatus;
|
||||
previewGenerationStatus?: DriveGenerationStatus;
|
||||
fingerprintGenerationStatus?: DriveGenerationStatus;
|
||||
thumbnailReadyCount: number;
|
||||
thumbnailPendingCount: number;
|
||||
thumbnailFailedCount: number;
|
||||
thumbnailDurationPendingCount: number;
|
||||
teaserReadyCount: number;
|
||||
teaserPendingCount: number;
|
||||
teaserFailedCount: number;
|
||||
fingerprintReadyCount: number;
|
||||
fingerprintPendingCount: number;
|
||||
fingerprintFailedCount: number;
|
||||
};
|
||||
|
||||
export type DriveGenerationStatus = {
|
||||
@@ -121,10 +139,9 @@ export function getDriveStorage() {
|
||||
|
||||
export type UpsertDriveInput = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "spider91";
|
||||
kind: "quark" | "p115" | "p123" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
name: string;
|
||||
rootId: string;
|
||||
scanRootId: string;
|
||||
credentials: Record<string, string>;
|
||||
/**
|
||||
* 可选:写入"扫描跳过目录"集合。`undefined` 表示不变(沿用服务端旧值),
|
||||
@@ -141,9 +158,14 @@ export function upsertDrive(body: UpsertDriveInput) {
|
||||
});
|
||||
}
|
||||
|
||||
export function deleteDrive(id: string) {
|
||||
return request<{ ok: boolean }>(`/drives/${encodeURIComponent(id)}`, {
|
||||
export type DeleteDriveInput = {
|
||||
deleteVideos: true;
|
||||
};
|
||||
|
||||
export function deleteDrive(id: string, body: DeleteDriveInput) {
|
||||
return request<{ ok: boolean; deletedVideos: number }>(`/drives/${encodeURIComponent(id)}`, {
|
||||
method: "DELETE",
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -154,10 +176,44 @@ export function rescan(id: string) {
|
||||
);
|
||||
}
|
||||
|
||||
export function stopDriveTasks(id: string) {
|
||||
return request<{ ok: boolean; stopped: boolean }>(
|
||||
`/drives/${encodeURIComponent(id)}/tasks/stop`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
export type P123QRSession = {
|
||||
loginUuid: string;
|
||||
uniID: string;
|
||||
qrCodeUrl: string;
|
||||
qrImageDataUrl: string;
|
||||
expiresAt?: string;
|
||||
};
|
||||
|
||||
export type P123QRStatus = {
|
||||
loginStatus: number;
|
||||
statusText: string;
|
||||
scanPlatform?: number;
|
||||
platformText?: string;
|
||||
accessToken?: string;
|
||||
};
|
||||
|
||||
export function startP123QRLogin() {
|
||||
return request<P123QRSession>("/drives/p123/qr", { method: "POST" });
|
||||
}
|
||||
|
||||
export function getP123QRStatus(uniID: string, loginUuid: string) {
|
||||
const qs = new URLSearchParams({ loginUuid });
|
||||
return request<P123QRStatus>(
|
||||
`/drives/p123/qr/${encodeURIComponent(uniID)}?${qs.toString()}`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 切换某个云盘的 teaser 生成开关。点击网盘列表里行内的 toggle 按钮时调用。
|
||||
* 切换某个云盘的预览视频生成开关。点击网盘列表里行内的 toggle 按钮时调用。
|
||||
*
|
||||
* 后端会写 catalog.drives.teaser_enabled,并在从关到开时立刻补扫该盘 pending teaser;
|
||||
* 后端会写 catalog.drives.teaser_enabled,并在从关到开时立刻补扫该盘 pending 预览视频;
|
||||
* 关闭分支不补做任何事,新的入队判断会自动停。
|
||||
*/
|
||||
export function setDriveTeaserEnabled(id: string, enabled: boolean) {
|
||||
@@ -216,7 +272,7 @@ export function regenFailedPreviews(id: string) {
|
||||
|
||||
/**
|
||||
* 触发某 drive 下所有 thumbnail_status=failed 的封面重新入队生成。
|
||||
* 与 regenFailedPreviews 行为对称(一个管 teaser,一个管封面)。
|
||||
* 与 regenFailedPreviews 行为对称(一个管预览视频,一个管封面)。
|
||||
*
|
||||
* 后端立即返回 202;实际状态变化在下次 listDrives 拉到的 thumbnailFailedCount /
|
||||
* thumbnailGenerationStatus 字段里观察。
|
||||
@@ -228,6 +284,13 @@ export function regenFailedThumbnails(id: string) {
|
||||
);
|
||||
}
|
||||
|
||||
export function regenFailedFingerprints(id: string) {
|
||||
return request<{ ok: boolean }>(
|
||||
`/drives/${encodeURIComponent(id)}/fingerprints/failed/regenerate`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- Videos ----------
|
||||
|
||||
export type AdminVideo = {
|
||||
@@ -261,11 +324,12 @@ export type AdminVideoList = {
|
||||
size: number;
|
||||
};
|
||||
|
||||
export function listVideos(params: { driveId?: string; page?: number; size?: number } = {}) {
|
||||
export function listVideos(params: { driveId?: string; page?: number; size?: number; keyword?: string } = {}) {
|
||||
const qs = new URLSearchParams();
|
||||
if (params.driveId) qs.set("driveId", params.driveId);
|
||||
if (params.page) qs.set("page", String(params.page));
|
||||
if (params.size) qs.set("size", String(params.size));
|
||||
if (params.keyword) qs.set("keyword", params.keyword);
|
||||
const suffix = qs.toString() ? `?${qs.toString()}` : "";
|
||||
return request<AdminVideoList>(`/videos${suffix}`);
|
||||
}
|
||||
@@ -289,6 +353,13 @@ export function updateVideo(id: string, body: UpdateVideoInput) {
|
||||
});
|
||||
}
|
||||
|
||||
export function deleteVideo(id: string) {
|
||||
return request<{ ok: boolean; deletedSource: boolean }>(
|
||||
`/videos/${encodeURIComponent(id)}`,
|
||||
{ method: "DELETE" }
|
||||
);
|
||||
}
|
||||
|
||||
export function regenPreview(id: string) {
|
||||
return request<{ ok: boolean }>(
|
||||
`/videos/${encodeURIComponent(id)}/regen-preview`,
|
||||
@@ -317,6 +388,13 @@ export function createTag(label: string, aliases: string[]) {
|
||||
});
|
||||
}
|
||||
|
||||
export function deleteTag(id: number) {
|
||||
return request<{ ok: boolean; removedVideos: number }>(
|
||||
`/tags/${encodeURIComponent(String(id))}`,
|
||||
{ method: "DELETE" }
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- Settings ----------
|
||||
|
||||
export type Theme = "dark" | "pink";
|
||||
@@ -324,9 +402,9 @@ export type Theme = "dark" | "pink";
|
||||
export type Settings = {
|
||||
theme: Theme;
|
||||
/**
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak 或 p115 drive)。
|
||||
* - 空字符串:自动模式。系统中如果只挂着一个 pikpak/p115 drive 就用它;多个并存时迁移会跳过。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115}。
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115、p123 或 onedrive drive)。
|
||||
* - 空字符串:本地保存,不上传到云盘。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, p123, onedrive}。
|
||||
*/
|
||||
spider91UploadDriveId: string;
|
||||
};
|
||||
@@ -353,10 +431,32 @@ export function updateSettings(body: Partial<Settings>) {
|
||||
|
||||
/**
|
||||
* 立即触发一次完整的凌晨流水线(Phase1 扫盘 + Phase2 91 爬虫 + Phase3 迁移),
|
||||
* 不论当前时间或今日是否已跑。立即返回 202;进度通过 backend 日志观察。
|
||||
* 不论当前时间或今日是否已跑。立即返回 202;进度通过任务状态和 backend 日志观察。
|
||||
*
|
||||
* 流水线已在跑时后端会丢弃此次触发。
|
||||
* 流水线已在跑或已排队时,后端会拒绝重复触发。
|
||||
*/
|
||||
export function runNightlyJob() {
|
||||
return request<{ ok: boolean }>("/jobs/nightly/run", { method: "POST" });
|
||||
export type NightlyJobStatus = {
|
||||
state: "idle" | "queued" | "running" | "running_queued";
|
||||
running: boolean;
|
||||
queued: boolean;
|
||||
startedAt?: string;
|
||||
lastFinishedAt?: string;
|
||||
};
|
||||
|
||||
export function getNightlyJobStatus() {
|
||||
return request<NightlyJobStatus>("/jobs/nightly/status");
|
||||
}
|
||||
|
||||
export function runNightlyJob() {
|
||||
return request<{ ok: boolean; accepted: boolean; status: NightlyJobStatus }>(
|
||||
"/jobs/nightly/run",
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
export function stopAllTasks() {
|
||||
return request<{ ok: boolean; stoppedDrives: number; status: NightlyJobStatus }>(
|
||||
"/tasks/stop",
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { Trash2 } from "lucide-react";
|
||||
import * as api from "../api";
|
||||
import { Modal } from "../Modal";
|
||||
|
||||
export function DeleteDriveModal({
|
||||
drive,
|
||||
deleting,
|
||||
onCancel,
|
||||
onConfirm,
|
||||
}: {
|
||||
drive: api.AdminDrive | null;
|
||||
deleting: boolean;
|
||||
onCancel: () => void;
|
||||
onConfirm: () => void;
|
||||
}) {
|
||||
const name = drive?.name || drive?.id || "";
|
||||
const isSpider91 = drive?.kind === "spider91";
|
||||
const title = isSpider91 ? "删除 91Spider" : "删除存储";
|
||||
const primaryText = deleting ? "删除中..." : "确认删除";
|
||||
|
||||
return (
|
||||
<Modal
|
||||
open={!!drive}
|
||||
title={title}
|
||||
onClose={onCancel}
|
||||
className="admin-modal--delete-confirm"
|
||||
footer={
|
||||
<>
|
||||
<button className="admin-btn" onClick={onCancel} disabled={deleting}>
|
||||
取消
|
||||
</button>
|
||||
<button className="admin-btn is-danger" onClick={onConfirm} disabled={deleting}>
|
||||
<Trash2 size={13} />
|
||||
{primaryText}
|
||||
</button>
|
||||
</>
|
||||
}
|
||||
>
|
||||
<div className="admin-confirm is-message-centered">
|
||||
<div className="admin-confirm__content">
|
||||
<p className="admin-confirm__message">{`确定要删除「${name}」吗?`}</p>
|
||||
</div>
|
||||
</div>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user