mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-16 01:05:42 +08:00
Compare commits
61 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 858abd7532 | |||
| 91352ac681 | |||
| d70cea7320 | |||
| 6654a1b730 | |||
| bcc887e088 | |||
| 2c0cfe1d15 | |||
| a276e6b32d | |||
| fa4ea469c3 | |||
| 92885748fd | |||
| 093724a59d | |||
| 9cd30c8059 | |||
| bec6d9496c | |||
| f187302b8e | |||
| 739baf1294 | |||
| af18bbbf4c | |||
| 309b621084 | |||
| 286329c446 | |||
| 1d5b5c2495 | |||
| fac60b0054 | |||
| 19a939e80f | |||
| 16a2a7e03c | |||
| b9b6c5e098 | |||
| 4200919774 | |||
| 33d970a322 | |||
| 59e9b435a0 | |||
| dcda0e2e36 | |||
| 87709792f1 | |||
| 655da05b94 | |||
| 674a92be16 | |||
| 9ada39debb | |||
| 24d1244bc3 | |||
| 0dd0c45509 | |||
| ef6eadd0a6 | |||
| 2c5a3342cc | |||
| 7ace5f8bc7 | |||
| 66b33b2a31 | |||
| 27aefc870f | |||
| 02d82e9a62 | |||
| 492431164b | |||
| 051f1555d5 | |||
| 85292ea095 | |||
| fea5e984d1 | |||
| 46bb1fa9f2 | |||
| 7f4407ac28 | |||
| 79b5b5c37d | |||
| f3180f7b3c | |||
| 353b01b8e7 | |||
| d27eae9c62 | |||
| 003efa301b | |||
| f72898f530 | |||
| 641d29e008 | |||
| fed46b51bb | |||
| 304559203c | |||
| 62ccd6a998 | |||
| 720a92af7a | |||
| e33384c786 | |||
| 2e7c761aaf | |||
| 0d6c3c6ac9 | |||
| 10426e5483 | |||
| 863abe2064 | |||
| 60d0640a01 |
@@ -0,0 +1,21 @@
|
||||
.git
|
||||
.github
|
||||
.gitattributes
|
||||
.gitignore
|
||||
|
||||
node_modules
|
||||
dist
|
||||
release
|
||||
data
|
||||
backend/data
|
||||
backend/config.yaml
|
||||
config.yaml
|
||||
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
*.log
|
||||
*.tmp
|
||||
|
||||
tests
|
||||
video-site-implementation-plan.md
|
||||
@@ -0,0 +1,82 @@
|
||||
name: Docker
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
tags:
|
||||
- "v*"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract Docker metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=tag
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=sha,prefix=sha-
|
||||
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
|
||||
- name: Determine image version
|
||||
id: version
|
||||
shell: bash
|
||||
run: |
|
||||
if [[ "$GITHUB_REF" == refs/tags/v* ]]; then
|
||||
version="$GITHUB_REF_NAME"
|
||||
else
|
||||
version="$(git describe --tags --always --dirty 2>/dev/null || git rev-parse --short=12 HEAD)"
|
||||
fi
|
||||
echo "version=$version" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
VERSION=${{ steps.version.outputs.version }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
@@ -15,6 +15,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
@@ -36,8 +38,11 @@ jobs:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ github.ref_name }}
|
||||
run: |
|
||||
if gh release view "$TAG" >/dev/null 2>&1; then
|
||||
gh release upload "$TAG" release/*.tar.gz --clobber
|
||||
else
|
||||
gh release create "$TAG" release/*.tar.gz --title "$TAG" --notes "Prebuilt Linux release packages."
|
||||
git tag -d "$TAG" >/dev/null 2>&1 || true
|
||||
git fetch --force origin "refs/tags/$TAG:refs/tags/$TAG"
|
||||
NOTES="$(git tag -l "$TAG" --format='%(contents)')"
|
||||
if [ -z "$NOTES" ]; then
|
||||
NOTES="Prebuilt Linux release packages."
|
||||
fi
|
||||
gh release delete "$TAG" --yes >/dev/null 2>&1 || true
|
||||
gh release create "$TAG" release/*.tar.gz --title "$TAG" --notes "$NOTES" --verify-tag
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
- 视频直链 (MP4)
|
||||
|
||||
依赖安装:
|
||||
pip install requests beautifulsoup4 lxml
|
||||
pip install requests beautifulsoup4 lxml PySocks
|
||||
|
||||
使用方法:
|
||||
# 全量爬取(默认行为,从 page=1 一直爬到末尾,写到 OUTPUT_FILE)
|
||||
@@ -68,6 +68,7 @@ import time
|
||||
import random
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import html
|
||||
from urllib.parse import urljoin, unquote, urlparse
|
||||
@@ -80,6 +81,28 @@ except ImportError:
|
||||
print("请运行: pip install beautifulsoup4 lxml")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def prefer_ipv4_for_plain_socks5_proxy():
|
||||
"""PySocks may pick IPv6 first for socks5://; some SOCKS5 servers only accept IPv4."""
|
||||
proxy_envs = (
|
||||
os.environ.get("HTTPS_PROXY", ""),
|
||||
os.environ.get("HTTP_PROXY", ""),
|
||||
os.environ.get("https_proxy", ""),
|
||||
os.environ.get("http_proxy", ""),
|
||||
)
|
||||
uses_plain_socks5 = any(v.strip().lower().startswith("socks5://") for v in proxy_envs)
|
||||
if not uses_plain_socks5 or getattr(socket, "_spider91_ipv4_first", False):
|
||||
return
|
||||
|
||||
original_getaddrinfo = socket.getaddrinfo
|
||||
|
||||
def getaddrinfo_ipv4_first(*args, **kwargs):
|
||||
infos = original_getaddrinfo(*args, **kwargs)
|
||||
return sorted(infos, key=lambda info: 0 if info[0] == socket.AF_INET else 1)
|
||||
|
||||
socket.getaddrinfo = getaddrinfo_ipv4_first
|
||||
socket._spider91_ipv4_first = True
|
||||
|
||||
# ===================== 配置区域 =====================
|
||||
BASE_URL = "https://www.91porn.com/v.php"
|
||||
LIST_PARAMS = {
|
||||
@@ -706,7 +729,7 @@ def print_help():
|
||||
- 视频直链 (MP4)
|
||||
|
||||
依赖安装:
|
||||
pip install requests beautifulsoup4 lxml
|
||||
pip install requests beautifulsoup4 lxml PySocks
|
||||
|
||||
使用方法:
|
||||
python spider_91porn.py
|
||||
@@ -757,13 +780,15 @@ def main():
|
||||
"日志改走 stderr。配合 backend 边读边下载使用。")
|
||||
|
||||
args, _ = parser.parse_known_args()
|
||||
cli_out = sys.stderr if args.stream_output else sys.stdout
|
||||
prefer_ipv4_for_plain_socks5_proxy()
|
||||
|
||||
print("""
|
||||
================================================
|
||||
91porn 视频爬虫启动中...
|
||||
================================================
|
||||
按 Ctrl+C 可随时中断并保存进度
|
||||
""")
|
||||
""", file=cli_out)
|
||||
|
||||
# 加载已知 ID(来自 backend 的 catalog 已入库列表;兼容旧参数名)
|
||||
seen_viewkeys = []
|
||||
@@ -775,9 +800,9 @@ def main():
|
||||
if line:
|
||||
seen_viewkeys.append(line)
|
||||
except FileNotFoundError:
|
||||
print(f"警告: --seen-viewkeys-file 不存在: {args.seen_viewkeys_file}")
|
||||
print(f"警告: --seen-viewkeys-file 不存在: {args.seen_viewkeys_file}", file=cli_out)
|
||||
except Exception as e:
|
||||
print(f"警告: 读取 --seen-viewkeys-file 失败: {e}")
|
||||
print(f"警告: 读取 --seen-viewkeys-file 失败: {e}", file=cli_out)
|
||||
|
||||
# 决定运行模式
|
||||
if args.target_new is not None:
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Common Commands
|
||||
|
||||
### Full Local Development
|
||||
|
||||
```bash
|
||||
npm install
|
||||
./start.sh
|
||||
```
|
||||
|
||||
`./start.sh` starts the Go backend on `127.0.0.1:9192` and the frontend on `0.0.0.0:9191`. By default it builds the frontend and runs Vite preview mode. Use hot reload with:
|
||||
|
||||
```bash
|
||||
FRONTEND_MODE=dev ./start.sh --restart
|
||||
```
|
||||
|
||||
Useful variants:
|
||||
|
||||
```bash
|
||||
./start.sh --status
|
||||
./start.sh --restart
|
||||
./start.sh --stop
|
||||
```
|
||||
|
||||
### Frontend
|
||||
|
||||
Run from the repository root:
|
||||
|
||||
```bash
|
||||
npm run dev # Vite dev server; default port comes from Vite unless overridden
|
||||
npm run dev:raw # Vite dev server on 127.0.0.1:5173
|
||||
npm run build # tsc -b && vite build
|
||||
npm run preview # Vite preview; vite.config.ts uses port 9191
|
||||
npm run lint # TypeScript no-emit check
|
||||
npm test # node --import tsx --test tests/*.test.ts
|
||||
```
|
||||
|
||||
Run one frontend test:
|
||||
|
||||
```bash
|
||||
node --import tsx --test tests/previewIntent.test.ts
|
||||
```
|
||||
|
||||
### Backend
|
||||
|
||||
Run from `backend/` unless noted:
|
||||
|
||||
```bash
|
||||
go run ./cmd/server
|
||||
go test ./... -count=1
|
||||
go build -o video-server ./cmd/server
|
||||
```
|
||||
|
||||
Run one backend package or test:
|
||||
|
||||
```bash
|
||||
go test ./internal/scanner -count=1
|
||||
go test ./internal/scanner -run TestParse -count=1
|
||||
```
|
||||
|
||||
The backend requires Go 1.23+ and uses vendored dependencies in `backend/vendor/`, so keep `go mod vendor` in sync after dependency changes.
|
||||
|
||||
### Release and Deployment
|
||||
|
||||
```bash
|
||||
scripts/build-release.sh # builds Linux amd64/arm64 release tarballs into release/
|
||||
sudo bash install.sh # prebuilt installer flow used by README
|
||||
sudo bash deploy.sh # build from current checkout and install systemd services
|
||||
```
|
||||
|
||||
Docker uses the root `Dockerfile` and `docker-compose.yml`. The runtime image exposes port `9191` and stores persistent data under `/opt/video-site-91/data`.
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
This is a private video aggregation site with a React/Vite frontend and a Go backend.
|
||||
|
||||
### Frontend
|
||||
|
||||
The frontend is a React 18 SPA under `src/`. `src/main.tsx` mounts `BrowserRouter`, `ToastProvider`, and `AuthProvider`, then renders `src/App.tsx`. `App.tsx` defines the public app routes (`/`, `/list`, `/shorts`, `/upload`, `/video/:id`) and admin routes under `/admin`; both main-site and admin pages are wrapped in `RequireAuth`, while `/login` is public.
|
||||
|
||||
Frontend API calls are split by surface:
|
||||
|
||||
- `src/data/videos.ts` calls the main authenticated API under `/api` and upload/proxy-related endpoints.
|
||||
- `src/admin/api.ts` is the admin API client for `/admin/api`, always sending cookies and raising `UnauthorizedError` on `401`.
|
||||
|
||||
`vite.config.ts` proxies `/api`, `/p`, and `/admin/api` to `http://127.0.0.1:9192`, with frontend dev/preview served on port `9191` by default. The alias `@` maps to `src`.
|
||||
|
||||
Styling is plain CSS loaded from `src/main.tsx` in token/base/layout/navigation/search/video/admin layers. Shared UI lives in `src/components`, page-level screens in `src/pages`, and admin screens in `src/admin`.
|
||||
|
||||
### Backend
|
||||
|
||||
The backend entrypoint is `backend/cmd/server/main.go`. It loads `config.yaml` or `VIDEO_CONFIG`, creates the SQLite catalog and preview directories, builds the app state, registers API routes, starts the nightly runner, and then asynchronously attaches configured external drives so slow upstream login checks do not block port binding.
|
||||
|
||||
Important backend packages:
|
||||
|
||||
- `internal/config`: YAML config loading and first-run admin credential setup.
|
||||
- `internal/catalog`: SQLite catalog, schema migration, video metadata, settings, tags, drive records, generation status, and deduplication state. It opens SQLite with WAL and a busy timeout.
|
||||
- `internal/drives`: provider abstraction. Implementations include `quark`, `p115`, `pikpak`, `wopan`, `onedrive`, `localstorage`, `localupload`, and `spider91`.
|
||||
- `internal/scanner`: recursively lists drive directories, parses filenames/tags, upserts catalog videos, applies skip-directory rules, and enqueues newly discovered videos.
|
||||
- `internal/preview`: ffprobe/ffmpeg thumbnail and teaser generation workers. Generated assets are local files under the configured preview directory.
|
||||
- `internal/fingerprint`: asynchronous sampled SHA-256 worker used for cross-drive duplicate detection.
|
||||
- `internal/proxy`: `/p/*` media serving. Some providers redirect with `302` to signed CDN URLs, while providers requiring backend-held headers are reverse-proxied with Range support.
|
||||
- `internal/api`: main API and admin API route handlers.
|
||||
- `internal/nightly`: daily pipeline for drive scans, spider91 crawl, migration, queue drain, and duplicate asset cleanup.
|
||||
- `internal/spider91migrate`: migration from spider91 downloads to a configured cloud drive.
|
||||
|
||||
### Runtime Flow
|
||||
|
||||
1. Admin adds or edits drives through `/admin/drives`, which persists drive config in the catalog.
|
||||
2. The server attaches the drive implementation into the proxy registry and can trigger scans.
|
||||
3. Scans convert provider files into catalog video rows, parse titles/authors/tags from filenames, and queue preview/fingerprint work.
|
||||
4. The frontend lists videos through `/api/home`, `/api/list`, `/api/video/:id`, and streams media through `/p/*` endpoints.
|
||||
5. The nightly runner performs the scheduled end-to-end maintenance pipeline; admins can trigger it manually through `/admin/api/jobs/nightly/run`.
|
||||
|
||||
### Configuration and Data
|
||||
|
||||
Backend defaults come from `backend/config.example.yaml`. On first backend start, `config.yaml` is created automatically if missing. Default local development paths are:
|
||||
|
||||
- Backend listen address: `127.0.0.1:9192`
|
||||
- SQLite DB: `backend/data/video-site.db`
|
||||
- Generated previews/thumbs: `backend/data/previews`
|
||||
|
||||
Docker and installer deployments rewrite config paths so data lives under `/opt/video-site-91/data` or the mounted `./data` directory.
|
||||
|
||||
`VIDEO_FRONTEND_DIR` controls where the Go server looks for built frontend assets. If unset, it serves `./dist` when present. Backend routes (`/api`, `/admin/api`, `/p`) are excluded from the SPA fallback.
|
||||
|
||||
## Notes for Changes
|
||||
|
||||
- Main-site API routes and proxy routes require authentication; only login/setup and `/api/settings/theme` are intentionally public.
|
||||
- When adding a new drive provider, implement `internal/drives.Drive`, persist any needed config through catalog/admin APIs, attach it in `cmd/server`, and decide whether `/p/stream` should redirect or reverse-proxy in `internal/proxy`.
|
||||
- Generated thumbnails and teasers are local runtime assets; do not treat them as source files.
|
||||
- Frontend tests use Node's built-in test runner with `tsx`; TypeScript linting only checks `src` through the root `tsconfig.json`.
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
# ---- Stage 1: Build frontend ----
|
||||
FROM node:20-slim AS frontend
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tsconfig.json vite.config.ts index.html ./
|
||||
COPY public/ public/
|
||||
COPY src/ src/
|
||||
RUN npm run build
|
||||
|
||||
# ---- Stage 2: Build backend ----
|
||||
FROM golang:1.23-bookworm AS backend
|
||||
|
||||
WORKDIR /app/backend
|
||||
|
||||
COPY backend/go.mod backend/go.sum ./
|
||||
COPY backend/vendor/ vendor/
|
||||
COPY backend/cmd/ cmd/
|
||||
COPY backend/internal/ internal/
|
||||
RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server
|
||||
|
||||
# ---- Stage 3: Runtime ----
|
||||
FROM debian:bookworm-slim AS runtime
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
ffmpeg \
|
||||
openssl \
|
||||
python3 \
|
||||
python3-bs4 \
|
||||
python3-lxml \
|
||||
python3-requests \
|
||||
python3-socks \
|
||||
tar \
|
||||
tzdata \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /opt/video-site-91
|
||||
|
||||
COPY --from=backend /out/server ./server
|
||||
COPY --from=frontend /app/dist ./dist
|
||||
COPY backend/config.example.yaml ./config.example.yaml
|
||||
COPY 91VideoSpider/ ./91VideoSpider/
|
||||
COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
ENV VIDEO_CONFIG=/opt/video-site-91/data/config.yaml \
|
||||
VIDEO_FRONTEND_DIR=/opt/video-site-91/dist \
|
||||
VIDEO_GITHUB_REPO=nianzhibai/91 \
|
||||
VIDEO_IMAGE_VERSION=${VERSION} \
|
||||
VIDEO_LISTEN_PORT=9191 \
|
||||
VIDEO_VERSION_FILE=/opt/video-site-91/data/.version
|
||||
|
||||
RUN chmod +x ./server /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
VOLUME ["/opt/video-site-91/data"]
|
||||
EXPOSE 9191
|
||||
|
||||
ENTRYPOINT ["docker-entrypoint.sh"]
|
||||
CMD ["./server"]
|
||||
@@ -1,171 +1,203 @@
|
||||
# 视频聚合站
|
||||
# 91
|
||||
|
||||
把散落在不同网盘里的视频,整理成一个可以自己登录、自己浏览、自己管理的私人视频站。
|
||||
<p align="center">
|
||||
<img width="120" height="120" alt="91" src="https://github.com/user-attachments/assets/5b323c94-bbd3-4dce-bbc8-adc86935b7de" />
|
||||
</p>
|
||||
|
||||
网盘适合存东西,却不适合慢慢看东西。文件多了以后,你很难记住它们在哪里、叫什么、有没有看过、还能不能快速预览。这个项目做的是中间那一层:文件仍然留在原来的网盘里,但你可以用一个更像视频站的界面去搜索、筛选、预览和管理它们。
|
||||
<p align="center">
|
||||
😄 个人私有视频站 😄
|
||||
</p>
|
||||
|
||||
它不是另一个网盘客户端,也不是内容平台。它更像是给你自己的视频收藏做一个入口:安静、集中、可控。
|
||||
<p align="center">
|
||||
<a href="#快速开始">快速开始</a> ·
|
||||
<a href="#功能特性">功能特性</a> ·
|
||||
<a href="#预览图">预览图</a> ·
|
||||
<a href="#数据存放位置">数据目录</a>
|
||||
</p>
|
||||
|
||||
## 它能做什么
|
||||
---
|
||||
|
||||
- **统一入口**:把 115、PikPak、夸克、联通沃盘、OneDrive、本地上传和可选的 91 爬虫源放在同一个站里浏览。
|
||||
- **像视频站一样浏览**:首页推荐、最新视频、列表页、搜索、标签筛选、详情播放和相关推荐都已经接好。
|
||||
- **自动生成预览**:后端会用 ffmpeg 在本地生成封面和短 teaser,扫到新视频后不用一条条手动整理。
|
||||
- **保留网盘本身**:视频文件不需要搬家,播放时由后端按来源取链和代理。
|
||||
- **后台可管理**:在管理后台添加网盘、扫描所有网盘、编辑视频信息、维护标签、切换主题。
|
||||
- **首次部署更直接**:第一次访问时会要求设置管理员用户名和密码,设置后保存到本地配置文件。
|
||||
- **适合长期运行**:扫描、预览、隐藏视频、标签归类这些重复工作,都尽量交给系统处理。
|
||||
## 功能特性
|
||||
|
||||
## 适合谁
|
||||
- **多后端支持** — 兼容 115 云盘、PikPak 云盘、OneDrive、Google Drive 和本地存储
|
||||
- **低带宽播放** — 115 云盘、PikPak 云盘、OneDrive 都支持302模式,在线播放视频时,不占用服务器带宽,播放体验不受服务器带宽影响;Google Drive 不支持302模式,走服务器中转,观看体验会受服务器带宽影响
|
||||
- **封面 & 预览片段** — 自动为每个视频生成封面图和预览片段,首页快速选片
|
||||
- **91 爬虫** — 内置爬虫,支持抓取 91 本月最热视频
|
||||
- **双主题** — 黑黄经典主题 / 粉白清新主题,随时切换
|
||||
- **短视频模式** — 一键切换抖音风格,沉浸刷片
|
||||
- **低资源占用** — 2C2G 服务器稳定运行,主要性能消耗就是封面图和预览视频的生成
|
||||
|
||||
如果你有一批视频散落在多个网盘里,想把它们整理成一个自己的私有站点,这个项目会比较合适。
|
||||
---
|
||||
|
||||
如果你只是想临时播放单个文件,直接用网盘客户端更简单;如果你想做公开视频网站,这个项目也不是为那个场景设计的。它的重点是个人部署、个人管理、个人观看。
|
||||
## 预览图
|
||||
|
||||
## 支持的来源
|
||||
### 电脑端
|
||||
|
||||
- 115 网盘
|
||||
- PikPak
|
||||
- 91 爬虫源
|
||||
- 夸克网盘
|
||||
- 联通沃盘
|
||||
- OneDrive
|
||||
- 本地上传
|
||||
<p>
|
||||
<img width="49%" alt="首页" src="https://github.com/user-attachments/assets/9808fceb-760b-4dd5-b7d2-8622b95b90d5" />
|
||||
<img width="49%" alt="播放页" src="https://github.com/user-attachments/assets/859db4aa-1fba-44f2-bb46-1db07c2f964f" />
|
||||
</p>
|
||||
|
||||
91 爬虫源是一种特殊存储来源,用来把爬虫抓到的视频和封面接入站内目录。它不是必须项;如果你只想管理自己的网盘,可以完全不启用。
|
||||
<p>
|
||||
<img width="49%" alt="主题切换" src="https://github.com/user-attachments/assets/96bea37a-8764-413e-9b70-1856b4ae0cd2" />
|
||||
<img width="49%" alt="管理页" src="https://github.com/user-attachments/assets/29c1e27a-7651-4dfc-93dd-556331844214" />
|
||||
</p>
|
||||
|
||||
### 手机端
|
||||
|
||||
<p align="center">
|
||||
<img width="1284" height="1134" alt="手机端" src="https://github.com/user-attachments/assets/bdb7a86c-a4e5-483e-a307-e02c0bb34dac" />
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
## 快速开始
|
||||
|
||||
需要先准备:
|
||||
|
||||
- Node.js 18+
|
||||
- Go 1.23+
|
||||
- ffmpeg 和 ffprobe
|
||||
|
||||
启动项目:
|
||||
### 方式一:一键安装脚本(推荐)
|
||||
|
||||
```bash
|
||||
npm install
|
||||
./start.sh
|
||||
```
|
||||
|
||||
默认访问地址:
|
||||
|
||||
- 前台:`http://127.0.0.1:9191/`
|
||||
- 后台:`http://127.0.0.1:9191/admin`
|
||||
- 后端:`127.0.0.1:9192`
|
||||
|
||||
第一次打开时,如果还没有设置管理员账号,页面会引导你创建用户名和密码。保存后会写入本地的 `backend/config.yaml`。
|
||||
|
||||
常用命令:
|
||||
|
||||
```bash
|
||||
./start.sh --status
|
||||
./start.sh --restart
|
||||
./start.sh --stop
|
||||
```
|
||||
|
||||
需要前端热更新时:
|
||||
|
||||
```bash
|
||||
FRONTEND_MODE=dev ./start.sh --restart
|
||||
```
|
||||
|
||||
## 新服务器一键安装
|
||||
|
||||
如果你只是想在一台 Ubuntu / Debian 服务器上尽快跑起来,推荐使用预编译安装脚本。普通用户不需要安装 Go、Node.js,也不需要自己编译;脚本会按服务器 CPU 架构下载 GitHub Release 里的预编译包,安装运行依赖,写入 systemd 服务并启动。
|
||||
|
||||
```bash
|
||||
sudo apt update
|
||||
sudo apt install -y curl ca-certificates
|
||||
sudo apt update && sudo apt install -y curl ca-certificates
|
||||
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/install.sh -o install.sh
|
||||
sudo bash install.sh
|
||||
```
|
||||
|
||||
部署完成后访问:
|
||||
|
||||
- 前台:`http://服务器IP:9191/`
|
||||
- 后台:`http://服务器IP:9191/admin`
|
||||
| 地址 | 说明 |
|
||||
|------|------|
|
||||
| `http://服务器IP:9191/` | 前台 |
|
||||
| `http://服务器IP:9191/admin` | 后台管理 |
|
||||
|
||||
第一次打开后台会要求设置管理员用户名和密码。常用维护命令:
|
||||
**注意:如果首次访问,显示502,可以运行 `91 restart` 重启一下服务**
|
||||
|
||||
安装后自动注册 `91` 管理命令:
|
||||
|
||||
```bash
|
||||
sudo bash install.sh status
|
||||
sudo bash install.sh logs
|
||||
sudo bash install.sh update
|
||||
sudo bash install.sh restart
|
||||
sudo bash install.sh stop
|
||||
91 # 打开管理菜单
|
||||
91 status # 查看运行状态
|
||||
91 logs # 查看日志
|
||||
91 update # 更新到最新版本
|
||||
91 restart # 重启服务
|
||||
91 stop # 停止服务
|
||||
```
|
||||
|
||||
安装后会自动创建 `91` 指令,和 OpenList 的管理指令类似:
|
||||
> `video-site-91` 为等效别名,两者可互换使用。
|
||||
|
||||
```bash
|
||||
91 # 打开管理菜单
|
||||
91 status # 查看状态
|
||||
91 logs # 查看日志
|
||||
91 update # 更新
|
||||
91 restart # 重启
|
||||
91 stop # 停止
|
||||
```
|
||||
|
||||
同时也保留 `video-site-91` 作为同等别名。
|
||||
|
||||
想换端口:
|
||||
**自定义端口:**
|
||||
|
||||
```bash
|
||||
FRONTEND_PORT=8080 sudo -E bash install.sh
|
||||
```
|
||||
|
||||
如果服务器还有云厂商安全组,请记得放行对应端口,默认是 `9191/tcp`。
|
||||
**旧版本升级(v0.0.2 之前):**
|
||||
|
||||
如果你是项目维护者,要预先编译发布包:
|
||||
旧版脚本直接执行 `91 update` 可能失败,先执行以下修复命令:
|
||||
|
||||
```bash
|
||||
scripts/build-release.sh
|
||||
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/install.sh -o /tmp/install-91.sh
|
||||
sudo bash /tmp/install-91.sh update
|
||||
```
|
||||
|
||||
它会生成:
|
||||
---
|
||||
|
||||
- `release/video-site-91-linux-amd64.tar.gz`
|
||||
- `release/video-site-91-linux-arm64.tar.gz`
|
||||
### 方式二:Docker Compose 部署
|
||||
|
||||
把这两个文件上传到 GitHub Release 后,`install.sh` 就能自动下载。仓库也带了 GitHub Actions:推送 `v*` 标签时会自动构建并上传这两个 Release 包。
|
||||
**1. 准备目录**
|
||||
|
||||
源码部署仍然保留在 `deploy.sh`,适合你想在服务器上直接 clone、编译和调试时使用。
|
||||
```bash
|
||||
mkdir -p video-site-91 && cd video-site-91
|
||||
```
|
||||
|
||||
## 第一次使用
|
||||
**2. 创建 `docker-compose.yml`**
|
||||
|
||||
1. 打开 `http://127.0.0.1:9191/`,先完成管理员账号设置。
|
||||
2. 进入 `/admin`,在网盘管理里新建一个来源。
|
||||
3. 填入名称和对应凭证,保存。
|
||||
4. 点击“扫描所有网盘”,等待视频入库。
|
||||
5. 回到前台,用首页、搜索、标签和详情页浏览内容。
|
||||
```yaml
|
||||
services:
|
||||
video-site-91:
|
||||
image: ghcr.io/nianzhibai/91:stable
|
||||
container_name: video-site-91
|
||||
ports:
|
||||
- "9191:9191"
|
||||
volumes:
|
||||
- ./data:/opt/video-site-91/data
|
||||
restart: unless-stopped
|
||||
```
|
||||
创建yml文件后运行下面指令
|
||||
```bash
|
||||
docker compose pull
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
## 数据放在哪里
|
||||
如果想固定某个 Release 版本,可以改成明确的 tag,例如:
|
||||
|
||||
项目会把运行数据保存在本地:
|
||||
```yaml
|
||||
image: ghcr.io/nianzhibai/91:v0.0.6
|
||||
```
|
||||
|
||||
- `backend/config.yaml`:本地配置、管理员账号、网盘凭证。
|
||||
- `backend/data/video-site.db`:SQLite 数据库。
|
||||
- `backend/data/previews/`:本地生成的封面和 teaser。
|
||||
或直接拉取仓库内置配置:
|
||||
|
||||
这些文件不应该提交到公开仓库。仓库里的 `backend/config.example.yaml` 只是模板,不应该放真实账号、Cookie、Token 或密码。
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/nianzhibai/91/main/docker-compose.yml -o docker-compose.yml
|
||||
```
|
||||
|
||||
**3. 启动**
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
**常用命令:**
|
||||
|
||||
```bash
|
||||
docker compose logs -f # 查看日志
|
||||
docker compose pull # 拉取最新正式版 stable 镜像
|
||||
docker compose up -d # 更新并重启
|
||||
```
|
||||
|
||||
> 所有配置、数据库、封面、预览及上传文件均保存在 `./data/` 目录下。
|
||||
|
||||
---
|
||||
|
||||
## 数据存放位置
|
||||
|
||||
### 一键脚本部署
|
||||
|
||||
| 路径 | 内容 |
|
||||
|------|------|
|
||||
| `/opt/video-site-91/config.yaml` | 配置文件、管理员账号、网盘凭证 |
|
||||
| `/opt/video-site-91/data/video-site.db` | SQLite 数据库 |
|
||||
| `/opt/video-site-91/data/previews/` | 封面图和预览片段 |
|
||||
|
||||
### Docker Compose 部署
|
||||
|
||||
| 路径 | 内容 |
|
||||
|------|------|
|
||||
| `./data/config.yaml` | 配置文件、管理员账号、网盘凭证 |
|
||||
| `./data/video-site.db` | SQLite 数据库 |
|
||||
| `./data/previews/` | 封面图和预览片段 |
|
||||
| `./data/uploads/` | 本地上传的视频文件 |
|
||||
| `./data/spider91/` | 91 爬虫抓取的视频文件 |
|
||||
|
||||
---
|
||||
|
||||
## 更多文档
|
||||
|
||||
根目录 README 只保留项目介绍和最短上手路径。更细的实现、接口、网盘字段和部署方式可以看:
|
||||
| 文档 | 内容 |
|
||||
|------|------|
|
||||
| [backend/README.md](backend/README.md) | 后端实现、接口说明、网盘字段 |
|
||||
| [video-site-implementation-plan.md](video-site-implementation-plan.md) | 完整实现方案 |
|
||||
|
||||
- [backend/README.md](backend/README.md)
|
||||
- [video-site-implementation-plan.md](video-site-implementation-plan.md)
|
||||
---
|
||||
|
||||
## 开发验证
|
||||
## 使用须知
|
||||
|
||||
```bash
|
||||
npm run lint
|
||||
npm test
|
||||
cd backend && go test ./... -count=1
|
||||
```
|
||||
本项目面向**个人私有部署**,请仅接入你有权访问和管理的内容,并遵守对应网盘、站点的服务条款及所在地法律法规。
|
||||
|
||||
## 使用边界
|
||||
> 不对外传播,仅限个人使用。
|
||||
|
||||
这个项目面向个人私有部署。请只接入你有权访问和管理的内容,并遵守对应网盘、站点服务条款以及所在地法律法规。
|
||||
---
|
||||
|
||||
## 致谢
|
||||
|
||||
- [OpenList](https://github.com/OpenListTeam/OpenList) — 优秀的开源项目
|
||||
- [LinuxDo](https://linux.do/) — 学 AI 上 L 站
|
||||
- [NodeSeek](https://nodeseek.com/) — MJJ 上 N 站
|
||||
|
||||
+34
-11
@@ -2,7 +2,7 @@
|
||||
|
||||
视频聚合站的 Go 后端。提供三件事:
|
||||
|
||||
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive)
|
||||
1. 多家网盘统一抽象(夸克 / 115 / PikPak / 联通沃盘 / OneDrive / Google Drive / 本地存储)
|
||||
2. 视频元数据目录(SQLite)+ 扫描 + teaser 预生成
|
||||
3. REST API(前台)+ 管理后台 + 直链代理
|
||||
4. 标签池、视频隐藏、按网盘统计和详情页来源网盘类型展示能力
|
||||
@@ -21,6 +21,8 @@ internal/
|
||||
pikpak/ PikPak(自己实现,参考 OpenList pikpak)
|
||||
wopan/ 联通沃盘(壳子 + OpenListTeam/wopan-sdk-go)
|
||||
onedrive/ OneDrive(OpenList 在线续期 + Microsoft Graph 文件接口)
|
||||
googledrive/ Google Drive(OpenList 在线续期 + Google Drive API;播放走后端代理)
|
||||
localstorage/ 本地目录扫描(服务器已有视频目录)
|
||||
scanner/ 扫目录 → 落库
|
||||
preview/ ffmpeg 抽封面和生成多段 teaser
|
||||
proxy/ /p/stream/*、/p/preview/* 代理
|
||||
@@ -91,7 +93,6 @@ go run ./cmd/server 后端 9192
|
||||
"kind": "quark",
|
||||
"name": "我的夸克盘",
|
||||
"rootId": "0",
|
||||
"scanRootId": "0",
|
||||
"credentials": {
|
||||
"cookie": "粘贴浏览器 F12 复制的 pan.quark.cn Cookie"
|
||||
}
|
||||
@@ -105,9 +106,11 @@ go run ./cmd/server 后端 9192
|
||||
|--------|---------------------------------------------------------------|
|
||||
| quark | `cookie` |
|
||||
| p115 | `cookie`(形如 `UID=...; CID=...; SEID=...; KID=...`) |
|
||||
| pikpak | `username`、`password`,可选 `refresh_token`、`captcha_token`、`device_id`、`platform`、`disable_media_link` |
|
||||
| pikpak | `username`、`password`(token、验证码和设备 ID 由服务端自动处理并保存) |
|
||||
| wopan | `access_token`、`refresh_token`,可选 `family_id` |
|
||||
| onedrive | `refresh_token`,可选 `access_token`、`api_url_address`、`region`、`is_sharepoint`、`site_id` |
|
||||
| onedrive | `refresh_token` |
|
||||
| googledrive | `refresh_token` |
|
||||
| localstorage | `path`(服务器上的已有视频目录,如 `/mnt/videos`) |
|
||||
|
||||
### PikPak 速度说明
|
||||
|
||||
@@ -115,24 +118,44 @@ go run ./cmd/server 后端 9192
|
||||
|
||||
当前服务器同时存在 sing-box TUN 透明代理,PikPak 默认出站会被 `tun0` 接管;但强制直连物理网卡并没有更快,慢速的主要差异来自 PikPak 取链方式。media/cache CDN 节点仍有波动,偶尔可能遇到慢节点;如果播放变慢,可重新获取直链或重新挂载 PikPak 后再测。
|
||||
|
||||
OneDrive 按 OpenList 默认方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。OpenList 代刷得到的 refresh token 可以直接填到本项目。普通 OneDrive 的 `rootId` / `scanRootId` 可填 `root`;SharePoint 文档库需要额外设置 `is_sharepoint=true` 和 `site_id`。
|
||||
OneDrive 按 OpenList 默认应用方式调用 `https://api.oplist.org/onedrive/renewapi` 在线刷新 token,不需要配置 Azure 应用的 `client_id` / `client_secret` / `redirect_uri`。后台新建 OneDrive 时只需要填 OpenList 代刷得到的 `refresh_token`;服务端会默认挂载根目录并自动回写新 token。
|
||||
|
||||
Google Drive 按 OpenList 在线 API 调用 `https://api.oplist.org/googleui/renewapi` 刷新 token。后台新建 Google Drive 时只需要填 OpenList Google Drive 获取到的 `refresh_token`。Google Drive 下载地址必须携带 `Authorization` 头,浏览器不能直接 302 使用,所以本站会由后端代理 `/p/stream` 播放,不加入零带宽 302 白名单。
|
||||
|
||||
## 文件名约定
|
||||
|
||||
扫描器按以下顺序解析文件名:
|
||||
扫描器按以下顺序解析文件名,用于提取标题和作者:
|
||||
|
||||
1. `[tag1,tag2] 标题 - 作者.mp4`
|
||||
2. `[tag1,tag2] 标题.mp4`
|
||||
1. `[前缀] 标题 - 作者.mp4`
|
||||
2. `[前缀] 标题.mp4`
|
||||
3. `标题 - 作者.mp4`
|
||||
4. `标题.mp4`
|
||||
|
||||
标签分隔符支持 `, , 、` 和空格。解析结果会和系统标签池匹配,常见番号类噪声会归并到 `AV` 等系统标签,避免把每个番号都变成独立标签。解析结果可在管理后台覆盖。
|
||||
开头的 `[前缀]` 只会从标题里剥离,不会按分隔符作为任意标签入库。视频标签来自三类规则:
|
||||
|
||||
1. 文件名、作者和目录名命中系统标签或已有标签的标签名 / 别名。
|
||||
2. 符合条件的目录名会自动创建 `collection` 合集标签,并给同目录视频打上该标签。
|
||||
3. 常见番号类噪声会统一归并到 `AV`,避免把每个番号都变成独立标签。
|
||||
|
||||
当前内置系统标签为:`后入`、`奶子`、`口交`、`臀`、`人妻`、`女大`、`AV`。解析结果可在管理后台覆盖;手动保存后,该视频会标记为人工标签,后续扫描不会再自动覆盖。
|
||||
|
||||
## 视频去重
|
||||
|
||||
项目有三层去重:
|
||||
|
||||
1. 同一网盘同一文件按 `(drive_id, file_id)` 形成稳定视频 ID,重复扫描只更新同一行。
|
||||
2. 扫描时优先按网盘侧 `content_hash` 去重;没有 hash 时退化为 `file_name + size_bytes`。
|
||||
3. 扫描、爬虫、本地上传或服务启动挂载网盘后,后台指纹 worker 会异步读取视频的少量 Range 片段,生成 `sampled_sha256`。前台列表、首页、搜索、推荐会按 `size_bytes + sampled_sha256` 只展示最早入库的 canonical 视频。
|
||||
|
||||
`sampled_sha256` 是文件级去重:适合识别同一个视频文件被复制到 115 / PikPak / OneDrive 等不同网盘的情况。它不会删除任何网盘文件,也不用于识别转码、裁剪、加水印后的同源视频。
|
||||
|
||||
封面和 teaser 仍然优先生成,不等待指纹完成。夜间流水线最后会做一次重复资产清理:对 `size_bytes + sampled_sha256` 命中的非 canonical 视频,只删除本机生成的重复封面和 teaser,并把对应字段重置为 `pending`。网盘原文件和视频元数据记录不会被删除;如果 canonical 视频以后被移除,这些重复项会重新进入生成队列。
|
||||
|
||||
## 管理能力
|
||||
|
||||
- `/admin/drives`:新增、编辑、删除网盘,触发扫描。
|
||||
- `/admin/videos`:按网盘筛选视频,每页 100 条分页,查看各网盘 Teaser 统计,编辑标题/作者/分类/标签,单条或全量重生 teaser。
|
||||
- `/admin/tags`:新增标签并用内置规则自动匹配已有视频。
|
||||
- `/admin/tags`:新增标签并用内置规则自动匹配已有视频;删除非系统标签时会从所有视频上同步移除该标签。
|
||||
- 播放页视频信息会展示来源网盘类型;同时提供“不再展示”,点击后会把视频标记为全局隐藏。隐藏视频不会再出现在首页、列表、搜索、相关推荐和详情接口中。目前没有管理后台恢复入口,如需恢复可把数据库里对应视频的 `hidden` 字段改回 `0`。
|
||||
|
||||
## Teaser 生成
|
||||
@@ -147,7 +170,7 @@ ffmpeg -ss <起点> -headers "UA/Cookie/Referer" -i <直链> \
|
||||
|
||||
当前策略是每段固定 3 秒;30 秒以下最多 3 段,30 秒及以上固定 4 段;长视频在 20% 到 80% 区间均匀取段。生成的 teaser 和封面都只保存在本地 `data/previews/`,不会回写到网盘;旧数据中的 `preview_file_id` 会被忽略。
|
||||
|
||||
服务启动或网盘重新挂载时,如果 Teaser 开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 直链生成 teaser 时可能触发 Microsoft 429 限流;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。
|
||||
服务启动或网盘重新挂载时,如果 Teaser 开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 扫盘和直链生成 teaser / 封面时可能触发 Microsoft Graph 429、`TooManyRequests`、`activityLimitReached` 或 throttled 文本;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。扫盘阶段会按 `Retry-After` 或默认冷却时间等待后继续当前目录。
|
||||
|
||||
前端卡片的 `previewSrc` 统一指向 `/p/preview/<videoID>`,后端只从本地 `preview_local` 文件读取。
|
||||
|
||||
|
||||
@@ -67,3 +67,40 @@ func TestFrontendHandlerDoesNotSwallowBackendRoutes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveFrontendDirFallsBackToParentDist(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
backendDir := filepath.Join(workspace, "backend")
|
||||
distDir := filepath.Join(workspace, "dist")
|
||||
if err := os.MkdirAll(backendDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir backend: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(distDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir dist: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(distDir, "index.html"), []byte("<html>app</html>"), 0o644); err != nil {
|
||||
t.Fatalf("write index: %v", err)
|
||||
}
|
||||
|
||||
oldWD, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("getwd: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := os.Chdir(oldWD); err != nil {
|
||||
t.Fatalf("restore wd: %v", err)
|
||||
}
|
||||
})
|
||||
t.Setenv("VIDEO_FRONTEND_DIR", "")
|
||||
if err := os.Chdir(backendDir); err != nil {
|
||||
t.Fatalf("chdir backend: %v", err)
|
||||
}
|
||||
|
||||
got, ok := resolveFrontendDir()
|
||||
if !ok {
|
||||
t.Fatal("resolveFrontendDir ok = false, want true")
|
||||
}
|
||||
if got != "../dist" {
|
||||
t.Fatalf("frontend dir = %q, want ../dist", got)
|
||||
}
|
||||
}
|
||||
|
||||
+547
-152
File diff suppressed because it is too large
Load Diff
@@ -1,9 +1,13 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
)
|
||||
|
||||
func TestSpider91IntCredFallbacks(t *testing.T) {
|
||||
@@ -30,3 +34,56 @@ func TestSpider91IntCredFallbacks(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
|
||||
reg := proxy.NewRegistry()
|
||||
reg.Set("p115-one", &spider91UploadTargetFakeDrive{id: "p115-one", kind: "p115"})
|
||||
reg.Set("onedrive-one", &spider91UploadTargetFakeDrive{id: "onedrive-one", kind: "onedrive"})
|
||||
|
||||
app := &App{registry: reg}
|
||||
if got := app.Spider91UploadDriveID(); got != "" {
|
||||
t.Fatalf("empty upload target selected %q, want local-only empty target", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "p115-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "p115-one" {
|
||||
t.Fatalf("explicit upload target = %q, want p115-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "onedrive-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "onedrive-one" {
|
||||
t.Fatalf("explicit onedrive upload target = %q, want onedrive-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "missing"
|
||||
if got := app.Spider91UploadDriveID(); got != "" {
|
||||
t.Fatalf("missing upload target = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
type spider91UploadTargetFakeDrive struct {
|
||||
id string
|
||||
kind string
|
||||
}
|
||||
|
||||
func (d *spider91UploadTargetFakeDrive) Kind() string { return d.kind }
|
||||
func (d *spider91UploadTargetFakeDrive) ID() string { return d.id }
|
||||
func (d *spider91UploadTargetFakeDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) RootID() string { return "root" }
|
||||
|
||||
+368
-37
@@ -13,7 +13,9 @@ import (
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/config"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/fingerprint"
|
||||
"github.com/video-site/backend/internal/preview"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
)
|
||||
|
||||
func TestRegisterPreviewWorkerBackfillsPendingWhenDriveTeaserEnabled(t *testing.T) {
|
||||
@@ -53,7 +55,7 @@ func TestRegisterPreviewWorkerBackfillsPendingWhenDriveTeaserEnabled(t *testing.
|
||||
worker := preview.NewWorker(&serverFakeTeaserGenerator{}, cat, &serverFakeDrive{})
|
||||
go worker.Run(ctx)
|
||||
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, nil, func() {})
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, nil, nil, func() {})
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
@@ -77,7 +79,7 @@ func TestRegisterPreviewWorkerBackfillsPendingWhenDriveTeaserEnabled(t *testing.
|
||||
t.Fatalf("preview status = %q, want ready", got.PreviewStatus)
|
||||
}
|
||||
|
||||
func TestRegisterPreviewWorkersGenerateThumbnailsBeforePreviews(t *testing.T) {
|
||||
func TestRegisterPreviewWorkersRunThumbnailsAndPreviewsIndependently(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
@@ -93,16 +95,18 @@ func TestRegisterPreviewWorkersGenerateThumbnailsBeforePreviews(t *testing.T) {
|
||||
|
||||
seedDriveWithTeaser(t, cat, "drive-id", true)
|
||||
now := time.Now()
|
||||
for _, v := range []*catalog.Video{
|
||||
{ID: "video-1", DriveID: "drive-id", FileID: "file-1", Title: "Clip 1", PreviewStatus: "pending"},
|
||||
{ID: "video-2", DriveID: "drive-id", FileID: "file-2", Title: "Clip 2", PreviewStatus: "pending"},
|
||||
} {
|
||||
v.PublishedAt = now
|
||||
v.CreatedAt = now
|
||||
v.UpdatedAt = now
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed video %s: %v", v.ID, err)
|
||||
}
|
||||
video := &catalog.Video{
|
||||
ID: "video-1",
|
||||
DriveID: "drive-id",
|
||||
FileID: "file-1",
|
||||
Title: "Clip 1",
|
||||
PreviewStatus: "pending",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if err := cat.UpsertVideo(ctx, video); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
app := &App{
|
||||
@@ -110,47 +114,234 @@ func TestRegisterPreviewWorkersGenerateThumbnailsBeforePreviews(t *testing.T) {
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
}
|
||||
gen := &serverFakeTeaserGenerator{}
|
||||
gen := &serverBlockingThumbGenerator{
|
||||
started: make(chan string, 1),
|
||||
release: make(chan struct{}),
|
||||
}
|
||||
drv := &serverFakeDrive{}
|
||||
worker := preview.NewWorker(gen, cat, drv)
|
||||
thumbWorker := preview.NewThumbWorker(gen, cat, drv)
|
||||
go worker.Run(ctx)
|
||||
go thumbWorker.Run(ctx)
|
||||
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, func() {})
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, nil, func() {})
|
||||
|
||||
select {
|
||||
case got := <-gen.started:
|
||||
if got != video.ID {
|
||||
t.Fatalf("thumbnail started for %q, want %q", got, video.ID)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("thumbnail generation did not start")
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
first, err := cat.GetVideo(ctx, "video-1")
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video-1: %v", err)
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
second, err := cat.GetVideo(ctx, "video-2")
|
||||
if err != nil {
|
||||
t.Fatalf("get video-2: %v", err)
|
||||
}
|
||||
if first.ThumbnailURL != "" && second.ThumbnailURL != "" &&
|
||||
first.PreviewStatus == "ready" && second.PreviewStatus == "ready" {
|
||||
events := gen.Events()
|
||||
if len(events) != 4 {
|
||||
t.Fatalf("events = %#v, want 4 generation events", events)
|
||||
}
|
||||
for i, event := range events[:2] {
|
||||
if event[:6] != "thumb:" {
|
||||
t.Fatalf("event %d = %q, want thumbnail before previews; all events=%#v", i, event, events)
|
||||
}
|
||||
}
|
||||
for i, event := range events[2:] {
|
||||
if event[:8] != "preview:" {
|
||||
t.Fatalf("event %d = %q, want previews after thumbnails; all events=%#v", i+2, event, events)
|
||||
}
|
||||
if got.PreviewStatus == "ready" {
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail url = %q, want preview ready while thumbnail is still blocked", got.ThumbnailURL)
|
||||
}
|
||||
close(gen.release)
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
t.Fatalf("generation did not finish, events=%#v", gen.Events())
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video after timeout: %v", err)
|
||||
}
|
||||
t.Fatalf("preview status=%q thumbnail=%q, want preview ready before thumbnail finishes", got.PreviewStatus, got.ThumbnailURL)
|
||||
}
|
||||
|
||||
func TestRegisterPreviewWorkersBackfillsHistoricalFingerprints(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
dataPath := filepath.Join(t.TempDir(), "video.mp4")
|
||||
data := []byte("historical video content for fingerprint")
|
||||
if err := os.WriteFile(dataPath, data, 0o644); err != nil {
|
||||
t.Fatalf("write video data: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
video := &catalog.Video{
|
||||
ID: "historical-video",
|
||||
DriveID: "drive-id",
|
||||
FileID: "file-id",
|
||||
Title: "Historical",
|
||||
Size: int64(len(data)),
|
||||
FingerprintStatus: "pending",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if err := cat.UpsertVideo(ctx, video); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
app := &App{
|
||||
cat: cat,
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
fingerprintWorkers: make(map[string]*fingerprint.Worker),
|
||||
}
|
||||
drv := &serverFingerprintFakeDrive{path: dataPath}
|
||||
fingerprintWorker := fingerprint.NewWorker(cat, drv, fingerprint.Config{})
|
||||
go fingerprintWorker.Run(ctx)
|
||||
|
||||
app.registerPreviewWorkers(ctx, "drive-id", nil, nil, fingerprintWorker, func() {})
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.SampledSHA256 != "" && got.FingerprintStatus == "ready" {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video after timeout: %v", err)
|
||||
}
|
||||
t.Fatalf("fingerprint status=%q sampled=%q, want ready with hash", got.FingerprintStatus, got.SampledSHA256)
|
||||
}
|
||||
|
||||
func TestRunScanStartsFingerprintBeforeThumbnailAndPreviewDrain(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
seedDriveWithTeaser(t, cat, "drive-id", true)
|
||||
|
||||
dataPath := filepath.Join(t.TempDir(), "scan-video.mp4")
|
||||
data := []byte("scan video content for independent fingerprint")
|
||||
if err := os.WriteFile(dataPath, data, 0o644); err != nil {
|
||||
t.Fatalf("write video data: %v", err)
|
||||
}
|
||||
|
||||
drv := &serverScanFingerprintFakeDrive{
|
||||
serverFingerprintFakeDrive: serverFingerprintFakeDrive{path: dataPath},
|
||||
entries: []drives.Entry{{
|
||||
ID: "file-id",
|
||||
Name: "scan-video.mp4",
|
||||
Size: int64(len(data)),
|
||||
ParentID: "root",
|
||||
}},
|
||||
}
|
||||
registry := proxy.NewRegistry()
|
||||
registry.Set("drive-id", drv)
|
||||
|
||||
gen := &serverFakeTeaserGenerator{}
|
||||
worker := preview.NewWorker(gen, cat, drv)
|
||||
thumbWorker := preview.NewThumbWorker(gen, cat, drv)
|
||||
fingerprintWorker := fingerprint.NewWorker(cat, drv, fingerprint.Config{})
|
||||
go fingerprintWorker.Run(ctx)
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{
|
||||
Scanner: config.Scanner{VideoExtensions: []string{".mp4"}},
|
||||
},
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
workers: map[string]*preview.Worker{"drive-id": worker},
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{"drive-id": thumbWorker},
|
||||
fingerprintWorkers: map[string]*fingerprint.Worker{"drive-id": fingerprintWorker},
|
||||
}
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
app.runScan(ctx, "drive-id")
|
||||
}()
|
||||
|
||||
videoID := "fake-drive-id-file-id"
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
got, err := cat.GetVideo(ctx, videoID)
|
||||
if err == nil && got.SampledSHA256 != "" && got.FingerprintStatus == "ready" {
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("scan did not stop after context cancel")
|
||||
}
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail url = %q, want fingerprint before thumbnail generation", got.ThumbnailURL)
|
||||
}
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("scan did not stop after context cancel")
|
||||
}
|
||||
got, err := cat.GetVideo(context.Background(), videoID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video after timeout: %v", err)
|
||||
}
|
||||
t.Fatalf("fingerprint status=%q sampled=%q, want ready before thumbnail/preview drain", got.FingerprintStatus, got.SampledSHA256)
|
||||
}
|
||||
|
||||
func TestNightlyTargetsComeFromCatalogBeforeDriveAttach(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, d := range []*catalog.Drive{
|
||||
{ID: "115", Kind: "p115", Name: "115", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "pikpak", Kind: "pikpak", Name: "PikPak", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "91-spider", Kind: "spider91", Name: "91 Spider", RootID: "0", TeaserEnabled: true},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
t.Fatalf("seed drive %s: %v", d.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
app := &App{cat: cat}
|
||||
scanIDs := app.listScanTargetIDs(ctx)
|
||||
if len(scanIDs) != 2 || scanIDs[0] != "115" || scanIDs[1] != "pikpak" {
|
||||
t.Fatalf("scan target ids = %#v, want 115 and pikpak from catalog", scanIDs)
|
||||
}
|
||||
spiderIDs := app.listSpider91DriveIDs(ctx)
|
||||
if len(spiderIDs) != 1 || spiderIDs[0] != "91-spider" {
|
||||
t.Fatalf("spider91 ids = %#v, want catalog spider drive", spiderIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedThumbnailsDoNotBlockPreviewGeneration(t *testing.T) {
|
||||
@@ -205,7 +396,7 @@ func TestFailedThumbnailsDoNotBlockPreviewGeneration(t *testing.T) {
|
||||
go worker.Run(ctx)
|
||||
go thumbWorker.Run(ctx)
|
||||
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, func() {})
|
||||
app.registerPreviewWorkers(ctx, "drive-id", worker, thumbWorker, nil, func() {})
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
@@ -480,6 +671,106 @@ func TestCleanupMissingPikPakVideosRemovesDatabaseRowsAndLocalAssets(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupDuplicateVideoAssetsRemovesOnlyDuplicateLocalAssets(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
localDir := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
canonicalPreview := filepath.Join(localDir, "canonical.mp4")
|
||||
duplicatePreview := filepath.Join(localDir, "duplicate.mp4")
|
||||
canonicalThumb := filepath.Join(localDir, "thumbs", "canonical-video.jpg")
|
||||
duplicateThumb := filepath.Join(localDir, "thumbs", "duplicate-video.jpg")
|
||||
for _, path := range []string{canonicalPreview, duplicatePreview, canonicalThumb, duplicateThumb} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("asset"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
|
||||
for _, v := range []*catalog.Video{
|
||||
{
|
||||
ID: "canonical-video",
|
||||
DriveID: "115",
|
||||
FileID: "file-a",
|
||||
Title: "Canonical",
|
||||
Size: 2048,
|
||||
ThumbnailURL: "/p/thumb/canonical-video",
|
||||
PreviewLocal: canonicalPreview,
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "duplicate-video",
|
||||
DriveID: "onedrive",
|
||||
FileID: "file-b",
|
||||
Title: "Duplicate",
|
||||
Size: 2048,
|
||||
ThumbnailURL: "/p/thumb/duplicate-video",
|
||||
PreviewLocal: duplicatePreview,
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, v.ID, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
}
|
||||
if err := app.cleanupDuplicateVideoAssets(ctx); err != nil {
|
||||
t.Fatalf("cleanup duplicate video assets: %v", err)
|
||||
}
|
||||
|
||||
for _, path := range []string{canonicalPreview, canonicalThumb} {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("canonical asset %s missing: %v", path, err)
|
||||
}
|
||||
}
|
||||
for _, path := range []string{duplicatePreview, duplicateThumb} {
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("duplicate asset %s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
dup, err := cat.GetVideo(ctx, "duplicate-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get duplicate: %v", err)
|
||||
}
|
||||
if dup.PreviewLocal != "" || dup.PreviewStatus != "pending" {
|
||||
t.Fatalf("duplicate preview local=%q status=%q, want empty pending", dup.PreviewLocal, dup.PreviewStatus)
|
||||
}
|
||||
if dup.ThumbnailURL != "" {
|
||||
t.Fatalf("duplicate thumbnail url = %q, want empty", dup.ThumbnailURL)
|
||||
}
|
||||
canon, err := cat.GetVideo(ctx, "canonical-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get canonical: %v", err)
|
||||
}
|
||||
if canon.PreviewLocal != canonicalPreview || canon.ThumbnailURL != "/p/thumb/canonical-video" {
|
||||
t.Fatalf("canonical changed: preview=%q thumb=%q", canon.PreviewLocal, canon.ThumbnailURL)
|
||||
}
|
||||
}
|
||||
|
||||
type serverFakeTeaserGenerator struct {
|
||||
mu sync.Mutex
|
||||
events []string
|
||||
@@ -520,6 +811,28 @@ func (g *serverFakeTeaserGenerator) GenerateThumbnail(_ context.Context, _ *driv
|
||||
return "/tmp/" + videoID + ".jpg", nil
|
||||
}
|
||||
|
||||
type serverBlockingThumbGenerator struct {
|
||||
serverFakeTeaserGenerator
|
||||
started chan string
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (g *serverBlockingThumbGenerator) GenerateThumbnail(ctx context.Context, _ *drives.StreamLink, videoID string, _ float64) (string, error) {
|
||||
g.record("thumb:" + videoID)
|
||||
if g.started != nil {
|
||||
select {
|
||||
case g.started <- videoID:
|
||||
default:
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-g.release:
|
||||
return "/tmp/" + videoID + ".jpg", nil
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
type serverFakeDrive struct{}
|
||||
|
||||
func (d *serverFakeDrive) Kind() string { return "fake" }
|
||||
@@ -544,6 +857,24 @@ func (d *serverFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
}
|
||||
func (d *serverFakeDrive) RootID() string { return "root" }
|
||||
|
||||
type serverFingerprintFakeDrive struct {
|
||||
serverFakeDrive
|
||||
path string
|
||||
}
|
||||
|
||||
func (d *serverFingerprintFakeDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
|
||||
return &drives.StreamLink{URL: d.path}, nil
|
||||
}
|
||||
|
||||
type serverScanFingerprintFakeDrive struct {
|
||||
serverFingerprintFakeDrive
|
||||
entries []drives.Entry
|
||||
}
|
||||
|
||||
func (d *serverScanFingerprintFakeDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return d.entries, nil
|
||||
}
|
||||
|
||||
type serverLocalUploadFakeDrive struct {
|
||||
serverFakeDrive
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ preview:
|
||||
width: 480
|
||||
|
||||
# 盘列表。上线后请通过管理后台添加,本文件可留空。
|
||||
# kind 支持 quark / p115 / pikpak / wopan / onedrive。
|
||||
# kind 支持 quark / p115 / pikpak / wopan / onedrive / googledrive / localstorage。
|
||||
# OneDrive 示例:
|
||||
# - id: "my-onedrive"
|
||||
# kind: "onedrive"
|
||||
@@ -67,6 +67,18 @@ preview:
|
||||
# root_id: "root"
|
||||
# params:
|
||||
# refresh_token: "..."
|
||||
# api_url_address: "https://api.oplist.org/onedrive/renewapi"
|
||||
# region: "global"
|
||||
# Google Drive 示例:
|
||||
# - id: "my-google"
|
||||
# kind: "googledrive"
|
||||
# name: "我的 Google Drive"
|
||||
# root_id: "root"
|
||||
# params:
|
||||
# refresh_token: "..."
|
||||
# 本地存储示例:
|
||||
# - id: "local-media"
|
||||
# kind: "localstorage"
|
||||
# name: "本地视频目录"
|
||||
# root_id: "/"
|
||||
# params:
|
||||
# path: "/mnt/videos"
|
||||
drives: []
|
||||
|
||||
+3
-3
@@ -7,15 +7,17 @@ toolchain go1.23.4
|
||||
require (
|
||||
github.com/OpenListTeam/wopan-sdk-go v0.2.0
|
||||
github.com/SheltonZhu/115driver v1.3.2
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
|
||||
github.com/go-chi/chi/v5 v5.1.0
|
||||
github.com/go-resty/resty/v2 v2.14.0
|
||||
golang.org/x/net v0.27.0
|
||||
golang.org/x/sys v0.30.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
modernc.org/sqlite v1.33.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aead/ecdh v0.2.0 // indirect
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect
|
||||
github.com/andreburgaud/crypt2go v1.1.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
@@ -28,8 +30,6 @@ require (
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
|
||||
golang.org/x/crypto v0.25.0 // indirect
|
||||
golang.org/x/net v0.27.0 // indirect
|
||||
golang.org/x/sys v0.30.0 // indirect
|
||||
golang.org/x/time v0.8.0 // indirect
|
||||
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
|
||||
modernc.org/libc v1.55.3 // indirect
|
||||
|
||||
+300
-36
@@ -5,9 +5,13 @@ import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
@@ -18,6 +22,17 @@ import (
|
||||
type AdminServer struct {
|
||||
Catalog *catalog.Catalog
|
||||
Auth *auth.Authenticator
|
||||
// VersionFilePath points to the installer-written .version file.
|
||||
VersionFilePath string
|
||||
// ImageVersion is the Docker image version injected at build/runtime.
|
||||
// It takes precedence over VersionFilePath because Docker data volumes can
|
||||
// keep an older .version file across image upgrades.
|
||||
ImageVersion string
|
||||
// GitHubRepo is the owner/name repo used for update checks.
|
||||
GitHubRepo string
|
||||
// ReleaseAPIURL and HTTPClient are injectable for tests. Production code leaves them empty.
|
||||
ReleaseAPIURL string
|
||||
HTTPClient *http.Client
|
||||
// SetupRequired 表示当前是否仍处于首次部署初始化状态。
|
||||
SetupRequired func() bool
|
||||
// OnSetup 持久化首次部署时设置的管理员账号密码,并更新运行中认证器。
|
||||
@@ -32,6 +47,7 @@ type AdminServer struct {
|
||||
OnRegenAllPreviews func()
|
||||
OnRegenFailedPreviews func(driveID string)
|
||||
OnRegenFailedThumbnails func(driveID string)
|
||||
OnRegenFailedFingerprints func(driveID string)
|
||||
GetDriveGenerationStatuses func() map[string]DriveGenerationStatuses
|
||||
// OnTeaserEnabledChanged 在 per-drive teaser 开关被切换后调用。
|
||||
// enabled=true 时上层应该重新把 pending teaser 入队(类似旧的全局开关从关到开);
|
||||
@@ -40,13 +56,15 @@ type AdminServer struct {
|
||||
// Theme 读写("dark" | "pink")
|
||||
GetTheme func() string
|
||||
SetTheme func(theme string) error
|
||||
// Spider91 → PikPak 上传目标 drive ID 读写
|
||||
// Spider91 → 115/PikPak 上传目标 drive ID 读写
|
||||
GetSpider91UploadDriveID func() string
|
||||
SetSpider91UploadDriveID func(driveID string) error
|
||||
// OnRunNightlyJob 触发一次完整的凌晨流水线(Phase1 扫盘 + Phase2 91 爬虫 +
|
||||
// Phase3 迁移)。立即返回 —— 实际任务在后台跑,admin 在日志或下次状态查询里
|
||||
// 看进度。重复点击会被 Runner.TryLock 丢弃。
|
||||
OnRunNightlyJob func()
|
||||
// 看进度。若流水线正在跑或已排队,Runner 会拒绝重复触发。
|
||||
OnRunNightlyJob func() bool
|
||||
// GetNightlyJobStatus 返回凌晨流水线当前状态,用于前端禁用重复触发按钮。
|
||||
GetNightlyJobStatus func() NightlyJobStatus
|
||||
// ListDriveDirChildren 列出某个 drive 在 parentID 目录下的直接子目录。
|
||||
// parentID 为空时使用 drive 的 RootID。返回 (子目录列表, error)。
|
||||
// 用于"设置跳过目录"弹窗按需展开浏览网盘目录树;只返回目录条目,文件忽略。
|
||||
@@ -68,8 +86,17 @@ type GenerationStatus struct {
|
||||
}
|
||||
|
||||
type DriveGenerationStatuses struct {
|
||||
Thumbnail GenerationStatus `json:"thumbnail"`
|
||||
Preview GenerationStatus `json:"preview"`
|
||||
Thumbnail GenerationStatus `json:"thumbnail"`
|
||||
Preview GenerationStatus `json:"preview"`
|
||||
Fingerprint GenerationStatus `json:"fingerprint"`
|
||||
}
|
||||
|
||||
type NightlyJobStatus struct {
|
||||
State string `json:"state"`
|
||||
Running bool `json:"running"`
|
||||
Queued bool `json:"queued"`
|
||||
StartedAt string `json:"startedAt,omitempty"`
|
||||
LastFinishedAt string `json:"lastFinishedAt,omitempty"`
|
||||
}
|
||||
|
||||
func (a *AdminServer) Register(r chi.Router) {
|
||||
@@ -96,6 +123,7 @@ func (a *AdminServer) Register(r chi.Router) {
|
||||
r.Get("/drives/{id}/dirtree", a.handleListDriveDirTree)
|
||||
r.Post("/drives/{id}/previews/failed/regenerate", a.handleRegenFailedPreviews)
|
||||
r.Post("/drives/{id}/thumbnails/failed/regenerate", a.handleRegenFailedThumbnails)
|
||||
r.Post("/drives/{id}/fingerprints/failed/regenerate", a.handleRegenFailedFingerprints)
|
||||
|
||||
// 视频
|
||||
r.Get("/videos", a.handleAdminListVideos)
|
||||
@@ -106,17 +134,33 @@ func (a *AdminServer) Register(r chi.Router) {
|
||||
// 标签
|
||||
r.Get("/tags", a.handleListTags)
|
||||
r.Post("/tags", a.handleCreateTag)
|
||||
r.Delete("/tags/{id}", a.handleDeleteTag)
|
||||
|
||||
// 运行时设置
|
||||
r.Get("/settings", a.handleGetSettings)
|
||||
r.Put("/settings", a.handlePutSettings)
|
||||
|
||||
// 运维任务
|
||||
r.Get("/update/check", a.handleCheckUpdate)
|
||||
r.Get("/jobs/nightly/status", a.handleNightlyJobStatus)
|
||||
r.Post("/jobs/nightly/run", a.handleRunNightlyJob)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
type updateCheckDTO struct {
|
||||
CurrentVersion string `json:"currentVersion"`
|
||||
LatestVersion string `json:"latestVersion"`
|
||||
HasUpdate bool `json:"hasUpdate"`
|
||||
ReleaseURL string `json:"releaseUrl,omitempty"`
|
||||
CheckedAt string `json:"checkedAt"`
|
||||
}
|
||||
|
||||
type githubReleaseDTO struct {
|
||||
TagName string `json:"tag_name"`
|
||||
HTMLURL string `json:"html_url"`
|
||||
}
|
||||
|
||||
type loginReq struct {
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
@@ -221,6 +265,94 @@ func (a *AdminServer) handleMe(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"authenticated": ok})
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleCheckUpdate(w http.ResponseWriter, r *http.Request) {
|
||||
info, err := a.checkUpdate(r.Context())
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusBadGateway, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, http.StatusOK, info)
|
||||
}
|
||||
|
||||
func (a *AdminServer) checkUpdate(ctx context.Context) (updateCheckDTO, error) {
|
||||
current := a.installedVersion()
|
||||
if current == "" {
|
||||
current = "unknown"
|
||||
}
|
||||
release, err := a.latestRelease(ctx)
|
||||
if err != nil {
|
||||
return updateCheckDTO{
|
||||
CurrentVersion: current,
|
||||
CheckedAt: time.Now().Format(time.RFC3339),
|
||||
}, err
|
||||
}
|
||||
latest := strings.TrimSpace(release.TagName)
|
||||
return updateCheckDTO{
|
||||
CurrentVersion: current,
|
||||
LatestVersion: latest,
|
||||
HasUpdate: current != "unknown" && latest != "" && current != latest,
|
||||
ReleaseURL: release.HTMLURL,
|
||||
CheckedAt: time.Now().Format(time.RFC3339),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *AdminServer) installedVersion() string {
|
||||
if version := strings.TrimSpace(a.ImageVersion); version != "" {
|
||||
return version
|
||||
}
|
||||
path := strings.TrimSpace(a.VersionFilePath)
|
||||
if path == "" {
|
||||
path = ".version"
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
lines := strings.Split(strings.ReplaceAll(string(data), "\r\n", "\n"), "\n")
|
||||
if len(lines) == 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(lines[0])
|
||||
}
|
||||
|
||||
func (a *AdminServer) latestRelease(ctx context.Context) (githubReleaseDTO, error) {
|
||||
url := strings.TrimSpace(a.ReleaseAPIURL)
|
||||
if url == "" {
|
||||
repo := strings.TrimSpace(a.GitHubRepo)
|
||||
if repo == "" {
|
||||
repo = "nianzhibai/91"
|
||||
}
|
||||
url = "https://api.github.com/repos/" + repo + "/releases/latest"
|
||||
}
|
||||
client := a.HTTPClient
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 8 * time.Second}
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return githubReleaseDTO{}, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/vnd.github+json")
|
||||
req.Header.Set("User-Agent", "video-site-91")
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return githubReleaseDTO{}, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
return githubReleaseDTO{}, fmt.Errorf("github release check failed: HTTP %d", res.StatusCode)
|
||||
}
|
||||
var release githubReleaseDTO
|
||||
if err := json.NewDecoder(res.Body).Decode(&release); err != nil {
|
||||
return githubReleaseDTO{}, err
|
||||
}
|
||||
if strings.TrimSpace(release.TagName) == "" {
|
||||
return githubReleaseDTO{}, errors.New("github release check returned empty tag")
|
||||
}
|
||||
return release, nil
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
drives, err := a.Catalog.ListDrives(r.Context())
|
||||
if err != nil {
|
||||
@@ -237,6 +369,11 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
fingerprintCounts, err := a.Catalog.CountFingerprintsByDrive(r.Context())
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
generationStatuses := map[string]DriveGenerationStatuses{}
|
||||
if a.GetDriveGenerationStatuses != nil {
|
||||
generationStatuses = a.GetDriveGenerationStatuses()
|
||||
@@ -259,20 +396,27 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
SkipDirIDs []string `json:"skipDirIds"`
|
||||
// LastCrawlAt 是 spider91 上次成功爬取的 unix 秒(来自 credentials.last_crawl_at)。
|
||||
// 其它 kind 留 0;前端用它显示"上次抓取: N 小时前"。
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
Spider91Proxy string `json:"spider91Proxy,omitempty"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
ThumbnailDurationPendingCount int `json:"thumbnailDurationPendingCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
FingerprintReadyCount int `json:"fingerprintReadyCount"`
|
||||
FingerprintPendingCount int `json:"fingerprintPendingCount"`
|
||||
FingerprintFailedCount int `json:"fingerprintFailedCount"`
|
||||
}
|
||||
list := make([]out, 0, len(drives))
|
||||
for _, d := range drives {
|
||||
counts := teaserCounts[d.ID]
|
||||
thumbCounts := thumbnailCounts[d.ID]
|
||||
fingerprintCount := fingerprintCounts[d.ID]
|
||||
generation := generationStatuses[d.ID]
|
||||
if generation.Thumbnail.State == "" {
|
||||
generation.Thumbnail.State = "idle"
|
||||
@@ -280,6 +424,9 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
if generation.Preview.State == "" {
|
||||
generation.Preview.State = "idle"
|
||||
}
|
||||
if generation.Fingerprint.State == "" {
|
||||
generation.Fingerprint.State = "idle"
|
||||
}
|
||||
// spider91 没有用户凭证概念;只要存在 drive 行就视为"已配置"。
|
||||
// last_crawl_at 是后端自动写入的运行状态字段,不计入 hasCredential 判定。
|
||||
hasCred := false
|
||||
@@ -305,28 +452,35 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
ID: d.ID, Kind: d.Kind, Name: d.Name,
|
||||
RootID: d.RootID, ScanRootID: d.ScanRootID,
|
||||
Status: d.Status, LastError: d.LastError,
|
||||
HasCredential: hasCred,
|
||||
TeaserEnabled: d.TeaserEnabled,
|
||||
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
|
||||
LastCrawlAt: lastCrawlAt,
|
||||
ThumbnailGenerationStatus: generation.Thumbnail,
|
||||
PreviewGenerationStatus: generation.Preview,
|
||||
ThumbnailReadyCount: thumbCounts.Ready,
|
||||
ThumbnailPendingCount: thumbCounts.Pending,
|
||||
ThumbnailFailedCount: thumbCounts.Failed,
|
||||
TeaserReadyCount: counts.Ready,
|
||||
TeaserPendingCount: counts.Pending,
|
||||
TeaserFailedCount: counts.Failed,
|
||||
HasCredential: hasCred,
|
||||
TeaserEnabled: d.TeaserEnabled,
|
||||
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
|
||||
Spider91Proxy: spider91ProxyForDrive(d),
|
||||
LastCrawlAt: lastCrawlAt,
|
||||
ThumbnailGenerationStatus: generation.Thumbnail,
|
||||
PreviewGenerationStatus: generation.Preview,
|
||||
FingerprintGenerationStatus: generation.Fingerprint,
|
||||
ThumbnailReadyCount: thumbCounts.Ready,
|
||||
ThumbnailPendingCount: thumbCounts.Pending,
|
||||
ThumbnailFailedCount: thumbCounts.Failed,
|
||||
ThumbnailDurationPendingCount: thumbCounts.DurationPending,
|
||||
TeaserReadyCount: counts.Ready,
|
||||
TeaserPendingCount: counts.Pending,
|
||||
TeaserFailedCount: counts.Failed,
|
||||
FingerprintReadyCount: fingerprintCount.Ready,
|
||||
FingerprintPendingCount: fingerprintCount.Pending,
|
||||
FingerprintFailedCount: fingerprintCount.Failed,
|
||||
})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, list)
|
||||
}
|
||||
|
||||
type upsertDriveReq struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
RootID string `json:"rootId"`
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
RootID string `json:"rootId"`
|
||||
// Deprecated: 扫描起点已固定为 rootId;保留字段只为兼容旧客户端请求体。
|
||||
ScanRootID string `json:"scanRootId"`
|
||||
Credentials map[string]string `json:"credentials"`
|
||||
// TeaserEnabled 是 per-drive teaser/封面生成开关。
|
||||
@@ -354,7 +508,14 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
|
||||
if existingDrive, err := a.Catalog.GetDrive(r.Context(), body.ID); err == nil {
|
||||
existing = existingDrive
|
||||
}
|
||||
if len(body.Credentials) == 0 && existing != nil && len(existing.Credentials) > 0 {
|
||||
if body.Kind == "spider91" {
|
||||
credentials, err := mergeSpider91Credentials(existing, body.Credentials)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
body.Credentials = credentials
|
||||
} else if len(body.Credentials) == 0 && existing != nil && len(existing.Credentials) > 0 {
|
||||
body.Credentials = existing.Credentials
|
||||
}
|
||||
|
||||
@@ -384,7 +545,7 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
d := &catalog.Drive{
|
||||
ID: body.ID, Kind: body.Kind, Name: body.Name,
|
||||
RootID: body.RootID, ScanRootID: body.ScanRootID,
|
||||
RootID: body.RootID,
|
||||
Credentials: body.Credentials,
|
||||
Status: "disconnected",
|
||||
TeaserEnabled: teaserEnabled,
|
||||
@@ -403,6 +564,58 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
|
||||
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
func spider91ProxyForDrive(d *catalog.Drive) string {
|
||||
if d == nil || d.Kind != "spider91" || d.Credentials == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(d.Credentials["proxy"])
|
||||
}
|
||||
|
||||
func mergeSpider91Credentials(existing *catalog.Drive, incoming map[string]string) (map[string]string, error) {
|
||||
merged := map[string]string{}
|
||||
if existing != nil {
|
||||
for k, v := range existing.Credentials {
|
||||
merged[k] = v
|
||||
}
|
||||
}
|
||||
for k, v := range incoming {
|
||||
if strings.TrimSpace(k) == "" {
|
||||
continue
|
||||
}
|
||||
if k == "proxy" {
|
||||
proxy, err := normalizeSpider91ProxyURL(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if proxy == "" {
|
||||
delete(merged, "proxy")
|
||||
} else {
|
||||
merged["proxy"] = proxy
|
||||
}
|
||||
continue
|
||||
}
|
||||
merged[k] = v
|
||||
}
|
||||
return merged, nil
|
||||
}
|
||||
|
||||
func normalizeSpider91ProxyURL(raw string) (string, error) {
|
||||
proxy := strings.TrimSpace(raw)
|
||||
if proxy == "" {
|
||||
return "", nil
|
||||
}
|
||||
u, err := url.Parse(proxy)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return "", fmt.Errorf("91Spider 代理地址格式无效,请填写类似 http://127.0.0.1:7890 的地址")
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https", "socks5", "socks5h":
|
||||
return proxy, nil
|
||||
default:
|
||||
return "", fmt.Errorf("91Spider 代理地址仅支持 http://、https://、socks5:// 或 socks5h://")
|
||||
}
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleDeleteDrive(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
if err := a.Catalog.DeleteDrive(r.Context(), id); err != nil {
|
||||
@@ -425,12 +638,32 @@ func (a *AdminServer) handleRescan(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// handleRunNightlyJob 触发一次完整的凌晨流水线(不论当前时间,不论今日是否已跑)。
|
||||
// 立即返回 202;进度通过 backend 日志和下次 GET /admin/api/drives 的状态变化观察。
|
||||
// 流水线已在跑时 Runner 会丢弃此次触发并记日志。
|
||||
// 流水线已在跑或已排队时,Runner 会拒绝重复触发。
|
||||
func (a *AdminServer) handleRunNightlyJob(w http.ResponseWriter, r *http.Request) {
|
||||
accepted := false
|
||||
if a.OnRunNightlyJob != nil {
|
||||
a.OnRunNightlyJob()
|
||||
accepted = a.OnRunNightlyJob()
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"ok": true,
|
||||
"accepted": accepted,
|
||||
"status": a.nightlyJobStatus(),
|
||||
})
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleNightlyJobStatus(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.nightlyJobStatus())
|
||||
}
|
||||
|
||||
func (a *AdminServer) nightlyJobStatus() NightlyJobStatus {
|
||||
if a.GetNightlyJobStatus == nil {
|
||||
return NightlyJobStatus{State: "idle"}
|
||||
}
|
||||
status := a.GetNightlyJobStatus()
|
||||
if status.State == "" {
|
||||
status.State = "idle"
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
// teaserEnabledReq 是 POST /admin/api/drives/{id}/teaser-enabled 的入参。
|
||||
@@ -611,6 +844,27 @@ func (a *AdminServer) handleCreateTag(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleDeleteTag(w http.ResponseWriter, r *http.Request) {
|
||||
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
||||
if err != nil || id <= 0 {
|
||||
writeErr(w, http.StatusBadRequest, errors.New("invalid tag id"))
|
||||
return
|
||||
}
|
||||
removedVideos, err := a.Catalog.DeleteTag(r.Context(), id)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
writeErr(w, http.StatusNotFound, err)
|
||||
case errors.Is(err, catalog.ErrSystemTag):
|
||||
writeErr(w, http.StatusBadRequest, err)
|
||||
default:
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "removedVideos": removedVideos})
|
||||
}
|
||||
|
||||
type updateVideoReq struct {
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
@@ -717,6 +971,16 @@ func (a *AdminServer) handleRegenFailedThumbnails(w http.ResponseWriter, r *http
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
// handleRegenFailedFingerprints triggers regeneration for all failed sampled
|
||||
// fingerprints on a drive. It mirrors the failed teaser/thumbnail retry endpoints.
|
||||
func (a *AdminServer) handleRegenFailedFingerprints(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
if a.OnRegenFailedFingerprints != nil {
|
||||
a.OnRegenFailedFingerprints(id)
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
// ---------- Settings ----------
|
||||
|
||||
// settingsDTO 是 GET/PUT /admin/api/settings 的入参/出参。
|
||||
@@ -748,7 +1012,7 @@ func (a *AdminServer) handleGetSettings(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
func (a *AdminServer) handlePutSettings(w http.ResponseWriter, r *http.Request) {
|
||||
// 用 map 区分"没传"和"传了空字符串"两种语义;空 spider91 上传 ID 表示
|
||||
// 清除显式设置(回退到自动模式)。
|
||||
// 本地保存不上传。
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.NewDecoder(r.Body).Decode(&raw); err != nil {
|
||||
writeErr(w, http.StatusBadRequest, err)
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -115,6 +116,185 @@ func TestHandleSetupStoresCredentialsAndCreatesSession(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleCheckUpdateReportsNewRelease(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
versionFile := filepath.Join(dir, ".version")
|
||||
if err := os.WriteFile(versionFile, []byte("v0.1.0\n2026-05-29 12:00:00\n"), 0o644); err != nil {
|
||||
t.Fatalf("write version file: %v", err)
|
||||
}
|
||||
releaseServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("User-Agent") == "" {
|
||||
http.Error(w, "missing user agent", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"tag_name": "v0.2.0",
|
||||
"html_url": "https://github.com/nianzhibai/91/releases/tag/v0.2.0",
|
||||
})
|
||||
}))
|
||||
t.Cleanup(releaseServer.Close)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/update/check", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{
|
||||
VersionFilePath: versionFile,
|
||||
ReleaseAPIURL: releaseServer.URL,
|
||||
}).handleCheckUpdate(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got updateCheckDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.CurrentVersion != "v0.1.0" {
|
||||
t.Fatalf("currentVersion = %q, want v0.1.0", got.CurrentVersion)
|
||||
}
|
||||
if got.LatestVersion != "v0.2.0" {
|
||||
t.Fatalf("latestVersion = %q, want v0.2.0", got.LatestVersion)
|
||||
}
|
||||
if !got.HasUpdate {
|
||||
t.Fatalf("hasUpdate = false, want true")
|
||||
}
|
||||
if got.ReleaseURL == "" {
|
||||
t.Fatalf("releaseUrl is empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleCheckUpdateReportsUpToDate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
versionFile := filepath.Join(dir, ".version")
|
||||
if err := os.WriteFile(versionFile, []byte("v0.2.0\n"), 0o644); err != nil {
|
||||
t.Fatalf("write version file: %v", err)
|
||||
}
|
||||
releaseServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"tag_name": "v0.2.0",
|
||||
"html_url": "https://github.com/nianzhibai/91/releases/tag/v0.2.0",
|
||||
})
|
||||
}))
|
||||
t.Cleanup(releaseServer.Close)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/update/check", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{
|
||||
VersionFilePath: versionFile,
|
||||
ReleaseAPIURL: releaseServer.URL,
|
||||
}).handleCheckUpdate(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got updateCheckDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.HasUpdate {
|
||||
t.Fatalf("hasUpdate = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleCheckUpdateUsesDockerImageVersion(t *testing.T) {
|
||||
releaseServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"tag_name": "v0.2.0",
|
||||
"html_url": "https://github.com/nianzhibai/91/releases/tag/v0.2.0",
|
||||
})
|
||||
}))
|
||||
t.Cleanup(releaseServer.Close)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/update/check", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{
|
||||
ImageVersion: "v0.1.0",
|
||||
ReleaseAPIURL: releaseServer.URL,
|
||||
}).handleCheckUpdate(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got updateCheckDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.CurrentVersion != "v0.1.0" {
|
||||
t.Fatalf("currentVersion = %q, want v0.1.0", got.CurrentVersion)
|
||||
}
|
||||
if !got.HasUpdate {
|
||||
t.Fatalf("hasUpdate = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstalledVersionPrefersDockerImageVersionOverVersionFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
versionFile := filepath.Join(dir, ".version")
|
||||
if err := os.WriteFile(versionFile, []byte("v0.1.0\n"), 0o644); err != nil {
|
||||
t.Fatalf("write version file: %v", err)
|
||||
}
|
||||
|
||||
got := (&AdminServer{
|
||||
VersionFilePath: versionFile,
|
||||
ImageVersion: "v0.2.0",
|
||||
}).installedVersion()
|
||||
|
||||
if got != "v0.2.0" {
|
||||
t.Fatalf("installedVersion = %q, want v0.2.0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRunNightlyJobReturnsAcceptedStatus(t *testing.T) {
|
||||
called := false
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/jobs/nightly/run", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
OnRunNightlyJob: func() bool {
|
||||
called = true
|
||||
return true
|
||||
},
|
||||
GetNightlyJobStatus: func() NightlyJobStatus {
|
||||
return NightlyJobStatus{State: "queued", Queued: true}
|
||||
},
|
||||
}).handleRunNightlyJob(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, want 202; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !called {
|
||||
t.Fatal("OnRunNightlyJob was not called")
|
||||
}
|
||||
var got struct {
|
||||
OK bool `json:"ok"`
|
||||
Accepted bool `json:"accepted"`
|
||||
Status NightlyJobStatus `json:"status"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if !got.OK || !got.Accepted || got.Status.State != "queued" || !got.Status.Queued {
|
||||
t.Fatalf("response = %#v, want accepted queued status", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleNightlyJobStatusDefaultsToIdle(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/jobs/nightly/status", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{}).handleNightlyJobStatus(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got NightlyJobStatus
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.State != "idle" || got.Running || got.Queued {
|
||||
t.Fatalf("status = %#v, want idle", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertDrivePreservesExistingCredentialsWhenRequestCredentialsEmpty(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -163,14 +343,52 @@ func TestHandleUpsertDrivePreservesExistingCredentialsWhenRequestCredentialsEmpt
|
||||
if got.Name != "New name" {
|
||||
t.Fatalf("name = %q, want New name", got.Name)
|
||||
}
|
||||
if got.ScanRootID != "scan-root" {
|
||||
t.Fatalf("scanRootId = %q, want scan-root", got.ScanRootID)
|
||||
if got.ScanRootID != "0" {
|
||||
t.Fatalf("scanRootId = %q, want rootId 0", got.ScanRootID)
|
||||
}
|
||||
if got.Credentials["cookie"] != "existing-cookie" {
|
||||
t.Fatalf("cookie credential = %q, want existing-cookie", got.Credentials["cookie"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertDriveDefaultsEmptyRootID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "onedrive-main",
|
||||
"kind": "onedrive",
|
||||
"name": "OneDrive",
|
||||
"rootId": "",
|
||||
"credentials": {"refresh_token": "token"}
|
||||
}`))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, "onedrive-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.RootID != "root" {
|
||||
t.Fatalf("rootId = %q, want root", got.RootID)
|
||||
}
|
||||
if got.ScanRootID != got.RootID {
|
||||
t.Fatalf("scanRootId = %q, want rootId %q", got.ScanRootID, got.RootID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertDriveReplacesExistingCredentialsWhenProvided(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -221,6 +439,190 @@ func TestHandleUpsertDriveReplacesExistingCredentialsWhenProvided(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertSpider91ProxyPreservesRuntimeCredentials(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "spider91-main",
|
||||
Kind: "spider91",
|
||||
Name: "91 Spider",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"last_crawl_at": "1800000000",
|
||||
"proxy": "http://old-proxy.local:7890",
|
||||
"script_path": "/opt/video-site-91/91VideoSpider/spider_91porn.py",
|
||||
},
|
||||
Status: "ok",
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"kind": "spider91",
|
||||
"name": "91 Spider",
|
||||
"rootId": "/",
|
||||
"credentials": {"proxy": " socks5h://proxy-user:proxy-pass@127.0.0.1:7891 "}
|
||||
}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
got, err := cat.GetDrive(ctx, "spider91-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.Credentials["proxy"] != "socks5h://proxy-user:proxy-pass@127.0.0.1:7891" {
|
||||
t.Fatalf("proxy = %q, want trimmed new proxy", got.Credentials["proxy"])
|
||||
}
|
||||
if got.Credentials["last_crawl_at"] != "1800000000" {
|
||||
t.Fatalf("last_crawl_at = %q, want preserved", got.Credentials["last_crawl_at"])
|
||||
}
|
||||
if got.Credentials["script_path"] == "" {
|
||||
t.Fatalf("script_path should be preserved")
|
||||
}
|
||||
|
||||
req = httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"kind": "spider91",
|
||||
"name": "91 Spider",
|
||||
"rootId": "/",
|
||||
"credentials": {"proxy": " "}
|
||||
}`))
|
||||
rr = httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("clear status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
got, err = cat.GetDrive(ctx, "spider91-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get cleared drive: %v", err)
|
||||
}
|
||||
if _, ok := got.Credentials["proxy"]; ok {
|
||||
t.Fatalf("proxy should be removed after empty save, got %q", got.Credentials["proxy"])
|
||||
}
|
||||
if got.Credentials["last_crawl_at"] != "1800000000" {
|
||||
t.Fatalf("last_crawl_at after clear = %q, want preserved", got.Credentials["last_crawl_at"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertSpider91RejectsUnsupportedProxyScheme(t *testing.T) {
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"kind": "spider91",
|
||||
"name": "91 Spider",
|
||||
"rootId": "/",
|
||||
"credentials": {"proxy": "ftp://127.0.0.1:21"}
|
||||
}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), "socks5:// 或 socks5h://") {
|
||||
t.Fatalf("body = %q, want supported schemes message", rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListDrivesIncludesSpider91Proxy(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, d := range []*catalog.Drive{
|
||||
{
|
||||
ID: "spider91-main",
|
||||
Kind: "spider91",
|
||||
Name: "91 Spider",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"last_crawl_at": "1800000000",
|
||||
"proxy": " http://127.0.0.1:7890 ",
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
{
|
||||
ID: "onedrive-main",
|
||||
Kind: "onedrive",
|
||||
Name: "OneDrive",
|
||||
RootID: "root",
|
||||
Credentials: map[string]string{
|
||||
"proxy": "http://should-not-leak.local:7890",
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
t.Fatalf("seed drive %s: %v", d.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/drives", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleListDrives(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
var got []struct {
|
||||
ID string `json:"id"`
|
||||
Spider91Proxy string `json:"spider91Proxy"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
byID := map[string]struct {
|
||||
Spider91Proxy string
|
||||
LastCrawlAt int64
|
||||
}{}
|
||||
for _, d := range got {
|
||||
byID[d.ID] = struct {
|
||||
Spider91Proxy string
|
||||
LastCrawlAt int64
|
||||
}{Spider91Proxy: d.Spider91Proxy, LastCrawlAt: d.LastCrawlAt}
|
||||
}
|
||||
if byID["spider91-main"].Spider91Proxy != "http://127.0.0.1:7890" {
|
||||
t.Fatalf("spider91 proxy = %q, want trimmed proxy", byID["spider91-main"].Spider91Proxy)
|
||||
}
|
||||
if byID["spider91-main"].LastCrawlAt != 1800000000 {
|
||||
t.Fatalf("lastCrawlAt = %d, want 1800000000", byID["spider91-main"].LastCrawlAt)
|
||||
}
|
||||
if byID["onedrive-main"].Spider91Proxy != "" {
|
||||
t.Fatalf("onedrive spider91Proxy = %q, want empty", byID["onedrive-main"].Spider91Proxy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -244,11 +646,11 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
|
||||
now := time.Now()
|
||||
videos := []*catalog.Video{
|
||||
{ID: "od-ready-1", DriveID: "OneDrive", FileID: "od-file-1", Title: "OD Ready 1", ThumbnailURL: "/p/thumb/od-ready-1", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-ready-2", DriveID: "OneDrive", FileID: "od-file-2", Title: "OD Ready 2", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-pending", DriveID: "OneDrive", FileID: "od-file-3", Title: "OD Pending", PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "pp-pending", DriveID: "PikPak", FileID: "pp-file-1", Title: "PP Pending", PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "pp-failed", DriveID: "PikPak", FileID: "pp-file-2", Title: "PP Failed", ThumbnailURL: "/p/thumb/pp-failed", PreviewStatus: "failed", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-ready-1", DriveID: "OneDrive", FileID: "od-file-1", Title: "OD Ready 1", Size: 100, ThumbnailURL: "/p/thumb/od-ready-1", PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-ready-2", DriveID: "OneDrive", FileID: "od-file-2", Title: "OD Ready 2", Size: 100, PreviewStatus: "ready", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "od-pending", DriveID: "OneDrive", FileID: "od-file-3", Title: "OD Pending", Size: 100, PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "pp-pending", DriveID: "PikPak", FileID: "pp-file-1", Title: "PP Pending", Size: 100, PreviewStatus: "pending", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "pp-failed", DriveID: "PikPak", FileID: "pp-file-2", Title: "PP Failed", Size: 100, ThumbnailURL: "/p/thumb/pp-failed", PreviewStatus: "failed", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
}
|
||||
for _, v := range videos {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
@@ -258,6 +660,12 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
if err := cat.UpdateVideoMeta(ctx, "od-ready-2", catalog.VideoMetaPatch{ThumbnailStatus: "failed"}); err != nil {
|
||||
t.Fatalf("mark thumbnail failed: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "od-ready-1", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "ready", ""); err != nil {
|
||||
t.Fatalf("mark fingerprint ready: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "od-ready-2", "", "failed", "sample failed"); err != nil {
|
||||
t.Fatalf("mark fingerprint failed: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/drives", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
@@ -266,8 +674,9 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
GetDriveGenerationStatuses: func() map[string]DriveGenerationStatuses {
|
||||
return map[string]DriveGenerationStatuses{
|
||||
"OneDrive": {
|
||||
Thumbnail: GenerationStatus{State: "cooling", QueueLength: 3, CooldownUntil: "2026-05-16T21:00:00+08:00"},
|
||||
Preview: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
|
||||
Thumbnail: GenerationStatus{State: "cooling", QueueLength: 3, CooldownUntil: "2026-05-16T21:00:00+08:00"},
|
||||
Preview: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
|
||||
Fingerprint: GenerationStatus{State: "generating", CurrentTitle: "OD Pending"},
|
||||
},
|
||||
}
|
||||
},
|
||||
@@ -277,48 +686,68 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got []struct {
|
||||
ID string `json:"id"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
ID string `json:"id"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
FingerprintGenerationStatus GenerationStatus `json:"fingerprintGenerationStatus"`
|
||||
ThumbnailReadyCount int `json:"thumbnailReadyCount"`
|
||||
ThumbnailPendingCount int `json:"thumbnailPendingCount"`
|
||||
ThumbnailFailedCount int `json:"thumbnailFailedCount"`
|
||||
ThumbnailDurationPendingCount int `json:"thumbnailDurationPendingCount"`
|
||||
TeaserReadyCount int `json:"teaserReadyCount"`
|
||||
TeaserPendingCount int `json:"teaserPendingCount"`
|
||||
TeaserFailedCount int `json:"teaserFailedCount"`
|
||||
FingerprintReadyCount int `json:"fingerprintReadyCount"`
|
||||
FingerprintPendingCount int `json:"fingerprintPendingCount"`
|
||||
FingerprintFailedCount int `json:"fingerprintFailedCount"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
byID := map[string]struct {
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
ThumbnailDurationPending int
|
||||
FingerprintReady int
|
||||
FingerprintPending int
|
||||
FingerprintFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
Fingerprint GenerationStatus
|
||||
}{}
|
||||
for _, d := range got {
|
||||
byID[d.ID] = struct {
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
TeaserReady int
|
||||
TeaserPending int
|
||||
TeaserFailed int
|
||||
ThumbnailReady int
|
||||
ThumbnailPending int
|
||||
ThumbnailFailed int
|
||||
ThumbnailDurationPending int
|
||||
FingerprintReady int
|
||||
FingerprintPending int
|
||||
FingerprintFailed int
|
||||
Thumbnail GenerationStatus
|
||||
Preview GenerationStatus
|
||||
Fingerprint GenerationStatus
|
||||
}{
|
||||
TeaserReady: d.TeaserReadyCount,
|
||||
TeaserPending: d.TeaserPendingCount,
|
||||
TeaserFailed: d.TeaserFailedCount,
|
||||
ThumbnailReady: d.ThumbnailReadyCount,
|
||||
ThumbnailPending: d.ThumbnailPendingCount,
|
||||
ThumbnailFailed: d.ThumbnailFailedCount,
|
||||
Thumbnail: d.ThumbnailGenerationStatus,
|
||||
Preview: d.PreviewGenerationStatus,
|
||||
TeaserReady: d.TeaserReadyCount,
|
||||
TeaserPending: d.TeaserPendingCount,
|
||||
TeaserFailed: d.TeaserFailedCount,
|
||||
ThumbnailReady: d.ThumbnailReadyCount,
|
||||
ThumbnailPending: d.ThumbnailPendingCount,
|
||||
ThumbnailFailed: d.ThumbnailFailedCount,
|
||||
ThumbnailDurationPending: d.ThumbnailDurationPendingCount,
|
||||
FingerprintReady: d.FingerprintReadyCount,
|
||||
FingerprintPending: d.FingerprintPendingCount,
|
||||
FingerprintFailed: d.FingerprintFailedCount,
|
||||
Thumbnail: d.ThumbnailGenerationStatus,
|
||||
Preview: d.PreviewGenerationStatus,
|
||||
Fingerprint: d.FingerprintGenerationStatus,
|
||||
}
|
||||
}
|
||||
if byID["OneDrive"].TeaserReady != 2 || byID["OneDrive"].TeaserPending != 1 || byID["OneDrive"].TeaserFailed != 0 {
|
||||
@@ -327,20 +756,54 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
if byID["OneDrive"].ThumbnailReady != 1 || byID["OneDrive"].ThumbnailPending != 1 || byID["OneDrive"].ThumbnailFailed != 1 {
|
||||
t.Fatalf("OneDrive thumbnail counts = %#v, want ready=1 pending=1 failed=1", byID["OneDrive"])
|
||||
}
|
||||
if byID["OneDrive"].ThumbnailDurationPending != 1 {
|
||||
t.Fatalf("OneDrive thumbnail duration pending = %#v, want 1", byID["OneDrive"])
|
||||
}
|
||||
if byID["OneDrive"].Thumbnail.State != "cooling" || byID["OneDrive"].Preview.State != "generating" {
|
||||
t.Fatalf("OneDrive generation statuses = %#v, want thumbnail cooling and preview generating", byID["OneDrive"])
|
||||
}
|
||||
if byID["OneDrive"].FingerprintReady != 1 || byID["OneDrive"].FingerprintPending != 1 || byID["OneDrive"].FingerprintFailed != 1 {
|
||||
t.Fatalf("OneDrive fingerprint counts = %#v, want ready=1 pending=1 failed=1", byID["OneDrive"])
|
||||
}
|
||||
if byID["OneDrive"].Fingerprint.State != "generating" {
|
||||
t.Fatalf("OneDrive fingerprint status = %#v, want generating", byID["OneDrive"].Fingerprint)
|
||||
}
|
||||
if byID["PikPak"].TeaserReady != 0 || byID["PikPak"].TeaserPending != 1 || byID["PikPak"].TeaserFailed != 1 {
|
||||
t.Fatalf("PikPak counts = %#v, want ready=0 pending=1 failed=1", byID["PikPak"])
|
||||
}
|
||||
if byID["PikPak"].ThumbnailReady != 1 || byID["PikPak"].ThumbnailPending != 1 || byID["PikPak"].ThumbnailFailed != 0 {
|
||||
t.Fatalf("PikPak thumbnail counts = %#v, want ready=1 pending=1 failed=0", byID["PikPak"])
|
||||
}
|
||||
if byID["PikPak"].Thumbnail.State != "idle" || byID["PikPak"].Preview.State != "idle" {
|
||||
if byID["PikPak"].FingerprintPending != 2 {
|
||||
t.Fatalf("PikPak fingerprint counts = %#v, want pending=2", byID["PikPak"])
|
||||
}
|
||||
if byID["PikPak"].Thumbnail.State != "idle" || byID["PikPak"].Preview.State != "idle" || byID["PikPak"].Fingerprint.State != "idle" {
|
||||
t.Fatalf("PikPak generation statuses = %#v, want idle defaults", byID["PikPak"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRegenFailedFingerprintsInvokesHook(t *testing.T) {
|
||||
called := ""
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives/drive-one/fingerprints/failed/regenerate", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "drive-one")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
OnRegenFailedFingerprints: func(driveID string) {
|
||||
called = driveID
|
||||
},
|
||||
}).handleRegenFailedFingerprints(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if called != "drive-one" {
|
||||
t.Fatalf("called drive = %q, want drive-one", called)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleDriveStorageReportsLocalMediaUsage(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
@@ -504,6 +967,64 @@ func TestHandleCreateTagClassifiesExistingVideos(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleDeleteTagRemovesTagFromVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "video-1",
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
Title: "清纯短发",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
if _, err := cat.CreateTagAndClassify(ctx, "清纯", nil, "user"); err != nil {
|
||||
t.Fatalf("create tag: %v", err)
|
||||
}
|
||||
tags, err := cat.ListTags(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list tags: %v", err)
|
||||
}
|
||||
var tagID int64
|
||||
for _, tag := range tags {
|
||||
if tag.Label == "清纯" {
|
||||
tagID = tag.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
if tagID == 0 {
|
||||
t.Fatal("created tag not found")
|
||||
}
|
||||
|
||||
req := requestWithRouteParam(http.MethodDelete, "/admin/api/tags/1", "id", strconv.FormatInt(tagID, 10), strings.NewReader(``))
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleDeleteTag(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
video, err := cat.GetVideo(ctx, "video-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if len(video.Tags) != 0 {
|
||||
t.Fatalf("video tags = %#v, want none", video.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleAdminListVideosFiltersByDriveID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
+202
-69
@@ -21,6 +21,7 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/auth"
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives/localstorage"
|
||||
"github.com/video-site/backend/internal/drives/localupload"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
@@ -39,7 +40,7 @@ var allowedUploadExtensions = map[string]struct{}{
|
||||
var allowedUploadTags = map[string]struct{}{
|
||||
"奶子": {},
|
||||
"臀": {},
|
||||
"口角": {},
|
||||
"口交": {},
|
||||
"女大": {},
|
||||
"人妻": {},
|
||||
"AV": {},
|
||||
@@ -155,26 +156,86 @@ func (s *Server) handleGetTheme(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) {
|
||||
// 拉一批候选(按发布时间倒序,覆盖最近 200 个),然后随机洗牌取前 homePageSize 个。
|
||||
// 如果库内不足 200 个会自动按实际数量返回,最后裁剪到 homePageSize。
|
||||
const candidatePool = 200
|
||||
items, _, err := s.Catalog.ListVideos(r.Context(), catalog.ListParams{
|
||||
Sort: "latest", Page: 1, PageSize: candidatePool,
|
||||
})
|
||||
// 首页优先从全量已有封面的视频里随机抽取,避免只在最近一小段候选中反复出现。
|
||||
excludeIDs := parseVideoIDQuery(r, "exclude", 120)
|
||||
items, err := s.Catalog.RandomVideosWithReadyThumbnailsExcluding(r.Context(), excludeIDs, homePageSize)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
rand.Shuffle(len(items), func(i, j int) {
|
||||
items[i], items[j] = items[j], items[i]
|
||||
})
|
||||
if len(items) > homePageSize {
|
||||
items = items[:homePageSize]
|
||||
if len(items) < homePageSize {
|
||||
fallbackExclude := append([]string{}, excludeIDs...)
|
||||
for _, item := range items {
|
||||
if item != nil {
|
||||
fallbackExclude = append(fallbackExclude, item.ID)
|
||||
}
|
||||
}
|
||||
fallback, err := s.Catalog.RandomVideosExcluding(r.Context(), fallbackExclude, homePageSize-len(items))
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
items = appendUniqueVideos(items, fallback, homePageSize)
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, http.StatusOK, mapVideos(items))
|
||||
}
|
||||
|
||||
func parseVideoIDQuery(r *http.Request, key string, limit int) []string {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
values := r.URL.Query()[key]
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
out := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
for _, id := range strings.Split(value, ",") {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
out = append(out, id)
|
||||
if limit > 0 && len(out) >= limit {
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func appendUniqueVideos(dst []*catalog.Video, candidates []*catalog.Video, limit int) []*catalog.Video {
|
||||
if len(dst) >= limit {
|
||||
return dst[:limit]
|
||||
}
|
||||
seen := make(map[string]struct{}, len(dst))
|
||||
for _, v := range dst {
|
||||
if v != nil {
|
||||
seen[v.ID] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, v := range candidates {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
dst = append(dst, v)
|
||||
seen[v.ID] = struct{}{}
|
||||
if len(dst) >= limit {
|
||||
return dst
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
func (s *Server) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
page, _ := strconv.Atoi(q.Get("page"))
|
||||
@@ -182,14 +243,18 @@ func (s *Server) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
if size <= 0 {
|
||||
size = 24
|
||||
}
|
||||
sort := q.Get("sort")
|
||||
params := catalog.ListParams{
|
||||
Keyword: q.Get("q"),
|
||||
Tag: q.Get("tag"),
|
||||
Category: q.Get("cat"),
|
||||
Sort: q.Get("sort"),
|
||||
Sort: sort,
|
||||
Page: page,
|
||||
PageSize: size,
|
||||
}
|
||||
if sort == "" || sort == "latest" {
|
||||
params.PreferReadyThumbnails = true
|
||||
}
|
||||
items, total, err := s.Catalog.ListVideos(r.Context(), params)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
@@ -241,7 +306,8 @@ func (s *Server) handleVideoDetail(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// pickRelatedVideos 选 total 个推荐视频。
|
||||
// 一半(向上取整)来自同标签命中,剩下用全库随机补齐;不会重复,也不会包含当前视频。
|
||||
// 一半来自同标签命中,剩下用全库随机补齐;两段都优先取已有封面的视频,
|
||||
// 不够时再回退到未生成封面的候选。结果不会重复,也不会包含当前视频。
|
||||
func (s *Server) pickRelatedVideos(ctx context.Context, current *catalog.Video, total int) []*catalog.Video {
|
||||
if total <= 0 || current == nil {
|
||||
return nil
|
||||
@@ -254,67 +320,124 @@ func (s *Server) pickRelatedVideos(ctx context.Context, current *catalog.Video,
|
||||
picked := make([]*catalog.Video, 0, total)
|
||||
seen := map[string]struct{}{current.ID: {}}
|
||||
|
||||
// 1) 同标签候选:对每个 tag 取一批,合并去重,洗牌后取 tagQuota 个
|
||||
// 1) 同标签候选:先取已有封面的候选,数量不够再从全部候选里补。
|
||||
if tagQuota > 0 && len(current.Tags) > 0 {
|
||||
var tagPool []*catalog.Video
|
||||
for _, tag := range current.Tags {
|
||||
if tag == "" {
|
||||
continue
|
||||
}
|
||||
items, _, err := s.Catalog.ListVideos(ctx, catalog.ListParams{
|
||||
Tag: tag, Sort: "latest", Page: 1, PageSize: 30,
|
||||
})
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, v := range items {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[v.ID] = struct{}{}
|
||||
tagPool = append(tagPool, v)
|
||||
}
|
||||
picked = appendRandomRelated(
|
||||
picked,
|
||||
s.relatedTagPool(ctx, current.Tags, seen, true),
|
||||
tagQuota,
|
||||
seen,
|
||||
)
|
||||
if len(picked) < tagQuota {
|
||||
picked = appendRandomRelated(
|
||||
picked,
|
||||
s.relatedTagPool(ctx, current.Tags, seen, false),
|
||||
tagQuota,
|
||||
seen,
|
||||
)
|
||||
}
|
||||
rand.Shuffle(len(tagPool), func(i, j int) {
|
||||
tagPool[i], tagPool[j] = tagPool[j], tagPool[i]
|
||||
})
|
||||
if len(tagPool) > tagQuota {
|
||||
tagPool = tagPool[:tagQuota]
|
||||
}
|
||||
picked = append(picked, tagPool...)
|
||||
}
|
||||
|
||||
// 2) 随机补齐:从全库取一批(避开已选 ID),洗牌后取剩下的名额
|
||||
remaining := total - len(picked)
|
||||
if remaining > 0 {
|
||||
// 2) 随机补齐:同样优先已有封面的全库候选,不够再回退。
|
||||
if len(picked) < total {
|
||||
picked = appendRandomRelated(
|
||||
picked,
|
||||
s.relatedListPool(ctx, seen, true, 200),
|
||||
total,
|
||||
seen,
|
||||
)
|
||||
}
|
||||
if len(picked) < total {
|
||||
picked = appendRandomRelated(
|
||||
picked,
|
||||
s.relatedListPool(ctx, seen, false, 200),
|
||||
total,
|
||||
seen,
|
||||
)
|
||||
}
|
||||
|
||||
return picked
|
||||
}
|
||||
|
||||
func (s *Server) relatedTagPool(ctx context.Context, tags []string, seen map[string]struct{}, readyOnly bool) []*catalog.Video {
|
||||
var pool []*catalog.Video
|
||||
poolSeen := make(map[string]struct{})
|
||||
for _, tag := range tags {
|
||||
if tag == "" {
|
||||
continue
|
||||
}
|
||||
items, _, err := s.Catalog.ListVideos(ctx, catalog.ListParams{
|
||||
Sort: "latest", Page: 1, PageSize: 200,
|
||||
Tag: tag,
|
||||
Sort: "latest",
|
||||
Page: 1,
|
||||
PageSize: 30,
|
||||
ThumbnailReadyOnly: readyOnly,
|
||||
PreferReadyThumbnails: !readyOnly,
|
||||
})
|
||||
if err == nil {
|
||||
var randomPool []*catalog.Video
|
||||
for _, v := range items {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[v.ID] = struct{}{}
|
||||
randomPool = append(randomPool, v)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, v := range items {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
rand.Shuffle(len(randomPool), func(i, j int) {
|
||||
randomPool[i], randomPool[j] = randomPool[j], randomPool[i]
|
||||
})
|
||||
if len(randomPool) > remaining {
|
||||
randomPool = randomPool[:remaining]
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
picked = append(picked, randomPool...)
|
||||
if _, ok := poolSeen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
poolSeen[v.ID] = struct{}{}
|
||||
pool = append(pool, v)
|
||||
}
|
||||
}
|
||||
return pool
|
||||
}
|
||||
|
||||
func (s *Server) relatedListPool(ctx context.Context, seen map[string]struct{}, readyOnly bool, pageSize int) []*catalog.Video {
|
||||
items, _, err := s.Catalog.ListVideos(ctx, catalog.ListParams{
|
||||
Sort: "latest",
|
||||
Page: 1,
|
||||
PageSize: pageSize,
|
||||
ThumbnailReadyOnly: readyOnly,
|
||||
PreferReadyThumbnails: !readyOnly,
|
||||
})
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
pool := make([]*catalog.Video, 0, len(items))
|
||||
for _, v := range items {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
pool = append(pool, v)
|
||||
}
|
||||
return pool
|
||||
}
|
||||
|
||||
func appendRandomRelated(picked []*catalog.Video, pool []*catalog.Video, targetLen int, seen map[string]struct{}) []*catalog.Video {
|
||||
if len(picked) >= targetLen || len(pool) == 0 {
|
||||
return picked
|
||||
}
|
||||
rand.Shuffle(len(pool), func(i, j int) {
|
||||
pool[i], pool[j] = pool[j], pool[i]
|
||||
})
|
||||
for _, v := range pool {
|
||||
if len(picked) >= targetLen {
|
||||
break
|
||||
}
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v.ID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[v.ID] = struct{}{}
|
||||
picked = append(picked, v)
|
||||
}
|
||||
return picked
|
||||
}
|
||||
|
||||
@@ -336,11 +459,12 @@ func (s *Server) handleTags(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// shortsNextReq 客户端把当前轮已看过的 video id 列表传上来,
|
||||
// 服务器从未在列表中的视频里随机抽 count 个返回。
|
||||
// shortsNextReq 客户端把当前轮已看过的 video id 列表传上来。
|
||||
// PreferredFromVideoID 来自短视频页最近一次点赞成功的视频,用于优先推荐相似标签。
|
||||
type shortsNextReq struct {
|
||||
SeenIDs []string `json:"seenIds"`
|
||||
Count int `json:"count"`
|
||||
SeenIDs []string `json:"seenIds"`
|
||||
Count int `json:"count"`
|
||||
PreferredFromVideoID string `json:"preferredFromVideoId"`
|
||||
}
|
||||
|
||||
// ShortsItemDTO 是短视频流单条的精简结构。比 VideoDTO 多 videoSrc / poster,
|
||||
@@ -386,7 +510,12 @@ func (s *Server) handleShortsNext(w http.ResponseWriter, r *http.Request) {
|
||||
exclude = nil
|
||||
}
|
||||
|
||||
items, err := s.Catalog.RandomVideosExcluding(r.Context(), exclude, count)
|
||||
var items []*catalog.Video
|
||||
if strings.TrimSpace(body.PreferredFromVideoID) != "" {
|
||||
items, err = s.Catalog.RandomVideosForPreferredVideoExcluding(r.Context(), body.PreferredFromVideoID, exclude, count)
|
||||
} else {
|
||||
items, err = s.Catalog.RandomVideosExcluding(r.Context(), exclude, count)
|
||||
}
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
@@ -796,6 +925,10 @@ func driveKindLabel(kind string) string {
|
||||
return "联通沃盘"
|
||||
case "onedrive":
|
||||
return "OneDrive"
|
||||
case "googledrive":
|
||||
return "Google Drive"
|
||||
case localstorage.Kind:
|
||||
return "本地存储"
|
||||
case spider91.Kind:
|
||||
return "91 爬虫"
|
||||
default:
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -98,6 +99,199 @@ func TestPreviewURLFallsBackWithoutUpdatedAt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleHomePrioritizesVideosWithReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < 20; i++ {
|
||||
id := "pending-video-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
PublishedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
CreatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
UpdatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed pending video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < homePageSize+2; i++ {
|
||||
id := "ready-video-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
|
||||
PublishedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
CreatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
UpdatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/home", nil)
|
||||
(&Server{Catalog: cat}).handleHome(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got []VideoDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(got) != homePageSize {
|
||||
t.Fatalf("home items = %d, want %d", len(got), homePageSize)
|
||||
}
|
||||
for _, item := range got {
|
||||
if !strings.HasPrefix(item.ID, "ready-video-") {
|
||||
t.Fatalf("home returned %q without a ready thumbnail; items=%#v", item.ID, got)
|
||||
}
|
||||
if !strings.HasPrefix(item.Thumbnail, "https://thumb.example/") {
|
||||
t.Fatalf("thumbnail for %q = %q, want ready thumbnail URL", item.ID, item.Thumbnail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleHomeExcludesRecentlyShownVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < homePageSize+4; i++ {
|
||||
id := "ready-video-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
|
||||
PublishedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
CreatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
UpdatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/home?exclude=ready-video-0&exclude=ready-video-1", nil)
|
||||
(&Server{Catalog: cat}).handleHome(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got []VideoDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(got) != homePageSize {
|
||||
t.Fatalf("home items = %d, want %d", len(got), homePageSize)
|
||||
}
|
||||
for _, item := range got {
|
||||
if item.ID == "ready-video-0" || item.ID == "ready-video-1" {
|
||||
t.Fatalf("home returned excluded video %q; items=%#v", item.ID, got)
|
||||
}
|
||||
if !strings.HasPrefix(item.ID, "ready-video-") {
|
||||
t.Fatalf("home returned %q without a ready thumbnail; items=%#v", item.ID, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListLatestPrefersReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < 20; i++ {
|
||||
id := "pending-latest-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
PublishedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
CreatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
UpdatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed pending video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < 12; i++ {
|
||||
id := "ready-latest-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
|
||||
PublishedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
CreatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
UpdatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/list?page=1&size=12&sort=latest", nil)
|
||||
(&Server{Catalog: cat}).handleList(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
Items []VideoDTO `json:"items"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if got.Total != 32 {
|
||||
t.Fatalf("total = %d, want all matching videos included", got.Total)
|
||||
}
|
||||
if len(got.Items) != 12 {
|
||||
t.Fatalf("items = %d, want 12", len(got.Items))
|
||||
}
|
||||
for _, item := range got.Items {
|
||||
if !strings.HasPrefix(item.ID, "ready-latest-") {
|
||||
t.Fatalf("latest list returned %q before ready thumbnails; items=%#v", item.ID, got.Items)
|
||||
}
|
||||
if !strings.HasPrefix(item.Thumbnail, "https://thumb.example/") {
|
||||
t.Fatalf("thumbnail for %q = %q, want ready thumbnail URL", item.ID, item.Thumbnail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUploadVideoSavesFileVideoTagsAndQueuesPreview(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -120,7 +314,7 @@ func TestHandleUploadVideoSavesFileVideoTagsAndQueuesPreview(t *testing.T) {
|
||||
}
|
||||
req := multipartUploadRequest(t, map[string]string{
|
||||
"title": "用户上传标题",
|
||||
"tags": "奶子,AV,女大",
|
||||
"tags": "奶子,口交,AV,女大",
|
||||
}, "clip.mp4", "video-bytes")
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
@@ -146,7 +340,7 @@ func TestHandleUploadVideoSavesFileVideoTagsAndQueuesPreview(t *testing.T) {
|
||||
if got.Title != "用户上传标题" {
|
||||
t.Fatalf("title = %q, want submitted title", got.Title)
|
||||
}
|
||||
if !sameStringSet(got.Tags, []string{"奶子", "AV", "女大"}) {
|
||||
if !sameStringSet(got.Tags, []string{"奶子", "口交", "AV", "女大"}) {
|
||||
t.Fatalf("tags = %#v, want selected tags", got.Tags)
|
||||
}
|
||||
if got.PreviewStatus != "pending" {
|
||||
@@ -382,6 +576,66 @@ func TestHandleTagsReturnsUnifiedTagPool(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleShortsNextUsesPreferredVideoLeastPopulatedTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*catalog.Video{
|
||||
{ID: "current", DriveID: "drive", FileID: "f-current", Title: "current", Tags: []string{"common", "rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-1", DriveID: "drive", FileID: "f-common-1", Title: "common 1", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-2", DriveID: "drive", FileID: "f-common-2", Title: "common 2", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "rare-1", DriveID: "drive", FileID: "f-rare-1", Title: "rare 1", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/shorts/next", strings.NewReader(`{"seenIds":["current"],"count":3,"preferredFromVideoId":"current"}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&Server{Catalog: cat}).handleShortsNext(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
Items []ShortsItemDTO `json:"items"`
|
||||
Total int `json:"total"`
|
||||
RoundComplete bool `json:"roundComplete"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
ids := make([]string, 0, len(got.Items))
|
||||
for _, item := range got.Items {
|
||||
ids = append(ids, item.ID)
|
||||
}
|
||||
if got.Total != 4 {
|
||||
t.Fatalf("total = %d, want 4", got.Total)
|
||||
}
|
||||
if got.RoundComplete {
|
||||
t.Fatalf("roundComplete = true, want false with fallback-filled batch")
|
||||
}
|
||||
if !containsString(ids, "rare-1") {
|
||||
t.Fatalf("ids = %#v, want rare-1 from least populated tag", ids)
|
||||
}
|
||||
if containsString(ids, "current") {
|
||||
t.Fatalf("ids = %#v, should exclude current", ids)
|
||||
}
|
||||
if len(ids) != 3 {
|
||||
t.Fatalf("ids = %#v, want 3 items", ids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpdateVideoTagsRejectsUnknownTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -509,6 +763,88 @@ func TestHandleVideoDetailIncludesDriveKindLabel(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleVideoDetailRecommendationsPreferReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "current-video",
|
||||
DriveID: "drive",
|
||||
FileID: "current-video",
|
||||
Title: "Current",
|
||||
Tags: []string{"same-tag"},
|
||||
ThumbnailURL: "https://thumb.example/current-video.jpg",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed current video: %v", err)
|
||||
}
|
||||
for i := 0; i < 20; i++ {
|
||||
id := "pending-related-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
Tags: []string{"same-tag"},
|
||||
PublishedAt: now.Add(time.Duration(i+1) * time.Minute),
|
||||
CreatedAt: now.Add(time.Duration(i+1) * time.Minute),
|
||||
UpdatedAt: now.Add(time.Duration(i+1) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed pending related video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < 8; i++ {
|
||||
id := "ready-related-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
Tags: []string{"same-tag"},
|
||||
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
|
||||
PublishedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
CreatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
UpdatedAt: now.Add(-time.Duration(i+1) * time.Hour),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready related video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
req := requestWithVideoID(http.MethodGet, "/api/video/current-video", "current-video", strings.NewReader(``))
|
||||
rr := httptest.NewRecorder()
|
||||
(&Server{Catalog: cat}).handleVideoDetail(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got VideoDetailDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(got.RelatedVideos) != 6 {
|
||||
t.Fatalf("related videos = %d, want 6; items=%#v", len(got.RelatedVideos), got.RelatedVideos)
|
||||
}
|
||||
for _, item := range got.RelatedVideos {
|
||||
if !strings.HasPrefix(item.ID, "ready-related-") {
|
||||
t.Fatalf("related returned %q before ready thumbnails; items=%#v", item.ID, got.RelatedVideos)
|
||||
}
|
||||
if !strings.HasPrefix(item.Thumbnail, "https://thumb.example/") {
|
||||
t.Fatalf("thumbnail for %q = %q, want ready thumbnail URL", item.ID, item.Thumbnail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleHideVideoRemovesVideoFromPublicListAndDetail(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
@@ -41,35 +41,38 @@ func (c *Catalog) Close() error { return c.db.Close() }
|
||||
// ---------- Video ----------
|
||||
|
||||
type Video struct {
|
||||
ID string `json:"id"`
|
||||
DriveID string `json:"driveId"`
|
||||
FileID string `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
ContentHash string `json:"contentHash"`
|
||||
ParentID string `json:"parentId"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Tags []string `json:"tags"`
|
||||
DurationSeconds int `json:"durationSeconds"`
|
||||
Size int64 `json:"size"`
|
||||
Ext string `json:"ext"`
|
||||
Quality string `json:"quality"`
|
||||
ThumbnailURL string `json:"thumbnailUrl"`
|
||||
PreviewFileID string `json:"previewFileId"`
|
||||
PreviewLocal string `json:"previewLocal"`
|
||||
PreviewStatus string `json:"previewStatus"`
|
||||
Views int `json:"views"`
|
||||
Favorites int `json:"favorites"`
|
||||
Comments int `json:"comments"`
|
||||
Likes int `json:"likes"`
|
||||
Dislikes int `json:"dislikes"`
|
||||
Category string `json:"category"`
|
||||
Hidden bool `json:"hidden"`
|
||||
Badges []string `json:"badges"`
|
||||
Description string `json:"description"`
|
||||
PublishedAt time.Time `json:"publishedAt"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
ID string `json:"id"`
|
||||
DriveID string `json:"driveId"`
|
||||
FileID string `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
ContentHash string `json:"contentHash"`
|
||||
SampledSHA256 string `json:"sampledSha256"`
|
||||
FingerprintStatus string `json:"fingerprintStatus"`
|
||||
FingerprintError string `json:"fingerprintError"`
|
||||
ParentID string `json:"parentId"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Tags []string `json:"tags"`
|
||||
DurationSeconds int `json:"durationSeconds"`
|
||||
Size int64 `json:"size"`
|
||||
Ext string `json:"ext"`
|
||||
Quality string `json:"quality"`
|
||||
ThumbnailURL string `json:"thumbnailUrl"`
|
||||
PreviewFileID string `json:"previewFileId"`
|
||||
PreviewLocal string `json:"previewLocal"`
|
||||
PreviewStatus string `json:"previewStatus"`
|
||||
Views int `json:"views"`
|
||||
Favorites int `json:"favorites"`
|
||||
Comments int `json:"comments"`
|
||||
Likes int `json:"likes"`
|
||||
Dislikes int `json:"dislikes"`
|
||||
Category string `json:"category"`
|
||||
Hidden bool `json:"hidden"`
|
||||
Badges []string `json:"badges"`
|
||||
Description string `json:"description"`
|
||||
PublishedAt time.Time `json:"publishedAt"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
func (c *Catalog) UpsertVideo(ctx context.Context, v *Video) error {
|
||||
@@ -109,6 +112,18 @@ ON CONFLICT(id) DO UPDATE SET
|
||||
WHEN excluded.content_hash != '' THEN excluded.content_hash
|
||||
ELSE videos.content_hash
|
||||
END,
|
||||
sampled_sha256 = CASE
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN ''
|
||||
ELSE videos.sampled_sha256
|
||||
END,
|
||||
fingerprint_status = CASE
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN 'pending'
|
||||
ELSE COALESCE(videos.fingerprint_status, 'pending')
|
||||
END,
|
||||
fingerprint_error = CASE
|
||||
WHEN videos.size_bytes != excluded.size_bytes THEN ''
|
||||
ELSE COALESCE(videos.fingerprint_error, '')
|
||||
END,
|
||||
duration_seconds= excluded.duration_seconds,
|
||||
size_bytes = excluded.size_bytes,
|
||||
ext = excluded.ext,
|
||||
@@ -199,8 +214,8 @@ func (c *Catalog) MigrateVideoToDrive(ctx context.Context, videoID, newDriveID,
|
||||
}
|
||||
|
||||
// ListVideosByDriveID 列出指定 drive 下所有未隐藏的视频,按 published_at 倒序。
|
||||
// 给 spider91 → PikPak 迁移 worker 用:扫描 spider91 drive 下所有视频,
|
||||
// 检查哪些还有本地文件,依次上传到 PikPak。
|
||||
// 给 spider91 → 115/PikPak 迁移 worker 用:扫描 spider91 drive 下所有视频,
|
||||
// 检查哪些还有本地文件,依次上传到目标盘。
|
||||
func (c *Catalog) ListVideosByDriveID(ctx context.Context, driveID string, limit int) ([]*Video, error) {
|
||||
if driveID == "" {
|
||||
return nil, fmt.Errorf("catalog: list videos by drive: empty drive id")
|
||||
@@ -306,14 +321,15 @@ func (c *Catalog) IncrementView(ctx context.Context, id string) (int, error) {
|
||||
|
||||
// VideoMetaPatch 轻量更新视频元数据(仅非零值字段会被写入)
|
||||
type VideoMetaPatch struct {
|
||||
ThumbnailURL string
|
||||
ThumbnailStatus string
|
||||
DurationSeconds int
|
||||
Category string
|
||||
ContentHash string
|
||||
FileName string
|
||||
Tags []string
|
||||
TagsSet bool
|
||||
ThumbnailURL string
|
||||
ThumbnailStatus string
|
||||
ResetThumbnailFailures bool
|
||||
DurationSeconds int
|
||||
Category string
|
||||
ContentHash string
|
||||
FileName string
|
||||
Tags []string
|
||||
TagsSet bool
|
||||
}
|
||||
|
||||
func (c *Catalog) UpdateVideoMeta(ctx context.Context, id string, p VideoMetaPatch) error {
|
||||
@@ -327,8 +343,12 @@ func (c *Catalog) UpdateVideoMeta(ctx context.Context, id string, p VideoMetaPat
|
||||
case p.ThumbnailStatus != "":
|
||||
// 调用方显式指定 status —— 信任之;典型是 worker 把状态置 'failed' 或
|
||||
// 在重试时显式置 'pending'。
|
||||
status := nullableStatus(p.ThumbnailStatus)
|
||||
parts = append(parts, "thumbnail_status = ?")
|
||||
args = append(args, nullableStatus(p.ThumbnailStatus))
|
||||
args = append(args, status)
|
||||
if status == "ready" {
|
||||
p.ResetThumbnailFailures = true
|
||||
}
|
||||
case p.ThumbnailURL != "":
|
||||
// 调用方写了 url 但没显式给 status —— 视为"封面就绪"。url 非空意味着
|
||||
// 浏览器访问那个 URL 能拿到图(要么是本地 /p/thumb/<id>,要么是网盘 API
|
||||
@@ -336,6 +356,10 @@ func (c *Catalog) UpdateVideoMeta(ctx context.Context, id string, p VideoMetaPat
|
||||
// 仍是 'pending' 的脏状态(修过的历史 bug)。
|
||||
parts = append(parts, "thumbnail_status = ?")
|
||||
args = append(args, nullableStatus("ready"))
|
||||
p.ResetThumbnailFailures = true
|
||||
}
|
||||
if p.ResetThumbnailFailures {
|
||||
parts = append(parts, "thumbnail_failures = 0")
|
||||
}
|
||||
if p.DurationSeconds > 0 {
|
||||
parts = append(parts, "duration_seconds = ?")
|
||||
@@ -374,6 +398,38 @@ func (c *Catalog) UpdateVideoMeta(ctx context.Context, id string, p VideoMetaPat
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Catalog) IncrementThumbnailFailures(ctx context.Context, id string) (int, error) {
|
||||
tx, err := c.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
res, err := tx.ExecContext(ctx,
|
||||
`UPDATE videos
|
||||
SET thumbnail_failures = COALESCE(thumbnail_failures, 0) + 1,
|
||||
updated_at = ?
|
||||
WHERE id = ?`,
|
||||
time.Now().UnixMilli(), id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
|
||||
return 0, sql.ErrNoRows
|
||||
}
|
||||
|
||||
var failures int
|
||||
if err := tx.QueryRowContext(ctx,
|
||||
`SELECT COALESCE(thumbnail_failures, 0) FROM videos WHERE id = ?`,
|
||||
id).Scan(&failures); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return failures, nil
|
||||
}
|
||||
|
||||
// ListCategories 聚合所有 category,按视频数降序
|
||||
type CategoryStat struct {
|
||||
Category string
|
||||
@@ -486,8 +542,14 @@ func (c *Catalog) ListVideosByThumbnailStatus(ctx context.Context, driveID, stat
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ListVideosNeedingThumbnail returns videos that still need a thumbnail attempt.
|
||||
// ListVideosNeedingThumbnail returns videos that still need thumbnail-worker work.
|
||||
// Besides missing thumbnails, this includes videos with an existing thumbnail but
|
||||
// missing duration metadata, because the thumbnail worker probes duration while
|
||||
// it already has a stream link.
|
||||
// Failed thumbnails are reported separately and should not block teaser generation.
|
||||
// Videos whose local assets were cleared because they are fingerprint duplicates
|
||||
// stay pending in the DB, but uniqueVideoWhereSQL keeps them out of this queue
|
||||
// while their canonical sibling still exists.
|
||||
func (c *Catalog) ListVideosNeedingThumbnail(ctx context.Context, driveID string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10000
|
||||
@@ -495,8 +557,11 @@ func (c *Catalog) ListVideosNeedingThumbnail(ctx context.Context, driveID string
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND COALESCE(thumbnail_url, '') = ''
|
||||
AND COALESCE(thumbnail_status, 'pending') != 'failed'
|
||||
AND (
|
||||
COALESCE(thumbnail_url, '') = ''
|
||||
OR COALESCE(duration_seconds, 0) <= 0
|
||||
)
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('failed', 'skipped')
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
AND `+uniqueVideoWhereSQL+`
|
||||
ORDER BY created_at ASC
|
||||
@@ -522,8 +587,11 @@ func (c *Catalog) CountVideosNeedingThumbnail(ctx context.Context, driveID strin
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND COALESCE(thumbnail_url, '') = ''
|
||||
AND COALESCE(thumbnail_status, 'pending') != 'failed'
|
||||
AND (
|
||||
COALESCE(thumbnail_url, '') = ''
|
||||
OR COALESCE(duration_seconds, 0) <= 0
|
||||
)
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('failed', 'skipped')
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
AND `+uniqueVideoWhereSQL,
|
||||
driveID).Scan(&count)
|
||||
@@ -668,14 +736,101 @@ func (c *Catalog) FindVideoByFileSignature(ctx context.Context, fileName string,
|
||||
return scanVideo(row)
|
||||
}
|
||||
|
||||
func (c *Catalog) ListVideosNeedingFingerprint(ctx context.Context, driveID string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10000
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND size_bytes > 0
|
||||
AND COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = 'pending'
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT ?`,
|
||||
driveID, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListVideosByFingerprintStatus lists visible videos on a drive by fingerprint status.
|
||||
// It is used by the admin "retry failed fingerprints" action to reset failed rows
|
||||
// back to pending and enqueue them again.
|
||||
func (c *Catalog) ListVideosByFingerprintStatus(ctx context.Context, driveID, status string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10000
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = ?
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT ?`,
|
||||
driveID, status, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) UpdateVideoFingerprint(ctx context.Context, id, sampledSHA256, status, errText string) error {
|
||||
sampledSHA256 = normalizeContentHash(sampledSHA256)
|
||||
if status == "" {
|
||||
status = "pending"
|
||||
}
|
||||
if len(errText) > 500 {
|
||||
errText = errText[:500]
|
||||
}
|
||||
res, err := c.db.ExecContext(ctx,
|
||||
`UPDATE videos
|
||||
SET sampled_sha256 = ?,
|
||||
fingerprint_status = ?,
|
||||
fingerprint_error = ?,
|
||||
updated_at = ?
|
||||
WHERE id = ?`,
|
||||
sampledSHA256, status, errText, time.Now().UnixMilli(), id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rows, err := res.RowsAffected(); err == nil && rows == 0 {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type ListParams struct {
|
||||
Keyword string
|
||||
DriveID string
|
||||
Tag string
|
||||
Category string
|
||||
Sort string // latest | hot | week | long
|
||||
Page int
|
||||
PageSize int
|
||||
Keyword string
|
||||
DriveID string
|
||||
Tag string
|
||||
Category string
|
||||
Sort string // latest | hot | week | long
|
||||
ThumbnailReadyOnly bool
|
||||
PreferReadyThumbnails bool
|
||||
Page int
|
||||
PageSize int
|
||||
}
|
||||
|
||||
func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int, error) {
|
||||
@@ -698,33 +853,36 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
|
||||
args = append(args, p.DriveID)
|
||||
}
|
||||
if p.Tag != "" {
|
||||
where = append(where, `EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
JOIN tags t ON t.id = vt.tag_id
|
||||
WHERE vt.video_id = videos.id AND t.label = ? COLLATE NOCASE
|
||||
)`)
|
||||
where = append(where, videoMatchesTagLabelSQL("videos"))
|
||||
args = append(args, p.Tag)
|
||||
}
|
||||
if p.Category != "" && p.Category != "all" {
|
||||
where = append(where, "category = ?")
|
||||
args = append(args, p.Category)
|
||||
}
|
||||
if p.ThumbnailReadyOnly {
|
||||
where = append(where, "COALESCE(thumbnail_url, '') != ''")
|
||||
}
|
||||
where = append(where, "COALESCE(hidden, 0) = 0")
|
||||
where = append(where, uniqueVideoWhereSQL)
|
||||
|
||||
whereSQL := ""
|
||||
whereSQL = " WHERE " + strings.Join(where, " AND ")
|
||||
|
||||
orderBy := " ORDER BY published_at DESC"
|
||||
readyOrderPrefix := ""
|
||||
if p.PreferReadyThumbnails {
|
||||
readyOrderPrefix = "CASE WHEN COALESCE(thumbnail_url, '') != '' THEN 0 ELSE 1 END, "
|
||||
}
|
||||
|
||||
orderBy := " ORDER BY " + readyOrderPrefix + "published_at DESC"
|
||||
switch p.Sort {
|
||||
case "hot":
|
||||
// 热度 = 点赞数,点赞相同按最新
|
||||
orderBy = " ORDER BY likes DESC, published_at DESC"
|
||||
orderBy = " ORDER BY " + readyOrderPrefix + "likes DESC, published_at DESC"
|
||||
case "week":
|
||||
orderBy = " ORDER BY likes DESC"
|
||||
orderBy = " ORDER BY " + readyOrderPrefix + "likes DESC"
|
||||
case "long":
|
||||
orderBy = " ORDER BY duration_seconds DESC"
|
||||
orderBy = " ORDER BY " + readyOrderPrefix + "duration_seconds DESC"
|
||||
}
|
||||
|
||||
// count
|
||||
@@ -774,28 +932,25 @@ func (c *Catalog) CountVisibleVideos(ctx context.Context) (int, error) {
|
||||
// 如果剩余可选数量 < limit,就返回所有可选项;调用方负责判断是否需要开新一轮。
|
||||
// limit <= 0 时返回 nil, nil。
|
||||
func (c *Catalog) RandomVideosExcluding(ctx context.Context, excludeIDs []string, limit int) ([]*Video, error) {
|
||||
return c.randomVideosExcluding(ctx, excludeIDs, limit, false)
|
||||
}
|
||||
|
||||
func (c *Catalog) RandomVideosWithReadyThumbnailsExcluding(ctx context.Context, excludeIDs []string, limit int) ([]*Video, error) {
|
||||
return c.randomVideosExcluding(ctx, excludeIDs, limit, true)
|
||||
}
|
||||
|
||||
func (c *Catalog) randomVideosExcluding(ctx context.Context, excludeIDs []string, limit int, thumbnailReadyOnly bool) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// 去重 excludeIDs,过滤空串
|
||||
seen := make(map[string]struct{}, len(excludeIDs))
|
||||
cleaned := make([]string, 0, len(excludeIDs))
|
||||
for _, id := range excludeIDs {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
cleaned = append(cleaned, id)
|
||||
}
|
||||
|
||||
cleaned := cleanVideoIDs(excludeIDs)
|
||||
args := make([]any, 0, len(cleaned)+1)
|
||||
whereSQL := `WHERE COALESCE(hidden, 0) = 0
|
||||
AND ` + uniqueVideoWhereSQL
|
||||
if thumbnailReadyOnly {
|
||||
whereSQL += " AND COALESCE(thumbnail_url, '') != ''"
|
||||
}
|
||||
if len(cleaned) > 0 {
|
||||
placeholders := strings.Repeat("?,", len(cleaned))
|
||||
placeholders = placeholders[:len(placeholders)-1]
|
||||
@@ -830,6 +985,175 @@ func (c *Catalog) RandomVideosExcluding(ctx context.Context, excludeIDs []string
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func cleanVideoIDs(ids []string) []string {
|
||||
seen := make(map[string]struct{}, len(ids))
|
||||
cleaned := make([]string, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
cleaned = append(cleaned, id)
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func cleanTagLabels(labels []string) []string {
|
||||
seen := make(map[string]struct{}, len(labels))
|
||||
cleaned := make([]string, 0, len(labels))
|
||||
for _, label := range labels {
|
||||
label = strings.TrimSpace(label)
|
||||
if label == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(label)
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
cleaned = append(cleaned, label)
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func (c *Catalog) LeastPopulatedVisibleUniqueTag(ctx context.Context, labels []string) (string, error) {
|
||||
cleaned := cleanTagLabels(labels)
|
||||
bestLabel := ""
|
||||
bestCount := 0
|
||||
for _, label := range cleaned {
|
||||
var count int
|
||||
if err := c.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*)
|
||||
FROM videos
|
||||
WHERE COALESCE(hidden, 0) = 0
|
||||
AND `+uniqueVideoWhereSQL+`
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
JOIN tags t ON t.id = vt.tag_id
|
||||
WHERE vt.video_id = videos.id
|
||||
AND t.label = ? COLLATE NOCASE
|
||||
)`,
|
||||
label,
|
||||
).Scan(&count); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if count == 0 {
|
||||
continue
|
||||
}
|
||||
if bestLabel == "" || count < bestCount {
|
||||
bestLabel = label
|
||||
bestCount = count
|
||||
}
|
||||
}
|
||||
return bestLabel, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) RandomVideosByTagExcluding(ctx context.Context, tag string, excludeIDs []string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
tag = strings.TrimSpace(tag)
|
||||
if tag == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
cleaned := cleanVideoIDs(excludeIDs)
|
||||
args := make([]any, 0, len(cleaned)+2)
|
||||
args = append(args, tag)
|
||||
whereSQL := `WHERE COALESCE(hidden, 0) = 0
|
||||
AND ` + uniqueVideoWhereSQL + `
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
JOIN tags t ON t.id = vt.tag_id
|
||||
WHERE vt.video_id = videos.id
|
||||
AND t.label = ? COLLATE NOCASE
|
||||
)`
|
||||
if len(cleaned) > 0 {
|
||||
placeholders := strings.Repeat("?,", len(cleaned))
|
||||
placeholders = placeholders[:len(placeholders)-1]
|
||||
whereSQL += " AND id NOT IN (" + placeholders + ")"
|
||||
for _, id := range cleaned {
|
||||
args = append(args, id)
|
||||
}
|
||||
}
|
||||
args = append(args, limit)
|
||||
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos `+whereSQL+`
|
||||
ORDER BY RANDOM() LIMIT ?`,
|
||||
args...,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) RandomVideosForPreferredVideoExcluding(ctx context.Context, preferredVideoID string, excludeIDs []string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
preferredVideoID = strings.TrimSpace(preferredVideoID)
|
||||
if preferredVideoID == "" {
|
||||
return c.RandomVideosExcluding(ctx, excludeIDs, limit)
|
||||
}
|
||||
|
||||
preferredExclude := append([]string{}, excludeIDs...)
|
||||
preferredExclude = append(preferredExclude, preferredVideoID)
|
||||
|
||||
preferred, err := c.GetVideo(ctx, preferredVideoID)
|
||||
if err != nil || preferred == nil || preferred.Hidden || len(preferred.Tags) == 0 {
|
||||
return c.RandomVideosExcluding(ctx, preferredExclude, limit)
|
||||
}
|
||||
tag, err := c.LeastPopulatedVisibleUniqueTag(ctx, preferred.Tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tag == "" {
|
||||
return c.RandomVideosExcluding(ctx, preferredExclude, limit)
|
||||
}
|
||||
|
||||
items, err := c.RandomVideosByTagExcluding(ctx, tag, preferredExclude, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(items) >= limit {
|
||||
return items, nil
|
||||
}
|
||||
|
||||
mergedExclude := make([]string, 0, len(preferredExclude)+len(items))
|
||||
mergedExclude = append(mergedExclude, preferredExclude...)
|
||||
for _, item := range items {
|
||||
if item != nil {
|
||||
mergedExclude = append(mergedExclude, item.ID)
|
||||
}
|
||||
}
|
||||
fallback, err := c.RandomVideosExcluding(ctx, mergedExclude, limit-len(items))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(items, fallback...), nil
|
||||
}
|
||||
|
||||
type DriveTeaserCounts struct {
|
||||
Ready int
|
||||
Pending int
|
||||
@@ -837,6 +1161,13 @@ type DriveTeaserCounts struct {
|
||||
}
|
||||
|
||||
type DriveThumbnailCounts struct {
|
||||
Ready int
|
||||
Pending int
|
||||
Failed int
|
||||
DurationPending int
|
||||
}
|
||||
|
||||
type DriveFingerprintCounts struct {
|
||||
Ready int
|
||||
Pending int
|
||||
Failed int
|
||||
@@ -877,9 +1208,12 @@ func (c *Catalog) CountThumbnailsByDrive(ctx context.Context) (map[string]DriveT
|
||||
`SELECT drive_id,
|
||||
COUNT(CASE WHEN COALESCE(thumbnail_url, '') != '' THEN 1 END) AS ready_count,
|
||||
COUNT(CASE WHEN COALESCE(thumbnail_url, '') = ''
|
||||
AND COALESCE(thumbnail_status, 'pending') != 'failed' THEN 1 END) AS pending_count,
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('failed', 'skipped') THEN 1 END) AS pending_count,
|
||||
COUNT(CASE WHEN COALESCE(thumbnail_url, '') = ''
|
||||
AND COALESCE(thumbnail_status, 'pending') = 'failed' THEN 1 END) AS failed_count
|
||||
AND COALESCE(thumbnail_status, 'pending') = 'failed' THEN 1 END) AS failed_count,
|
||||
COUNT(CASE WHEN COALESCE(thumbnail_url, '') != ''
|
||||
AND COALESCE(duration_seconds, 0) <= 0
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('failed', 'skipped') THEN 1 END) AS duration_pending_count
|
||||
FROM videos
|
||||
WHERE COALESCE(hidden, 0) = 0
|
||||
AND `+uniqueVideoWhereSQL+`
|
||||
@@ -893,6 +1227,39 @@ func (c *Catalog) CountThumbnailsByDrive(ctx context.Context) (map[string]DriveT
|
||||
for rows.Next() {
|
||||
var driveID string
|
||||
var counts DriveThumbnailCounts
|
||||
if err := rows.Scan(&driveID, &counts.Ready, &counts.Pending, &counts.Failed, &counts.DurationPending); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[driveID] = counts
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) CountFingerprintsByDrive(ctx context.Context) (map[string]DriveFingerprintCounts, error) {
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT drive_id,
|
||||
COUNT(CASE WHEN COALESCE(sampled_sha256, '') != ''
|
||||
OR COALESCE(fingerprint_status, 'pending') = 'ready' THEN 1 END) AS ready_count,
|
||||
COUNT(CASE WHEN size_bytes > 0
|
||||
AND COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = 'pending' THEN 1 END) AS pending_count,
|
||||
COUNT(CASE WHEN COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = 'failed' THEN 1 END) AS failed_count
|
||||
FROM videos
|
||||
WHERE COALESCE(hidden, 0) = 0
|
||||
GROUP BY drive_id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make(map[string]DriveFingerprintCounts)
|
||||
for rows.Next() {
|
||||
var driveID string
|
||||
var counts DriveFingerprintCounts
|
||||
if err := rows.Scan(&driveID, &counts.Ready, &counts.Pending, &counts.Failed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -904,6 +1271,19 @@ func (c *Catalog) CountThumbnailsByDrive(ctx context.Context) (map[string]DriveT
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) CountVideosNeedingFingerprint(ctx context.Context, driveID string) (int, error) {
|
||||
var count int
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND size_bytes > 0
|
||||
AND COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = 'pending'
|
||||
AND COALESCE(hidden, 0) = 0`,
|
||||
driveID).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
type LocalMediaRef struct {
|
||||
DriveID string
|
||||
VideoID string
|
||||
@@ -933,13 +1313,132 @@ func (c *Catalog) ListLocalMediaRefs(ctx context.Context) ([]LocalMediaRef, erro
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// DuplicateAssetCleanupCandidate points at a non-canonical video in a
|
||||
// size+sampled_sha256 duplicate group that still owns generated local assets.
|
||||
// The cleanup job uses this to remove duplicate thumbnails/teasers without
|
||||
// touching the original cloud file or deleting the catalog row.
|
||||
type DuplicateAssetCleanupCandidate struct {
|
||||
VideoID string
|
||||
DriveID string
|
||||
Title string
|
||||
PreviewLocal string
|
||||
ThumbnailURL string
|
||||
CanonicalID string
|
||||
SampledSHA256 string
|
||||
Size int64
|
||||
}
|
||||
|
||||
// ListDuplicateAssetCleanupCandidates returns duplicate videos whose own local
|
||||
// generated assets can be cleared. A group canonical is the same representative
|
||||
// used by uniqueVideoWhereSQL: earliest created_at, then lexicographically
|
||||
// smallest id.
|
||||
func (c *Catalog) ListDuplicateAssetCleanupCandidates(ctx context.Context, limit int) ([]DuplicateAssetCleanupCandidate, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10000
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
WITH canonical AS (
|
||||
SELECT v.id, v.size_bytes, v.sampled_sha256
|
||||
FROM videos v
|
||||
WHERE v.size_bytes > 0
|
||||
AND COALESCE(v.sampled_sha256, '') != ''
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM videos earlier
|
||||
WHERE earlier.size_bytes = v.size_bytes
|
||||
AND earlier.sampled_sha256 = v.sampled_sha256
|
||||
AND COALESCE(earlier.sampled_sha256, '') != ''
|
||||
AND earlier.size_bytes > 0
|
||||
AND (
|
||||
earlier.created_at < v.created_at
|
||||
OR (earlier.created_at = v.created_at AND earlier.id < v.id)
|
||||
)
|
||||
)
|
||||
)
|
||||
SELECT dup.id,
|
||||
dup.drive_id,
|
||||
dup.title,
|
||||
COALESCE(dup.preview_local, ''),
|
||||
COALESCE(dup.thumbnail_url, ''),
|
||||
canonical.id,
|
||||
dup.sampled_sha256,
|
||||
dup.size_bytes
|
||||
FROM videos dup
|
||||
JOIN canonical
|
||||
ON canonical.size_bytes = dup.size_bytes
|
||||
AND canonical.sampled_sha256 = dup.sampled_sha256
|
||||
WHERE dup.id != canonical.id
|
||||
AND dup.size_bytes > 0
|
||||
AND COALESCE(dup.sampled_sha256, '') != ''
|
||||
AND (
|
||||
COALESCE(dup.preview_local, '') != ''
|
||||
OR COALESCE(dup.thumbnail_url, '') = '/p/thumb/' || dup.id
|
||||
)
|
||||
ORDER BY dup.created_at ASC, dup.id ASC
|
||||
LIMIT ?`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []DuplicateAssetCleanupCandidate
|
||||
for rows.Next() {
|
||||
var item DuplicateAssetCleanupCandidate
|
||||
if err := rows.Scan(
|
||||
&item.VideoID,
|
||||
&item.DriveID,
|
||||
&item.Title,
|
||||
&item.PreviewLocal,
|
||||
&item.ThumbnailURL,
|
||||
&item.CanonicalID,
|
||||
&item.SampledSHA256,
|
||||
&item.Size,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ClearGeneratedAssets clears DB references to generated local assets for a
|
||||
// video. The statuses go back to pending so the video can regenerate assets if
|
||||
// it later becomes the canonical item after its older duplicate is removed.
|
||||
func (c *Catalog) ClearGeneratedAssets(ctx context.Context, videoID string, clearPreview, clearThumbnail bool) error {
|
||||
parts := []string{}
|
||||
args := []any{}
|
||||
if clearPreview {
|
||||
parts = append(parts, "preview_file_id = ''", "preview_local = ''", "preview_status = 'pending'")
|
||||
}
|
||||
if clearThumbnail {
|
||||
parts = append(parts, "thumbnail_url = ''", "thumbnail_status = 'pending'")
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return nil
|
||||
}
|
||||
parts = append(parts, "updated_at = ?")
|
||||
args = append(args, time.Now().UnixMilli(), videoID)
|
||||
res, err := c.db.ExecContext(ctx, `UPDATE videos SET `+strings.Join(parts, ", ")+` WHERE id = ?`, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rows, err := res.RowsAffected(); err == nil && rows == 0 {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------- Drive ----------
|
||||
|
||||
type Drive struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
RootID string `json:"rootId"`
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
RootID string `json:"rootId"`
|
||||
// Deprecated: 扫描入口固定等于 RootID;字段保留用于兼容旧数据/API。
|
||||
ScanRootID string `json:"scanRootId"`
|
||||
Credentials map[string]string `json:"credentials,omitempty"`
|
||||
Status string `json:"status"`
|
||||
@@ -957,6 +1456,7 @@ type Drive struct {
|
||||
}
|
||||
|
||||
func (c *Catalog) UpsertDrive(ctx context.Context, d *Drive) error {
|
||||
normalizeDriveRootFields(d)
|
||||
cred, _ := json.Marshal(d.Credentials)
|
||||
skipDirs := d.SkipDirIDs
|
||||
if skipDirs == nil {
|
||||
@@ -987,6 +1487,46 @@ ON CONFLICT(id) DO UPDATE SET
|
||||
return err
|
||||
}
|
||||
|
||||
func normalizeDriveRootFields(d *Drive) {
|
||||
if d == nil {
|
||||
return
|
||||
}
|
||||
d.RootID = normalizeDriveRootID(d.Kind, d.RootID)
|
||||
d.ScanRootID = d.RootID
|
||||
}
|
||||
|
||||
func normalizeDriveRootID(kind, rootID string) string {
|
||||
rootID = strings.TrimSpace(rootID)
|
||||
switch kind {
|
||||
case "pikpak":
|
||||
if rootID == "0" {
|
||||
return ""
|
||||
}
|
||||
return rootID
|
||||
case "onedrive", "googledrive":
|
||||
if rootID == "" {
|
||||
return "root"
|
||||
}
|
||||
return rootID
|
||||
case "localstorage", "spider91":
|
||||
return "/"
|
||||
default:
|
||||
if rootID == "" {
|
||||
return "0"
|
||||
}
|
||||
return rootID
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Catalog) syncDriveScanRootIDToRootID(ctx context.Context) error {
|
||||
_, err := c.db.ExecContext(ctx, `
|
||||
UPDATE drives
|
||||
SET scan_root_id = root_id,
|
||||
updated_at = ?
|
||||
WHERE COALESCE(scan_root_id, '') != COALESCE(root_id, '')`, time.Now().UnixMilli())
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) ListDrives(ctx context.Context) ([]*Drive, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT id, kind, name, root_id, COALESCE(scan_root_id, ''), COALESCE(credentials, '{}'), status, COALESCE(last_error, ''), COALESCE(teaser_enabled, 1), COALESCE(skip_dir_ids, '[]'), created_at, updated_at FROM drives ORDER BY created_at ASC`)
|
||||
if err != nil {
|
||||
@@ -1004,6 +1544,7 @@ func (c *Catalog) ListDrives(ctx context.Context) ([]*Drive, error) {
|
||||
}
|
||||
_ = json.Unmarshal([]byte(credsStr), &d.Credentials)
|
||||
_ = json.Unmarshal([]byte(skipDirsStr), &d.SkipDirIDs)
|
||||
normalizeDriveRootFields(d)
|
||||
d.TeaserEnabled = teaserEnabled != 0
|
||||
d.CreatedAt = time.UnixMilli(createdAt)
|
||||
d.UpdatedAt = time.UnixMilli(updatedAt)
|
||||
@@ -1023,6 +1564,7 @@ func (c *Catalog) GetDrive(ctx context.Context, id string) (*Drive, error) {
|
||||
}
|
||||
_ = json.Unmarshal([]byte(credsStr), &d.Credentials)
|
||||
_ = json.Unmarshal([]byte(skipDirsStr), &d.SkipDirIDs)
|
||||
normalizeDriveRootFields(d)
|
||||
d.TeaserEnabled = teaserEnabled != 0
|
||||
d.CreatedAt = time.UnixMilli(createdAt)
|
||||
d.UpdatedAt = time.UnixMilli(updatedAt)
|
||||
@@ -1161,7 +1703,9 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.upd
|
||||
// ---------- helpers ----------
|
||||
|
||||
const allVideoCols = `
|
||||
id, drive_id, file_id, COALESCE(file_name, ''), COALESCE(content_hash, ''), COALESCE(parent_id, ''), title, COALESCE(author, ''), COALESCE(tags, '[]'),
|
||||
id, drive_id, file_id, COALESCE(file_name, ''), COALESCE(content_hash, ''),
|
||||
COALESCE(sampled_sha256, ''), COALESCE(fingerprint_status, 'pending'), COALESCE(fingerprint_error, ''),
|
||||
COALESCE(parent_id, ''), title, COALESCE(author, ''), COALESCE(tags, '[]'),
|
||||
duration_seconds, size_bytes, COALESCE(ext, ''), COALESCE(quality, ''), COALESCE(thumbnail_url, ''),
|
||||
COALESCE(preview_file_id, ''), COALESCE(preview_local, ''), COALESCE(preview_status, 'pending'),
|
||||
views, favorites, comments, likes, dislikes,
|
||||
@@ -1180,6 +1724,20 @@ const uniqueVideoWhereSQL = `((COALESCE(videos.content_hash, '') = ''
|
||||
OR (dup.created_at = videos.created_at AND dup.id < videos.id)
|
||||
)
|
||||
))
|
||||
AND (COALESCE(videos.sampled_sha256, '') = ''
|
||||
OR videos.size_bytes <= 0
|
||||
OR NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM videos AS dup
|
||||
WHERE dup.sampled_sha256 = videos.sampled_sha256
|
||||
AND dup.size_bytes = videos.size_bytes
|
||||
AND COALESCE(dup.sampled_sha256, '') != ''
|
||||
AND dup.size_bytes > 0
|
||||
AND (
|
||||
dup.created_at < videos.created_at
|
||||
OR (dup.created_at = videos.created_at AND dup.id < videos.id)
|
||||
)
|
||||
))
|
||||
AND (COALESCE(videos.file_name, '') = ''
|
||||
OR videos.size_bytes <= 0
|
||||
OR NOT EXISTS (
|
||||
@@ -1205,7 +1763,9 @@ func scanVideo(row rowScanner) (*Video, error) {
|
||||
var publishedAt, createdAt, updatedAt int64
|
||||
var hidden int
|
||||
err := row.Scan(
|
||||
&v.ID, &v.DriveID, &v.FileID, &v.FileName, &v.ContentHash, &v.ParentID, &v.Title, &v.Author, &tagsJSON,
|
||||
&v.ID, &v.DriveID, &v.FileID, &v.FileName, &v.ContentHash,
|
||||
&v.SampledSHA256, &v.FingerprintStatus, &v.FingerprintError,
|
||||
&v.ParentID, &v.Title, &v.Author, &tagsJSON,
|
||||
&v.DurationSeconds, &v.Size, &v.Ext, &v.Quality, &v.ThumbnailURL,
|
||||
&v.PreviewFileID, &v.PreviewLocal, &v.PreviewStatus,
|
||||
&v.Views, &v.Favorites, &v.Comments, &v.Likes, &v.Dislikes,
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUpsertDriveUsesRootIDAsScanRootID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: "drive",
|
||||
Kind: "p115",
|
||||
Name: "115",
|
||||
RootID: "root-folder",
|
||||
ScanRootID: "ignored-scan-root",
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert drive: %v", err)
|
||||
}
|
||||
|
||||
got, err := cat.GetDrive(ctx, "drive")
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.RootID != "root-folder" {
|
||||
t.Fatalf("rootId = %q, want root-folder", got.RootID)
|
||||
}
|
||||
if got.ScanRootID != "root-folder" {
|
||||
t.Fatalf("scanRootId = %q, want root-folder", got.ScanRootID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertDriveDefaultsRootIDByKind(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
cases := []struct {
|
||||
id string
|
||||
kind string
|
||||
want string
|
||||
}{
|
||||
{id: "p115", kind: "p115", want: "0"},
|
||||
{id: "pikpak", kind: "pikpak", want: ""},
|
||||
{id: "onedrive", kind: "onedrive", want: "root"},
|
||||
{id: "googledrive", kind: "googledrive", want: "root"},
|
||||
{id: "localstorage", kind: "localstorage", want: "/"},
|
||||
{id: "spider91", kind: "spider91", want: "/"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: tc.id,
|
||||
Kind: tc.kind,
|
||||
Name: tc.kind,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert %s: %v", tc.kind, err)
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, tc.id)
|
||||
if err != nil {
|
||||
t.Fatalf("get %s: %v", tc.kind, err)
|
||||
}
|
||||
if got.RootID != tc.want {
|
||||
t.Fatalf("%s rootId = %q, want %q", tc.kind, got.RootID, tc.want)
|
||||
}
|
||||
if got.ScanRootID != tc.want {
|
||||
t.Fatalf("%s scanRootId = %q, want %q", tc.kind, got.ScanRootID, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertDriveIgnoresRootIDForLocalStorageAndSpider91(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, tc := range []struct {
|
||||
id string
|
||||
kind string
|
||||
}{
|
||||
{id: "localstorage", kind: "localstorage"},
|
||||
{id: "spider91", kind: "spider91"},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: tc.id,
|
||||
Kind: tc.kind,
|
||||
Name: tc.kind,
|
||||
RootID: "manual-root",
|
||||
ScanRootID: "manual-scan-root",
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert %s: %v", tc.kind, err)
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, tc.id)
|
||||
if err != nil {
|
||||
t.Fatalf("get %s: %v", tc.kind, err)
|
||||
}
|
||||
if got.RootID != "/" {
|
||||
t.Fatalf("%s rootId = %q, want /", tc.kind, got.RootID)
|
||||
}
|
||||
if got.ScanRootID != "/" {
|
||||
t.Fatalf("%s scanRootId = %q, want /", tc.kind, got.ScanRootID)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestListVideosDeduplicatesBySampledSHA256(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{
|
||||
ID: "drive-a-file-a",
|
||||
DriveID: "drive-a",
|
||||
FileID: "file-a",
|
||||
FileName: "first-name.mp4",
|
||||
Title: "First",
|
||||
Size: 1234,
|
||||
PublishedAt: now.Add(-time.Minute),
|
||||
CreatedAt: now.Add(-time.Minute),
|
||||
UpdatedAt: now.Add(-time.Minute),
|
||||
},
|
||||
{
|
||||
ID: "drive-b-file-b",
|
||||
DriveID: "drive-b",
|
||||
FileID: "file-b",
|
||||
FileName: "second-name.mp4",
|
||||
Title: "Second",
|
||||
Size: 1234,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("upsert %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list before fingerprint: %v", err)
|
||||
}
|
||||
if total != 2 || len(items) != 2 {
|
||||
t.Fatalf("before fingerprint total=%d len=%d, want 2", total, len(items))
|
||||
}
|
||||
|
||||
const sampled = "abc123"
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "drive-a-file-a", sampled, "ready", ""); err != nil {
|
||||
t.Fatalf("update a fingerprint: %v", err)
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "drive-b-file-b", sampled, "ready", ""); err != nil {
|
||||
t.Fatalf("update b fingerprint: %v", err)
|
||||
}
|
||||
|
||||
items, total, err = cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list after fingerprint: %v", err)
|
||||
}
|
||||
if total != 1 || len(items) != 1 {
|
||||
t.Fatalf("after fingerprint total=%d len=%d, want 1", total, len(items))
|
||||
}
|
||||
if items[0].ID != "drive-a-file-a" {
|
||||
t.Fatalf("canonical id = %q, want earliest created video", items[0].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuplicateAssetCleanupCandidates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
base := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
|
||||
videos := []*Video{
|
||||
{
|
||||
ID: "drive-a-canonical",
|
||||
DriveID: "drive-a",
|
||||
FileID: "file-a",
|
||||
FileName: "canonical.mp4",
|
||||
Title: "Canonical",
|
||||
Size: 1234,
|
||||
ThumbnailURL: "/p/thumb/drive-a-canonical",
|
||||
PreviewLocal: "/tmp/previews/canonical.mp4",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: base,
|
||||
CreatedAt: base,
|
||||
UpdatedAt: base,
|
||||
},
|
||||
{
|
||||
ID: "drive-b-duplicate",
|
||||
DriveID: "drive-b",
|
||||
FileID: "file-b",
|
||||
FileName: "duplicate.mp4",
|
||||
Title: "Duplicate",
|
||||
Size: 1234,
|
||||
ThumbnailURL: "/p/thumb/drive-b-duplicate",
|
||||
PreviewLocal: "/tmp/previews/duplicate.mp4",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: base.Add(time.Second),
|
||||
CreatedAt: base.Add(time.Second),
|
||||
UpdatedAt: base.Add(time.Second),
|
||||
},
|
||||
{
|
||||
ID: "drive-c-remote-thumb",
|
||||
DriveID: "drive-c",
|
||||
FileID: "file-c",
|
||||
FileName: "remote-thumb.mp4",
|
||||
Title: "Remote Thumbnail",
|
||||
Size: 1234,
|
||||
ThumbnailURL: "https://thumb.example/file-c.jpg",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: base.Add(2 * time.Second),
|
||||
CreatedAt: base.Add(2 * time.Second),
|
||||
UpdatedAt: base.Add(2 * time.Second),
|
||||
},
|
||||
}
|
||||
for _, v := range videos {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
const sampled = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
for _, v := range videos {
|
||||
if err := cat.UpdateVideoFingerprint(ctx, v.ID, sampled, "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, err := cat.ListDuplicateAssetCleanupCandidates(ctx, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list cleanup candidates: %v", err)
|
||||
}
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("candidates = %#v, want only local duplicate", items)
|
||||
}
|
||||
item := items[0]
|
||||
if item.VideoID != "drive-b-duplicate" || item.CanonicalID != "drive-a-canonical" {
|
||||
t.Fatalf("candidate = %#v, want duplicate with canonical", item)
|
||||
}
|
||||
|
||||
if err := cat.ClearGeneratedAssets(ctx, item.VideoID, true, true); err != nil {
|
||||
t.Fatalf("clear generated assets: %v", err)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, item.VideoID)
|
||||
if err != nil {
|
||||
t.Fatalf("get duplicate: %v", err)
|
||||
}
|
||||
if got.PreviewLocal != "" || got.PreviewStatus != "pending" {
|
||||
t.Fatalf("preview after cleanup local=%q status=%q, want empty pending", got.PreviewLocal, got.PreviewStatus)
|
||||
}
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail after cleanup = %q, want empty", got.ThumbnailURL)
|
||||
}
|
||||
var thumbStatus string
|
||||
if err := cat.db.QueryRowContext(ctx, `SELECT thumbnail_status FROM videos WHERE id = ?`, item.VideoID).Scan(&thumbStatus); err != nil {
|
||||
t.Fatalf("query thumbnail status: %v", err)
|
||||
}
|
||||
if thumbStatus != "pending" {
|
||||
t.Fatalf("thumbnail_status = %q, want pending", thumbStatus)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,9 @@ CREATE TABLE IF NOT EXISTS videos (
|
||||
file_id TEXT NOT NULL,
|
||||
file_name TEXT DEFAULT '', -- 网盘侧原始文件名,用于同名同大小去重
|
||||
content_hash TEXT DEFAULT '',
|
||||
sampled_sha256 TEXT DEFAULT '', -- 跨网盘统一采样指纹(size + sampled bytes)
|
||||
fingerprint_status TEXT DEFAULT 'pending', -- pending / ready / failed
|
||||
fingerprint_error TEXT DEFAULT '',
|
||||
parent_id TEXT,
|
||||
title TEXT NOT NULL,
|
||||
author TEXT,
|
||||
@@ -14,7 +17,8 @@ CREATE TABLE IF NOT EXISTS videos (
|
||||
ext TEXT,
|
||||
quality TEXT, -- HD / SD
|
||||
thumbnail_url TEXT,
|
||||
thumbnail_status TEXT DEFAULT 'pending', -- pending / ready / failed
|
||||
thumbnail_status TEXT DEFAULT 'pending', -- pending / ready / failed / skipped
|
||||
thumbnail_failures INTEGER DEFAULT 0, -- consecutive transient thumbnail generation failures
|
||||
preview_file_id TEXT, -- deprecated: 旧版回写网盘后的 teaser file id
|
||||
preview_local TEXT, -- 本地 teaser 路径(兜底)
|
||||
preview_status TEXT DEFAULT 'pending', -- pending / ready / failed
|
||||
@@ -58,13 +62,21 @@ CREATE TABLE IF NOT EXISTS video_tags (
|
||||
CREATE INDEX IF NOT EXISTS idx_video_tags_tag ON video_tags(tag_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_video_tags_video ON video_tags(video_id);
|
||||
|
||||
-- 用户手动删除过的非系统标签。自动扫描/迁移不再重新创建同名标签;
|
||||
-- 管理员手动新建同名标签时会移除这里的记录。
|
||||
CREATE TABLE IF NOT EXISTS deleted_tags (
|
||||
label TEXT PRIMARY KEY COLLATE NOCASE,
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
deleted_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- 网盘账户
|
||||
CREATE TABLE IF NOT EXISTS drives (
|
||||
id TEXT PRIMARY KEY,
|
||||
kind TEXT NOT NULL, -- quark / p115 / pikpak / wopan / onedrive / spider91
|
||||
kind TEXT NOT NULL, -- quark / p115 / pikpak / wopan / onedrive / googledrive / localstorage / spider91
|
||||
name TEXT NOT NULL,
|
||||
root_id TEXT NOT NULL DEFAULT '0',
|
||||
scan_root_id TEXT, -- 扫描起点(默认 root_id)
|
||||
scan_root_id TEXT, -- deprecated: 扫描起点固定等于 root_id
|
||||
credentials TEXT, -- JSON: cookie / refresh_token 等
|
||||
status TEXT DEFAULT 'disconnected', -- disconnected / ok / error
|
||||
last_error TEXT,
|
||||
|
||||
@@ -109,3 +109,227 @@ func TestRandomVideosExcluding(t *testing.T) {
|
||||
t.Fatalf("limit 0 should return nil, got %v", got4)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosWithReadyThumbnailsExcluding(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < 4; i++ {
|
||||
id := "ready-" + string(rune('a'+i))
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: "f-" + id,
|
||||
Title: id,
|
||||
ThumbnailURL: "/p/thumb/" + id,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
id := "pending-" + string(rune('a'+i))
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: "f-" + id,
|
||||
Title: id,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosWithReadyThumbnailsExcluding(ctx, []string{"ready-a"}, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("random ready excluding: %v", err)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("ready random count = %d, want 3", len(got))
|
||||
}
|
||||
for _, v := range got {
|
||||
if v.ID == "ready-a" {
|
||||
t.Fatal("excluded ready video was returned")
|
||||
}
|
||||
if v.ThumbnailURL == "" {
|
||||
t.Fatalf("pending video %q was returned", v.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoChoosesLeastPopulatedTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "current", DriveID: "drive", FileID: "f-current", Title: "current", Tags: []string{"common", "rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-1", DriveID: "drive", FileID: "f-common-1", Title: "common 1", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-2", DriveID: "drive", FileID: "f-common-2", Title: "common 2", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "rare-1", DriveID: "drive", FileID: "f-rare-1", Title: "rare 1", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
tag, err := cat.LeastPopulatedVisibleUniqueTag(ctx, []string{"common", "rare"})
|
||||
if err != nil {
|
||||
t.Fatalf("least populated tag: %v", err)
|
||||
}
|
||||
if tag != "rare" {
|
||||
t.Fatalf("least populated tag = %q, want rare", tag)
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosForPreferredVideoExcluding(ctx, "current", []string{"current"}, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("random preferred: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID != "rare-1" {
|
||||
t.Fatalf("preferred result = %#v, want rare-1", videoIDs(got))
|
||||
}
|
||||
|
||||
got, err = cat.RandomVideosForPreferredVideoExcluding(ctx, "current", nil, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("random preferred without explicit exclude: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID == "current" {
|
||||
t.Fatalf("preferred result without explicit exclude = %#v, should not return current", videoIDs(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoFallsBackToFillBatch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "current", DriveID: "drive", FileID: "f-current", Title: "current", Tags: []string{"common", "rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-1", DriveID: "drive", FileID: "f-common-1", Title: "common 1", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "common-2", DriveID: "drive", FileID: "f-common-2", Title: "common 2", Tags: []string{"common"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "rare-1", DriveID: "drive", FileID: "f-rare-1", Title: "rare 1", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "hidden-rare", DriveID: "drive", FileID: "f-hidden-rare", Title: "hidden rare", Tags: []string{"rare"}, PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
if err := cat.HideVideo(ctx, "hidden-rare"); err != nil {
|
||||
t.Fatalf("hide hidden-rare: %v", err)
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosForPreferredVideoExcluding(ctx, "current", []string{"current"}, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("random preferred: %v", err)
|
||||
}
|
||||
ids := videoIDs(got)
|
||||
if len(ids) != 3 {
|
||||
t.Fatalf("result ids = %#v, want 3 items", ids)
|
||||
}
|
||||
for _, excluded := range []string{"current", "hidden-rare"} {
|
||||
if hasVideoID(ids, excluded) {
|
||||
t.Fatalf("result ids = %#v, should not include %s", ids, excluded)
|
||||
}
|
||||
}
|
||||
if !hasVideoID(ids, "rare-1") {
|
||||
t.Fatalf("result ids = %#v, want rare-1 from least populated tag", ids)
|
||||
}
|
||||
if len(uniqueVideoIDs(ids)) != len(ids) {
|
||||
t.Fatalf("result ids = %#v, want no duplicates", ids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoFallbacksWhenPreferenceUnavailable(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "untagged", DriveID: "drive", FileID: "f-untagged", Title: "untagged", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "visible-1", DriveID: "drive", FileID: "f-visible-1", Title: "visible 1", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "visible-2", DriveID: "drive", FileID: "f-visible-2", Title: "visible 2", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosForPreferredVideoExcluding(ctx, "missing", []string{"untagged"}, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("random missing preferred: %v", err)
|
||||
}
|
||||
if !sameVideoIDSet(videoIDs(got), []string{"visible-1", "visible-2"}) {
|
||||
t.Fatalf("missing preferred ids = %#v, want visible fallback videos", videoIDs(got))
|
||||
}
|
||||
|
||||
got, err = cat.RandomVideosForPreferredVideoExcluding(ctx, "untagged", []string{"untagged"}, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("random untagged preferred: %v", err)
|
||||
}
|
||||
if !sameVideoIDSet(videoIDs(got), []string{"visible-1", "visible-2"}) {
|
||||
t.Fatalf("untagged preferred ids = %#v, want visible fallback videos", videoIDs(got))
|
||||
}
|
||||
}
|
||||
|
||||
func videoIDs(videos []*Video) []string {
|
||||
ids := make([]string, 0, len(videos))
|
||||
for _, v := range videos {
|
||||
ids = append(ids, v.ID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func hasVideoID(ids []string, want string) bool {
|
||||
for _, id := range ids {
|
||||
if id == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func uniqueVideoIDs(ids []string) map[string]struct{} {
|
||||
seen := make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
seen[id] = struct{}{}
|
||||
}
|
||||
return seen
|
||||
}
|
||||
|
||||
func sameVideoIDSet(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
seen := make(map[string]int, len(a))
|
||||
for _, value := range a {
|
||||
seen[value]++
|
||||
}
|
||||
for _, value := range b {
|
||||
if seen[value] == 0 {
|
||||
return false
|
||||
}
|
||||
seen[value]--
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -17,6 +17,8 @@ import (
|
||||
)
|
||||
|
||||
var ErrUnknownTag = errors.New("unknown tag")
|
||||
var ErrSystemTag = errors.New("system tag cannot be deleted")
|
||||
var ErrDeletedTag = errors.New("tag was previously deleted")
|
||||
|
||||
const avTagLabel = "AV"
|
||||
|
||||
@@ -43,6 +45,15 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "content_hash", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "sampled_sha256", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "fingerprint_status", "TEXT DEFAULT 'pending'"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "fingerprint_error", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "file_name", "TEXT DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -52,6 +63,9 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "thumbnail_status", "TEXT DEFAULT 'pending'"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "thumbnail_failures", "INTEGER DEFAULT 0"); err != nil {
|
||||
return err
|
||||
}
|
||||
// drives.teaser_enabled:每盘 teaser 开关,替代旧的全局 preview.enabled。
|
||||
// 升级路径:直接让 ALTER TABLE 的 DEFAULT 1 兜底 —— 每个现存 drive 都默认开启,
|
||||
// 不读旧的 settings.preview.enabled 字段。这样老用户即便之前关过全局开关,
|
||||
@@ -65,6 +79,9 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "drives", "skip_dir_ids", "TEXT NOT NULL DEFAULT '[]'"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.syncDriveScanRootIDToRootID(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
// 一次性修正:早期版本(短暂存在过)会把现存 drive 的 teaser_enabled 同步成
|
||||
// 旧的全局 preview.enabled 值,导致升级后所有 drive 都是关。"默认开启"约定下,
|
||||
// 这里一次性把所有 drive 强制重置为 1,并用 marker setting 记号,避免之后
|
||||
@@ -83,6 +100,9 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_content_hash ON videos(content_hash)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_sampled_sha256 ON videos(size_bytes, sampled_sha256)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_hidden ON videos(hidden)`); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -189,8 +209,9 @@ func (c *Catalog) resetDriveTeaserEnabledToDefaultOnce(ctx context.Context) erro
|
||||
// - 管理员凭直觉认知字段名时会被误导
|
||||
//
|
||||
// 修正策略:
|
||||
// - thumbnail_url 非空 + status 非 'ready' + status 非 'failed' → 改成 'ready'
|
||||
// - thumbnail_url 非空 + status 非 'ready' + status 非 'failed' + status 非 'skipped' → 改成 'ready'
|
||||
// - status='failed' 不动(这是 worker 显式标的失败,要保留以便管理员手动重生)
|
||||
// - status='skipped' 不动(已有封面但时长探测不可用,避免重启后重复排队)
|
||||
//
|
||||
// 幂等保证:marker setting 写过就不再跑,避免每次重启都 update 一遍。
|
||||
func (c *Catalog) reconcileThumbnailStatusOnce(ctx context.Context) error {
|
||||
@@ -207,7 +228,7 @@ UPDATE videos
|
||||
SET thumbnail_status = 'ready',
|
||||
updated_at = ?
|
||||
WHERE COALESCE(thumbnail_url, '') != ''
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('ready', 'failed')
|
||||
AND COALESCE(thumbnail_status, 'pending') NOT IN ('ready', 'failed', 'skipped')
|
||||
`, time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
return fmt.Errorf("reconcile thumbnail_status: %w", err)
|
||||
@@ -281,7 +302,15 @@ func (c *Catalog) classifySystemTags(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (c *Catalog) backfillVideoTags(ctx context.Context) error {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT id, COALESCE(tags, '[]') FROM videos`)
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT id, COALESCE(tags, '[]')
|
||||
FROM videos
|
||||
WHERE COALESCE(tags, '') NOT IN ('', '[]', 'null')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
WHERE vt.video_id = videos.id
|
||||
)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -298,11 +327,14 @@ func (c *Catalog) backfillVideoTags(ctx context.Context) error {
|
||||
if len(labels) == 0 {
|
||||
continue
|
||||
}
|
||||
if err := c.addVideoTags(ctx, videoID, labels, "legacy", true); err != nil {
|
||||
added, err := c.addVideoTags(ctx, videoID, labels, "legacy", true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return err
|
||||
if added {
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -350,6 +382,9 @@ GROUP BY category`)
|
||||
if !LooksLikeCollectionTag(stat.category) {
|
||||
continue
|
||||
}
|
||||
if c.tagDeleted(ctx, stat.category) {
|
||||
continue
|
||||
}
|
||||
if _, err := c.ensureTag(ctx, stat.category, nil, "collection"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -368,12 +403,178 @@ func (c *Catalog) CreateTagAndClassify(ctx context.Context, label string, aliase
|
||||
return c.classifyTag(ctx, tag)
|
||||
}
|
||||
|
||||
func (c *Catalog) EnsureTagForVideoIDPrefix(ctx context.Context, prefix, label string, aliases []string, source string) (int, error) {
|
||||
prefix = strings.TrimSpace(prefix)
|
||||
if prefix == "" {
|
||||
return 0, errors.New("video id prefix is required")
|
||||
}
|
||||
tag, err := c.ensureTag(ctx, label, aliases, source)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT v.id
|
||||
FROM videos v
|
||||
WHERE v.id LIKE ? || '%'
|
||||
AND COALESCE(v.tags_manual, 0) = 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
WHERE vt.video_id = v.id
|
||||
AND vt.tag_id = ?
|
||||
)
|
||||
ORDER BY v.id ASC`, prefix, tag.ID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var videoIDs []string
|
||||
for rows.Next() {
|
||||
var videoID string
|
||||
if err := rows.Scan(&videoID); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
videoIDs = append(videoIDs, videoID)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, videoID := range videoIDs {
|
||||
if err := c.insertVideoTag(ctx, videoID, tag.ID, "auto"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return len(videoIDs), nil
|
||||
}
|
||||
|
||||
func (c *Catalog) DeleteTag(ctx context.Context, tagID int64) (int, error) {
|
||||
tx, err := c.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
tag, err := c.getTagByIDTx(ctx, tx, tagID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if tag.Source == "system" {
|
||||
return 0, ErrSystemTag
|
||||
}
|
||||
|
||||
rows, err := tx.QueryContext(ctx, `SELECT video_id FROM video_tags WHERE tag_id = ?`, tagID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var videoIDs []string
|
||||
for rows.Next() {
|
||||
var videoID string
|
||||
if err := rows.Scan(&videoID); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
videoIDs = append(videoIDs, videoID)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM video_tags WHERE tag_id = ?`, tagID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM tags WHERE id = ?`, tagID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := markDeletedTagTx(ctx, tx, tag); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
for _, videoID := range videoIDs {
|
||||
manual := hasManualTagsTx(ctx, tx, videoID)
|
||||
if err := syncVideoTagsJSONTx(ctx, tx, videoID, manual); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(videoIDs), nil
|
||||
}
|
||||
|
||||
func (c *Catalog) ListTags(ctx context.Context) ([]Tag, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT t.id, t.label, t.aliases, t.source, COUNT(v.id) AS cnt
|
||||
WITH tagged_tags AS (
|
||||
SELECT vt.tag_id,
|
||||
tagged.id,
|
||||
COALESCE(tagged.content_hash, '') AS content_hash,
|
||||
COALESCE(tagged.sampled_sha256, '') AS sampled_sha256,
|
||||
tagged.size_bytes,
|
||||
COALESCE(tagged.file_name, '') AS file_name
|
||||
FROM video_tags vt
|
||||
JOIN videos tagged ON tagged.id = vt.video_id
|
||||
WHERE COALESCE(tagged.hidden, 0) = 0
|
||||
),
|
||||
tag_candidates AS (
|
||||
SELECT tag_id, id AS video_id
|
||||
FROM tagged_tags
|
||||
UNION ALL
|
||||
SELECT tag_id,
|
||||
(SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_tags.content_hash != ''
|
||||
AND canonical.content_hash = tagged_tags.content_hash
|
||||
AND COALESCE(canonical.content_hash, '') != ''
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_tags
|
||||
WHERE content_hash != ''
|
||||
UNION ALL
|
||||
SELECT tag_id,
|
||||
(SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_tags.sampled_sha256 != ''
|
||||
AND tagged_tags.size_bytes > 0
|
||||
AND canonical.sampled_sha256 = tagged_tags.sampled_sha256
|
||||
AND canonical.size_bytes = tagged_tags.size_bytes
|
||||
AND COALESCE(canonical.sampled_sha256, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_tags
|
||||
WHERE sampled_sha256 != '' AND size_bytes > 0
|
||||
UNION ALL
|
||||
SELECT tag_id,
|
||||
(SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_tags.file_name != ''
|
||||
AND tagged_tags.size_bytes > 0
|
||||
AND canonical.file_name = tagged_tags.file_name
|
||||
AND canonical.size_bytes = tagged_tags.size_bytes
|
||||
AND COALESCE(canonical.file_name, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_tags
|
||||
WHERE file_name != '' AND size_bytes > 0
|
||||
)
|
||||
SELECT t.id, t.label, t.aliases, t.source, COUNT(DISTINCT videos.id) AS cnt
|
||||
FROM tags t
|
||||
LEFT JOIN video_tags vt ON vt.tag_id = t.id
|
||||
LEFT JOIN videos v ON v.id = vt.video_id AND COALESCE(v.hidden, 0) = 0
|
||||
LEFT JOIN tag_candidates tc ON tc.tag_id = t.id AND tc.video_id IS NOT NULL
|
||||
LEFT JOIN videos ON videos.id = tc.video_id
|
||||
AND COALESCE(videos.hidden, 0) = 0
|
||||
AND `+uniqueVideoWhereSQL+`
|
||||
GROUP BY t.id, t.label, t.aliases, t.source
|
||||
ORDER BY cnt DESC, t.label ASC`)
|
||||
if err != nil {
|
||||
@@ -391,6 +592,66 @@ ORDER BY cnt DESC, t.label ASC`)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func videoMatchesTagLabelSQL(videoAlias string) string {
|
||||
return fmt.Sprintf(`%s.id IN (
|
||||
WITH tagged_videos AS (
|
||||
SELECT tagged.id,
|
||||
COALESCE(tagged.content_hash, '') AS content_hash,
|
||||
COALESCE(tagged.sampled_sha256, '') AS sampled_sha256,
|
||||
tagged.size_bytes,
|
||||
COALESCE(tagged.file_name, '') AS file_name
|
||||
FROM video_tags vt
|
||||
JOIN tags tag_filter ON tag_filter.id = vt.tag_id
|
||||
JOIN videos tagged ON tagged.id = vt.video_id
|
||||
WHERE tag_filter.label = ? COLLATE NOCASE
|
||||
AND COALESCE(tagged.hidden, 0) = 0
|
||||
),
|
||||
tag_candidates AS (
|
||||
SELECT id AS video_id
|
||||
FROM tagged_videos
|
||||
UNION ALL
|
||||
SELECT (SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_videos.content_hash != ''
|
||||
AND canonical.content_hash = tagged_videos.content_hash
|
||||
AND COALESCE(canonical.content_hash, '') != ''
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_videos
|
||||
WHERE content_hash != ''
|
||||
UNION ALL
|
||||
SELECT (SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_videos.sampled_sha256 != ''
|
||||
AND tagged_videos.size_bytes > 0
|
||||
AND canonical.sampled_sha256 = tagged_videos.sampled_sha256
|
||||
AND canonical.size_bytes = tagged_videos.size_bytes
|
||||
AND COALESCE(canonical.sampled_sha256, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_videos
|
||||
WHERE sampled_sha256 != '' AND size_bytes > 0
|
||||
UNION ALL
|
||||
SELECT (SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_videos.file_name != ''
|
||||
AND tagged_videos.size_bytes > 0
|
||||
AND canonical.file_name = tagged_videos.file_name
|
||||
AND canonical.size_bytes = tagged_videos.size_bytes
|
||||
AND COALESCE(canonical.file_name, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_videos
|
||||
WHERE file_name != '' AND size_bytes > 0
|
||||
)
|
||||
SELECT video_id
|
||||
FROM tag_candidates
|
||||
WHERE video_id IS NOT NULL
|
||||
)`, videoAlias)
|
||||
}
|
||||
|
||||
func (c *Catalog) SetManualVideoTags(ctx context.Context, videoID string, labels []string) error {
|
||||
if _, err := c.GetVideo(ctx, videoID); err != nil {
|
||||
return err
|
||||
@@ -441,6 +702,9 @@ func (c *Catalog) EnsureCollectionTag(ctx context.Context, label string) (string
|
||||
if !LooksLikeCollectionTag(label) {
|
||||
return "", false, nil
|
||||
}
|
||||
if c.tagDeleted(ctx, label) {
|
||||
return "", false, nil
|
||||
}
|
||||
if !c.tagExists(ctx, label) {
|
||||
count, err := c.categoryVideoCount(ctx, label)
|
||||
if err != nil {
|
||||
@@ -472,6 +736,14 @@ func (c *Catalog) ensureTag(ctx context.Context, label string, aliases []string,
|
||||
if source == "" {
|
||||
source = "user"
|
||||
}
|
||||
if source != "system" && source != "user" && c.tagDeleted(ctx, label) {
|
||||
return Tag{}, ErrDeletedTag
|
||||
}
|
||||
if source == "system" || source == "user" {
|
||||
if err := c.restoreDeletedTag(ctx, label); err != nil {
|
||||
return Tag{}, err
|
||||
}
|
||||
}
|
||||
aliases = cleanAliases(aliases, label)
|
||||
aliasesJSON, _ := json.Marshal(aliases)
|
||||
now := time.Now().UnixMilli()
|
||||
@@ -498,6 +770,10 @@ func (c *Catalog) getTagByLabel(ctx context.Context, label string) (Tag, error)
|
||||
}
|
||||
|
||||
func (c *Catalog) classifyTag(ctx context.Context, tag Tag) (int, error) {
|
||||
existingIDs, err := c.videoIDSetForTagID(ctx, tag.ID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT id, title, COALESCE(author, ''), COALESCE(category, ''), COALESCE(tags_manual, 0)
|
||||
FROM videos`)
|
||||
@@ -529,13 +805,14 @@ FROM videos`)
|
||||
continue
|
||||
}
|
||||
}
|
||||
added, err := c.addVideoTag(ctx, videoID, tag.Label, "auto", false)
|
||||
if err != nil {
|
||||
if existingIDs[videoID] {
|
||||
continue
|
||||
}
|
||||
if err := c.insertVideoTag(ctx, videoID, tag.ID, "auto"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if added {
|
||||
classified++
|
||||
}
|
||||
existingIDs[videoID] = true
|
||||
classified++
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -545,9 +822,15 @@ FROM videos`)
|
||||
|
||||
func (c *Catalog) replaceVideoTags(ctx context.Context, videoID string, labels []string, source string, manual bool, createMissing bool) error {
|
||||
labels = uniqueStrings(cleanLabels(labels))
|
||||
if source != "manual" {
|
||||
labels = c.filterDeletedTagLabels(ctx, labels)
|
||||
}
|
||||
if createMissing {
|
||||
for _, label := range labels {
|
||||
if _, err := c.ensureTag(ctx, label, nil, "legacy"); err != nil {
|
||||
if errors.Is(err, ErrDeletedTag) {
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -589,18 +872,33 @@ func (c *Catalog) replaceVideoTags(ctx context.Context, videoID string, labels [
|
||||
return c.syncVideoTagsJSON(ctx, videoID, manual)
|
||||
}
|
||||
|
||||
func (c *Catalog) addVideoTags(ctx context.Context, videoID string, labels []string, source string, createMissing bool) error {
|
||||
for _, label := range uniqueStrings(cleanLabels(labels)) {
|
||||
if _, err := c.addVideoTag(ctx, videoID, label, source, createMissing); err != nil {
|
||||
return err
|
||||
func (c *Catalog) addVideoTags(ctx context.Context, videoID string, labels []string, source string, createMissing bool) (bool, error) {
|
||||
labels = uniqueStrings(cleanLabels(labels))
|
||||
if source != "manual" {
|
||||
labels = c.filterDeletedTagLabels(ctx, labels)
|
||||
}
|
||||
changed := false
|
||||
for _, label := range labels {
|
||||
added, err := c.addVideoTag(ctx, videoID, label, source, createMissing)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if added {
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) addVideoTag(ctx context.Context, videoID, label, source string, createMissing bool) (bool, error) {
|
||||
if source != "manual" && c.tagDeleted(ctx, label) {
|
||||
return false, nil
|
||||
}
|
||||
if createMissing {
|
||||
if _, err := c.ensureTag(ctx, label, nil, "legacy"); err != nil {
|
||||
if errors.Is(err, ErrDeletedTag) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
@@ -619,12 +917,33 @@ func (c *Catalog) addVideoTag(ctx context.Context, videoID, label, source string
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) insertVideoTag(ctx context.Context, videoID string, tagID int64, source string) error {
|
||||
_, err := c.db.ExecContext(ctx,
|
||||
`INSERT OR IGNORE INTO video_tags (video_id, tag_id, source, created_at) VALUES (?, ?, ?, ?)`,
|
||||
videoID, tagID, source, time.Now().UnixMilli())
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) addCollectionTagToVideos(ctx context.Context, category string) error {
|
||||
return c.addTagToVideosByCategory(ctx, category, category, "auto")
|
||||
}
|
||||
|
||||
func (c *Catalog) addTagToVideosByCategory(ctx context.Context, category, label, source string) error {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT id FROM videos WHERE category = ? AND COALESCE(tags_manual, 0) = 0`, category)
|
||||
tag, err := c.getTagByLabel(ctx, label)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT v.id
|
||||
FROM videos v
|
||||
WHERE v.category = ?
|
||||
AND COALESCE(v.tags_manual, 0) = 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
WHERE vt.video_id = v.id
|
||||
AND vt.tag_id = ?
|
||||
)`, category, tag.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -643,7 +962,7 @@ func (c *Catalog) addTagToVideosByCategory(ctx context.Context, category, label,
|
||||
return err
|
||||
}
|
||||
for _, videoID := range videoIDs {
|
||||
if _, err := c.addVideoTag(ctx, videoID, label, source, false); err != nil {
|
||||
if err := c.insertVideoTag(ctx, videoID, tag.ID, source); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
@@ -727,6 +1046,23 @@ func (c *Catalog) videoIDsForTagID(ctx context.Context, tagID int64) ([]string,
|
||||
return videoIDs, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) videoIDSetForTagID(ctx context.Context, tagID int64) (map[string]bool, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT video_id FROM video_tags WHERE tag_id = ?`, tagID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var videoID string
|
||||
if err := rows.Scan(&videoID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[videoID] = true
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) validateTagsExist(ctx context.Context, labels []string) error {
|
||||
for _, label := range labels {
|
||||
if _, err := c.getTagByLabel(ctx, label); err != nil {
|
||||
@@ -792,6 +1128,39 @@ func (c *Catalog) tagExists(ctx context.Context, label string) bool {
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (c *Catalog) tagDeleted(ctx context.Context, label string) bool {
|
||||
label = cleanTagLabel(label)
|
||||
if label == "" {
|
||||
return false
|
||||
}
|
||||
var exists int
|
||||
err := c.db.QueryRowContext(ctx, `SELECT 1 FROM deleted_tags WHERE label = ? COLLATE NOCASE`, label).Scan(&exists)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (c *Catalog) filterDeletedTagLabels(ctx context.Context, labels []string) []string {
|
||||
if len(labels) == 0 {
|
||||
return labels
|
||||
}
|
||||
out := labels[:0]
|
||||
for _, label := range labels {
|
||||
if c.tagDeleted(ctx, label) {
|
||||
continue
|
||||
}
|
||||
out = append(out, label)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (c *Catalog) restoreDeletedTag(ctx context.Context, label string) error {
|
||||
label = cleanTagLabel(label)
|
||||
if label == "" {
|
||||
return nil
|
||||
}
|
||||
_, err := c.db.ExecContext(ctx, `DELETE FROM deleted_tags WHERE label = ? COLLATE NOCASE`, label)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) categoryVideoCount(ctx context.Context, category string) (int, error) {
|
||||
var count int
|
||||
err := c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM videos WHERE category = ?`, category).Scan(&count)
|
||||
@@ -805,6 +1174,71 @@ func (c *Catalog) getTagByLabelTx(ctx context.Context, tx *sql.Tx, label string)
|
||||
return scanTag(row)
|
||||
}
|
||||
|
||||
func (c *Catalog) getTagByIDTx(ctx context.Context, tx *sql.Tx, id int64) (Tag, error) {
|
||||
row := tx.QueryRowContext(ctx,
|
||||
`SELECT id, label, aliases, source, 0 FROM tags WHERE id = ?`,
|
||||
id)
|
||||
return scanTag(row)
|
||||
}
|
||||
|
||||
func hasManualTagsTx(ctx context.Context, tx *sql.Tx, videoID string) bool {
|
||||
var manual int
|
||||
err := tx.QueryRowContext(ctx, `SELECT COALESCE(tags_manual, 0) FROM videos WHERE id = ?`, videoID).Scan(&manual)
|
||||
return err == nil && manual == 1
|
||||
}
|
||||
|
||||
func markDeletedTagTx(ctx context.Context, tx *sql.Tx, tag Tag) error {
|
||||
label := cleanTagLabel(tag.Label)
|
||||
if label == "" {
|
||||
return nil
|
||||
}
|
||||
now := time.Now().UnixMilli()
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO deleted_tags (label, source, deleted_at)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(label) DO UPDATE SET
|
||||
source = excluded.source,
|
||||
deleted_at = excluded.deleted_at`, label, tag.Source, now)
|
||||
return err
|
||||
}
|
||||
|
||||
func syncVideoTagsJSONTx(ctx context.Context, tx *sql.Tx, videoID string, manual bool) error {
|
||||
rows, err := tx.QueryContext(ctx, `
|
||||
SELECT t.label
|
||||
FROM video_tags vt
|
||||
JOIN tags t ON t.id = vt.tag_id
|
||||
WHERE vt.video_id = ?
|
||||
ORDER BY t.id ASC`, videoID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var labels []string
|
||||
for rows.Next() {
|
||||
var label string
|
||||
if err := rows.Scan(&label); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
labels = append(labels, label)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
labelsJSON, _ := json.Marshal(labels)
|
||||
manualValue := 0
|
||||
if manual {
|
||||
manualValue = 1
|
||||
}
|
||||
_, err = tx.ExecContext(ctx,
|
||||
`UPDATE videos SET tags = ?, tags_manual = ?, updated_at = ? WHERE id = ?`,
|
||||
string(labelsJSON), manualValue, time.Now().UnixMilli(), videoID)
|
||||
return err
|
||||
}
|
||||
|
||||
type tagRowScanner interface {
|
||||
Scan(dest ...any) error
|
||||
}
|
||||
|
||||
@@ -3,10 +3,121 @@ package catalog
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestListVideosNeedingThumbnailIncludesExistingThumbnailMissingDuration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
videos := []*Video{
|
||||
{
|
||||
ID: "duration-only",
|
||||
DriveID: "drive",
|
||||
FileID: "file-duration-only",
|
||||
Title: "Duration Only",
|
||||
ThumbnailURL: "/p/thumb/duration-only",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "complete",
|
||||
DriveID: "drive",
|
||||
FileID: "file-complete",
|
||||
Title: "Complete",
|
||||
DurationSeconds: 12,
|
||||
ThumbnailURL: "/p/thumb/complete",
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
{
|
||||
ID: "missing-thumb",
|
||||
DriveID: "drive",
|
||||
FileID: "file-missing-thumb",
|
||||
Title: "Missing Thumb",
|
||||
DurationSeconds: 18,
|
||||
PublishedAt: now.Add(2 * time.Second),
|
||||
CreatedAt: now.Add(2 * time.Second),
|
||||
UpdatedAt: now.Add(2 * time.Second),
|
||||
},
|
||||
{
|
||||
ID: "failed",
|
||||
DriveID: "drive",
|
||||
FileID: "file-failed",
|
||||
Title: "Failed",
|
||||
PublishedAt: now.Add(3 * time.Second),
|
||||
CreatedAt: now.Add(3 * time.Second),
|
||||
UpdatedAt: now.Add(3 * time.Second),
|
||||
},
|
||||
}
|
||||
for _, v := range videos {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
if err := cat.UpdateVideoMeta(ctx, "failed", VideoMetaPatch{ThumbnailStatus: "failed"}); err != nil {
|
||||
t.Fatalf("mark failed thumbnail: %v", err)
|
||||
}
|
||||
|
||||
items, err := cat.ListVideosNeedingThumbnail(ctx, "drive", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list videos needing thumbnail: %v", err)
|
||||
}
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("items = %#v, want duration-only and missing-thumb", items)
|
||||
}
|
||||
if items[0].ID != "duration-only" || items[1].ID != "missing-thumb" {
|
||||
t.Fatalf("item ids = %q, %q; want duration-only, missing-thumb", items[0].ID, items[1].ID)
|
||||
}
|
||||
|
||||
count, err := cat.CountVideosNeedingThumbnail(ctx, "drive")
|
||||
if err != nil {
|
||||
t.Fatalf("count videos needing thumbnail: %v", err)
|
||||
}
|
||||
if count != 2 {
|
||||
t.Fatalf("count = %d, want 2", count)
|
||||
}
|
||||
|
||||
counts, err := cat.CountThumbnailsByDrive(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("count thumbnails by drive: %v", err)
|
||||
}
|
||||
if got := counts["drive"]; got.Ready != 2 || got.Pending != 1 || got.Failed != 1 || got.DurationPending != 1 {
|
||||
t.Fatalf("thumbnail counts = %#v, want ready=2 pending=1 failed=1 durationPending=1", got)
|
||||
}
|
||||
|
||||
if err := cat.UpdateVideoMeta(ctx, "duration-only", VideoMetaPatch{ThumbnailStatus: "skipped"}); err != nil {
|
||||
t.Fatalf("mark duration-only skipped: %v", err)
|
||||
}
|
||||
count, err = cat.CountVideosNeedingThumbnail(ctx, "drive")
|
||||
if err != nil {
|
||||
t.Fatalf("count videos needing thumbnail after skip: %v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("count after skip = %d, want 1", count)
|
||||
}
|
||||
counts, err = cat.CountThumbnailsByDrive(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("count thumbnails by drive after skip: %v", err)
|
||||
}
|
||||
if got := counts["drive"]; got.Ready != 2 || got.Pending != 1 || got.Failed != 1 || got.DurationPending != 0 {
|
||||
t.Fatalf("thumbnail counts after skip = %#v, want ready=2 pending=1 failed=1 durationPending=0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateTagAndClassifyAddsTagToMatchingExistingVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
@@ -70,6 +181,242 @@ func TestCreateTagAndClassifyAddsTagToMatchingExistingVideos(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteTagRemovesTagFromVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: "video-1",
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
Title: "清纯短发",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
if _, err := cat.CreateTagAndClassify(ctx, "清纯", nil, "user"); err != nil {
|
||||
t.Fatalf("create tag: %v", err)
|
||||
}
|
||||
|
||||
tag := mustTagByLabel(t, ctx, cat, "清纯")
|
||||
removed, err := cat.DeleteTag(ctx, tag.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("delete tag: %v", err)
|
||||
}
|
||||
if removed != 1 {
|
||||
t.Fatalf("removed = %d, want 1", removed)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(ctx, "video-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if len(got.Tags) != 0 {
|
||||
t.Fatalf("video tags = %#v, want none", got.Tags)
|
||||
}
|
||||
for _, tag := range mustListTags(t, ctx, cat) {
|
||||
if tag.Label == "清纯" {
|
||||
t.Fatal("deleted tag still appears in ListTags")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteTagSuppressesAutomaticCollectionRecreation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, id := range []string{"video-1", "video-2"} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: "合集视频",
|
||||
Category: "sunny",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
if label, ok, err := cat.EnsureCollectionTag(ctx, "sunny"); err != nil || !ok || label != "sunny" {
|
||||
t.Fatalf("ensure collection = %q, %v, %v; want sunny true nil", label, ok, err)
|
||||
}
|
||||
tag := mustTagByLabel(t, ctx, cat, "sunny")
|
||||
if _, err := cat.DeleteTag(ctx, tag.ID); err != nil {
|
||||
t.Fatalf("delete tag: %v", err)
|
||||
}
|
||||
|
||||
if label, ok, err := cat.EnsureCollectionTag(ctx, "sunny"); err != nil || ok || label != "" {
|
||||
t.Fatalf("ensure deleted collection = %q, %v, %v; want empty false nil", label, ok, err)
|
||||
}
|
||||
for _, tag := range mustListTags(t, ctx, cat) {
|
||||
if tag.Label == "sunny" {
|
||||
t.Fatal("deleted collection tag was recreated automatically")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateTagAndClassifyRestoresDeletedTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: "video-1",
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
Title: "清纯短发",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
if _, err := cat.CreateTagAndClassify(ctx, "清纯", nil, "user"); err != nil {
|
||||
t.Fatalf("create tag: %v", err)
|
||||
}
|
||||
tag := mustTagByLabel(t, ctx, cat, "清纯")
|
||||
if _, err := cat.DeleteTag(ctx, tag.ID); err != nil {
|
||||
t.Fatalf("delete tag: %v", err)
|
||||
}
|
||||
|
||||
classified, err := cat.CreateTagAndClassify(ctx, "清纯", nil, "user")
|
||||
if err != nil {
|
||||
t.Fatalf("recreate tag: %v", err)
|
||||
}
|
||||
if classified != 1 {
|
||||
t.Fatalf("classified = %d, want 1", classified)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, "video-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if !sameStrings(got.Tags, []string{"清纯"}) {
|
||||
t.Fatalf("video tags = %#v, want 清纯", got.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureTagForVideoIDPrefixBackfillsSourceTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, seed := range []struct {
|
||||
id string
|
||||
manual bool
|
||||
}{
|
||||
{id: "spider91-91-spider-1200001"},
|
||||
{id: "spider91-91-spider-1200002", manual: true},
|
||||
{id: "spider91-other-1200003"},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: seed.id,
|
||||
DriveID: "91-spider",
|
||||
FileID: seed.id + ".mp4",
|
||||
Title: "legacy title without source text",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", seed.id, err)
|
||||
}
|
||||
if seed.manual {
|
||||
if err := cat.SetManualVideoTags(ctx, seed.id, nil); err != nil {
|
||||
t.Fatalf("mark %s manual: %v", seed.id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
added, err := cat.EnsureTagForVideoIDPrefix(ctx, "spider91-91-spider-", "91porn", nil, "system")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure prefix tag: %v", err)
|
||||
}
|
||||
if added != 1 {
|
||||
t.Fatalf("added = %d, want 1", added)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, "spider91-91-spider-1200001")
|
||||
if err != nil {
|
||||
t.Fatalf("get tagged video: %v", err)
|
||||
}
|
||||
if !sameStrings(got.Tags, []string{"91porn"}) {
|
||||
t.Fatalf("tagged video tags = %#v, want 91porn", got.Tags)
|
||||
}
|
||||
manual, err := cat.GetVideo(ctx, "spider91-91-spider-1200002")
|
||||
if err != nil {
|
||||
t.Fatalf("get manual video: %v", err)
|
||||
}
|
||||
if len(manual.Tags) != 0 {
|
||||
t.Fatalf("manual video tags = %#v, want unchanged", manual.Tags)
|
||||
}
|
||||
other, err := cat.GetVideo(ctx, "spider91-other-1200003")
|
||||
if err != nil {
|
||||
t.Fatalf("get other prefix video: %v", err)
|
||||
}
|
||||
if len(other.Tags) != 0 {
|
||||
t.Fatalf("other prefix video tags = %#v, want unchanged", other.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteTagRejectsSystemTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
tag := mustTagByLabel(t, ctx, cat, "AV")
|
||||
if _, err := cat.DeleteTag(ctx, tag.ID); !errors.Is(err, ErrSystemTag) {
|
||||
t.Fatalf("delete system tag err = %v, want ErrSystemTag", err)
|
||||
}
|
||||
|
||||
if tag := mustTagByLabel(t, ctx, cat, "AV"); tag.Source != "system" {
|
||||
t.Fatalf("AV source = %q, want system", tag.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenClassifiesSystemTagsForExistingVideos(t *testing.T) {
|
||||
path := t.TempDir() + "/catalog.db"
|
||||
db, err := sql.Open("sqlite", path)
|
||||
@@ -120,6 +467,84 @@ VALUES
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateDoesNotRewriteAlreadySyncedVideoTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, id := range []string{"video-1", "video-2", "video-3"} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: "巨乳后入合集",
|
||||
Category: "Better Call Saul S03",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
if err := cat.migrate(ctx); err != nil {
|
||||
t.Fatalf("first migrate: %v", err)
|
||||
}
|
||||
|
||||
before := videoUpdatedAtByID(t, ctx, cat, "video-1", "video-2", "video-3")
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
if err := cat.migrate(ctx); err != nil {
|
||||
t.Fatalf("second migrate: %v", err)
|
||||
}
|
||||
after := videoUpdatedAtByID(t, ctx, cat, "video-1", "video-2", "video-3")
|
||||
for id, want := range before {
|
||||
if got := after[id]; got != want {
|
||||
t.Fatalf("%s updated_at changed on no-op migrate: got %d, want %d", id, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateBackfillsLegacyTagsWithoutRelations(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
if _, err := cat.db.ExecContext(ctx, `
|
||||
INSERT INTO videos (id, drive_id, file_id, title, tags, tags_manual, published_at, created_at, updated_at)
|
||||
VALUES ('legacy-video', 'drive', 'file-legacy', 'legacy title', '["legacy-tag"]', 0, ?, ?, ?)`,
|
||||
now, now, now); err != nil {
|
||||
t.Fatalf("seed legacy video: %v", err)
|
||||
}
|
||||
if err := cat.migrate(ctx); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
|
||||
tag := mustTagByLabel(t, ctx, cat, "legacy-tag")
|
||||
var count int
|
||||
if err := cat.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM video_tags WHERE video_id = 'legacy-video' AND tag_id = ?`, tag.ID).Scan(&count); err != nil {
|
||||
t.Fatalf("count video tag: %v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("legacy video tag relation count = %d, want 1", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenMigratesLegacyVideosWithoutFileName(t *testing.T) {
|
||||
path := t.TempDir() + "/catalog.db"
|
||||
db, err := sql.Open("sqlite", path)
|
||||
@@ -581,6 +1006,151 @@ func TestListVideosHidesDuplicateContentHashes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTagFilterMatchesCanonicalDuplicateVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{
|
||||
ID: "pikpak-canonical",
|
||||
DriveID: "pikpak",
|
||||
FileID: "canonical.mp4",
|
||||
Title: "Canonical",
|
||||
Size: 1024,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "spider91-dup-1",
|
||||
DriveID: "91-spider",
|
||||
FileID: "dup-1.mp4",
|
||||
Title: "Spider duplicate 1",
|
||||
Tags: []string{"91porn"},
|
||||
Size: 1024,
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
{
|
||||
ID: "spider91-dup-2",
|
||||
DriveID: "91-spider",
|
||||
FileID: "dup-2.mp4",
|
||||
Title: "Spider duplicate 2",
|
||||
Tags: []string{"91porn"},
|
||||
Size: 1024,
|
||||
PublishedAt: now.Add(2 * time.Second),
|
||||
CreatedAt: now.Add(2 * time.Second),
|
||||
UpdatedAt: now.Add(2 * time.Second),
|
||||
},
|
||||
{
|
||||
ID: "spider91-visible",
|
||||
DriveID: "91-spider",
|
||||
FileID: "visible.mp4",
|
||||
Title: "Spider visible",
|
||||
Tags: []string{"91porn"},
|
||||
Size: 2048,
|
||||
PublishedAt: now.Add(3 * time.Second),
|
||||
CreatedAt: now.Add(3 * time.Second),
|
||||
UpdatedAt: now.Add(3 * time.Second),
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"pikpak-canonical", "spider91-dup-1", "spider91-dup-2"} {
|
||||
if err := cat.UpdateVideoFingerprint(ctx, id, "same-sampled-sha256", "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "spider91-visible", "unique-sampled-sha256", "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint visible: %v", err)
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{Tag: "91porn", Page: 1, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list videos by tag: %v", err)
|
||||
}
|
||||
if total != 2 || len(items) != 2 {
|
||||
t.Fatalf("tagged videos total=%d len=%d, want 2", total, len(items))
|
||||
}
|
||||
gotIDs := map[string]bool{}
|
||||
for _, item := range items {
|
||||
gotIDs[item.ID] = true
|
||||
}
|
||||
for _, want := range []string{"pikpak-canonical", "spider91-visible"} {
|
||||
if !gotIDs[want] {
|
||||
t.Fatalf("tagged video ids = %#v, want %s", gotIDs, want)
|
||||
}
|
||||
}
|
||||
if got := mustTagByLabel(t, ctx, cat, "91porn").Count; got != 2 {
|
||||
t.Fatalf("91porn count = %d, want 2 visible canonical videos", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListVideosCanFilterReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{
|
||||
ID: "ready-video",
|
||||
DriveID: "drive",
|
||||
FileID: "file-ready",
|
||||
Title: "Ready",
|
||||
ThumbnailURL: "/p/thumb/ready-video",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "pending-video",
|
||||
DriveID: "drive",
|
||||
FileID: "file-pending",
|
||||
Title: "Pending",
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed video %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{
|
||||
Page: 1, PageSize: 10, ThumbnailReadyOnly: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list videos: %v", err)
|
||||
}
|
||||
if total != 1 || len(items) != 1 {
|
||||
t.Fatalf("ready videos total=%d len=%d, want 1", total, len(items))
|
||||
}
|
||||
if items[0].ID != "ready-video" {
|
||||
t.Fatalf("ready video id = %q, want ready-video", items[0].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func sameStrings(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
@@ -593,6 +1163,39 @@ func sameStrings(a, b []string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func mustListTags(t *testing.T, ctx context.Context, cat *Catalog) []Tag {
|
||||
t.Helper()
|
||||
tags, err := cat.ListTags(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list tags: %v", err)
|
||||
}
|
||||
return tags
|
||||
}
|
||||
|
||||
func mustTagByLabel(t *testing.T, ctx context.Context, cat *Catalog, label string) Tag {
|
||||
t.Helper()
|
||||
for _, tag := range mustListTags(t, ctx, cat) {
|
||||
if tag.Label == label {
|
||||
return tag
|
||||
}
|
||||
}
|
||||
t.Fatalf("tag %q not found", label)
|
||||
return Tag{}
|
||||
}
|
||||
|
||||
func videoUpdatedAtByID(t *testing.T, ctx context.Context, cat *Catalog, ids ...string) map[string]int64 {
|
||||
t.Helper()
|
||||
out := make(map[string]int64, len(ids))
|
||||
for _, id := range ids {
|
||||
var updatedAt int64
|
||||
if err := cat.db.QueryRowContext(ctx, `SELECT updated_at FROM videos WHERE id = ?`, id).Scan(&updatedAt); err != nil {
|
||||
t.Fatalf("read updated_at for %s: %v", id, err)
|
||||
}
|
||||
out[id] = updatedAt
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// 删除 collection 标签的最后一个引用视频后,标签应当自动从 tags 表里消失。
|
||||
// user/system 标签不受影响:用户/系统标签的语义由人维护,孤儿状态保留。
|
||||
func TestDeleteVideoPrunesOrphanCollectionTag(t *testing.T) {
|
||||
@@ -786,11 +1389,12 @@ func TestReconcileThumbnailStatusOnce(t *testing.T) {
|
||||
id, url, status string
|
||||
wantStatus string
|
||||
}{
|
||||
{"v-pending-url", "/p/thumb/v-pending-url", "pending", "ready"}, // 主要修复目标
|
||||
{"v-empty-url-pending", "", "pending", "pending"}, // 没 url 不动
|
||||
{"v-failed-with-url", "/p/thumb/v-failed-with-url", "failed", "failed"}, // 显式失败保留
|
||||
{"v-empty-url-failed", "", "failed", "failed"}, // 失败 + 没 url 也保留
|
||||
{"v-already-ready", "/p/thumb/v-already-ready", "ready", "ready"}, // 幂等
|
||||
{"v-pending-url", "/p/thumb/v-pending-url", "pending", "ready"}, // 主要修复目标
|
||||
{"v-empty-url-pending", "", "pending", "pending"}, // 没 url 不动
|
||||
{"v-failed-with-url", "/p/thumb/v-failed-with-url", "failed", "failed"}, // 显式失败保留
|
||||
{"v-empty-url-failed", "", "failed", "failed"}, // 失败 + 没 url 也保留
|
||||
{"v-skipped-with-url", "/p/thumb/v-skipped-with-url", "skipped", "skipped"}, // 已跳过的时长补全保留
|
||||
{"v-already-ready", "/p/thumb/v-already-ready", "ready", "ready"}, // 幂等
|
||||
}
|
||||
for _, c := range cases {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
|
||||
@@ -202,7 +202,7 @@ type Nightly struct {
|
||||
// 这里保留 yaml 中的静态定义,用于启动时预置盘。生产建议只在 DB 里维护。
|
||||
type Drive struct {
|
||||
ID string `yaml:"id"`
|
||||
Kind string `yaml:"kind"` // quark / p115 / pikpak / wopan / onedrive
|
||||
Kind string `yaml:"kind"` // quark / p115 / pikpak / wopan / onedrive / googledrive / localstorage
|
||||
Name string `yaml:"name"`
|
||||
RootID string `yaml:"root_id"`
|
||||
Params map[string]string `yaml:"params,omitempty"`
|
||||
|
||||
@@ -0,0 +1,505 @@
|
||||
package googledrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-resty/resty/v2"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
const (
|
||||
Kind = "googledrive"
|
||||
defaultAPIBaseURL = "https://www.googleapis.com/drive/v3"
|
||||
defaultOAuthURL = "https://www.googleapis.com/oauth2/v4/token"
|
||||
defaultRenewAPIURL = "https://api.oplist.org/googleui/renewapi"
|
||||
defaultListInterval = 1 * time.Second
|
||||
defaultListCooldown = 5 * time.Minute
|
||||
|
||||
filesListFields = "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken"
|
||||
fileInfoFields = "id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum"
|
||||
)
|
||||
|
||||
type Driver struct {
|
||||
id string
|
||||
rootID string
|
||||
refreshToken string
|
||||
accessToken string
|
||||
clientID string
|
||||
clientSecret string
|
||||
useOnlineAPI bool
|
||||
renewAPIURL string
|
||||
oauthURL string
|
||||
apiBaseURL string
|
||||
client *resty.Client
|
||||
onTokenUpdate func(access, refresh string)
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
listInterval time.Duration
|
||||
listCooldown time.Duration
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ID string
|
||||
RootID string
|
||||
RefreshToken string
|
||||
AccessToken string
|
||||
ClientID string
|
||||
ClientSecret string
|
||||
UseOnlineAPI bool
|
||||
RenewAPIURL string
|
||||
OAuthURL string
|
||||
APIBaseURL string
|
||||
|
||||
OnTokenUpdate func(access, refresh string)
|
||||
}
|
||||
|
||||
func New(c Config) *Driver {
|
||||
rootID := strings.TrimSpace(c.RootID)
|
||||
if rootID == "" {
|
||||
rootID = "root"
|
||||
}
|
||||
renewAPIURL := strings.TrimSpace(c.RenewAPIURL)
|
||||
if renewAPIURL == "" {
|
||||
renewAPIURL = defaultRenewAPIURL
|
||||
}
|
||||
oauthURL := strings.TrimSpace(c.OAuthURL)
|
||||
if oauthURL == "" {
|
||||
oauthURL = defaultOAuthURL
|
||||
}
|
||||
apiBaseURL := strings.TrimRight(strings.TrimSpace(c.APIBaseURL), "/")
|
||||
if apiBaseURL == "" {
|
||||
apiBaseURL = defaultAPIBaseURL
|
||||
}
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
rootID: rootID,
|
||||
refreshToken: strings.TrimSpace(c.RefreshToken),
|
||||
accessToken: strings.TrimSpace(c.AccessToken),
|
||||
clientID: strings.TrimSpace(c.ClientID),
|
||||
clientSecret: strings.TrimSpace(c.ClientSecret),
|
||||
useOnlineAPI: c.UseOnlineAPI,
|
||||
renewAPIURL: renewAPIURL,
|
||||
oauthURL: oauthURL,
|
||||
apiBaseURL: apiBaseURL,
|
||||
onTokenUpdate: c.OnTokenUpdate,
|
||||
client: resty.New().
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
listInterval: defaultListInterval,
|
||||
listCooldown: defaultListCooldown,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Kind() string { return Kind }
|
||||
func (d *Driver) ID() string { return d.id }
|
||||
func (d *Driver) RootID() string { return d.rootID }
|
||||
|
||||
func (d *Driver) Init(ctx context.Context) error {
|
||||
if d.refreshToken == "" {
|
||||
return errors.New("googledrive init: refresh_token is required")
|
||||
}
|
||||
return d.refresh(ctx)
|
||||
}
|
||||
|
||||
func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error) {
|
||||
if dirID == "" {
|
||||
dirID = d.rootID
|
||||
}
|
||||
d.listMu.Lock()
|
||||
defer d.listMu.Unlock()
|
||||
|
||||
pageToken := ""
|
||||
out := make([]drives.Entry, 0)
|
||||
for {
|
||||
if err := d.waitForListSlotLocked(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var resp filesResp
|
||||
err := d.request(ctx, d.filesURL(), http.MethodGet, func(req *resty.Request) {
|
||||
params := map[string]string{
|
||||
"fields": filesListFields,
|
||||
"pageSize": "1000",
|
||||
"q": fmt.Sprintf("'%s' in parents and trashed = false", strings.ReplaceAll(dirID, "'", "\\'")),
|
||||
"orderBy": "folder,name,modifiedTime desc",
|
||||
}
|
||||
if pageToken != "" {
|
||||
params["pageToken"] = pageToken
|
||||
}
|
||||
req.SetQueryParams(params)
|
||||
}, &resp)
|
||||
if err != nil {
|
||||
if wait, ok := drives.RateLimitRetryAfter(err); ok {
|
||||
if wait <= 0 {
|
||||
wait = d.listCooldown
|
||||
}
|
||||
if sleepErr := sleepContext(ctx, wait); sleepErr != nil {
|
||||
return nil, sleepErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("googledrive list: %w", err)
|
||||
}
|
||||
if err := d.fillShortcutFileMetadata(ctx, resp.Files); err != nil {
|
||||
return nil, fmt.Errorf("googledrive shortcut metadata: %w", err)
|
||||
}
|
||||
for _, f := range resp.Files {
|
||||
out = append(out, fileToEntry(f, dirID))
|
||||
}
|
||||
pageToken = resp.NextPageToken
|
||||
if pageToken == "" {
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) waitForListSlotLocked(ctx context.Context) error {
|
||||
if d.listInterval <= 0 || d.lastListAt.IsZero() {
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
next := d.lastListAt.Add(d.listInterval)
|
||||
now := time.Now()
|
||||
if now.Before(next) {
|
||||
if err := sleepContext(ctx, next.Sub(now)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, d time.Duration) error {
|
||||
if d <= 0 {
|
||||
return ctx.Err()
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
|
||||
var f driveFile
|
||||
if err := d.request(ctx, d.fileURL(fileID), http.MethodGet, func(req *resty.Request) {
|
||||
req.SetQueryParam("fields", fileInfoFields)
|
||||
}, &f); err != nil {
|
||||
return nil, fmt.Errorf("googledrive stat: %w", err)
|
||||
}
|
||||
e := fileToEntry(f, "")
|
||||
return &e, nil
|
||||
}
|
||||
|
||||
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
if fileID == "" {
|
||||
return nil, errors.New("googledrive stream: empty file id")
|
||||
}
|
||||
if _, err := d.Stat(ctx, fileID); err != nil {
|
||||
return nil, fmt.Errorf("googledrive stream: %w", err)
|
||||
}
|
||||
u := d.fileURL(fileID) + "?alt=media&acknowledgeAbuse=true&supportsAllDrives=true"
|
||||
return &drives.StreamLink{
|
||||
URL: u,
|
||||
Headers: http.Header{
|
||||
"Authorization": []string{"Bearer " + d.accessToken},
|
||||
},
|
||||
Expires: time.Now().Add(30 * time.Minute),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) refresh(ctx context.Context) error {
|
||||
if d.useOnlineAPI && d.renewAPIURL != "" {
|
||||
var out tokenResp
|
||||
res, err := d.client.R().
|
||||
SetContext(ctx).
|
||||
SetQueryParams(map[string]string{
|
||||
"refresh_ui": d.refreshToken,
|
||||
"server_use": "true",
|
||||
"driver_txt": "googleui_go",
|
||||
}).
|
||||
SetResult(&out).
|
||||
SetError(&out).
|
||||
Get(d.renewAPIURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("googledrive refresh token: %w", err)
|
||||
}
|
||||
if err := tokenResponseError("googledrive refresh token", res, out, true); err != nil {
|
||||
return err
|
||||
}
|
||||
d.applyToken(out)
|
||||
return nil
|
||||
}
|
||||
if d.clientID == "" || d.clientSecret == "" {
|
||||
return errors.New("googledrive refresh token: client_id and client_secret are required when online API is disabled")
|
||||
}
|
||||
var out tokenResp
|
||||
res, err := d.client.R().
|
||||
SetContext(ctx).
|
||||
SetFormData(map[string]string{
|
||||
"client_id": d.clientID,
|
||||
"client_secret": d.clientSecret,
|
||||
"refresh_token": d.refreshToken,
|
||||
"grant_type": "refresh_token",
|
||||
}).
|
||||
SetResult(&out).
|
||||
SetError(&out).
|
||||
Post(d.oauthURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("googledrive refresh token: %w", err)
|
||||
}
|
||||
if err := tokenResponseError("googledrive refresh token", res, out, false); err != nil {
|
||||
return err
|
||||
}
|
||||
d.applyToken(out)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) applyToken(out tokenResp) {
|
||||
d.accessToken = out.AccessToken
|
||||
if strings.TrimSpace(out.RefreshToken) != "" {
|
||||
d.refreshToken = out.RefreshToken
|
||||
}
|
||||
if d.onTokenUpdate != nil {
|
||||
d.onTokenUpdate(d.accessToken, d.refreshToken)
|
||||
}
|
||||
}
|
||||
|
||||
func tokenResponseError(prefix string, res *resty.Response, out tokenResp, requireRefresh bool) error {
|
||||
if out.Text != "" {
|
||||
return fmt.Errorf("%s: %s", prefix, out.Text)
|
||||
}
|
||||
if out.Error != "" {
|
||||
if out.ErrorDescription != "" {
|
||||
return fmt.Errorf("%s: %s", prefix, out.ErrorDescription)
|
||||
}
|
||||
return fmt.Errorf("%s: %s", prefix, out.Error)
|
||||
}
|
||||
if res != nil && res.IsError() {
|
||||
return fmt.Errorf("%s: status=%d body=%s", prefix, res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
if out.AccessToken == "" || (requireRefresh && out.RefreshToken == "") {
|
||||
return fmt.Errorf("%s: empty token", prefix)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) request(ctx context.Context, rawURL, method string, configure func(*resty.Request), out any) error {
|
||||
return d.requestOnce(ctx, rawURL, method, configure, out, true)
|
||||
}
|
||||
|
||||
func (d *Driver) requestOnce(ctx context.Context, rawURL, method string, configure func(*resty.Request), out any, retry bool) error {
|
||||
req := d.client.R().
|
||||
SetContext(ctx).
|
||||
SetHeader("Authorization", "Bearer "+d.accessToken).
|
||||
SetQueryParam("includeItemsFromAllDrives", "true").
|
||||
SetQueryParam("supportsAllDrives", "true")
|
||||
if configure != nil {
|
||||
configure(req)
|
||||
}
|
||||
if out != nil {
|
||||
req.SetResult(out)
|
||||
}
|
||||
var apiErr apiErrorResp
|
||||
req.SetError(&apiErr)
|
||||
res, err := req.Execute(method, rawURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isGoogleRateLimit(res, apiErr.Error) {
|
||||
return googleRateLimitError(res, apiErr.Error.Message)
|
||||
}
|
||||
if apiErr.Error.Code != 0 {
|
||||
if apiErr.Error.Code == http.StatusUnauthorized && retry {
|
||||
if err := d.refresh(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return d.requestOnce(ctx, rawURL, method, configure, out, false)
|
||||
}
|
||||
return googleAPIError(apiErr.Error)
|
||||
}
|
||||
if res.IsError() {
|
||||
return fmt.Errorf("google drive api error: status=%d body=%s", res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) fillShortcutFileMetadata(ctx context.Context, files []driveFile) error {
|
||||
for i := range files {
|
||||
f := &files[i]
|
||||
if f.MimeType != "application/vnd.google-apps.shortcut" ||
|
||||
f.Shortcut.TargetID == "" ||
|
||||
f.Shortcut.TargetMimeType == "application/vnd.google-apps.folder" {
|
||||
continue
|
||||
}
|
||||
var target driveFile
|
||||
if err := d.request(ctx, d.fileURL(f.Shortcut.TargetID), http.MethodGet, func(req *resty.Request) {
|
||||
req.SetQueryParam("fields", fileInfoFields)
|
||||
}, &target); err != nil {
|
||||
return err
|
||||
}
|
||||
if target.Size != "" {
|
||||
f.Size = target.Size
|
||||
}
|
||||
if target.MD5Checksum != "" {
|
||||
f.MD5Checksum = target.MD5Checksum
|
||||
}
|
||||
if target.SHA1Checksum != "" {
|
||||
f.SHA1Checksum = target.SHA1Checksum
|
||||
}
|
||||
if target.SHA256Checksum != "" {
|
||||
f.SHA256Checksum = target.SHA256Checksum
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) filesURL() string {
|
||||
return d.apiBaseURL + "/files"
|
||||
}
|
||||
|
||||
func (d *Driver) fileURL(fileID string) string {
|
||||
return d.filesURL() + "/" + url.PathEscape(fileID)
|
||||
}
|
||||
|
||||
func fileToEntry(f driveFile, fallbackParentID string) drives.Entry {
|
||||
id := f.ID
|
||||
isDir := f.MimeType == "application/vnd.google-apps.folder"
|
||||
if f.MimeType == "application/vnd.google-apps.shortcut" && f.Shortcut.TargetID != "" {
|
||||
id = f.Shortcut.TargetID
|
||||
isDir = f.Shortcut.TargetMimeType == "application/vnd.google-apps.folder"
|
||||
}
|
||||
size, _ := strconv.ParseInt(f.Size, 10, 64)
|
||||
hash := f.MD5Checksum
|
||||
if hash == "" {
|
||||
hash = f.SHA1Checksum
|
||||
}
|
||||
if hash == "" {
|
||||
hash = f.SHA256Checksum
|
||||
}
|
||||
return drives.Entry{
|
||||
ID: id,
|
||||
Name: f.Name,
|
||||
Size: size,
|
||||
Hash: hash,
|
||||
IsDir: isDir,
|
||||
ParentID: fallbackParentID,
|
||||
MimeType: mimeType(f),
|
||||
ModTime: f.ModifiedTime,
|
||||
ThumbnailURL: f.ThumbnailLink,
|
||||
}
|
||||
}
|
||||
|
||||
func mimeType(f driveFile) string {
|
||||
if f.MimeType != "" && f.MimeType != "application/vnd.google-apps.shortcut" {
|
||||
return f.MimeType
|
||||
}
|
||||
if f.Shortcut.TargetMimeType != "" {
|
||||
return f.Shortcut.TargetMimeType
|
||||
}
|
||||
ext := strings.ToLower(path.Ext(f.Name))
|
||||
switch ext {
|
||||
case ".mp4":
|
||||
return "video/mp4"
|
||||
case ".mkv":
|
||||
return "video/x-matroska"
|
||||
case ".mov":
|
||||
return "video/quicktime"
|
||||
case ".webm":
|
||||
return "video/webm"
|
||||
case ".avi":
|
||||
return "video/x-msvideo"
|
||||
case ".jpg", ".jpeg":
|
||||
return "image/jpeg"
|
||||
case ".png":
|
||||
return "image/png"
|
||||
default:
|
||||
return "application/octet-stream"
|
||||
}
|
||||
}
|
||||
|
||||
func isGoogleRateLimit(res *resty.Response, body apiErrorBody) bool {
|
||||
if res != nil && res.StatusCode() == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if body.Code == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
for _, e := range body.Errors {
|
||||
reason := strings.ToLower(strings.TrimSpace(e.Reason))
|
||||
switch reason {
|
||||
case "ratelimitexceeded", "userratelimitexceeded", "downloadquotaexceeded", "sharingratelimitexceeded":
|
||||
return true
|
||||
}
|
||||
}
|
||||
msg := strings.ToLower(body.Message)
|
||||
return strings.Contains(msg, "rate limit") || strings.Contains(msg, "too many requests") || strings.Contains(msg, "quota exceeded")
|
||||
}
|
||||
|
||||
func googleRateLimitError(res *resty.Response, message string) error {
|
||||
if strings.TrimSpace(message) == "" {
|
||||
message = "google drive rate limited"
|
||||
}
|
||||
if res != nil && strings.TrimSpace(res.String()) != "" {
|
||||
message = fmt.Sprintf("%s: status=%d body=%s", message, res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: parseRetryAfter(res),
|
||||
Err: errors.New(message),
|
||||
}
|
||||
}
|
||||
|
||||
func googleAPIError(body apiErrorBody) error {
|
||||
if body.Message != "" {
|
||||
return errors.New(body.Message)
|
||||
}
|
||||
if body.Code != 0 {
|
||||
return fmt.Errorf("google drive api error: code=%d", body.Code)
|
||||
}
|
||||
return errors.New("google drive api error")
|
||||
}
|
||||
|
||||
func parseRetryAfter(res *resty.Response) time.Duration {
|
||||
if res == nil {
|
||||
return 0
|
||||
}
|
||||
raw := strings.TrimSpace(res.Header().Get("Retry-After"))
|
||||
if raw == "" {
|
||||
return 0
|
||||
}
|
||||
if seconds, err := strconv.Atoi(raw); err == nil && seconds > 0 {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
if when, err := http.ParseTime(raw); err == nil {
|
||||
d := time.Until(when)
|
||||
if d > 0 {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
@@ -0,0 +1,190 @@
|
||||
package googledrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestInitUsesOnlineRenewAPI(t *testing.T) {
|
||||
var savedAccess, savedRefresh string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/renew" {
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
if got := r.URL.Query().Get("refresh_ui"); got != "old-refresh" {
|
||||
t.Fatalf("refresh_ui = %q", got)
|
||||
}
|
||||
if got := r.URL.Query().Get("server_use"); got != "true" {
|
||||
t.Fatalf("server_use = %q", got)
|
||||
}
|
||||
if got := r.URL.Query().Get("driver_txt"); got != "googleui_go" {
|
||||
t.Fatalf("driver_txt = %q", got)
|
||||
}
|
||||
writeTestJSON(w, tokenResp{
|
||||
AccessToken: "new-access",
|
||||
RefreshToken: "new-refresh",
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "g",
|
||||
RefreshToken: "old-refresh",
|
||||
UseOnlineAPI: true,
|
||||
RenewAPIURL: srv.URL + "/renew",
|
||||
OnTokenUpdate: func(access, refresh string) {
|
||||
savedAccess = access
|
||||
savedRefresh = refresh
|
||||
},
|
||||
})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("Init() error = %v", err)
|
||||
}
|
||||
if d.accessToken != "new-access" || d.refreshToken != "new-refresh" {
|
||||
t.Fatalf("tokens not applied: access=%q refresh=%q", d.accessToken, d.refreshToken)
|
||||
}
|
||||
if savedAccess != "new-access" || savedRefresh != "new-refresh" {
|
||||
t.Fatalf("tokens not persisted: access=%q refresh=%q", savedAccess, savedRefresh)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListMapsGoogleDriveFiles(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("Authorization = %q", got)
|
||||
}
|
||||
if r.URL.Path != "/drive/v3/files" {
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
if !strings.Contains(r.URL.Query().Get("q"), "'root' in parents") {
|
||||
t.Fatalf("unexpected q = %q", r.URL.Query().Get("q"))
|
||||
}
|
||||
writeTestJSON(w, filesResp{Files: []driveFile{
|
||||
{ID: "folder-1", Name: "Movies", MimeType: "application/vnd.google-apps.folder"},
|
||||
{
|
||||
ID: "file-1",
|
||||
Name: "clip.mp4",
|
||||
MimeType: "video/mp4",
|
||||
Size: "1234",
|
||||
MD5Checksum: "abc",
|
||||
ThumbnailLink: "https://thumb.example/1",
|
||||
},
|
||||
}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", RootID: "root", APIBaseURL: srv.URL + "/drive/v3"})
|
||||
d.accessToken = "access"
|
||||
d.listInterval = -1
|
||||
|
||||
entries, err := d.List(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("List() error = %v", err)
|
||||
}
|
||||
if len(entries) != 2 {
|
||||
t.Fatalf("len(entries) = %d", len(entries))
|
||||
}
|
||||
if !entries[0].IsDir || entries[0].ID != "folder-1" {
|
||||
t.Fatalf("folder entry = %+v", entries[0])
|
||||
}
|
||||
if entries[1].ID != "file-1" || entries[1].Size != 1234 || entries[1].Hash != "abc" || entries[1].ThumbnailURL == "" {
|
||||
t.Fatalf("file entry = %+v", entries[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLReturnsAuthenticatedMediaLinkWithoutRedirectRequirement(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("Authorization = %q", got)
|
||||
}
|
||||
if r.URL.Path != "/drive/v3/files/file-1" {
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
writeTestJSON(w, driveFile{
|
||||
ID: "file-1",
|
||||
Name: "clip.mp4",
|
||||
MimeType: "video/mp4",
|
||||
Size: "1234",
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{ID: "g", APIBaseURL: srv.URL + "/drive/v3"})
|
||||
d.accessToken = "access"
|
||||
|
||||
link, err := d.StreamURL(context.Background(), "file-1")
|
||||
if err != nil {
|
||||
t.Fatalf("StreamURL() error = %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(link.URL, srv.URL+"/drive/v3/files/file-1?") {
|
||||
t.Fatalf("link URL = %q", link.URL)
|
||||
}
|
||||
if !strings.Contains(link.URL, "alt=media") {
|
||||
t.Fatalf("link URL missing alt=media: %q", link.URL)
|
||||
}
|
||||
if got := link.Headers.Get("Authorization"); got != "Bearer access" {
|
||||
t.Fatalf("link Authorization = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestRefreshesOnUnauthorized(t *testing.T) {
|
||||
var fileCalls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/renew":
|
||||
writeTestJSON(w, tokenResp{
|
||||
AccessToken: "new-access",
|
||||
RefreshToken: "new-refresh",
|
||||
})
|
||||
case "/drive/v3/files/file-1":
|
||||
fileCalls++
|
||||
if fileCalls == 1 {
|
||||
writeTestJSONStatus(w, http.StatusUnauthorized, apiErrorResp{Error: apiErrorBody{
|
||||
Code: http.StatusUnauthorized,
|
||||
Message: "Invalid Credentials",
|
||||
}})
|
||||
return
|
||||
}
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer new-access" {
|
||||
t.Fatalf("Authorization after refresh = %q", got)
|
||||
}
|
||||
writeTestJSON(w, driveFile{ID: "file-1", Name: "clip.mp4", Size: "1"})
|
||||
default:
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "g",
|
||||
RefreshToken: "old-refresh",
|
||||
UseOnlineAPI: true,
|
||||
RenewAPIURL: srv.URL + "/renew",
|
||||
APIBaseURL: srv.URL + "/drive/v3",
|
||||
})
|
||||
d.accessToken = "old-access"
|
||||
|
||||
if _, err := d.Stat(context.Background(), "file-1"); err != nil {
|
||||
t.Fatalf("Stat() error = %v", err)
|
||||
}
|
||||
if fileCalls != 2 {
|
||||
t.Fatalf("fileCalls = %d", fileCalls)
|
||||
}
|
||||
if d.accessToken != "new-access" || d.refreshToken != "new-refresh" {
|
||||
t.Fatalf("tokens not refreshed: access=%q refresh=%q", d.accessToken, d.refreshToken)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestJSON(w http.ResponseWriter, v any) {
|
||||
writeTestJSONStatus(w, http.StatusOK, v)
|
||||
}
|
||||
|
||||
func writeTestJSONStatus(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package googledrive
|
||||
|
||||
import "time"
|
||||
|
||||
type tokenResp struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
RefreshToken string `json:"refresh_token"`
|
||||
ExpiresIn int64 `json:"expires_in"`
|
||||
Error string `json:"error"`
|
||||
ErrorDescription string `json:"error_description"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
type filesResp struct {
|
||||
NextPageToken string `json:"nextPageToken"`
|
||||
Files []driveFile `json:"files"`
|
||||
Error apiErrorBody `json:"error"`
|
||||
}
|
||||
|
||||
type driveFile struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
MimeType string `json:"mimeType"`
|
||||
ModifiedTime time.Time `json:"modifiedTime"`
|
||||
CreatedTime time.Time `json:"createdTime"`
|
||||
Size string `json:"size"`
|
||||
ThumbnailLink string `json:"thumbnailLink"`
|
||||
MD5Checksum string `json:"md5Checksum"`
|
||||
SHA1Checksum string `json:"sha1Checksum"`
|
||||
SHA256Checksum string `json:"sha256Checksum"`
|
||||
Shortcut struct {
|
||||
TargetID string `json:"targetId"`
|
||||
TargetMimeType string `json:"targetMimeType"`
|
||||
} `json:"shortcutDetails"`
|
||||
}
|
||||
|
||||
type apiErrorResp struct {
|
||||
Error apiErrorBody `json:"error"`
|
||||
}
|
||||
|
||||
type apiErrorBody struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Errors []struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
} `json:"errors"`
|
||||
}
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
// Drive 是多家网盘统一抽象。上层不区分盘,只区分 Kind。
|
||||
type Drive interface {
|
||||
// Kind 返回驱动代号:"quark" / "p115" / "pikpak" / "wopan" / "onedrive"
|
||||
// Kind 返回驱动代号:"quark" / "p115" / "pikpak" / "wopan" / "onedrive" / "googledrive" / "localstorage"
|
||||
Kind() string
|
||||
|
||||
// ID 返回该盘在 catalog 中的唯一标识
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
// Package localstorage exposes an existing server-side directory as a Drive.
|
||||
package localstorage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
const Kind = "localstorage"
|
||||
|
||||
type Config struct {
|
||||
ID string
|
||||
RootPath string
|
||||
}
|
||||
|
||||
type Driver struct {
|
||||
id string
|
||||
rootPath string
|
||||
}
|
||||
|
||||
func New(c Config) *Driver {
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
rootPath: c.RootPath,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Kind() string { return Kind }
|
||||
|
||||
func (d *Driver) ID() string { return d.id }
|
||||
|
||||
func (d *Driver) RootID() string { return "/" }
|
||||
|
||||
func (d *Driver) Init(context.Context) error {
|
||||
root, err := d.root()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("localstorage: stat root: %w", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("localstorage: root is not a directory: %s", root)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error) {
|
||||
dir, rel, err := d.pathForID(dirID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]drives.Entry, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Symlinks can escape the configured root or create cycles. Keep the
|
||||
// local storage drive predictable by scanning real files/directories only.
|
||||
if entry.Type()&os.ModeSymlink != 0 {
|
||||
continue
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !info.IsDir() && !info.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
childRel := joinRel(rel, entry.Name())
|
||||
out = append(out, drives.Entry{
|
||||
ID: encodeRel(childRel),
|
||||
Name: entry.Name(),
|
||||
Size: sizeForEntry(info),
|
||||
IsDir: info.IsDir(),
|
||||
ParentID: idForRel(rel),
|
||||
ModTime: info.ModTime(),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
|
||||
p, rel, err := d.pathForID(fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &drives.Entry{
|
||||
ID: idForRel(rel),
|
||||
Name: filepath.Base(p),
|
||||
Size: sizeForEntry(info),
|
||||
IsDir: info.IsDir(),
|
||||
ParentID: idForRel(parentRel(rel)),
|
||||
ModTime: info.ModTime(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
p, _, err := d.pathForID(fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.IsDir() || !info.Mode().IsRegular() || info.Size() <= 0 {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return &drives.StreamLink{
|
||||
URL: p,
|
||||
Expires: time.Now().Add(24 * time.Hour),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) root() (string, error) {
|
||||
raw := strings.TrimSpace(d.rootPath)
|
||||
if raw == "" {
|
||||
return "", errors.New("localstorage: empty path")
|
||||
}
|
||||
raw = os.ExpandEnv(raw)
|
||||
if strings.HasPrefix(raw, "~") {
|
||||
if home, err := os.UserHomeDir(); err == nil && home != "" {
|
||||
switch {
|
||||
case raw == "~":
|
||||
raw = home
|
||||
case strings.HasPrefix(raw, "~/") || strings.HasPrefix(raw, `~\`):
|
||||
raw = filepath.Join(home, raw[2:])
|
||||
}
|
||||
}
|
||||
}
|
||||
return filepath.Abs(raw)
|
||||
}
|
||||
|
||||
func (d *Driver) pathForID(id string) (string, string, error) {
|
||||
root, err := d.root()
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
rel, err := decodeRel(id)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if rel == "" {
|
||||
return root, "", nil
|
||||
}
|
||||
p, err := filepath.Abs(filepath.Join(root, filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if p != root && !strings.HasPrefix(p, root+string(os.PathSeparator)) {
|
||||
return "", "", errors.New("localstorage: path escapes root")
|
||||
}
|
||||
return p, rel, nil
|
||||
}
|
||||
|
||||
func decodeRel(id string) (string, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" || id == "/" {
|
||||
return "", nil
|
||||
}
|
||||
raw, err := base64.RawURLEncoding.DecodeString(id)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("localstorage: invalid file id: %w", err)
|
||||
}
|
||||
rel := filepath.ToSlash(filepath.Clean(filepath.FromSlash(string(raw))))
|
||||
if rel == "." {
|
||||
return "", nil
|
||||
}
|
||||
if strings.HasPrefix(rel, "../") || rel == ".." || strings.HasPrefix(rel, "/") {
|
||||
return "", errors.New("localstorage: invalid relative path")
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
|
||||
func encodeRel(rel string) string {
|
||||
rel = filepath.ToSlash(filepath.Clean(filepath.FromSlash(rel)))
|
||||
if rel == "." || rel == "" {
|
||||
return "/"
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(rel))
|
||||
}
|
||||
|
||||
func idForRel(rel string) string {
|
||||
if rel == "" {
|
||||
return "/"
|
||||
}
|
||||
return encodeRel(rel)
|
||||
}
|
||||
|
||||
func joinRel(parent, name string) string {
|
||||
if parent == "" {
|
||||
return filepath.ToSlash(name)
|
||||
}
|
||||
return filepath.ToSlash(filepath.Join(filepath.FromSlash(parent), name))
|
||||
}
|
||||
|
||||
func parentRel(rel string) string {
|
||||
if rel == "" {
|
||||
return ""
|
||||
}
|
||||
parent := filepath.ToSlash(filepath.Dir(filepath.FromSlash(rel)))
|
||||
if parent == "." {
|
||||
return ""
|
||||
}
|
||||
return parent
|
||||
}
|
||||
|
||||
func sizeForEntry(info os.FileInfo) int64 {
|
||||
if info == nil || info.IsDir() {
|
||||
return 0
|
||||
}
|
||||
return info.Size()
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
@@ -0,0 +1,119 @@
|
||||
package localstorage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/scanner"
|
||||
)
|
||||
|
||||
func TestListEncodesRelativePathsAndStreamURLResolvesFile(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
sub := filepath.Join(root, "clips")
|
||||
if err := os.MkdirAll(sub, 0o755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
videoPath := filepath.Join(sub, "sample.mp4")
|
||||
if err := os.WriteFile(videoPath, []byte("video"), 0o644); err != nil {
|
||||
t.Fatalf("write video: %v", err)
|
||||
}
|
||||
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
if err := drv.Init(context.Background()); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
rootEntries, err := drv.List(context.Background(), drv.RootID())
|
||||
if err != nil {
|
||||
t.Fatalf("list root: %v", err)
|
||||
}
|
||||
if len(rootEntries) != 1 || !rootEntries[0].IsDir {
|
||||
t.Fatalf("root entries = %#v, want one directory", rootEntries)
|
||||
}
|
||||
if strings.Contains(rootEntries[0].ID, "/") {
|
||||
t.Fatalf("encoded dir id contains slash: %q", rootEntries[0].ID)
|
||||
}
|
||||
|
||||
fileEntries, err := drv.List(context.Background(), rootEntries[0].ID)
|
||||
if err != nil {
|
||||
t.Fatalf("list subdir: %v", err)
|
||||
}
|
||||
if len(fileEntries) != 1 || fileEntries[0].Name != "sample.mp4" {
|
||||
t.Fatalf("file entries = %#v, want sample.mp4", fileEntries)
|
||||
}
|
||||
if strings.Contains(fileEntries[0].ID, "/") {
|
||||
t.Fatalf("encoded file id contains slash: %q", fileEntries[0].ID)
|
||||
}
|
||||
|
||||
link, err := drv.StreamURL(context.Background(), fileEntries[0].ID)
|
||||
if err != nil {
|
||||
t.Fatalf("stream url: %v", err)
|
||||
}
|
||||
if link.URL != videoPath {
|
||||
t.Fatalf("url = %q, want %q", link.URL, videoPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLRejectsEscapingID(t *testing.T) {
|
||||
drv := New(Config{ID: "local", RootPath: t.TempDir()})
|
||||
escaped := base64.RawURLEncoding.EncodeToString([]byte("../secret.mp4"))
|
||||
|
||||
_, err := drv.StreamURL(context.Background(), escaped)
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid relative path") {
|
||||
t.Fatalf("error = %v, want invalid relative path", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitRequiresExistingDirectory(t *testing.T) {
|
||||
drv := New(Config{ID: "local", RootPath: filepath.Join(t.TempDir(), "missing")})
|
||||
|
||||
err := drv.Init(context.Background())
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "stat root") {
|
||||
t.Fatalf("error = %v, want stat root failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerPersistsLocalStorageVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(root, "collection"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir collection: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "collection", "clip.mp4"), []byte("video"), 0o644); err != nil {
|
||||
t.Fatalf("write video: %v", err)
|
||||
}
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
drv := New(Config{ID: "local", RootPath: root})
|
||||
sc := scanner.New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
stats, err := sc.Run(ctx, drv.RootID())
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Added != 1 {
|
||||
t.Fatalf("added = %d, want 1", stats.Added)
|
||||
}
|
||||
|
||||
fileID := encodeRel("collection/clip.mp4")
|
||||
got, err := cat.GetVideo(ctx, Kind+"-local-"+fileID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DriveID != "local" || got.FileID != fileID || got.Category != "collection" {
|
||||
t.Fatalf("video = %#v, want local drive video in collection", got)
|
||||
}
|
||||
}
|
||||
@@ -3,14 +3,19 @@ package onedrive
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-resty/resty/v2"
|
||||
@@ -18,8 +23,17 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
maxSmallUploadSize = 250 * 1024 * 1024
|
||||
defaultRenewAPIURL = "https://api.oplist.org/onedrive/renewapi"
|
||||
maxSmallUploadSize = 250 * 1024 * 1024
|
||||
defaultUploadSessionChunk = 10 * 1024 * 1024
|
||||
uploadSessionRetryAttempts = 3
|
||||
defaultRenewAPIURL = "https://api.oplist.org/onedrive/renewapi"
|
||||
onedriveListCooldown = 5 * time.Minute
|
||||
onedriveListInterval = 1 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
smallUploadThreshold = int64(maxSmallUploadSize)
|
||||
uploadSessionChunk = int64(defaultUploadSessionChunk)
|
||||
)
|
||||
|
||||
type Driver struct {
|
||||
@@ -34,6 +48,11 @@ type Driver struct {
|
||||
renewAPIURL string
|
||||
client *resty.Client
|
||||
onTokenUpdate func(access, refresh string)
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
listInterval time.Duration
|
||||
listCooldown time.Duration
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -85,6 +104,8 @@ func New(c Config) *Driver {
|
||||
client: resty.New().
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
listInterval: onedriveListInterval,
|
||||
listCooldown: onedriveListCooldown,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,10 +127,16 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
|
||||
if dirID == "" {
|
||||
dirID = d.rootID
|
||||
}
|
||||
d.listMu.Lock()
|
||||
defer d.listMu.Unlock()
|
||||
|
||||
nextLink := d.childrenURL(dirID)
|
||||
first := true
|
||||
out := make([]drives.Entry, 0)
|
||||
for nextLink != "" {
|
||||
if err := d.waitForListSlotLocked(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var resp filesResp
|
||||
err := d.request(ctx, nextLink, http.MethodGet, func(req *resty.Request) {
|
||||
if first {
|
||||
@@ -120,6 +147,19 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
|
||||
}
|
||||
}, &resp)
|
||||
if err != nil {
|
||||
if wait, ok := drives.RateLimitRetryAfter(err); ok {
|
||||
if wait <= 0 {
|
||||
wait = d.listCooldown
|
||||
if wait <= 0 {
|
||||
wait = onedriveListCooldown
|
||||
}
|
||||
}
|
||||
log.Printf("[onedrive] list cooling down drive=%s dir=%s cooldown=%s err=%v", d.id, dirID, wait, err)
|
||||
if err := sleepContext(ctx, wait); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("onedrive list: %w", err)
|
||||
}
|
||||
for _, item := range resp.Value {
|
||||
@@ -131,6 +171,36 @@ func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *Driver) waitForListSlotLocked(ctx context.Context) error {
|
||||
if d.listInterval <= 0 || d.lastListAt.IsZero() {
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
next := d.lastListAt.Add(d.listInterval)
|
||||
now := time.Now()
|
||||
if now.Before(next) {
|
||||
if err := sleepContext(ctx, next.Sub(now)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, d time.Duration) error {
|
||||
if d <= 0 {
|
||||
return ctx.Err()
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
|
||||
var item graphItem
|
||||
if err := d.request(ctx, d.itemURL(fileID), http.MethodGet, nil, &item); err != nil {
|
||||
@@ -156,15 +226,49 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
res, err := d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return res.FileID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
parentID, name, err := d.normalizeUploadArgs(parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
threshold := smallUploadThreshold
|
||||
if threshold <= 0 {
|
||||
threshold = maxSmallUploadSize
|
||||
}
|
||||
if size <= threshold {
|
||||
return d.uploadSmallAndReportHash(ctx, parentID, name, r, size, threshold)
|
||||
}
|
||||
return d.uploadSessionAndReportHash(ctx, parentID, name, r, size)
|
||||
}
|
||||
|
||||
func (d *Driver) normalizeUploadArgs(parentID, name string, r io.Reader, size int64) (string, string, error) {
|
||||
if r == nil {
|
||||
return "", "", errors.New("onedrive upload: body is required")
|
||||
}
|
||||
if size < 0 {
|
||||
return "", "", fmt.Errorf("onedrive upload: invalid size %d", size)
|
||||
}
|
||||
if parentID == "" {
|
||||
parentID = d.rootID
|
||||
}
|
||||
if size > maxSmallUploadSize {
|
||||
return "", fmt.Errorf("onedrive upload: files over %d bytes require upload session", maxSmallUploadSize)
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return "", "", errors.New("onedrive upload: empty file name")
|
||||
}
|
||||
data, err := readSmallUpload(r)
|
||||
return parentID, name, nil
|
||||
}
|
||||
|
||||
func (d *Driver) uploadSmallAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size, limit int64) (UploadResult, error) {
|
||||
data, hash, actualSize, err := readSmallUpload(r, size, limit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return UploadResult{}, err
|
||||
}
|
||||
u := fmt.Sprintf("%s/items/%s:/%s:/content", d.driveBaseURL(), url.PathEscape(parentID), url.PathEscape(name))
|
||||
var item graphItem
|
||||
@@ -173,26 +277,159 @@ func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader,
|
||||
req.SetContentLength(true)
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("onedrive upload: %w", err)
|
||||
return UploadResult{}, fmt.Errorf("onedrive upload: %w", err)
|
||||
}
|
||||
if item.ID == "" {
|
||||
return "", errors.New("onedrive upload: empty item id")
|
||||
return UploadResult{}, errors.New("onedrive upload: empty item id")
|
||||
}
|
||||
return item.ID, nil
|
||||
return UploadResult{FileID: item.ID, Hash: hash, Size: actualSize}, nil
|
||||
}
|
||||
|
||||
func readSmallUpload(r io.Reader) ([]byte, error) {
|
||||
if r == nil {
|
||||
return nil, errors.New("onedrive upload: body is required")
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(r, maxSmallUploadSize+1))
|
||||
func (d *Driver) uploadSessionAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
session, err := d.createUploadSession(ctx, parentID, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("onedrive upload: read body: %w", err)
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if len(data) > maxSmallUploadSize {
|
||||
return nil, fmt.Errorf("onedrive upload: files over %d bytes require upload session", maxSmallUploadSize)
|
||||
if strings.TrimSpace(session.UploadURL) == "" {
|
||||
return UploadResult{}, errors.New("onedrive upload session: empty upload url")
|
||||
}
|
||||
return data, nil
|
||||
|
||||
chunkSize := uploadSessionChunk
|
||||
if chunkSize <= 0 {
|
||||
chunkSize = defaultUploadSessionChunk
|
||||
}
|
||||
buf := make([]byte, int(chunkSize))
|
||||
hasher := sha1.New()
|
||||
var finalItem graphItem
|
||||
var offset int64
|
||||
for offset < size {
|
||||
partSize := minInt64(chunkSize, size-offset)
|
||||
chunk := buf[:int(partSize)]
|
||||
n, err := io.ReadFull(r, chunk)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return UploadResult{}, fmt.Errorf("onedrive upload: size mismatch: declared %d, copied %d", size, offset+int64(n))
|
||||
}
|
||||
return UploadResult{}, fmt.Errorf("onedrive upload: read body: %w", err)
|
||||
}
|
||||
chunk = chunk[:n]
|
||||
_, _ = hasher.Write(chunk)
|
||||
item, err := d.putUploadSessionChunkWithRetry(ctx, session.UploadURL, offset, size, chunk)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if item != nil {
|
||||
finalItem = *item
|
||||
}
|
||||
offset += int64(n)
|
||||
}
|
||||
if finalItem.ID == "" {
|
||||
return UploadResult{}, errors.New("onedrive upload session: empty item id")
|
||||
}
|
||||
return UploadResult{
|
||||
FileID: finalItem.ID,
|
||||
Hash: hex.EncodeToString(hasher.Sum(nil)),
|
||||
Size: offset,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) createUploadSession(ctx context.Context, parentID, name string) (uploadSessionResp, error) {
|
||||
u := fmt.Sprintf("%s/items/%s:/%s:/createUploadSession", d.driveBaseURL(), url.PathEscape(parentID), url.PathEscape(name))
|
||||
body := map[string]any{
|
||||
"item": map[string]any{
|
||||
"@microsoft.graph.conflictBehavior": "rename",
|
||||
},
|
||||
}
|
||||
var out uploadSessionResp
|
||||
err := d.request(ctx, u, http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(body)
|
||||
}, &out)
|
||||
if err != nil {
|
||||
return uploadSessionResp{}, fmt.Errorf("onedrive upload session: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunkWithRetry(ctx context.Context, uploadURL string, start, total int64, data []byte) (*graphItem, error) {
|
||||
var last error
|
||||
for attempt := 0; attempt < uploadSessionRetryAttempts; attempt++ {
|
||||
if attempt > 0 {
|
||||
if err := sleepContext(ctx, time.Duration(attempt)*time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
item, retryable, err := d.putUploadSessionChunk(ctx, uploadURL, start, total, data)
|
||||
if err == nil {
|
||||
return item, nil
|
||||
}
|
||||
last = err
|
||||
if !retryable {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if last == nil {
|
||||
last = errors.New("onedrive upload session: retry attempts exhausted")
|
||||
}
|
||||
return nil, last
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunk(ctx context.Context, uploadURL string, start, total int64, data []byte) (*graphItem, bool, error) {
|
||||
end := start + int64(len(data)) - 1
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
req.ContentLength = int64(len(data))
|
||||
req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, total))
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK, http.StatusCreated:
|
||||
var item graphItem
|
||||
if err := json.NewDecoder(res.Body).Decode(&item); err != nil {
|
||||
return nil, false, fmt.Errorf("onedrive upload session: decode completed item: %w", err)
|
||||
}
|
||||
return &item, false, nil
|
||||
case http.StatusAccepted:
|
||||
return nil, false, nil
|
||||
default:
|
||||
body, _ := io.ReadAll(io.LimitReader(res.Body, 4096))
|
||||
err := fmt.Errorf("onedrive upload session: status=%d body=%s", res.StatusCode, strings.TrimSpace(string(body)))
|
||||
retryable := res.StatusCode == http.StatusTooManyRequests || (res.StatusCode >= 500 && res.StatusCode <= 504)
|
||||
return nil, retryable, err
|
||||
}
|
||||
}
|
||||
|
||||
func readSmallUpload(r io.Reader, declaredSize, limit int64) ([]byte, string, int64, error) {
|
||||
if r == nil {
|
||||
return nil, "", 0, errors.New("onedrive upload: body is required")
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = maxSmallUploadSize
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(r, limit+1))
|
||||
if err != nil {
|
||||
return nil, "", 0, fmt.Errorf("onedrive upload: read body: %w", err)
|
||||
}
|
||||
if int64(len(data)) > limit {
|
||||
return nil, "", 0, fmt.Errorf("onedrive upload: files over %d bytes require upload session", limit)
|
||||
}
|
||||
if declaredSize >= 0 && int64(len(data)) != declaredSize {
|
||||
return nil, "", 0, fmt.Errorf("onedrive upload: size mismatch: declared %d, copied %d", declaredSize, len(data))
|
||||
}
|
||||
sum := sha1.Sum(data)
|
||||
return data, hex.EncodeToString(sum[:]), int64(len(data)), nil
|
||||
}
|
||||
|
||||
func minInt64(a, b int64) int64 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
@@ -245,6 +482,25 @@ func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, er
|
||||
return item.ID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("onedrive rename: empty file id")
|
||||
}
|
||||
newName = strings.TrimSpace(newName)
|
||||
if newName == "" {
|
||||
return errors.New("onedrive rename: empty new name")
|
||||
}
|
||||
var item graphItem
|
||||
err := d.request(ctx, d.itemURL(fileID), http.MethodPatch, func(req *resty.Request) {
|
||||
req.SetBody(map[string]string{"name": newName})
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return fmt.Errorf("onedrive rename: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) request(ctx context.Context, rawURL, method string, configure func(*resty.Request), out any) error {
|
||||
return d.requestOnce(ctx, rawURL, method, configure, out, true)
|
||||
}
|
||||
@@ -265,7 +521,7 @@ func (d *Driver) requestOnce(ctx context.Context, rawURL, method string, configu
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isRateLimitResponse(res, graphErr.Error.Code) {
|
||||
if isRateLimitResponse(res, graphErr.Error.Code, graphErr.Error.Message) {
|
||||
return onedriveRateLimitError(res, graphErr.Error.Message)
|
||||
}
|
||||
if graphErr.Error.Code != "" {
|
||||
@@ -327,11 +583,54 @@ func (d *Driver) refresh(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func isRateLimitResponse(res *resty.Response, code string) bool {
|
||||
if code == "TooManyRequests" || code == "activityLimitReached" {
|
||||
func isRateLimitResponse(res *resty.Response, code, message string) bool {
|
||||
if isRateLimitCode(code) || isRateLimitMessage(message) {
|
||||
return true
|
||||
}
|
||||
return res != nil && res.StatusCode() == http.StatusTooManyRequests
|
||||
if res == nil {
|
||||
return false
|
||||
}
|
||||
if res.StatusCode() == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if res.Header().Get("Retry-After") == "" {
|
||||
return false
|
||||
}
|
||||
switch res.StatusCode() {
|
||||
case http.StatusServiceUnavailable, http.StatusGatewayTimeout:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isRateLimitCode(code string) bool {
|
||||
normalized := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(code), "_", ""))
|
||||
normalized = strings.ReplaceAll(normalized, "-", "")
|
||||
switch normalized {
|
||||
case "toomanyrequests",
|
||||
"activitylimitreached",
|
||||
"throttledrequest",
|
||||
"requestthrottled",
|
||||
"resourcethrottled",
|
||||
"applicationthrottled",
|
||||
"tenantthrottled":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isRateLimitMessage(message string) bool {
|
||||
text := strings.ToLower(strings.TrimSpace(message))
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "throttl") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "activity limit") ||
|
||||
strings.Contains(text, "temporarily blocked")
|
||||
}
|
||||
|
||||
func onedriveRateLimitError(res *resty.Response, message string) error {
|
||||
|
||||
@@ -2,6 +2,8 @@ package onedrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
@@ -199,7 +201,7 @@ func TestGraph429ReturnsRateLimitErrorWithRetryAfter(t *testing.T) {
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
|
||||
_, err := d.List(context.Background(), "root")
|
||||
_, err := d.StreamURL(context.Background(), "file-id")
|
||||
if err == nil {
|
||||
t.Fatal("list succeeded, want rate limit error")
|
||||
}
|
||||
@@ -212,6 +214,92 @@ func TestGraph429ReturnsRateLimitErrorWithRetryAfter(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphThrottleMessageReturnsRateLimitError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{
|
||||
"error": map[string]any{
|
||||
"code": "generalException",
|
||||
"message": "The request has been throttled. Please try again later.",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("write json: %v", err)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
|
||||
_, err := d.StreamURL(context.Background(), "file-id")
|
||||
if err == nil {
|
||||
t.Fatal("list succeeded, want rate limit error")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListCoolsDownAndRetriesOneDriveRateLimit(t *testing.T) {
|
||||
var calls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1.0/me/drive/items/root/children" {
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
|
||||
}
|
||||
calls++
|
||||
if calls == 1 {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{
|
||||
"error": map[string]any{
|
||||
"code": "TooManyRequests",
|
||||
"message": "throttled",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("write json: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
writeJSON(t, w, map[string]any{
|
||||
"value": []map[string]any{
|
||||
{
|
||||
"id": "file-id",
|
||||
"name": "demo.mp4",
|
||||
"size": 100,
|
||||
"file": map[string]any{"mimeType": "video/mp4"},
|
||||
},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
d.listInterval = 0
|
||||
d.listCooldown = time.Millisecond
|
||||
|
||||
got, err := d.List(context.Background(), "root")
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("calls = %d, want retry after rate limit", calls)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID != "file-id" {
|
||||
t.Fatalf("entries = %#v, want retried file", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatAndStreamURLUseDriveItemMetadata(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
@@ -320,6 +408,36 @@ func TestEnsureDirCreatesMissingFolders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenamePatchesDriveItemName(t *testing.T) {
|
||||
var body map[string]string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPatch || r.URL.EscapedPath() != "/v1.0/me/drive/items/file-id" {
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
|
||||
}
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
t.Fatalf("authorization = %q, want bearer token", got)
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode body: %v", err)
|
||||
}
|
||||
writeJSON(t, w, map[string]any{"id": "file-id", "name": "new name.mp4"})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
if err := d.Rename(context.Background(), "file-id", "new name.mp4"); err != nil {
|
||||
t.Fatalf("rename: %v", err)
|
||||
}
|
||||
if body["name"] != "new name.mp4" {
|
||||
t.Fatalf("rename body = %#v, want new name", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadSmallFileReturnsNewItemID(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
@@ -358,6 +476,86 @@ func TestUploadSmallFileReturnsNewItemID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadLargeFileUsesUploadSessionAndReportsHash(t *testing.T) {
|
||||
oldThreshold := smallUploadThreshold
|
||||
oldChunk := uploadSessionChunk
|
||||
smallUploadThreshold = 8
|
||||
uploadSessionChunk = 4
|
||||
t.Cleanup(func() {
|
||||
smallUploadThreshold = oldThreshold
|
||||
uploadSessionChunk = oldChunk
|
||||
})
|
||||
|
||||
body := "0123456789abc"
|
||||
var ranges []string
|
||||
var chunks []string
|
||||
var createdSession bool
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost && r.URL.EscapedPath() == "/v1.0/me/drive/items/parent-id:/big.mp4:/createUploadSession":
|
||||
createdSession = true
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
t.Fatalf("authorization = %q, want bearer token", got)
|
||||
}
|
||||
writeJSON(t, w, map[string]any{"uploadUrl": srv.URL + "/upload-session"})
|
||||
case r.Method == http.MethodPut && r.URL.Path == "/upload-session":
|
||||
ranges = append(ranges, r.Header.Get("Content-Range"))
|
||||
data, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read chunk: %v", err)
|
||||
}
|
||||
chunks = append(chunks, string(data))
|
||||
if len(ranges) < 4 {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
if _, err := w.Write([]byte(`{"nextExpectedRanges":["0-"]}`)); err != nil {
|
||||
t.Fatalf("write accepted: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{"id": "uploaded-big-id"}); err != nil {
|
||||
t.Fatalf("write final item: %v", err)
|
||||
}
|
||||
default:
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
got, err := d.UploadAndReportHash(context.Background(), "parent-id", "big.mp4", strings.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
if !createdSession {
|
||||
t.Fatal("createUploadSession was not called")
|
||||
}
|
||||
wantRanges := []string{
|
||||
"bytes 0-3/13",
|
||||
"bytes 4-7/13",
|
||||
"bytes 8-11/13",
|
||||
"bytes 12-12/13",
|
||||
}
|
||||
if strings.Join(ranges, "|") != strings.Join(wantRanges, "|") {
|
||||
t.Fatalf("ranges = %#v, want %#v", ranges, wantRanges)
|
||||
}
|
||||
if strings.Join(chunks, "") != body {
|
||||
t.Fatalf("uploaded chunks = %q, want %q", strings.Join(chunks, ""), body)
|
||||
}
|
||||
sum := sha1.Sum([]byte(body))
|
||||
if got.FileID != "uploaded-big-id" || got.Size != int64(len(body)) || got.Hash != hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("upload result = %#v, want file id/hash/size for body", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadRefreshesExpiredTokenAndReplaysBody(t *testing.T) {
|
||||
var uploadAttempts int
|
||||
var tokenRefreshes int
|
||||
|
||||
@@ -82,3 +82,13 @@ type filesResp struct {
|
||||
Value []graphItem `json:"value"`
|
||||
NextLink string `json:"@odata.nextLink"`
|
||||
}
|
||||
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
Hash string
|
||||
Size int64
|
||||
}
|
||||
|
||||
type uploadSessionResp struct {
|
||||
UploadURL string `json:"uploadUrl"`
|
||||
}
|
||||
|
||||
@@ -199,9 +199,8 @@ func (d *Driver) refreshCaptchaToken(ctx context.Context, action string, meta ma
|
||||
|
||||
// refreshCaptchaTokenOnce 调 /v1/shield/captcha/init 申请新 captcha token。
|
||||
//
|
||||
// 如果 retry=true 且服务端返回 4002(captcha_token expired,意味着 body 里
|
||||
// 携带的 d.captchaToken 已经过期),就清空缓存的 captcha_token 后再调一次;
|
||||
// 这次 body 里 captcha_token 为空,服务端会直接发一个新的。这覆盖
|
||||
// 如果 retry=true 且服务端返回 captcha 失效错误(4002 或 9),就清空缓存的
|
||||
// captcha_token 后再调一次;这次 body 里 captcha_token 为空,服务端会直接发一个新的。这覆盖
|
||||
// driver 重启后 Init() 用持久化的旧 captcha_token 调 captcha init 失败的
|
||||
// 场景。
|
||||
func (d *Driver) refreshCaptchaTokenOnce(ctx context.Context, action string, meta map[string]string, retry bool) error {
|
||||
@@ -230,7 +229,7 @@ func (d *Driver) refreshCaptchaTokenOnce(ctx context.Context, action string, met
|
||||
return err
|
||||
}
|
||||
if e.isError() {
|
||||
if retry && e.ErrorCode == 4002 && d.captchaToken != "" {
|
||||
if retry && isCaptchaTokenRejectedCode(e.ErrorCode) && d.captchaToken != "" {
|
||||
d.captchaToken = ""
|
||||
return d.refreshCaptchaTokenOnce(ctx, action, meta, false)
|
||||
}
|
||||
|
||||
@@ -96,6 +96,65 @@ func TestRefreshCaptchaTokenRecoversFrom4002(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefreshCaptchaTokenRecoversFrom9 覆盖 PikPak 返回 error_code=9
|
||||
// captcha_invalid 的路径。这个错误和 4002 一样表示当前 captcha_token 已被拒绝;
|
||||
// 重试 captcha/init 前必须先清空旧 token,否则服务端会继续拒绝。
|
||||
func TestRefreshCaptchaTokenRecoversFrom9(t *testing.T) {
|
||||
var calls int32
|
||||
type bodyShape struct {
|
||||
CaptchaToken string `json:"captcha_token"`
|
||||
}
|
||||
var (
|
||||
firstBody bodyShape
|
||||
secondBody bodyShape
|
||||
)
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v1/shield/captcha/init", func(w http.ResponseWriter, r *http.Request) {
|
||||
n := atomic.AddInt32(&calls, 1)
|
||||
switch n {
|
||||
case 1:
|
||||
_ = json.NewDecoder(r.Body).Decode(&firstBody)
|
||||
writeErrorJSON(w, `{
|
||||
"error_code": 9,
|
||||
"error": "captcha_invalid",
|
||||
"error_description": "Verification code is invalid"
|
||||
}`)
|
||||
case 2:
|
||||
_ = json.NewDecoder(r.Body).Decode(&secondBody)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"captcha_token": "fresh-captcha",
|
||||
"expires_in": 300
|
||||
}`))
|
||||
default:
|
||||
t.Errorf("unexpected captcha init call #%d", n)
|
||||
}
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
d.captchaToken = "expired-captcha"
|
||||
|
||||
if err := d.refreshCaptchaTokenAtLogin(context.Background(), "GET:/drive/v1/files", "user-1"); err != nil {
|
||||
t.Fatalf("refreshCaptchaTokenAtLogin: %v", err)
|
||||
}
|
||||
|
||||
if got := atomic.LoadInt32(&calls); got != 2 {
|
||||
t.Fatalf("captcha init called %d times, want 2", got)
|
||||
}
|
||||
if firstBody.CaptchaToken != "expired-captcha" {
|
||||
t.Errorf("first body captcha_token = %q, want \"expired-captcha\"", firstBody.CaptchaToken)
|
||||
}
|
||||
if secondBody.CaptchaToken != "" {
|
||||
t.Errorf("second body captcha_token = %q, want empty (cleared after error_code=9)", secondBody.CaptchaToken)
|
||||
}
|
||||
if d.captchaToken != "fresh-captcha" {
|
||||
t.Errorf("d.captchaToken = %q, want \"fresh-captcha\"", d.captchaToken)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefreshCaptchaTokenDoesNotLoopOn4002WithEmptyToken 防止退化成无限重试:
|
||||
// 如果调用方一开始 captchaToken 就是空,又遇上 4002,不应该再清空一次重试
|
||||
// (清空后还是空,再发会拿到同样的错误),应该直接返回错误让上层处理。
|
||||
@@ -121,6 +180,141 @@ func TestRefreshCaptchaTokenDoesNotLoopOn4002WithEmptyToken(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitWithRefreshTokenDoesNotSendPersistedCaptchaToken(t *testing.T) {
|
||||
var captchaCalls int32
|
||||
var captchaBody struct {
|
||||
CaptchaToken string `json:"captcha_token"`
|
||||
}
|
||||
var persisted struct {
|
||||
access, refresh, captcha string
|
||||
calls int
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v1/auth/token", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"access_token": "fresh-access",
|
||||
"refresh_token": "fresh-refresh",
|
||||
"sub": "user-1"
|
||||
}`))
|
||||
})
|
||||
mux.HandleFunc("/v1/shield/captcha/init", func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&captchaCalls, 1)
|
||||
_ = json.NewDecoder(r.Body).Decode(&captchaBody)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"captcha_token": "fresh-captcha",
|
||||
"expires_in": 300
|
||||
}`))
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
d.captchaToken = "persisted-stale-captcha"
|
||||
d.onTokenUpdate = func(access, refresh, captcha, deviceID string) {
|
||||
persisted.access = access
|
||||
persisted.refresh = refresh
|
||||
persisted.captcha = captcha
|
||||
persisted.calls++
|
||||
}
|
||||
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
|
||||
if got := atomic.LoadInt32(&captchaCalls); got != 1 {
|
||||
t.Fatalf("captcha init calls = %d, want 1", got)
|
||||
}
|
||||
if captchaBody.CaptchaToken != "" {
|
||||
t.Errorf("captcha init body captcha_token = %q, want empty", captchaBody.CaptchaToken)
|
||||
}
|
||||
if d.captchaToken != "fresh-captcha" {
|
||||
t.Errorf("d.captchaToken = %q, want \"fresh-captcha\"", d.captchaToken)
|
||||
}
|
||||
if persisted.access != "fresh-access" || persisted.refresh != "fresh-refresh" || persisted.captcha != "fresh-captcha" {
|
||||
t.Errorf("persisted tokens = (%q, %q, %q), want fresh values", persisted.access, persisted.refresh, persisted.captcha)
|
||||
}
|
||||
if persisted.calls < 2 {
|
||||
t.Errorf("persist callback calls = %d, want at least 2 (clear stale + persist fresh)", persisted.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitFallsBackToLoginWhenRefreshReturnsCaptchaInvalid(t *testing.T) {
|
||||
var (
|
||||
tokenCalls int32
|
||||
captchaCalls int32
|
||||
signinCalls int32
|
||||
)
|
||||
var signinBody struct {
|
||||
CaptchaToken string `json:"captcha_token"`
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v1/auth/token", func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&tokenCalls, 1)
|
||||
writeErrorJSON(w, `{
|
||||
"error_code": 4002,
|
||||
"error": "captcha_invalid",
|
||||
"error_description": "Code(4002) - captcha_token expired"
|
||||
}`)
|
||||
})
|
||||
mux.HandleFunc("/v1/shield/captcha/init", func(w http.ResponseWriter, r *http.Request) {
|
||||
n := atomic.AddInt32(&captchaCalls, 1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch n {
|
||||
case 1:
|
||||
_, _ = w.Write([]byte(`{
|
||||
"captcha_token": "login-captcha",
|
||||
"expires_in": 300
|
||||
}`))
|
||||
case 2:
|
||||
_, _ = w.Write([]byte(`{
|
||||
"captcha_token": "files-captcha",
|
||||
"expires_in": 300
|
||||
}`))
|
||||
default:
|
||||
t.Errorf("unexpected captcha init call #%d", n)
|
||||
}
|
||||
})
|
||||
mux.HandleFunc("/v1/auth/signin", func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&signinCalls, 1)
|
||||
_ = json.NewDecoder(r.Body).Decode(&signinBody)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"access_token": "login-access",
|
||||
"refresh_token": "login-refresh",
|
||||
"sub": "user-1"
|
||||
}`))
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
d.captchaToken = "persisted-stale-captcha"
|
||||
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
|
||||
if got := atomic.LoadInt32(&tokenCalls); got != 1 {
|
||||
t.Fatalf("token refresh calls = %d, want 1", got)
|
||||
}
|
||||
if got := atomic.LoadInt32(&signinCalls); got != 1 {
|
||||
t.Fatalf("signin calls = %d, want 1", got)
|
||||
}
|
||||
if got := atomic.LoadInt32(&captchaCalls); got != 2 {
|
||||
t.Fatalf("captcha init calls = %d, want 2 (login + post-login files action)", got)
|
||||
}
|
||||
if signinBody.CaptchaToken != "login-captcha" {
|
||||
t.Errorf("signin captcha_token = %q, want \"login-captcha\"", signinBody.CaptchaToken)
|
||||
}
|
||||
if d.accessToken != "login-access" || d.refreshToken != "login-refresh" || d.captchaToken != "files-captcha" {
|
||||
t.Errorf("driver tokens = (%q, %q, %q), want login/files tokens", d.accessToken, d.refreshToken, d.captchaToken)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRequestOnceRecoversFrom4002OnAPICall 验证一个普通 API 调用收到 4002
|
||||
// 时,requestOnce 会先清空 captchaToken、再走 captcha 刷新,最后用新 token
|
||||
// 重试请求,最终成功返回。
|
||||
@@ -196,6 +390,76 @@ func TestRequestOnceRecoversFrom4002OnAPICall(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRequestOnceRecoversFrom9OnAPICall 验证普通 API 调用收到 error_code=9
|
||||
// 时,会先清空旧 captchaToken,再刷新 captcha 并重试原请求。
|
||||
func TestRequestOnceRecoversFrom9OnAPICall(t *testing.T) {
|
||||
var (
|
||||
filesCalls int32
|
||||
captchaCalls int32
|
||||
)
|
||||
type capturedFiles struct {
|
||||
captchaHeader string
|
||||
}
|
||||
var firstFiles, secondFiles capturedFiles
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
n := atomic.AddInt32(&filesCalls, 1)
|
||||
switch n {
|
||||
case 1:
|
||||
firstFiles.captchaHeader = r.Header.Get("X-Captcha-Token")
|
||||
writeErrorJSON(w, `{
|
||||
"error_code": 9,
|
||||
"error": "captcha_invalid",
|
||||
"error_description": "Verification code is invalid"
|
||||
}`)
|
||||
case 2:
|
||||
secondFiles.captchaHeader = r.Header.Get("X-Captcha-Token")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"files": [], "next_page_token": ""}`))
|
||||
default:
|
||||
t.Errorf("unexpected /drive/v1/files call #%d", n)
|
||||
}
|
||||
})
|
||||
mux.HandleFunc("/v1/shield/captcha/init", func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&captchaCalls, 1)
|
||||
var body struct {
|
||||
CaptchaToken string `json:"captcha_token"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
if body.CaptchaToken != "" {
|
||||
t.Errorf("captcha init body captcha_token = %q, want empty (error_code=9 path should clear cache)", body.CaptchaToken)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"captcha_token": "fresh-captcha", "expires_in": 300}`))
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
d.captchaToken = "expired-captcha"
|
||||
|
||||
if _, err := d.List(context.Background(), "any-parent"); err != nil {
|
||||
t.Fatalf("List: %v", err)
|
||||
}
|
||||
|
||||
if got := atomic.LoadInt32(&filesCalls); got != 2 {
|
||||
t.Fatalf("/drive/v1/files calls = %d, want 2 (initial + retry)", got)
|
||||
}
|
||||
if got := atomic.LoadInt32(&captchaCalls); got != 1 {
|
||||
t.Fatalf("captcha init calls = %d, want 1", got)
|
||||
}
|
||||
if firstFiles.captchaHeader != "expired-captcha" {
|
||||
t.Errorf("first request X-Captcha-Token = %q, want \"expired-captcha\"", firstFiles.captchaHeader)
|
||||
}
|
||||
if secondFiles.captchaHeader != "fresh-captcha" {
|
||||
t.Errorf("retry X-Captcha-Token = %q, want \"fresh-captcha\"", secondFiles.captchaHeader)
|
||||
}
|
||||
if d.captchaToken != "fresh-captcha" {
|
||||
t.Errorf("d.captchaToken after recovery = %q, want \"fresh-captcha\"", d.captchaToken)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRequestOnceDoesNotRetryTwiceOn4002 验证 4002 恢复路径只重试一次;
|
||||
// 如果重试请求依然失败(哪怕是再来一个 4002),也不会再次进入恢复逻辑,
|
||||
// 而是把错误返回出去,避免无限循环。
|
||||
|
||||
@@ -121,9 +121,28 @@ func (d *Driver) ID() string { return d.id }
|
||||
func (d *Driver) RootID() string { return d.rootID }
|
||||
|
||||
func (d *Driver) Init(ctx context.Context) error {
|
||||
clearPersistedCaptcha := func() {
|
||||
if d.captchaToken == "" {
|
||||
return
|
||||
}
|
||||
d.captchaToken = ""
|
||||
d.persistTokens()
|
||||
}
|
||||
|
||||
if d.refreshToken != "" {
|
||||
if err := d.refresh(ctx, d.refreshToken); err != nil {
|
||||
return err
|
||||
if !IsCaptchaError(err) || d.username == "" || d.password == "" {
|
||||
return err
|
||||
}
|
||||
clearPersistedCaptcha()
|
||||
if err := d.login(ctx); err != nil {
|
||||
return fmt.Errorf("pikpak refresh captcha recovery login: %w", err)
|
||||
}
|
||||
} else {
|
||||
// Persisted captcha tokens are short-lived. With a refresh token we can
|
||||
// safely request a fresh captcha token after auth, and avoiding the
|
||||
// stored value prevents known-stale tokens from poisoning startup.
|
||||
clearPersistedCaptcha()
|
||||
}
|
||||
} else {
|
||||
if err := d.login(ctx); err != nil {
|
||||
@@ -336,7 +355,60 @@ func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
currentID := d.rootID
|
||||
for _, name := range splitPath(pathFromRoot) {
|
||||
childID, err := d.findChildDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if childID == "" {
|
||||
childID, err = d.makeDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
currentID = childID
|
||||
}
|
||||
return currentID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) findChildDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
entries, err := d.List(ctx, parentID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir && e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
var out file
|
||||
err := d.request(ctx, filesURL, http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(map[string]any{
|
||||
"kind": "drive#folder",
|
||||
"parent_id": parentID,
|
||||
"name": name,
|
||||
})
|
||||
}, &out)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("pikpak mkdir %s: %w", name, err)
|
||||
}
|
||||
if out.ID == "" {
|
||||
return "", fmt.Errorf("pikpak mkdir %s: empty folder id", name)
|
||||
}
|
||||
return out.ID, nil
|
||||
}
|
||||
|
||||
func splitPath(p string) []string {
|
||||
p = strings.Trim(p, "/")
|
||||
if p == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(p, "/")
|
||||
}
|
||||
|
||||
func (d *Driver) getFiles(ctx context.Context, parentID string) ([]file, error) {
|
||||
@@ -408,14 +480,15 @@ func (d *Driver) requestOnce(ctx context.Context, url, method string, configure
|
||||
// serialized. Once we hold the lock, if d.captchaToken has
|
||||
// already moved past staleToken, another goroutine has refreshed
|
||||
// it for us — we skip the refresh and just retry. Otherwise we
|
||||
// clear the cached token (4002 means "the value in the body is
|
||||
// expired"; sending it again will keep returning 4002) and ask
|
||||
// /v1/shield/captcha/init for a fresh one.
|
||||
// clear the cached token before asking /v1/shield/captcha/init
|
||||
// for a fresh one. PikPak may report stale captcha as either
|
||||
// 4002 or 9, and sending the rejected token into captcha init can
|
||||
// keep returning captcha_invalid.
|
||||
staleToken := d.captchaToken
|
||||
d.captchaMu.Lock()
|
||||
var refreshErr error
|
||||
if d.captchaToken == staleToken {
|
||||
if e.ErrorCode == 4002 {
|
||||
if d.captchaToken != "" {
|
||||
d.captchaToken = ""
|
||||
}
|
||||
refreshErr = d.refreshCaptchaTokenAtLogin(ctx, getAction(method, url), d.userID)
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package pikpak
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestNewDefaults(t *testing.T) {
|
||||
@@ -95,11 +97,85 @@ func TestFolderToEntry(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDirStillUnsupported(t *testing.T) {
|
||||
d := New(Config{ID: "pikpak-main"})
|
||||
func TestEnsureDirReusesExistingFolder(t *testing.T) {
|
||||
var postCalled bool
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
if got := r.URL.Query().Get("parent_id"); got != "root-id" {
|
||||
t.Fatalf("parent_id = %q, want root-id", got)
|
||||
}
|
||||
writePikPakJSON(t, w, map[string]any{
|
||||
"files": []map[string]any{{
|
||||
"id": "existing-folder-id",
|
||||
"kind": "drive#folder",
|
||||
"name": "91 Spider",
|
||||
}},
|
||||
})
|
||||
case http.MethodPost:
|
||||
postCalled = true
|
||||
t.Fatalf("existing folder should not be created again")
|
||||
default:
|
||||
t.Fatalf("unexpected method %s", r.Method)
|
||||
}
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
if _, err := d.EnsureDir(nil, "/previews"); err != drives.ErrNotSupported {
|
||||
t.Fatalf("EnsureDir error = %v, want ErrNotSupported", err)
|
||||
d := newTestDriver(t, srv)
|
||||
got, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure dir: %v", err)
|
||||
}
|
||||
if got != "existing-folder-id" {
|
||||
t.Fatalf("dir id = %q, want existing-folder-id", got)
|
||||
}
|
||||
if postCalled {
|
||||
t.Fatal("POST should not be called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDirCreatesMissingFolder(t *testing.T) {
|
||||
var got uploadRequestBody
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
writePikPakJSON(t, w, map[string]any{"files": []map[string]any{}})
|
||||
case http.MethodPost:
|
||||
if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode create folder body: %v", err)
|
||||
}
|
||||
writePikPakJSON(t, w, map[string]any{
|
||||
"id": "new-folder-id",
|
||||
"kind": "drive#folder",
|
||||
"name": "91 Spider",
|
||||
})
|
||||
default:
|
||||
t.Fatalf("unexpected method %s", r.Method)
|
||||
}
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
d := newTestDriver(t, srv)
|
||||
id, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure dir: %v", err)
|
||||
}
|
||||
if id != "new-folder-id" {
|
||||
t.Fatalf("dir id = %q, want new-folder-id", id)
|
||||
}
|
||||
if got.Kind != "drive#folder" || got.ParentID != "root-id" || got.Name != "91 Spider" {
|
||||
t.Fatalf("create folder body = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func writePikPakJSON(t *testing.T, w http.ResponseWriter, body any) {
|
||||
t.Helper()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(body); err != nil {
|
||||
t.Fatalf("write json: %v", err)
|
||||
}
|
||||
// Upload 的真实实现见 upload_test.go。
|
||||
}
|
||||
|
||||
@@ -59,6 +59,10 @@ func (e *errResp) Error() string {
|
||||
return fmt.Sprintf("pikpak error_code=%d error=%s description=%s", e.ErrorCode, e.ErrorMsg, e.ErrorDescription)
|
||||
}
|
||||
|
||||
func isCaptchaTokenRejectedCode(code int64) bool {
|
||||
return code == 9 || code == 4002
|
||||
}
|
||||
|
||||
// APIError is the public alias for the PikPak API error response. Callers
|
||||
// outside this package (e.g. the spider91→PikPak migrator, tests) can either
|
||||
// construct it for fakes or unwrap it via errors.As. Prefer IsCaptchaError
|
||||
@@ -76,7 +80,7 @@ func IsCaptchaError(err error) bool {
|
||||
}
|
||||
var e *errResp
|
||||
if errors.As(err, &e) {
|
||||
return e != nil && (e.ErrorCode == 4002 || e.ErrorCode == 9)
|
||||
return e != nil && isCaptchaTokenRejectedCode(e.ErrorCode)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -20,6 +21,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"golang.org/x/net/proxy"
|
||||
)
|
||||
|
||||
// 默认 author/tag 标签,便于在前端筛选 spider91 来源的视频。
|
||||
@@ -79,29 +81,123 @@ func NewCrawler(cfg CrawlerConfig) *Crawler {
|
||||
cfg.DownloadTimeout = 30 * time.Minute
|
||||
}
|
||||
if cfg.HTTPClient == nil {
|
||||
// 选 proxy 函数:显式 ProxyURL > 环境变量 > 直连
|
||||
proxyFn := http.ProxyFromEnvironment
|
||||
if strings.TrimSpace(cfg.ProxyURL) != "" {
|
||||
if u, err := url.Parse(cfg.ProxyURL); err == nil {
|
||||
proxyFn = http.ProxyURL(u)
|
||||
} else {
|
||||
log.Printf("[spider91] invalid proxy URL %q, falling back to env: %v", cfg.ProxyURL, err)
|
||||
}
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
ResponseHeaderTimeout: 60 * time.Second,
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
}
|
||||
if err := configureExplicitProxy(transport, cfg.ProxyURL); err != nil {
|
||||
log.Printf("[spider91] invalid configured proxy URL, falling back to env: %v", err)
|
||||
}
|
||||
cfg.HTTPClient = &http.Client{
|
||||
// 不限制总下载时长,靠 ctx 控制;只挡 dial / handshake / header
|
||||
Timeout: 0,
|
||||
Transport: &http.Transport{
|
||||
Proxy: proxyFn,
|
||||
ResponseHeaderTimeout: 60 * time.Second,
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
},
|
||||
Timeout: 0,
|
||||
Transport: transport,
|
||||
}
|
||||
}
|
||||
return &Crawler{cfg: cfg}
|
||||
}
|
||||
|
||||
func configureExplicitProxy(transport *http.Transport, raw string) error {
|
||||
proxyURL := strings.TrimSpace(raw)
|
||||
if proxyURL == "" {
|
||||
return nil
|
||||
}
|
||||
u, err := url.Parse(proxyURL)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return fmt.Errorf("invalid proxy URL")
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https":
|
||||
transport.Proxy = http.ProxyURL(u)
|
||||
transport.DialContext = nil
|
||||
return nil
|
||||
case "socks5", "socks5h":
|
||||
dialContext, err := socksProxyDialContext(u)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
transport.Proxy = nil
|
||||
transport.DialContext = dialContext
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("unsupported proxy scheme %q", u.Scheme)
|
||||
}
|
||||
}
|
||||
|
||||
func socksProxyDialContext(proxyURL *url.URL) (func(context.Context, string, string) (net.Conn, error), error) {
|
||||
var auth *proxy.Auth
|
||||
if proxyURL.User != nil {
|
||||
username := proxyURL.User.Username()
|
||||
password, _ := proxyURL.User.Password()
|
||||
auth = &proxy.Auth{User: username, Password: password}
|
||||
}
|
||||
dialer, err := proxy.SOCKS5("tcp", proxyURL.Host, auth, &net.Dialer{Timeout: 60 * time.Second})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remoteDNS := strings.EqualFold(proxyURL.Scheme, "socks5h")
|
||||
return func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
target := addr
|
||||
if !remoteDNS {
|
||||
resolved, err := resolveSocksTarget(ctx, addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target = resolved
|
||||
}
|
||||
if ctxDialer, ok := dialer.(proxy.ContextDialer); ok {
|
||||
return ctxDialer.DialContext(ctx, network, target)
|
||||
}
|
||||
type result struct {
|
||||
conn net.Conn
|
||||
err error
|
||||
}
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
conn, err := dialer.Dial(network, target)
|
||||
ch <- result{conn: conn, err: err}
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case res := <-ch:
|
||||
return res.conn, res.err
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveSocksTarget(ctx context.Context, addr string) (string, error) {
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil || net.ParseIP(host) != nil {
|
||||
return addr, nil
|
||||
}
|
||||
ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
ip := selectSocksTargetIP(ips)
|
||||
if ip == nil {
|
||||
return "", fmt.Errorf("resolve %s: no address", host)
|
||||
}
|
||||
return net.JoinHostPort(ip.String(), port), nil
|
||||
}
|
||||
|
||||
func selectSocksTargetIP(ips []net.IPAddr) net.IP {
|
||||
for _, addr := range ips {
|
||||
if ip4 := addr.IP.To4(); ip4 != nil {
|
||||
return ip4
|
||||
}
|
||||
}
|
||||
for _, addr := range ips {
|
||||
if addr.IP != nil {
|
||||
return addr.IP
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CrawlResult 汇总一次 RunOnce 的结果。
|
||||
type CrawlResult struct {
|
||||
// TargetNew 是本次 RunOnce 的目标新增数(来自 drive.Credentials.target_new)。
|
||||
@@ -324,6 +420,16 @@ func (c *Crawler) startSpiderTargetNew(ctx context.Context, targetNew int, seenP
|
||||
if c.cfg.WorkDir != "" {
|
||||
cmd.Dir = c.cfg.WorkDir
|
||||
}
|
||||
if proxyURL := strings.TrimSpace(c.cfg.ProxyURL); proxyURL != "" {
|
||||
cmd.Env = append(os.Environ(),
|
||||
"HTTP_PROXY="+proxyURL,
|
||||
"HTTPS_PROXY="+proxyURL,
|
||||
"http_proxy="+proxyURL,
|
||||
"https_proxy="+proxyURL,
|
||||
"NO_PROXY=",
|
||||
"no_proxy=",
|
||||
)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("stdout pipe: %w", err)
|
||||
@@ -427,6 +533,17 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
}
|
||||
}
|
||||
|
||||
title := strings.TrimSpace(item.Title)
|
||||
if title == "" {
|
||||
title = sourceID
|
||||
}
|
||||
tags := []string{DefaultTag}
|
||||
if matched, err := c.cfg.Catalog.MatchTags(ctx, title+" "+DefaultAuthor); err == nil {
|
||||
tags = mergeCatalogTags(tags, matched)
|
||||
} else {
|
||||
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s match tags: %v", c.cfg.Driver.ID(), viewkey, sourceID, err)
|
||||
}
|
||||
|
||||
// 入库
|
||||
now := time.Now()
|
||||
v := &catalog.Video{
|
||||
@@ -434,9 +551,9 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
DriveID: c.cfg.Driver.ID(),
|
||||
FileID: videoFile,
|
||||
FileName: videoFile,
|
||||
Title: strings.TrimSpace(item.Title),
|
||||
Title: title,
|
||||
Author: DefaultAuthor,
|
||||
Tags: []string{DefaultTag},
|
||||
Tags: tags,
|
||||
Ext: strings.TrimPrefix(videoExt, "."),
|
||||
Quality: "HD",
|
||||
Size: videoSize,
|
||||
@@ -445,9 +562,6 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if v.Title == "" {
|
||||
v.Title = sourceID
|
||||
}
|
||||
if thumbReady {
|
||||
// 设了 ThumbnailURL 后 thumb worker 会跳过这条视频,
|
||||
// 不再尝试用 ffmpeg 抽帧(封面已经是网站原图)。
|
||||
@@ -463,8 +577,7 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
// 网站封面下载失败的视频:spider91 drive 的 thumb worker 按设计不
|
||||
// 处理 spider91 视频(封面应是网站原图直接保存),所以没人接手。
|
||||
// 显式标 'failed' 让 CountVideosNeedingThumbnail 排除(条件 status
|
||||
// != 'failed'),否则 enqueueDriveGeneration → waitForThumbnailsBeforePreview
|
||||
// 会因为 count > 0 把 teaser 入队永远卡在等待循环里。
|
||||
// != 'failed'),避免后续封面补队列一直重复捞到这条视频。
|
||||
_ = c.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
ThumbnailStatus: "failed",
|
||||
})
|
||||
@@ -889,6 +1002,26 @@ func copyFileAtomic(src, dst string) error {
|
||||
return os.Rename(tmp, dst)
|
||||
}
|
||||
|
||||
func mergeCatalogTags(lists ...[]string) []string {
|
||||
out := []string{}
|
||||
seen := map[string]bool{}
|
||||
for _, list := range lists {
|
||||
for _, tag := range list {
|
||||
tag = strings.TrimSpace(tag)
|
||||
if tag == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(tag)
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, tag)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BuildVideoID 给定 driveID + 91 源视频 ID,按统一规则生成 catalog 中 videos.id。
|
||||
// 与 scanner 用法一致:<kind>-<driveID>-<fileID>。
|
||||
func BuildVideoID(driveID, sourceID string) string {
|
||||
|
||||
@@ -3,6 +3,8 @@ package spider91
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
@@ -53,7 +55,7 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
|
||||
// 同时仍写 --output 文件作归档。
|
||||
videoEntries := []map[string]string{
|
||||
{
|
||||
"title": "Video One",
|
||||
"title": "Video One 口交",
|
||||
"thumb_url": srv.URL + "/thumb/not-120001.jpg",
|
||||
"video_url": srv.URL + "/videos/120001.mp4",
|
||||
"viewkey": "vk-001",
|
||||
@@ -94,6 +96,9 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert drive: %v", err)
|
||||
}
|
||||
if _, err := cat.CreateTagAndClassify(context.Background(), "Video One", nil, "user"); err != nil {
|
||||
t.Fatalf("create user tag: %v", err)
|
||||
}
|
||||
|
||||
var newVideos []*catalog.Video
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
@@ -188,6 +193,17 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
|
||||
if !hasDefaultTag {
|
||||
t.Fatalf("video %s tags = %v, want contain %q", videoID, v.Tags, DefaultTag)
|
||||
}
|
||||
if sourceID == "120001" {
|
||||
if !containsString(v.Tags, "口交") {
|
||||
t.Fatalf("video %s tags = %v, want contain built-in tag 口交", videoID, v.Tags)
|
||||
}
|
||||
if !containsString(v.Tags, "Video One") {
|
||||
t.Fatalf("video %s tags = %v, want contain user tag Video One", videoID, v.Tags)
|
||||
}
|
||||
}
|
||||
if sourceID == "120002" && (containsString(v.Tags, "口交") || containsString(v.Tags, "Video One")) {
|
||||
t.Fatalf("video %s tags = %v, should not inherit tags from other spider91 videos", videoID, v.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
// 7. 第二次 RunOnce:源视频 ID 已存在 → 全部 skipped,无新文件下载
|
||||
@@ -233,14 +249,116 @@ func TestCrawlerRunOnceMissingScript(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerPassesProxyToSpiderProcess(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell-based fake script only on unix")
|
||||
}
|
||||
|
||||
tmp := t.TempDir()
|
||||
scriptPath := filepath.Join(tmp, "print_proxy_env.sh")
|
||||
script := `#!/bin/sh
|
||||
printf 'HTTP_PROXY=%s\n' "$HTTP_PROXY"
|
||||
printf 'HTTPS_PROXY=%s\n' "$HTTPS_PROXY"
|
||||
printf 'http_proxy=%s\n' "$http_proxy"
|
||||
printf 'https_proxy=%s\n' "$https_proxy"
|
||||
printf 'NO_PROXY=%s\n' "$NO_PROXY"
|
||||
printf 'no_proxy=%s\n' "$no_proxy"
|
||||
`
|
||||
if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
|
||||
proxyURL := "socks5h://proxy.local:1080"
|
||||
drv := New(Config{ID: "proxy-drive", RootDir: filepath.Join(tmp, "proxy-drive")})
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
PythonPath: "sh",
|
||||
ScriptPath: scriptPath,
|
||||
ProxyURL: proxyURL,
|
||||
})
|
||||
cmd, stdout, err := c.startSpiderTargetNew(
|
||||
context.Background(),
|
||||
1,
|
||||
filepath.Join(tmp, "seen.txt"),
|
||||
filepath.Join(tmp, "out.json"),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("startSpiderTargetNew: %v", err)
|
||||
}
|
||||
raw, err := io.ReadAll(stdout)
|
||||
if err != nil {
|
||||
t.Fatalf("read stdout: %v", err)
|
||||
}
|
||||
if err := cmd.Wait(); err != nil {
|
||||
t.Fatalf("wait: %v", err)
|
||||
}
|
||||
|
||||
want := strings.Join([]string{
|
||||
"HTTP_PROXY=" + proxyURL,
|
||||
"HTTPS_PROXY=" + proxyURL,
|
||||
"http_proxy=" + proxyURL,
|
||||
"https_proxy=" + proxyURL,
|
||||
"NO_PROXY=",
|
||||
"no_proxy=",
|
||||
}, "\n") + "\n"
|
||||
if string(raw) != want {
|
||||
t.Fatalf("proxy env = %q, want %q", string(raw), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigureExplicitProxySupportsSocksSchemes(t *testing.T) {
|
||||
for _, raw := range []string{
|
||||
"socks5://127.0.0.1:1080",
|
||||
"socks5h://proxy-user:proxy-pass@127.0.0.1:1080",
|
||||
} {
|
||||
t.Run(raw, func(t *testing.T) {
|
||||
transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
|
||||
if err := configureExplicitProxy(transport, raw); err != nil {
|
||||
t.Fatalf("configureExplicitProxy: %v", err)
|
||||
}
|
||||
if transport.Proxy != nil {
|
||||
t.Fatalf("Transport.Proxy should be nil for SOCKS proxy")
|
||||
}
|
||||
if transport.DialContext == nil {
|
||||
t.Fatalf("Transport.DialContext should be set for SOCKS proxy")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
|
||||
if err := configureExplicitProxy(transport, "http://127.0.0.1:7890"); err != nil {
|
||||
t.Fatalf("configureExplicitProxy http: %v", err)
|
||||
}
|
||||
if transport.Proxy == nil {
|
||||
t.Fatalf("Transport.Proxy should be set for HTTP proxy")
|
||||
}
|
||||
if transport.DialContext != nil {
|
||||
t.Fatalf("Transport.DialContext should not be set for HTTP proxy")
|
||||
}
|
||||
|
||||
if err := configureExplicitProxy(&http.Transport{}, "ftp://127.0.0.1:21"); err == nil {
|
||||
t.Fatalf("expected unsupported proxy scheme error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSocksTargetIPPrefersIPv4(t *testing.T) {
|
||||
got := selectSocksTargetIP([]net.IPAddr{
|
||||
{IP: net.ParseIP("2606:4700:20::681a:229")},
|
||||
{IP: net.ParseIP("104.26.3.41")},
|
||||
})
|
||||
if got == nil || got.String() != "104.26.3.41" {
|
||||
t.Fatalf("selectSocksTargetIP = %v, want IPv4 104.26.3.41", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCrawlerThumbDownloadFailureMarksStatusFailed 验证:网站封面下载失败时
|
||||
// crawler 把 thumbnail_status 显式标 'failed',避免 enqueueDriveGeneration 的
|
||||
// waitForThumbnailsBeforePreview 因为 count > 0 把 teaser 卡死等待。
|
||||
// crawler 把 thumbnail_status 显式标 'failed',避免后续封面补队列一直重复
|
||||
// 捞到这条 spider91 视频。
|
||||
//
|
||||
// 历史 bug:之前 thumb 下载失败仅打 log,url=”, status 走 schema DEFAULT 'pending'。
|
||||
// CountVideosNeedingThumbnail 条件是 url=” AND status != 'failed' → count=1。
|
||||
// spider91 drive 的 thumb worker 按设计不处理 spider91 视频 → 没人会改 status。
|
||||
// 结果 teaser 永远卡在 [preview] waiting for 1 thumbnails before teaser generation。
|
||||
// spider91 drive 的 thumb worker 按设计不处理 spider91 视频 → 没人会改 status,
|
||||
// 后续补队列会一直认为它还缺封面。
|
||||
func TestCrawlerThumbDownloadFailureMarksStatusFailed(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell-based fake script only on unix")
|
||||
@@ -317,8 +435,7 @@ func TestCrawlerThumbDownloadFailureMarksStatusFailed(t *testing.T) {
|
||||
|
||||
// 关键断言:CountVideosNeedingThumbnail 应该返回 0。
|
||||
// 该函数的 SQL 条件是 `url = '' AND status != 'failed'`;如果 crawler 没把
|
||||
// status 标 'failed'(schema DEFAULT 'pending'),count 就会是 1,外层
|
||||
// waitForThumbnailsBeforePreview 会因为 count > 0 把 teaser 卡死等待。
|
||||
// status 标 'failed'(schema DEFAULT 'pending'),count 就会是 1。
|
||||
count, err := cat.CountVideosNeedingThumbnail(context.Background(), driveID)
|
||||
if err != nil {
|
||||
t.Fatalf("count: %v", err)
|
||||
@@ -659,3 +776,12 @@ func buildFakeSpiderScript(entries []map[string]string) string {
|
||||
sb.WriteString("fi\n")
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func containsString(values []string, want string) bool {
|
||||
for _, value := range values {
|
||||
if value == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,468 @@
|
||||
package fingerprint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultSampleSizeBytes int64 = 512 * 1024
|
||||
defaultFullHashMaxSize int64 = 8 * 1024 * 1024
|
||||
defaultCooldown = 5 * time.Minute
|
||||
defaultWorkerQueueSize = 10000
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
SampleSizeBytes int64
|
||||
FullHashMaxSize int64
|
||||
RateLimitCooldown time.Duration
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
type Worker struct {
|
||||
Catalog *catalog.Catalog
|
||||
Drive drives.Drive
|
||||
Config Config
|
||||
|
||||
ch chan *catalog.Video
|
||||
queue videoQueue
|
||||
activity taskActivity
|
||||
cooldown cooldownState
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
type TaskStatus struct {
|
||||
State string
|
||||
CurrentTitle string
|
||||
QueueLength int
|
||||
CooldownUntil time.Time
|
||||
}
|
||||
|
||||
func NewWorker(cat *catalog.Catalog, drv drives.Drive, cfg Config) *Worker {
|
||||
hc := cfg.HTTPClient
|
||||
if hc == nil {
|
||||
hc = &http.Client{Timeout: 0}
|
||||
}
|
||||
if cfg.SampleSizeBytes <= 0 {
|
||||
cfg.SampleSizeBytes = defaultSampleSizeBytes
|
||||
}
|
||||
if cfg.FullHashMaxSize <= 0 {
|
||||
cfg.FullHashMaxSize = defaultFullHashMaxSize
|
||||
}
|
||||
if cfg.RateLimitCooldown <= 0 {
|
||||
cfg.RateLimitCooldown = defaultCooldown
|
||||
}
|
||||
return &Worker{
|
||||
Catalog: cat,
|
||||
Drive: drv,
|
||||
Config: cfg,
|
||||
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
|
||||
http: hc,
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) Enqueue(v *catalog.Video) bool {
|
||||
if v == nil {
|
||||
return false
|
||||
}
|
||||
if !w.queue.reserve(v.ID) {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case w.ch <- v:
|
||||
return true
|
||||
default:
|
||||
w.queue.release(v.ID)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) EnqueueBlocking(ctx context.Context, v *catalog.Video) bool {
|
||||
if v == nil {
|
||||
return false
|
||||
}
|
||||
if !w.queue.reserve(v.ID) {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case w.ch <- v:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
w.queue.release(v.ID)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) Run(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case v := <-w.ch:
|
||||
w.processQueued(ctx, v)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) Status() TaskStatus {
|
||||
if w == nil {
|
||||
return TaskStatus{State: "idle"}
|
||||
}
|
||||
currentID, currentTitle := w.activity.current()
|
||||
status := TaskStatus{
|
||||
State: "idle",
|
||||
CurrentTitle: currentTitle,
|
||||
QueueLength: w.queue.lengthExcluding(currentID),
|
||||
}
|
||||
if until, ok := w.cooldown.active(time.Now()); ok {
|
||||
status.State = "cooling"
|
||||
status.CooldownUntil = until
|
||||
return status
|
||||
}
|
||||
if currentID != "" {
|
||||
status.State = "generating"
|
||||
return status
|
||||
}
|
||||
if status.QueueLength > 0 {
|
||||
status.State = "queued"
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
defer w.queue.release(v.ID)
|
||||
if w.Catalog == nil || w.Drive == nil || v == nil || v.ID == "" {
|
||||
return
|
||||
}
|
||||
current, err := w.Catalog.GetVideo(ctx, v.ID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if current.SampledSHA256 != "" || current.FingerprintStatus == "ready" || current.Hidden {
|
||||
return
|
||||
}
|
||||
w.activity.start(current)
|
||||
defer w.activity.done()
|
||||
sum, err := Compute(ctx, w.Drive, current, w.Config, w.http)
|
||||
if err != nil {
|
||||
var rl *drives.RateLimitError
|
||||
if errors.As(err, &rl) {
|
||||
wait := rl.RetryAfter
|
||||
if wait <= 0 {
|
||||
wait = w.Config.RateLimitCooldown
|
||||
}
|
||||
until := time.Now().Add(wait)
|
||||
w.cooldown.set(until)
|
||||
log.Printf("[fingerprint] drive=%s rate limited; keep video=%s pending and cool down for %s: %v", w.Drive.ID(), current.ID, wait, err)
|
||||
sleepContext(ctx, wait)
|
||||
w.cooldown.clear(until)
|
||||
return
|
||||
}
|
||||
log.Printf("[fingerprint] video=%s failed: %v", current.ID, err)
|
||||
_ = w.Catalog.UpdateVideoFingerprint(ctx, current.ID, "", "failed", err.Error())
|
||||
return
|
||||
}
|
||||
if err := w.Catalog.UpdateVideoFingerprint(ctx, current.ID, sum, "ready", ""); err != nil {
|
||||
log.Printf("[fingerprint] update video=%s: %v", current.ID, err)
|
||||
return
|
||||
}
|
||||
log.Printf("[fingerprint] video=%s ready sampled_sha256=%s", current.ID, sum)
|
||||
}
|
||||
|
||||
func Compute(ctx context.Context, drv drives.Drive, v *catalog.Video, cfg Config, hc *http.Client) (string, error) {
|
||||
if drv == nil {
|
||||
return "", errors.New("fingerprint: nil drive")
|
||||
}
|
||||
if v == nil {
|
||||
return "", errors.New("fingerprint: nil video")
|
||||
}
|
||||
if v.Size <= 0 {
|
||||
return "", errors.New("fingerprint: video size is empty")
|
||||
}
|
||||
if cfg.SampleSizeBytes <= 0 {
|
||||
cfg.SampleSizeBytes = defaultSampleSizeBytes
|
||||
}
|
||||
if cfg.FullHashMaxSize <= 0 {
|
||||
cfg.FullHashMaxSize = defaultFullHashMaxSize
|
||||
}
|
||||
if hc == nil {
|
||||
hc = &http.Client{Timeout: 0}
|
||||
}
|
||||
link, err := drv.StreamURL(ctx, v.FileID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fingerprint: stream url: %w", err)
|
||||
}
|
||||
if link == nil || strings.TrimSpace(link.URL) == "" {
|
||||
return "", errors.New("fingerprint: empty stream url")
|
||||
}
|
||||
ranges := sampleRanges(v.Size, cfg.SampleSizeBytes, cfg.FullHashMaxSize)
|
||||
h := sha256.New()
|
||||
writeHashHeader(h, v.Size, ranges)
|
||||
for _, r := range ranges {
|
||||
data, err := readRange(ctx, hc, link, r)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if int64(len(data)) != r.length {
|
||||
return "", fmt.Errorf("fingerprint: short sample at %d: got %d want %d", r.start, len(data), r.length)
|
||||
}
|
||||
_, _ = h.Write([]byte(fmt.Sprintf("offset=%d length=%d\n", r.start, r.length)))
|
||||
_, _ = h.Write(data)
|
||||
_, _ = h.Write([]byte("\n"))
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
type byteRange struct {
|
||||
start int64
|
||||
length int64
|
||||
}
|
||||
|
||||
func sampleRanges(size, sampleSize, fullHashMax int64) []byteRange {
|
||||
if size <= fullHashMax {
|
||||
return []byteRange{{start: 0, length: size}}
|
||||
}
|
||||
if sampleSize > size {
|
||||
sampleSize = size
|
||||
}
|
||||
maxStart := size - sampleSize
|
||||
percents := []int64{0, 20, 40, 60, 80}
|
||||
out := make([]byteRange, 0, len(percents))
|
||||
seen := make(map[int64]struct{}, len(percents))
|
||||
for _, pct := range percents {
|
||||
start := maxStart * pct / 100
|
||||
if _, ok := seen[start]; ok {
|
||||
continue
|
||||
}
|
||||
seen[start] = struct{}{}
|
||||
out = append(out, byteRange{start: start, length: sampleSize})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func writeHashHeader(w io.Writer, size int64, ranges []byteRange) {
|
||||
_, _ = fmt.Fprintf(w, "video-site-sampled-sha256-v1\nsize=%d\nsamples=%d\n", size, len(ranges))
|
||||
}
|
||||
|
||||
func readRange(ctx context.Context, hc *http.Client, link *drives.StreamLink, r byteRange) ([]byte, error) {
|
||||
u, err := url.Parse(link.URL)
|
||||
if err == nil && (u.Scheme == "http" || u.Scheme == "https") {
|
||||
return readHTTPRange(ctx, hc, link, r)
|
||||
}
|
||||
path := link.URL
|
||||
if err == nil && u.Scheme == "file" {
|
||||
path = u.Path
|
||||
}
|
||||
return readLocalRange(path, r)
|
||||
}
|
||||
|
||||
func readLocalRange(path string, r byteRange) ([]byte, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fingerprint: open local stream: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
buf := make([]byte, r.length)
|
||||
n, err := f.ReadAt(buf, r.start)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, fmt.Errorf("fingerprint: read local sample: %w", err)
|
||||
}
|
||||
if int64(n) != r.length {
|
||||
return nil, fmt.Errorf("fingerprint: read local sample at %d: got %d want %d", r.start, n, r.length)
|
||||
}
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
func readHTTPRange(ctx context.Context, hc *http.Client, link *drives.StreamLink, r byteRange) ([]byte, error) {
|
||||
end := r.start + r.length - 1
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, link.URL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for k, vs := range link.Headers {
|
||||
for _, v := range vs {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", r.start, end))
|
||||
resp, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fingerprint: read remote sample: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
return nil, &drives.RateLimitError{
|
||||
Provider: "fingerprint",
|
||||
RetryAfter: parseRetryAfter(resp.Header.Get("Retry-After")),
|
||||
Err: fmt.Errorf("remote sample rate limited: status=%d", resp.StatusCode),
|
||||
}
|
||||
}
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
if resp.StatusCode == http.StatusOK && r.start == 0 {
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, r.length+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) == r.length {
|
||||
return data, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("fingerprint: range request got status=%d for bytes=%d-%d", resp.StatusCode, r.start, end)
|
||||
}
|
||||
return io.ReadAll(io.LimitReader(resp.Body, r.length))
|
||||
}
|
||||
|
||||
func parseRetryAfter(raw string) time.Duration {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return 0
|
||||
}
|
||||
if seconds, err := strconv.Atoi(raw); err == nil && seconds > 0 {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
if when, err := http.ParseTime(raw); err == nil {
|
||||
d := time.Until(when)
|
||||
if d > 0 {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, d time.Duration) bool {
|
||||
if d <= 0 {
|
||||
return true
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-timer.C:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
type taskActivity struct {
|
||||
mu sync.Mutex
|
||||
currentID string
|
||||
currentTitle string
|
||||
}
|
||||
|
||||
func (a *taskActivity) start(v *catalog.Video) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if v == nil {
|
||||
a.currentID = ""
|
||||
a.currentTitle = ""
|
||||
return
|
||||
}
|
||||
a.currentID = v.ID
|
||||
a.currentTitle = v.Title
|
||||
}
|
||||
|
||||
func (a *taskActivity) done() {
|
||||
a.mu.Lock()
|
||||
a.currentID = ""
|
||||
a.currentTitle = ""
|
||||
a.mu.Unlock()
|
||||
}
|
||||
|
||||
func (a *taskActivity) current() (string, string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.currentID, a.currentTitle
|
||||
}
|
||||
|
||||
type cooldownState struct {
|
||||
mu sync.Mutex
|
||||
until time.Time
|
||||
}
|
||||
|
||||
func (s *cooldownState) set(until time.Time) {
|
||||
s.mu.Lock()
|
||||
s.until = until
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *cooldownState) clear(until time.Time) {
|
||||
s.mu.Lock()
|
||||
if s.until.Equal(until) {
|
||||
s.until = time.Time{}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *cooldownState) active(now time.Time) (time.Time, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.until.IsZero() || !s.until.After(now) {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return s.until, true
|
||||
}
|
||||
|
||||
type videoQueue struct {
|
||||
mu sync.Mutex
|
||||
ids map[string]struct{}
|
||||
}
|
||||
|
||||
func (q *videoQueue) reserve(id string) bool {
|
||||
if id == "" {
|
||||
return true
|
||||
}
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
if q.ids == nil {
|
||||
q.ids = make(map[string]struct{})
|
||||
}
|
||||
if _, ok := q.ids[id]; ok {
|
||||
return false
|
||||
}
|
||||
q.ids[id] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
func (q *videoQueue) release(id string) {
|
||||
if id == "" {
|
||||
return
|
||||
}
|
||||
q.mu.Lock()
|
||||
delete(q.ids, id)
|
||||
q.mu.Unlock()
|
||||
}
|
||||
|
||||
func (q *videoQueue) lengthExcluding(currentID string) int {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
n := len(q.ids)
|
||||
if currentID != "" {
|
||||
if _, ok := q.ids[currentID]; ok {
|
||||
n--
|
||||
}
|
||||
}
|
||||
if n < 0 {
|
||||
return 0
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package fingerprint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestComputeLocalFilesWithSameContentMatch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
body := []byte("same video bytes")
|
||||
a := filepath.Join(dir, "a.mp4")
|
||||
b := filepath.Join(dir, "b.mp4")
|
||||
if err := os.WriteFile(a, body, 0o644); err != nil {
|
||||
t.Fatalf("write a: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(b, body, 0o644); err != nil {
|
||||
t.Fatalf("write b: %v", err)
|
||||
}
|
||||
|
||||
sumA, err := Compute(ctx, &fakeDrive{paths: map[string]string{"a": a}}, &catalog.Video{ID: "a", FileID: "a", Size: int64(len(body))}, Config{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("compute a: %v", err)
|
||||
}
|
||||
sumB, err := Compute(ctx, &fakeDrive{paths: map[string]string{"b": b}}, &catalog.Video{ID: "b", FileID: "b", Size: int64(len(body))}, Config{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("compute b: %v", err)
|
||||
}
|
||||
if sumA == "" || sumA != sumB {
|
||||
t.Fatalf("fingerprints = %q / %q, want same non-empty", sumA, sumB)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeRemoteUsesRangeSamples(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
data := make([]byte, 10*1024*1024)
|
||||
for i := range data {
|
||||
data[i] = byte(i % 251)
|
||||
}
|
||||
var ranges []string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
rawRange := r.Header.Get("Range")
|
||||
ranges = append(ranges, rawRange)
|
||||
var start, end int
|
||||
if _, err := fmt.Sscanf(rawRange, "bytes=%d-%d", &start, &end); err != nil {
|
||||
t.Fatalf("bad range %q: %v", rawRange, err)
|
||||
}
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write(data[start : end+1])
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
drv := &fakeDrive{paths: map[string]string{"remote": srv.URL + "/video.mp4"}}
|
||||
sum, err := Compute(ctx, drv, &catalog.Video{ID: "remote", FileID: "remote", Size: int64(len(data))}, Config{
|
||||
SampleSizeBytes: 4,
|
||||
FullHashMaxSize: 8,
|
||||
HTTPClient: srv.Client(),
|
||||
}, srv.Client())
|
||||
if err != nil {
|
||||
t.Fatalf("compute remote: %v", err)
|
||||
}
|
||||
if sum == "" {
|
||||
t.Fatal("fingerprint should not be empty")
|
||||
}
|
||||
want := []string{
|
||||
"bytes=0-3",
|
||||
"bytes=2097151-2097154",
|
||||
"bytes=4194302-4194305",
|
||||
"bytes=6291453-6291456",
|
||||
"bytes=8388604-8388607",
|
||||
}
|
||||
if fmt.Sprint(ranges) != fmt.Sprint(want) {
|
||||
t.Fatalf("ranges = %#v, want %#v", ranges, want)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeDrive struct {
|
||||
paths map[string]string
|
||||
}
|
||||
|
||||
func (d *fakeDrive) Kind() string { return "fake" }
|
||||
func (d *fakeDrive) ID() string { return "fake" }
|
||||
func (d *fakeDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *fakeDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeDrive) StreamURL(_ context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
return &drives.StreamLink{URL: d.paths[fileID], Expires: time.Now().Add(time.Minute)}, nil
|
||||
}
|
||||
func (d *fakeDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeDrive) RootID() string { return "root" }
|
||||
@@ -12,6 +12,8 @@
|
||||
// wait until teaser queues are idle
|
||||
// Phase 3: spider91 → cloud migration (single sweep, captcha cooldown still
|
||||
// honored within this call)
|
||||
// Phase 4: cleanup duplicate local preview/thumbnail assets after sampled
|
||||
// fingerprints have identified canonical videos
|
||||
//
|
||||
// A 6h soft deadline guards each pipeline run; phases check deadline at their
|
||||
// boundaries and exit cleanly if exceeded (no in-flight ffmpeg / upload is
|
||||
@@ -85,15 +87,34 @@ type Config struct {
|
||||
// RunMigration runs spider91migrate.Migrator.RunOnce for Phase 3.
|
||||
RunMigration func(ctx context.Context) error
|
||||
|
||||
// RunDedupeAssetCleanup removes generated local assets from non-canonical
|
||||
// videos in size+sampled_sha256 duplicate groups. It must not delete cloud
|
||||
// files or catalog rows.
|
||||
RunDedupeAssetCleanup func(ctx context.Context) error
|
||||
|
||||
// Now is injected for tests; nil → time.Now.
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type Status struct {
|
||||
State string
|
||||
Running bool
|
||||
Queued bool
|
||||
StartedAt time.Time
|
||||
LastFinishedAt time.Time
|
||||
}
|
||||
|
||||
// Runner drives the nightly pipeline.
|
||||
type Runner struct {
|
||||
cfg Config
|
||||
trigger chan struct{} // buffered(1); manual "run now"
|
||||
runMu sync.Mutex // prevents overlapping pipeline runs
|
||||
|
||||
stateMu sync.Mutex
|
||||
running bool
|
||||
queued bool
|
||||
startedAt time.Time
|
||||
lastFinishedAt time.Time
|
||||
}
|
||||
|
||||
// New constructs a Runner. cfg is shallow-copied; defaults are applied.
|
||||
@@ -131,13 +152,53 @@ func (r *Runner) Run(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// TriggerNow asks the running loop to fire a pipeline ASAP. If a pipeline is
|
||||
// already in progress (or another trigger is already pending), the request
|
||||
// is dropped — the in-progress run will absorb the intent.
|
||||
func (r *Runner) TriggerNow() {
|
||||
// TriggerNow asks the running loop to fire a pipeline ASAP. Only one manual
|
||||
// trigger can be active at a time: if a pipeline is already running or waiting
|
||||
// in the trigger channel, the request is ignored and returns false.
|
||||
func (r *Runner) TriggerNow() bool {
|
||||
r.stateMu.Lock()
|
||||
if r.running || r.queued {
|
||||
r.stateMu.Unlock()
|
||||
return false
|
||||
}
|
||||
r.queued = true
|
||||
r.stateMu.Unlock()
|
||||
|
||||
select {
|
||||
case r.trigger <- struct{}{}:
|
||||
return true
|
||||
default:
|
||||
r.stateMu.Lock()
|
||||
r.queued = false
|
||||
r.stateMu.Unlock()
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) Status() Status {
|
||||
r.stateMu.Lock()
|
||||
running := r.running
|
||||
queued := r.queued
|
||||
startedAt := r.startedAt
|
||||
lastFinishedAt := r.lastFinishedAt
|
||||
r.stateMu.Unlock()
|
||||
|
||||
state := "idle"
|
||||
switch {
|
||||
case running && queued:
|
||||
state = "running_queued"
|
||||
case running:
|
||||
state = "running"
|
||||
case queued:
|
||||
state = "queued"
|
||||
}
|
||||
|
||||
return Status{
|
||||
State: state,
|
||||
Running: running,
|
||||
Queued: queued,
|
||||
StartedAt: startedAt,
|
||||
LastFinishedAt: lastFinishedAt,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,9 +236,13 @@ func (r *Runner) runPipelineLocked(ctx context.Context, manual bool) {
|
||||
log.Printf("[nightly] another pipeline is already running, skipping this trigger")
|
||||
return
|
||||
}
|
||||
defer r.runMu.Unlock()
|
||||
|
||||
started := r.cfg.Now()
|
||||
r.markStarted(started)
|
||||
defer func() {
|
||||
r.markFinished(r.cfg.Now())
|
||||
r.runMu.Unlock()
|
||||
}()
|
||||
|
||||
mode := "scheduled"
|
||||
if manual {
|
||||
@@ -199,6 +264,22 @@ func (r *Runner) runPipelineLocked(ctx context.Context, manual bool) {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) markStarted(started time.Time) {
|
||||
r.stateMu.Lock()
|
||||
defer r.stateMu.Unlock()
|
||||
r.running = true
|
||||
r.queued = false
|
||||
r.startedAt = started
|
||||
}
|
||||
|
||||
func (r *Runner) markFinished(finished time.Time) {
|
||||
r.stateMu.Lock()
|
||||
defer r.stateMu.Unlock()
|
||||
r.running = false
|
||||
r.startedAt = time.Time{}
|
||||
r.lastFinishedAt = finished
|
||||
}
|
||||
|
||||
// runPipeline executes the three phases. It returns when the pipeline finishes
|
||||
// OR ctx is done (deadline / cancel). Errors are logged but not propagated —
|
||||
// each phase is best-effort; downstream phases still attempt to run unless ctx
|
||||
@@ -240,6 +321,7 @@ func (r *Runner) runPipeline(ctx context.Context) {
|
||||
}
|
||||
if len(spiderIDs) == 0 {
|
||||
log.Printf("[nightly] phase 2/3 skipped: no spider91 drive configured")
|
||||
r.runDedupeAssetCleanupPhase(ctx)
|
||||
return
|
||||
}
|
||||
log.Printf("[nightly] phase 2: crawling %d spider91 drive(s)", len(spiderIDs))
|
||||
@@ -266,6 +348,8 @@ func (r *Runner) runPipeline(ctx context.Context) {
|
||||
log.Printf("[nightly] phase 3 migration: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
r.runDedupeAssetCleanupPhase(ctx)
|
||||
}
|
||||
|
||||
// checkDeadline returns true when ctx is already done (runner shutting down or
|
||||
@@ -291,6 +375,19 @@ func (r *Runner) waitIdle(ctx context.Context, phase string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runner) runDedupeAssetCleanupPhase(ctx context.Context) {
|
||||
if r.checkDeadline(ctx, "phase 4") {
|
||||
return
|
||||
}
|
||||
if r.cfg.RunDedupeAssetCleanup == nil {
|
||||
return
|
||||
}
|
||||
log.Printf("[nightly] phase 4: duplicate asset cleanup")
|
||||
if err := r.cfg.RunDedupeAssetCleanup(ctx); err != nil {
|
||||
log.Printf("[nightly] phase 4 duplicate asset cleanup: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// readLastRunDate reads the persisted last_run_date or returns "" when unset.
|
||||
func (r *Runner) readLastRunDate(ctx context.Context) (string, error) {
|
||||
if r.cfg.Settings == nil {
|
||||
|
||||
@@ -114,6 +114,10 @@ func TestRunPipelineHonoursPhaseOrder(t *testing.T) {
|
||||
rec.push("migrate")
|
||||
return nil
|
||||
},
|
||||
RunDedupeAssetCleanup: func(context.Context) error {
|
||||
rec.push("dedupe-cleanup")
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
r.runPipeline(context.Background())
|
||||
@@ -128,6 +132,7 @@ func TestRunPipelineHonoursPhaseOrder(t *testing.T) {
|
||||
"crawl:sp-1",
|
||||
"wait-idle", // after phase 2
|
||||
"migrate",
|
||||
"dedupe-cleanup",
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("call sequence len = %d, want %d; got=%v", len(got), len(want), got)
|
||||
@@ -156,6 +161,10 @@ func TestRunPipelineSkipsMigrationWhenNoSpider91(t *testing.T) {
|
||||
rec.push("migrate")
|
||||
return nil
|
||||
},
|
||||
RunDedupeAssetCleanup: func(context.Context) error {
|
||||
rec.push("dedupe-cleanup")
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
r.runPipeline(context.Background())
|
||||
@@ -165,6 +174,15 @@ func TestRunPipelineSkipsMigrationWhenNoSpider91(t *testing.T) {
|
||||
t.Fatalf("phase 2/3 should be skipped when no spider91 drive, got call %q", c)
|
||||
}
|
||||
}
|
||||
foundCleanup := false
|
||||
for _, c := range rec.snapshot() {
|
||||
if c == "dedupe-cleanup" {
|
||||
foundCleanup = true
|
||||
}
|
||||
}
|
||||
if !foundCleanup {
|
||||
t.Fatalf("dedupe cleanup should still run when spider91 is absent; calls=%v", rec.snapshot())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
|
||||
@@ -186,6 +204,7 @@ func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
|
||||
RunSpider91Crawl: func(context.Context, string) { rec.push("crawl") },
|
||||
WaitPreviewQueuesIdle: func(context.Context) error { rec.push("wait-idle"); return nil },
|
||||
RunMigration: func(context.Context) error { rec.push("migrate"); return nil },
|
||||
RunDedupeAssetCleanup: func(context.Context) error { rec.push("dedupe-cleanup"); return nil },
|
||||
})
|
||||
|
||||
r.runPipeline(ctx)
|
||||
@@ -200,6 +219,9 @@ func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
|
||||
if c == "crawl" || c == "migrate" {
|
||||
t.Fatalf("subsequent phase should not run after cancel, got call %q", c)
|
||||
}
|
||||
if c == "dedupe-cleanup" {
|
||||
t.Fatalf("dedupe cleanup should not run after cancel, got call %q", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -290,11 +312,14 @@ func TestCtxCancelPreventsLaterPhases(t *testing.T) {
|
||||
func TestTriggerNowIsNonBlocking(t *testing.T) {
|
||||
r := New(Config{Settings: newStubSettings()})
|
||||
// fill the trigger channel
|
||||
r.TriggerNow()
|
||||
if !r.TriggerNow() {
|
||||
t.Fatal("first TriggerNow should be accepted")
|
||||
}
|
||||
// Second call must not block
|
||||
done := make(chan struct{})
|
||||
var accepted bool
|
||||
go func() {
|
||||
r.TriggerNow()
|
||||
accepted = r.TriggerNow()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
@@ -302,4 +327,98 @@ func TestTriggerNowIsNonBlocking(t *testing.T) {
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
t.Fatal("TriggerNow blocked when channel is full")
|
||||
}
|
||||
if accepted {
|
||||
t.Fatal("second TriggerNow should be ignored when trigger channel is full")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusTracksQueuedRunningAndFinished(t *testing.T) {
|
||||
blockScan := make(chan struct{})
|
||||
scanStarted := make(chan struct{})
|
||||
var startedOnce sync.Once
|
||||
r := New(Config{
|
||||
Settings: newStubSettings(),
|
||||
ListScanTargets: func(context.Context) []string {
|
||||
return []string{"drive"}
|
||||
},
|
||||
RunScan: func(context.Context, string) {
|
||||
startedOnce.Do(func() { close(scanStarted) })
|
||||
<-blockScan
|
||||
},
|
||||
})
|
||||
|
||||
if got := r.Status(); got.State != "idle" || got.Running || got.Queued {
|
||||
t.Fatalf("initial status = %#v, want idle", got)
|
||||
}
|
||||
|
||||
if !r.TriggerNow() {
|
||||
t.Fatal("TriggerNow should queue a manual run")
|
||||
}
|
||||
if got := r.Status(); got.State != "queued" || got.Running || !got.Queued {
|
||||
t.Fatalf("queued status = %#v, want queued", got)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go r.Run(ctx)
|
||||
|
||||
select {
|
||||
case <-scanStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("pipeline did not start")
|
||||
}
|
||||
|
||||
if got := r.Status(); got.State != "running" || !got.Running || got.Queued || got.StartedAt.IsZero() {
|
||||
t.Fatalf("running status = %#v, want running with startedAt", got)
|
||||
}
|
||||
|
||||
if r.TriggerNow() {
|
||||
t.Fatal("TriggerNow during a run should be ignored")
|
||||
}
|
||||
if got := r.Status(); got.State != "running" || !got.Running || got.Queued {
|
||||
t.Fatalf("status after ignored trigger = %#v, want running", got)
|
||||
}
|
||||
|
||||
close(blockScan)
|
||||
deadline := time.After(time.Second)
|
||||
for {
|
||||
got := r.Status()
|
||||
if !got.Running && !got.Queued && !got.LastFinishedAt.IsZero() {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatalf("status did not finish; got=%#v", got)
|
||||
default:
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTriggerNowAcceptsOnlyOneConcurrentRequest(t *testing.T) {
|
||||
r := New(Config{Settings: newStubSettings()})
|
||||
|
||||
const callers = 16
|
||||
start := make(chan struct{})
|
||||
results := make(chan bool, callers)
|
||||
for i := 0; i < callers; i++ {
|
||||
go func() {
|
||||
<-start
|
||||
results <- r.TriggerNow()
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
|
||||
accepted := 0
|
||||
for i := 0; i < callers; i++ {
|
||||
if <-results {
|
||||
accepted++
|
||||
}
|
||||
}
|
||||
if accepted != 1 {
|
||||
t.Fatalf("accepted triggers = %d, want 1", accepted)
|
||||
}
|
||||
if got := r.Status(); got.State != "queued" || got.Running || !got.Queued {
|
||||
t.Fatalf("status = %#v, want one queued trigger", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package preview
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -289,7 +290,7 @@ func (g *Generator) generateThumbnailAtOffset(ctx context.Context, link *drives.
|
||||
args = append(args,
|
||||
"-i", ffmpegLink.URL,
|
||||
"-frames:v", "1",
|
||||
"-vf", fmt.Sprintf("scale=%d:-2", g.cfg.Width),
|
||||
"-vf", thumbnailVideoFilter(g.cfg.Width),
|
||||
"-q:v", "3",
|
||||
"-y", dst,
|
||||
)
|
||||
@@ -307,6 +308,12 @@ func (g *Generator) generateThumbnailAtOffset(ctx context.Context, link *drives.
|
||||
return nil
|
||||
}
|
||||
|
||||
func thumbnailVideoFilter(width int) string {
|
||||
// FFmpeg 7 rejects non-full-range YUV for MJPEG/JPEG output. Force the
|
||||
// scaled frame into a JPEG-friendly full-range pixel format before encode.
|
||||
return fmt.Sprintf("scale=%d:-2:out_range=pc,format=yuvj420p", width)
|
||||
}
|
||||
|
||||
func thumbnailOffsetFallbackAllowed(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
@@ -339,9 +346,15 @@ func (g *Generator) Probe(ctx context.Context, link *drives.StreamLink) (float64
|
||||
args = append(args, ffmpegLink.URL)
|
||||
|
||||
cmd := exec.CommandContext(ctx2, g.cfg.FFprobePath, args...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return 0, ffmpegCommandError("ffprobe", err, out)
|
||||
errOut := stderr.Bytes()
|
||||
if len(errOut) == 0 {
|
||||
errOut = out
|
||||
}
|
||||
return 0, ffmpegCommandError("ffprobe", err, errOut)
|
||||
}
|
||||
raw := strings.TrimSpace(string(out))
|
||||
if raw == "" || raw == "N/A" {
|
||||
@@ -923,6 +936,7 @@ func ffmpegOutputLooksRateLimited(output []byte) bool {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "throttl") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "rate-limit") ||
|
||||
strings.Contains(text, "server returned 429")
|
||||
@@ -968,7 +982,6 @@ type Worker struct {
|
||||
queue videoQueue
|
||||
|
||||
RateLimitCooldown time.Duration
|
||||
BeforeTask func(context.Context) bool
|
||||
rateLimit rateLimitState
|
||||
activity taskActivity
|
||||
}
|
||||
@@ -978,7 +991,7 @@ func NewWorker(gen TeaserGenerator, cat *catalog.Catalog, drv drives.Drive) *Wor
|
||||
Gen: gen,
|
||||
Catalog: cat,
|
||||
Drive: drv,
|
||||
ch: make(chan *catalog.Video, 4096),
|
||||
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1027,10 +1040,12 @@ type ThumbWorker struct {
|
||||
}
|
||||
|
||||
const (
|
||||
defaultTransientMediaCooldown = 5 * time.Minute
|
||||
defaultGenerationRateLimitCooldown = 5 * time.Minute
|
||||
maxPreviewTeaserSizeBytes int64 = 5 * 1024 * 1024 * 1024
|
||||
previewStatusSkipped = "skipped"
|
||||
defaultTransientMediaCooldown = 5 * time.Minute
|
||||
defaultGenerationRateLimitCooldown = 5 * time.Minute
|
||||
defaultThumbTransientMediaMaxFailures = 3
|
||||
defaultWorkerQueueSize = 10000
|
||||
maxPreviewTeaserSizeBytes int64 = 5 * 1024 * 1024 * 1024
|
||||
previewStatusSkipped = "skipped"
|
||||
)
|
||||
|
||||
type rateLimitState struct {
|
||||
@@ -1168,7 +1183,7 @@ func NewThumbWorker(gen ThumbnailGenerator, cat *catalog.Catalog, drv drives.Dri
|
||||
Gen: gen,
|
||||
Catalog: cat,
|
||||
Drive: drv,
|
||||
ch: make(chan *catalog.Video, 4096),
|
||||
ch: make(chan *catalog.Video, defaultWorkerQueueSize),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1323,10 +1338,6 @@ func (w *ThumbWorker) Run(ctx context.Context) {
|
||||
|
||||
func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
defer w.queue.release(v)
|
||||
if w.BeforeTask != nil && !w.BeforeTask(ctx) {
|
||||
return
|
||||
}
|
||||
|
||||
w.activity.start(v)
|
||||
defer w.activity.done()
|
||||
if !waitForRateLimitCooldown(ctx, &w.rateLimit, "preview", w.Drive) {
|
||||
@@ -1336,13 +1347,16 @@ func (w *Worker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) processQueued(ctx context.Context, v *catalog.Video) {
|
||||
defer w.queue.release(v)
|
||||
w.activity.start(v)
|
||||
defer w.activity.done()
|
||||
if !waitForRateLimitCooldown(ctx, &w.rateLimit, "thumb", w.Drive) {
|
||||
return
|
||||
retry := false
|
||||
if waitForRateLimitCooldown(ctx, &w.rateLimit, "thumb", w.Drive) {
|
||||
retry = w.process(ctx, v)
|
||||
}
|
||||
w.activity.done()
|
||||
w.queue.release(v)
|
||||
if retry && ctx.Err() == nil {
|
||||
w.EnqueueBlocking(ctx, v)
|
||||
}
|
||||
w.process(ctx, v)
|
||||
}
|
||||
|
||||
func waitForRateLimitCooldown(ctx context.Context, state *rateLimitState, label string, drive drives.Drive) bool {
|
||||
@@ -1424,15 +1438,34 @@ func (w *ThumbWorker) pauseForRateLimit(err error, step, title string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) pauseForRecoverableError(err error, step, title string) bool {
|
||||
func (w *ThumbWorker) pauseForRecoverableError(ctx context.Context, v *catalog.Video, err error, step string) bool {
|
||||
title := ""
|
||||
videoID := ""
|
||||
if v != nil {
|
||||
title = v.Title
|
||||
videoID = v.ID
|
||||
}
|
||||
if w.pauseForRateLimit(err, step, title) {
|
||||
return true
|
||||
}
|
||||
if !driveErrorShouldCooldown(w.Drive, err) {
|
||||
return false
|
||||
}
|
||||
failures := 1
|
||||
if w.Catalog != nil && videoID != "" {
|
||||
count, countErr := w.Catalog.IncrementThumbnailFailures(ctx, videoID)
|
||||
if countErr != nil {
|
||||
log.Printf("[thumb] drive=%s transient media source error count failed step=%s video=%s: %v", w.Drive.ID(), step, title, countErr)
|
||||
} else {
|
||||
failures = count
|
||||
}
|
||||
}
|
||||
if failures >= defaultThumbTransientMediaMaxFailures {
|
||||
log.Printf("[thumb] drive=%s transient media source error reached retry limit failures=%d/%d step=%s video=%s: %v", w.Drive.ID(), failures, defaultThumbTransientMediaMaxFailures, step, title, err)
|
||||
return false
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), w.RateLimitCooldown)
|
||||
log.Printf("[thumb] drive=%s transient media source error until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err)
|
||||
log.Printf("[thumb] drive=%s transient media source error until=%s failures=%d/%d step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), failures, defaultThumbTransientMediaMaxFailures, step, title, err)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -1478,44 +1511,110 @@ func driveErrorShouldCooldown(d drives.Drive, err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
|
||||
func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) bool {
|
||||
if w.skipIfRateLimited(v) {
|
||||
return
|
||||
return false
|
||||
}
|
||||
if current, err := w.Catalog.GetVideo(ctx, v.ID); err == nil {
|
||||
if current.ThumbnailURL != "" {
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
|
||||
return
|
||||
queued := v
|
||||
current := v
|
||||
if loaded, err := w.Catalog.GetVideo(ctx, v.ID); err == nil {
|
||||
if loaded.PreviewLocal == "" {
|
||||
loaded.PreviewLocal = queued.PreviewLocal
|
||||
}
|
||||
current = loaded
|
||||
v = loaded
|
||||
if loaded.ThumbnailURL != "" && loaded.DurationSeconds > 0 {
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
|
||||
return false
|
||||
}
|
||||
}
|
||||
if current.ThumbnailURL != "" {
|
||||
durationBackfillFailed := false
|
||||
if current.DurationSeconds <= 0 {
|
||||
link, err := w.streamLink(ctx, current)
|
||||
if err != nil {
|
||||
if w.pauseForRecoverableError(ctx, current, err, "streamURL") {
|
||||
return true
|
||||
}
|
||||
log.Printf("[thumb] probe streamURL %s: %v", current.Title, err)
|
||||
durationBackfillFailed = true
|
||||
} else if w.probeDuration(ctx, current, link) {
|
||||
return true
|
||||
} else if current.DurationSeconds <= 0 {
|
||||
durationBackfillFailed = true
|
||||
}
|
||||
}
|
||||
if durationBackfillFailed {
|
||||
log.Printf("[thumb] skip duration backfill %s: thumbnail already exists but duration could not be probed", current.Title)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, current.ID, catalog.VideoMetaPatch{ThumbnailStatus: "skipped"})
|
||||
return false
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, current.ID, catalog.VideoMetaPatch{ThumbnailStatus: "ready"})
|
||||
return false
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "pending"})
|
||||
link, err := w.Drive.StreamURL(ctx, v.FileID)
|
||||
link, err := w.streamLink(ctx, v)
|
||||
if err != nil {
|
||||
if localLink, ok := localPreviewLink(v); ok {
|
||||
link = localLink
|
||||
} else {
|
||||
if w.pauseForRecoverableError(err, "streamURL", v.Title) {
|
||||
return
|
||||
}
|
||||
log.Printf("[thumb] streamURL %s: %v", v.Title, err)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
|
||||
return
|
||||
if w.pauseForRecoverableError(ctx, v, err, "streamURL") {
|
||||
return true
|
||||
}
|
||||
log.Printf("[thumb] streamURL %s: %v", v.Title, err)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
|
||||
return false
|
||||
}
|
||||
if w.probeDuration(ctx, v, link) {
|
||||
return true
|
||||
}
|
||||
|
||||
if err := w.generateThumbnailFromLink(ctx, v, link); err != nil {
|
||||
if localLink, ok := localPreviewLink(v); ok && link.URL != localLink.URL {
|
||||
if w.probeDuration(ctx, v, localLink) {
|
||||
return true
|
||||
}
|
||||
if localErr := w.generateThumbnailFromLink(ctx, v, localLink); localErr == nil {
|
||||
return
|
||||
return false
|
||||
}
|
||||
}
|
||||
if w.pauseForRecoverableError(err, "generate", v.Title) {
|
||||
return
|
||||
if w.pauseForRecoverableError(ctx, v, err, "generate") {
|
||||
return true
|
||||
}
|
||||
log.Printf("[thumb] generate %s: %v", v.Title, err)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
|
||||
return
|
||||
return false
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) streamLink(ctx context.Context, v *catalog.Video) (*drives.StreamLink, error) {
|
||||
link, err := w.Drive.StreamURL(ctx, v.FileID)
|
||||
if err == nil {
|
||||
return link, nil
|
||||
}
|
||||
if localLink, ok := localPreviewLink(v); ok {
|
||||
return localLink, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) probeDuration(ctx context.Context, v *catalog.Video, link *drives.StreamLink) bool {
|
||||
if v.DurationSeconds > 0 {
|
||||
return false
|
||||
}
|
||||
dur, err := w.Gen.Probe(ctx, link)
|
||||
if err == nil {
|
||||
if dur > 0 {
|
||||
v.DurationSeconds = int(dur)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
DurationSeconds: int(dur),
|
||||
})
|
||||
}
|
||||
return false
|
||||
}
|
||||
if w.pauseForRecoverableError(ctx, v, err, "probe") {
|
||||
return true
|
||||
}
|
||||
log.Printf("[thumb] probe %s: %v", v.Title, err)
|
||||
return false
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) generateThumbnailFromLink(ctx context.Context, v *catalog.Video, link *drives.StreamLink) error {
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"errors"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -95,6 +97,24 @@ func TestTinyVideoPreviewPlanUsesWholeVideoAsSingleSegment(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeIgnoresStderrWarnings(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ffprobePath := filepath.Join(dir, "ffprobe")
|
||||
script := "#!/bin/sh\nprintf '%s\\n' 'h264 warning' >&2\nprintf '%s\\n' '364.800000'\n"
|
||||
if err := os.WriteFile(ffprobePath, []byte(script), 0o755); err != nil {
|
||||
t.Fatalf("write ffprobe stub: %v", err)
|
||||
}
|
||||
|
||||
gen := New(Config{FFprobePath: ffprobePath})
|
||||
got, err := gen.Probe(context.Background(), &drives.StreamLink{URL: filepath.Join(dir, "video.mp4")})
|
||||
if err != nil {
|
||||
t.Fatalf("probe: %v", err)
|
||||
}
|
||||
if got != 364.8 {
|
||||
t.Fatalf("duration = %v, want 364.8", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTeaserCandidateStartsKeepPrimaryAndAddFallbacks(t *testing.T) {
|
||||
primary := []float64{10.2, 64.65, 119.1, 173.55}
|
||||
got := teaserCandidateStarts(204, primary, 3)
|
||||
@@ -161,6 +181,16 @@ func TestThumbnailOffsetsUseFiveSecondsWithEarlyFallbacks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailVideoFilterUsesFullRangeJPEGPixelFormat(t *testing.T) {
|
||||
got := thumbnailVideoFilter(480)
|
||||
if !strings.Contains(got, "scale=480:-2:out_range=pc") {
|
||||
t.Fatalf("thumbnail filter = %q, want full-range scale output", got)
|
||||
}
|
||||
if !strings.Contains(got, "format=yuvj420p") {
|
||||
t.Fatalf("thumbnail filter = %q, want JPEG-friendly pixel format", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailOffsetFallbackAllowedForEmptyOutputAndTimeouts(t *testing.T) {
|
||||
for _, err := range []error{
|
||||
errors.New("ffmpeg thumb produced empty file, stderr: "),
|
||||
|
||||
@@ -13,11 +13,11 @@ import (
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
|
||||
func TestThumbWorkerUpdatesThumbnailAndDurationWithoutChangingPreviewStatus(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-video")
|
||||
|
||||
gen := &fakeThumbGenerator{}
|
||||
gen := &fakeThumbGenerator{probeDuration: 42}
|
||||
drv := &previewFakeDrive{}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
@@ -33,8 +33,8 @@ func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
|
||||
if got.PreviewStatus != "pending" {
|
||||
t.Fatalf("preview status = %q, want pending", got.PreviewStatus)
|
||||
}
|
||||
if got.DurationSeconds != 0 {
|
||||
t.Fatalf("duration = %d, want unchanged", got.DurationSeconds)
|
||||
if got.DurationSeconds != 42 {
|
||||
t.Fatalf("duration = %d, want probed duration", got.DurationSeconds)
|
||||
}
|
||||
if gen.thumbnailVideoID != video.ID {
|
||||
t.Fatalf("thumbnail video id = %q, want %q", gen.thumbnailVideoID, video.ID)
|
||||
@@ -42,14 +42,93 @@ func TestThumbWorkerUpdatesThumbnailWithoutChangingPreviewStatus(t *testing.T) {
|
||||
if gen.thumbnailDuration != 0 {
|
||||
t.Fatalf("thumbnail duration = %.1f, want fixed-offset thumbnail generation", gen.thumbnailDuration)
|
||||
}
|
||||
if gen.probeCalls != 0 {
|
||||
t.Fatalf("probe calls = %d, want 0 for thumbnail generation", gen.probeCalls)
|
||||
if gen.probeCalls != 1 {
|
||||
t.Fatalf("probe calls = %d, want 1 for thumbnail generation", gen.probeCalls)
|
||||
}
|
||||
if drv.streamFileID != video.FileID {
|
||||
t.Fatalf("stream file id = %q, want %q", drv.streamFileID, video.FileID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerBackfillsDurationWhenThumbnailAlreadyExists(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-existing-thumbnail")
|
||||
video.ThumbnailURL = "/p/thumb/" + video.ID
|
||||
if err := cat.UpsertVideo(ctx, video); err != nil {
|
||||
t.Fatalf("update video: %v", err)
|
||||
}
|
||||
|
||||
gen := &fakeThumbGenerator{probeDuration: 19}
|
||||
drv := &previewFakeDrive{}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
worker.process(ctx, video)
|
||||
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DurationSeconds != 19 {
|
||||
t.Fatalf("duration = %d, want probed duration", got.DurationSeconds)
|
||||
}
|
||||
if got.ThumbnailURL != "/p/thumb/"+video.ID {
|
||||
t.Fatalf("thumbnail = %q, want unchanged existing thumbnail", got.ThumbnailURL)
|
||||
}
|
||||
ready, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "ready", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list ready thumbnails: %v", err)
|
||||
}
|
||||
if len(ready) != 1 || ready[0].ID != video.ID {
|
||||
t.Fatalf("ready thumbnails = %#v, want only %s", ready, video.ID)
|
||||
}
|
||||
if gen.probeCalls != 1 {
|
||||
t.Fatalf("probe calls = %d, want 1", gen.probeCalls)
|
||||
}
|
||||
if gen.thumbnailVideoID != "" {
|
||||
t.Fatalf("thumbnail generation video id = %q, want no regeneration", gen.thumbnailVideoID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerSkipsDurationBackfillWhenExistingThumbnailCannotBeProbed(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-existing-thumbnail-probe-fails")
|
||||
video.ThumbnailURL = "/p/thumb/" + video.ID
|
||||
if err := cat.UpsertVideo(ctx, video); err != nil {
|
||||
t.Fatalf("update video: %v", err)
|
||||
}
|
||||
|
||||
gen := &fakeThumbGenerator{probeErr: errors.New("invalid media")}
|
||||
drv := &previewFakeDrive{}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
worker.process(ctx, video)
|
||||
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "/p/thumb/"+video.ID {
|
||||
t.Fatalf("thumbnail = %q, want unchanged existing thumbnail", got.ThumbnailURL)
|
||||
}
|
||||
if got.DurationSeconds != 0 {
|
||||
t.Fatalf("duration = %d, want still unknown", got.DurationSeconds)
|
||||
}
|
||||
skipped, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "skipped", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list skipped thumbnails: %v", err)
|
||||
}
|
||||
if len(skipped) != 1 || skipped[0].ID != video.ID {
|
||||
t.Fatalf("skipped thumbnails = %#v, want only %s", skipped, video.ID)
|
||||
}
|
||||
missing, err := cat.CountVideosNeedingThumbnail(ctx, video.DriveID)
|
||||
if err != nil {
|
||||
t.Fatalf("count videos needing thumbnail: %v", err)
|
||||
}
|
||||
if missing != 0 {
|
||||
t.Fatalf("missing thumbnails = %d, want 0 after duration backfill is skipped", missing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerFallsBackToLocalPreviewWhenDriveStreamFails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-local-preview")
|
||||
@@ -377,6 +456,113 @@ func TestThumbWorkerRateLimitCoolsDownFiveMinutes(t *testing.T) {
|
||||
assertCooldownAround(t, worker.Status().CooldownUntil, before, 5*time.Minute)
|
||||
}
|
||||
|
||||
func TestThumbWorkerP115TransientErrorFailsAfterRetryLimit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-p115-transient")
|
||||
|
||||
gen := &fakeThumbGenerator{
|
||||
generateErr: errors.New("ffmpeg thumb: exit status 183, stderr: partial file Cannot determine format of input 0:0 after EOF"),
|
||||
}
|
||||
drv := &previewFakeDrive{kind: "p115"}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
for attempt := 1; attempt <= defaultThumbTransientMediaMaxFailures; attempt++ {
|
||||
worker.rateLimit = rateLimitState{}
|
||||
worker.process(ctx, video)
|
||||
|
||||
if attempt < defaultThumbTransientMediaMaxFailures {
|
||||
pending, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "pending", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list pending thumbnails: %v", err)
|
||||
}
|
||||
if len(pending) != 1 || pending[0].ID != video.ID {
|
||||
t.Fatalf("attempt %d pending thumbnails = %#v, want only %s", attempt, pending, video.ID)
|
||||
}
|
||||
missing, err := cat.CountVideosNeedingThumbnail(ctx, video.DriveID)
|
||||
if err != nil {
|
||||
t.Fatalf("count missing thumbnails: %v", err)
|
||||
}
|
||||
if missing != 1 {
|
||||
t.Fatalf("attempt %d missing thumbnails = %d, want 1 before retry limit", attempt, missing)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list failed thumbnails: %v", err)
|
||||
}
|
||||
if len(failed) != 1 || failed[0].ID != video.ID {
|
||||
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
|
||||
}
|
||||
missing, err := cat.CountVideosNeedingThumbnail(ctx, video.DriveID)
|
||||
if err != nil {
|
||||
t.Fatalf("count missing thumbnails: %v", err)
|
||||
}
|
||||
if missing != 0 {
|
||||
t.Fatalf("missing thumbnails = %d, want 0 after retry limit marks failed", missing)
|
||||
}
|
||||
}
|
||||
|
||||
if gen.generateCalls != defaultThumbTransientMediaMaxFailures {
|
||||
t.Fatalf("generate calls = %d, want %d", gen.generateCalls, defaultThumbTransientMediaMaxFailures)
|
||||
}
|
||||
|
||||
if err := cat.UpdateVideoMeta(ctx, video.ID, catalog.VideoMetaPatch{
|
||||
ThumbnailStatus: "pending",
|
||||
ResetThumbnailFailures: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("reset thumbnail status: %v", err)
|
||||
}
|
||||
worker.rateLimit = rateLimitState{}
|
||||
worker.process(ctx, video)
|
||||
|
||||
pending, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "pending", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list pending thumbnails after reset: %v", err)
|
||||
}
|
||||
if len(pending) != 1 || pending[0].ID != video.ID {
|
||||
t.Fatalf("pending thumbnails after reset = %#v, want only %s", pending, video.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerRequeuesP115TransientErrorBeforeRetryLimit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-p115-requeue")
|
||||
|
||||
gen := &fakeThumbGenerator{
|
||||
generateErr: errors.New("ffmpeg thumb: partial file Cannot determine format of input 0:0 after EOF"),
|
||||
}
|
||||
drv := &previewFakeDrive{kind: "p115"}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
worker.processQueued(ctx, video)
|
||||
|
||||
select {
|
||||
case queued := <-worker.ch:
|
||||
if queued.ID != video.ID {
|
||||
t.Fatalf("requeued video id = %q, want %q", queued.ID, video.ID)
|
||||
}
|
||||
default:
|
||||
t.Fatal("expected transient thumbnail failure to requeue the same video")
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want empty after transient failure", got.ThumbnailURL)
|
||||
}
|
||||
pending, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "pending", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list pending thumbnails: %v", err)
|
||||
}
|
||||
if len(pending) != 1 || pending[0].ID != video.ID {
|
||||
t.Fatalf("pending thumbnails = %#v, want only %s", pending, video.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewWorkerP115TransientErrorKeepsVideoPending(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "preview-p115-transient")
|
||||
@@ -469,15 +655,22 @@ type fakeThumbGenerator struct {
|
||||
thumbnailDuration float64
|
||||
thumbnailURL string
|
||||
probeCalls int
|
||||
generateCalls int
|
||||
probeDuration float64
|
||||
probeErr error
|
||||
generateErr error
|
||||
}
|
||||
|
||||
func (g *fakeThumbGenerator) Probe(context.Context, *drives.StreamLink) (float64, error) {
|
||||
g.probeCalls++
|
||||
return 42, nil
|
||||
if g.probeErr != nil {
|
||||
return 0, g.probeErr
|
||||
}
|
||||
return g.probeDuration, nil
|
||||
}
|
||||
|
||||
func (g *fakeThumbGenerator) GenerateThumbnail(_ context.Context, link *drives.StreamLink, videoID string, duration float64) (string, error) {
|
||||
g.generateCalls++
|
||||
g.thumbnailVideoID = videoID
|
||||
g.thumbnailDuration = duration
|
||||
if link != nil {
|
||||
@@ -568,7 +761,6 @@ func (d *previewFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
}
|
||||
func (d *previewFakeDrive) RootID() string { return "root" }
|
||||
|
||||
|
||||
func TestWorkerWaitIdleReturnsImmediatelyWhenQueueEmpty(t *testing.T) {
|
||||
worker := NewWorker(&fakeTeaserGenerator{}, nil, &previewFakeDrive{})
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -144,13 +145,15 @@ func (p *Proxy) ServeStream(w http.ResponseWriter, r *http.Request, driveID, fil
|
||||
// 302 之后浏览器用自己的 UA 直连,CDN 仍然认签名
|
||||
// - pikpak:与 OpenList 一致,WebContentLink / media link 都是自签 URL,
|
||||
// CDN 不校验请求头,直连可获得最佳带宽并避免占用 backend 出站
|
||||
// - onedrive:Microsoft Graph 返回的 @microsoft.graph.downloadUrl 是短期
|
||||
// 免鉴权下载 URL,不需要后端继续代传视频字节
|
||||
//
|
||||
// 其余网盘(如 OneDrive / 沃盘 / 夸克等)仍走反代,因为它们的下载
|
||||
// 其余网盘(如沃盘 / 夸克等)仍走反代,因为它们的下载
|
||||
// 链接通常需要随请求带上后端持有的 Cookie / Authorization / Range
|
||||
// 的特殊处理,浏览器拿不到这些上下文。
|
||||
func shouldRedirect(d drives.Drive) bool {
|
||||
switch d.Kind() {
|
||||
case "p115", "pikpak":
|
||||
case "p115", "pikpak", "onedrive":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -169,6 +172,11 @@ func (p *Proxy) serve(w http.ResponseWriter, r *http.Request, link *drives.Strea
|
||||
http.Error(w, "bad upstream url", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
if localPath, ok := localFilePath(u, link.URL); ok {
|
||||
w.Header().Set("Cache-Control", "private, max-age=300")
|
||||
http.ServeFile(w, r, localPath)
|
||||
return
|
||||
}
|
||||
req, err := http.NewRequestWithContext(r.Context(), r.Method, u.String(), nil)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
@@ -211,6 +219,19 @@ func (p *Proxy) ServeLocal(w http.ResponseWriter, r *http.Request, path string)
|
||||
http.ServeFile(w, r, path)
|
||||
}
|
||||
|
||||
func localFilePath(u *url.URL, raw string) (string, bool) {
|
||||
if u == nil {
|
||||
return "", false
|
||||
}
|
||||
if u.Scheme == "file" && u.Path != "" {
|
||||
return u.Path, true
|
||||
}
|
||||
if u.Scheme == "" && u.Host == "" && filepath.IsAbs(raw) {
|
||||
return raw, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
var errDriveNotFound = &httpError{Code: http.StatusNotFound, Msg: "drive not found"}
|
||||
|
||||
type httpError struct {
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -149,6 +151,61 @@ func TestServeStreamPikPakSetsRedirectHeaders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamRedirectsOneDrive(t *testing.T) {
|
||||
reg := NewRegistry()
|
||||
drv := &proxyFakeSimpleDrive{
|
||||
kind: "onedrive",
|
||||
url: "https://public.onedrive.example/video.mp4",
|
||||
}
|
||||
reg.Set("onedrive", drv)
|
||||
|
||||
p := New(reg)
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/onedrive/file-1", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
p.ServeStream(rr, req, "onedrive", "file-1")
|
||||
|
||||
if rr.Code != http.StatusFound {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusFound)
|
||||
}
|
||||
if got := rr.Header().Get("Location"); got != "https://public.onedrive.example/video.mp4" {
|
||||
t.Fatalf("Location = %q", got)
|
||||
}
|
||||
if drv.calls != 1 {
|
||||
t.Fatalf("link calls = %d, want 1", drv.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamServesLocalFilePath(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "video.mp4")
|
||||
if err := os.WriteFile(path, []byte("0123456789"), 0o644); err != nil {
|
||||
t.Fatalf("write local file: %v", err)
|
||||
}
|
||||
reg := NewRegistry()
|
||||
drv := &proxyFakeSimpleDrive{
|
||||
kind: "localstorage",
|
||||
url: path,
|
||||
}
|
||||
reg.Set("local", drv)
|
||||
|
||||
p := New(reg)
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/local/file-1", nil)
|
||||
req.Header.Set("Range", "bytes=2-5")
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
p.ServeStream(rr, req, "local", "file-1")
|
||||
|
||||
if rr.Code != http.StatusPartialContent {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusPartialContent)
|
||||
}
|
||||
if got := rr.Body.String(); got != "2345" {
|
||||
t.Fatalf("body = %q, want range bytes", got)
|
||||
}
|
||||
if drv.calls != 1 {
|
||||
t.Fatalf("link calls = %d, want 1", drv.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func requestPikPak(t *testing.T, p *Proxy, driveID, fileID, ua string) {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/"+driveID+"/"+fileID, nil)
|
||||
@@ -192,3 +249,36 @@ func (d *proxyFakePikPakDrive) EnsureDir(context.Context, string) (string, error
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakePikPakDrive) RootID() string { return "0" }
|
||||
|
||||
type proxyFakeSimpleDrive struct {
|
||||
kind string
|
||||
url string
|
||||
calls int
|
||||
}
|
||||
|
||||
func (d *proxyFakeSimpleDrive) Kind() string { return d.kind }
|
||||
func (d *proxyFakeSimpleDrive) ID() string { return d.kind }
|
||||
func (d *proxyFakeSimpleDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
|
||||
d.calls++
|
||||
return &drives.StreamLink{
|
||||
URL: d.url,
|
||||
Headers: http.Header{},
|
||||
Expires: time.Now().Add(10 * time.Minute),
|
||||
}, nil
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *proxyFakeSimpleDrive) RootID() string { return "0" }
|
||||
|
||||
@@ -16,11 +16,11 @@ type ParsedName struct {
|
||||
}
|
||||
|
||||
var (
|
||||
reTags = regexp.MustCompile(`^\s*\[([^\]]+)\]\s*`) // [tag1,tag2]
|
||||
reTags = regexp.MustCompile(`^\s*\[([^\]]+)\]\s*`) // [前缀]
|
||||
reAuthor = regexp.MustCompile(`\s*-\s*([^-]+?)\s*$`) // - author
|
||||
)
|
||||
|
||||
// Parse 按约定解析:[tag1,tag2] 标题 - 作者.ext
|
||||
// Parse 按约定解析:[前缀] 标题 - 作者.ext
|
||||
// 任何字段缺失都能降级
|
||||
func Parse(filename string) ParsedName {
|
||||
name := strings.TrimSuffix(filename, path.Ext(filename))
|
||||
|
||||
@@ -254,6 +254,93 @@ func TestRunAddsShortCollectionDirectoryAsTag(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDoesNotRecreateDeletedCollectionDirectoryTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
now := time.Now()
|
||||
for _, id := range []string{"existing-1", "existing-2"} {
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: "Existing",
|
||||
Category: "sunny",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed existing sunny video: %v", err)
|
||||
}
|
||||
}
|
||||
if label, ok, err := cat.EnsureCollectionTag(ctx, "sunny"); err != nil || !ok || label != "sunny" {
|
||||
t.Fatalf("ensure collection = %q, %v, %v; want sunny true nil", label, ok, err)
|
||||
}
|
||||
tags, err := cat.ListTags(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list tags: %v", err)
|
||||
}
|
||||
var tagID int64
|
||||
for _, tag := range tags {
|
||||
if tag.Label == "sunny" {
|
||||
tagID = tag.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
if tagID == 0 {
|
||||
t.Fatal("sunny tag not found before delete")
|
||||
}
|
||||
if _, err := cat.DeleteTag(ctx, tagID); err != nil {
|
||||
t.Fatalf("delete tag: %v", err)
|
||||
}
|
||||
|
||||
drv := &scannerTreeFakeDrive{
|
||||
entries: map[string][]drives.Entry{
|
||||
"root": {{
|
||||
ID: "dir-1",
|
||||
Name: "sunny",
|
||||
IsDir: true,
|
||||
}},
|
||||
"dir-1": {{
|
||||
ID: "file-1",
|
||||
ParentID: "dir-1",
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
ModTime: now,
|
||||
}},
|
||||
},
|
||||
}
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
|
||||
if _, err := sc.Run(ctx, ""); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(ctx, "fake-drive-file-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if len(got.Tags) != 0 {
|
||||
t.Fatalf("tags = %#v, want none", got.Tags)
|
||||
}
|
||||
tags, err = cat.ListTags(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list tags after scan: %v", err)
|
||||
}
|
||||
for _, tag := range tags {
|
||||
if tag.Label == "sunny" {
|
||||
t.Fatal("deleted collection tag was recreated during scan")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMapsAVCodeDirectoryToAVTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Package spider91migrate 周期性把 spider91 drive 下载到本地的视频
|
||||
// 上传到一个指定的目标 drive 目录(PikPak 或 115),上传成功后:
|
||||
// 上传到一个指定的目标 drive 目录(PikPak、115 或 OneDrive),上传成功后:
|
||||
//
|
||||
// - 改写 catalog 行:drive_id / file_id / content_hash 改成目标盘的;
|
||||
// 视频自身的 id 不变(仍是 spider91-<driveID>-<viewkey>),video_tags、
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/onedrive"
|
||||
"github.com/video-site/backend/internal/drives/p115"
|
||||
"github.com/video-site/backend/internal/drives/pikpak"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
@@ -39,12 +40,14 @@ import (
|
||||
// 这一层抽象把"迁移调用方"和"具体盘的 SDK 协议"解耦:
|
||||
// - PikPak 走 GCID + OSS PutObject(pikpak.UploadResult)
|
||||
// - 115 走 SHA1 + 秒传 / OSS / 分片(p115.UploadResult)
|
||||
// - OneDrive 走 SHA1 + 小文件 PUT / 大文件 upload session
|
||||
//
|
||||
// 两个返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
|
||||
// 各家返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
|
||||
type uploadTarget interface {
|
||||
ID() string
|
||||
Kind() string
|
||||
RootID() string
|
||||
EnsureDir(ctx context.Context, pathFromRoot string) (string, error)
|
||||
UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error)
|
||||
Rename(ctx context.Context, fileID, newName string) error
|
||||
}
|
||||
@@ -52,7 +55,7 @@ type uploadTarget interface {
|
||||
// UploadResult 是 uploadTarget.UploadAndReportHash 的归一返回。
|
||||
//
|
||||
// FileID 目标盘上的新文件 ID;
|
||||
// Hash GCID(PikPak)或 SHA1 HEX 大写(115),写入 catalog.content_hash 用于跨盘去重;
|
||||
// Hash GCID(PikPak)或 SHA1 HEX(115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;
|
||||
// Size 实际上传字节数。
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
@@ -60,7 +63,9 @@ type UploadResult struct {
|
||||
Size int64
|
||||
}
|
||||
|
||||
// pikpakAdapter / p115Adapter 把具体 driver 包装成 uploadTarget。
|
||||
const spider91UploadDirName = "91 Spider"
|
||||
|
||||
// pikpakAdapter / p115Adapter / onedriveAdapter 把具体 driver 包装成 uploadTarget。
|
||||
//
|
||||
// 之所以不让 driver 直接实现 uploadTarget:
|
||||
//
|
||||
@@ -74,6 +79,9 @@ type pikpakAdapter struct {
|
||||
func (a *pikpakAdapter) ID() string { return a.d.ID() }
|
||||
func (a *pikpakAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *pikpakAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *pikpakAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *pikpakAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
@@ -92,6 +100,9 @@ type p115Adapter struct {
|
||||
func (a *p115Adapter) ID() string { return a.d.ID() }
|
||||
func (a *p115Adapter) Kind() string { return a.d.Kind() }
|
||||
func (a *p115Adapter) RootID() string { return a.d.RootID() }
|
||||
func (a *p115Adapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *p115Adapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportSha1(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
@@ -103,6 +114,27 @@ func (a *p115Adapter) Rename(ctx context.Context, fileID, newName string) error
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
type onedriveAdapter struct {
|
||||
d *onedrive.Driver
|
||||
}
|
||||
|
||||
func (a *onedriveAdapter) ID() string { return a.d.ID() }
|
||||
func (a *onedriveAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *onedriveAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *onedriveAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *onedriveAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
return UploadResult{FileID: res.FileID, Hash: res.Hash, Size: res.Size}, nil
|
||||
}
|
||||
func (a *onedriveAdapter) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
// adaptUploadTarget 把通用 drive 包装成 uploadTarget。
|
||||
// 不支持的盘 kind 返回 error;调用方静默跳过。
|
||||
func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
@@ -111,6 +143,8 @@ func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
return &pikpakAdapter{d: v}, nil
|
||||
case *p115.Driver:
|
||||
return &p115Adapter{d: v}, nil
|
||||
case *onedrive.Driver:
|
||||
return &onedriveAdapter{d: v}, nil
|
||||
case uploadTarget:
|
||||
// 测试或自定义实现可以直接传入;优先使用具体类型分支以拿到适配器。
|
||||
return v, nil
|
||||
@@ -272,7 +306,7 @@ func (m *Migrator) runOnce(ctx context.Context) {
|
||||
|
||||
target, pp, err := m.resolveTarget()
|
||||
if err != nil {
|
||||
// 没目标就静默 —— 用户可能还没配 PikPak drive
|
||||
// 没目标就静默 —— 用户选择了本地保存,或还没配 115/PikPak drive。
|
||||
return
|
||||
}
|
||||
|
||||
@@ -382,7 +416,7 @@ func (m *Migrator) spider91Drives() []*spider91.Driver {
|
||||
// - 列出 spider91 drive 本地 videos/ 目录所有 mp4 文件,按 mtime 降序排
|
||||
// - 跳过最新 KeepLatestN 个:这些是用户希望保留在本地的最新爬取
|
||||
// - 对剩下的(更旧)逐个处理:
|
||||
// - 还没迁移(drive_id 仍是 src.ID())→ 上传到 PikPak + 改 catalog + 删本地
|
||||
// - 还没迁移(drive_id 仍是 src.ID())→ 上传到目标盘 + 改 catalog + 删本地
|
||||
// - 已经迁移过但本地还有残留 → 仅删本地(兜底)
|
||||
//
|
||||
// KeepLatestN < 0 时不保护任何本地文件,全部尝试迁移(旧行为,主要给测试用)。
|
||||
@@ -484,7 +518,7 @@ func (m *Migrator) migrateDrive(ctx context.Context, src *spider91.Driver, targe
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
// migrateOne 把单条 spider91 视频上传到 PikPak 并改写 catalog。
|
||||
// migrateOne 把单条 spider91 视频上传到目标盘并改写 catalog。
|
||||
// 返回 (true, nil) 表示真的迁了一条;(false, nil) 表示跳过(本地文件已不在等);
|
||||
// (false, err) 表示真出错。
|
||||
func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider91.Driver, targetDriveID string, pp uploadTarget) (bool, error) {
|
||||
@@ -511,15 +545,19 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// 上传到目标盘的根目录(用户配置的目标 drive 的 rootID)。
|
||||
// 上传到目标盘 rootID 下的固定 "91 Spider" 子目录。若用户把目标盘 rootID
|
||||
// 配成某个自定义目录,这里会在该自定义目录下查找/创建 "91 Spider"。
|
||||
// 上传名走 desiredPikPakName 算出来的方案 B 格式:
|
||||
//
|
||||
// <sanitized title>-<viewkey 后 8 位>.<ext>
|
||||
//
|
||||
// 这样网盘 Web 端列出来的文件名能直接看出是哪个视频,
|
||||
// 又用 viewkey 后 8 位避免同标题撞名。两个目标盘(PikPak / 115)共用同一格式,
|
||||
// 又用 viewkey 后 8 位避免同标题撞名。所有目标盘共用同一格式,
|
||||
// 简化前端 / catalog 的认知。
|
||||
parent := pp.RootID()
|
||||
parent, err := pp.EnsureDir(ctx, spider91UploadDirName)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s ensure %q dir: %w", pp.Kind(), spider91UploadDirName, err)
|
||||
}
|
||||
uploadName := desiredPikPakName(v.Title, extractViewKey(v.ID), v.Ext)
|
||||
res, err := pp.UploadAndReportHash(ctx, parent, uploadName, f, info.Size())
|
||||
if err != nil {
|
||||
@@ -639,7 +677,7 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driv
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// backfillFileNames 扫描目标 drive(PikPak 或 115)下所有 spider91-* 起始 ID 的视频,
|
||||
// backfillFileNames 扫描目标 drive(PikPak、115 或 OneDrive)下所有 spider91-* 起始 ID 的视频,
|
||||
// 对文件名不是 desiredPikPakName(...) 期望格式的,调 target.Rename 修正,
|
||||
// 并把 catalog.file_name 同步到新名字。
|
||||
//
|
||||
|
||||
@@ -53,6 +53,8 @@ type fakePikPak struct {
|
||||
uploadFunc func(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error)
|
||||
mu sync.Mutex
|
||||
gotBodies map[string][]byte
|
||||
gotParents map[string]string
|
||||
ensureCalls []string
|
||||
// renameCalls 记录每次 Rename 的 fileID->newName 历史,用于 backfill 测试断言。
|
||||
renameCalls map[string]string
|
||||
}
|
||||
@@ -62,6 +64,7 @@ func newFakePikPak(id, rootID string) *fakePikPak {
|
||||
id: id,
|
||||
rootID: rootID,
|
||||
gotBodies: make(map[string][]byte),
|
||||
gotParents: make(map[string]string),
|
||||
renameCalls: make(map[string]string),
|
||||
}
|
||||
}
|
||||
@@ -80,8 +83,11 @@ func (d *fakePikPak) StreamURL(context.Context, string) (*drives.StreamLink, err
|
||||
func (d *fakePikPak) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakePikPak) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
func (d *fakePikPak) EnsureDir(_ context.Context, pathFromRoot string) (string, error) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.ensureCalls = append(d.ensureCalls, pathFromRoot)
|
||||
return d.rootID + "/" + pathFromRoot, nil
|
||||
}
|
||||
func (d *fakePikPak) Rename(_ context.Context, fileID, newName string) error {
|
||||
d.mu.Lock()
|
||||
@@ -99,6 +105,7 @@ func (d *fakePikPak) UploadAndReportHash(ctx context.Context, parentID, name str
|
||||
body, _ := io.ReadAll(r)
|
||||
d.mu.Lock()
|
||||
d.gotBodies[name] = body
|
||||
d.gotParents[name] = parentID
|
||||
d.mu.Unlock()
|
||||
return UploadResult{
|
||||
FileID: "remote-" + name,
|
||||
@@ -127,6 +134,19 @@ func (d *fakeP115) Kind() string { return "p115" }
|
||||
var _ drives.Drive = (*fakeP115)(nil)
|
||||
var _ uploadTarget = (*fakeP115)(nil)
|
||||
|
||||
type fakeOneDrive struct {
|
||||
*fakePikPak
|
||||
}
|
||||
|
||||
func newFakeOneDrive(id, rootID string) *fakeOneDrive {
|
||||
return &fakeOneDrive{fakePikPak: newFakePikPak(id, rootID)}
|
||||
}
|
||||
|
||||
func (d *fakeOneDrive) Kind() string { return "onedrive" }
|
||||
|
||||
var _ drives.Drive = (*fakeOneDrive)(nil)
|
||||
var _ uploadTarget = (*fakeOneDrive)(nil)
|
||||
|
||||
// TestBackfillFileNamesRenamesOnlyMismatchedSpider91Videos 验证回填逻辑:
|
||||
//
|
||||
// - 已经是期望格式的不会再调 Rename(幂等)
|
||||
@@ -347,6 +367,12 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
|
||||
if _, ok := pp.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("PikPak did not receive expected upload name %q (got names: %v)", wantName, keysOf(pp.gotBodies))
|
||||
}
|
||||
if gotParent := pp.gotParents[wantName]; gotParent != "pikpak-root-id/"+spider91UploadDirName {
|
||||
t.Fatalf("upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(pp.ensureCalls) != 1 || pp.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("ensure calls = %#v, want %q", pp.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
@@ -588,7 +614,7 @@ func TestRunOnceKeepsAllLocalWhenWithinKeepWindow(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestRunOnceMigratesOnlyOlderFilesBeyondKeepWindow 验证:本地文件数 > KeepLatestN 时
|
||||
// 按 mtime 降序保留最新 N 个,超出部分(更旧的)才上传到 PikPak。
|
||||
// 按 mtime 降序保留最新 N 个,超出部分(更旧的)才上传到目标盘。
|
||||
func TestRunOnceMigratesOnlyOlderFilesBeyondKeepWindow(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src, _ := setupSpider91(t)
|
||||
@@ -841,7 +867,6 @@ func TestNonCaptchaErrorDoesNotTriggerCooldown(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// TestRunOnceMigratesToP115Target 验证:当目标 drive 是 115(kind="p115")时,
|
||||
// migrator 也能正确把 spider91 视频上传过去并改写 catalog。
|
||||
//
|
||||
@@ -885,6 +910,12 @@ func TestRunOnceMigratesToP115Target(t *testing.T) {
|
||||
if _, ok := target.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("p115 did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
|
||||
}
|
||||
if gotParent := target.gotParents[wantName]; gotParent != "p115-root-cid/"+spider91UploadDirName {
|
||||
t.Fatalf("p115 upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("p115 ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
@@ -906,7 +937,67 @@ func TestRunOnceMigratesToP115Target(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak 也不是 115 时,
|
||||
func TestRunOnceMigratesToOneDriveTarget(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src, _ := setupSpider91(t)
|
||||
target := newFakeOneDrive("onedrive-target", "onedrive-root")
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(target)
|
||||
|
||||
now := time.Now()
|
||||
id := writeSpider91Video(t, cat, src, "vk-od-001", ".mp4", []byte("video bytes onedrive"), now)
|
||||
|
||||
m := New(Config{
|
||||
Catalog: cat,
|
||||
Registry: reg,
|
||||
GetTargetDriveID: func() string { return target.ID() },
|
||||
KeepLatestN: -1,
|
||||
})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
if target.uploadCalls != 1 {
|
||||
t.Fatalf("onedrive upload calls = %d, want 1", target.uploadCalls)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DriveID != target.ID() {
|
||||
t.Fatalf("drive_id = %q, want %q", got.DriveID, target.ID())
|
||||
}
|
||||
wantName := "Sample vk-od-001-001.mp4"
|
||||
if _, ok := target.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("onedrive did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
|
||||
}
|
||||
if gotParent := target.gotParents[wantName]; gotParent != "onedrive-root/"+spider91UploadDirName {
|
||||
t.Fatalf("onedrive upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("onedrive ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
if got.FileName != wantName {
|
||||
t.Fatalf("file_name = %q, want %q", got.FileName, wantName)
|
||||
}
|
||||
if got.ContentHash == "" {
|
||||
t.Fatal("content_hash should be set after onedrive migration")
|
||||
}
|
||||
|
||||
videoPath, _ := src.VideoPath("vk-od-001.mp4")
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local mp4 still exists after onedrive migration or stat error: %v", err)
|
||||
}
|
||||
thumbPath, _ := src.ThumbPath("vk-od-001.jpg")
|
||||
if _, err := os.Stat(thumbPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local thumb still exists after onedrive migration or stat error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115 也不是 OneDrive 时,
|
||||
// resolveTarget 拒绝并返回 error,让 runOnce 静默跳过(不会做破坏性变更)。
|
||||
func TestResolveTargetRejectsUnsupportedKind(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
// Copyright 2018 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package socks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
noDeadline = time.Time{}
|
||||
aLongTimeAgo = time.Unix(1, 0)
|
||||
)
|
||||
|
||||
func (d *Dialer) connect(ctx context.Context, c net.Conn, address string) (_ net.Addr, ctxErr error) {
|
||||
host, port, err := splitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if deadline, ok := ctx.Deadline(); ok && !deadline.IsZero() {
|
||||
c.SetDeadline(deadline)
|
||||
defer c.SetDeadline(noDeadline)
|
||||
}
|
||||
if ctx != context.Background() {
|
||||
errCh := make(chan error, 1)
|
||||
done := make(chan struct{})
|
||||
defer func() {
|
||||
close(done)
|
||||
if ctxErr == nil {
|
||||
ctxErr = <-errCh
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
c.SetDeadline(aLongTimeAgo)
|
||||
errCh <- ctx.Err()
|
||||
case <-done:
|
||||
errCh <- nil
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
b := make([]byte, 0, 6+len(host)) // the size here is just an estimate
|
||||
b = append(b, Version5)
|
||||
if len(d.AuthMethods) == 0 || d.Authenticate == nil {
|
||||
b = append(b, 1, byte(AuthMethodNotRequired))
|
||||
} else {
|
||||
ams := d.AuthMethods
|
||||
if len(ams) > 255 {
|
||||
return nil, errors.New("too many authentication methods")
|
||||
}
|
||||
b = append(b, byte(len(ams)))
|
||||
for _, am := range ams {
|
||||
b = append(b, byte(am))
|
||||
}
|
||||
}
|
||||
if _, ctxErr = c.Write(b); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if _, ctxErr = io.ReadFull(c, b[:2]); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
if b[0] != Version5 {
|
||||
return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0])))
|
||||
}
|
||||
am := AuthMethod(b[1])
|
||||
if am == AuthMethodNoAcceptableMethods {
|
||||
return nil, errors.New("no acceptable authentication methods")
|
||||
}
|
||||
if d.Authenticate != nil {
|
||||
if ctxErr = d.Authenticate(ctx, c, am); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
b = b[:0]
|
||||
b = append(b, Version5, byte(d.cmd), 0)
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
b = append(b, AddrTypeIPv4)
|
||||
b = append(b, ip4...)
|
||||
} else if ip6 := ip.To16(); ip6 != nil {
|
||||
b = append(b, AddrTypeIPv6)
|
||||
b = append(b, ip6...)
|
||||
} else {
|
||||
return nil, errors.New("unknown address type")
|
||||
}
|
||||
} else {
|
||||
if len(host) > 255 {
|
||||
return nil, errors.New("FQDN too long")
|
||||
}
|
||||
b = append(b, AddrTypeFQDN)
|
||||
b = append(b, byte(len(host)))
|
||||
b = append(b, host...)
|
||||
}
|
||||
b = append(b, byte(port>>8), byte(port))
|
||||
if _, ctxErr = c.Write(b); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if _, ctxErr = io.ReadFull(c, b[:4]); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
if b[0] != Version5 {
|
||||
return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0])))
|
||||
}
|
||||
if cmdErr := Reply(b[1]); cmdErr != StatusSucceeded {
|
||||
return nil, errors.New("unknown error " + cmdErr.String())
|
||||
}
|
||||
if b[2] != 0 {
|
||||
return nil, errors.New("non-zero reserved field")
|
||||
}
|
||||
l := 2
|
||||
var a Addr
|
||||
switch b[3] {
|
||||
case AddrTypeIPv4:
|
||||
l += net.IPv4len
|
||||
a.IP = make(net.IP, net.IPv4len)
|
||||
case AddrTypeIPv6:
|
||||
l += net.IPv6len
|
||||
a.IP = make(net.IP, net.IPv6len)
|
||||
case AddrTypeFQDN:
|
||||
if _, err := io.ReadFull(c, b[:1]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l += int(b[0])
|
||||
default:
|
||||
return nil, errors.New("unknown address type " + strconv.Itoa(int(b[3])))
|
||||
}
|
||||
if cap(b) < l {
|
||||
b = make([]byte, l)
|
||||
} else {
|
||||
b = b[:l]
|
||||
}
|
||||
if _, ctxErr = io.ReadFull(c, b); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
if a.IP != nil {
|
||||
copy(a.IP, b)
|
||||
} else {
|
||||
a.Name = string(b[:len(b)-2])
|
||||
}
|
||||
a.Port = int(b[len(b)-2])<<8 | int(b[len(b)-1])
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
func splitHostPort(address string) (string, int, error) {
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
portnum, err := strconv.Atoi(port)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if 1 > portnum || portnum > 0xffff {
|
||||
return "", 0, errors.New("port number out of range " + port)
|
||||
}
|
||||
return host, portnum, nil
|
||||
}
|
||||
+317
@@ -0,0 +1,317 @@
|
||||
// Copyright 2018 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package socks provides a SOCKS version 5 client implementation.
|
||||
//
|
||||
// SOCKS protocol version 5 is defined in RFC 1928.
|
||||
// Username/Password authentication for SOCKS version 5 is defined in
|
||||
// RFC 1929.
|
||||
package socks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// A Command represents a SOCKS command.
|
||||
type Command int
|
||||
|
||||
func (cmd Command) String() string {
|
||||
switch cmd {
|
||||
case CmdConnect:
|
||||
return "socks connect"
|
||||
case cmdBind:
|
||||
return "socks bind"
|
||||
default:
|
||||
return "socks " + strconv.Itoa(int(cmd))
|
||||
}
|
||||
}
|
||||
|
||||
// An AuthMethod represents a SOCKS authentication method.
|
||||
type AuthMethod int
|
||||
|
||||
// A Reply represents a SOCKS command reply code.
|
||||
type Reply int
|
||||
|
||||
func (code Reply) String() string {
|
||||
switch code {
|
||||
case StatusSucceeded:
|
||||
return "succeeded"
|
||||
case 0x01:
|
||||
return "general SOCKS server failure"
|
||||
case 0x02:
|
||||
return "connection not allowed by ruleset"
|
||||
case 0x03:
|
||||
return "network unreachable"
|
||||
case 0x04:
|
||||
return "host unreachable"
|
||||
case 0x05:
|
||||
return "connection refused"
|
||||
case 0x06:
|
||||
return "TTL expired"
|
||||
case 0x07:
|
||||
return "command not supported"
|
||||
case 0x08:
|
||||
return "address type not supported"
|
||||
default:
|
||||
return "unknown code: " + strconv.Itoa(int(code))
|
||||
}
|
||||
}
|
||||
|
||||
// Wire protocol constants.
|
||||
const (
|
||||
Version5 = 0x05
|
||||
|
||||
AddrTypeIPv4 = 0x01
|
||||
AddrTypeFQDN = 0x03
|
||||
AddrTypeIPv6 = 0x04
|
||||
|
||||
CmdConnect Command = 0x01 // establishes an active-open forward proxy connection
|
||||
cmdBind Command = 0x02 // establishes a passive-open forward proxy connection
|
||||
|
||||
AuthMethodNotRequired AuthMethod = 0x00 // no authentication required
|
||||
AuthMethodUsernamePassword AuthMethod = 0x02 // use username/password
|
||||
AuthMethodNoAcceptableMethods AuthMethod = 0xff // no acceptable authentication methods
|
||||
|
||||
StatusSucceeded Reply = 0x00
|
||||
)
|
||||
|
||||
// An Addr represents a SOCKS-specific address.
|
||||
// Either Name or IP is used exclusively.
|
||||
type Addr struct {
|
||||
Name string // fully-qualified domain name
|
||||
IP net.IP
|
||||
Port int
|
||||
}
|
||||
|
||||
func (a *Addr) Network() string { return "socks" }
|
||||
|
||||
func (a *Addr) String() string {
|
||||
if a == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
port := strconv.Itoa(a.Port)
|
||||
if a.IP == nil {
|
||||
return net.JoinHostPort(a.Name, port)
|
||||
}
|
||||
return net.JoinHostPort(a.IP.String(), port)
|
||||
}
|
||||
|
||||
// A Conn represents a forward proxy connection.
|
||||
type Conn struct {
|
||||
net.Conn
|
||||
|
||||
boundAddr net.Addr
|
||||
}
|
||||
|
||||
// BoundAddr returns the address assigned by the proxy server for
|
||||
// connecting to the command target address from the proxy server.
|
||||
func (c *Conn) BoundAddr() net.Addr {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
return c.boundAddr
|
||||
}
|
||||
|
||||
// A Dialer holds SOCKS-specific options.
|
||||
type Dialer struct {
|
||||
cmd Command // either CmdConnect or cmdBind
|
||||
proxyNetwork string // network between a proxy server and a client
|
||||
proxyAddress string // proxy server address
|
||||
|
||||
// ProxyDial specifies the optional dial function for
|
||||
// establishing the transport connection.
|
||||
ProxyDial func(context.Context, string, string) (net.Conn, error)
|
||||
|
||||
// AuthMethods specifies the list of request authentication
|
||||
// methods.
|
||||
// If empty, SOCKS client requests only AuthMethodNotRequired.
|
||||
AuthMethods []AuthMethod
|
||||
|
||||
// Authenticate specifies the optional authentication
|
||||
// function. It must be non-nil when AuthMethods is not empty.
|
||||
// It must return an error when the authentication is failed.
|
||||
Authenticate func(context.Context, io.ReadWriter, AuthMethod) error
|
||||
}
|
||||
|
||||
// DialContext connects to the provided address on the provided
|
||||
// network.
|
||||
//
|
||||
// The returned error value may be a net.OpError. When the Op field of
|
||||
// net.OpError contains "socks", the Source field contains a proxy
|
||||
// server address and the Addr field contains a command target
|
||||
// address.
|
||||
//
|
||||
// See func Dial of the net package of standard library for a
|
||||
// description of the network and address parameters.
|
||||
func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
if err := d.validateTarget(network, address); err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
if ctx == nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")}
|
||||
}
|
||||
var err error
|
||||
var c net.Conn
|
||||
if d.ProxyDial != nil {
|
||||
c, err = d.ProxyDial(ctx, d.proxyNetwork, d.proxyAddress)
|
||||
} else {
|
||||
var dd net.Dialer
|
||||
c, err = dd.DialContext(ctx, d.proxyNetwork, d.proxyAddress)
|
||||
}
|
||||
if err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
a, err := d.connect(ctx, c, address)
|
||||
if err != nil {
|
||||
c.Close()
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
return &Conn{Conn: c, boundAddr: a}, nil
|
||||
}
|
||||
|
||||
// DialWithConn initiates a connection from SOCKS server to the target
|
||||
// network and address using the connection c that is already
|
||||
// connected to the SOCKS server.
|
||||
//
|
||||
// It returns the connection's local address assigned by the SOCKS
|
||||
// server.
|
||||
func (d *Dialer) DialWithConn(ctx context.Context, c net.Conn, network, address string) (net.Addr, error) {
|
||||
if err := d.validateTarget(network, address); err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
if ctx == nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")}
|
||||
}
|
||||
a, err := d.connect(ctx, c, address)
|
||||
if err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// Dial connects to the provided address on the provided network.
|
||||
//
|
||||
// Unlike DialContext, it returns a raw transport connection instead
|
||||
// of a forward proxy connection.
|
||||
//
|
||||
// Deprecated: Use DialContext or DialWithConn instead.
|
||||
func (d *Dialer) Dial(network, address string) (net.Conn, error) {
|
||||
if err := d.validateTarget(network, address); err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
var err error
|
||||
var c net.Conn
|
||||
if d.ProxyDial != nil {
|
||||
c, err = d.ProxyDial(context.Background(), d.proxyNetwork, d.proxyAddress)
|
||||
} else {
|
||||
c, err = net.Dial(d.proxyNetwork, d.proxyAddress)
|
||||
}
|
||||
if err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
if _, err := d.DialWithConn(context.Background(), c, network, address); err != nil {
|
||||
c.Close()
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (d *Dialer) validateTarget(network, address string) error {
|
||||
switch network {
|
||||
case "tcp", "tcp6", "tcp4":
|
||||
default:
|
||||
return errors.New("network not implemented")
|
||||
}
|
||||
switch d.cmd {
|
||||
case CmdConnect, cmdBind:
|
||||
default:
|
||||
return errors.New("command not implemented")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Dialer) pathAddrs(address string) (proxy, dst net.Addr, err error) {
|
||||
for i, s := range []string{d.proxyAddress, address} {
|
||||
host, port, err := splitHostPort(s)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
a := &Addr{Port: port}
|
||||
a.IP = net.ParseIP(host)
|
||||
if a.IP == nil {
|
||||
a.Name = host
|
||||
}
|
||||
if i == 0 {
|
||||
proxy = a
|
||||
} else {
|
||||
dst = a
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// NewDialer returns a new Dialer that dials through the provided
|
||||
// proxy server's network and address.
|
||||
func NewDialer(network, address string) *Dialer {
|
||||
return &Dialer{proxyNetwork: network, proxyAddress: address, cmd: CmdConnect}
|
||||
}
|
||||
|
||||
const (
|
||||
authUsernamePasswordVersion = 0x01
|
||||
authStatusSucceeded = 0x00
|
||||
)
|
||||
|
||||
// UsernamePassword are the credentials for the username/password
|
||||
// authentication method.
|
||||
type UsernamePassword struct {
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
// Authenticate authenticates a pair of username and password with the
|
||||
// proxy server.
|
||||
func (up *UsernamePassword) Authenticate(ctx context.Context, rw io.ReadWriter, auth AuthMethod) error {
|
||||
switch auth {
|
||||
case AuthMethodNotRequired:
|
||||
return nil
|
||||
case AuthMethodUsernamePassword:
|
||||
if len(up.Username) == 0 || len(up.Username) > 255 || len(up.Password) > 255 {
|
||||
return errors.New("invalid username/password")
|
||||
}
|
||||
b := []byte{authUsernamePasswordVersion}
|
||||
b = append(b, byte(len(up.Username)))
|
||||
b = append(b, up.Username...)
|
||||
b = append(b, byte(len(up.Password)))
|
||||
b = append(b, up.Password...)
|
||||
// TODO(mikio): handle IO deadlines and cancelation if
|
||||
// necessary
|
||||
if _, err := rw.Write(b); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.ReadFull(rw, b[:2]); err != nil {
|
||||
return err
|
||||
}
|
||||
if b[0] != authUsernamePasswordVersion {
|
||||
return errors.New("invalid username/password version")
|
||||
}
|
||||
if b[1] != authStatusSucceeded {
|
||||
return errors.New("username/password authentication failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errors.New("unsupported authentication method " + strconv.Itoa(int(auth)))
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
// Copyright 2019 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
)
|
||||
|
||||
// A ContextDialer dials using a context.
|
||||
type ContextDialer interface {
|
||||
DialContext(ctx context.Context, network, address string) (net.Conn, error)
|
||||
}
|
||||
|
||||
// Dial works like DialContext on net.Dialer but using a dialer returned by FromEnvironment.
|
||||
//
|
||||
// The passed ctx is only used for returning the Conn, not the lifetime of the Conn.
|
||||
//
|
||||
// Custom dialers (registered via RegisterDialerType) that do not implement ContextDialer
|
||||
// can leak a goroutine for as long as it takes the underlying Dialer implementation to timeout.
|
||||
//
|
||||
// A Conn returned from a successful Dial after the context has been cancelled will be immediately closed.
|
||||
func Dial(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
d := FromEnvironment()
|
||||
if xd, ok := d.(ContextDialer); ok {
|
||||
return xd.DialContext(ctx, network, address)
|
||||
}
|
||||
return dialContext(ctx, d, network, address)
|
||||
}
|
||||
|
||||
// WARNING: this can leak a goroutine for as long as the underlying Dialer implementation takes to timeout
|
||||
// A Conn returned from a successful Dial after the context has been cancelled will be immediately closed.
|
||||
func dialContext(ctx context.Context, d Dialer, network, address string) (net.Conn, error) {
|
||||
var (
|
||||
conn net.Conn
|
||||
done = make(chan struct{}, 1)
|
||||
err error
|
||||
)
|
||||
go func() {
|
||||
conn, err = d.Dial(network, address)
|
||||
close(done)
|
||||
if conn != nil && ctx.Err() != nil {
|
||||
conn.Close()
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
case <-done:
|
||||
}
|
||||
return conn, err
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
)
|
||||
|
||||
type direct struct{}
|
||||
|
||||
// Direct implements Dialer by making network connections directly using net.Dial or net.DialContext.
|
||||
var Direct = direct{}
|
||||
|
||||
var (
|
||||
_ Dialer = Direct
|
||||
_ ContextDialer = Direct
|
||||
)
|
||||
|
||||
// Dial directly invokes net.Dial with the supplied parameters.
|
||||
func (direct) Dial(network, addr string) (net.Conn, error) {
|
||||
return net.Dial(network, addr)
|
||||
}
|
||||
|
||||
// DialContext instantiates a net.Dialer and invokes its DialContext receiver with the supplied parameters.
|
||||
func (direct) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, network, addr)
|
||||
}
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A PerHost directs connections to a default Dialer unless the host name
|
||||
// requested matches one of a number of exceptions.
|
||||
type PerHost struct {
|
||||
def, bypass Dialer
|
||||
|
||||
bypassNetworks []*net.IPNet
|
||||
bypassIPs []net.IP
|
||||
bypassZones []string
|
||||
bypassHosts []string
|
||||
}
|
||||
|
||||
// NewPerHost returns a PerHost Dialer that directs connections to either
|
||||
// defaultDialer or bypass, depending on whether the connection matches one of
|
||||
// the configured rules.
|
||||
func NewPerHost(defaultDialer, bypass Dialer) *PerHost {
|
||||
return &PerHost{
|
||||
def: defaultDialer,
|
||||
bypass: bypass,
|
||||
}
|
||||
}
|
||||
|
||||
// Dial connects to the address addr on the given network through either
|
||||
// defaultDialer or bypass.
|
||||
func (p *PerHost) Dial(network, addr string) (c net.Conn, err error) {
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return p.dialerForRequest(host).Dial(network, addr)
|
||||
}
|
||||
|
||||
// DialContext connects to the address addr on the given network through either
|
||||
// defaultDialer or bypass.
|
||||
func (p *PerHost) DialContext(ctx context.Context, network, addr string) (c net.Conn, err error) {
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := p.dialerForRequest(host)
|
||||
if x, ok := d.(ContextDialer); ok {
|
||||
return x.DialContext(ctx, network, addr)
|
||||
}
|
||||
return dialContext(ctx, d, network, addr)
|
||||
}
|
||||
|
||||
func (p *PerHost) dialerForRequest(host string) Dialer {
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
for _, net := range p.bypassNetworks {
|
||||
if net.Contains(ip) {
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
for _, bypassIP := range p.bypassIPs {
|
||||
if bypassIP.Equal(ip) {
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
return p.def
|
||||
}
|
||||
|
||||
for _, zone := range p.bypassZones {
|
||||
if strings.HasSuffix(host, zone) {
|
||||
return p.bypass
|
||||
}
|
||||
if host == zone[1:] {
|
||||
// For a zone ".example.com", we match "example.com"
|
||||
// too.
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
for _, bypassHost := range p.bypassHosts {
|
||||
if bypassHost == host {
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
return p.def
|
||||
}
|
||||
|
||||
// AddFromString parses a string that contains comma-separated values
|
||||
// specifying hosts that should use the bypass proxy. Each value is either an
|
||||
// IP address, a CIDR range, a zone (*.example.com) or a host name
|
||||
// (localhost). A best effort is made to parse the string and errors are
|
||||
// ignored.
|
||||
func (p *PerHost) AddFromString(s string) {
|
||||
hosts := strings.Split(s, ",")
|
||||
for _, host := range hosts {
|
||||
host = strings.TrimSpace(host)
|
||||
if len(host) == 0 {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(host, "/") {
|
||||
// We assume that it's a CIDR address like 127.0.0.0/8
|
||||
if _, net, err := net.ParseCIDR(host); err == nil {
|
||||
p.AddNetwork(net)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
p.AddIP(ip)
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(host, "*.") {
|
||||
p.AddZone(host[1:])
|
||||
continue
|
||||
}
|
||||
p.AddHost(host)
|
||||
}
|
||||
}
|
||||
|
||||
// AddIP specifies an IP address that will use the bypass proxy. Note that
|
||||
// this will only take effect if a literal IP address is dialed. A connection
|
||||
// to a named host will never match an IP.
|
||||
func (p *PerHost) AddIP(ip net.IP) {
|
||||
p.bypassIPs = append(p.bypassIPs, ip)
|
||||
}
|
||||
|
||||
// AddNetwork specifies an IP range that will use the bypass proxy. Note that
|
||||
// this will only take effect if a literal IP address is dialed. A connection
|
||||
// to a named host will never match.
|
||||
func (p *PerHost) AddNetwork(net *net.IPNet) {
|
||||
p.bypassNetworks = append(p.bypassNetworks, net)
|
||||
}
|
||||
|
||||
// AddZone specifies a DNS suffix that will use the bypass proxy. A zone of
|
||||
// "example.com" matches "example.com" and all of its subdomains.
|
||||
func (p *PerHost) AddZone(zone string) {
|
||||
zone = strings.TrimSuffix(zone, ".")
|
||||
if !strings.HasPrefix(zone, ".") {
|
||||
zone = "." + zone
|
||||
}
|
||||
p.bypassZones = append(p.bypassZones, zone)
|
||||
}
|
||||
|
||||
// AddHost specifies a host name that will use the bypass proxy.
|
||||
func (p *PerHost) AddHost(host string) {
|
||||
host = strings.TrimSuffix(host, ".")
|
||||
p.bypassHosts = append(p.bypassHosts, host)
|
||||
}
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package proxy provides support for a variety of protocols to proxy network
|
||||
// data.
|
||||
package proxy // import "golang.org/x/net/proxy"
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// A Dialer is a means to establish a connection.
|
||||
// Custom dialers should also implement ContextDialer.
|
||||
type Dialer interface {
|
||||
// Dial connects to the given address via the proxy.
|
||||
Dial(network, addr string) (c net.Conn, err error)
|
||||
}
|
||||
|
||||
// Auth contains authentication parameters that specific Dialers may require.
|
||||
type Auth struct {
|
||||
User, Password string
|
||||
}
|
||||
|
||||
// FromEnvironment returns the dialer specified by the proxy-related
|
||||
// variables in the environment and makes underlying connections
|
||||
// directly.
|
||||
func FromEnvironment() Dialer {
|
||||
return FromEnvironmentUsing(Direct)
|
||||
}
|
||||
|
||||
// FromEnvironmentUsing returns the dialer specify by the proxy-related
|
||||
// variables in the environment and makes underlying connections
|
||||
// using the provided forwarding Dialer (for instance, a *net.Dialer
|
||||
// with desired configuration).
|
||||
func FromEnvironmentUsing(forward Dialer) Dialer {
|
||||
allProxy := allProxyEnv.Get()
|
||||
if len(allProxy) == 0 {
|
||||
return forward
|
||||
}
|
||||
|
||||
proxyURL, err := url.Parse(allProxy)
|
||||
if err != nil {
|
||||
return forward
|
||||
}
|
||||
proxy, err := FromURL(proxyURL, forward)
|
||||
if err != nil {
|
||||
return forward
|
||||
}
|
||||
|
||||
noProxy := noProxyEnv.Get()
|
||||
if len(noProxy) == 0 {
|
||||
return proxy
|
||||
}
|
||||
|
||||
perHost := NewPerHost(proxy, forward)
|
||||
perHost.AddFromString(noProxy)
|
||||
return perHost
|
||||
}
|
||||
|
||||
// proxySchemes is a map from URL schemes to a function that creates a Dialer
|
||||
// from a URL with such a scheme.
|
||||
var proxySchemes map[string]func(*url.URL, Dialer) (Dialer, error)
|
||||
|
||||
// RegisterDialerType takes a URL scheme and a function to generate Dialers from
|
||||
// a URL with that scheme and a forwarding Dialer. Registered schemes are used
|
||||
// by FromURL.
|
||||
func RegisterDialerType(scheme string, f func(*url.URL, Dialer) (Dialer, error)) {
|
||||
if proxySchemes == nil {
|
||||
proxySchemes = make(map[string]func(*url.URL, Dialer) (Dialer, error))
|
||||
}
|
||||
proxySchemes[scheme] = f
|
||||
}
|
||||
|
||||
// FromURL returns a Dialer given a URL specification and an underlying
|
||||
// Dialer for it to make network requests.
|
||||
func FromURL(u *url.URL, forward Dialer) (Dialer, error) {
|
||||
var auth *Auth
|
||||
if u.User != nil {
|
||||
auth = new(Auth)
|
||||
auth.User = u.User.Username()
|
||||
if p, ok := u.User.Password(); ok {
|
||||
auth.Password = p
|
||||
}
|
||||
}
|
||||
|
||||
switch u.Scheme {
|
||||
case "socks5", "socks5h":
|
||||
addr := u.Hostname()
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
port = "1080"
|
||||
}
|
||||
return SOCKS5("tcp", net.JoinHostPort(addr, port), auth, forward)
|
||||
}
|
||||
|
||||
// If the scheme doesn't match any of the built-in schemes, see if it
|
||||
// was registered by another package.
|
||||
if proxySchemes != nil {
|
||||
if f, ok := proxySchemes[u.Scheme]; ok {
|
||||
return f(u, forward)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.New("proxy: unknown scheme: " + u.Scheme)
|
||||
}
|
||||
|
||||
var (
|
||||
allProxyEnv = &envOnce{
|
||||
names: []string{"ALL_PROXY", "all_proxy"},
|
||||
}
|
||||
noProxyEnv = &envOnce{
|
||||
names: []string{"NO_PROXY", "no_proxy"},
|
||||
}
|
||||
)
|
||||
|
||||
// envOnce looks up an environment variable (optionally by multiple
|
||||
// names) once. It mitigates expensive lookups on some platforms
|
||||
// (e.g. Windows).
|
||||
// (Borrowed from net/http/transport.go)
|
||||
type envOnce struct {
|
||||
names []string
|
||||
once sync.Once
|
||||
val string
|
||||
}
|
||||
|
||||
func (e *envOnce) Get() string {
|
||||
e.once.Do(e.init)
|
||||
return e.val
|
||||
}
|
||||
|
||||
func (e *envOnce) init() {
|
||||
for _, n := range e.names {
|
||||
e.val = os.Getenv(n)
|
||||
if e.val != "" {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reset is used by tests
|
||||
func (e *envOnce) reset() {
|
||||
e.once = sync.Once{}
|
||||
e.val = ""
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
|
||||
"golang.org/x/net/internal/socks"
|
||||
)
|
||||
|
||||
// SOCKS5 returns a Dialer that makes SOCKSv5 connections to the given
|
||||
// address with an optional username and password.
|
||||
// See RFC 1928 and RFC 1929.
|
||||
func SOCKS5(network, address string, auth *Auth, forward Dialer) (Dialer, error) {
|
||||
d := socks.NewDialer(network, address)
|
||||
if forward != nil {
|
||||
if f, ok := forward.(ContextDialer); ok {
|
||||
d.ProxyDial = func(ctx context.Context, network string, address string) (net.Conn, error) {
|
||||
return f.DialContext(ctx, network, address)
|
||||
}
|
||||
} else {
|
||||
d.ProxyDial = func(ctx context.Context, network string, address string) (net.Conn, error) {
|
||||
return dialContext(ctx, forward, network, address)
|
||||
}
|
||||
}
|
||||
}
|
||||
if auth != nil {
|
||||
up := socks.UsernamePassword{
|
||||
Username: auth.User,
|
||||
Password: auth.Password,
|
||||
}
|
||||
d.AuthMethods = []socks.AuthMethod{
|
||||
socks.AuthMethodNotRequired,
|
||||
socks.AuthMethodUsernamePassword,
|
||||
}
|
||||
d.Authenticate = up.Authenticate
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
Vendored
+2
@@ -67,6 +67,8 @@ github.com/skip2/go-qrcode/reedsolomon
|
||||
golang.org/x/crypto/curve25519
|
||||
# golang.org/x/net v0.27.0
|
||||
## explicit; go 1.18
|
||||
golang.org/x/net/internal/socks
|
||||
golang.org/x/net/proxy
|
||||
golang.org/x/net/publicsuffix
|
||||
# golang.org/x/sys v0.30.0
|
||||
## explicit; go 1.18
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
services:
|
||||
video-site-91:
|
||||
image: ghcr.io/nianzhibai/91:stable
|
||||
container_name: video-site-91
|
||||
ports:
|
||||
- "9191:9191"
|
||||
volumes:
|
||||
- ./data:/opt/video-site-91/data
|
||||
restart: unless-stopped
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
APP_DIR="/opt/video-site-91"
|
||||
DATA_DIR="${VIDEO_DATA_DIR:-$APP_DIR/data}"
|
||||
CONFIG="${VIDEO_CONFIG:-$DATA_DIR/config.yaml}"
|
||||
EXAMPLE="$APP_DIR/config.example.yaml"
|
||||
PORT="${VIDEO_LISTEN_PORT:-9191}"
|
||||
|
||||
mkdir -p "$DATA_DIR" "$DATA_DIR/previews" "$DATA_DIR/uploads" "$DATA_DIR/spider91"
|
||||
|
||||
if [ ! -f "$CONFIG" ]; then
|
||||
if [ ! -f "$EXAMPLE" ]; then
|
||||
echo "[entrypoint] missing config template: $EXAMPLE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$(dirname "$CONFIG")"
|
||||
cp "$EXAMPLE" "$CONFIG"
|
||||
|
||||
SECRET="$(openssl rand -hex 32)"
|
||||
sed -i -E "s#^([[:space:]]*listen:[[:space:]]*).*\$#\1\"0.0.0.0:${PORT}\"#" "$CONFIG"
|
||||
sed -i -E "s#^([[:space:]]*session_secret:[[:space:]]*).*\$#\1\"${SECRET}\"#" "$CONFIG"
|
||||
sed -i -E "s#^([[:space:]]*db_path:[[:space:]]*).*\$#\1\"${DATA_DIR}/video-site.db\"#" "$CONFIG"
|
||||
sed -i -E "s#^([[:space:]]*local_preview_dir:[[:space:]]*).*\$#\1\"${DATA_DIR}/previews\"#" "$CONFIG"
|
||||
chmod 600 "$CONFIG"
|
||||
|
||||
echo "[entrypoint] generated $CONFIG"
|
||||
else
|
||||
echo "[entrypoint] using existing $CONFIG"
|
||||
fi
|
||||
|
||||
if [ -n "${VIDEO_VERSION_FILE:-}" ] && [ -n "${VIDEO_IMAGE_VERSION:-}" ]; then
|
||||
mkdir -p "$(dirname "$VIDEO_VERSION_FILE")"
|
||||
printf '%s\n' "$VIDEO_IMAGE_VERSION" > "$VIDEO_VERSION_FILE"
|
||||
fi
|
||||
|
||||
exec "$@"
|
||||
+420
-38
@@ -11,6 +11,12 @@ VERSION="${VERSION:-latest}"
|
||||
GH_PROXY="${GH_PROXY:-}"
|
||||
CONFIGURE_UFW="${CONFIGURE_UFW:-1}"
|
||||
INSTALL_DEPS="${INSTALL_DEPS:-1}"
|
||||
SELF_UPDATE="${SELF_UPDATE:-1}"
|
||||
FORCE_UPDATE="${FORCE_UPDATE:-0}"
|
||||
INSTALL_SCRIPT_REF="${INSTALL_SCRIPT_REF:-main}"
|
||||
INSTALL_SCRIPT_URL="${INSTALL_SCRIPT_URL:-${GH_PROXY}https://raw.githubusercontent.com/${GITHUB_REPO}/${INSTALL_SCRIPT_REF}/install.sh}"
|
||||
VIDEO_SITE_SKIP_SELF_UPDATE="${VIDEO_SITE_SKIP_SELF_UPDATE:-0}"
|
||||
SERVICE_READY_TIMEOUT="${SERVICE_READY_TIMEOUT:-90}"
|
||||
VERSION_FILE="$INSTALL_PATH/.version"
|
||||
MANAGER_PATH="/usr/local/sbin/${APP_NAME}-manager"
|
||||
COMMAND_LINK="/usr/local/bin/91"
|
||||
@@ -47,7 +53,7 @@ Default action:
|
||||
|
||||
Actions:
|
||||
install Install to $INSTALL_PATH
|
||||
update Download latest release and replace program files, keeping config/data
|
||||
update Refresh manager script, download latest release, and keep config/data
|
||||
restart Restart service
|
||||
stop Stop service
|
||||
status Show service status
|
||||
@@ -62,6 +68,12 @@ Options via environment:
|
||||
GH_PROXY=$GH_PROXY
|
||||
INSTALL_DEPS=$INSTALL_DEPS
|
||||
CONFIGURE_UFW=$CONFIGURE_UFW
|
||||
SELF_UPDATE=$SELF_UPDATE
|
||||
FORCE_UPDATE=$FORCE_UPDATE
|
||||
UNINSTALL_DELETE_FILES=0 Set to 1 for non-interactive uninstall to delete $INSTALL_PATH
|
||||
INSTALL_SCRIPT_REF=$INSTALL_SCRIPT_REF
|
||||
INSTALL_SCRIPT_URL=$INSTALL_SCRIPT_URL
|
||||
SERVICE_READY_TIMEOUT=$SERVICE_READY_TIMEOUT
|
||||
|
||||
Examples:
|
||||
sudo bash install.sh
|
||||
@@ -118,7 +130,7 @@ install_deps() {
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
log "installing runtime dependencies"
|
||||
apt-get update
|
||||
apt-get install -y ca-certificates curl tar ffmpeg openssl iproute2 python3 python3-requests python3-bs4 python3-lxml
|
||||
apt-get install -y ca-certificates curl tar ffmpeg openssl iproute2 python3 python3-requests python3-bs4 python3-lxml python3-socks
|
||||
return
|
||||
fi
|
||||
|
||||
@@ -158,6 +170,30 @@ download_file() {
|
||||
return 1
|
||||
}
|
||||
|
||||
backup_install_files() {
|
||||
local backup="$1"
|
||||
mkdir -p "$backup"
|
||||
cp -a "$INSTALL_PATH/server" "$backup/server"
|
||||
for item in dist config.example.yaml 91VideoSpider config.yaml .version; do
|
||||
if [[ -e "$INSTALL_PATH/$item" ]]; then
|
||||
cp -a "$INSTALL_PATH/$item" "$backup/$item"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
restore_install_files() {
|
||||
local backup="$1"
|
||||
mkdir -p "$INSTALL_PATH"
|
||||
cp -a "$backup/server" "$INSTALL_PATH/server"
|
||||
for item in dist config.example.yaml 91VideoSpider config.yaml .version; do
|
||||
rm -rf "${INSTALL_PATH:?}/$item"
|
||||
if [[ -e "$backup/$item" ]]; then
|
||||
cp -a "$backup/$item" "$INSTALL_PATH/$item"
|
||||
fi
|
||||
done
|
||||
chmod +x "$INSTALL_PATH/server"
|
||||
}
|
||||
|
||||
prepare_config() {
|
||||
local cfg="$INSTALL_PATH/config.yaml"
|
||||
local example="$INSTALL_PATH/config.example.yaml"
|
||||
@@ -200,6 +236,8 @@ RestartSec=5
|
||||
TimeoutStopSec=20
|
||||
Environment=VIDEO_CONFIG=${INSTALL_PATH}/config.yaml
|
||||
Environment=VIDEO_FRONTEND_DIR=${INSTALL_PATH}/dist
|
||||
Environment=VIDEO_VERSION_FILE=${VERSION_FILE}
|
||||
Environment=VIDEO_GITHUB_REPO=${GITHUB_REPO}
|
||||
Environment=HOME=/root
|
||||
Environment=PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
|
||||
LimitNOFILE=65536
|
||||
@@ -217,23 +255,297 @@ EOF
|
||||
install_cli() {
|
||||
local src
|
||||
src="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")"
|
||||
if [[ -f "$src" ]]; then
|
||||
cp "$src" "$MANAGER_PATH"
|
||||
chmod 755 "$MANAGER_PATH"
|
||||
ln -sf "$MANAGER_PATH" "$COMMAND_LINK"
|
||||
ln -sf "$MANAGER_PATH" "$APP_COMMAND_LINK"
|
||||
install_cli_from_file "$src"
|
||||
}
|
||||
|
||||
install_cli_from_file() {
|
||||
local src="$1"
|
||||
local tmp
|
||||
[[ -f "$src" ]] || return 0
|
||||
mkdir -p "$(dirname "$MANAGER_PATH")" "$(dirname "$COMMAND_LINK")" "$(dirname "$APP_COMMAND_LINK")"
|
||||
tmp="${MANAGER_PATH}.tmp.$$"
|
||||
cp "$src" "$tmp"
|
||||
chmod 755 "$tmp"
|
||||
mv "$tmp" "$MANAGER_PATH"
|
||||
ln -sfn "$MANAGER_PATH" "$COMMAND_LINK"
|
||||
ln -sfn "$MANAGER_PATH" "$APP_COMMAND_LINK"
|
||||
}
|
||||
|
||||
self_update_manager() {
|
||||
[[ "$SELF_UPDATE" == "1" ]] || return 1
|
||||
[[ "$VIDEO_SITE_SKIP_SELF_UPDATE" != "1" ]] || return 1
|
||||
[[ -n "$INSTALL_SCRIPT_URL" ]] || return 1
|
||||
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
log "checking latest manager script"
|
||||
if ! download_file "$INSTALL_SCRIPT_URL" "$tmp"; then
|
||||
warn "manager self-update skipped: cannot download $INSTALL_SCRIPT_URL"
|
||||
rm -f "$tmp"
|
||||
return 1
|
||||
fi
|
||||
if ! bash -n "$tmp"; then
|
||||
warn "manager self-update skipped: downloaded script has syntax errors"
|
||||
rm -f "$tmp"
|
||||
return 1
|
||||
fi
|
||||
if [[ -f "$MANAGER_PATH" ]] && cmp -s "$tmp" "$MANAGER_PATH"; then
|
||||
rm -f "$tmp"
|
||||
return 1
|
||||
fi
|
||||
|
||||
install_cli_from_file "$tmp"
|
||||
rm -f "$tmp"
|
||||
log "manager script updated"
|
||||
return 0
|
||||
}
|
||||
|
||||
exec_latest_manager_update() {
|
||||
local env_args=(
|
||||
"VIDEO_SITE_SKIP_SELF_UPDATE=1"
|
||||
"APP_NAME=$APP_NAME"
|
||||
"GITHUB_REPO=$GITHUB_REPO"
|
||||
"INSTALL_PATH=$INSTALL_PATH"
|
||||
"SERVICE_NAME=$SERVICE_NAME"
|
||||
"VERSION=$VERSION"
|
||||
"GH_PROXY=$GH_PROXY"
|
||||
"CONFIGURE_UFW=$CONFIGURE_UFW"
|
||||
"INSTALL_DEPS=$INSTALL_DEPS"
|
||||
"SELF_UPDATE=$SELF_UPDATE"
|
||||
"FORCE_UPDATE=$FORCE_UPDATE"
|
||||
"INSTALL_SCRIPT_REF=$INSTALL_SCRIPT_REF"
|
||||
"INSTALL_SCRIPT_URL=$INSTALL_SCRIPT_URL"
|
||||
"SERVICE_READY_TIMEOUT=$SERVICE_READY_TIMEOUT"
|
||||
)
|
||||
if [[ -n "$FRONTEND_PORT_WAS_SET" ]]; then
|
||||
env_args+=("FRONTEND_PORT=$FRONTEND_PORT")
|
||||
fi
|
||||
exec env "${env_args[@]}" bash "$MANAGER_PATH" update
|
||||
}
|
||||
|
||||
open_firewall_port() {
|
||||
[[ "$CONFIGURE_UFW" == "1" ]] || return
|
||||
command -v ufw >/dev/null 2>&1 || return
|
||||
[[ "$CONFIGURE_UFW" == "1" ]] || return 0
|
||||
command -v ufw >/dev/null 2>&1 || return 0
|
||||
if ufw status 2>/dev/null | grep -qi "Status: active"; then
|
||||
log "allowing ${FRONTEND_PORT}/tcp in UFW"
|
||||
ufw allow "${FRONTEND_PORT}/tcp"
|
||||
fi
|
||||
}
|
||||
|
||||
listen_port_from_config() {
|
||||
local cfg="$INSTALL_PATH/config.yaml"
|
||||
local listen="" port
|
||||
if [[ -f "$cfg" ]]; then
|
||||
listen="$(sed -nE 's/^[[:space:]]*listen:[[:space:]]*"?([^" #]+)"?.*/\1/p' "$cfg" | head -n1)"
|
||||
fi
|
||||
port="${listen##*:}"
|
||||
if [[ "$port" =~ ^[0-9]+$ ]]; then
|
||||
printf '%s' "$port"
|
||||
return
|
||||
fi
|
||||
printf '%s' "$FRONTEND_PORT"
|
||||
}
|
||||
|
||||
append_unique() {
|
||||
local value="$1"
|
||||
shift
|
||||
for existing in "$@"; do
|
||||
[[ "$existing" == "$value" ]] && return 1
|
||||
done
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
app_service_names() {
|
||||
local names=()
|
||||
local name
|
||||
for name in "$SERVICE_NAME" "$APP_NAME" video-site-91 video-site-backend video-site-frontend; do
|
||||
[[ -n "$name" ]] || continue
|
||||
if append_unique "$name" "${names[@]}" >/dev/null; then
|
||||
names+=("$name")
|
||||
fi
|
||||
done
|
||||
printf '%s\n' "${names[@]}"
|
||||
}
|
||||
|
||||
stop_app_services() {
|
||||
local name unit
|
||||
while IFS= read -r name; do
|
||||
[[ -n "$name" ]] || continue
|
||||
unit="${name}.service"
|
||||
systemctl disable --now "$unit" 2>/dev/null || systemctl stop "$unit" 2>/dev/null || true
|
||||
rm -f "/etc/systemd/system/$unit"
|
||||
done < <(app_service_names)
|
||||
systemctl daemon-reload
|
||||
}
|
||||
|
||||
remove_app_containers() {
|
||||
command -v docker >/dev/null 2>&1 || return 0
|
||||
|
||||
local names=()
|
||||
local name
|
||||
for name in "$SERVICE_NAME" "$APP_NAME" video-site-91; do
|
||||
[[ -n "$name" ]] || continue
|
||||
if append_unique "$name" "${names[@]}" >/dev/null; then
|
||||
names+=("$name")
|
||||
fi
|
||||
done
|
||||
|
||||
for name in "${names[@]}"; do
|
||||
if docker ps -a --format '{{.Names}}' 2>/dev/null | grep -Fxq "$name"; then
|
||||
log "removing docker container $name"
|
||||
docker rm -f "$name" >/dev/null 2>&1 || true
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
pids_listening_on_port() {
|
||||
local port="$1"
|
||||
[[ "$port" =~ ^[0-9]+$ ]] || return 0
|
||||
command -v ss >/dev/null 2>&1 || return 0
|
||||
|
||||
ss -ltnp 2>/dev/null \
|
||||
| awk -v port="$port" '$4 ~ ":" port "$" {print}' \
|
||||
| grep -oE 'pid=[0-9]+' \
|
||||
| cut -d= -f2 \
|
||||
| sort -u || true
|
||||
}
|
||||
|
||||
process_looks_like_app() {
|
||||
local pid="$1"
|
||||
local exe="" cmd=""
|
||||
exe="$(readlink "/proc/$pid/exe" 2>/dev/null || true)"
|
||||
cmd="$(tr '\0' ' ' <"/proc/$pid/cmdline" 2>/dev/null || true)"
|
||||
|
||||
[[ "$exe" == "$INSTALL_PATH/server" ]] && return 0
|
||||
[[ "$cmd" == *"$INSTALL_PATH"* ]] && return 0
|
||||
[[ "$cmd" == *"VIDEO_FRONTEND_DIR=$INSTALL_PATH/dist"* ]] && return 0
|
||||
[[ "$cmd" == *"VIDEO_CONFIG=$INSTALL_PATH/config.yaml"* ]] && return 0
|
||||
[[ "$cmd" == *"video-site-91"* ]] && return 0
|
||||
[[ "$cmd" == *"91VideoSpider"* ]] && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
stop_lingering_app_processes() {
|
||||
local ports=("$@")
|
||||
local port pid pids=()
|
||||
|
||||
for port in "${ports[@]}"; do
|
||||
[[ "$port" =~ ^[0-9]+$ ]] || continue
|
||||
while IFS= read -r pid; do
|
||||
[[ -n "$pid" ]] || continue
|
||||
process_looks_like_app "$pid" || continue
|
||||
if append_unique "$pid" "${pids[@]}" >/dev/null; then
|
||||
pids+=("$pid")
|
||||
fi
|
||||
done < <(pids_listening_on_port "$port")
|
||||
done
|
||||
|
||||
if (( ${#pids[@]} == 0 )); then
|
||||
return
|
||||
fi
|
||||
|
||||
warn "stopping lingering app process(es): ${pids[*]}"
|
||||
kill "${pids[@]}" 2>/dev/null || true
|
||||
sleep 1
|
||||
|
||||
local alive=()
|
||||
for pid in "${pids[@]}"; do
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
alive+=("$pid")
|
||||
fi
|
||||
done
|
||||
if (( ${#alive[@]} > 0 )); then
|
||||
warn "force killing lingering app process(es): ${alive[*]}"
|
||||
kill -9 "${alive[@]}" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
warn_remaining_listeners() {
|
||||
local ports=("$@")
|
||||
local port pid cmd
|
||||
for port in "${ports[@]}"; do
|
||||
[[ "$port" =~ ^[0-9]+$ ]] || continue
|
||||
while IFS= read -r pid; do
|
||||
[[ -n "$pid" ]] || continue
|
||||
cmd="$(tr '\0' ' ' <"/proc/$pid/cmdline" 2>/dev/null || true)"
|
||||
warn "port $port is still listening after uninstall: pid=$pid ${cmd:-unknown}"
|
||||
done < <(pids_listening_on_port "$port")
|
||||
done
|
||||
}
|
||||
|
||||
has_interactive_tty() {
|
||||
[[ -t 0 ]]
|
||||
}
|
||||
|
||||
confirm_uninstall_app() {
|
||||
if ! has_interactive_tty; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local confirm=""
|
||||
printf '确认卸载 91 吗?这会停止服务、移除管理命令,并可选择是否删除项目文件。[y/N]: ' >/dev/tty
|
||||
IFS= read -r confirm </dev/tty || confirm=""
|
||||
case "$confirm" in
|
||||
[yY]) return 0 ;;
|
||||
*)
|
||||
log "uninstall cancelled"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
delete_install_path_requested() {
|
||||
if [[ "${UNINSTALL_DELETE_FILES:-0}" == "1" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if ! has_interactive_tty; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
local confirm=""
|
||||
printf '删除 %s 里的程序、配置和数据吗?[y/N]: ' "$INSTALL_PATH" >/dev/tty
|
||||
IFS= read -r confirm </dev/tty || confirm=""
|
||||
case "$confirm" in
|
||||
[yY]) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
service_health_url() {
|
||||
printf 'http://127.0.0.1:%s/admin/api/setup' "$(listen_port_from_config)"
|
||||
}
|
||||
|
||||
wait_for_service_ready() {
|
||||
local url deadline
|
||||
url="$(service_health_url)"
|
||||
deadline=$((SECONDS + SERVICE_READY_TIMEOUT))
|
||||
log "waiting for service at $url"
|
||||
while (( SECONDS < deadline )); do
|
||||
if curl -fsS --connect-timeout 2 --max-time 5 "$url" >/dev/null 2>&1; then
|
||||
log "service is ready"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
restart_service_ready() {
|
||||
if systemctl restart "${SERVICE_NAME}.service" && wait_for_service_ready; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
warn "service did not become ready; retrying restart"
|
||||
if systemctl restart "${SERVICE_NAME}.service" && wait_for_service_ready; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
warn "service failed to become ready"
|
||||
systemctl --no-pager --full status "${SERVICE_NAME}.service" || true
|
||||
journalctl -u "${SERVICE_NAME}.service" -n 80 --no-pager || true
|
||||
return 1
|
||||
}
|
||||
|
||||
fetch_and_unpack() {
|
||||
local tmp archive url root
|
||||
tmp="$(mktemp -d)"
|
||||
@@ -271,19 +583,63 @@ fetch_and_unpack() {
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
current_version_from_github() {
|
||||
installed_version() {
|
||||
if [[ -f "$VERSION_FILE" ]]; then
|
||||
head -n1 "$VERSION_FILE" 2>/dev/null | tr -d '\r'
|
||||
fi
|
||||
}
|
||||
|
||||
target_version() {
|
||||
if [[ "$VERSION" != "latest" ]]; then
|
||||
printf '%s' "$VERSION"
|
||||
return
|
||||
fi
|
||||
curl -fsSL "https://api.github.com/repos/${GITHUB_REPO}/releases/latest" \
|
||||
|
||||
local body version effective_url
|
||||
body="$(curl -fsSL \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "User-Agent: video-site-91-installer" \
|
||||
"https://api.github.com/repos/${GITHUB_REPO}/releases/latest" 2>/dev/null || true)"
|
||||
version="$(printf '%s\n' "$body" \
|
||||
| sed -nE 's/.*"tag_name"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/p' \
|
||||
| head -n1)"
|
||||
if [[ -n "$version" ]]; then
|
||||
printf '%s' "$version"
|
||||
return
|
||||
fi
|
||||
|
||||
effective_url="$(curl -fsSLI -o /dev/null -w '%{url_effective}' "$(download_base_url)/$(asset_name)" 2>/dev/null || true)"
|
||||
printf '%s\n' "$effective_url" \
|
||||
| sed -nE 's#.*/releases/download/([^/]+)/.*#\1#p' \
|
||||
| head -n1
|
||||
}
|
||||
|
||||
should_skip_update() {
|
||||
[[ "$FORCE_UPDATE" != "1" ]] || return 1
|
||||
|
||||
local current target
|
||||
current="$(installed_version)"
|
||||
target="$(target_version || true)"
|
||||
|
||||
if [[ -z "$target" ]]; then
|
||||
warn "cannot determine target version; continuing update"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -z "$current" ]]; then
|
||||
log "installed version: unknown"
|
||||
log "target version: $target"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "installed version: $current"
|
||||
log "target version: $target"
|
||||
[[ "$current" == "$target" ]]
|
||||
}
|
||||
|
||||
record_version() {
|
||||
local version
|
||||
version="$(current_version_from_github || true)"
|
||||
version="$(target_version || true)"
|
||||
[[ -n "$version" ]] || version="$VERSION"
|
||||
{
|
||||
echo "$version"
|
||||
@@ -298,7 +654,7 @@ show_success() {
|
||||
version="$(head -n1 "$VERSION_FILE" 2>/dev/null || echo unknown)"
|
||||
|
||||
echo
|
||||
printf "${GREEN}安装完成${RESET}\n"
|
||||
printf '%b安装完成%b\n' "$GREEN" "$RESET"
|
||||
echo "版本:$version"
|
||||
[[ -n "$local_ip" ]] && echo "局域网:http://${local_ip}:${FRONTEND_PORT}/"
|
||||
[[ -n "$public_ip" ]] && echo "公网: http://${public_ip}:${FRONTEND_PORT}/"
|
||||
@@ -319,57 +675,79 @@ install_app() {
|
||||
write_service
|
||||
install_cli
|
||||
open_firewall_port
|
||||
restart_service_ready || die "service failed to start"
|
||||
record_version
|
||||
systemctl restart "${SERVICE_NAME}.service"
|
||||
show_success
|
||||
}
|
||||
|
||||
update_app() {
|
||||
check_system
|
||||
check_disk_space
|
||||
install_deps
|
||||
[[ -f "$INSTALL_PATH/server" ]] || die "not installed at $INSTALL_PATH"
|
||||
|
||||
if self_update_manager; then
|
||||
log "re-running update with latest manager script"
|
||||
exec_latest_manager_update
|
||||
fi
|
||||
|
||||
install_deps
|
||||
|
||||
if should_skip_update; then
|
||||
log "already up to date; skipped app update"
|
||||
return 0
|
||||
fi
|
||||
|
||||
check_disk_space
|
||||
|
||||
local backup
|
||||
backup="$(mktemp -d)"
|
||||
cp "$INSTALL_PATH/server" "$backup/server"
|
||||
[[ -d "$INSTALL_PATH/dist" ]] && cp -R "$INSTALL_PATH/dist" "$backup/dist"
|
||||
backup_install_files "$backup"
|
||||
|
||||
systemctl stop "${SERVICE_NAME}.service" 2>/dev/null || true
|
||||
if ! fetch_and_unpack; then
|
||||
if ! (fetch_and_unpack && prepare_config && write_service && install_cli); then
|
||||
warn "update failed; restoring previous files"
|
||||
cp "$backup/server" "$INSTALL_PATH/server"
|
||||
rm -rf "$INSTALL_PATH/dist"
|
||||
[[ -d "$backup/dist" ]] && cp -R "$backup/dist" "$INSTALL_PATH/dist"
|
||||
restore_install_files "$backup"
|
||||
systemctl start "${SERVICE_NAME}.service" 2>/dev/null || true
|
||||
rm -rf "$backup"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
prepare_config
|
||||
write_service
|
||||
install_cli
|
||||
if ! restart_service_ready; then
|
||||
warn "new version failed to start; restoring previous files"
|
||||
restore_install_files "$backup"
|
||||
restart_service_ready 2>/dev/null || true
|
||||
rm -rf "$backup"
|
||||
exit 1
|
||||
fi
|
||||
record_version
|
||||
systemctl restart "${SERVICE_NAME}.service"
|
||||
rm -rf "$backup"
|
||||
log "updated"
|
||||
}
|
||||
|
||||
uninstall_app() {
|
||||
systemctl disable --now "${SERVICE_NAME}.service" 2>/dev/null || true
|
||||
rm -f "/etc/systemd/system/${SERVICE_NAME}.service"
|
||||
systemctl daemon-reload
|
||||
local listen_port port ports=()
|
||||
confirm_uninstall_app || return 1
|
||||
|
||||
listen_port="$(listen_port_from_config)"
|
||||
for port in "$listen_port" "$FRONTEND_PORT" 9191 9192; do
|
||||
[[ "$port" =~ ^[0-9]+$ ]] || continue
|
||||
if append_unique "$port" "${ports[@]}" >/dev/null; then
|
||||
ports+=("$port")
|
||||
fi
|
||||
done
|
||||
|
||||
stop_app_services
|
||||
remove_app_containers
|
||||
stop_lingering_app_processes "${ports[@]}"
|
||||
rm -f "$COMMAND_LINK" "$APP_COMMAND_LINK" "$MANAGER_PATH"
|
||||
|
||||
if [[ -t 0 ]]; then
|
||||
read -r -p "删除 $INSTALL_PATH 里的程序、配置和数据吗?[y/N]: " confirm
|
||||
case "$confirm" in
|
||||
[yY]) rm -rf "$INSTALL_PATH" ;;
|
||||
*) log "kept $INSTALL_PATH" ;;
|
||||
esac
|
||||
if delete_install_path_requested; then
|
||||
rm -rf "$INSTALL_PATH"
|
||||
log "removed $INSTALL_PATH"
|
||||
else
|
||||
log "removed service; kept $INSTALL_PATH"
|
||||
log "kept $INSTALL_PATH"
|
||||
fi
|
||||
|
||||
warn_remaining_listeners "${ports[@]}"
|
||||
}
|
||||
|
||||
show_menu() {
|
||||
@@ -399,7 +777,11 @@ show_menu() {
|
||||
3) main update ;;
|
||||
4) main restart ;;
|
||||
5) main stop ;;
|
||||
6) main uninstall ;;
|
||||
6)
|
||||
if main uninstall; then
|
||||
exit 0
|
||||
fi
|
||||
;;
|
||||
0) exit 0 ;;
|
||||
*) echo "无效的选项" ;;
|
||||
esac
|
||||
@@ -430,7 +812,7 @@ main() {
|
||||
;;
|
||||
restart)
|
||||
need_root "$@"
|
||||
systemctl restart "${SERVICE_NAME}.service"
|
||||
restart_service_ready || die "service failed to start"
|
||||
;;
|
||||
stop)
|
||||
need_root "$@"
|
||||
|
||||
Generated
+829
-1531
File diff suppressed because it is too large
Load Diff
+4
-4
@@ -15,14 +15,14 @@
|
||||
"lucide-react": "0.453.0",
|
||||
"react": "18.3.1",
|
||||
"react-dom": "18.3.1",
|
||||
"react-router-dom": "6.26.2"
|
||||
"react-router-dom": "^6.30.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "18.3.12",
|
||||
"@types/react-dom": "18.3.1",
|
||||
"@vitejs/plugin-react": "4.3.3",
|
||||
"tsx": "^4.19.2",
|
||||
"@vitejs/plugin-react": "^6.0.2",
|
||||
"tsx": "^4.22.3",
|
||||
"typescript": "5.6.3",
|
||||
"vite": "5.4.10"
|
||||
"vite": "^8.0.14"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,6 +61,7 @@ build_package() {
|
||||
)
|
||||
|
||||
cp "$ROOT_DIR/backend/config.example.yaml" "$work/config.example.yaml"
|
||||
cp "$ROOT_DIR/install.sh" "$work/install.sh"
|
||||
cp -R "$ROOT_DIR/dist" "$work/dist"
|
||||
mkdir -p "$work/91VideoSpider"
|
||||
cp "$ROOT_DIR/91VideoSpider/spider_91porn.py" "$work/91VideoSpider/spider_91porn.py"
|
||||
@@ -69,10 +70,11 @@ build_package() {
|
||||
$APP_NAME $VERSION
|
||||
|
||||
This is a prebuilt release package.
|
||||
Use install.sh from the repository to install it on a Linux server.
|
||||
Use install.sh in this package or from the repository to install it on a Linux server.
|
||||
EOF
|
||||
|
||||
chmod +x "$work/server"
|
||||
chmod +x "$work/install.sh"
|
||||
tar -C "$OUT_DIR/.work" -czf "$OUT_DIR/$artifact.tar.gz" "$artifact"
|
||||
log "wrote $OUT_DIR/$artifact.tar.gz"
|
||||
}
|
||||
|
||||
@@ -1,5 +1,16 @@
|
||||
import { useState } from "react";
|
||||
import { NavLink, Outlet, useNavigate } from "react-router-dom";
|
||||
import { HardDrive, Film, LogOut, Play, Home, Tags, Palette } from "lucide-react";
|
||||
import {
|
||||
HardDrive,
|
||||
Film,
|
||||
LogOut,
|
||||
Play,
|
||||
Home,
|
||||
Tags,
|
||||
Palette,
|
||||
RefreshCw,
|
||||
} from "lucide-react";
|
||||
import * as api from "./api";
|
||||
import { useAuth } from "./AuthContext";
|
||||
import { useToast } from "./ToastContext";
|
||||
|
||||
@@ -7,6 +18,31 @@ export function AdminLayout() {
|
||||
const { logout } = useAuth();
|
||||
const navigate = useNavigate();
|
||||
const { show } = useToast();
|
||||
const [checkingUpdate, setCheckingUpdate] = useState(false);
|
||||
|
||||
async function handleCheckUpdate() {
|
||||
if (checkingUpdate) return;
|
||||
setCheckingUpdate(true);
|
||||
try {
|
||||
const result = await api.checkUpdate();
|
||||
if (result.hasUpdate) {
|
||||
show(
|
||||
`发现新版本 ${result.latestVersion},当前 ${result.currentVersion}`,
|
||||
"success"
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (result.currentVersion === "unknown") {
|
||||
show(`当前版本未知,GitHub 最新版本为 ${result.latestVersion}`, "info");
|
||||
return;
|
||||
}
|
||||
show(`当前已是最新版本:${result.currentVersion}`, "success");
|
||||
} catch {
|
||||
show("检查更新失败,请稍后重试", "error");
|
||||
} finally {
|
||||
setCheckingUpdate(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleLogout() {
|
||||
try {
|
||||
@@ -65,8 +101,16 @@ export function AdminLayout() {
|
||||
</NavLink>
|
||||
</nav>
|
||||
<div className="admin-sidebar__footer">
|
||||
<button
|
||||
className="admin-sidebar__check-update"
|
||||
onClick={handleCheckUpdate}
|
||||
disabled={checkingUpdate}
|
||||
>
|
||||
<RefreshCw size={14} />
|
||||
{checkingUpdate ? "检查中" : "检查更新"}
|
||||
</button>
|
||||
<button className="admin-sidebar__logout" onClick={handleLogout}>
|
||||
<LogOut size={14} style={{ verticalAlign: -2, marginRight: 4 }} />
|
||||
<LogOut size={14} />
|
||||
退出登录
|
||||
</button>
|
||||
</div>
|
||||
|
||||
+207
-168
@@ -26,6 +26,8 @@ const kindLabel: Record<string, string> = {
|
||||
pikpak: "PikPak",
|
||||
wopan: "联通沃盘",
|
||||
onedrive: "OneDrive",
|
||||
googledrive: "Google Drive",
|
||||
localstorage: "本地存储",
|
||||
spider91: "91 爬虫",
|
||||
};
|
||||
|
||||
@@ -40,7 +42,6 @@ type FormState = {
|
||||
kind: Kind;
|
||||
name: string;
|
||||
rootId: string;
|
||||
scanRootId: string;
|
||||
creds: Record<string, string>;
|
||||
/**
|
||||
* spider91 专用字段:把视频迁移到云盘的目标 drive ID。
|
||||
@@ -48,7 +49,7 @@ type FormState = {
|
||||
* 单独通过 PUT /admin/api/settings 写到全局 setting。在 form state 里维护它
|
||||
* 是为了让 DriveForm 能读写同一份编辑状态。
|
||||
*
|
||||
* 空字符串 = 自动模式(系统中唯一的 pikpak/p115 drive)。
|
||||
* 空字符串 = 本地保存,不上传。
|
||||
*/
|
||||
spider91UploadDriveId: string;
|
||||
};
|
||||
@@ -57,47 +58,71 @@ const emptyForm: FormState = {
|
||||
id: "",
|
||||
kind: "p115",
|
||||
name: "",
|
||||
rootId: "0",
|
||||
scanRootId: "0",
|
||||
rootId: "",
|
||||
creds: {},
|
||||
spider91UploadDriveId: "",
|
||||
};
|
||||
|
||||
const idleNightlyStatus: api.NightlyJobStatus = {
|
||||
state: "idle",
|
||||
running: false,
|
||||
queued: false,
|
||||
};
|
||||
|
||||
function nightlyButtonText(status: api.NightlyJobStatus, triggering: boolean) {
|
||||
if (triggering) return "触发中...";
|
||||
if (status.running) return "扫描运行中";
|
||||
if (status.queued) return "扫描已排队";
|
||||
return "扫描所有网盘";
|
||||
}
|
||||
|
||||
function nightlyBusyText(status: api.NightlyJobStatus) {
|
||||
if (status.running) return "扫描任务正在运行";
|
||||
if (status.queued) return "扫描任务已排队";
|
||||
return "";
|
||||
}
|
||||
|
||||
export function DrivesPage() {
|
||||
const [list, setList] = useState<api.AdminDrive[]>([]);
|
||||
const [storage, setStorage] = useState<api.AdminDriveStorage | null>(null);
|
||||
const [settings, setSettings] = useState<api.Settings | null>(null);
|
||||
const [nightlyStatus, setNightlyStatus] =
|
||||
useState<api.NightlyJobStatus>(idleNightlyStatus);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [modalOpen, setModalOpen] = useState(false);
|
||||
const [form, setForm] = useState<FormState>(emptyForm);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [regenFailedId, setRegenFailedId] = useState("");
|
||||
// 与 regenFailedId 并列:失败封面重新入队按钮的 disable 状态。两套独立按钮 →
|
||||
// 两个 state 互不阻塞,避免操作 teaser 时锁住封面那条按钮(反之亦然)。
|
||||
// 失败重试按钮各自维护 pending 状态,避免操作 teaser / 封面 / 指纹时互相锁住。
|
||||
const [regenFailedThumbId, setRegenFailedThumbId] = useState("");
|
||||
const [regenFailedFingerprintId, setRegenFailedFingerprintId] = useState("");
|
||||
// togglingTeaserId 在请求未返回前禁用按钮,避免连点导致两次切换互相覆盖。
|
||||
const [togglingTeaserId, setTogglingTeaserId] = useState("");
|
||||
const [scanningAll, setScanningAll] = useState(false);
|
||||
const [selectedDriveId, setSelectedDriveId] = useState<string | null>(null);
|
||||
const { show } = useToast();
|
||||
const nightlyBusy = scanningAll || nightlyStatus.running || nightlyStatus.queued;
|
||||
|
||||
// 当前系统中可作为 spider91 上传目标的 drive 列表(pikpak ∪ p115)。
|
||||
// 用户保存 spider91 drive 时从这里挑一个;空表示走"自动"模式。
|
||||
// 当前系统中可作为 spider91 上传目标的 drive 列表(pikpak ∪ p115 ∪ onedrive)。
|
||||
// 用户保存 spider91 drive 时从这里挑一个;空表示本地保存不上传。
|
||||
const uploadTargets = useMemo(
|
||||
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115"),
|
||||
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115" || d.kind === "onedrive"),
|
||||
[list]
|
||||
);
|
||||
|
||||
async function refresh() {
|
||||
setLoading(true);
|
||||
try {
|
||||
const [data, storageData, settingsData] = await Promise.all([
|
||||
const [data, storageData, settingsData, jobStatus] = await Promise.all([
|
||||
api.listDrives(),
|
||||
api.getDriveStorage(),
|
||||
api.getSettings().catch(() => null),
|
||||
api.getNightlyJobStatus().catch(() => null),
|
||||
]);
|
||||
setList(data ?? []);
|
||||
setStorage(storageData);
|
||||
if (settingsData) setSettings(settingsData);
|
||||
if (jobStatus) setNightlyStatus(jobStatus);
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "加载失败", "error");
|
||||
} finally {
|
||||
@@ -107,8 +132,12 @@ export function DrivesPage() {
|
||||
|
||||
async function refreshDriveList() {
|
||||
try {
|
||||
const data = await api.listDrives();
|
||||
const [data, jobStatus] = await Promise.all([
|
||||
api.listDrives(),
|
||||
api.getNightlyJobStatus().catch(() => null),
|
||||
]);
|
||||
setList(data ?? []);
|
||||
if (jobStatus) setNightlyStatus(jobStatus);
|
||||
} catch {
|
||||
// 保持当前页面状态,下一次轮询或手动操作再刷新。
|
||||
}
|
||||
@@ -126,7 +155,7 @@ export function DrivesPage() {
|
||||
|
||||
function openCreate() {
|
||||
// 创建时把全局 setting 当前值带进表单,方便用户在新建第一个 spider91 drive 时
|
||||
// 直接看到当前的上传目标选择(一般是空 = 自动)。
|
||||
// 直接看到当前的上传目标选择(一般是空 = 本地保存)。
|
||||
setForm({
|
||||
...emptyForm,
|
||||
spider91UploadDriveId: settings?.spider91UploadDriveId ?? "",
|
||||
@@ -140,8 +169,7 @@ export function DrivesPage() {
|
||||
kind: d.kind,
|
||||
name: d.name,
|
||||
rootId: d.rootId,
|
||||
scanRootId: d.scanRootId || d.rootId,
|
||||
creds: {},
|
||||
creds: d.kind === "spider91" ? { proxy: d.spider91Proxy ?? "" } : {},
|
||||
spider91UploadDriveId: settings?.spider91UploadDriveId ?? "",
|
||||
});
|
||||
setModalOpen(true);
|
||||
@@ -157,6 +185,9 @@ export function DrivesPage() {
|
||||
const driveID = existing
|
||||
? form.id
|
||||
: makeUniqueDriveId(form.kind, name, list);
|
||||
const rootId = usesRootDirectoryID(form.kind)
|
||||
? form.rootId.trim() || defaultRootId(form.kind)
|
||||
: defaultRootId(form.kind);
|
||||
// 若编辑且没有提供凭证,提示一下但仍允许保存(不改凭证)
|
||||
setSaving(true);
|
||||
try {
|
||||
@@ -164,8 +195,7 @@ export function DrivesPage() {
|
||||
id: driveID,
|
||||
kind: form.kind,
|
||||
name,
|
||||
rootId: form.rootId || defaultRootId(form.kind),
|
||||
scanRootId: form.scanRootId || form.rootId || defaultRootId(form.kind),
|
||||
rootId,
|
||||
credentials: form.creds,
|
||||
});
|
||||
|
||||
@@ -232,14 +262,26 @@ export function DrivesPage() {
|
||||
/**
|
||||
* 立即触发完整凌晨流水线(Phase1 扫所有云盘 → Phase2 spider91 爬虫 →
|
||||
* Phase3 spider91 → 云盘迁移)。后端立即返回 202;进度看 backend 日志。
|
||||
* 已在跑时后端会丢弃此次触发,按钮再点也不会重复进入。
|
||||
* 如果当前已有流水线在跑或已排队,前端只提示,不再提交新任务。
|
||||
*/
|
||||
async function handleRunNightly() {
|
||||
if (nightlyBusy) {
|
||||
show(nightlyBusyText(nightlyStatus) || "当前已有扫描所有网盘任务", "info");
|
||||
return;
|
||||
}
|
||||
setScanningAll(true);
|
||||
try {
|
||||
await api.runNightlyJob();
|
||||
show("已触发扫描所有网盘,耗时较长,可在 backend 日志观察进度", "success");
|
||||
const resp = await api.runNightlyJob();
|
||||
setNightlyStatus(resp.status);
|
||||
if (resp.accepted) {
|
||||
show("已触发扫描所有网盘,耗时较长,可在任务状态和 backend 日志观察进度", "success");
|
||||
} else {
|
||||
show("当前已有扫描所有网盘任务", "info");
|
||||
}
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "触发失败", "error");
|
||||
} finally {
|
||||
setScanningAll(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -270,6 +312,19 @@ export function DrivesPage() {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleRegenFailedFingerprints(d: api.AdminDrive) {
|
||||
setRegenFailedFingerprintId(d.id);
|
||||
try {
|
||||
await api.regenFailedFingerprints(d.id);
|
||||
show("已触发失败指纹重新生成", "success");
|
||||
refresh();
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "触发失败", "error");
|
||||
} finally {
|
||||
setRegenFailedFingerprintId("");
|
||||
}
|
||||
}
|
||||
|
||||
async function handleToggleTeaser(d: api.AdminDrive) {
|
||||
const next = !d.teaserEnabled;
|
||||
setTogglingTeaserId(d.id);
|
||||
@@ -355,16 +410,12 @@ export function DrivesPage() {
|
||||
<span className="admin-detail-label">网盘 ID</span>
|
||||
<span className="admin-detail-value admin-mono-cell">{d.id}</span>
|
||||
</div>
|
||||
{d.kind !== "spider91" && (
|
||||
{usesRootDirectoryID(d.kind) && (
|
||||
<>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">根目录 ID</span>
|
||||
<span className="admin-detail-value admin-mono-cell">{d.rootId}</span>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">扫描起点 ID</span>
|
||||
<span className="admin-detail-value admin-mono-cell">{d.scanRootId || d.rootId}</span>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
{d.kind === "spider91" && (
|
||||
@@ -427,13 +478,13 @@ export function DrivesPage() {
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* 右栏:Teaser / 封面 与 缓存占用 */}
|
||||
{/* 右栏:Teaser / 封面 / 指纹 与 缓存占用 */}
|
||||
<div>
|
||||
<div className="admin-detail-card">
|
||||
<header className="admin-detail-card__title">
|
||||
<div className="admin-detail-card__title-left">
|
||||
<PlayCircle size={16} />
|
||||
<span>Teaser 预览与封面</span>
|
||||
<span>生成状态</span>
|
||||
</div>
|
||||
<div className="admin-detail-actions-inline">
|
||||
<button
|
||||
@@ -443,14 +494,14 @@ export function DrivesPage() {
|
||||
style={{ padding: "4px 10px", fontSize: "11px" }}
|
||||
>
|
||||
{d.teaserEnabled ? <Power size={11} /> : <PowerOff size={11} />}
|
||||
<span>{d.teaserEnabled ? "Teaser: 开" : "Teaser: 关"}</span>
|
||||
<span>{d.teaserEnabled ? "预览视频生成:开" : "预览视频生成:关"}</span>
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div className="admin-detail-grid">
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">封面状态</span>
|
||||
<span className="admin-detail-label">封面生成状态</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationStatusLine label="封面" status={d.thumbnailGenerationStatus} />
|
||||
</div>
|
||||
@@ -462,17 +513,18 @@ export function DrivesPage() {
|
||||
ready={d.thumbnailReadyCount}
|
||||
pending={d.thumbnailPendingCount}
|
||||
failed={d.thumbnailFailedCount}
|
||||
durationPending={d.thumbnailDurationPendingCount}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">Teaser 状态</span>
|
||||
<span className="admin-detail-label">预览视频生成状态</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationStatusLine label="预览" status={d.previewGenerationStatus} />
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">Teaser 数量</span>
|
||||
<span className="admin-detail-label">预览视频数量</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationCounts
|
||||
ready={d.teaserReadyCount}
|
||||
@@ -481,6 +533,22 @@ export function DrivesPage() {
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">视频指纹生成状态</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationStatusLine label="指纹" status={d.fingerprintGenerationStatus} />
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">视频指纹数量</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationCounts
|
||||
ready={d.fingerprintReadyCount}
|
||||
pending={d.fingerprintPendingCount}
|
||||
failed={d.fingerprintFailedCount}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="admin-detail-actions">
|
||||
@@ -500,6 +568,17 @@ export function DrivesPage() {
|
||||
<RotateCcw size={13} />
|
||||
<span>重试失败封面</span>
|
||||
</button>
|
||||
<button
|
||||
className="admin-btn"
|
||||
disabled={
|
||||
(d.fingerprintFailedCount ?? 0) <= 0 ||
|
||||
regenFailedFingerprintId === d.id
|
||||
}
|
||||
onClick={() => handleRegenFailedFingerprints(d)}
|
||||
>
|
||||
<RotateCcw size={13} />
|
||||
<span>重试失败指纹</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -517,7 +596,7 @@ export function DrivesPage() {
|
||||
<span className="admin-detail-value">{formatBytes(driveStorage?.thumbnailBytes ?? 0)}</span>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">Teaser 占用</span>
|
||||
<span className="admin-detail-label">预览视频占用</span>
|
||||
<span className="admin-detail-value">{formatBytes(driveStorage?.teaserBytes ?? 0)}</span>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
@@ -570,9 +649,10 @@ export function DrivesPage() {
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={handleRunNightly}
|
||||
title="立即扫描所有网盘。耗时较长,期间不要重复触发。"
|
||||
disabled={scanningAll}
|
||||
title={nightlyBusyText(nightlyStatus) || "立即扫描所有网盘。耗时较长,期间不要重复触发。"}
|
||||
>
|
||||
<PlayCircle size={14} /> 扫描所有网盘
|
||||
<PlayCircle size={14} /> {nightlyButtonText(nightlyStatus, scanningAll)}
|
||||
</button>
|
||||
<button className="admin-btn is-primary" onClick={openCreate}>
|
||||
<Plus size={14} /> 新建网盘
|
||||
@@ -586,7 +666,7 @@ export function DrivesPage() {
|
||||
<div className="admin-empty">加载中...</div>
|
||||
) : list.length === 0 ? (
|
||||
<div className="admin-card admin-empty">
|
||||
还没有配置任何网盘。点击右上角「新建」,选择夸克 / 115 / PikPak / 沃盘 / OneDrive,填入凭证即可。
|
||||
还没有配置任何网盘。点击右上角「新建」,选择夸克 / 115 / PikPak / 沃盘 / OneDrive / 本地存储,填入凭证或路径即可。
|
||||
</div>
|
||||
) : (
|
||||
<div className="admin-drives-grid">
|
||||
@@ -625,6 +705,15 @@ export function DrivesPage() {
|
||||
</span>
|
||||
</strong>
|
||||
</div>
|
||||
<div className="admin-drive-card__metric">
|
||||
<span>指纹数 (就绪/失败)</span>
|
||||
<strong>
|
||||
{d.fingerprintReadyCount ?? 0}
|
||||
<span style={{ fontSize: "11px", fontWeight: "normal", color: "var(--text-faint)" }}>
|
||||
{" "}/ {d.fingerprintFailedCount ?? 0}
|
||||
</span>
|
||||
</strong>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="admin-drive-card__footer">
|
||||
@@ -676,7 +765,7 @@ function StorageSummary({ storage }: { storage: api.AdminDriveStorage }) {
|
||||
<strong>{formatBytes(storage.thumbnailBytes)}</strong>
|
||||
</div>
|
||||
<div className="admin-storage-summary__metric">
|
||||
<span>Teaser 占用</span>
|
||||
<span>预览视频占用</span>
|
||||
<strong>{formatBytes(storage.teaserBytes)}</strong>
|
||||
</div>
|
||||
<div className="admin-storage-summary__metric">
|
||||
@@ -695,10 +784,12 @@ function GenerationCounts({
|
||||
ready,
|
||||
pending,
|
||||
failed,
|
||||
durationPending,
|
||||
}: {
|
||||
ready?: number;
|
||||
pending?: number;
|
||||
failed?: number;
|
||||
durationPending?: number;
|
||||
}) {
|
||||
return (
|
||||
<div className="admin-generation-counts">
|
||||
@@ -711,6 +802,11 @@ function GenerationCounts({
|
||||
<span className="admin-drive-teaser__metric is-failed">
|
||||
失败 {failed ?? 0}
|
||||
</span>
|
||||
{(durationPending ?? 0) > 0 && (
|
||||
<span className="admin-drive-teaser__metric">
|
||||
待补时长 {durationPending}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -726,7 +822,7 @@ function GenerationStatusLine({
|
||||
const queueLength = status?.queueLength ?? 0;
|
||||
const detail = generationDetail(status);
|
||||
const title = generationTitle(status, detail);
|
||||
const countText = queueLength > 0 ? `${label === "封面" ? "剩余" : "队列"} ${queueLength}` : "";
|
||||
const countText = queueLength > 0 ? `${label === "封面" ? "待处理" : "队列"} ${queueLength}` : "";
|
||||
|
||||
return (
|
||||
<div className="admin-generation-row" title={title}>
|
||||
@@ -848,8 +944,7 @@ function DriveForm({
|
||||
onChange({
|
||||
...form,
|
||||
kind: v,
|
||||
rootId: defaultRootId(v),
|
||||
scanRootId: defaultRootId(v),
|
||||
rootId: "",
|
||||
creds: {},
|
||||
});
|
||||
}
|
||||
@@ -873,34 +968,26 @@ function DriveForm({
|
||||
>
|
||||
<option value="p115">115 网盘</option>
|
||||
<option value="pikpak">PikPak</option>
|
||||
<option value="spider91">91 爬虫</option>
|
||||
<option value="onedrive">OneDrive</option>
|
||||
<option value="googledrive">Google Drive</option>
|
||||
<option value="localstorage">本地存储</option>
|
||||
<option value="spider91">91 Spider</option>
|
||||
<option value="quark">夸克网盘</option>
|
||||
<option value="wopan">联通沃盘</option>
|
||||
<option value="onedrive">OneDrive</option>
|
||||
</select>
|
||||
</div>
|
||||
{form.kind !== "spider91" && (
|
||||
<>
|
||||
<div className="admin-form__row">
|
||||
<label>根目录 ID</label>
|
||||
<input
|
||||
value={form.rootId}
|
||||
onChange={(e) => set("rootId", e.target.value)}
|
||||
placeholder={form.kind === "pikpak" ? "留空表示根目录" : form.kind === "onedrive" ? "root" : "0"}
|
||||
/>
|
||||
{usesRootDirectoryID(form.kind) && (
|
||||
<div className="admin-form__row">
|
||||
<label>根目录 ID</label>
|
||||
<input
|
||||
value={form.rootId}
|
||||
onChange={(e) => set("rootId", e.target.value)}
|
||||
placeholder={rootIdPlaceholder(form.kind)}
|
||||
/>
|
||||
<div className="admin-form__help">
|
||||
留空时使用该网盘类型的默认根目录,具体目录ID获取方式请参考OpenList文档
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>扫描起点目录 ID</label>
|
||||
<input
|
||||
value={form.scanRootId}
|
||||
onChange={(e) => set("scanRootId", e.target.value)}
|
||||
placeholder="留空则使用根目录"
|
||||
/>
|
||||
<div className="admin-form__help">
|
||||
可以指定一个子目录作为视频库入口,避免扫描整个网盘
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{(help || fields.length > 0) && (
|
||||
@@ -953,10 +1040,9 @@ function DriveForm({
|
||||
* Spider91UploadTargetField 是 spider91 drive 表单专属的"上传目标"下拉。
|
||||
*
|
||||
* 行为:
|
||||
* - 选项 = "(自动)" + 系统中所有 pikpak/p115 drive
|
||||
* - "自动" 模式(value="")下,后端在迁移 worker 启动时挑唯一的目标盘;
|
||||
* 系统中如果同时挂着 pikpak 和 p115 drive 必须显式选定,否则不会上传
|
||||
* - 没有任何 pikpak/p115 drive 时给一行提示文字,告诉用户先去添加目标盘
|
||||
* - 选项 = "本地保存,不上传" + 系统中所有 pikpak/p115/onedrive drive
|
||||
* - value="" 时后端不迁移上传,视频保存在服务器本地
|
||||
* - 没有任何 pikpak/p115/onedrive drive 时仍允许选择本地保存
|
||||
* - 该字段写入的是全局 setting `spider91.upload_drive_id`,不是 drive 自己的
|
||||
* credentials —— 所有 spider91 drive 共享同一个上传目标
|
||||
*/
|
||||
@@ -969,53 +1055,20 @@ function Spider91UploadTargetField({
|
||||
onChange: (v: string) => void;
|
||||
uploadTargets: api.AdminDrive[];
|
||||
}) {
|
||||
// 文案根据系统中实际挂载的目标盘 kind 自适应:
|
||||
// - 只挂了 PikPak → 文案只讲 "PikPak"
|
||||
// - 只挂了 115 → 文案只讲 "115 网盘"
|
||||
// - 两类都挂 → 文案讲 "PikPak / 115 网盘"
|
||||
// 这样在单一类型场景下用户不会被另一类的字样干扰。
|
||||
const kindsPresent = new Set(uploadTargets.map((d) => d.kind));
|
||||
const hasPikPak = kindsPresent.has("pikpak");
|
||||
const has115 = kindsPresent.has("p115");
|
||||
const presentLabel =
|
||||
hasPikPak && has115
|
||||
? "PikPak / 115 网盘"
|
||||
: hasPikPak
|
||||
? "PikPak"
|
||||
: has115
|
||||
? "115 网盘"
|
||||
: "PikPak 或 115 网盘";
|
||||
|
||||
return (
|
||||
<div className="admin-form__row">
|
||||
<label>视频上传目标</label>
|
||||
{uploadTargets.length === 0 ? (
|
||||
<>
|
||||
<select value="" disabled>
|
||||
<option value="">(请先添加 {presentLabel})</option>
|
||||
</select>
|
||||
<div className="admin-form__help">
|
||||
目前系统里还没有 {presentLabel} drive。可以先保存 91 爬虫,之后再回来选择上传目标。
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<select value={value} onChange={(e) => onChange(e.target.value)}>
|
||||
<option value="">(自动:唯一的 {presentLabel})</option>
|
||||
{uploadTargets.map((d) => (
|
||||
<option key={d.id} value={d.id}>
|
||||
{kindLabel[d.kind] ?? d.kind} · {d.name || d.id}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<div className="admin-form__help">
|
||||
爬取后的旧视频会上传到该云盘根目录。该设置全局生效;
|
||||
{uploadTargets.length > 1
|
||||
? `如果同时挂着多个 ${presentLabel} drive,"自动"模式不会工作,必须显式选定一个。`
|
||||
: `当前只挂着 1 个 ${presentLabel},"自动"模式会直接选用它。`}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
<select value={value} onChange={(e) => onChange(e.target.value)}>
|
||||
<option value="">本地保存,不上传</option>
|
||||
{uploadTargets.map((d) => (
|
||||
<option key={d.id} value={d.id}>
|
||||
{kindLabel[d.kind] ?? d.kind} · {d.name || d.id}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<div className="admin-form__help">
|
||||
选择本地保存时,爬取视频只保存在服务器本地;选择 115 网盘、PikPak 或 OneDrive 后,较早的视频会上传到该云盘根目录下的 91 Spider 文件夹。该设置全局生效。
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1028,13 +1081,17 @@ function credentialHelp(kind: Kind, isEdit: boolean): string {
|
||||
case "p115":
|
||||
return `登录 115.com 后复制 Cookie,形如 "UID=...; CID=...; SEID=...; KID=..."。${note}`;
|
||||
case "pikpak":
|
||||
return `参考 OpenList 的 PikPak 登录方式。可填用户名和密码首次登录,也可填 refresh_token;如返回验证码链接,打开验证后把 captcha_token 粘贴回来。${note}`;
|
||||
return `填写 PikPak 账号和密码即可。平台、设备 ID、验证码 token 和 refresh token 会由服务端自动处理并保存。${note}`;
|
||||
case "wopan":
|
||||
return `需要 access_token 和 refresh_token。后续会加扫码/短信登录入口,第一版只能手工粘贴。${note}`;
|
||||
case "onedrive":
|
||||
return `按 OpenList 默认方式,通过 api.oplist.org 在线刷新 token。只需要 refresh_token;保存后会自动回写新的 access_token / refresh_token。${note}`;
|
||||
return `按 OpenList 默认应用在线挂载,只需要 refresh_token;保存时会自动刷新并保存 token。${note}`;
|
||||
case "googledrive":
|
||||
return `按 OpenList 在线 API 挂载,只需要 Google Drive refresh_token;保存时会自动刷新并保存 token。播放不走 302,会由后端带 Authorization 代理转发。${note}`;
|
||||
case "localstorage":
|
||||
return `把服务器上的一个已有目录作为视频来源扫描。填写绝对路径,例如 /mnt/videos;系统会读取该目录及子目录中的视频,并生成封面、Teaser 和指纹。${note}`;
|
||||
case "spider91":
|
||||
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;它不是外部网盘,不需要填写 Cookie 或目录 ID。后续流水线会把较早的视频上传到你选择的 115 / PikPak 目标盘。";
|
||||
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;可按服务器网络情况单独配置代理。后续流水线会把较早的视频上传到你选择的 115 / PikPak / OneDrive 目标盘。";
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
@@ -1073,42 +1130,15 @@ function credentialFields(kind: Kind): Array<{
|
||||
return [
|
||||
{
|
||||
key: "username",
|
||||
label: "用户名 / 邮箱(无 refresh_token 时必填)",
|
||||
label: "用户名 / 邮箱",
|
||||
placeholder: "user@example.com",
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
key: "password",
|
||||
label: "密码(无 refresh_token 时必填)",
|
||||
label: "密码",
|
||||
placeholder: "PikPak 密码",
|
||||
},
|
||||
{
|
||||
key: "platform",
|
||||
label: "platform",
|
||||
placeholder: "web(可选:android / web / pc)",
|
||||
help: "默认 web;如果登录或直链异常,可尝试 android 或 pc。",
|
||||
},
|
||||
{
|
||||
key: "refresh_token",
|
||||
label: "refresh_token(可选)",
|
||||
placeholder: "已有 token 时可直接粘贴",
|
||||
multiline: true,
|
||||
},
|
||||
{
|
||||
key: "captcha_token",
|
||||
label: "captcha_token(可选)",
|
||||
placeholder: "遇到验证码校验时粘贴",
|
||||
multiline: true,
|
||||
},
|
||||
{
|
||||
key: "device_id",
|
||||
label: "device_id(可选)",
|
||||
placeholder: "留空自动生成并保存",
|
||||
},
|
||||
{
|
||||
key: "disable_media_link",
|
||||
label: "disable_media_link",
|
||||
placeholder: "true",
|
||||
help: "默认 true,使用原始下载链接;填 false 可尝试使用媒体缓存链接。",
|
||||
required: true,
|
||||
},
|
||||
];
|
||||
case "wopan":
|
||||
@@ -1140,48 +1170,57 @@ function credentialFields(kind: Kind): Array<{
|
||||
multiline: true,
|
||||
required: true,
|
||||
},
|
||||
];
|
||||
case "googledrive":
|
||||
return [
|
||||
{
|
||||
key: "access_token",
|
||||
label: "access_token(可选)",
|
||||
placeholder: "留空也可以,保存时会在线刷新",
|
||||
key: "refresh_token",
|
||||
label: "refresh_token",
|
||||
placeholder: "OpenList Google Drive refresh_token",
|
||||
multiline: true,
|
||||
required: true,
|
||||
},
|
||||
];
|
||||
case "localstorage":
|
||||
return [
|
||||
{
|
||||
key: "api_url_address",
|
||||
label: "api_url_address(可选)",
|
||||
placeholder: "https://api.oplist.org/onedrive/renewapi",
|
||||
help: "默认使用 OpenList 的在线刷新 API;除非你有自建兼容服务,否则留空。",
|
||||
},
|
||||
{
|
||||
key: "region",
|
||||
label: "region(可选)",
|
||||
placeholder: "global(可选:global / cn / us / de)",
|
||||
help: "默认 global;世纪互联填 cn,美国政府云填 us,德国云填 de。",
|
||||
},
|
||||
{
|
||||
key: "is_sharepoint",
|
||||
label: "is_sharepoint(可选)",
|
||||
placeholder: "false",
|
||||
help: "普通 OneDrive 留空或 false;SharePoint 文档库填 true,并填写 site_id。",
|
||||
},
|
||||
{
|
||||
key: "site_id",
|
||||
label: "site_id(SharePoint 必填)",
|
||||
placeholder: "SharePoint site id",
|
||||
key: "path",
|
||||
label: "本地目录路径",
|
||||
placeholder: "/mnt/videos",
|
||||
required: true,
|
||||
help: "路径必须是后端服务器上的已有目录;保存后可手动重扫,系统会递归扫描支持的视频格式。",
|
||||
},
|
||||
];
|
||||
case "spider91":
|
||||
return [];
|
||||
return [
|
||||
{
|
||||
key: "proxy",
|
||||
label: "代理地址(可选)",
|
||||
placeholder: "http://127.0.0.1:7890",
|
||||
help: "仅用于 91Spider 的列表/详情请求和视频、封面下载;留空则使用服务器环境变量 HTTP_PROXY / HTTPS_PROXY 或直连。支持 http://、https://、socks5:// 或 socks5h://。",
|
||||
},
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
function defaultRootId(kind: Kind): string {
|
||||
if (kind === "pikpak") return "";
|
||||
if (kind === "onedrive") return "root";
|
||||
if (kind === "googledrive") return "root";
|
||||
if (kind === "localstorage") return "/";
|
||||
if (kind === "spider91") return "/";
|
||||
return "0";
|
||||
}
|
||||
|
||||
function usesRootDirectoryID(kind: Kind): boolean {
|
||||
return kind !== "localstorage" && kind !== "spider91";
|
||||
}
|
||||
|
||||
function rootIdPlaceholder(kind: Kind): string {
|
||||
const rootId = defaultRootId(kind);
|
||||
return rootId ? `默认:${rootId}` : "留空表示根目录";
|
||||
}
|
||||
|
||||
|
||||
// ---------- SkipDirsModal ----------
|
||||
//
|
||||
|
||||
@@ -82,11 +82,6 @@ export function LoginPage() {
|
||||
<Play size={18} fill="currentColor" /> {setupRequired ? "首次设置管理员" : "登录"}
|
||||
</h1>
|
||||
<div className="admin-form">
|
||||
{setupRequired && (
|
||||
<div className="admin-form__help admin-form__help--lead">
|
||||
请先设置后台管理员账号。保存后会写入本机配置文件,之后使用该账号登录。
|
||||
</div>
|
||||
)}
|
||||
<div className="admin-form__row">
|
||||
<label>用户名</label>
|
||||
<input
|
||||
|
||||
+268
-37
@@ -1,16 +1,26 @@
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { Film, Plus, RefreshCw, Search, Tags } from "lucide-react";
|
||||
import { CheckSquare, Film, Plus, RefreshCw, Search, Tags, Trash2 } from "lucide-react";
|
||||
import * as api from "./api";
|
||||
import { useToast } from "./ToastContext";
|
||||
|
||||
const DESKTOP_TAGS_PAGE_SIZE = 25;
|
||||
const MOBILE_TAGS_PAGE_SIZE = 8;
|
||||
const TAGS_MOBILE_QUERY = "(max-width: 640px)";
|
||||
|
||||
export function TagsPage() {
|
||||
const [tags, setTags] = useState<api.AdminTag[]>([]);
|
||||
const [label, setLabel] = useState("");
|
||||
const [aliases, setAliases] = useState("");
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [deletingId, setDeletingId] = useState<number | null>(null);
|
||||
const [searchQuery, setSearchQuery] = useState("");
|
||||
const [filterSource, setFilterSource] = useState<string>("all");
|
||||
const [selectMode, setSelectMode] = useState(false);
|
||||
const [selected, setSelected] = useState<Set<number>>(new Set());
|
||||
const [bulkDeleting, setBulkDeleting] = useState(false);
|
||||
const pageSize = useTagsPageSize();
|
||||
const [page, setPage] = useState(1);
|
||||
const { show } = useToast();
|
||||
|
||||
async function refresh() {
|
||||
@@ -45,6 +55,63 @@ export function TagsPage() {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleDelete(tag: api.AdminTag) {
|
||||
if (tag.source === "system") return;
|
||||
if (!window.confirm(`确定删除标签「${tag.label}」吗?此操作会从所有视频上移除该标签。`)) {
|
||||
return;
|
||||
}
|
||||
setDeletingId(tag.id);
|
||||
try {
|
||||
const r = await api.deleteTag(tag.id);
|
||||
show(`已删除标签,并从 ${r.removedVideos} 个视频移除`, "success");
|
||||
await refresh();
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "删除标签失败", "error");
|
||||
} finally {
|
||||
setDeletingId(null);
|
||||
}
|
||||
}
|
||||
|
||||
function toggleSelectMode() {
|
||||
setSelectMode((m) => !m);
|
||||
setSelected(new Set());
|
||||
}
|
||||
|
||||
function toggleSelect(id: number) {
|
||||
setSelected((prev) => {
|
||||
const next = new Set(prev);
|
||||
next.has(id) ? next.delete(id) : next.add(id);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
async function handleBulkDelete() {
|
||||
const ids = [...selected];
|
||||
if (ids.length === 0) return;
|
||||
if (!window.confirm(`确定删除选中的 ${ids.length} 个标签吗?此操作会从所有视频上移除这些标签。`)) {
|
||||
return;
|
||||
}
|
||||
setBulkDeleting(true);
|
||||
try {
|
||||
let ok = 0;
|
||||
for (const id of ids) {
|
||||
try {
|
||||
await api.deleteTag(id);
|
||||
ok += 1;
|
||||
} catch {
|
||||
/* 统计失败数,继续删除其余标签 */
|
||||
}
|
||||
}
|
||||
const failed = ids.length - ok;
|
||||
show(failed ? `已删除 ${ok} 个,${failed} 个失败` : `已删除 ${ok} 个标签`, failed ? "error" : "success");
|
||||
setSelected(new Set());
|
||||
setSelectMode(false);
|
||||
await refresh();
|
||||
} finally {
|
||||
setBulkDeleting(false);
|
||||
}
|
||||
}
|
||||
|
||||
const stats = useMemo(() => {
|
||||
let totalVideos = 0;
|
||||
let systemCount = 0;
|
||||
@@ -82,6 +149,41 @@ export function TagsPage() {
|
||||
});
|
||||
}, [tags, searchQuery, filterSource]);
|
||||
|
||||
const totalPages = Math.max(1, Math.ceil(filteredTags.length / pageSize));
|
||||
const currentPage = Math.min(page, totalPages);
|
||||
const pageStartIndex = (currentPage - 1) * pageSize;
|
||||
const pageEndIndex = pageStartIndex + pageSize;
|
||||
const pagedTags = useMemo(
|
||||
() => filteredTags.slice(pageStartIndex, pageEndIndex),
|
||||
[filteredTags, pageStartIndex, pageEndIndex]
|
||||
);
|
||||
const pageStart = filteredTags.length === 0 ? 0 : pageStartIndex + 1;
|
||||
const pageEnd = Math.min(filteredTags.length, pageEndIndex);
|
||||
|
||||
useEffect(() => {
|
||||
setPage(1);
|
||||
}, [searchQuery, filterSource, pageSize]);
|
||||
|
||||
useEffect(() => {
|
||||
setPage((p) => Math.min(Math.max(1, p), totalPages));
|
||||
}, [totalPages]);
|
||||
|
||||
const deletablePageTags = useMemo(
|
||||
() => pagedTags.filter((t) => t.source !== "system"),
|
||||
[pagedTags]
|
||||
);
|
||||
const allSelected =
|
||||
deletablePageTags.length > 0 && deletablePageTags.every((t) => selected.has(t.id));
|
||||
|
||||
function toggleSelectAll() {
|
||||
setSelected((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (allSelected) deletablePageTags.forEach((t) => next.delete(t.id));
|
||||
else deletablePageTags.forEach((t) => next.add(t.id));
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<section>
|
||||
<header className="admin-page__header">
|
||||
@@ -130,7 +232,7 @@ export function TagsPage() {
|
||||
|
||||
<div className="admin-card">
|
||||
<div className="admin-card__title">
|
||||
<Tags size={15} /> 系统标签统计
|
||||
<Tags size={15} /> 标签总览
|
||||
</div>
|
||||
<div className="admin-tag-stats-list">
|
||||
<div className="admin-tag-stat-item">
|
||||
@@ -141,14 +243,6 @@ export function TagsPage() {
|
||||
<span>关联视频次</span>
|
||||
<strong>{stats.totalVideos}</strong>
|
||||
</div>
|
||||
<div className="admin-tag-stat-item">
|
||||
<span>系统提取</span>
|
||||
<strong>{stats.systemCount}</strong>
|
||||
</div>
|
||||
<div className="admin-tag-stat-item">
|
||||
<span>用户创建</span>
|
||||
<strong>{stats.userCount}</strong>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -195,9 +289,52 @@ export function TagsPage() {
|
||||
>
|
||||
合集 ({stats.collectionCount})
|
||||
</button>
|
||||
{stats.legacyCount > 0 && (
|
||||
<button
|
||||
type="button"
|
||||
className={`admin-tags-filter-tab ${filterSource === "legacy" ? "is-active" : ""}`}
|
||||
onClick={() => setFilterSource("legacy")}
|
||||
>
|
||||
旧数据 ({stats.legacyCount})
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<button
|
||||
type="button"
|
||||
className={`admin-btn ${selectMode ? "is-primary" : ""}`}
|
||||
onClick={toggleSelectMode}
|
||||
>
|
||||
<CheckSquare size={13} /> {selectMode ? "退出批量" : "批量删除"}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{selectMode && (
|
||||
<div className="admin-tags-bulkbar">
|
||||
<label className="admin-check">
|
||||
<input type="checkbox" checked={allSelected} onChange={toggleSelectAll} />
|
||||
<span>全选本页 ({deletablePageTags.length})</span>
|
||||
</label>
|
||||
<span className="admin-tags-bulkbar__count">已选 {selected.size} 个</span>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setSelected(new Set())}
|
||||
disabled={selected.size === 0}
|
||||
>
|
||||
清空
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn is-danger"
|
||||
onClick={handleBulkDelete}
|
||||
disabled={selected.size === 0 || bulkDeleting}
|
||||
>
|
||||
<Trash2 size={13} /> {bulkDeleting ? "删除中..." : `删除选中 (${selected.size})`}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{loading ? (
|
||||
<div className="admin-empty">加载中...</div>
|
||||
) : filteredTags.length === 0 ? (
|
||||
@@ -205,36 +342,110 @@ export function TagsPage() {
|
||||
没有找到匹配的标签。
|
||||
</div>
|
||||
) : (
|
||||
<div className="admin-tags-grid">
|
||||
{filteredTags.map((tag) => (
|
||||
<div key={tag.id} className="admin-tag-card">
|
||||
<div className="admin-tag-card__head">
|
||||
<span className="admin-tag-card__title">{tag.label}</span>
|
||||
<span className="admin-tag-card__source-badge" data-source={tag.source}>
|
||||
{sourceLabel(tag.source)}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{tag.aliases && tag.aliases.length > 0 && (
|
||||
<div className="admin-tag-card__aliases">
|
||||
{tag.aliases.map((alias) => (
|
||||
<span key={alias} className="admin-tag-card__alias-pill">
|
||||
{alias}
|
||||
<>
|
||||
<div className="admin-tags-grid">
|
||||
{pagedTags.map((tag) => {
|
||||
const selectable = selectMode && tag.source !== "system";
|
||||
const isSelected = selected.has(tag.id);
|
||||
return (
|
||||
<div
|
||||
key={tag.id}
|
||||
className={`admin-tag-card${selectable ? " is-selectable" : ""}${
|
||||
selectable && isSelected ? " is-selected" : ""
|
||||
}`}
|
||||
onClick={selectable ? () => toggleSelect(tag.id) : undefined}
|
||||
>
|
||||
<div className="admin-tag-card__head">
|
||||
{selectable && (
|
||||
<input
|
||||
type="checkbox"
|
||||
className="admin-tag-card__check"
|
||||
checked={isSelected}
|
||||
readOnly
|
||||
/>
|
||||
)}
|
||||
<span className="admin-tag-card__title">{tag.label}</span>
|
||||
<span className="admin-tag-card__source-badge" data-source={tag.source}>
|
||||
{sourceLabel(tag.source)}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="admin-tag-card__footer">
|
||||
<span>ID: {tag.id}</span>
|
||||
<span className="admin-tag-card__count">
|
||||
<Film size={11} />
|
||||
<strong>{tag.count} 视频</strong>
|
||||
</span>
|
||||
</div>
|
||||
{tag.aliases && tag.aliases.length > 0 && (
|
||||
<div className="admin-tag-card__aliases">
|
||||
{tag.aliases.map((alias) => (
|
||||
<span key={alias} className="admin-tag-card__alias-pill">
|
||||
{alias}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="admin-tag-card__footer">
|
||||
<span className="admin-tag-card__count">
|
||||
<Film size={13} />
|
||||
<strong>{tag.count}</strong> 视频
|
||||
</span>
|
||||
<div className="admin-tag-card__footer-actions">
|
||||
<span className="admin-tag-card__id">#{tag.id}</span>
|
||||
{!selectMode && tag.source !== "system" && (
|
||||
<button
|
||||
type="button"
|
||||
className="admin-tag-card__delete"
|
||||
onClick={() => handleDelete(tag)}
|
||||
disabled={deletingId === tag.id}
|
||||
aria-label={`删除标签 ${tag.label}`}
|
||||
>
|
||||
<Trash2 size={11} />
|
||||
<span>{deletingId === tag.id ? "删除中" : "删除"}</span>
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
{totalPages > 1 && (
|
||||
<div className="admin-table-pagination admin-tags-pagination">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage(1)}
|
||||
disabled={currentPage <= 1}
|
||||
>
|
||||
首页
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage((p) => Math.max(1, p - 1))}
|
||||
disabled={currentPage <= 1}
|
||||
>
|
||||
上一页
|
||||
</button>
|
||||
<span className="admin-table-pagination__info">
|
||||
第 {currentPage} / {totalPages} 页,显示 {pageStart}-{pageEnd} / {filteredTags.length},每页 {pageSize} 个
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage((p) => Math.min(totalPages, p + 1))}
|
||||
disabled={currentPage >= totalPages}
|
||||
>
|
||||
下一页
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => setPage(totalPages)}
|
||||
disabled={currentPage >= totalPages}
|
||||
>
|
||||
末页
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
@@ -242,6 +453,26 @@ export function TagsPage() {
|
||||
);
|
||||
}
|
||||
|
||||
function useTagsPageSize() {
|
||||
const [pageSize, setPageSize] = useState(() =>
|
||||
window.matchMedia(TAGS_MOBILE_QUERY).matches
|
||||
? MOBILE_TAGS_PAGE_SIZE
|
||||
: DESKTOP_TAGS_PAGE_SIZE
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const media = window.matchMedia(TAGS_MOBILE_QUERY);
|
||||
const update = () => {
|
||||
setPageSize(media.matches ? MOBILE_TAGS_PAGE_SIZE : DESKTOP_TAGS_PAGE_SIZE);
|
||||
};
|
||||
update();
|
||||
media.addEventListener("change", update);
|
||||
return () => media.removeEventListener("change", update);
|
||||
}, []);
|
||||
|
||||
return pageSize;
|
||||
}
|
||||
|
||||
function splitList(s: string): string[] {
|
||||
return s
|
||||
.split(/[,,、\s]+/)
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
useCallback,
|
||||
useContext,
|
||||
useEffect,
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
|
||||
@@ -18,13 +19,25 @@ const ToastCtx = createContext<Ctx | null>(null);
|
||||
|
||||
export function ToastProvider({ children }: { children: ReactNode }) {
|
||||
const [items, setItems] = useState<Toast[]>([]);
|
||||
const timers = useRef<Record<string, ReturnType<typeof window.setTimeout>>>({});
|
||||
|
||||
// Deduplicate: same text won't stack, just resets the dismiss timer
|
||||
const show = useCallback((text: string, kind: ToastKind = "info") => {
|
||||
// Reset timer if duplicate
|
||||
if (timers.current[text]) {
|
||||
window.clearTimeout(timers.current[text]);
|
||||
timers.current[text] = window.setTimeout(() => {
|
||||
setItems((list) => list.filter((t) => t.text !== text));
|
||||
delete timers.current[text];
|
||||
}, 2600);
|
||||
return;
|
||||
}
|
||||
const id = Date.now() + Math.random();
|
||||
setItems((list) => [...list, { id, kind, text }]);
|
||||
window.setTimeout(() => {
|
||||
timers.current[text] = window.setTimeout(() => {
|
||||
setItems((list) => list.filter((t) => t.id !== id));
|
||||
delete timers.current[text];
|
||||
}, 2600);
|
||||
setItems((list) => [...list, { id, kind, text }]);
|
||||
}, []);
|
||||
|
||||
return (
|
||||
|
||||
+57
-11
@@ -61,14 +61,25 @@ export function me() {
|
||||
return request<{ authenticated: boolean }>("/me");
|
||||
}
|
||||
|
||||
export type UpdateCheck = {
|
||||
currentVersion: string;
|
||||
latestVersion: string;
|
||||
hasUpdate: boolean;
|
||||
releaseUrl?: string;
|
||||
checkedAt: string;
|
||||
};
|
||||
|
||||
export function checkUpdate() {
|
||||
return request<UpdateCheck>("/update/check");
|
||||
}
|
||||
|
||||
// ---------- Drives ----------
|
||||
|
||||
export type AdminDrive = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "spider91";
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
name: string;
|
||||
rootId: string;
|
||||
scanRootId: string;
|
||||
status: string;
|
||||
lastError?: string;
|
||||
hasCredential: boolean;
|
||||
@@ -82,14 +93,21 @@ export type AdminDrive = {
|
||||
skipDirIds: string[];
|
||||
// spider91 上次成功爬取时间(unix 秒);其它 kind 留空。
|
||||
lastCrawlAt?: number;
|
||||
// spider91 专用代理地址;仅后台管理接口返回,用于编辑表单回显。
|
||||
spider91Proxy?: string;
|
||||
thumbnailGenerationStatus?: DriveGenerationStatus;
|
||||
previewGenerationStatus?: DriveGenerationStatus;
|
||||
fingerprintGenerationStatus?: DriveGenerationStatus;
|
||||
thumbnailReadyCount: number;
|
||||
thumbnailPendingCount: number;
|
||||
thumbnailFailedCount: number;
|
||||
thumbnailDurationPendingCount: number;
|
||||
teaserReadyCount: number;
|
||||
teaserPendingCount: number;
|
||||
teaserFailedCount: number;
|
||||
fingerprintReadyCount: number;
|
||||
fingerprintPendingCount: number;
|
||||
fingerprintFailedCount: number;
|
||||
};
|
||||
|
||||
export type DriveGenerationStatus = {
|
||||
@@ -121,10 +139,9 @@ export function getDriveStorage() {
|
||||
|
||||
export type UpsertDriveInput = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "spider91";
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
name: string;
|
||||
rootId: string;
|
||||
scanRootId: string;
|
||||
credentials: Record<string, string>;
|
||||
/**
|
||||
* 可选:写入"扫描跳过目录"集合。`undefined` 表示不变(沿用服务端旧值),
|
||||
@@ -228,6 +245,13 @@ export function regenFailedThumbnails(id: string) {
|
||||
);
|
||||
}
|
||||
|
||||
export function regenFailedFingerprints(id: string) {
|
||||
return request<{ ok: boolean }>(
|
||||
`/drives/${encodeURIComponent(id)}/fingerprints/failed/regenerate`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- Videos ----------
|
||||
|
||||
export type AdminVideo = {
|
||||
@@ -317,6 +341,13 @@ export function createTag(label: string, aliases: string[]) {
|
||||
});
|
||||
}
|
||||
|
||||
export function deleteTag(id: number) {
|
||||
return request<{ ok: boolean; removedVideos: number }>(
|
||||
`/tags/${encodeURIComponent(String(id))}`,
|
||||
{ method: "DELETE" }
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- Settings ----------
|
||||
|
||||
export type Theme = "dark" | "pink";
|
||||
@@ -324,9 +355,9 @@ export type Theme = "dark" | "pink";
|
||||
export type Settings = {
|
||||
theme: Theme;
|
||||
/**
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak 或 p115 drive)。
|
||||
* - 空字符串:自动模式。系统中如果只挂着一个 pikpak/p115 drive 就用它;多个并存时迁移会跳过。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115}。
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115 或 onedrive drive)。
|
||||
* - 空字符串:本地保存,不上传到云盘。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, onedrive}。
|
||||
*/
|
||||
spider91UploadDriveId: string;
|
||||
};
|
||||
@@ -353,10 +384,25 @@ export function updateSettings(body: Partial<Settings>) {
|
||||
|
||||
/**
|
||||
* 立即触发一次完整的凌晨流水线(Phase1 扫盘 + Phase2 91 爬虫 + Phase3 迁移),
|
||||
* 不论当前时间或今日是否已跑。立即返回 202;进度通过 backend 日志观察。
|
||||
* 不论当前时间或今日是否已跑。立即返回 202;进度通过任务状态和 backend 日志观察。
|
||||
*
|
||||
* 流水线已在跑时后端会丢弃此次触发。
|
||||
* 流水线已在跑或已排队时,后端会拒绝重复触发。
|
||||
*/
|
||||
export function runNightlyJob() {
|
||||
return request<{ ok: boolean }>("/jobs/nightly/run", { method: "POST" });
|
||||
export type NightlyJobStatus = {
|
||||
state: "idle" | "queued" | "running" | "running_queued";
|
||||
running: boolean;
|
||||
queued: boolean;
|
||||
startedAt?: string;
|
||||
lastFinishedAt?: string;
|
||||
};
|
||||
|
||||
export function getNightlyJobStatus() {
|
||||
return request<NightlyJobStatus>("/jobs/nightly/status");
|
||||
}
|
||||
|
||||
export function runNightlyJob() {
|
||||
return request<{ ok: boolean; accepted: boolean; status: NightlyJobStatus }>(
|
||||
"/jobs/nightly/run",
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -294,5 +294,6 @@ function sourceKindFromLabel(label: string): string {
|
||||
if (value.includes("pikpak")) return "pikpak";
|
||||
if (value.includes("沃盘") || value.includes("wopan") || value.includes("联通")) return "wopan";
|
||||
if (value.includes("onedrive") || value.includes("one drive")) return "onedrive";
|
||||
if (value.includes("本地") || value.includes("localstorage") || value.includes("local storage")) return "localstorage";
|
||||
return "";
|
||||
}
|
||||
|
||||
@@ -72,5 +72,7 @@ function sourceKindFromLabel(label: string): string {
|
||||
if (value.includes("沃盘") || value.includes("wopan") || value.includes("联通"))
|
||||
return "wopan";
|
||||
if (value.includes("onedrive") || value.includes("one drive")) return "onedrive";
|
||||
if (value.includes("本地") || value.includes("localstorage") || value.includes("local storage"))
|
||||
return "localstorage";
|
||||
return "";
|
||||
}
|
||||
|
||||
+12
-5
@@ -1,8 +1,13 @@
|
||||
import type { VideoDetail, VideoItem } from "@/types";
|
||||
|
||||
// 真实后端接口调用。未配置网盘时,各接口返回空数据。
|
||||
export function fetchHomeVideos(): Promise<VideoItem[]> {
|
||||
return apiGet<VideoItem[]>("/api/home").catch(() => []);
|
||||
export function fetchHomeVideos(excludeIds?: string[]): Promise<VideoItem[]> {
|
||||
const qs = new URLSearchParams();
|
||||
for (const id of excludeIds ?? []) {
|
||||
if (id.trim()) qs.append("exclude", id.trim());
|
||||
}
|
||||
const suffix = qs.toString() ? `?${qs.toString()}` : "";
|
||||
return apiGet<VideoItem[]>(`/api/home${suffix}`).catch(() => []);
|
||||
}
|
||||
|
||||
export function fetchListing(
|
||||
@@ -93,17 +98,19 @@ export type ShortsNextResponse = {
|
||||
|
||||
/**
|
||||
* 拉取短视频流的下一批候选。把当前轮已看过的 video id 列表传给后端,
|
||||
* 服务器从未在列表中的视频里随机抽 count 条返回。
|
||||
* 服务器从未在列表中的视频里随机抽 count 条返回。preferredFromVideoId
|
||||
* 来自用户最近一次点赞成功的视频,用于按相似标签优先推荐。
|
||||
*
|
||||
* 失败时返回空批 + roundComplete=false,由调用方决定是否重试。
|
||||
*/
|
||||
export function fetchShortsNext(
|
||||
seenIds: string[],
|
||||
count: number
|
||||
count: number,
|
||||
preferredFromVideoId?: string
|
||||
): Promise<ShortsNextResponse> {
|
||||
return apiJSON<ShortsNextResponse>("/api/shorts/next", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ seenIds, count }),
|
||||
body: JSON.stringify({ seenIds, count, preferredFromVideoId }),
|
||||
}).catch(() => ({ items: [], total: 0, roundComplete: false }));
|
||||
}
|
||||
|
||||
|
||||
+34
-1
@@ -10,6 +10,8 @@ import type { VideoItem } from "@/types";
|
||||
|
||||
const DESKTOP_COUNT = 12;
|
||||
const MOBILE_COUNT = 8;
|
||||
const HOME_RECENT_KEY = "home.random.recentVideoIds";
|
||||
const HOME_RECENT_LIMIT = 72;
|
||||
|
||||
function useIsMobile() {
|
||||
const [mobile, setMobile] = useState(window.innerWidth <= 640);
|
||||
@@ -26,6 +28,35 @@ function useIsMobile() {
|
||||
let cachedRanking: VideoItem[] | null = null;
|
||||
let cachedLatest: VideoItem[] | null = null;
|
||||
|
||||
function loadRecentHomeVideoIds(): string[] {
|
||||
try {
|
||||
const raw = window.localStorage.getItem(HOME_RECENT_KEY);
|
||||
const parsed = raw ? JSON.parse(raw) : [];
|
||||
return Array.isArray(parsed)
|
||||
? parsed.filter((id): id is string => typeof id === "string" && id.length > 0)
|
||||
: [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function rememberHomeVideos(items: VideoItem[]) {
|
||||
const merged = [...items.map((item) => item.id), ...loadRecentHomeVideoIds()];
|
||||
const seen = new Set<string>();
|
||||
const recent: string[] = [];
|
||||
for (const id of merged) {
|
||||
if (!id || seen.has(id)) continue;
|
||||
seen.add(id);
|
||||
recent.push(id);
|
||||
if (recent.length >= HOME_RECENT_LIMIT) break;
|
||||
}
|
||||
try {
|
||||
window.localStorage.setItem(HOME_RECENT_KEY, JSON.stringify(recent));
|
||||
} catch {
|
||||
// localStorage 不可用时只影响连续刷新去重,不影响首页展示。
|
||||
}
|
||||
}
|
||||
|
||||
export default function HomePage() {
|
||||
const [rankingVideos, setRankingVideos] = useState<VideoItem[]>(cachedRanking ?? []);
|
||||
const [latestVideos, setLatestVideos] = useState<VideoItem[]>(cachedLatest ?? []);
|
||||
@@ -40,11 +71,13 @@ export default function HomePage() {
|
||||
|
||||
let active = true;
|
||||
setLoading(true);
|
||||
const excludeIds = loadRecentHomeVideoIds();
|
||||
Promise.all([
|
||||
fetchHomeVideos(),
|
||||
fetchHomeVideos(excludeIds),
|
||||
fetchListing(1, DESKTOP_COUNT, { sort: "latest" }),
|
||||
]).then(([rankingItems, latestResult]) => {
|
||||
if (!active) return;
|
||||
rememberHomeVideos(rankingItems);
|
||||
cachedRanking = rankingItems;
|
||||
cachedLatest = latestResult.items;
|
||||
setRankingVideos(rankingItems);
|
||||
|
||||
+83
-12
@@ -120,12 +120,16 @@ export default function ShortsPage() {
|
||||
|
||||
// seenIds 用 ref 维护,方便在异步 callback 里读到最新值
|
||||
const seenIdsRef = useRef<string[]>(loadSeenIds());
|
||||
const preferredFromVideoIdRef = useRef<string | null>(null);
|
||||
|
||||
const containerRef = useRef<HTMLDivElement | null>(null);
|
||||
// 整个页面根元素,用于 requestFullscreen
|
||||
const pageRef = useRef<HTMLDivElement | null>(null);
|
||||
// index → video element,用来精确控制播放/暂停
|
||||
const videoRefs = useRef<Map<number, HTMLVideoElement>>(new Map());
|
||||
const activeIndexRef = useRef(0);
|
||||
const ignoreIntersectionUntilRef = useRef(0);
|
||||
const fullscreenRestoreTimersRef = useRef<number[]>([]);
|
||||
|
||||
// 当前是否处在浏览器全屏(Fullscreen API)状态。
|
||||
// iOS Safari 不支持元素级 Fullscreen API,这里会一直保持 false,
|
||||
@@ -139,6 +143,10 @@ export default function ShortsPage() {
|
||||
// 用户在操作栏点取消时会从这里移除,允许之后再次点赞。
|
||||
const likedIdsRef = useRef<Set<string>>(new Set());
|
||||
|
||||
useEffect(() => {
|
||||
activeIndexRef.current = activeIndex;
|
||||
}, [activeIndex]);
|
||||
|
||||
/**
|
||||
* 切换点赞状态。
|
||||
* - liked=true:发 POST /api/video/:id/like
|
||||
@@ -163,6 +171,11 @@ export default function ShortsPage() {
|
||||
);
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const data = (await res.json()) as { likes?: number };
|
||||
if (liked) {
|
||||
preferredFromVideoIdRef.current = videoId;
|
||||
} else if (preferredFromVideoIdRef.current === videoId) {
|
||||
preferredFromVideoIdRef.current = null;
|
||||
}
|
||||
return typeof data.likes === "number" ? data.likes : null;
|
||||
} catch {
|
||||
// 请求失败:回滚集合,让 Slide 自己回滚 UI
|
||||
@@ -191,7 +204,11 @@ export default function ShortsPage() {
|
||||
setLoading(true);
|
||||
try {
|
||||
const seen = seenIdsRef.current;
|
||||
const resp = await fetchShortsNext(seen, BATCH_SIZE);
|
||||
const resp = await fetchShortsNext(
|
||||
seen,
|
||||
BATCH_SIZE,
|
||||
preferredFromVideoIdRef.current ?? undefined
|
||||
);
|
||||
if (resp.items.length === 0) {
|
||||
setEmpty((prev) => prev || true /* 维持 true 即可 */);
|
||||
setRoundComplete(true);
|
||||
@@ -250,6 +267,8 @@ export default function ShortsPage() {
|
||||
|
||||
const observer = new IntersectionObserver(
|
||||
(entries) => {
|
||||
if (Date.now() < ignoreIntersectionUntilRef.current) return;
|
||||
|
||||
let bestIndex = -1;
|
||||
let bestRatio = 0.6;
|
||||
for (const entry of entries) {
|
||||
@@ -437,11 +456,37 @@ export default function ShortsPage() {
|
||||
};
|
||||
}, []);
|
||||
|
||||
function clearFullscreenRestoreTimers() {
|
||||
for (const timer of fullscreenRestoreTimersRef.current) {
|
||||
window.clearTimeout(timer);
|
||||
}
|
||||
fullscreenRestoreTimersRef.current = [];
|
||||
}
|
||||
|
||||
function restoreActiveSlideIntoView() {
|
||||
const idx = activeIndexRef.current;
|
||||
const slide = containerRef.current?.querySelector<HTMLElement>(
|
||||
`[data-index="${idx}"]`
|
||||
);
|
||||
if (!slide) return;
|
||||
slide.scrollIntoView({ block: "start", inline: "nearest", behavior: "auto" });
|
||||
}
|
||||
|
||||
function scheduleFullscreenActiveRestore() {
|
||||
ignoreIntersectionUntilRef.current = Date.now() + 700;
|
||||
clearFullscreenRestoreTimers();
|
||||
restoreActiveSlideIntoView();
|
||||
fullscreenRestoreTimersRef.current = [80, 220, 520].map((delay) =>
|
||||
window.setTimeout(restoreActiveSlideIntoView, delay)
|
||||
);
|
||||
}
|
||||
|
||||
// ---- 浏览器全屏(Fullscreen API) ----
|
||||
// 监听全屏状态变化,保持 React state 同步。
|
||||
// 用户按 ESC / 系统返回 / 浏览器退出全屏按钮 时也会走这里。
|
||||
useEffect(() => {
|
||||
function handleChange() {
|
||||
scheduleFullscreenActiveRestore();
|
||||
setIsFullscreen(
|
||||
document.fullscreenElement !== null ||
|
||||
// Safari (desktop) 旧前缀
|
||||
@@ -454,6 +499,7 @@ export default function ShortsPage() {
|
||||
return () => {
|
||||
document.removeEventListener("fullscreenchange", handleChange);
|
||||
document.removeEventListener("webkitfullscreenchange", handleChange);
|
||||
clearFullscreenRestoreTimers();
|
||||
};
|
||||
}, []);
|
||||
|
||||
@@ -530,6 +576,7 @@ export default function ShortsPage() {
|
||||
}
|
||||
|
||||
function toggleFullscreen() {
|
||||
scheduleFullscreenActiveRestore();
|
||||
if (isFullscreen) exitPageFullscreen();
|
||||
else requestPageFullscreen();
|
||||
}
|
||||
@@ -695,6 +742,7 @@ function ShortsSlide({
|
||||
const [duration, setDuration] = useState(0);
|
||||
const [currentTime, setCurrentTime] = useState(0);
|
||||
const [scrubbing, setScrubbing] = useState(false);
|
||||
const scrubbingRef = useRef(false);
|
||||
// 拖动开始时是否在播:用于拖完后判断要不要 resume
|
||||
const wasPlayingRef = useRef(true);
|
||||
|
||||
@@ -735,6 +783,7 @@ function ShortsSlide({
|
||||
if (!isActive) {
|
||||
setPaused(false);
|
||||
setScrubbing(false);
|
||||
scrubbingRef.current = false;
|
||||
setIsBuffering(false);
|
||||
setPlayPauseHud(null);
|
||||
}
|
||||
@@ -769,7 +818,7 @@ function ShortsSlide({
|
||||
};
|
||||
const handleTime = () => {
|
||||
// 拖动期间不要被 timeupdate 覆盖 UI
|
||||
if (!scrubbing) setCurrentTime(video.currentTime);
|
||||
if (!scrubbingRef.current) setCurrentTime(video.currentTime);
|
||||
};
|
||||
const handleWaiting = () => {
|
||||
setIsBuffering(true);
|
||||
@@ -811,7 +860,7 @@ function ShortsSlide({
|
||||
video.removeEventListener("canplay", handlePlayingOrCanPlay);
|
||||
video.removeEventListener("volumechange", handleVolumeChange);
|
||||
};
|
||||
}, [shouldMount, scrubbing, muted, volume, setMuted, setVolume]);
|
||||
}, [muted, volume, setMuted, setVolume]);
|
||||
|
||||
// 长按 2 倍速:直接绑原生事件
|
||||
useEffect(() => {
|
||||
@@ -1021,11 +1070,16 @@ function ShortsSlide({
|
||||
// ---- 进度条拖动 ----
|
||||
// 触摸进度条时:暂停 → 跟随手指更新 currentTime → 松手 resume
|
||||
function handleProgressPointerDown(e: React.PointerEvent<HTMLDivElement>) {
|
||||
const video = localRef.current;
|
||||
if (!video || !duration) return;
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
(e.currentTarget as HTMLElement).setPointerCapture(e.pointerId);
|
||||
const video = localRef.current;
|
||||
const seekDuration = getSeekDuration(video);
|
||||
if (!video || !seekDuration) return;
|
||||
try {
|
||||
(e.currentTarget as HTMLElement).setPointerCapture(e.pointerId);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
wasPlayingRef.current = !video.paused;
|
||||
if (!video.paused) {
|
||||
try {
|
||||
@@ -1034,17 +1088,19 @@ function ShortsSlide({
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
scrubbingRef.current = true;
|
||||
setScrubbing(true);
|
||||
applyProgressFromEvent(e);
|
||||
applyProgressFromEvent(e, seekDuration);
|
||||
}
|
||||
function handleProgressPointerMove(e: React.PointerEvent<HTMLDivElement>) {
|
||||
if (!scrubbing) return;
|
||||
if (!scrubbingRef.current) return;
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
applyProgressFromEvent(e);
|
||||
}
|
||||
function handleProgressPointerEnd(e: React.PointerEvent<HTMLDivElement>) {
|
||||
if (!scrubbing) return;
|
||||
if (!scrubbingRef.current) return;
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
try {
|
||||
(e.currentTarget as HTMLElement).releasePointerCapture(e.pointerId);
|
||||
@@ -1052,17 +1108,30 @@ function ShortsSlide({
|
||||
// ignore
|
||||
}
|
||||
const video = localRef.current;
|
||||
scrubbingRef.current = false;
|
||||
setScrubbing(false);
|
||||
if (video && wasPlayingRef.current) {
|
||||
video.play().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
function applyProgressFromEvent(e: React.PointerEvent<HTMLDivElement>) {
|
||||
function getSeekDuration(video: HTMLVideoElement | null) {
|
||||
if (duration > 0) return duration;
|
||||
if (video && Number.isFinite(video.duration) && video.duration > 0) {
|
||||
setDuration(video.duration);
|
||||
return video.duration;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
function applyProgressFromEvent(
|
||||
e: React.PointerEvent<HTMLDivElement>,
|
||||
knownDuration?: number
|
||||
) {
|
||||
const video = localRef.current;
|
||||
if (!video || !duration) return;
|
||||
const seekDuration = knownDuration ?? getSeekDuration(video);
|
||||
if (!video || !seekDuration) return;
|
||||
const rect = e.currentTarget.getBoundingClientRect();
|
||||
const ratio = clamp((e.clientX - rect.left) / rect.width, 0, 1);
|
||||
const next = ratio * duration;
|
||||
const next = ratio * seekDuration;
|
||||
setCurrentTime(next);
|
||||
try {
|
||||
video.currentTime = next;
|
||||
@@ -1235,6 +1304,7 @@ function ShortsSlide({
|
||||
onPointerMove={handleProgressPointerMove}
|
||||
onPointerUp={handleProgressPointerEnd}
|
||||
onPointerCancel={handleProgressPointerEnd}
|
||||
onLostPointerCapture={handleProgressPointerEnd}
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
>
|
||||
<div
|
||||
@@ -1287,6 +1357,7 @@ function getDriveShortName(source: string): string {
|
||||
if (s.includes("quark") || s.includes("夸克")) return "Quak";
|
||||
if (s.includes("onedrive")) return "OneDrive";
|
||||
if (s.includes("wopan") || s.includes("沃盘")) return "沃盘";
|
||||
if (s.includes("localstorage") || s.includes("本地")) return "本地";
|
||||
if (s.includes("spider") || s.includes("爬虫")) return "爬虫";
|
||||
return source.substring(0, 4);
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import { uploadVideo } from "@/data/videos";
|
||||
import { defaultUploadTitleFromFileName } from "@/lib/uploadTitle";
|
||||
import type { VideoItem } from "@/types";
|
||||
|
||||
const UPLOAD_TAGS = ["奶子", "臀", "口角", "女大", "人妻", "AV"];
|
||||
const UPLOAD_TAGS = ["奶子", "臀", "口交", "女大", "人妻", "AV"];
|
||||
|
||||
export default function UploadPage() {
|
||||
const [file, setFile] = useState<File | null>(null);
|
||||
|
||||
+153
-20
@@ -121,10 +121,45 @@
|
||||
border-top: 1px solid var(--border-subtle);
|
||||
}
|
||||
|
||||
.admin-sidebar__check-update {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
width: 100%;
|
||||
min-height: 34px;
|
||||
padding: 8px 12px;
|
||||
border-radius: var(--radius-sm);
|
||||
color: var(--text-muted);
|
||||
font-size: var(--font-sm);
|
||||
font-weight: var(--weight-medium);
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.admin-sidebar__check-update:hover:not(:disabled) {
|
||||
color: var(--text-strong);
|
||||
background: rgba(255, 255, 255, 0.04);
|
||||
}
|
||||
|
||||
.admin-sidebar__check-update:disabled {
|
||||
opacity: 0.6;
|
||||
cursor: wait;
|
||||
}
|
||||
|
||||
.admin-sidebar__check-update:disabled svg {
|
||||
animation: admin-update-spin 0.9s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes admin-update-spin {
|
||||
to {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
|
||||
.admin-sidebar__logout {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
width: 100%;
|
||||
padding: 8px 12px;
|
||||
margin-top: 8px;
|
||||
border-radius: var(--radius-sm);
|
||||
@@ -953,12 +988,16 @@
|
||||
* ========================================================= */
|
||||
@media (max-width: 768px) {
|
||||
.admin-shell {
|
||||
grid-template-columns: minmax(0, 1fr);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
min-height: 100dvh;
|
||||
}
|
||||
|
||||
.admin-sidebar {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
flex: 0 0 48px;
|
||||
width: 100%;
|
||||
height: 48px;
|
||||
min-height: 48px;
|
||||
flex-direction: row;
|
||||
@@ -1023,7 +1062,14 @@
|
||||
.admin-nav__link.is-active::before { display: none; }
|
||||
|
||||
.admin-main {
|
||||
padding: var(--space-4) var(--space-3);
|
||||
flex: 1 1 auto;
|
||||
width: 100%;
|
||||
padding: var(--space-2) var(--space-3) var(--space-4);
|
||||
}
|
||||
|
||||
.admin-page__header {
|
||||
gap: var(--space-2);
|
||||
margin-bottom: var(--space-3);
|
||||
}
|
||||
|
||||
.admin-page__title {
|
||||
@@ -1779,6 +1825,7 @@
|
||||
.admin-drive-card__brand-icon[data-kind="pikpak"] { background: var(--drive-pikpak); }
|
||||
.admin-drive-card__brand-icon[data-kind="wopan"] { background: var(--drive-wopan); }
|
||||
.admin-drive-card__brand-icon[data-kind="onedrive"] { background: var(--drive-onedrive); }
|
||||
.admin-drive-card__brand-icon[data-kind="localstorage"] { background: var(--drive-localstorage); }
|
||||
.admin-drive-card__brand-icon[data-kind="spider91"] { background: var(--accent); }
|
||||
|
||||
.admin-drive-card__info {
|
||||
@@ -2081,28 +2128,51 @@
|
||||
}
|
||||
|
||||
.admin-tags-grid {
|
||||
column-count: 3;
|
||||
column-gap: var(--space-3);
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(240px, 1fr));
|
||||
gap: var(--space-3);
|
||||
align-items: stretch;
|
||||
}
|
||||
|
||||
@media (max-width: 1200px) {
|
||||
.admin-tags-grid {
|
||||
column-count: 2;
|
||||
}
|
||||
.admin-tags-bulkbar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--space-3);
|
||||
flex-wrap: wrap;
|
||||
margin-bottom: var(--space-4);
|
||||
padding: var(--space-2) var(--space-3);
|
||||
background: var(--bg-sunken);
|
||||
border: 1px solid var(--border-subtle);
|
||||
border-radius: var(--radius-sm);
|
||||
}
|
||||
|
||||
@media (max-width: 600px) {
|
||||
.admin-tags-grid {
|
||||
column-count: 1;
|
||||
}
|
||||
.admin-tags-bulkbar__count {
|
||||
font-size: var(--font-xs);
|
||||
color: var(--text-muted);
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
.admin-tag-card.is-selectable {
|
||||
cursor: pointer;
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
.admin-tag-card.is-selected {
|
||||
border-color: var(--border-accent);
|
||||
box-shadow: 0 0 0 1px var(--border-accent), var(--shadow-md);
|
||||
}
|
||||
|
||||
.admin-tag-card__check {
|
||||
width: 15px;
|
||||
height: 15px;
|
||||
accent-color: var(--accent);
|
||||
cursor: pointer;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.admin-tag-card {
|
||||
break-inside: avoid;
|
||||
display: inline-flex;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
width: 100%;
|
||||
margin-bottom: var(--space-3);
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border-subtle);
|
||||
border-radius: var(--radius-sm);
|
||||
@@ -2129,6 +2199,7 @@
|
||||
font-size: var(--font-md);
|
||||
font-weight: var(--weight-bold);
|
||||
color: var(--text-strong);
|
||||
margin-right: auto;
|
||||
}
|
||||
|
||||
.admin-tag-card__source-badge {
|
||||
@@ -2181,7 +2252,7 @@
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-top: var(--space-2);
|
||||
margin-top: auto;
|
||||
padding-top: var(--space-2);
|
||||
border-top: 1px solid var(--border-subtle);
|
||||
font-size: var(--font-xs);
|
||||
@@ -2190,9 +2261,71 @@
|
||||
|
||||
.admin-tag-card__count {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
align-items: baseline;
|
||||
gap: 4px;
|
||||
font-weight: var(--weight-semibold);
|
||||
color: var(--text-default);
|
||||
font-size: var(--font-xs);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
.admin-tag-card__count svg {
|
||||
align-self: center;
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.admin-tag-card__count strong {
|
||||
font-size: var(--font-md);
|
||||
font-weight: var(--weight-bold);
|
||||
color: var(--text-strong);
|
||||
}
|
||||
|
||||
.admin-tag-card__id {
|
||||
font-size: 10px;
|
||||
color: var(--text-faint);
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.admin-tag-card__footer-actions {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
}
|
||||
|
||||
.admin-tag-card__delete {
|
||||
border: 1px solid var(--danger);
|
||||
background: var(--danger-soft);
|
||||
color: var(--danger);
|
||||
border-radius: var(--radius-xs);
|
||||
padding: 3px 6px;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 3px;
|
||||
font-size: 10px;
|
||||
font-weight: var(--weight-semibold);
|
||||
cursor: pointer;
|
||||
transition: all var(--transition-fast);
|
||||
}
|
||||
|
||||
.admin-tag-card__delete:hover:not(:disabled) {
|
||||
background: var(--danger);
|
||||
color: white;
|
||||
}
|
||||
|
||||
/* 仅在支持悬停的设备上隐藏删除按钮,悬停或键盘聚焦卡片时显示,避免满屏删除按钮 */
|
||||
@media (hover: hover) {
|
||||
.admin-tag-card__delete {
|
||||
opacity: 0;
|
||||
pointer-events: none;
|
||||
transition: opacity var(--transition-fast);
|
||||
}
|
||||
|
||||
.admin-tag-card:hover .admin-tag-card__delete,
|
||||
.admin-tag-card:focus-within .admin-tag-card__delete {
|
||||
opacity: 1;
|
||||
pointer-events: auto;
|
||||
}
|
||||
}
|
||||
|
||||
.admin-tag-card__delete:disabled {
|
||||
opacity: 0.55;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
@@ -134,6 +134,7 @@
|
||||
--drive-pikpak: #8a6dff;
|
||||
--drive-wopan: #ff8a3c;
|
||||
--drive-onedrive: #4cabea;
|
||||
--drive-localstorage: #35b88f;
|
||||
|
||||
/* ----- 阴影 ----- */
|
||||
--shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.4);
|
||||
@@ -213,6 +214,7 @@
|
||||
--drive-pikpak: #8466e6;
|
||||
--drive-wopan: #e57a36;
|
||||
--drive-onedrive: #2f95cf;
|
||||
--drive-localstorage: #239978;
|
||||
|
||||
/* ----- 阴影(粉色柔投影 + 一层中性) ----- */
|
||||
--shadow-sm: 0 1px 2px rgba(180, 90, 120, 0.08);
|
||||
|
||||
@@ -306,6 +306,14 @@
|
||||
--drive-shadow: rgba(76, 171, 234, 0.15);
|
||||
--drive-shadow-strong: rgba(76, 171, 234, 0.4);
|
||||
}
|
||||
.source-badge[data-kind="localstorage"] {
|
||||
--drive-color: var(--drive-localstorage);
|
||||
--drive-bg: rgba(53, 184, 143, 0.12);
|
||||
--drive-text: #a8efdc;
|
||||
--drive-border: rgba(53, 184, 143, 0.35);
|
||||
--drive-shadow: rgba(53, 184, 143, 0.15);
|
||||
--drive-shadow-strong: rgba(53, 184, 143, 0.4);
|
||||
}
|
||||
|
||||
/* ----- Preview overlay ----- */
|
||||
.preview-loader {
|
||||
@@ -441,6 +449,7 @@
|
||||
.video-card__source[data-kind="pikpak"] { --source-color: var(--drive-pikpak); }
|
||||
.video-card__source[data-kind="wopan"] { --source-color: var(--drive-wopan); }
|
||||
.video-card__source[data-kind="onedrive"] { --source-color: var(--drive-onedrive); }
|
||||
.video-card__source[data-kind="localstorage"] { --source-color: var(--drive-localstorage); }
|
||||
|
||||
/* =========================================================
|
||||
* Empty / Loading / Skeleton
|
||||
|
||||
@@ -351,6 +351,12 @@
|
||||
color: var(--drive-onedrive);
|
||||
}
|
||||
|
||||
.vd-meta__chip[data-tone="localstorage"] {
|
||||
background: rgba(53, 184, 143, 0.14);
|
||||
border-color: rgba(53, 184, 143, 0.3);
|
||||
color: var(--drive-localstorage);
|
||||
}
|
||||
|
||||
/* ---------- Actions toolbar ---------- */
|
||||
.vd-actions {
|
||||
display: flex;
|
||||
|
||||
@@ -8,8 +8,119 @@ const drivesPageSource = readFileSync(
|
||||
);
|
||||
|
||||
test("spider91 drive form does not expose advanced crawler credentials", () => {
|
||||
assert.match(drivesPageSource, /key: "proxy"/);
|
||||
assert.match(drivesPageSource, /label: "代理地址(可选)"/);
|
||||
assert.match(drivesPageSource, /支持 http:\/\/、https:\/\/、socks5:\/\/ 或 socks5h:\/\//);
|
||||
assert.doesNotMatch(drivesPageSource, /target_new/);
|
||||
assert.doesNotMatch(drivesPageSource, /crawl_hour/);
|
||||
assert.doesNotMatch(drivesPageSource, /python_path/);
|
||||
assert.doesNotMatch(drivesPageSource, /script_path/);
|
||||
});
|
||||
|
||||
test("spider91 upload target uses explicit local-save option instead of auto target", () => {
|
||||
assert.match(drivesPageSource, /本地保存,不上传/);
|
||||
assert.match(
|
||||
drivesPageSource,
|
||||
/d\.kind === "pikpak" \|\| d\.kind === "p115" \|\| d\.kind === "onedrive"/
|
||||
);
|
||||
assert.doesNotMatch(drivesPageSource, /自动:唯一/);
|
||||
assert.doesNotMatch(drivesPageSource, /自动模式/);
|
||||
});
|
||||
|
||||
test("drive form hides root directory id for localstorage and spider91", () => {
|
||||
assert.match(drivesPageSource, /<label>根目录 ID<\/label>/);
|
||||
assert.match(
|
||||
drivesPageSource,
|
||||
/function usesRootDirectoryID\(kind: Kind\): boolean \{\s*return kind !== "localstorage" && kind !== "spider91";\s*\}/
|
||||
);
|
||||
assert.match(drivesPageSource, /\{usesRootDirectoryID\(form\.kind\) && \(/);
|
||||
assert.match(drivesPageSource, /\{usesRootDirectoryID\(d\.kind\) && \(/);
|
||||
assert.match(drivesPageSource, /placeholder=\{rootIdPlaceholder\(form\.kind\)\}/);
|
||||
assert.doesNotMatch(drivesPageSource, /扫描起点目录 ID/);
|
||||
assert.doesNotMatch(drivesPageSource, /set\("scanRootId"/);
|
||||
});
|
||||
|
||||
test("onedrive drive form only exposes required default-app fields", () => {
|
||||
const match =
|
||||
/function credentialFields[\s\S]*?case "onedrive":\s*return \[([\s\S]*?)\];\s*case "googledrive":/.exec(
|
||||
drivesPageSource
|
||||
);
|
||||
assert.ok(match, "onedrive credential field block should be present");
|
||||
const fields = match[1];
|
||||
|
||||
assert.match(fields, /key: "refresh_token"/);
|
||||
assert.doesNotMatch(fields, /key: "access_token"/);
|
||||
assert.doesNotMatch(fields, /key: "api_url_address"/);
|
||||
assert.doesNotMatch(fields, /key: "region"/);
|
||||
assert.doesNotMatch(fields, /key: "is_sharepoint"/);
|
||||
assert.doesNotMatch(fields, /key: "site_id"/);
|
||||
});
|
||||
|
||||
test("googledrive drive form only exposes refresh token", () => {
|
||||
assert.match(drivesPageSource, /<option value="googledrive">Google Drive<\/option>/);
|
||||
|
||||
const match =
|
||||
/case "googledrive":\s*return \[([\s\S]*?)\];\s*case "localstorage":/.exec(
|
||||
drivesPageSource
|
||||
);
|
||||
assert.ok(match, "googledrive credential field block should be present");
|
||||
const fields = match[1];
|
||||
|
||||
assert.match(fields, /key: "refresh_token"/);
|
||||
assert.doesNotMatch(fields, /key: "access_token"/);
|
||||
assert.doesNotMatch(fields, /key: "api_url_address"/);
|
||||
assert.doesNotMatch(fields, /key: "client_id"/);
|
||||
assert.doesNotMatch(fields, /key: "client_secret"/);
|
||||
});
|
||||
|
||||
test("pikpak drive form only exposes account login fields", () => {
|
||||
const match =
|
||||
/case "pikpak":\s*return \[([\s\S]*?)\];\s*case "wopan":/.exec(
|
||||
drivesPageSource
|
||||
);
|
||||
assert.ok(match, "pikpak credential field block should be present");
|
||||
const fields = match[1];
|
||||
|
||||
assert.match(fields, /key: "username"/);
|
||||
assert.match(fields, /key: "password"/);
|
||||
assert.doesNotMatch(fields, /key: "platform"/);
|
||||
assert.doesNotMatch(fields, /key: "refresh_token"/);
|
||||
assert.doesNotMatch(fields, /key: "captcha_token"/);
|
||||
assert.doesNotMatch(fields, /key: "device_id"/);
|
||||
assert.doesNotMatch(fields, /key: "disable_media_link"/);
|
||||
});
|
||||
|
||||
test("localstorage drive form asks for a server directory path", () => {
|
||||
assert.match(drivesPageSource, /<option value="localstorage">本地存储<\/option>/);
|
||||
|
||||
const match =
|
||||
/case "localstorage":\s*return \[([\s\S]*?)\];\s*case "spider91":/.exec(
|
||||
drivesPageSource
|
||||
);
|
||||
assert.ok(match, "localstorage credential field block should be present");
|
||||
const fields = match[1];
|
||||
|
||||
assert.match(fields, /key: "path"/);
|
||||
assert.match(fields, /label: "本地目录路径"/);
|
||||
assert.match(drivesPageSource, /if \(kind === "localstorage"\) return "\/"/);
|
||||
assert.match(drivesPageSource, /kind !== "localstorage" && kind !== "spider91"/);
|
||||
});
|
||||
|
||||
test("drive type selector keeps primary source order", () => {
|
||||
const options = Array.from(
|
||||
drivesPageSource.matchAll(/<option value="([^"]+)">([^<]+)<\/option>/g),
|
||||
(match) => ({ value: match[1], label: match[2] })
|
||||
);
|
||||
const driveOptions = options.slice(0, 8);
|
||||
|
||||
assert.deepEqual(driveOptions, [
|
||||
{ value: "p115", label: "115 网盘" },
|
||||
{ value: "pikpak", label: "PikPak" },
|
||||
{ value: "onedrive", label: "OneDrive" },
|
||||
{ value: "googledrive", label: "Google Drive" },
|
||||
{ value: "localstorage", label: "本地存储" },
|
||||
{ value: "spider91", label: "91 Spider" },
|
||||
{ value: "quark", label: "夸克网盘" },
|
||||
{ value: "wopan", label: "联通沃盘" },
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -82,10 +82,14 @@ test("admin modals and action footers adapt on mobile", () => {
|
||||
test("mobile admin top navigation stays compact", () => {
|
||||
const css = mobileCss();
|
||||
|
||||
assert.match(ruleBody(css, ".admin-shell"), /display\s*:\s*flex/);
|
||||
assert.match(ruleBody(css, ".admin-shell"), /flex-direction\s*:\s*column/);
|
||||
assert.match(ruleBody(css, ".admin-sidebar"), /height\s*:\s*48px/);
|
||||
assert.match(ruleBody(css, ".admin-sidebar"), /min-height\s*:\s*48px/);
|
||||
assert.match(ruleBody(css, ".admin-nav"), /align-items\s*:\s*center/);
|
||||
assert.match(ruleBody(css, ".admin-nav__link"), /height\s*:\s*34px/);
|
||||
assert.match(ruleBody(css, ".admin-nav__link"), /line-height\s*:\s*1/);
|
||||
assert.match(ruleBody(css, ".admin-nav__link"), /flex\s*:\s*0\s+0\s+auto/);
|
||||
assert.match(ruleBody(css, ".admin-main"), /padding\s*:\s*var\(--space-2\)\s+var\(--space-3\)\s+var\(--space-4\)/);
|
||||
assert.match(ruleBody(css, ".admin-page__header"), /margin-bottom\s*:\s*var\(--space-3\)/);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import test from "node:test";
|
||||
|
||||
const tagsPageSource = readFileSync(
|
||||
new URL("../src/admin/TagsPage.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
|
||||
test("admin tags page limits visible tags by viewport", () => {
|
||||
assert.match(tagsPageSource, /const DESKTOP_TAGS_PAGE_SIZE = 25;/);
|
||||
assert.match(tagsPageSource, /const MOBILE_TAGS_PAGE_SIZE = 8;/);
|
||||
assert.match(tagsPageSource, /const TAGS_MOBILE_QUERY = "\(max-width: 640px\)";/);
|
||||
assert.match(tagsPageSource, /window\.matchMedia\(TAGS_MOBILE_QUERY\)/);
|
||||
});
|
||||
|
||||
test("admin tags page renders only the current page", () => {
|
||||
assert.match(tagsPageSource, /filteredTags\.slice\(pageStartIndex, pageEndIndex\)/);
|
||||
assert.match(tagsPageSource, /pagedTags\.map\(\(tag\) =>/);
|
||||
assert.doesNotMatch(tagsPageSource, /filteredTags\.map\(\(tag\) =>/);
|
||||
assert.match(tagsPageSource, /全选本页/);
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import test from "node:test";
|
||||
|
||||
const shortsPageSource = readFileSync(
|
||||
new URL("../src/pages/ShortsPage.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
|
||||
test("shorts recommendation preference follows successful likes instead of watch time", () => {
|
||||
assert.doesNotMatch(shortsPageSource, /currentTime\s*>=\s*3/);
|
||||
assert.doesNotMatch(shortsPageSource, /onPreferenceReady/);
|
||||
|
||||
const match = /const handleLikeToggle[\s\S]*?const hasLiked/.exec(
|
||||
shortsPageSource
|
||||
);
|
||||
assert.ok(match, "handleLikeToggle block should be present");
|
||||
|
||||
assert.match(
|
||||
match[0],
|
||||
/if \(liked\) \{\s*preferredFromVideoIdRef\.current = videoId;\s*\} else if \(preferredFromVideoIdRef\.current === videoId\) \{\s*preferredFromVideoIdRef\.current = null;/
|
||||
);
|
||||
});
|
||||
|
||||
test("shorts progress dragging uses immediate pointer state", () => {
|
||||
assert.match(shortsPageSource, /const scrubbingRef = useRef\(false\)/);
|
||||
assert.match(shortsPageSource, /scrubbingRef\.current = true;/);
|
||||
assert.match(shortsPageSource, /if \(!scrubbingRef\.current\) return;/);
|
||||
assert.doesNotMatch(shortsPageSource, /if \(!scrubbing\) return;/);
|
||||
assert.match(shortsPageSource, /function getSeekDuration/);
|
||||
assert.match(shortsPageSource, /onLostPointerCapture=\{handleProgressPointerEnd\}/);
|
||||
});
|
||||
|
||||
test("shorts fullscreen changes preserve the active slide", () => {
|
||||
assert.match(shortsPageSource, /const activeIndexRef = useRef\(0\)/);
|
||||
assert.match(shortsPageSource, /const ignoreIntersectionUntilRef = useRef\(0\)/);
|
||||
assert.match(
|
||||
shortsPageSource,
|
||||
/if \(Date\.now\(\) < ignoreIntersectionUntilRef\.current\) return;/
|
||||
);
|
||||
assert.match(shortsPageSource, /function scheduleFullscreenActiveRestore\(\)/);
|
||||
assert.match(shortsPageSource, /scheduleFullscreenActiveRestore\(\);\s*setIsFullscreen/);
|
||||
assert.match(
|
||||
shortsPageSource,
|
||||
/function toggleFullscreen\(\) \{\s*scheduleFullscreenActiveRestore\(\);/
|
||||
);
|
||||
assert.match(shortsPageSource, /scrollIntoView\(\{ block: "start", inline: "nearest", behavior: "auto" \}\)/);
|
||||
});
|
||||
@@ -1441,7 +1441,7 @@ VideoProject/
|
||||
| 项 | 决定 |
|
||||
|---|---|
|
||||
| 登录方式 | **B**:管理后台做完整登录流程。115 扫码、夸克扫码或 Cookie 导入、沃盘手机号 + 短信验证。Token 持久化到 SQLite 并自动刷新。 |
|
||||
| 元数据来源 | **默认文件名解析**:`标题.mp4` 或 `[tag1,tag2] 标题 - 作者.mp4`;同时提供后台录入 API 覆盖字段 |
|
||||
| 元数据来源 | **默认文件名解析**:`标题.mp4`、`标题 - 作者.mp4`,或带前缀的 `[前缀] 标题 - 作者.mp4`;前缀只用于标题清理,不作为任意标签列表入库。标签来自系统 / 用户标签匹配和目录合集规则;同时提供后台录入 API 覆盖字段 |
|
||||
| Hover teaser | **C 预生成**:scanner 发现新视频时异步生成 10s teaser 并存回网盘的 `previews/` 目录,详情页和列表页 hover 都秒开 |
|
||||
| 部署目标 | Linux 服务器;本地 Windows 开发 |
|
||||
| 扫描策略 | 启动时全量 + 每 6 小时增量 + 支持手动触发 |
|
||||
@@ -1485,14 +1485,14 @@ type StreamLink struct {
|
||||
|
||||
### 15.5 文件名解析规则
|
||||
|
||||
默认解析顺序(取第一个匹配):
|
||||
默认解析顺序(取第一个匹配),用于提取 `title` / `author`:
|
||||
|
||||
1. 完整格式:`[tag1,tag2] 标题 - 作者.ext`
|
||||
2. 去作者:`[tag1,tag2] 标题.ext`
|
||||
3. 去标签:`标题 - 作者.ext`
|
||||
1. 带前缀和作者:`[前缀] 标题 - 作者.ext`
|
||||
2. 带前缀:`[前缀] 标题.ext`
|
||||
3. 带作者:`标题 - 作者.ext`
|
||||
4. 最简单:`标题.ext`
|
||||
|
||||
解析出的字段:`title` / `author` / `tags[]`。其余字段(`duration` / `views` / `favorites` 等)由 scanner 读取文件元数据或置默认值。
|
||||
开头的 `[前缀]` 只会从标题里剥离,不会按 `,` / `,` / `、` / 空格拆成任意标签入库。`tags[]` 由 scanner 另行生成:文件名、作者和目录名命中系统标签或已有标签的标签名 / 别名时自动打标;符合条件的目录名会创建 `collection` 合集标签;常见番号类文本会归并为 `AV`。当前内置系统标签是 `后入`、`奶子`、`口交`、`臀`、`人妻`、`女大`、`AV`。其余字段(`duration` / `views` / `favorites` 等)由 scanner 读取文件元数据或置默认值。
|
||||
|
||||
后台录入接口可用来覆盖解析结果:
|
||||
|
||||
@@ -1570,6 +1570,10 @@ POST /admin/api/videos # 手动新建
|
||||
PUT /admin/api/videos/:id # 修改元数据
|
||||
DELETE /admin/api/videos/:id
|
||||
POST /admin/api/videos/:id/regen-preview
|
||||
|
||||
GET /admin/api/tags # 标签列表
|
||||
POST /admin/api/tags # 新增标签并自动归类历史视频
|
||||
DELETE /admin/api/tags/:id # 删除非系统标签,并从所有视频上移除
|
||||
```
|
||||
|
||||
登录流程三家各不相同:
|
||||
@@ -1684,9 +1688,10 @@ Teaser 不再是"固定从第 10 秒抽 10 秒",改为按视频时长分段挑
|
||||
- `backend/internal/catalog/tags.go` `migrate` / `pruneOrphanCollectionTags` / `pruneOrphanCollectionTagsByID` / `collectVideoTagIDs`
|
||||
- 测试:`backend/internal/catalog/tags_test.go` `TestDeleteVideoPrunesOrphanCollectionTag` / `TestMigratePrunesPreexistingOrphanCollectionTags`
|
||||
|
||||
**已知不在本次范围**:
|
||||
- `/admin/api/tags` 仍只有 `GET` / `POST`,没有 `DELETE`。如果将来要让管理员手动删 `user` 标签,再加 endpoint。
|
||||
- 数据迁移:上线时对运行中数据库一次性执行同样的 `DELETE` 即可(已对当前实例执行:清掉 10 条 `Season N` / `Better Call Saul SXX` / `东京爱情故事(1991)`,`tags` 总数 153 → 143)。
|
||||
**手动删除标签**:
|
||||
- `/admin/api/tags/{id}` 支持 `DELETE`。管理员可手动删除非系统标签,删除时同步清理 `video_tags` 并刷新相关视频的 `videos.tags` JSON。
|
||||
- `system` 标签由固定标签池维护,不开放删除;`user` / `collection` / `legacy` 标签可由管理员按需删除。
|
||||
- 历史孤儿 `collection` 标签仍由迁移自愈逻辑自动清理。
|
||||
|
||||
### 14.7 取消浏览器内本机转码,全部走 302 直链 + VLC 外部播放器按钮(2026-05-21)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user