mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-25 13:12:39 +08:00
Compare commits
16 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2adaac3d7d | |||
| ee8af315b0 | |||
| 6884473dbf | |||
| f0458f7043 | |||
| e32da9016b | |||
| 2427f58165 | |||
| 00aaeed736 | |||
| 5efbceb205 | |||
| 0faeaf408f | |||
| 1b5eda92b0 | |||
| 840a858dbd | |||
| 1ee5ee35be | |||
| 12b737b6fe | |||
| bd33d26a1f | |||
| 36fe32cb84 | |||
| 194d98895a |
@@ -30,9 +30,6 @@ tmp/
|
||||
|
||||
# 91 爬虫脚本独立运行时的默认输出文件(backend 跑时会显式 --output 到 backend/data/spider91/,所以不会落在这里)
|
||||
91porn_videos.json
|
||||
91VideoSpider/91porn_videos.json
|
||||
91VideoSpider/data/
|
||||
91VideoSpider/__pycache__/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
|
||||
@@ -1,988 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
91porn 视频爬虫脚本
|
||||
===================
|
||||
爬取 https://www.91porn.com/v.php?category=top&viewtype=basic 下的所有视频信息:
|
||||
- 视频名称
|
||||
- 封面图直链
|
||||
- 视频直链 (MP4)
|
||||
|
||||
依赖安装:
|
||||
pip install requests beautifulsoup4 lxml PySocks
|
||||
|
||||
使用方法:
|
||||
# 作为 video-site-91 通用爬虫脚本运行(后台会自动这样调用)
|
||||
python spider_91porn.py --job /path/to/job.json
|
||||
|
||||
# 全量爬取(默认行为,从 page=1 一直爬到末尾,写到 OUTPUT_FILE)
|
||||
python spider_91porn.py
|
||||
|
||||
# 只爬指定页(单页模式,手动调试用)
|
||||
python spider_91porn.py --page 1 --output /tmp/spider91_page1.json
|
||||
|
||||
# 凑够 N 个新视频模式(backend 凌晨任务用)
|
||||
python spider_91porn.py --target-new 15 --seen-viewkeys-file /tmp/seen.txt --output /tmp/new.json
|
||||
|
||||
CLI 参数:
|
||||
--job FILE crawler.v1 job JSON 路径;后台爬虫管理会使用此模式
|
||||
--page N 只爬第 N 页,配合 --output 用于手动调试
|
||||
--target-new N 从 page 1 起翻页直到凑够 N 个新视频(不在 seen 列表里的)
|
||||
--seen-viewkeys-file FILE 每行一个已知 viewkey 或 mp4 源 ID,命中即跳过;与 --target-new 配合使用
|
||||
--output FILE 输出 JSON 路径,覆盖默认的 OUTPUT_FILE
|
||||
--no-resume 禁用断点续爬(单页/target-new 模式下自动禁用)
|
||||
--quiet 压缩日志,每条视频只输出一行
|
||||
-h / --help 帮助
|
||||
|
||||
配置说明 (编辑脚本内 "配置区域"):
|
||||
- MIN_PAGE_DELAY / MAX_PAGE_DELAY : 列表页请求间隔 (默认 3-6 秒)
|
||||
- MIN_DETAIL_DELAY / MAX_DETAIL_DELAY : 详情页请求间隔 (默认 2-5 秒)
|
||||
- MAX_PAGES : 限制最大爬取页数 (None=不限, 如 5=只爬前5页)
|
||||
- OUTPUT_FILE : 输出文件名
|
||||
|
||||
输出格式 (JSON):
|
||||
--job 模式下 stdout 输出 crawler.v1 JSON Lines,日志全部写到 stderr。
|
||||
手动运行模式仍会写传统 JSON 文件:
|
||||
{
|
||||
"videos": [
|
||||
{
|
||||
"title": "视频标题",
|
||||
"thumb_url": "https://...thumb/xxxx.jpg",
|
||||
"video_url": "https://...mp43/xxxx.mp4?st=...",
|
||||
"viewkey": "abc123...",
|
||||
"source_id": "xxxx",
|
||||
"detail_url": "https://...view_video.php?viewkey=..."
|
||||
},
|
||||
...
|
||||
]
|
||||
}
|
||||
|
||||
注意:
|
||||
1. 视频直链包含时效性token (e参数为过期时间戳),会过期,需定期重新爬取
|
||||
2. 脚本已内置随机延时,请勿移除,避免对服务器造成压力
|
||||
3. 网站有Cloudflare保护,如遇到403/5xx错误,可能需要使用带cookie的session
|
||||
4. 本脚本仅供学习交流,请遵守当地法律法规
|
||||
|
||||
作者: OpenCode
|
||||
日期: 2026-05-22
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import requests
|
||||
import re
|
||||
import time
|
||||
import random
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import html
|
||||
from urllib.parse import urljoin, unquote, urlparse
|
||||
from datetime import datetime
|
||||
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except ImportError:
|
||||
print("错误: 缺少依赖库 beautifulsoup4", file=sys.stderr)
|
||||
print("请运行: pip install beautifulsoup4 lxml", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def prefer_ipv4_for_plain_socks5_proxy():
|
||||
"""PySocks may pick IPv6 first for socks5://; some SOCKS5 servers only accept IPv4."""
|
||||
proxy_envs = (
|
||||
os.environ.get("HTTPS_PROXY", ""),
|
||||
os.environ.get("HTTP_PROXY", ""),
|
||||
os.environ.get("https_proxy", ""),
|
||||
os.environ.get("http_proxy", ""),
|
||||
)
|
||||
uses_plain_socks5 = any(v.strip().lower().startswith("socks5://") for v in proxy_envs)
|
||||
if not uses_plain_socks5 or getattr(socket, "_spider91_ipv4_first", False):
|
||||
return
|
||||
|
||||
original_getaddrinfo = socket.getaddrinfo
|
||||
|
||||
def getaddrinfo_ipv4_first(*args, **kwargs):
|
||||
infos = original_getaddrinfo(*args, **kwargs)
|
||||
return sorted(infos, key=lambda info: 0 if info[0] == socket.AF_INET else 1)
|
||||
|
||||
socket.getaddrinfo = getaddrinfo_ipv4_first
|
||||
socket._spider91_ipv4_first = True
|
||||
|
||||
# ===================== 配置区域 =====================
|
||||
BASE_URL = "https://www.91porn.com/v.php"
|
||||
LIST_PARAMS = {
|
||||
"category": "top",
|
||||
"viewtype": "basic"
|
||||
}
|
||||
|
||||
# 请求头 (模拟真实浏览器)
|
||||
HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/125.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": (
|
||||
"text/html,application/xhtml+xml,application/xml;"
|
||||
"q=0.9,image/avif,image/webp,image/apng,*/*;"
|
||||
"q=0.8,application/signed-exchange;v=b3;q=0.7"
|
||||
),
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
# 注意: 不要包含 "br" (brotli),除非安装了 brotli 库
|
||||
# "Accept-Encoding": "gzip, deflate, br",
|
||||
"Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
}
|
||||
|
||||
# 延时配置 (秒) - 控制爬取频率,避免被封
|
||||
MIN_PAGE_DELAY = 3.0 # 列表页之间最小延时
|
||||
MAX_PAGE_DELAY = 6.0 # 列表页之间最大延时
|
||||
MIN_DETAIL_DELAY = 2.0 # 详情页之间最小延时
|
||||
MAX_DETAIL_DELAY = 5.0 # 详情页之间最大延时
|
||||
|
||||
# 重试配置
|
||||
MAX_RETRIES = 3
|
||||
RETRY_DELAY = 5.0
|
||||
|
||||
# 输出配置
|
||||
OUTPUT_FILE = "91porn_videos.json"
|
||||
MAX_PAGES = None # 设置为 None 爬取所有页,或设置整数如 5 只爬前5页
|
||||
RESUME = True # 是否跳过输出文件中已存在的 viewkey (断点续爬)
|
||||
MAX_EMPTY_PAGES = 2 # 连续空页数达到此值时停止爬取
|
||||
CRAWLER_NAME = "91Porn"
|
||||
CRAWLER_PROTOCOL = "crawler.v1"
|
||||
# ===================================================
|
||||
|
||||
|
||||
def crawler_source_id(raw: str) -> str:
|
||||
"""Return a backend-safe source_id, preserving existing numeric 91 IDs."""
|
||||
value = str(raw or "").strip()
|
||||
if not value:
|
||||
return ""
|
||||
safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", value).strip("._-")
|
||||
return safe[:160]
|
||||
|
||||
|
||||
def write_jsonl(event: dict):
|
||||
print(json.dumps(event, ensure_ascii=False), flush=True)
|
||||
|
||||
|
||||
class Porn91Spider:
|
||||
def __init__(
|
||||
self,
|
||||
output_file: str = None,
|
||||
start_page: int = 1,
|
||||
max_pages: int = None,
|
||||
resume: bool = None,
|
||||
max_empty_pages: int = None,
|
||||
quiet: bool = False,
|
||||
target_new: int = None,
|
||||
seen_viewkeys: list = None,
|
||||
stream_output: bool = False,
|
||||
stream_protocol: str = "legacy",
|
||||
):
|
||||
"""
|
||||
构造函数。所有参数都有默认值,等同于使用脚本顶部的全局配置。
|
||||
backend 调用时会传 output_file/seen_viewkeys/target_new,等价于:
|
||||
"从第 1 页开始爬,跳过 seen_viewkeys 里的视频,凑够 target_new 个新视频后停止"
|
||||
|
||||
stream_output=True 时(backend 流水线用):
|
||||
- 每凑齐一个 video 直链就把该 entry 作为一行 JSON 写到 stdout 并 flush,
|
||||
便于上层(Go crawler)边读边下载,不再等所有详情页处理完。
|
||||
- 所有日志改走 stderr,避免与 stdout JSONL 流混合。
|
||||
- --output 仍生效,作为离线归档用(脚本退出时一次性写完整 JSON)。
|
||||
"""
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(HEADERS)
|
||||
# 91porn 没有固定 mode cookie 时,详情页首次请求可能返回与列表卡片
|
||||
# 不一致的视频源;固定桌面模式让列表页和详情页解析保持一致。
|
||||
self.session.cookies.set("mode", "d")
|
||||
|
||||
# 解析后的实际配置;优先使用构造参数,回退到模块级配置
|
||||
self.output_file = output_file if output_file is not None else OUTPUT_FILE
|
||||
self.start_page = max(1, int(start_page or 1))
|
||||
# max_pages=None 表示不限制;max_pages=N 表示从 start_page 起爬 N 页
|
||||
self.max_pages = max_pages if max_pages is None or max_pages > 0 else None
|
||||
# resume 默认跟模块配置;单页模式下调用方应该显式传 False
|
||||
self.resume = RESUME if resume is None else bool(resume)
|
||||
self.max_empty_pages = (
|
||||
MAX_EMPTY_PAGES if max_empty_pages is None else int(max_empty_pages)
|
||||
)
|
||||
# target_new 是 backend 触发时的核心模式:累计处理这么多新源视频后退出。
|
||||
self.target_new = target_new if target_new and target_new > 0 else None
|
||||
self.quiet = bool(quiet)
|
||||
# stream_output:每解析出一个 video 直链立即输出一行 JSON 到 stdout
|
||||
# (配合 backend Go 端 bufio.Scanner 实时消费,下载一个就开始下一个)。
|
||||
# 开启后所有 log 都走 stderr。
|
||||
self.stream_output = bool(stream_output)
|
||||
self.stream_protocol = stream_protocol or "legacy"
|
||||
|
||||
# 添加重试适配器
|
||||
try:
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
retry_strategy = Retry(
|
||||
total=MAX_RETRIES,
|
||||
backoff_factor=1,
|
||||
status_forcelist=[429, 500, 502, 503, 504],
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry_strategy)
|
||||
self.session.mount("https://", adapter)
|
||||
self.session.mount("http://", adapter)
|
||||
except ImportError:
|
||||
pass # urllib3 版本可能较低
|
||||
|
||||
self.results = []
|
||||
self.pages_crawled = 0
|
||||
self.processed_videos = 0
|
||||
self.skipped_videos = 0
|
||||
self.failed_videos = 0
|
||||
self.skip_viewkeys = set()
|
||||
|
||||
# backend 通过 --seen-viewkeys-file 传进来一批已入库的历史 ID。
|
||||
# 兼容旧名:文件里可能是 viewkey,也可能是新逻辑使用的 mp4 源 ID。
|
||||
if seen_viewkeys:
|
||||
for vk in seen_viewkeys:
|
||||
if not vk:
|
||||
continue
|
||||
vk = vk.strip()
|
||||
if vk:
|
||||
self.skip_viewkeys.add(vk)
|
||||
|
||||
# 断点续爬:加载已有结果,跳过已处理的 viewkey
|
||||
if self.resume and os.path.exists(self.output_file):
|
||||
try:
|
||||
with open(self.output_file, 'r', encoding='utf-8') as f:
|
||||
existing_data = json.load(f)
|
||||
existing_videos = existing_data.get('videos', [])
|
||||
self.results = existing_videos
|
||||
for v in existing_videos:
|
||||
vk = v.get('viewkey', '')
|
||||
if vk:
|
||||
self.skip_viewkeys.add(vk)
|
||||
self.processed_videos = existing_data.get('successful', 0)
|
||||
self.failed_videos = existing_data.get('failed', 0)
|
||||
self.log(f"加载已有数据: {len(self.results)} 个视频, 将跳过已处理项")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def log(self, message: str):
|
||||
"""带时间戳的日志输出。stream_output 模式下走 stderr,避免污染 stdout JSONL。"""
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
line = f"[{timestamp}] {message}"
|
||||
if self.stream_output:
|
||||
print(line, file=sys.stderr, flush=True)
|
||||
else:
|
||||
print(line)
|
||||
|
||||
def emit_stream_video(self, video: dict):
|
||||
"""stream_output 模式下把单条 video entry 作为一行 JSON 写到 stdout 并立即刷盘。
|
||||
Go 端 bufio.Scanner 按行读取,每收到一行就立即下载视频和封面。"""
|
||||
if not self.stream_output:
|
||||
return
|
||||
try:
|
||||
if self.stream_protocol == "crawler.v1":
|
||||
source_id = crawler_source_id(video.get("source_id") or video.get("viewkey") or "")
|
||||
item = {
|
||||
"title": video.get("title") or "",
|
||||
"detail_url": video.get("detail_url") or "",
|
||||
"author": "91porn",
|
||||
"tags": ["91porn"],
|
||||
"media_url": video.get("video_url") or "",
|
||||
"thumbnail_url": video.get("thumb_url") or "",
|
||||
"headers": {
|
||||
"Referer": video.get("detail_url") or BASE_URL,
|
||||
},
|
||||
}
|
||||
if source_id:
|
||||
item["source_id"] = source_id
|
||||
event = {
|
||||
"type": "item",
|
||||
"item": item,
|
||||
}
|
||||
write_jsonl(event)
|
||||
else:
|
||||
print(json.dumps(video, ensure_ascii=False), flush=True)
|
||||
except Exception as e:
|
||||
# stdout 异常基本只在管道断开时发生(消费方进程死了);
|
||||
# 写到 stderr 让 backend 看到,然后让 crawl 循环自己 break。
|
||||
print(f"[stream] emit failed: {e}", file=sys.stderr, flush=True)
|
||||
|
||||
def random_sleep(self, min_sec: float, max_sec: float):
|
||||
"""随机延时,模拟人类行为"""
|
||||
delay = random.uniform(min_sec, max_sec)
|
||||
if not self.quiet:
|
||||
self.log(f" 随机延时 {delay:.2f} 秒...")
|
||||
time.sleep(delay)
|
||||
|
||||
def fetch_page(self, url: str, description: str = "", referer: str = "") -> str:
|
||||
"""
|
||||
获取页面HTML内容,带错误处理和重试
|
||||
"""
|
||||
headers_extra = {}
|
||||
if referer:
|
||||
headers_extra["Referer"] = referer
|
||||
|
||||
for attempt in range(1, MAX_RETRIES + 1):
|
||||
try:
|
||||
self.log(f"正在请求: {description or url} (尝试 {attempt}/{MAX_RETRIES})")
|
||||
response = self.session.get(url, timeout=30, headers=headers_extra)
|
||||
|
||||
# 检查是否被Cloudflare拦截 (需在 raise_for_status 之前)
|
||||
if response.status_code == 403:
|
||||
self.log("警告: 收到 403 Forbidden,可能被拦截")
|
||||
if attempt < MAX_RETRIES:
|
||||
self.random_sleep(RETRY_DELAY, RETRY_DELAY + 3)
|
||||
continue
|
||||
return ""
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
# 优先使用 content.decode('utf-8'),避免 requests 编码检测问题
|
||||
try:
|
||||
html_content = response.content.decode('utf-8', errors='replace')
|
||||
except Exception:
|
||||
html_content = response.text
|
||||
|
||||
# Cloudflare 挑战检测:如果页面主要内容只有挑战页面,而非正常内容
|
||||
# 注意:网站本身会加载 challenge-platform 脚本,所以不能仅凭此判断
|
||||
is_cf_challenge = (
|
||||
"Just a moment" in html_content and
|
||||
len(html_content) < 8000
|
||||
)
|
||||
if is_cf_challenge:
|
||||
self.log("警告: 页面被Cloudflare挑战拦截,需要浏览器环境或正确cookie")
|
||||
if attempt < MAX_RETRIES:
|
||||
self.random_sleep(RETRY_DELAY, RETRY_DELAY + 5)
|
||||
continue
|
||||
return ""
|
||||
|
||||
return html_content
|
||||
except requests.exceptions.HTTPError as e:
|
||||
self.log(f"HTTP错误: {e}")
|
||||
if attempt < MAX_RETRIES:
|
||||
self.random_sleep(RETRY_DELAY, RETRY_DELAY + 3)
|
||||
else:
|
||||
return ""
|
||||
except requests.exceptions.RequestException as e:
|
||||
self.log(f"请求失败: {e}")
|
||||
if attempt < MAX_RETRIES:
|
||||
self.random_sleep(RETRY_DELAY, RETRY_DELAY + 3)
|
||||
else:
|
||||
self.log(f"达到最大重试次数,放弃: {url}")
|
||||
return ""
|
||||
return ""
|
||||
|
||||
def parse_list_page(self, html: str) -> list:
|
||||
"""
|
||||
解析列表页,提取视频基本信息
|
||||
返回: [{title, detail_url, thumb_url, viewkey}, ...]
|
||||
"""
|
||||
videos = []
|
||||
soup = BeautifulSoup(html, 'lxml')
|
||||
|
||||
# 只解析正常视频卡片。页面中还混有 col-lg-8 的异常大卡片,里面的标题、
|
||||
# thumb、detail URL 会串到其它视频,不能作为入库来源。
|
||||
video_cards = soup.select('div.col-xs-12.col-sm-4.col-md-3.col-lg-3')
|
||||
|
||||
seen_cards = set()
|
||||
|
||||
for card in video_cards:
|
||||
link = card.find('a', href=re.compile(r'view_video\.php\?viewkey='))
|
||||
if not link:
|
||||
continue
|
||||
href = link.get('href', '')
|
||||
if not href:
|
||||
continue
|
||||
|
||||
# 提取 viewkey
|
||||
match = re.search(r'viewkey=([^&]+)', href)
|
||||
if not match:
|
||||
continue
|
||||
viewkey = match.group(1)
|
||||
|
||||
detail_url = urljoin(BASE_URL, href)
|
||||
|
||||
# 提取标题
|
||||
title = self._extract_title(link)
|
||||
|
||||
# 提取列表卡片来源 ID 和封面图 URL
|
||||
thumb_url = ""
|
||||
source_id = ""
|
||||
overlay = link.find(id=re.compile(r'^playvthumb_\d+$'))
|
||||
if overlay:
|
||||
source_id = overlay.get('id', '').rsplit('_', 1)[-1]
|
||||
img = link.find('img', class_=re.compile(r'img-responsive'))
|
||||
if img:
|
||||
thumb_url = img.get('src', '') or img.get('data-original', '')
|
||||
if thumb_url:
|
||||
thumb_url = urljoin(BASE_URL, thumb_url)
|
||||
if not source_id and thumb_url:
|
||||
source_id = self._extract_thumb_source_id(thumb_url)
|
||||
|
||||
card_key = source_id or detail_url
|
||||
if card_key in seen_cards:
|
||||
continue
|
||||
seen_cards.add(card_key)
|
||||
|
||||
videos.append({
|
||||
"title": title,
|
||||
"detail_url": detail_url,
|
||||
"thumb_url": thumb_url,
|
||||
"viewkey": viewkey,
|
||||
"source_id": source_id
|
||||
})
|
||||
|
||||
return videos
|
||||
|
||||
def _extract_title(self, link) -> str:
|
||||
"""
|
||||
从视频链接标签中提取并清理标题
|
||||
"""
|
||||
# 优先从 span.video-title 获取 (已渲染的干净标题)
|
||||
title_el = link.find('span', class_=re.compile(r'video-title'))
|
||||
if title_el:
|
||||
title = title_el.get_text(strip=True)
|
||||
if title:
|
||||
return html.unescape(title)
|
||||
|
||||
# 备用: 从 link 的 title 属性提取
|
||||
title = link.get('title', '').strip()
|
||||
if title:
|
||||
return html.unescape(title)
|
||||
|
||||
# 最后手段: 从链接文本提取并清理前缀
|
||||
text = link.get_text(separator=' ', strip=True)
|
||||
# 去掉前缀: "HD" / "91" / 时间戳 "HH:MM:SS"
|
||||
text = re.sub(r'^(HD\s+|91\s+)?\d{2}:\d{2}:\d{2}\s*', '', text)
|
||||
text = re.sub(r'\s+', ' ', text).strip()
|
||||
return html.unescape(text)[:120]
|
||||
|
||||
def parse_detail_page(self, html: str) -> dict:
|
||||
"""
|
||||
解析详情页,提取视频直链
|
||||
返回: {"video_url": "...", "source_id": "...", "title": "..."} 或空字典
|
||||
"""
|
||||
result = {}
|
||||
|
||||
if not html:
|
||||
return result
|
||||
|
||||
title = self._extract_detail_title(html)
|
||||
if title:
|
||||
result["title"] = title
|
||||
|
||||
# 方法1: 解码 strencode2 (主要方式, 页面通过 document.write 动态写入 video 标签)
|
||||
# 格式: document.write(strencode2("%3c%73%6f..."));
|
||||
strencode_match = re.search(r'strencode2\(["\']([^"\']+)["\']\)', html)
|
||||
if strencode_match:
|
||||
encoded = strencode_match.group(1)
|
||||
try:
|
||||
# strencode2 在JS中等价于 unescape / decodeURIComponent
|
||||
decoded = unquote(encoded)
|
||||
|
||||
# 从解码后的 HTML 片段中提取 src
|
||||
src_match = re.search(r"src=['\"]([^'\"]+)['\"]", decoded)
|
||||
if src_match:
|
||||
video_url = src_match.group(1)
|
||||
# 规范化双斜杠 (如 https://host//path -> https://host/path)
|
||||
video_url = re.sub(r'(https?://[^/]+)//+', r'\1/', video_url)
|
||||
result["video_url"] = video_url
|
||||
result["source_id"] = self._extract_source_id(video_url)
|
||||
return result
|
||||
except Exception as e:
|
||||
self.log(f" 解码 strencode2 失败: {e}")
|
||||
|
||||
# 方法2: 通用正则匹配页面中的 mp4 链接 (备用, 过滤广告)
|
||||
mp4_match = re.search(
|
||||
r'https?://[^\s"\'<>]+\.mp4[^\s"\'<>]*',
|
||||
html
|
||||
)
|
||||
if mp4_match:
|
||||
url = mp4_match.group(0)
|
||||
if 'kwai' not in url and 'ad-' not in url.lower():
|
||||
result["video_url"] = url
|
||||
result["source_id"] = self._extract_source_id(url)
|
||||
return result
|
||||
|
||||
return result
|
||||
|
||||
def _extract_detail_title(self, html_text: str) -> str:
|
||||
soup = BeautifulSoup(html_text, 'lxml')
|
||||
title_el = soup.find('title')
|
||||
if not title_el:
|
||||
return ""
|
||||
title = title_el.get_text(" ", strip=True)
|
||||
title = re.sub(r'\s*-\s*91porn.*$', '', title, flags=re.IGNORECASE).strip()
|
||||
return html.unescape(title)[:160]
|
||||
|
||||
def _extract_source_id(self, video_url: str) -> str:
|
||||
path = urlparse(video_url or "").path
|
||||
name = os.path.basename(path)
|
||||
stem, ext = os.path.splitext(name)
|
||||
if ext.lower() not in {".mp4", ".m4v", ".mov", ".webm", ".mkv", ".avi"}:
|
||||
return ""
|
||||
source_id = re.sub(r'[^0-9]+', '', stem)
|
||||
if not source_id or source_id != stem:
|
||||
return ""
|
||||
return source_id
|
||||
|
||||
def _extract_thumb_source_id(self, thumb_url: str) -> str:
|
||||
path = urlparse(thumb_url or "").path
|
||||
match = re.search(r'/thumb/(\d+)\.[A-Za-z0-9]+$', path)
|
||||
return match.group(1) if match else ""
|
||||
|
||||
def _thumb_url_for_source(self, thumb_url: str, source_id: str) -> str:
|
||||
if not thumb_url or not source_id:
|
||||
return thumb_url
|
||||
parsed = urlparse(thumb_url)
|
||||
match = re.search(r'/thumb/([^/?#]+)\.[A-Za-z0-9]+$', parsed.path)
|
||||
if not match:
|
||||
return thumb_url
|
||||
current = match.group(1)
|
||||
if current == source_id:
|
||||
return thumb_url
|
||||
path = re.sub(
|
||||
r'/thumb/[^/?#]+\.[A-Za-z0-9]+$',
|
||||
f'/thumb/{source_id}.jpg',
|
||||
parsed.path,
|
||||
)
|
||||
return parsed._replace(path=path, query="", fragment="").geturl()
|
||||
|
||||
def crawl(self):
|
||||
"""
|
||||
主爬取流程。停止条件(任一满足即停):
|
||||
- 达到 max_pages 配置
|
||||
- 连续 max_empty_pages 页都没有视频
|
||||
- target_new 模式下,已经累计处理 target_new 个新视频
|
||||
"""
|
||||
self.log("=" * 60)
|
||||
self.log("91porn 视频爬虫启动")
|
||||
self.log("=" * 60)
|
||||
self.log(f"配置: 列表页延时 {MIN_PAGE_DELAY}-{MAX_PAGE_DELAY}s, 详情页延时 {MIN_DETAIL_DELAY}-{MAX_DETAIL_DELAY}s")
|
||||
self.log(f"配置: 最大重试 {MAX_RETRIES} 次, 连续空页上限 {self.max_empty_pages}")
|
||||
self.log(f"配置: 起始页 {self.start_page}, 最大爬取页数 {self.max_pages if self.max_pages else '不限'}")
|
||||
if self.target_new:
|
||||
self.log(f"配置: 目标新增视频数 {self.target_new}")
|
||||
self.log(f"配置: 输出文件 {os.path.abspath(self.output_file)}")
|
||||
if self.skip_viewkeys:
|
||||
self.log(f"配置: 已跳过 {len(self.skip_viewkeys)} 个已知 viewkey")
|
||||
self.log("")
|
||||
|
||||
page_num = self.start_page
|
||||
consecutive_empty = 0
|
||||
crawled_in_session = 0
|
||||
|
||||
while True:
|
||||
if self.max_pages is not None and crawled_in_session >= self.max_pages:
|
||||
self.log(f"达到配置的页数上限 {self.max_pages},停止")
|
||||
break
|
||||
if consecutive_empty >= self.max_empty_pages:
|
||||
self.log(f"连续 {self.max_empty_pages} 页无结果,已达到末尾")
|
||||
break
|
||||
if self.target_new is not None and self.processed_videos >= self.target_new:
|
||||
self.log(f"已累计 {self.processed_videos} 个新视频,达到目标 {self.target_new},停止")
|
||||
break
|
||||
|
||||
if page_num == 1:
|
||||
page_url = f"{BASE_URL}?category=top&viewtype=basic"
|
||||
else:
|
||||
page_url = f"{BASE_URL}?category=top&viewtype=basic&page={page_num}"
|
||||
|
||||
if crawled_in_session > 0:
|
||||
self.log("")
|
||||
self.random_sleep(MIN_PAGE_DELAY, MAX_PAGE_DELAY)
|
||||
|
||||
self.log(f"[页 {page_num}] 请求: {page_url}")
|
||||
page_html = self.fetch_page(page_url, f"列表页 第{page_num}页")
|
||||
|
||||
if not page_html:
|
||||
self.log(f"[页 {page_num}] 获取失败,跳过")
|
||||
consecutive_empty += 1
|
||||
page_num += 1
|
||||
crawled_in_session += 1
|
||||
continue
|
||||
|
||||
page_videos = self.parse_list_page(page_html)
|
||||
|
||||
# 判断页面是否真的没有视频(而非全部已处理)
|
||||
if not page_videos:
|
||||
self.log(f"[页 {page_num}] 页面无视频,可能已到末尾")
|
||||
consecutive_empty += 1
|
||||
page_num += 1
|
||||
crawled_in_session += 1
|
||||
continue
|
||||
|
||||
consecutive_empty = 0
|
||||
|
||||
# 过滤已处理的 viewkey,只保留新视频
|
||||
new_videos = [v for v in page_videos if v['viewkey'] not in self.skip_viewkeys]
|
||||
skipped_on_page = len(page_videos) - len(new_videos)
|
||||
|
||||
if skipped_on_page > 0:
|
||||
self.log(f"[页 {page_num}] 发现 {len(page_videos)} 个链接, 其中 {skipped_on_page} 个已处理, {len(new_videos)} 个新视频")
|
||||
else:
|
||||
self.log(f"[页 {page_num}] 发现 {len(new_videos)} 个视频")
|
||||
|
||||
if new_videos:
|
||||
self._process_video_list(new_videos, referer=page_url)
|
||||
self.pages_crawled += 1
|
||||
page_num += 1
|
||||
crawled_in_session += 1
|
||||
|
||||
self._save_results()
|
||||
self._print_summary()
|
||||
|
||||
def _process_video_list(self, videos: list, referer: str = ""):
|
||||
"""
|
||||
处理一批视频列表,逐个获取详情页
|
||||
"""
|
||||
for idx, video in enumerate(videos, 1):
|
||||
# target_new 模式下,凑够后立即停止,不再请求详情页
|
||||
if self.target_new is not None and self.processed_videos >= self.target_new:
|
||||
return
|
||||
# 跳过已处理的 viewkey (断点续爬)
|
||||
if video['viewkey'] in self.skip_viewkeys:
|
||||
self.log(f" [SKIP] 已处理过: {video['viewkey']}")
|
||||
self.skipped_videos += 1
|
||||
continue
|
||||
|
||||
self.log(f" 处理视频 {idx}/{len(videos)}: {video['title'][:40]}...")
|
||||
|
||||
# 延时控制 (同一批次内第一个视频不延时)
|
||||
if idx > 1:
|
||||
self.random_sleep(MIN_DETAIL_DELAY, MAX_DETAIL_DELAY)
|
||||
|
||||
# 获取详情页
|
||||
detail_html = self.fetch_page(video['detail_url'], f"详情页 viewkey={video['viewkey']}", referer=referer)
|
||||
|
||||
if not detail_html:
|
||||
self.log(f" [FAIL] 详情页获取失败: {video['viewkey']}")
|
||||
video["video_url"] = ""
|
||||
self.results.append(video)
|
||||
self.skip_viewkeys.add(video['viewkey'])
|
||||
self.failed_videos += 1
|
||||
continue
|
||||
|
||||
# 解析视频直链
|
||||
detail_info = self.parse_detail_page(detail_html)
|
||||
|
||||
if detail_info.get("video_url"):
|
||||
video["video_url"] = detail_info["video_url"]
|
||||
if detail_info.get("title"):
|
||||
video["title"] = detail_info["title"]
|
||||
list_source_id = video.get("source_id", "")
|
||||
detail_source_id = detail_info.get("source_id", "")
|
||||
if list_source_id and detail_source_id and list_source_id != detail_source_id:
|
||||
self.log(
|
||||
f" [FAIL] 详情页视频源不匹配: list_source_id={list_source_id} "
|
||||
f"detail_source_id={detail_source_id} viewkey={video['viewkey']}"
|
||||
)
|
||||
self.failed_videos += 1
|
||||
self.skip_viewkeys.add(video['viewkey'])
|
||||
continue
|
||||
if not list_source_id and detail_source_id:
|
||||
video["source_id"] = detail_source_id
|
||||
if video.get("source_id"):
|
||||
video["thumb_url"] = self._thumb_url_for_source(
|
||||
video.get("thumb_url", ""),
|
||||
video["source_id"],
|
||||
)
|
||||
if video["source_id"] in self.skip_viewkeys:
|
||||
self.log(f" [SKIP] 已处理过 source_id: {video['source_id']}")
|
||||
self.skipped_videos += 1
|
||||
continue
|
||||
self.results.append(video)
|
||||
self.skip_viewkeys.add(video['viewkey'])
|
||||
if video.get("source_id"):
|
||||
self.skip_viewkeys.add(video["source_id"])
|
||||
self.processed_videos += 1
|
||||
self.log(f" [OK] 成功提取视频直链")
|
||||
# 流式:立刻把这条 entry 交给 Go 端开始下载,不等本批余下视频
|
||||
self.emit_stream_video(video)
|
||||
else:
|
||||
self.log(f" [FAIL] 未找到视频直链: {video['viewkey']}")
|
||||
video["video_url"] = ""
|
||||
self.results.append(video)
|
||||
self.skip_viewkeys.add(video['viewkey'])
|
||||
self.failed_videos += 1
|
||||
|
||||
def _save_results(self):
|
||||
"""
|
||||
保存结果到JSON文件
|
||||
"""
|
||||
output_data = {
|
||||
"crawl_time": datetime.now().isoformat(),
|
||||
"source_url": BASE_URL,
|
||||
"pages_crawled": self.pages_crawled,
|
||||
"total_videos": len(self.results),
|
||||
"successful": self.processed_videos,
|
||||
"skipped": self.skipped_videos,
|
||||
"failed": self.failed_videos,
|
||||
"videos": self.results
|
||||
}
|
||||
|
||||
try:
|
||||
# 保证父目录存在;写入临时文件后原子 rename,避免读到半截 JSON
|
||||
out_path = self.output_file
|
||||
parent = os.path.dirname(os.path.abspath(out_path))
|
||||
if parent:
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
tmp_path = out_path + ".part"
|
||||
with open(tmp_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(output_data, f, ensure_ascii=False, indent=2)
|
||||
os.replace(tmp_path, out_path)
|
||||
self.log(f"结果已保存到: {os.path.abspath(out_path)}")
|
||||
except Exception as e:
|
||||
self.log(f"保存文件失败: {e}")
|
||||
# 尝试输出到控制台作为备份
|
||||
backup_out = sys.stderr if self.stream_output else sys.stdout
|
||||
print("\n--- 备份输出 ---", file=backup_out, flush=True)
|
||||
print(json.dumps(output_data, ensure_ascii=False, indent=2), file=backup_out, flush=True)
|
||||
|
||||
def _print_summary(self):
|
||||
"""
|
||||
打印爬取摘要
|
||||
"""
|
||||
self.log("")
|
||||
self.log("=" * 60)
|
||||
self.log("爬取完成!")
|
||||
self.log("=" * 60)
|
||||
self.log(f"爬取页数: {self.pages_crawled}")
|
||||
self.log(f"总视频数: {len(self.results)}")
|
||||
self.log(f"成功提取直链: {self.processed_videos}")
|
||||
self.log(f"跳过(已处理): {self.skipped_videos}")
|
||||
self.log(f"失败/缺失直链: {self.failed_videos}")
|
||||
self.log(f"输出文件: {os.path.abspath(self.output_file)}")
|
||||
self.log("=" * 60)
|
||||
|
||||
|
||||
def print_help():
|
||||
print("""
|
||||
================================================
|
||||
91porn 视频爬虫 v1.0
|
||||
================================================
|
||||
|
||||
本脚本将爬取 91porn "本月最热" 分类下的所有视频信息:
|
||||
- 视频名称
|
||||
- 封面图直链
|
||||
- 视频直链 (MP4)
|
||||
|
||||
依赖安装:
|
||||
pip install requests beautifulsoup4 lxml PySocks
|
||||
|
||||
使用方法:
|
||||
python spider_91porn.py
|
||||
|
||||
配置说明 (编辑脚本内 "配置区域"):
|
||||
MIN_PAGE_DELAY / MAX_PAGE_DELAY : 列表页请求间隔 (默认 3-6 秒)
|
||||
MIN_DETAIL_DELAY / MAX_DETAIL_DELAY : 详情页请求间隔 (默认 2-5 秒)
|
||||
MAX_PAGES : 限制最大爬取页数 (None=不限, 如 5=只爬前5页)
|
||||
OUTPUT_FILE : 输出文件名 (默认 91porn_videos.json)
|
||||
|
||||
按 Ctrl+C 可随时中断并保存已爬取的数据
|
||||
|
||||
注意:
|
||||
1. 视频直链包含时效性token,会过期,需定期重新爬取
|
||||
2. 脚本已内置随机延时,请勿移除,避免对服务器造成压力
|
||||
3. 如遇到Cloudflare拦截,需要先通过浏览器获取Cookie
|
||||
4. 本脚本仅供学习交流,请遵守当地法律法规
|
||||
================================================
|
||||
""")
|
||||
|
||||
|
||||
def run_job(job_path: str):
|
||||
"""Run as a crawler.v1 script plugin.
|
||||
|
||||
The Go host passes a job JSON file and expects stdout JSONL events. Logs go
|
||||
to stderr so stdout stays machine-readable.
|
||||
"""
|
||||
with open(job_path, "r", encoding="utf-8") as f:
|
||||
job = json.load(f)
|
||||
|
||||
if job.get("protocol") != CRAWLER_PROTOCOL:
|
||||
raise ValueError(f"unsupported crawler protocol: {job.get('protocol')!r}")
|
||||
if job.get("mode") not in ("", None, "crawl"):
|
||||
raise ValueError(f"unsupported crawler mode: {job.get('mode')!r}")
|
||||
|
||||
try:
|
||||
target_new = int(job.get("target_new") or 15)
|
||||
except (TypeError, ValueError):
|
||||
target_new = 15
|
||||
if target_new <= 0:
|
||||
target_new = 15
|
||||
seen_file = job.get("seen_source_ids_file") or ""
|
||||
output_dir = job.get("output_dir") or os.getcwd()
|
||||
run_id = job.get("run_id") or datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
output_file = os.path.join(output_dir, f"spider91-{run_id}.json")
|
||||
|
||||
network = job.get("network") if isinstance(job.get("network"), dict) else {}
|
||||
proxy_url = str(network.get("proxy_url") or "").strip()
|
||||
if proxy_url:
|
||||
os.environ["HTTP_PROXY"] = proxy_url
|
||||
os.environ["HTTPS_PROXY"] = proxy_url
|
||||
os.environ["http_proxy"] = proxy_url
|
||||
os.environ["https_proxy"] = proxy_url
|
||||
os.environ["NO_PROXY"] = ""
|
||||
os.environ["no_proxy"] = ""
|
||||
|
||||
seen_viewkeys = []
|
||||
if seen_file:
|
||||
try:
|
||||
with open(seen_file, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
seen_viewkeys.append(line)
|
||||
except FileNotFoundError:
|
||||
print(f"警告: seen_source_ids_file 不存在: {seen_file}", file=sys.stderr, flush=True)
|
||||
except Exception as e:
|
||||
print(f"警告: 读取 seen_source_ids_file 失败: {e}", file=sys.stderr, flush=True)
|
||||
|
||||
prefer_ipv4_for_plain_socks5_proxy()
|
||||
spider = Porn91Spider(
|
||||
output_file=output_file,
|
||||
start_page=1,
|
||||
max_pages=None,
|
||||
resume=False,
|
||||
quiet=True,
|
||||
target_new=target_new,
|
||||
seen_viewkeys=seen_viewkeys,
|
||||
stream_output=True,
|
||||
stream_protocol="crawler.v1",
|
||||
)
|
||||
try:
|
||||
spider.crawl()
|
||||
done = {
|
||||
"type": "done",
|
||||
"stats": {
|
||||
"emitted": spider.processed_videos,
|
||||
"failed": spider.failed_videos,
|
||||
"skipped": spider.skipped_videos,
|
||||
},
|
||||
}
|
||||
write_jsonl(done)
|
||||
except KeyboardInterrupt:
|
||||
spider.log("\n用户中断,正在保存已爬取的数据...")
|
||||
spider._save_results()
|
||||
raise
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) > 1 and sys.argv[1] in ('-h', '--help', 'help'):
|
||||
print_help()
|
||||
return
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="spider_91porn.py",
|
||||
description="91porn 视频元数据爬虫",
|
||||
add_help=False, # 让 -h/--help 走 print_help() 中文版本
|
||||
)
|
||||
parser.add_argument("--page", type=int, default=None,
|
||||
help="只爬指定页(单页模式,配合 --output 用于定时任务)")
|
||||
parser.add_argument("--output", type=str, default=None,
|
||||
help="输出 JSON 路径,覆盖默认 OUTPUT_FILE")
|
||||
parser.add_argument("--max-pages", type=int, default=None,
|
||||
help="单页模式下,从 --page 起最多再爬几页(默认 1)")
|
||||
parser.add_argument("--no-resume", action="store_true",
|
||||
help="禁用断点续爬(单页模式默认禁用)")
|
||||
parser.add_argument("--quiet", action="store_true",
|
||||
help="压缩日志,每条视频只输出关键事件")
|
||||
parser.add_argument("--target-new", type=int, default=None,
|
||||
help="目标新增模式:从 page 1 起翻页直到累计处理这么多新源视频后停止(backend 凌晨任务用)")
|
||||
parser.add_argument("--seen-viewkeys-file", type=str, default=None,
|
||||
help="文件路径,每行一个已处理过的 viewkey 或 mp4 源 ID;脚本会跳过这些视频")
|
||||
parser.add_argument("--stream-output", action="store_true",
|
||||
help="流式模式:每解析一条视频直链就立即把它作为一行 JSON 写到 stdout 并 flush;"
|
||||
"日志改走 stderr。配合 backend 边读边下载使用。")
|
||||
parser.add_argument("--job", type=str, default=None,
|
||||
help="crawler.v1 job JSON 路径;作为通用脚本爬虫运行。")
|
||||
|
||||
args, _ = parser.parse_known_args()
|
||||
if args.job:
|
||||
run_job(args.job)
|
||||
return
|
||||
|
||||
cli_out = sys.stderr if args.stream_output else sys.stdout
|
||||
prefer_ipv4_for_plain_socks5_proxy()
|
||||
|
||||
print("""
|
||||
================================================
|
||||
91porn 视频爬虫启动中...
|
||||
================================================
|
||||
按 Ctrl+C 可随时中断并保存进度
|
||||
""", file=cli_out)
|
||||
|
||||
# 加载已知 ID(来自 backend 的 catalog 已入库列表;兼容旧参数名)
|
||||
seen_viewkeys = []
|
||||
if args.seen_viewkeys_file:
|
||||
try:
|
||||
with open(args.seen_viewkeys_file, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
seen_viewkeys.append(line)
|
||||
except FileNotFoundError:
|
||||
print(f"警告: --seen-viewkeys-file 不存在: {args.seen_viewkeys_file}", file=cli_out)
|
||||
except Exception as e:
|
||||
print(f"警告: 读取 --seen-viewkeys-file 失败: {e}", file=cli_out)
|
||||
|
||||
# 决定运行模式
|
||||
if args.target_new is not None:
|
||||
# 凑够 N 个新视频模式:从 page 1 起翻页,直到累计 target_new 个新视频
|
||||
spider = Porn91Spider(
|
||||
output_file=args.output,
|
||||
start_page=1,
|
||||
max_pages=None,
|
||||
resume=False, # 凑够 N 模式靠 seen_viewkeys 去重,不读 OUTPUT_FILE
|
||||
quiet=args.quiet,
|
||||
target_new=args.target_new,
|
||||
seen_viewkeys=seen_viewkeys,
|
||||
stream_output=args.stream_output,
|
||||
)
|
||||
elif args.page is not None:
|
||||
# 单页模式(保留作手动调试用):start_page=N, max_pages=1
|
||||
start_page = max(1, args.page)
|
||||
max_pages = args.max_pages if args.max_pages and args.max_pages > 0 else 1
|
||||
spider = Porn91Spider(
|
||||
output_file=args.output,
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
resume=False,
|
||||
quiet=args.quiet,
|
||||
seen_viewkeys=seen_viewkeys,
|
||||
stream_output=args.stream_output,
|
||||
)
|
||||
else:
|
||||
# 全量模式(向后兼容):从 page 1 起爬到末尾
|
||||
spider = Porn91Spider(
|
||||
output_file=args.output,
|
||||
resume=False if args.no_resume else None,
|
||||
quiet=args.quiet,
|
||||
seen_viewkeys=seen_viewkeys,
|
||||
stream_output=args.stream_output,
|
||||
)
|
||||
|
||||
try:
|
||||
spider.crawl()
|
||||
except KeyboardInterrupt:
|
||||
spider.log("\n用户中断,正在保存已爬取的数据...")
|
||||
spider._save_results()
|
||||
spider._print_summary()
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
spider.log(f"发生未预料的错误: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
spider._save_results()
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+781
-463
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,32 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
)
|
||||
|
||||
func TestCrawlerIntCredFallbacks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
d *catalog.Drive
|
||||
key string
|
||||
def int
|
||||
want int
|
||||
}{
|
||||
{"nil drive", nil, "page", 1, 1},
|
||||
{"nil creds", &catalog.Drive{}, "page", 7, 7},
|
||||
{"empty value", &catalog.Drive{Credentials: map[string]string{"page": ""}}, "page", 5, 5},
|
||||
{"non-numeric", &catalog.Drive{Credentials: map[string]string{"page": "abc"}}, "page", 9, 9},
|
||||
{"happy", &catalog.Drive{Credentials: map[string]string{"page": "42"}}, "page", 1, 42},
|
||||
{"missing key", &catalog.Drive{Credentials: map[string]string{"a": "1"}}, "b", 99, 99},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := crawlerIntCred(tc.d, tc.key, tc.def)
|
||||
if got != tc.want {
|
||||
t.Fatalf("crawlerIntCred(%s) = %d, want %d", tc.name, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
)
|
||||
|
||||
func TestSpider91IntCredFallbacks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
d *catalog.Drive
|
||||
key string
|
||||
def int
|
||||
want int
|
||||
}{
|
||||
{"nil drive", nil, "page", 1, 1},
|
||||
{"nil creds", &catalog.Drive{}, "page", 7, 7},
|
||||
{"empty value", &catalog.Drive{Credentials: map[string]string{"page": ""}}, "page", 5, 5},
|
||||
{"non-numeric", &catalog.Drive{Credentials: map[string]string{"page": "abc"}}, "page", 9, 9},
|
||||
{"happy", &catalog.Drive{Credentials: map[string]string{"page": "42"}}, "page", 1, 42},
|
||||
{"missing key", &catalog.Drive{Credentials: map[string]string{"a": "1"}}, "b", 99, 99},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := spider91IntCred(tc.d, tc.key, tc.def)
|
||||
if got != tc.want {
|
||||
t.Fatalf("spider91IntCred(%s) = %d, want %d", tc.name, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
|
||||
reg := proxy.NewRegistry()
|
||||
reg.Set("p115-one", &spider91UploadTargetFakeDrive{id: "p115-one", kind: "p115"})
|
||||
reg.Set("p123-one", &spider91UploadTargetFakeDrive{id: "p123-one", kind: "p123"})
|
||||
reg.Set("onedrive-one", &spider91UploadTargetFakeDrive{id: "onedrive-one", kind: "onedrive"})
|
||||
reg.Set("wopan-one", &spider91UploadTargetFakeDrive{id: "wopan-one", kind: "wopan"})
|
||||
reg.Set("guangyapan-one", &spider91UploadTargetFakeDrive{id: "guangyapan-one", kind: "guangyapan"})
|
||||
|
||||
app := &App{registry: reg}
|
||||
if got := app.Spider91UploadDriveID(); got != "" {
|
||||
t.Fatalf("empty upload target selected %q, want local-only empty target", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "p115-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "p115-one" {
|
||||
t.Fatalf("explicit upload target = %q, want p115-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "p123-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "p123-one" {
|
||||
t.Fatalf("explicit p123 upload target = %q, want p123-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "onedrive-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "onedrive-one" {
|
||||
t.Fatalf("explicit onedrive upload target = %q, want onedrive-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "wopan-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "wopan-one" {
|
||||
t.Fatalf("explicit wopan upload target = %q, want wopan-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "guangyapan-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "guangyapan-one" {
|
||||
t.Fatalf("explicit guangyapan upload target = %q, want guangyapan-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "missing"
|
||||
if got := app.Spider91UploadDriveID(); got != "" {
|
||||
t.Fatalf("missing upload target = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
type spider91UploadTargetFakeDrive struct {
|
||||
id string
|
||||
kind string
|
||||
}
|
||||
|
||||
func (d *spider91UploadTargetFakeDrive) Kind() string { return d.kind }
|
||||
func (d *spider91UploadTargetFakeDrive) ID() string { return d.id }
|
||||
func (d *spider91UploadTargetFakeDrive) Init(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *spider91UploadTargetFakeDrive) RootID() string { return "root" }
|
||||
+373
-88
@@ -3,6 +3,9 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -15,7 +18,6 @@ import (
|
||||
"github.com/video-site/backend/internal/config"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/scriptcrawler"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/fingerprint"
|
||||
"github.com/video-site/backend/internal/preview"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
@@ -469,31 +471,54 @@ func TestGuangYaPanGenerationCooldowns(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSpider91MigrationAfterManualCrawlRequiresConfiguredUploadTarget(t *testing.T) {
|
||||
func TestRunCrawlerMigrationAfterManualCrawlRequiresCrawlerUploadTarget(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "crawler-main",
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Crawler",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"script_path": "/tmp/crawler.py",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed crawler: %v", err)
|
||||
}
|
||||
|
||||
registry := proxy.NewRegistry()
|
||||
migrator := &serverFakeSpider91MigrationRunner{}
|
||||
migrator := &serverFakeCrawlerUploadRunner{}
|
||||
app := &App{
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
spider91Migrator: migrator,
|
||||
crawlerUploader: migrator,
|
||||
workers: map[string]*preview.Worker{},
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{},
|
||||
fingerprintWorkers: map[string]*fingerprint.Worker{},
|
||||
}
|
||||
|
||||
app.runSpider91MigrationAfterManualCrawl(ctx, "91spider")
|
||||
app.runCrawlerMigrationAfterManualCrawl(ctx, "crawler-main")
|
||||
if migrator.called != 0 {
|
||||
t.Fatalf("migration called without upload target")
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "pikpak"
|
||||
app.runSpider91MigrationAfterManualCrawl(ctx, "91spider")
|
||||
if migrator.called != 0 {
|
||||
t.Fatalf("migration called when upload target is not attached")
|
||||
d, err := cat.GetDrive(ctx, "crawler-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get crawler: %v", err)
|
||||
}
|
||||
|
||||
registry.Set("pikpak", &serverFakeKindDrive{id: "pikpak", kind: "pikpak"})
|
||||
app.runSpider91MigrationAfterManualCrawl(ctx, "91spider")
|
||||
d.Credentials["upload_drive_id"] = "pikpak"
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
t.Fatalf("set upload target: %v", err)
|
||||
}
|
||||
app.runCrawlerMigrationAfterManualCrawl(ctx, "crawler-main")
|
||||
if migrator.called != 1 {
|
||||
t.Fatalf("migration calls = %d, want 1", migrator.called)
|
||||
}
|
||||
@@ -524,11 +549,11 @@ func TestScheduleCrawlerUploadMigrationRunsForConfiguredCrawler(t *testing.T) {
|
||||
}
|
||||
registry := proxy.NewRegistry()
|
||||
registry.Set("crawler-truvaze", &serverFakeKindDrive{id: "crawler-truvaze", kind: scriptcrawler.Kind})
|
||||
migrator := &serverFakeSpider91MigrationRunner{}
|
||||
migrator := &serverFakeCrawlerUploadRunner{}
|
||||
app := &App{
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
spider91Migrator: migrator,
|
||||
crawlerUploader: migrator,
|
||||
workers: map[string]*preview.Worker{},
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{},
|
||||
fingerprintWorkers: map[string]*fingerprint.Worker{},
|
||||
@@ -567,8 +592,8 @@ func TestScheduleCrawlerUploadMigrationSkipsWithoutUploadTarget(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatalf("seed crawler: %v", err)
|
||||
}
|
||||
migrator := &serverFakeSpider91MigrationRunner{}
|
||||
app := &App{cat: cat, registry: proxy.NewRegistry(), spider91Migrator: migrator}
|
||||
migrator := &serverFakeCrawlerUploadRunner{}
|
||||
app := &App{cat: cat, registry: proxy.NewRegistry(), crawlerUploader: migrator}
|
||||
|
||||
if app.scheduleCrawlerUploadMigration(ctx, "crawler-local") {
|
||||
t.Fatal("scheduleCrawlerUploadMigration returned true without upload target")
|
||||
@@ -578,6 +603,128 @@ func TestScheduleCrawlerUploadMigrationSkipsWithoutUploadTarget(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleManualCrawlerUploadMigrationRunsWhenAssetsReady(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "crawler-ready",
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Ready Crawler",
|
||||
RootID: "/",
|
||||
TeaserEnabled: true,
|
||||
Credentials: map[string]string{
|
||||
"script_path": "/tmp/ready.py",
|
||||
"upload_drive_id": "pikpak-target",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed crawler: %v", err)
|
||||
}
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: scriptcrawler.BuildVideoID("crawler-ready", "source-1"),
|
||||
DriveID: "crawler-ready",
|
||||
FileID: "source-1.mp4",
|
||||
FileName: "source-1.mp4",
|
||||
Title: "Source 1",
|
||||
Size: 123,
|
||||
Ext: "mp4",
|
||||
SampledSHA256: "sampled-source-1",
|
||||
FingerprintStatus: "ready",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: time.Now(),
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
registry := proxy.NewRegistry()
|
||||
registry.Set("crawler-ready", &serverFakeKindDrive{id: "crawler-ready", kind: scriptcrawler.Kind})
|
||||
registry.Set("pikpak-target", &serverFakeKindDrive{id: "pikpak-target", kind: "pikpak"})
|
||||
migrator := &serverFakeCrawlerUploadRunner{}
|
||||
app := &App{
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
crawlerUploader: migrator,
|
||||
workers: map[string]*preview.Worker{},
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{},
|
||||
fingerprintWorkers: map[string]*fingerprint.Worker{},
|
||||
}
|
||||
|
||||
accepted, message := app.scheduleManualCrawlerUploadMigration(ctx, "crawler-ready")
|
||||
if !accepted {
|
||||
t.Fatalf("accepted = false, message = %q", message)
|
||||
}
|
||||
deadline := time.After(time.Second)
|
||||
for migrator.called == 0 {
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatalf("migration calls = %d, want 1", migrator.called)
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleManualCrawlerUploadMigrationRejectsPendingFingerprint(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "crawler-pending",
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Pending Crawler",
|
||||
RootID: "/",
|
||||
TeaserEnabled: true,
|
||||
Credentials: map[string]string{
|
||||
"script_path": "/tmp/pending.py",
|
||||
"upload_drive_id": "pikpak-target",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed crawler: %v", err)
|
||||
}
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: scriptcrawler.BuildVideoID("crawler-pending", "source-1"),
|
||||
DriveID: "crawler-pending",
|
||||
FileID: "source-1.mp4",
|
||||
FileName: "source-1.mp4",
|
||||
Title: "Source 1",
|
||||
Size: 123,
|
||||
Ext: "mp4",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: time.Now(),
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
migrator := &serverFakeCrawlerUploadRunner{}
|
||||
app := &App{cat: cat, registry: proxy.NewRegistry(), crawlerUploader: migrator}
|
||||
|
||||
accepted, message := app.scheduleManualCrawlerUploadMigration(ctx, "crawler-pending")
|
||||
if accepted {
|
||||
t.Fatal("accepted = true, want false")
|
||||
}
|
||||
if !strings.Contains(message, "指纹") {
|
||||
t.Fatalf("message = %q, want fingerprint reason", message)
|
||||
}
|
||||
if migrator.called != 0 {
|
||||
t.Fatalf("migration calls = %d, want 0", migrator.called)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriveGenerationStatusUsesWorkerQueueNotPendingCatalogRows(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -765,9 +912,8 @@ func TestNightlyTargetsComeFromCatalogBeforeDriveAttach(t *testing.T) {
|
||||
for _, d := range []*catalog.Drive{
|
||||
{ID: "115", Kind: "p115", Name: "115", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "pikpak", Kind: "pikpak", Name: "PikPak", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "91-legacy", Kind: "spider91", Name: "91 Legacy", RootID: "0", TeaserEnabled: true},
|
||||
{ID: "91-crawler", Kind: scriptcrawler.Kind, Name: "91 Spider", RootID: "/", Credentials: map[string]string{"script_path": "/tmp/crawler.py"}, TeaserEnabled: true},
|
||||
{ID: "91-crawler-deleted", Kind: scriptcrawler.Kind, Name: "Deleted Spider", RootID: "/", Credentials: map[string]string{}, TeaserEnabled: true},
|
||||
{ID: "crawler-main", Kind: scriptcrawler.Kind, Name: "Crawler", RootID: "/", Credentials: map[string]string{"script_path": "/tmp/crawler.py"}, TeaserEnabled: true},
|
||||
{ID: "crawler-deleted", Kind: scriptcrawler.Kind, Name: "Deleted Crawler", RootID: "/", Credentials: map[string]string{}, TeaserEnabled: true},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
t.Fatalf("seed drive %s: %v", d.ID, err)
|
||||
@@ -779,13 +925,13 @@ func TestNightlyTargetsComeFromCatalogBeforeDriveAttach(t *testing.T) {
|
||||
if len(scanIDs) != 2 || scanIDs[0] != "115" || scanIDs[1] != "pikpak" {
|
||||
t.Fatalf("scan target ids = %#v, want 115 and pikpak from catalog", scanIDs)
|
||||
}
|
||||
spiderIDs := app.listSpider91DriveIDs(ctx)
|
||||
if len(spiderIDs) != 1 || spiderIDs[0] != "91-crawler" {
|
||||
t.Fatalf("spider91 ids = %#v, want crawler-page script drive", spiderIDs)
|
||||
crawlerIDs := app.listCrawlerDriveIDs(ctx)
|
||||
if len(crawlerIDs) != 1 || crawlerIDs[0] != "crawler-main" {
|
||||
t.Fatalf("crawler ids = %#v, want crawler-page script drive", crawlerIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAttachDriveRejectsLegacySpider91Storage(t *testing.T) {
|
||||
func TestAttachDriveRejectsUnknownKind(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -797,9 +943,9 @@ func TestAttachDriveRejectsLegacySpider91Storage(t *testing.T) {
|
||||
}
|
||||
})
|
||||
d := &catalog.Drive{
|
||||
ID: "91-legacy",
|
||||
Kind: spider91.Kind,
|
||||
Name: "91 Legacy",
|
||||
ID: "unknown-main",
|
||||
Kind: "unknown",
|
||||
Name: "Unknown",
|
||||
RootID: "/",
|
||||
TeaserEnabled: true,
|
||||
}
|
||||
@@ -809,18 +955,11 @@ func TestAttachDriveRejectsLegacySpider91Storage(t *testing.T) {
|
||||
|
||||
app := &App{cat: cat, registry: proxy.NewRegistry()}
|
||||
err = app.attachDrive(ctx, d)
|
||||
if err == nil || !strings.Contains(err.Error(), "爬虫管理") {
|
||||
t.Fatalf("attach err = %v, want crawler management guidance", err)
|
||||
if err == nil || !strings.Contains(err.Error(), "unknown drive kind: unknown") {
|
||||
t.Fatalf("attach err = %v, want unknown kind error", err)
|
||||
}
|
||||
if _, ok := app.registry.Get(d.ID); ok {
|
||||
t.Fatal("legacy spider91 drive should not be registered")
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, d.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.Status != "error" || !strings.Contains(got.LastError, "爬虫管理") {
|
||||
t.Fatalf("status/error = %q/%q, want deprecated error", got.Status, got.LastError)
|
||||
t.Fatal("unknown drive should not be registered")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1468,7 +1607,7 @@ func TestDeleteVideoUsesSourceRemoverWithCatalogMetadata(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteVideoRemovesSpider91SourceFile(t *testing.T) {
|
||||
func TestDeleteVideoRemovesScriptCrawlerSourceFile(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
localDir := filepath.Join(root, "previews")
|
||||
@@ -1479,23 +1618,28 @@ func TestDeleteVideoRemovesSpider91SourceFile(t *testing.T) {
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "spider-main",
|
||||
Kind: spider91.Kind,
|
||||
Name: "Spider",
|
||||
ID: "crawler-main",
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Crawler",
|
||||
RootID: "/",
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
registry: proxy.NewRegistry(),
|
||||
}
|
||||
sourceDir := app.spider91DriveDir("spider-main")
|
||||
sourceDir := app.scriptCrawlerDriveDir("crawler-main")
|
||||
app.registry.Set("crawler-main", scriptcrawler.New(scriptcrawler.Config{
|
||||
ID: "crawler-main",
|
||||
RootDir: sourceDir,
|
||||
}))
|
||||
sourceVideo := filepath.Join(sourceDir, "videos", "source.mp4")
|
||||
sourceThumb := filepath.Join(sourceDir, "thumbs", "source.jpg")
|
||||
previewPath := filepath.Join(localDir, "spider91-spider-main-source.mp4")
|
||||
commonThumb := filepath.Join(localDir, "thumbs", "spider91-spider-main-source.jpg")
|
||||
previewPath := filepath.Join(localDir, "scriptcrawler-crawler-main-source.mp4")
|
||||
commonThumb := filepath.Join(localDir, "thumbs", "scriptcrawler-crawler-main-source.jpg")
|
||||
for _, path := range []string{sourceVideo, sourceThumb, previewPath, commonThumb} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
@@ -1507,15 +1651,15 @@ func TestDeleteVideoRemovesSpider91SourceFile(t *testing.T) {
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "spider91-spider-main-source",
|
||||
DriveID: "spider-main",
|
||||
ID: "scriptcrawler-crawler-main-source",
|
||||
DriveID: "crawler-main",
|
||||
FileID: "source.mp4",
|
||||
FileName: "source.mp4",
|
||||
Ext: "mp4",
|
||||
Title: "Spider Source",
|
||||
Title: "Crawler Source",
|
||||
PreviewLocal: previewPath,
|
||||
PreviewStatus: "ready",
|
||||
ThumbnailURL: "/p/thumb/spider91-spider-main-source",
|
||||
ThumbnailURL: "/p/thumb/scriptcrawler-crawler-main-source",
|
||||
Size: 456,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
@@ -1524,9 +1668,9 @@ func TestDeleteVideoRemovesSpider91SourceFile(t *testing.T) {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
result, err := app.deleteVideo(ctx, "spider91-spider-main-source", true)
|
||||
result, err := app.deleteVideo(ctx, "scriptcrawler-crawler-main-source", true)
|
||||
if err != nil {
|
||||
t.Fatalf("delete spider video: %v", err)
|
||||
t.Fatalf("delete crawler video: %v", err)
|
||||
}
|
||||
if !result.OK || !result.DeletedSource {
|
||||
t.Fatalf("delete result = %#v, want source deleted", result)
|
||||
@@ -1536,23 +1680,23 @@ func TestDeleteVideoRemovesSpider91SourceFile(t *testing.T) {
|
||||
t.Fatalf("deleted file %s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "spider91-spider-main-source"); err != sql.ErrNoRows {
|
||||
if _, err := cat.GetVideo(ctx, "scriptcrawler-crawler-main-source"); err != sql.ErrNoRows {
|
||||
t.Fatalf("deleted video lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
deleted, err := cat.IsVideoDeleted(ctx, "spider91-spider-main-source")
|
||||
deleted, err := cat.IsVideoDeleted(ctx, "scriptcrawler-crawler-main-source")
|
||||
if err != nil {
|
||||
t.Fatalf("check tombstone: %v", err)
|
||||
}
|
||||
if !deleted {
|
||||
t.Fatal("deleted spider91 video tombstone missing")
|
||||
t.Fatal("deleted crawler video tombstone missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupDriveVideosForDeleteSpider91RemovesCrawledDirAndOriginRecords(t *testing.T) {
|
||||
func TestCleanupDriveVideosForDeleteScriptCrawlerRemovesOnlyLocalRows(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
localDir := filepath.Join(root, "previews")
|
||||
driveID := "spider-main"
|
||||
driveID := "crawler-main"
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
@@ -1565,22 +1709,19 @@ func TestCleanupDriveVideosForDeleteSpider91RemovesCrawledDirAndOriginRecords(t
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: driveID,
|
||||
Kind: "spider91",
|
||||
Name: "91 Spider",
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Crawler",
|
||||
RootID: "/",
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed spider91 drive: %v", err)
|
||||
t.Fatalf("seed crawler drive: %v", err)
|
||||
}
|
||||
|
||||
spiderDriveDir := filepath.Join(root, "spider91", driveID)
|
||||
sourceVideo := filepath.Join(spiderDriveDir, "videos", "source.mp4")
|
||||
sourceThumb := filepath.Join(spiderDriveDir, "thumbs", "source.jpg")
|
||||
localPreview := filepath.Join(localDir, "spider91-spider-main-source.mp4")
|
||||
localThumb := filepath.Join(localDir, "thumbs", "spider91-spider-main-source.jpg")
|
||||
migratedPreview := filepath.Join(localDir, "spider91-spider-main-migrated.mp4")
|
||||
migratedThumb := filepath.Join(localDir, "thumbs", "spider91-spider-main-migrated.jpg")
|
||||
for _, path := range []string{sourceVideo, sourceThumb, localPreview, localThumb, migratedPreview, migratedThumb} {
|
||||
localPreview := filepath.Join(localDir, "scriptcrawler-crawler-main-source.mp4")
|
||||
localThumb := filepath.Join(localDir, "thumbs", "scriptcrawler-crawler-main-source.jpg")
|
||||
migratedPreview := filepath.Join(localDir, "scriptcrawler-crawler-main-migrated.mp4")
|
||||
migratedThumb := filepath.Join(localDir, "thumbs", "scriptcrawler-crawler-main-migrated.jpg")
|
||||
for _, path := range []string{localPreview, localThumb, migratedPreview, migratedThumb} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
@@ -1592,22 +1733,22 @@ func TestCleanupDriveVideosForDeleteSpider91RemovesCrawledDirAndOriginRecords(t
|
||||
now := time.Now()
|
||||
for _, v := range []*catalog.Video{
|
||||
{
|
||||
ID: "spider91-spider-main-source",
|
||||
ID: "scriptcrawler-crawler-main-source",
|
||||
DriveID: driveID,
|
||||
FileID: "source.mp4",
|
||||
Title: "Source",
|
||||
PreviewLocal: localPreview,
|
||||
PreviewStatus: "ready",
|
||||
ThumbnailURL: "/p/thumb/spider91-spider-main-source",
|
||||
ThumbnailURL: "/p/thumb/scriptcrawler-crawler-main-source",
|
||||
},
|
||||
{
|
||||
ID: "spider91-spider-main-migrated",
|
||||
ID: "scriptcrawler-crawler-main-migrated",
|
||||
DriveID: "PikPak",
|
||||
FileID: "pikpak-file-id",
|
||||
Title: "Migrated",
|
||||
PreviewLocal: migratedPreview,
|
||||
PreviewStatus: "ready",
|
||||
ThumbnailURL: "/p/thumb/spider91-spider-main-migrated",
|
||||
ThumbnailURL: "/p/thumb/scriptcrawler-crawler-main-migrated",
|
||||
},
|
||||
{
|
||||
ID: "pikpak-PikPak-other",
|
||||
@@ -1635,24 +1776,30 @@ func TestCleanupDriveVideosForDeleteSpider91RemovesCrawledDirAndOriginRecords(t
|
||||
}
|
||||
removed, err := app.cleanupDriveVideosForDelete(ctx, driveID)
|
||||
if err != nil {
|
||||
t.Fatalf("cleanup spider91 videos: %v", err)
|
||||
t.Fatalf("cleanup crawler videos: %v", err)
|
||||
}
|
||||
if removed != 2 {
|
||||
t.Fatalf("removed = %d, want 2", removed)
|
||||
if removed != 1 {
|
||||
t.Fatalf("removed = %d, want 1", removed)
|
||||
}
|
||||
for _, id := range []string{"spider91-spider-main-source", "spider91-spider-main-migrated"} {
|
||||
if _, err := cat.GetVideo(ctx, id); err != sql.ErrNoRows {
|
||||
t.Fatalf("%s lookup error = %v, want sql.ErrNoRows", id, err)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "scriptcrawler-crawler-main-source"); err != sql.ErrNoRows {
|
||||
t.Fatalf("local crawler video lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "scriptcrawler-crawler-main-migrated"); err != nil {
|
||||
t.Fatalf("migrated crawler video missing: %v", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "pikpak-PikPak-other"); err != nil {
|
||||
t.Fatalf("unrelated pikpak video missing: %v", err)
|
||||
}
|
||||
for _, path := range []string{spiderDriveDir, localPreview, localThumb, migratedPreview, migratedThumb} {
|
||||
for _, path := range []string{localPreview, localThumb} {
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("%s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
for _, path := range []string{migratedPreview, migratedThumb} {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("%s missing, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupOrphanDriveVideosRemovesRowsAndGeneratedAssets(t *testing.T) {
|
||||
@@ -1743,7 +1890,7 @@ func TestCleanupOrphanDriveVideosRemovesRowsAndGeneratedAssets(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupDuplicateVideoAssetsRemovesOnlyDuplicateLocalAssets(t *testing.T) {
|
||||
func TestCleanupDuplicateVideoAssetsDeletesExactDuplicateRows(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
localDir := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
@@ -1824,15 +1971,22 @@ func TestCleanupDuplicateVideoAssetsRemovesOnlyDuplicateLocalAssets(t *testing.T
|
||||
t.Fatalf("duplicate asset %s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
dup, err := cat.GetVideo(ctx, "duplicate-video")
|
||||
if _, err := cat.GetVideo(ctx, "duplicate-video"); err != sql.ErrNoRows {
|
||||
t.Fatalf("duplicate lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
deleted, err := cat.IsVideoDeleted(ctx, "duplicate-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get duplicate: %v", err)
|
||||
t.Fatalf("check duplicate tombstone: %v", err)
|
||||
}
|
||||
if dup.PreviewLocal != "" || dup.PreviewStatus != "pending" {
|
||||
t.Fatalf("duplicate preview local=%q status=%q, want empty pending", dup.PreviewLocal, dup.PreviewStatus)
|
||||
if !deleted {
|
||||
t.Fatalf("duplicate tombstone missing")
|
||||
}
|
||||
if dup.ThumbnailURL != "" {
|
||||
t.Fatalf("duplicate thumbnail url = %q, want empty", dup.ThumbnailURL)
|
||||
deletedItems, _, err := cat.ListDeletedVideos(ctx, catalog.ListParams{Page: 1, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list deleted videos: %v", err)
|
||||
}
|
||||
if len(deletedItems) != 1 || deletedItems[0].ID != "duplicate-video" || deletedItems[0].Reason != catalog.DeletedVideoReasonDuplicate {
|
||||
t.Fatalf("duplicate tombstone = %#v, want reason %q", deletedItems, catalog.DeletedVideoReasonDuplicate)
|
||||
}
|
||||
canon, err := cat.GetVideo(ctx, "canonical-video")
|
||||
if err != nil {
|
||||
@@ -1843,6 +1997,137 @@ func TestCleanupDuplicateVideoAssetsRemovesOnlyDuplicateLocalAssets(t *testing.T
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupDuplicateVideoAssetsDeletesNearDuplicateRowsKeepingLargest(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
localDir := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
smallPreview := filepath.Join(localDir, "small-video.mp4")
|
||||
largePreview := filepath.Join(localDir, "large-video.mp4")
|
||||
smallThumb := filepath.Join(localDir, "thumbs", "small-video.jpg")
|
||||
largeThumb := filepath.Join(localDir, "thumbs", "large-video.jpg")
|
||||
for _, path := range []string{smallPreview, largePreview} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("preview"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
writeSolidJPEG(t, smallThumb, color.RGBA{R: 180, G: 80, B: 40, A: 255})
|
||||
writeSolidJPEG(t, largeThumb, color.RGBA{R: 180, G: 80, B: 40, A: 255})
|
||||
|
||||
now := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
|
||||
for _, v := range []*catalog.Video{
|
||||
{
|
||||
ID: "small-video",
|
||||
DriveID: "scriptcrawler-a",
|
||||
FileID: "file-small",
|
||||
FileName: "small.mp4",
|
||||
Title: "反差极品大二女友,叫声可射~,“射进小骚逼里面~” - 91porn",
|
||||
DurationSeconds: 313,
|
||||
Size: 1024,
|
||||
ThumbnailURL: "/p/thumb/small-video",
|
||||
PreviewLocal: smallPreview,
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "large-video",
|
||||
DriveID: "scriptcrawler-b",
|
||||
FileID: "file-large",
|
||||
FileName: "large.mp4",
|
||||
Title: "反差极品大二女友,叫声可射~,“射进小骚逼里面~”_91pinse",
|
||||
DurationSeconds: 313,
|
||||
Size: 4096,
|
||||
ThumbnailURL: "/p/thumb/large-video",
|
||||
PreviewLocal: largePreview,
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
}
|
||||
if err := app.cleanupDuplicateVideoAssets(ctx); err != nil {
|
||||
t.Fatalf("cleanup duplicate video assets: %v", err)
|
||||
}
|
||||
|
||||
if _, err := cat.GetVideo(ctx, "small-video"); err != sql.ErrNoRows {
|
||||
t.Fatalf("small duplicate lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
deleted, err := cat.IsVideoDeleted(ctx, "small-video")
|
||||
if err != nil {
|
||||
t.Fatalf("check small tombstone: %v", err)
|
||||
}
|
||||
if !deleted {
|
||||
t.Fatalf("small duplicate tombstone missing")
|
||||
}
|
||||
deletedItems, _, err := cat.ListDeletedVideos(ctx, catalog.ListParams{Page: 1, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list deleted videos: %v", err)
|
||||
}
|
||||
if len(deletedItems) != 1 || deletedItems[0].ID != "small-video" || deletedItems[0].Reason != catalog.DeletedVideoReasonDuplicate {
|
||||
t.Fatalf("small duplicate tombstone = %#v, want reason %q", deletedItems, catalog.DeletedVideoReasonDuplicate)
|
||||
}
|
||||
large, err := cat.GetVideo(ctx, "large-video")
|
||||
if err != nil {
|
||||
t.Fatalf("large canonical missing: %v", err)
|
||||
}
|
||||
if large.Size != 4096 {
|
||||
t.Fatalf("large canonical size = %d, want 4096", large.Size)
|
||||
}
|
||||
for _, path := range []string{smallPreview, smallThumb} {
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("small duplicate asset %s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
for _, path := range []string{largePreview, largeThumb} {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("large canonical asset %s missing: %v", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeSolidJPEG(t *testing.T, path string, c color.RGBA) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
t.Fatalf("create %s: %v", path, err)
|
||||
}
|
||||
defer f.Close()
|
||||
img := image.NewRGBA(image.Rect(0, 0, 64, 64))
|
||||
for y := 0; y < 64; y++ {
|
||||
for x := 0; x < 64; x++ {
|
||||
img.SetRGBA(x, y, c)
|
||||
}
|
||||
}
|
||||
if err := jpeg.Encode(f, img, &jpeg.Options{Quality: 95}); err != nil {
|
||||
t.Fatalf("encode %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
type serverFakeTeaserGenerator struct {
|
||||
mu sync.Mutex
|
||||
events []string
|
||||
@@ -1978,11 +2263,11 @@ func (d *serverSourceRemovableFakeDrive) Remove(ctx context.Context, fileID stri
|
||||
return nil
|
||||
}
|
||||
|
||||
type serverFakeSpider91MigrationRunner struct {
|
||||
type serverFakeCrawlerUploadRunner struct {
|
||||
called int
|
||||
}
|
||||
|
||||
func (r *serverFakeSpider91MigrationRunner) RunOnce(context.Context) error {
|
||||
func (r *serverFakeCrawlerUploadRunner) RunOnce(context.Context) error {
|
||||
r.called++
|
||||
return nil
|
||||
}
|
||||
|
||||
+242
-93
@@ -24,7 +24,6 @@ import (
|
||||
"github.com/video-site/backend/internal/drives/guangyapan"
|
||||
"github.com/video-site/backend/internal/drives/p123"
|
||||
"github.com/video-site/backend/internal/drives/scriptcrawler"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/drives/wopan"
|
||||
)
|
||||
|
||||
@@ -53,6 +52,7 @@ type AdminServer struct {
|
||||
OnDriveDeleteCleanup func(ctx context.Context, driveID string) (int, error)
|
||||
OnDriveRemoved func(driveID string)
|
||||
OnScanRequested func(driveID string) bool
|
||||
OnCrawlerUploadRequested func(driveID string) (bool, string)
|
||||
OnStopDriveTasks func(driveID string) bool
|
||||
OnStopAllTasks func() int
|
||||
OnRegenPreview func(videoID string)
|
||||
@@ -76,11 +76,8 @@ type AdminServer struct {
|
||||
// Theme 读写("dark" | "pink" | "sky")
|
||||
GetTheme func() string
|
||||
SetTheme func(theme string) error
|
||||
// Spider91 → 115/123/PikPak/OneDrive/Google Drive/联通网盘/光鸭网盘 上传目标 drive ID 读写
|
||||
GetSpider91UploadDriveID func() string
|
||||
SetSpider91UploadDriveID func(driveID string) error
|
||||
// OnRunNightlyJob 触发一次完整的凌晨流水线(Phase1 扫盘 + Phase2 91 爬虫 +
|
||||
// Phase3 迁移)。立即返回 —— 实际任务在后台跑,admin 在日志或下次状态查询里
|
||||
// OnRunNightlyJob 触发一次完整的凌晨流水线(Phase1 扫盘 + Phase2 爬虫 +
|
||||
// Phase3 上传)。立即返回 —— 实际任务在后台跑,admin 在日志或下次状态查询里
|
||||
// 看进度。若流水线正在跑或已排队,Runner 会拒绝重复触发。
|
||||
OnRunNightlyJob func() bool
|
||||
// GetNightlyJobStatus 返回凌晨流水线当前状态,用于前端禁用重复触发按钮。
|
||||
@@ -194,6 +191,7 @@ func (a *AdminServer) Register(r chi.Router) {
|
||||
r.Post("/crawlers/test-script", a.handleTestCrawlerScript)
|
||||
r.Delete("/crawlers/{id}", a.handleDeleteCrawler)
|
||||
r.Post("/crawlers/{id}/run", a.handleRunCrawler)
|
||||
r.Post("/crawlers/{id}/upload", a.handleUploadCrawlerVideos)
|
||||
r.Post("/crawlers/{id}/tasks/stop", a.handleStopCrawlerTasks)
|
||||
|
||||
// 视频
|
||||
@@ -476,12 +474,10 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
// SkipDirIDs 是用户在 admin 配置的"扫描跳过目录"集合(drive 侧目录 fileID)。
|
||||
// 前端用它在"设置跳过目录"弹窗里回显已选项;JSON 字段名 camelCase 与
|
||||
// catalog.Drive 保持一致。
|
||||
SkipDirIDs []string `json:"skipDirIds"`
|
||||
// LastCrawlAt 是 spider91 上次成功爬取的 unix 秒(来自 credentials.last_crawl_at)。
|
||||
// 其它 kind 留 0;前端用它显示"上次抓取: N 小时前"。
|
||||
Spider91Proxy string `json:"spider91Proxy,omitempty"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
GoogleDriveUseOnlineAPI *bool `json:"googleDriveUseOnlineAPI,omitempty"`
|
||||
SkipDirIDs []string `json:"skipDirIds"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
GoogleDriveUseOnlineAPI *bool `json:"googleDriveUseOnlineAPI,omitempty"`
|
||||
GoogleDriveOpenListAPIURL string `json:"googleDriveOpenListApiUrl,omitempty"`
|
||||
// STRMAllowOutsideRoot 是 localstorage 的 .strm 越root开关;其它 kind 省略。
|
||||
STRMAllowOutsideRoot *bool `json:"strmAllowOutsideRoot,omitempty"`
|
||||
ScanGenerationStatus GenerationStatus `json:"scanGenerationStatus"`
|
||||
@@ -529,7 +525,6 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
if generation.Transcode.State == "" {
|
||||
generation.Transcode.State = "idle"
|
||||
}
|
||||
// spider91 没有用户凭证概念;只要存在 drive 行就视为"已配置"。
|
||||
// last_crawl_at 是后端自动写入的运行状态字段,不计入 hasCredential 判定。
|
||||
hasCred := false
|
||||
userCredKeys := 0
|
||||
@@ -539,7 +534,7 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
userCredKeys++
|
||||
}
|
||||
hasCred = userCredKeys > 0 || d.Kind == "spider91"
|
||||
hasCred = userCredKeys > 0
|
||||
|
||||
var lastCrawlAt int64
|
||||
if d.Credentials != nil {
|
||||
@@ -557,9 +552,9 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
HasCredential: hasCred,
|
||||
TeaserEnabled: d.TeaserEnabled,
|
||||
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
|
||||
Spider91Proxy: spider91ProxyForDrive(d),
|
||||
LastCrawlAt: lastCrawlAt,
|
||||
GoogleDriveUseOnlineAPI: googleDriveUseOnlineAPIForDrive(d),
|
||||
GoogleDriveOpenListAPIURL: googleDriveOpenListAPIURLForDrive(d),
|
||||
STRMAllowOutsideRoot: strmAllowOutsideRootForDrive(d),
|
||||
ScanGenerationStatus: generation.Scan,
|
||||
ThumbnailGenerationStatus: generation.Thumbnail,
|
||||
@@ -618,17 +613,20 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
|
||||
if existingDrive, err := a.Catalog.GetDrive(r.Context(), body.ID); err == nil {
|
||||
existing = existingDrive
|
||||
}
|
||||
if body.Kind == "spider91" {
|
||||
http.Error(w, "91Spider 已不再支持通过网盘添加,请在爬虫管理页面添加爬虫脚本", http.StatusBadRequest)
|
||||
if !isSupportedDriveKind(body.Kind) {
|
||||
http.Error(w, "unsupported drive kind", http.StatusBadRequest)
|
||||
return
|
||||
} else if body.Kind == scriptcrawler.Kind {
|
||||
}
|
||||
if body.Kind == scriptcrawler.Kind {
|
||||
credentials, err := mergeScriptCrawlerCredentials(existing, body.Credentials)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
body.Credentials = credentials
|
||||
} else if body.Kind == "googledrive" || body.Kind == "localstorage" || body.Kind == "guangyapan" {
|
||||
} else if body.Kind == "googledrive" {
|
||||
body.Credentials = mergeGoogleDriveCredentials(existing, body.Credentials)
|
||||
} else if body.Kind == "localstorage" || body.Kind == "guangyapan" {
|
||||
// 按键合并、空值沿用旧值:这些网盘的编辑表单允许只改某几个字段,
|
||||
// 其它 token / 路径 / 开关字段应保留旧值。
|
||||
body.Credentials = mergeNonEmptyCredentials(existing, body.Credentials)
|
||||
@@ -810,7 +808,6 @@ func crawlerVideoIDPrefixes(d *catalog.Drive) []string {
|
||||
}
|
||||
return []string{
|
||||
scriptcrawler.Kind + "-" + d.ID + "-",
|
||||
spider91.Kind + "-" + d.ID + "-",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -956,7 +953,7 @@ func (a *AdminServer) validateCrawlerUploadDrive(ctx context.Context, driveID st
|
||||
|
||||
func isCrawlerUploadTargetKind(kind string) bool {
|
||||
switch strings.TrimSpace(kind) {
|
||||
case "p115", "pikpak", "p123", "googledrive", "onedrive", "wopan":
|
||||
case "p115", "pikpak", "p123", "googledrive", "onedrive", "wopan", "guangyapan":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
@@ -1285,6 +1282,104 @@ func (a *AdminServer) handleRunCrawler(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusAccepted, resp)
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleUploadCrawlerVideos(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
d, err := a.Catalog.GetDrive(r.Context(), id)
|
||||
if err != nil || d == nil || !isConfiguredCrawlerDrive(d) {
|
||||
http.Error(w, "crawler not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
status := a.nightlyJobStatus()
|
||||
if status.Running || status.Queued {
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"ok": true,
|
||||
"accepted": false,
|
||||
"message": fullScanBusyMessage,
|
||||
"status": status,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
assets, err := a.Catalog.CountCrawlerAssets(r.Context(), d.ID, crawlerVideoIDPrefixes(d))
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
generation := DriveGenerationStatuses{}
|
||||
if a.GetDriveGenerationStatuses != nil {
|
||||
generation = a.GetDriveGenerationStatuses()[d.ID]
|
||||
}
|
||||
if reason := crawlerUploadBlockedReason(d, assets, generation); reason != "" {
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"ok": true,
|
||||
"accepted": false,
|
||||
"message": reason,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
accepted := true
|
||||
message := ""
|
||||
if a.OnCrawlerUploadRequested != nil {
|
||||
accepted, message = a.OnCrawlerUploadRequested(id)
|
||||
}
|
||||
resp := map[string]any{"ok": true, "accepted": accepted}
|
||||
if !accepted {
|
||||
if strings.TrimSpace(message) == "" {
|
||||
message = driveTaskBusyMessage
|
||||
}
|
||||
resp["message"] = message
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, resp)
|
||||
}
|
||||
|
||||
func crawlerUploadBlockedReason(d *catalog.Drive, assets catalog.CrawlerAssetCounts, generation DriveGenerationStatuses) string {
|
||||
if d == nil || !isConfiguredCrawlerDrive(d) {
|
||||
return "爬虫不存在"
|
||||
}
|
||||
if strings.TrimSpace(d.Credentials["upload_drive_id"]) == "" {
|
||||
return "请先配置上传网盘"
|
||||
}
|
||||
if assets.Local <= 0 {
|
||||
return "没有待上传的本地视频"
|
||||
}
|
||||
if crawlerGenerationBusy(generation) {
|
||||
return "当前爬虫有正在进行的任务,请稍后重试"
|
||||
}
|
||||
if assets.Fingerprint.Pending > 0 {
|
||||
return "还有待生成的视频指纹"
|
||||
}
|
||||
if assets.Fingerprint.Failed > 0 {
|
||||
return "存在指纹生成失败的视频,请先重试或处理失败项"
|
||||
}
|
||||
if d.TeaserEnabled {
|
||||
if assets.Teaser.Pending > 0 {
|
||||
return "还有待生成的预览视频"
|
||||
}
|
||||
if assets.Teaser.Failed > 0 {
|
||||
return "存在预览视频生成失败的视频,请先重试或处理失败项"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func crawlerGenerationBusy(g DriveGenerationStatuses) bool {
|
||||
return generationBusy(g.Scan) ||
|
||||
generationBusy(g.Thumbnail) ||
|
||||
generationBusy(g.Preview) ||
|
||||
generationBusy(g.Fingerprint) ||
|
||||
generationBusy(g.Upload)
|
||||
}
|
||||
|
||||
func generationBusy(g GenerationStatus) bool {
|
||||
switch strings.TrimSpace(g.State) {
|
||||
case "", "idle":
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleStopCrawlerTasks(w http.ResponseWriter, r *http.Request) {
|
||||
a.handleStopDriveTasks(w, r)
|
||||
}
|
||||
@@ -1335,6 +1430,15 @@ func isCrawlerDriveKind(kind string) bool {
|
||||
return kind == scriptcrawler.Kind
|
||||
}
|
||||
|
||||
func isSupportedDriveKind(kind string) bool {
|
||||
switch kind {
|
||||
case "quark", "p115", "p123", "pikpak", "wopan", "guangyapan", "onedrive", "googledrive", "localstorage", scriptcrawler.Kind:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isConfiguredCrawlerDrive(d *catalog.Drive) bool {
|
||||
return d != nil &&
|
||||
isCrawlerDriveKind(d.Kind) &&
|
||||
@@ -1370,13 +1474,6 @@ func (a *AdminServer) removeImportedCrawlerScript(d *catalog.Drive) (bool, error
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func spider91ProxyForDrive(d *catalog.Drive) string {
|
||||
if d == nil || d.Kind != "spider91" || d.Credentials == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(d.Credentials["proxy"])
|
||||
}
|
||||
|
||||
// strmAllowOutsideRootForDrive 返回 localstorage 的 .strm 越root开关;
|
||||
// 其它 kind 返回 nil(JSON 省略)。未配置时默认 false。
|
||||
func strmAllowOutsideRootForDrive(d *catalog.Drive) *bool {
|
||||
@@ -1412,6 +1509,21 @@ func googleDriveUseOnlineAPIForDrive(d *catalog.Drive) *bool {
|
||||
return &result
|
||||
}
|
||||
|
||||
func googleDriveOpenListAPIURLForDrive(d *catalog.Drive) string {
|
||||
if d == nil || d.Kind != "googledrive" || d.Credentials == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(d.Credentials["api_url_address"])
|
||||
}
|
||||
|
||||
func mergeGoogleDriveCredentials(existing *catalog.Drive, incoming map[string]string) map[string]string {
|
||||
merged := mergeNonEmptyCredentials(existing, incoming)
|
||||
if _, ok := incoming["api_url_address"]; ok && strings.TrimSpace(incoming["api_url_address"]) == "" {
|
||||
delete(merged, "api_url_address")
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
// mergeNonEmptyCredentials 逐键合并凭证:incoming 里非空的键覆盖旧值,
|
||||
// 空值/缺失的键沿用旧值。googledrive、localstorage 和 guangyapan 的编辑表单都依赖
|
||||
// 这个语义(留空 = 不修改)。
|
||||
@@ -1436,34 +1548,6 @@ func mergeNonEmptyCredentials(existing *catalog.Drive, incoming map[string]strin
|
||||
return merged
|
||||
}
|
||||
|
||||
func mergeSpider91Credentials(existing *catalog.Drive, incoming map[string]string) (map[string]string, error) {
|
||||
merged := map[string]string{}
|
||||
if existing != nil {
|
||||
for k, v := range existing.Credentials {
|
||||
merged[k] = v
|
||||
}
|
||||
}
|
||||
for k, v := range incoming {
|
||||
if strings.TrimSpace(k) == "" {
|
||||
continue
|
||||
}
|
||||
if k == "proxy" {
|
||||
proxy, err := normalizeSpider91ProxyURL(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if proxy == "" {
|
||||
delete(merged, "proxy")
|
||||
} else {
|
||||
merged["proxy"] = proxy
|
||||
}
|
||||
continue
|
||||
}
|
||||
merged[k] = v
|
||||
}
|
||||
return merged, nil
|
||||
}
|
||||
|
||||
func mergeScriptCrawlerCredentials(existing *catalog.Drive, incoming map[string]string) (map[string]string, error) {
|
||||
merged := map[string]string{}
|
||||
if existing != nil {
|
||||
@@ -1525,10 +1609,6 @@ func mergeScriptCrawlerCredentials(existing *catalog.Drive, incoming map[string]
|
||||
return merged, nil
|
||||
}
|
||||
|
||||
func normalizeSpider91ProxyURL(raw string) (string, error) {
|
||||
return normalizeCrawlerProxyURL(raw, "91Spider")
|
||||
}
|
||||
|
||||
func normalizeCrawlerProxyURL(raw, label string) (string, error) {
|
||||
proxy := strings.TrimSpace(raw)
|
||||
if proxy == "" {
|
||||
@@ -1941,7 +2021,7 @@ func (a *AdminServer) handleAdminListVideos(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"items": items,
|
||||
"items": mapAdminVideos(items),
|
||||
"total": total,
|
||||
"page": page,
|
||||
"size": size,
|
||||
@@ -1972,7 +2052,12 @@ func (a *AdminServer) handleListBlacklist(w http.ResponseWriter, r *http.Request
|
||||
if size <= 0 || size > 100 {
|
||||
size = 100
|
||||
}
|
||||
items, total, err := a.Catalog.ListDeletedVideos(r.Context(), q.Get("keyword"), page, size)
|
||||
items, total, err := a.Catalog.ListDeletedVideos(r.Context(), catalog.ListParams{
|
||||
Keyword: q.Get("keyword"),
|
||||
DriveID: q.Get("driveId"),
|
||||
Page: page,
|
||||
PageSize: size,
|
||||
})
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
@@ -2055,7 +2140,6 @@ type updateVideoReq struct {
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Tags []string `json:"tags"`
|
||||
Category string `json:"category"`
|
||||
Badges []string `json:"badges"`
|
||||
Description string `json:"description"`
|
||||
Thumbnail string `json:"thumbnail"`
|
||||
@@ -2063,6 +2147,97 @@ type updateVideoReq struct {
|
||||
DurationSec int `json:"durationSeconds"`
|
||||
}
|
||||
|
||||
type adminVideoDTO struct {
|
||||
ID string `json:"id"`
|
||||
DriveID string `json:"driveId"`
|
||||
FileID string `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
ContentHash string `json:"contentHash"`
|
||||
SampledSHA256 string `json:"sampledSha256"`
|
||||
FingerprintStatus string `json:"fingerprintStatus"`
|
||||
FingerprintError string `json:"fingerprintError"`
|
||||
ParentID string `json:"parentId"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Tags []string `json:"tags"`
|
||||
DurationSeconds int `json:"durationSeconds"`
|
||||
Size int64 `json:"size"`
|
||||
Ext string `json:"ext"`
|
||||
Quality string `json:"quality"`
|
||||
ThumbnailURL string `json:"thumbnailUrl"`
|
||||
PreviewFileID string `json:"previewFileId"`
|
||||
PreviewLocal string `json:"previewLocal"`
|
||||
PreviewStatus string `json:"previewStatus"`
|
||||
TranscodeStatus string `json:"transcodeStatus"`
|
||||
TranscodeError string `json:"transcodeError"`
|
||||
TranscodedFileID string `json:"transcodedFileId"`
|
||||
TranscodedSize int64 `json:"transcodedSize"`
|
||||
Views int `json:"views"`
|
||||
LastViewedAt time.Time `json:"lastViewedAt"`
|
||||
Favorites int `json:"favorites"`
|
||||
Comments int `json:"comments"`
|
||||
Likes int `json:"likes"`
|
||||
Dislikes int `json:"dislikes"`
|
||||
Hidden bool `json:"hidden"`
|
||||
Badges []string `json:"badges"`
|
||||
Description string `json:"description"`
|
||||
PublishedAt time.Time `json:"publishedAt"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
func mapAdminVideo(v *catalog.Video) adminVideoDTO {
|
||||
if v == nil {
|
||||
return adminVideoDTO{}
|
||||
}
|
||||
return adminVideoDTO{
|
||||
ID: v.ID,
|
||||
DriveID: v.DriveID,
|
||||
FileID: v.FileID,
|
||||
FileName: v.FileName,
|
||||
ContentHash: v.ContentHash,
|
||||
SampledSHA256: v.SampledSHA256,
|
||||
FingerprintStatus: v.FingerprintStatus,
|
||||
FingerprintError: v.FingerprintError,
|
||||
ParentID: v.ParentID,
|
||||
Title: v.Title,
|
||||
Author: v.Author,
|
||||
Tags: v.Tags,
|
||||
DurationSeconds: v.DurationSeconds,
|
||||
Size: v.Size,
|
||||
Ext: v.Ext,
|
||||
Quality: v.Quality,
|
||||
ThumbnailURL: v.ThumbnailURL,
|
||||
PreviewFileID: v.PreviewFileID,
|
||||
PreviewLocal: v.PreviewLocal,
|
||||
PreviewStatus: v.PreviewStatus,
|
||||
TranscodeStatus: v.TranscodeStatus,
|
||||
TranscodeError: v.TranscodeError,
|
||||
TranscodedFileID: v.TranscodedFileID,
|
||||
TranscodedSize: v.TranscodedSize,
|
||||
Views: v.Views,
|
||||
LastViewedAt: v.LastViewedAt,
|
||||
Favorites: v.Favorites,
|
||||
Comments: v.Comments,
|
||||
Likes: v.Likes,
|
||||
Dislikes: v.Dislikes,
|
||||
Hidden: v.Hidden,
|
||||
Badges: v.Badges,
|
||||
Description: v.Description,
|
||||
PublishedAt: v.PublishedAt,
|
||||
CreatedAt: v.CreatedAt,
|
||||
UpdatedAt: v.UpdatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func mapAdminVideos(vs []*catalog.Video) []adminVideoDTO {
|
||||
out := make([]adminVideoDTO, 0, len(vs))
|
||||
for _, v := range vs {
|
||||
out = append(out, mapAdminVideo(v))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleUpdateVideo(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
var body updateVideoReq
|
||||
@@ -2081,9 +2256,6 @@ func (a *AdminServer) handleUpdateVideo(w http.ResponseWriter, r *http.Request)
|
||||
if body.Author != "" {
|
||||
v.Author = body.Author
|
||||
}
|
||||
if body.Category != "" {
|
||||
v.Category = body.Category
|
||||
}
|
||||
if body.Badges != nil {
|
||||
v.Badges = body.Badges
|
||||
}
|
||||
@@ -2118,7 +2290,7 @@ func (a *AdminServer) handleUpdateVideo(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
writeJSON(w, http.StatusOK, mapAdminVideo(v))
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleDeleteVideo(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -2212,10 +2384,9 @@ func (a *AdminServer) handleRegenFailedFingerprints(w http.ResponseWriter, r *ht
|
||||
//
|
||||
// 注意:早期的全局 previewEnabled 字段已经下沉为每盘 teaser_enabled,
|
||||
// 不再出现在这里;前端要切换某个盘的预览视频生成请用 POST /admin/api/drives 上传
|
||||
// teaserEnabled 字段。保留 settings 用作主题、spider91 上传目标这类全局配置。
|
||||
// teaserEnabled 字段。settings 目前只保留全站主题。
|
||||
type settingsDTO struct {
|
||||
Theme string `json:"theme"`
|
||||
Spider91UploadDriveID string `json:"spider91UploadDriveId"`
|
||||
Theme string `json:"theme"`
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleGetSettings(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -2225,19 +2396,12 @@ func (a *AdminServer) handleGetSettings(w http.ResponseWriter, r *http.Request)
|
||||
theme = v
|
||||
}
|
||||
}
|
||||
spider91UploadID := ""
|
||||
if a.GetSpider91UploadDriveID != nil {
|
||||
spider91UploadID = a.GetSpider91UploadDriveID()
|
||||
}
|
||||
writeJSON(w, http.StatusOK, settingsDTO{
|
||||
Theme: theme,
|
||||
Spider91UploadDriveID: spider91UploadID,
|
||||
Theme: theme,
|
||||
})
|
||||
}
|
||||
|
||||
func (a *AdminServer) handlePutSettings(w http.ResponseWriter, r *http.Request) {
|
||||
// 用 map 区分"没传"和"传了空字符串"两种语义;空 spider91 上传 ID 表示
|
||||
// 本地保存不上传。
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.NewDecoder(r.Body).Decode(&raw); err != nil {
|
||||
writeErr(w, http.StatusBadRequest, err)
|
||||
@@ -2258,25 +2422,10 @@ func (a *AdminServer) handlePutSettings(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
|
||||
if v, ok := raw["spider91UploadDriveId"]; ok && a.SetSpider91UploadDriveID != nil {
|
||||
var driveID string
|
||||
if err := json.Unmarshal(v, &driveID); err != nil {
|
||||
writeErr(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
if err := a.SetSpider91UploadDriveID(driveID); err != nil {
|
||||
writeErr(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 回显当前值
|
||||
resp := settingsDTO{}
|
||||
if a.GetTheme != nil {
|
||||
resp.Theme = a.GetTheme()
|
||||
}
|
||||
if a.GetSpider91UploadDriveID != nil {
|
||||
resp.Spider91UploadDriveID = a.GetSpider91UploadDriveID()
|
||||
}
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
@@ -732,9 +732,34 @@ func TestHandleUpsertGoogleDriveMergesOAuthCredentials(t *testing.T) {
|
||||
if got.Credentials["client_id"] != "google-client-id" || got.Credentials["client_secret"] != "google-client-secret" {
|
||||
t.Fatalf("oauth client credentials = %#v, want saved", got.Credentials)
|
||||
}
|
||||
if got.Credentials["api_url_address"] != "https://api.oplist.org/googleui/renewapi" {
|
||||
t.Fatalf("api_url_address = %q, want preserved", got.Credentials["api_url_address"])
|
||||
}
|
||||
|
||||
clearReq := httptest.NewRequest(http.MethodPost, "/admin/api/drives", bytes.NewBufferString(`{
|
||||
"id": "google-main",
|
||||
"kind": "googledrive",
|
||||
"name": "Google Drive",
|
||||
"rootId": "root",
|
||||
"credentials": {
|
||||
"api_url_address": ""
|
||||
}
|
||||
}`))
|
||||
clearRR := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(clearRR, clearReq)
|
||||
if clearRR.Code != http.StatusOK {
|
||||
t.Fatalf("clear status = %d, body = %s", clearRR.Code, clearRR.Body.String())
|
||||
}
|
||||
cleared, err := cat.GetDrive(ctx, "google-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get cleared drive: %v", err)
|
||||
}
|
||||
if _, ok := cleared.Credentials["api_url_address"]; ok {
|
||||
t.Fatalf("api_url_address was not cleared: %#v", cleared.Credentials)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertSpider91DriveIsRejected(t *testing.T) {
|
||||
func TestHandleUpsertUnknownDriveKindIsRejected(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -747,14 +772,12 @@ func TestHandleUpsertSpider91DriveIsRejected(t *testing.T) {
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "spider91-main",
|
||||
Kind: "spider91",
|
||||
Name: "91 Spider",
|
||||
ID: "unknown-main",
|
||||
Kind: "unknown",
|
||||
Name: "Unknown",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"last_crawl_at": "1800000000",
|
||||
"proxy": "http://old-proxy.local:7890",
|
||||
"script_path": "/opt/video-site-91/91VideoSpider/spider_91porn.py",
|
||||
"token": "old-token",
|
||||
},
|
||||
Status: "ok",
|
||||
}); err != nil {
|
||||
@@ -762,33 +785,27 @@ func TestHandleUpsertSpider91DriveIsRejected(t *testing.T) {
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"kind": "spider91",
|
||||
"name": "91 Spider",
|
||||
"id": "unknown-main",
|
||||
"kind": "unknown",
|
||||
"name": "Unknown",
|
||||
"rootId": "/",
|
||||
"credentials": {"proxy": " socks5h://proxy-user:proxy-pass@127.0.0.1:7891 "}
|
||||
"credentials": {"token": "new-token"}
|
||||
}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), "爬虫管理") {
|
||||
t.Fatalf("body = %q, want crawler management guidance", rr.Body.String())
|
||||
if rr.Body.String() != "unsupported drive kind\n" {
|
||||
t.Fatalf("body = %q, want unsupported kind", rr.Body.String())
|
||||
}
|
||||
|
||||
got, err := cat.GetDrive(ctx, "spider91-main")
|
||||
got, err := cat.GetDrive(ctx, "unknown-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.Credentials["proxy"] != "http://old-proxy.local:7890" {
|
||||
t.Fatalf("proxy = %q, want unchanged old proxy", got.Credentials["proxy"])
|
||||
}
|
||||
if got.Credentials["last_crawl_at"] != "1800000000" {
|
||||
t.Fatalf("last_crawl_at = %q, want preserved", got.Credentials["last_crawl_at"])
|
||||
}
|
||||
if got.Credentials["script_path"] == "" {
|
||||
t.Fatalf("script_path should be preserved")
|
||||
if got.Credentials["token"] != "old-token" {
|
||||
t.Fatalf("token = %q, want unchanged old token", got.Credentials["token"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -914,31 +931,18 @@ func TestHandleListCrawlersOnlyIncludesCrawlerPageScripts(t *testing.T) {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
scriptPath := filepath.Join(tmp, "spider_91porn.py")
|
||||
if err := os.WriteFile(scriptPath, []byte("CRAWLER_NAME = \"91Porn\"\n"), 0o644); err != nil {
|
||||
scriptPath := filepath.Join(tmp, "demo_crawler.py")
|
||||
if err := os.WriteFile(scriptPath, []byte("CRAWLER_NAME = \"Demo Crawler\"\n"), 0o644); err != nil {
|
||||
t.Fatalf("write crawler script: %v", err)
|
||||
}
|
||||
|
||||
for _, d := range []*catalog.Drive{
|
||||
{
|
||||
ID: "spider91-main",
|
||||
Kind: "spider91",
|
||||
Name: "91 Spider",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"last_crawl_at": "1800000000",
|
||||
"proxy": " http://127.0.0.1:7890 ",
|
||||
"script_path": scriptPath,
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
{
|
||||
ID: "crawler-spider91",
|
||||
ID: "crawler-main",
|
||||
Kind: "scriptcrawler",
|
||||
Name: "91 Spider",
|
||||
Name: "Crawler",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"builtin": "spider91",
|
||||
"last_crawl_at": "1800000000",
|
||||
"proxy": " http://127.0.0.1:7890 ",
|
||||
"script_path": scriptPath,
|
||||
@@ -980,27 +984,27 @@ func TestHandleListCrawlersOnlyIncludesCrawlerPageScripts(t *testing.T) {
|
||||
}
|
||||
for _, v := range []*catalog.Video{
|
||||
{
|
||||
ID: "spider91-crawler-spider91-local",
|
||||
DriveID: "crawler-spider91",
|
||||
ID: "scriptcrawler-crawler-main-local",
|
||||
DriveID: "crawler-main",
|
||||
FileID: "local.mp4",
|
||||
FileName: "local.mp4",
|
||||
Title: "Local",
|
||||
Size: 123,
|
||||
Ext: "mp4",
|
||||
ThumbnailURL: "/p/thumb/spider91-crawler-spider91-local",
|
||||
ThumbnailURL: "/p/thumb/scriptcrawler-crawler-main-local",
|
||||
PreviewStatus: "ready",
|
||||
DurationSeconds: 12,
|
||||
PublishedAt: time.Now(),
|
||||
},
|
||||
{
|
||||
ID: "scriptcrawler-crawler-spider91-migrated",
|
||||
ID: "scriptcrawler-crawler-main-migrated",
|
||||
DriveID: "p115-target",
|
||||
FileID: "uploaded-id",
|
||||
FileName: "migrated.mp4",
|
||||
Title: "Migrated",
|
||||
Size: 456,
|
||||
Ext: "mp4",
|
||||
ThumbnailURL: "/p/thumb/scriptcrawler-crawler-spider91-migrated",
|
||||
ThumbnailURL: "/p/thumb/scriptcrawler-crawler-main-migrated",
|
||||
PreviewStatus: "ready",
|
||||
DurationSeconds: 34,
|
||||
PublishedAt: time.Now(),
|
||||
@@ -1071,35 +1075,32 @@ func TestHandleListCrawlersOnlyIncludesCrawlerPageScripts(t *testing.T) {
|
||||
FingerprintReady: d.FingerprintReady,
|
||||
}
|
||||
}
|
||||
if _, ok := byID["spider91-main"]; ok {
|
||||
t.Fatal("legacy spider91 drive should not be returned by crawler list")
|
||||
}
|
||||
if _, ok := byID["crawler-script-deleted"]; ok {
|
||||
t.Fatal("crawler without script_path should not be returned by crawler list")
|
||||
}
|
||||
if byID["crawler-spider91"].Kind != "scriptcrawler" {
|
||||
t.Fatalf("crawler kind = %q, want scriptcrawler", byID["crawler-spider91"].Kind)
|
||||
if byID["crawler-main"].Kind != "scriptcrawler" {
|
||||
t.Fatalf("crawler kind = %q, want scriptcrawler", byID["crawler-main"].Kind)
|
||||
}
|
||||
if byID["crawler-spider91"].Name != "91Porn" {
|
||||
t.Fatalf("crawler name = %q, want script metadata name", byID["crawler-spider91"].Name)
|
||||
if byID["crawler-main"].Name != "Demo Crawler" {
|
||||
t.Fatalf("crawler name = %q, want script metadata name", byID["crawler-main"].Name)
|
||||
}
|
||||
if byID["crawler-spider91"].Proxy != "http://127.0.0.1:7890" {
|
||||
t.Fatalf("crawler proxy = %q, want trimmed proxy", byID["crawler-spider91"].Proxy)
|
||||
if byID["crawler-main"].Proxy != "http://127.0.0.1:7890" {
|
||||
t.Fatalf("crawler proxy = %q, want trimmed proxy", byID["crawler-main"].Proxy)
|
||||
}
|
||||
if byID["crawler-spider91"].UploadDriveID != "p115-target" {
|
||||
t.Fatalf("uploadDriveId = %q, want p115-target", byID["crawler-spider91"].UploadDriveID)
|
||||
if byID["crawler-main"].UploadDriveID != "p115-target" {
|
||||
t.Fatalf("uploadDriveId = %q, want p115-target", byID["crawler-main"].UploadDriveID)
|
||||
}
|
||||
if byID["crawler-spider91"].TeaserEnabled {
|
||||
if byID["crawler-main"].TeaserEnabled {
|
||||
t.Fatal("teaserEnabled = true, want false from crawler drive")
|
||||
}
|
||||
if byID["crawler-spider91"].LastCrawlAt != 1800000000 {
|
||||
t.Fatalf("lastCrawlAt = %d, want 1800000000", byID["crawler-spider91"].LastCrawlAt)
|
||||
if byID["crawler-main"].LastCrawlAt != 1800000000 {
|
||||
t.Fatalf("lastCrawlAt = %d, want 1800000000", byID["crawler-main"].LastCrawlAt)
|
||||
}
|
||||
if byID["crawler-spider91"].TotalCrawled != 2 || byID["crawler-spider91"].LocalVideos != 1 || byID["crawler-spider91"].MigratedVideo != 1 {
|
||||
t.Fatalf("crawler counts = total %d local %d migrated %d, want 2/1/1", byID["crawler-spider91"].TotalCrawled, byID["crawler-spider91"].LocalVideos, byID["crawler-spider91"].MigratedVideo)
|
||||
if byID["crawler-main"].TotalCrawled != 2 || byID["crawler-main"].LocalVideos != 1 || byID["crawler-main"].MigratedVideo != 1 {
|
||||
t.Fatalf("crawler counts = total %d local %d migrated %d, want 2/1/1", byID["crawler-main"].TotalCrawled, byID["crawler-main"].LocalVideos, byID["crawler-main"].MigratedVideo)
|
||||
}
|
||||
if byID["crawler-spider91"].ThumbnailReady != 2 || byID["crawler-spider91"].TeaserReady != 2 || byID["crawler-spider91"].FingerprintReady != 2 {
|
||||
t.Fatalf("asset ready counts = thumb %d teaser %d fingerprint %d, want 2/2/2", byID["crawler-spider91"].ThumbnailReady, byID["crawler-spider91"].TeaserReady, byID["crawler-spider91"].FingerprintReady)
|
||||
if byID["crawler-main"].ThumbnailReady != 2 || byID["crawler-main"].TeaserReady != 2 || byID["crawler-main"].FingerprintReady != 2 {
|
||||
t.Fatalf("asset ready counts = thumb %d teaser %d fingerprint %d, want 2/2/2", byID["crawler-main"].ThumbnailReady, byID["crawler-main"].TeaserReady, byID["crawler-main"].FingerprintReady)
|
||||
}
|
||||
if _, ok := byID["onedrive-main"]; ok {
|
||||
t.Fatal("onedrive should not be returned by crawler list")
|
||||
@@ -1121,10 +1122,7 @@ func TestHandleListCrawlersOnlyIncludesCrawlerPageScripts(t *testing.T) {
|
||||
for _, d := range drives {
|
||||
driveIDs[d.ID] = true
|
||||
}
|
||||
if !driveIDs["spider91-main"] {
|
||||
t.Fatal("legacy spider91 drive should remain visible in drive list for deletion")
|
||||
}
|
||||
if driveIDs["crawler-spider91"] {
|
||||
if driveIDs["crawler-main"] {
|
||||
t.Fatal("scriptcrawler should not be returned by drive list")
|
||||
}
|
||||
}
|
||||
@@ -1144,15 +1142,15 @@ func TestHandleUpsertCrawlerRequiresScriptPath(t *testing.T) {
|
||||
|
||||
srv := &AdminServer{Catalog: cat}
|
||||
scriptPath := filepath.Join(tmp, "custom.py")
|
||||
if err := os.WriteFile(scriptPath, []byte("CRAWLER_NAME = \"91 Spider\"\n"), 0o644); err != nil {
|
||||
if err := os.WriteFile(scriptPath, []byte("CRAWLER_NAME = \"Demo Crawler\"\n"), 0o644); err != nil {
|
||||
t.Fatalf("write crawler script: %v", err)
|
||||
}
|
||||
|
||||
// 不再内置任何爬虫:没有脚本路径的保存请求必须被拒绝,
|
||||
// 旧的 builtin 字段也不再有"免脚本"特权。
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/crawlers", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"builtin": "spider91",
|
||||
"id": "crawler-main",
|
||||
"builtin": "legacy",
|
||||
"scriptPath": "",
|
||||
"targetNew": "15"
|
||||
}`))
|
||||
@@ -1164,8 +1162,8 @@ func TestHandleUpsertCrawlerRequiresScriptPath(t *testing.T) {
|
||||
|
||||
// 带脚本路径时正常保存,且请求中的 builtin 字段被忽略,不会写入凭证。
|
||||
req = httptest.NewRequest(http.MethodPost, "/admin/api/crawlers", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"builtin": "spider91",
|
||||
"id": "crawler-main",
|
||||
"builtin": "legacy",
|
||||
"scriptPath": "`+scriptPath+`",
|
||||
"targetNew": "15",
|
||||
"teaserEnabled": false
|
||||
@@ -1176,7 +1174,7 @@ func TestHandleUpsertCrawlerRequiresScriptPath(t *testing.T) {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
got, err := cat.GetDrive(ctx, "spider91-main")
|
||||
got, err := cat.GetDrive(ctx, "crawler-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get crawler drive: %v", err)
|
||||
}
|
||||
@@ -1186,7 +1184,7 @@ func TestHandleUpsertCrawlerRequiresScriptPath(t *testing.T) {
|
||||
if got.Credentials["python_path"] != "" || got.Credentials["config_json"] != "" {
|
||||
t.Fatalf("legacy hidden credentials should not be saved: %+v", got.Credentials)
|
||||
}
|
||||
if got.Name != "91 Spider" {
|
||||
if got.Name != "Demo Crawler" {
|
||||
t.Fatalf("name = %q, want script metadata name", got.Name)
|
||||
}
|
||||
if got.Credentials["script_path"] != scriptPath {
|
||||
@@ -1271,6 +1269,7 @@ func TestHandleUpsertCrawlerPersistsAndValidatesUploadDrive(t *testing.T) {
|
||||
for _, d := range []*catalog.Drive{
|
||||
{ID: "p115-target", Kind: "p115", Name: "115", RootID: "0", Credentials: map[string]string{"cookie": "x"}},
|
||||
{ID: "wopan-target", Kind: "wopan", Name: "沃盘", RootID: "0", Credentials: map[string]string{"access_token": "a", "refresh_token": "r"}},
|
||||
{ID: "guangyapan-target", Kind: "guangyapan", Name: "光鸭", RootID: "", Credentials: map[string]string{"access_token": "a", "refresh_token": "r"}},
|
||||
{ID: "local-target", Kind: "localstorage", Name: "Local", RootID: "/", Credentials: map[string]string{"path": tmp}},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
@@ -1336,6 +1335,24 @@ func TestHandleUpsertCrawlerPersistsAndValidatesUploadDrive(t *testing.T) {
|
||||
t.Fatalf("teaser callback after preserved edit = %q, want none", teaserCallbackID)
|
||||
}
|
||||
|
||||
req = httptest.NewRequest(http.MethodPost, "/admin/api/crawlers", strings.NewReader(`{
|
||||
"id": "crawler-upload",
|
||||
"scriptPath": "`+scriptPath+`",
|
||||
"uploadDriveId": "guangyapan-target"
|
||||
}`))
|
||||
rr = httptest.NewRecorder()
|
||||
srv.handleUpsertCrawler(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("guangyapan target status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
got, err = cat.GetDrive(ctx, "crawler-upload")
|
||||
if err != nil {
|
||||
t.Fatalf("get crawler after guangyapan target: %v", err)
|
||||
}
|
||||
if got.Credentials["upload_drive_id"] != "guangyapan-target" {
|
||||
t.Fatalf("upload_drive_id = %q, want guangyapan-target", got.Credentials["upload_drive_id"])
|
||||
}
|
||||
|
||||
req = httptest.NewRequest(http.MethodPost, "/admin/api/crawlers", strings.NewReader(`{
|
||||
"id": "crawler-upload",
|
||||
"scriptPath": "`+scriptPath+`",
|
||||
@@ -1929,7 +1946,8 @@ func TestHandleListDrivesIncludesGoogleDriveOnlineAPIMode(t *testing.T) {
|
||||
Name: "Google Legacy",
|
||||
RootID: "root",
|
||||
Credentials: map[string]string{
|
||||
"refresh_token": "legacy-refresh",
|
||||
"refresh_token": "legacy-refresh",
|
||||
"api_url_address": "https://openlist-api.example/googleui/renewapi",
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
@@ -1960,15 +1978,18 @@ func TestHandleListDrivesIncludesGoogleDriveOnlineAPIMode(t *testing.T) {
|
||||
}
|
||||
|
||||
var got []struct {
|
||||
ID string `json:"id"`
|
||||
GoogleDriveUseOnlineAPI bool `json:"googleDriveUseOnlineAPI"`
|
||||
ID string `json:"id"`
|
||||
GoogleDriveUseOnlineAPI bool `json:"googleDriveUseOnlineAPI"`
|
||||
GoogleDriveOpenListAPIURL string `json:"googleDriveOpenListApiUrl"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
byID := map[string]bool{}
|
||||
byAPIURL := map[string]string{}
|
||||
for _, d := range got {
|
||||
byID[d.ID] = d.GoogleDriveUseOnlineAPI
|
||||
byAPIURL[d.ID] = d.GoogleDriveOpenListAPIURL
|
||||
}
|
||||
if !byID["google-legacy"] {
|
||||
t.Fatalf("legacy google drive use_online_api = false, want true")
|
||||
@@ -1976,6 +1997,9 @@ func TestHandleListDrivesIncludesGoogleDriveOnlineAPIMode(t *testing.T) {
|
||||
if byID["google-oauth"] {
|
||||
t.Fatalf("oauth google drive use_online_api = true, want false")
|
||||
}
|
||||
if byAPIURL["google-legacy"] != "https://openlist-api.example/googleui/renewapi" {
|
||||
t.Fatalf("legacy google drive openlist api url = %q, want custom URL", byAPIURL["google-legacy"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
@@ -2452,6 +2476,52 @@ func TestHandleAdminListVideosFiltersByDriveID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleAdminListVideosDoesNotExposeCategory(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "video-1",
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
Title: "Video",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/videos", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleAdminListVideos(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
Items []map[string]any `json:"items"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(got.Items) != 1 {
|
||||
t.Fatalf("items len = %d, want 1", len(got.Items))
|
||||
}
|
||||
if _, ok := got.Items[0]["category"]; ok {
|
||||
t.Fatalf("admin video response exposed category: %#v", got.Items[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleAdminListVideosPaginates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
@@ -25,7 +25,6 @@ import (
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives/localstorage"
|
||||
"github.com/video-site/backend/internal/drives/localupload"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
)
|
||||
@@ -94,7 +93,6 @@ type VideoDTO struct {
|
||||
Dislikes int `json:"dislikes"`
|
||||
PublishedAt string `json:"publishedAt"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Category string `json:"category,omitempty"`
|
||||
}
|
||||
|
||||
type TagDTO struct {
|
||||
@@ -153,7 +151,6 @@ func (s *Server) RegisterRoutes(r chi.Router, a *auth.Authenticator) {
|
||||
// 代理路由同样需要鉴权,防止绕过
|
||||
r.Get("/p/stream/{driveID}/*", s.handleStream)
|
||||
r.Get("/p/upload/{videoID}", s.handleUploadedVideo)
|
||||
r.Get("/p/spider91/{videoID}", s.handleSpider91Video)
|
||||
r.Get("/p/preview/{videoID}", s.handlePreview)
|
||||
r.Get("/p/thumb/{videoID}", s.handleThumb)
|
||||
})
|
||||
@@ -295,7 +292,6 @@ func (s *Server) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
params := catalog.ListParams{
|
||||
Keyword: q.Get("q"),
|
||||
Tag: q.Get("tag"),
|
||||
Category: q.Get("cat"),
|
||||
Sort: sort,
|
||||
Page: page,
|
||||
PageSize: size,
|
||||
@@ -833,44 +829,6 @@ func (s *Server) handleUploadedVideo(w http.ResponseWriter, r *http.Request) {
|
||||
http.ServeFile(w, r, path)
|
||||
}
|
||||
|
||||
// handleSpider91Video 服务 spider91 drive 下载到本地的视频文件。
|
||||
// 路径形如 /p/spider91/<videoID>,videoID = "spider91-<driveID>-<sourceID>"。
|
||||
// 通过 catalog 拿到 file_id("<sourceID>.mp4"),再让 driver 解析到绝对路径并 ServeFile。
|
||||
func (s *Server) handleSpider91Video(w http.ResponseWriter, r *http.Request) {
|
||||
videoID := routeParam(r, "videoID")
|
||||
v, err := s.Catalog.GetVideo(r.Context(), videoID)
|
||||
if err != nil || v.Hidden {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if s.Proxy == nil || s.Proxy.Registry == nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
d, ok := s.Proxy.Registry.Get(v.DriveID)
|
||||
if !ok || d.Kind() != spider91.Kind {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
sd, ok := d.(*spider91.Driver)
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
path, err := sd.VideoPath(v.FileID)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid video id", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil || info.IsDir() || info.Size() == 0 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Cache-Control", "private, max-age=300")
|
||||
http.ServeFile(w, r, path)
|
||||
}
|
||||
|
||||
func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) {
|
||||
videoID := routeParam(r, "videoID")
|
||||
v, err := s.Catalog.GetVideo(r.Context(), videoID)
|
||||
@@ -949,7 +907,6 @@ func mapVideo(v *catalog.Video) VideoDTO {
|
||||
Dislikes: v.Dislikes,
|
||||
PublishedAt: v.PublishedAt.Format("2006-01-02"),
|
||||
Tags: tags,
|
||||
Category: v.Category,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -988,14 +945,6 @@ func (s *Server) videoSource(v *catalog.Video) string {
|
||||
if v.DriveID == localUploadDriveID {
|
||||
return "/p/upload/" + pathSegment(v.ID)
|
||||
}
|
||||
if s.Proxy != nil && s.Proxy.Registry != nil {
|
||||
if d, ok := s.Proxy.Registry.Get(v.DriveID); ok {
|
||||
switch d.Kind() {
|
||||
case spider91.Kind:
|
||||
return "/p/spider91/" + pathSegment(v.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
if src, ok := transcodedSource(v); ok {
|
||||
return src
|
||||
}
|
||||
@@ -1076,8 +1025,6 @@ func driveKindLabel(kind string) string {
|
||||
return "Google Drive"
|
||||
case localstorage.Kind:
|
||||
return "本地存储"
|
||||
case spider91.Kind:
|
||||
return "91 爬虫"
|
||||
default:
|
||||
return kind
|
||||
}
|
||||
|
||||
@@ -498,6 +498,68 @@ func TestHandleListLatestPrefersReadyThumbnails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListIgnoresCategoryQueryAndDoesNotExposeCategory(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*catalog.Video{
|
||||
{
|
||||
ID: "video-a",
|
||||
DriveID: "drive",
|
||||
FileID: "file-a",
|
||||
Title: "A",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "video-b",
|
||||
DriveID: "drive",
|
||||
FileID: "file-b",
|
||||
Title: "B",
|
||||
PublishedAt: now.Add(-time.Hour),
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed video %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/list?page=1&size=24&cat=alpha", nil)
|
||||
(&Server{Catalog: cat}).handleList(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
Items []map[string]any `json:"items"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if got.Total != 2 || len(got.Items) != 2 {
|
||||
t.Fatalf("response total/items = %d/%d, want 2/2", got.Total, len(got.Items))
|
||||
}
|
||||
for _, item := range got.Items {
|
||||
if _, ok := item["category"]; ok {
|
||||
t.Fatalf("list response exposed category: %#v", item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUploadVideoSavesFileVideoTagsAndQueuesPreview(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -763,7 +825,6 @@ func TestHandleTagsReturnsUnifiedTagPool(t *testing.T) {
|
||||
FileID: "file-1",
|
||||
Title: "清纯女大后入",
|
||||
Tags: []string{"后入", "女大"},
|
||||
Category: "random-category",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
|
||||
+193
-114
@@ -78,11 +78,11 @@ type Video struct {
|
||||
TranscodedFileID string `json:"transcodedFileId"`
|
||||
TranscodedSize int64 `json:"transcodedSize"`
|
||||
Views int `json:"views"`
|
||||
LastViewedAt time.Time `json:"lastViewedAt"`
|
||||
Favorites int `json:"favorites"`
|
||||
Comments int `json:"comments"`
|
||||
Likes int `json:"likes"`
|
||||
Dislikes int `json:"dislikes"`
|
||||
Category string `json:"category"`
|
||||
Hidden bool `json:"hidden"`
|
||||
Badges []string `json:"badges"`
|
||||
Description string `json:"description"`
|
||||
@@ -111,16 +111,16 @@ func (c *Catalog) UpsertVideo(ctx context.Context, v *Video) error {
|
||||
INSERT INTO videos (
|
||||
id, drive_id, file_id, file_name, content_hash, sampled_sha256, fingerprint_status, fingerprint_error, parent_id, title, author, tags,
|
||||
duration_seconds, size_bytes, ext, quality, thumbnail_url, thumbnail_status,
|
||||
preview_file_id, preview_local, preview_status,
|
||||
views, favorites, comments, likes, dislikes,
|
||||
category, hidden, badges, description, published_at, created_at, updated_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, CASE WHEN COALESCE(?, '') != '' THEN 'ready' ELSE 'pending' END,
|
||||
?, ?, ?,
|
||||
?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?
|
||||
)
|
||||
preview_file_id, preview_local, preview_status,
|
||||
views, last_viewed_at, favorites, comments, likes, dislikes,
|
||||
hidden, badges, description, published_at, created_at, updated_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, CASE WHEN COALESCE(?, '') != '' THEN 'ready' ELSE 'pending' END,
|
||||
?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?
|
||||
)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
file_name = CASE
|
||||
WHEN excluded.file_name != '' THEN excluded.file_name
|
||||
@@ -161,16 +161,15 @@ ON CONFLICT(id) DO UPDATE SET
|
||||
WHEN COALESCE(excluded.thumbnail_url, '') != '' THEN 'ready'
|
||||
ELSE videos.thumbnail_status
|
||||
END,
|
||||
category = excluded.category,
|
||||
badges = excluded.badges,
|
||||
description = excluded.description,
|
||||
badges = excluded.badges,
|
||||
description = excluded.description,
|
||||
updated_at = excluded.updated_at
|
||||
`,
|
||||
v.ID, v.DriveID, v.FileID, v.FileName, v.ContentHash, v.SampledSHA256, fingerprintStatus, v.FingerprintError, v.ParentID, v.Title, v.Author, string(tagsJSON),
|
||||
v.DurationSeconds, v.Size, v.Ext, v.Quality, v.ThumbnailURL, v.ThumbnailURL,
|
||||
v.PreviewFileID, v.PreviewLocal, nullableStatus(v.PreviewStatus),
|
||||
v.Views, v.Favorites, v.Comments, v.Likes, v.Dislikes,
|
||||
v.Category, boolToInt(v.Hidden), string(badgesJSON), v.Description,
|
||||
v.Views, unixMilliOrZero(v.LastViewedAt), v.Favorites, v.Comments, v.Likes, v.Dislikes,
|
||||
boolToInt(v.Hidden), string(badgesJSON), v.Description,
|
||||
v.PublishedAt.UnixMilli(), v.CreatedAt.UnixMilli(), v.UpdatedAt.UnixMilli(),
|
||||
)
|
||||
if err != nil {
|
||||
@@ -308,10 +307,9 @@ func (c *Catalog) ListHiddenVideos(ctx context.Context) ([]*Video, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// MigrateVideoToDrive 把 catalog 里 id=videoID 这条视频迁移到另一个 drive。
|
||||
// 用于 spider91 → PikPak 的迁移:上传成功后改写 drive_id / file_id /
|
||||
// content_hash,保留视频自身的 id(spider91-<driveID>-<sourceID>),这样
|
||||
// 关联表 (video_tags / 收藏 / 点赞) 都不需要动。
|
||||
// MigrateVideoToDrive rewrites a crawler video row after it has been uploaded
|
||||
// to another drive. The video id is preserved so tags, favorites, likes and
|
||||
// view records keep pointing at the same logical video.
|
||||
//
|
||||
// scanner 后续看到 PikPak 目录下相同 hash / file_name 的文件时,会通过
|
||||
// findDuplicate 命中本行,不会再插入重复行。
|
||||
@@ -337,8 +335,8 @@ func (c *Catalog) MigrateVideoToDrive(ctx context.Context, videoID, newDriveID,
|
||||
}
|
||||
|
||||
// ListVideosByDriveID 列出指定 drive 下所有未隐藏的视频,按 published_at 倒序。
|
||||
// 给 spider91 → 115/PikPak 迁移 worker 用:扫描 spider91 drive 下所有视频,
|
||||
// 检查哪些还有本地文件,依次上传到目标盘。
|
||||
// crawler upload worker uses this to find local crawler rows before uploading
|
||||
// them to their configured target drive.
|
||||
func (c *Catalog) ListVideosByDriveID(ctx context.Context, driveID string, limit int) ([]*Video, error) {
|
||||
if driveID == "" {
|
||||
return nil, fmt.Errorf("catalog: list videos by drive: empty drive id")
|
||||
@@ -423,9 +421,10 @@ func (c *Catalog) IncrementView(ctx context.Context, id string) (int, error) {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
now := time.Now().UnixMilli()
|
||||
res, err := tx.ExecContext(ctx,
|
||||
`UPDATE videos SET views = views + 1, updated_at = ? WHERE id = ?`,
|
||||
time.Now().UnixMilli(), id)
|
||||
`UPDATE videos SET views = views + 1, last_viewed_at = ?, updated_at = ? WHERE id = ?`,
|
||||
now, now, id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -448,9 +447,12 @@ type VideoMetaPatch struct {
|
||||
ThumbnailStatus string
|
||||
ResetThumbnailFailures bool
|
||||
DurationSeconds int
|
||||
Category string
|
||||
ContentHash string
|
||||
FileName string
|
||||
Title string
|
||||
TitleSet bool
|
||||
Author string
|
||||
AuthorSet bool
|
||||
Tags []string
|
||||
TagsSet bool
|
||||
}
|
||||
@@ -488,10 +490,6 @@ func (c *Catalog) UpdateVideoMeta(ctx context.Context, id string, p VideoMetaPat
|
||||
parts = append(parts, "duration_seconds = ?")
|
||||
args = append(args, p.DurationSeconds)
|
||||
}
|
||||
if p.Category != "" {
|
||||
parts = append(parts, "category = ?")
|
||||
args = append(args, p.Category)
|
||||
}
|
||||
if p.ContentHash != "" {
|
||||
parts = append(parts, "content_hash = ?")
|
||||
args = append(args, normalizeContentHash(p.ContentHash))
|
||||
@@ -500,6 +498,14 @@ func (c *Catalog) UpdateVideoMeta(ctx context.Context, id string, p VideoMetaPat
|
||||
parts = append(parts, "file_name = ?")
|
||||
args = append(args, p.FileName)
|
||||
}
|
||||
if p.TitleSet {
|
||||
parts = append(parts, "title = ?")
|
||||
args = append(args, p.Title)
|
||||
}
|
||||
if p.AuthorSet {
|
||||
parts = append(parts, "author = ?")
|
||||
args = append(args, p.Author)
|
||||
}
|
||||
if p.TagsSet {
|
||||
tagsJSON, _ := json.Marshal(p.Tags)
|
||||
parts = append(parts, "tags = ?")
|
||||
@@ -553,35 +559,6 @@ func (c *Catalog) IncrementThumbnailFailures(ctx context.Context, id string) (in
|
||||
return failures, nil
|
||||
}
|
||||
|
||||
// ListCategories 聚合所有 category,按视频数降序
|
||||
type CategoryStat struct {
|
||||
Category string
|
||||
Count int
|
||||
}
|
||||
|
||||
func (c *Catalog) ListCategories(ctx context.Context) ([]CategoryStat, error) {
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT COALESCE(category, '') AS c, COUNT(*) AS cnt
|
||||
FROM videos
|
||||
WHERE category IS NOT NULL AND category != ''
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
GROUP BY c
|
||||
ORDER BY cnt DESC, c ASC`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []CategoryStat
|
||||
for rows.Next() {
|
||||
var s CategoryStat
|
||||
if err := rows.Scan(&s.Category, &s.Count); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type TagStat struct {
|
||||
Label string
|
||||
Count int
|
||||
@@ -745,6 +722,29 @@ func (c *Catalog) ListVideosByDrive(ctx context.Context, driveID string) ([]*Vid
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListVideoMaintenanceCandidates returns all current catalog videos without the
|
||||
// public listing dedupe filter. Nightly maintenance needs to see duplicate rows
|
||||
// that ListVideos intentionally hides from the frontend.
|
||||
func (c *Catalog) ListVideoMaintenanceCandidates(ctx context.Context) ([]*Video, error) {
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE COALESCE(hidden, 0) = 0
|
||||
ORDER BY created_at ASC, id ASC`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) ListVideosByIDPrefix(ctx context.Context, prefix string) ([]*Video, error) {
|
||||
prefix = strings.TrimSpace(prefix)
|
||||
if prefix == "" {
|
||||
@@ -816,21 +816,6 @@ func (c *Catalog) ListVideoFileIDsByDrive(ctx context.Context, driveID string) (
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListSpider91Viewkeys 列出某个 spider91 drive 历史上爬过的所有 ID 后缀。
|
||||
// 函数名保留历史叫法;新 spider91 数据的后缀是 91 mp4 源 ID,不再是 viewkey。
|
||||
//
|
||||
// 不能再用 ListVideoFileIDsByDrive:那个只看 drive_id,但 spider91 视频
|
||||
// 一旦被 spider91migrate 迁移到 PikPak,drive_id 就变成 PikPak 了。
|
||||
//
|
||||
// 这里按 video.id 前缀 "spider91-<driveID>-" 查,即使迁移后视频也仍能被
|
||||
// 找到——id 本身会保留 "spider91-<driveID>-<sourceID>" 这个来源前缀。
|
||||
//
|
||||
// 用途:crawler 把这个集合写到 seen 文件,让 Python/Go 跳过已爬过的视频,
|
||||
// 配合 --target-new 真正凑出 N 个未爬过的视频。
|
||||
func (c *Catalog) ListSpider91Viewkeys(ctx context.Context, driveID string) ([]string, error) {
|
||||
return c.ListCrawlerSourceIDs(ctx, "spider91", driveID)
|
||||
}
|
||||
|
||||
// ListCrawlerSourceIDs lists source IDs that were already imported by a
|
||||
// crawler-like drive. It reads both videos and deleted_videos so explicit admin
|
||||
// deletions remain tombstoned for future crawler runs.
|
||||
@@ -907,10 +892,19 @@ ON CONFLICT(kind, drive_id, source_id) DO UPDATE SET
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteVideoWithTombstone records that an administrator explicitly deleted a
|
||||
// video, then removes the visible catalog row. The tombstone is used by
|
||||
// scanners/crawlers to avoid importing the same source file again.
|
||||
const DeletedVideoReasonDuplicate = "duplicate"
|
||||
|
||||
// DeleteVideoWithTombstone records that a video was removed, then removes the
|
||||
// visible catalog row. The tombstone is used by scanners/crawlers to avoid
|
||||
// importing the same source file again.
|
||||
func (c *Catalog) DeleteVideoWithTombstone(ctx context.Context, id string) error {
|
||||
return c.DeleteVideoWithTombstoneReason(ctx, id, "")
|
||||
}
|
||||
|
||||
// DeleteVideoWithTombstoneReason is the same tombstone path with an optional
|
||||
// machine reason for admin UI hints. Empty reason means user/admin initiated.
|
||||
func (c *Catalog) DeleteVideoWithTombstoneReason(ctx context.Context, id, reason string) error {
|
||||
reason = normalizeDeletedVideoReason(reason)
|
||||
tx, err := c.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -934,7 +928,7 @@ SELECT id, drive_id, file_id, COALESCE(content_hash, ''), COALESCE(file_name, ''
|
||||
}
|
||||
v.ContentHash = normalizeContentHash(v.ContentHash)
|
||||
|
||||
// 先记录这次视频关联的 tag_id,便于事务末尾清理孤儿 collection 标签。
|
||||
// 先记录这次视频关联的 tag_id,便于事务末尾清理旧版本遗留的孤儿 collection 标签。
|
||||
tagIDs, err := collectVideoTagIDs(ctx, tx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -942,16 +936,17 @@ SELECT id, drive_id, file_id, COALESCE(content_hash, ''), COALESCE(file_name, ''
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO deleted_videos (id, drive_id, file_id, content_hash, file_name, size_bytes, deleted_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO deleted_videos (id, drive_id, file_id, content_hash, file_name, size_bytes, reason, deleted_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
drive_id = excluded.drive_id,
|
||||
file_id = excluded.file_id,
|
||||
content_hash = excluded.content_hash,
|
||||
file_name = excluded.file_name,
|
||||
size_bytes = excluded.size_bytes,
|
||||
reason = excluded.reason,
|
||||
deleted_at = excluded.deleted_at`,
|
||||
v.ID, v.DriveID, v.FileID, v.ContentHash, v.FileName, v.Size, now); err != nil {
|
||||
v.ID, v.DriveID, v.FileID, v.ContentHash, v.FileName, v.Size, reason, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM video_tags WHERE video_id = ?`, id); err != nil {
|
||||
@@ -977,7 +972,7 @@ func (c *Catalog) DeleteVideo(ctx context.Context, id string) error {
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
// 先记录这次视频关联的 tag_id,便于事务末尾清理孤儿 collection 标签
|
||||
// 先记录这次视频关联的 tag_id,便于事务末尾清理旧版本遗留的孤儿 collection 标签。
|
||||
tagIDs, err := collectVideoTagIDs(ctx, tx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -994,7 +989,7 @@ func (c *Catalog) DeleteVideo(ctx context.Context, id string) error {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
|
||||
// collection 标签是 scanner 按目录名机器生成的;视频删完后若不再被引用就一起回收。
|
||||
// collection 标签来自旧版本按目录名生成的标签;视频删完后若不再被引用就一起回收。
|
||||
// system / user / auto / legacy 不在此处删除,避免破坏管理员手动维护的标签语义。
|
||||
if err := pruneOrphanCollectionTagsByID(ctx, tx, tagIDs); err != nil {
|
||||
return err
|
||||
@@ -1011,24 +1006,29 @@ type DeletedVideo struct {
|
||||
FileID string `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
Size int64 `json:"size"`
|
||||
Reason string `json:"reason"`
|
||||
DeletedAt int64 `json:"deletedAt"` // unix 毫秒
|
||||
}
|
||||
|
||||
// ListDeletedVideos 分页列出黑名单视频,按拉黑时间倒序。
|
||||
// keyword 非空时按文件名模糊匹配。
|
||||
func (c *Catalog) ListDeletedVideos(ctx context.Context, keyword string, page, size int) ([]*DeletedVideo, int, error) {
|
||||
if size <= 0 {
|
||||
size = 50
|
||||
// Keyword 非空时按文件名模糊匹配,DriveID 非空时限定来源网盘。
|
||||
func (c *Catalog) ListDeletedVideos(ctx context.Context, p ListParams) ([]*DeletedVideo, int, error) {
|
||||
if p.PageSize <= 0 {
|
||||
p.PageSize = 50
|
||||
}
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
if p.Page <= 0 {
|
||||
p.Page = 1
|
||||
}
|
||||
var where []string
|
||||
var args []any
|
||||
if kw := strings.TrimSpace(keyword); kw != "" {
|
||||
if kw := strings.TrimSpace(p.Keyword); kw != "" {
|
||||
where = append(where, "file_name LIKE ?")
|
||||
args = append(args, "%"+kw+"%")
|
||||
}
|
||||
if driveID := strings.TrimSpace(p.DriveID); driveID != "" {
|
||||
where = append(where, "drive_id = ?")
|
||||
args = append(args, driveID)
|
||||
}
|
||||
whereSQL := ""
|
||||
if len(where) > 0 {
|
||||
whereSQL = " WHERE " + strings.Join(where, " AND ")
|
||||
@@ -1039,13 +1039,13 @@ func (c *Catalog) ListDeletedVideos(ctx context.Context, keyword string, page, s
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
offset := (page - 1) * size
|
||||
offset := (p.Page - 1) * p.PageSize
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT id, COALESCE(drive_id, ''), COALESCE(file_id, ''), COALESCE(file_name, ''), COALESCE(size_bytes, 0), deleted_at
|
||||
`SELECT id, COALESCE(drive_id, ''), COALESCE(file_id, ''), COALESCE(file_name, ''), COALESCE(size_bytes, 0), COALESCE(reason, ''), deleted_at
|
||||
FROM deleted_videos`+whereSQL+`
|
||||
ORDER BY deleted_at DESC
|
||||
LIMIT ? OFFSET ?`,
|
||||
append(args, size, offset)...)
|
||||
append(args, p.PageSize, offset)...)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
@@ -1054,7 +1054,7 @@ func (c *Catalog) ListDeletedVideos(ctx context.Context, keyword string, page, s
|
||||
var out []*DeletedVideo
|
||||
for rows.Next() {
|
||||
v := &DeletedVideo{}
|
||||
if err := rows.Scan(&v.ID, &v.DriveID, &v.FileID, &v.FileName, &v.Size, &v.DeletedAt); err != nil {
|
||||
if err := rows.Scan(&v.ID, &v.DriveID, &v.FileID, &v.FileName, &v.Size, &v.Reason, &v.DeletedAt); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out = append(out, v)
|
||||
@@ -1188,6 +1188,73 @@ func (c *Catalog) FindEquivalentVideo(ctx context.Context, source *Video) (*Vide
|
||||
return scanVideo(row)
|
||||
}
|
||||
|
||||
// FindVideoBySampledFingerprint returns the earliest visible video with the
|
||||
// same file size and sampled fingerprint as source.
|
||||
func (c *Catalog) FindVideoBySampledFingerprint(ctx context.Context, source *Video) (*Video, error) {
|
||||
if source == nil || source.Size <= 0 {
|
||||
return nil, sql.ErrNoRows
|
||||
}
|
||||
sampled := normalizeContentHash(source.SampledSHA256)
|
||||
if sampled == "" {
|
||||
return nil, sql.ErrNoRows
|
||||
}
|
||||
row := c.db.QueryRowContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE id != ?
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
AND COALESCE(file_id, '') != ''
|
||||
AND size_bytes = ?
|
||||
AND COALESCE(sampled_sha256, '') != ''
|
||||
AND sampled_sha256 = ?
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT 1`,
|
||||
source.ID, source.Size, sampled)
|
||||
return scanVideo(row)
|
||||
}
|
||||
|
||||
// ListNearDuplicateVideoCandidates returns visible videos that are cheap
|
||||
// candidates for perceptual duplicate checking: same-ish duration and a ready
|
||||
// thumbnail URL. Callers are expected to apply title similarity and image SSIM.
|
||||
func (c *Catalog) ListNearDuplicateVideoCandidates(ctx context.Context, source *Video, durationToleranceSeconds, limit int) ([]*Video, error) {
|
||||
if source == nil || strings.TrimSpace(source.Title) == "" || source.DurationSeconds <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if durationToleranceSeconds < 0 {
|
||||
durationToleranceSeconds = 0
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 200
|
||||
}
|
||||
minDuration := source.DurationSeconds - durationToleranceSeconds
|
||||
if minDuration < 1 {
|
||||
minDuration = 1
|
||||
}
|
||||
maxDuration := source.DurationSeconds + durationToleranceSeconds
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE id != ?
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
AND COALESCE(file_id, '') != ''
|
||||
AND COALESCE(thumbnail_url, '') != ''
|
||||
AND COALESCE(duration_seconds, 0) BETWEEN ? AND ?
|
||||
ORDER BY ABS(duration_seconds - ?) ASC, created_at ASC, id ASC
|
||||
LIMIT ?`,
|
||||
source.ID, minDuration, maxDuration, source.DurationSeconds, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// FindEquivalentVideoOnDrive returns a visible video on driveID that represents
|
||||
// the same content as source by strong hash or sampled fingerprint.
|
||||
func (c *Catalog) FindEquivalentVideoOnDrive(ctx context.Context, source *Video, driveID string) (*Video, error) {
|
||||
@@ -1351,8 +1418,7 @@ type ListParams struct {
|
||||
Keyword string
|
||||
DriveID string
|
||||
Tag string
|
||||
Category string
|
||||
Sort string // latest | hot | week | long
|
||||
Sort string // latest | hot | recent
|
||||
ThumbnailReadyOnly bool
|
||||
PreferReadyThumbnails bool
|
||||
SkipTotal bool
|
||||
@@ -1371,9 +1437,9 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
|
||||
var where []string
|
||||
var args []any
|
||||
if p.Keyword != "" {
|
||||
where = append(where, "(title LIKE ? OR author LIKE ?)")
|
||||
where = append(where, "(title LIKE ? OR author LIKE ? OR file_name LIKE ?)")
|
||||
like := "%" + p.Keyword + "%"
|
||||
args = append(args, like, like)
|
||||
args = append(args, like, like, like)
|
||||
}
|
||||
if p.DriveID != "" {
|
||||
where = append(where, "drive_id = ?")
|
||||
@@ -1383,10 +1449,6 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
|
||||
where = append(where, videoMatchesTagLabelSQL("videos"))
|
||||
args = append(args, p.Tag)
|
||||
}
|
||||
if p.Category != "" && p.Category != "all" {
|
||||
where = append(where, "category = ?")
|
||||
args = append(args, p.Category)
|
||||
}
|
||||
if p.ThumbnailReadyOnly {
|
||||
where = append(where, "COALESCE(thumbnail_url, '') != ''")
|
||||
}
|
||||
@@ -1407,10 +1469,8 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
|
||||
case "hot":
|
||||
// 热度 = 点赞数,点赞相同按最新
|
||||
orderBy = " ORDER BY " + readyOrderPrefix + "likes DESC, published_at DESC"
|
||||
case "week":
|
||||
orderBy = " ORDER BY " + readyOrderPrefix + "likes DESC"
|
||||
case "long":
|
||||
orderBy = " ORDER BY " + readyOrderPrefix + "duration_seconds DESC"
|
||||
case "recent":
|
||||
orderBy = " ORDER BY " + readyOrderPrefix + "COALESCE(last_viewed_at, 0) DESC, published_at DESC"
|
||||
}
|
||||
|
||||
var total int
|
||||
@@ -2002,7 +2062,7 @@ func normalizeDriveRootID(kind, rootID string) string {
|
||||
return "root"
|
||||
}
|
||||
return rootID
|
||||
case "localstorage", "spider91":
|
||||
case "localstorage", "scriptcrawler":
|
||||
return "/"
|
||||
default:
|
||||
if rootID == "" {
|
||||
@@ -2202,11 +2262,11 @@ COALESCE(sampled_sha256, ''), COALESCE(fingerprint_status, 'pending'), COALESCE(
|
||||
COALESCE(parent_id, ''), title, COALESCE(author, ''), COALESCE(tags, '[]'),
|
||||
duration_seconds, size_bytes, COALESCE(ext, ''), COALESCE(quality, ''), COALESCE(thumbnail_url, ''),
|
||||
COALESCE(preview_file_id, ''), COALESCE(preview_local, ''), COALESCE(preview_status, 'pending'),
|
||||
COALESCE(transcode_status, ''), COALESCE(transcode_error, ''), COALESCE(transcoded_file_id, ''), COALESCE(transcoded_size, 0),
|
||||
views, favorites, comments, likes, dislikes,
|
||||
COALESCE(category, ''), COALESCE(hidden, 0), COALESCE(badges, '[]'), COALESCE(description, ''),
|
||||
published_at, created_at, updated_at
|
||||
`
|
||||
COALESCE(transcode_status, ''), COALESCE(transcode_error, ''), COALESCE(transcoded_file_id, ''), COALESCE(transcoded_size, 0),
|
||||
views, COALESCE(last_viewed_at, 0), favorites, comments, likes, dislikes,
|
||||
COALESCE(hidden, 0), COALESCE(badges, '[]'), COALESCE(description, ''),
|
||||
published_at, created_at, updated_at
|
||||
`
|
||||
|
||||
const activeDriveWhereSQL = `(videos.drive_id = 'local-upload'
|
||||
OR EXISTS (
|
||||
@@ -2266,7 +2326,7 @@ type rowScanner interface {
|
||||
func scanVideo(row rowScanner) (*Video, error) {
|
||||
v := &Video{}
|
||||
var tagsJSON, badgesJSON string
|
||||
var publishedAt, createdAt, updatedAt int64
|
||||
var publishedAt, createdAt, updatedAt, lastViewedAt int64
|
||||
var hidden int
|
||||
err := row.Scan(
|
||||
&v.ID, &v.DriveID, &v.FileID, &v.FileName, &v.ContentHash,
|
||||
@@ -2275,8 +2335,8 @@ func scanVideo(row rowScanner) (*Video, error) {
|
||||
&v.DurationSeconds, &v.Size, &v.Ext, &v.Quality, &v.ThumbnailURL,
|
||||
&v.PreviewFileID, &v.PreviewLocal, &v.PreviewStatus,
|
||||
&v.TranscodeStatus, &v.TranscodeError, &v.TranscodedFileID, &v.TranscodedSize,
|
||||
&v.Views, &v.Favorites, &v.Comments, &v.Likes, &v.Dislikes,
|
||||
&v.Category, &hidden, &badgesJSON, &v.Description,
|
||||
&v.Views, &lastViewedAt, &v.Favorites, &v.Comments, &v.Likes, &v.Dislikes,
|
||||
&hidden, &badgesJSON, &v.Description,
|
||||
&publishedAt, &createdAt, &updatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
@@ -2288,6 +2348,9 @@ func scanVideo(row rowScanner) (*Video, error) {
|
||||
v.PublishedAt = time.UnixMilli(publishedAt)
|
||||
v.CreatedAt = time.UnixMilli(createdAt)
|
||||
v.UpdatedAt = time.UnixMilli(updatedAt)
|
||||
if lastViewedAt > 0 {
|
||||
v.LastViewedAt = time.UnixMilli(lastViewedAt)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
@@ -2295,6 +2358,22 @@ func normalizeContentHash(hash string) string {
|
||||
return strings.ToLower(strings.TrimSpace(hash))
|
||||
}
|
||||
|
||||
func normalizeDeletedVideoReason(reason string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(reason)) {
|
||||
case DeletedVideoReasonDuplicate:
|
||||
return DeletedVideoReasonDuplicate
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func unixMilliOrZero(t time.Time) int64 {
|
||||
if t.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return t.UnixMilli()
|
||||
}
|
||||
|
||||
func boolToInt(v bool) int {
|
||||
if v {
|
||||
return 1
|
||||
|
||||
@@ -62,7 +62,7 @@ func TestUpsertDriveDefaultsRootIDByKind(t *testing.T) {
|
||||
{id: "onedrive", kind: "onedrive", want: "root"},
|
||||
{id: "googledrive", kind: "googledrive", want: "root"},
|
||||
{id: "localstorage", kind: "localstorage", want: "/"},
|
||||
{id: "spider91", kind: "spider91", want: "/"},
|
||||
{id: "scriptcrawler", kind: "scriptcrawler", want: "/"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
@@ -85,7 +85,7 @@ func TestUpsertDriveDefaultsRootIDByKind(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertDriveIgnoresRootIDForLocalStorageAndSpider91(t *testing.T) {
|
||||
func TestUpsertDriveIgnoresRootIDForLocalStorageAndScriptCrawler(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -102,7 +102,7 @@ func TestUpsertDriveIgnoresRootIDForLocalStorageAndSpider91(t *testing.T) {
|
||||
kind string
|
||||
}{
|
||||
{id: "localstorage", kind: "localstorage"},
|
||||
{id: "spider91", kind: "spider91"},
|
||||
{id: "scriptcrawler", kind: "scriptcrawler"},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: tc.id,
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestListVideoFileIDsByDrive 校验 spider91 crawler 用到的轻量 file_id 查询:
|
||||
// TestListVideoFileIDsByDrive 校验上传 worker 用到的轻量 file_id 查询:
|
||||
// - 只返回指定 drive 的 file_id;不返回其它 drive 的
|
||||
// - 跳过 file_id 为空的视频
|
||||
// - 返回顺序无要求,但每个 file_id 只出现一次
|
||||
@@ -33,20 +33,20 @@ func TestListVideoFileIDsByDrive(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
insert("spider91-A-vk001", "spider-a", "vk001.mp4")
|
||||
insert("spider91-A-vk002", "spider-a", "vk002.flv")
|
||||
insert("spider91-A-vk003", "spider-a", "vk003.mp4")
|
||||
insert("scriptcrawler-A-source001", "crawler-a", "source001.mp4")
|
||||
insert("scriptcrawler-A-source002", "crawler-a", "source002.flv")
|
||||
insert("scriptcrawler-A-source003", "crawler-a", "source003.mp4")
|
||||
// 不同 drive 的视频不应出现
|
||||
insert("quark-other-fid", "drive-quark", "abcdef")
|
||||
// 空 file_id 应被过滤
|
||||
insert("spider91-A-empty", "spider-a", "")
|
||||
insert("scriptcrawler-A-empty", "crawler-a", "")
|
||||
|
||||
got, err := cat.ListVideoFileIDsByDrive(ctx, "spider-a")
|
||||
got, err := cat.ListVideoFileIDsByDrive(ctx, "crawler-a")
|
||||
if err != nil {
|
||||
t.Fatalf("ListVideoFileIDsByDrive: %v", err)
|
||||
}
|
||||
sort.Strings(got)
|
||||
want := []string{"vk001.mp4", "vk002.flv", "vk003.mp4"}
|
||||
want := []string{"source001.mp4", "source002.flv", "source003.mp4"}
|
||||
sort.Strings(want)
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("got %d ids, want %d: got=%v", len(got), len(want), got)
|
||||
@@ -67,11 +67,11 @@ func TestListVideoFileIDsByDrive(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestListSpider91ViewkeysFindsMigratedVideos 校验:即使 spider91 视频
|
||||
// 被迁移到 PikPak(drive_id 改了),ListSpider91Viewkeys 仍能通过 video.id
|
||||
// 前缀找到这些 viewkey。这是 crawler 写 seen 文件的关键不变量,
|
||||
// 否则下一次爬取会把已爬过的 viewkey 当作"新"的再爬一遍。
|
||||
func TestListSpider91ViewkeysFindsMigratedVideos(t *testing.T) {
|
||||
// TestListCrawlerSourceIDsFindsMigratedVideos 校验:即使爬虫视频被上传迁移
|
||||
// 到目标网盘(drive_id 改了),ListCrawlerSourceIDs 仍能通过 video.id 前缀
|
||||
// 找到这些 source_id。这是 crawler 写 seen 文件的关键不变量,否则下一次
|
||||
// 爬取会把已爬过的 source_id 当作"新"的再爬一遍。
|
||||
func TestListCrawlerSourceIDsFindsMigratedVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -92,25 +92,25 @@ func TestListSpider91ViewkeysFindsMigratedVideos(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1) 仍在 spider91 drive 下的视频(未迁移)
|
||||
insert("spider91-91Spider-vk001", "91Spider", "vk001.mp4")
|
||||
// 2) 已迁移到 PikPak 的视频:drive_id 变了,但 id 仍是 spider91-91Spider-...
|
||||
insert("spider91-91Spider-vk002", "PikPak", "PIKPAK-FILE-ID-2")
|
||||
insert("spider91-91Spider-vk003", "PikPak", "PIKPAK-FILE-ID-3")
|
||||
// 3) 别的 spider91 drive 的视频,不应混进来
|
||||
insert("spider91-OtherDrive-vk999", "OtherDrive", "vk999.mp4")
|
||||
// 1) 仍在本地爬虫 drive 下的视频(未上传)
|
||||
insert("scriptcrawler-crawler-a-source001", "crawler-a", "source001.mp4")
|
||||
// 2) 已上传到目标盘的视频:drive_id 变了,但 id 仍保留 crawler 来源前缀。
|
||||
insert("scriptcrawler-crawler-a-source002", "target-drive", "TARGET-FILE-ID-2")
|
||||
insert("scriptcrawler-crawler-a-source003", "target-drive", "TARGET-FILE-ID-3")
|
||||
// 3) 别的爬虫 drive 的视频,不应混进来
|
||||
insert("scriptcrawler-other-source999", "other-crawler", "source999.mp4")
|
||||
// 4) 完全无关的视频
|
||||
insert("quark-some-fid", "drive-quark", "abc")
|
||||
|
||||
got, err := cat.ListSpider91Viewkeys(ctx, "91Spider")
|
||||
got, err := cat.ListCrawlerSourceIDs(ctx, "scriptcrawler", "crawler-a")
|
||||
if err != nil {
|
||||
t.Fatalf("ListSpider91Viewkeys: %v", err)
|
||||
t.Fatalf("ListCrawlerSourceIDs: %v", err)
|
||||
}
|
||||
sort.Strings(got)
|
||||
want := []string{"vk001", "vk002", "vk003"}
|
||||
want := []string{"source001", "source002", "source003"}
|
||||
sort.Strings(want)
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("got %d viewkeys, want %d: got=%v", len(got), len(want), got)
|
||||
t.Fatalf("got %d source ids, want %d: got=%v", len(got), len(want), got)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != want[i] {
|
||||
@@ -119,9 +119,9 @@ func TestListSpider91ViewkeysFindsMigratedVideos(t *testing.T) {
|
||||
}
|
||||
|
||||
// 不存在的 drive 返回空列表
|
||||
other, err := cat.ListSpider91Viewkeys(ctx, "no-such-drive")
|
||||
other, err := cat.ListCrawlerSourceIDs(ctx, "scriptcrawler", "no-such-drive")
|
||||
if err != nil {
|
||||
t.Fatalf("ListSpider91Viewkeys empty: %v", err)
|
||||
t.Fatalf("ListCrawlerSourceIDs empty: %v", err)
|
||||
}
|
||||
if len(other) != 0 {
|
||||
t.Fatalf("non-existent drive: got %v, want empty", other)
|
||||
@@ -138,12 +138,12 @@ func TestDeleteVideoWithTombstonePreventsReimport(t *testing.T) {
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: "spider91-91Spider-vk004",
|
||||
DriveID: "91Spider",
|
||||
FileID: "vk004.mp4",
|
||||
FileName: "vk004.mp4",
|
||||
ID: "scriptcrawler-crawler-a-source004",
|
||||
DriveID: "crawler-a",
|
||||
FileID: "source004.mp4",
|
||||
FileName: "source004.mp4",
|
||||
ContentHash: "ABCDEF",
|
||||
Title: "Deleted Spider",
|
||||
Title: "Deleted Source",
|
||||
Size: 2048,
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now,
|
||||
@@ -153,24 +153,24 @@ func TestDeleteVideoWithTombstonePreventsReimport(t *testing.T) {
|
||||
t.Fatalf("upsert: %v", err)
|
||||
}
|
||||
|
||||
if err := cat.DeleteVideoWithTombstone(ctx, "spider91-91Spider-vk004"); err != nil {
|
||||
if err := cat.DeleteVideoWithTombstone(ctx, "scriptcrawler-crawler-a-source004"); err != nil {
|
||||
t.Fatalf("delete with tombstone: %v", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "spider91-91Spider-vk004"); err != sql.ErrNoRows {
|
||||
if _, err := cat.GetVideo(ctx, "scriptcrawler-crawler-a-source004"); err != sql.ErrNoRows {
|
||||
t.Fatalf("get deleted video error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
deleted, err := cat.IsDeletedVideoCandidate(ctx, "spider91-91Spider-vk004", "91Spider", "vk004.mp4", "abcdef", "vk004.mp4", 2048)
|
||||
deleted, err := cat.IsDeletedVideoCandidate(ctx, "scriptcrawler-crawler-a-source004", "crawler-a", "source004.mp4", "abcdef", "source004.mp4", 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("check deleted candidate: %v", err)
|
||||
}
|
||||
if !deleted {
|
||||
t.Fatal("deleted candidate was not recognized")
|
||||
}
|
||||
viewkeys, err := cat.ListSpider91Viewkeys(ctx, "91Spider")
|
||||
sourceIDs, err := cat.ListCrawlerSourceIDs(ctx, "scriptcrawler", "crawler-a")
|
||||
if err != nil {
|
||||
t.Fatalf("ListSpider91Viewkeys: %v", err)
|
||||
t.Fatalf("ListCrawlerSourceIDs: %v", err)
|
||||
}
|
||||
if len(viewkeys) != 1 || viewkeys[0] != "vk004" {
|
||||
t.Fatalf("viewkeys = %#v, want [vk004]", viewkeys)
|
||||
if len(sourceIDs) != 1 || sourceIDs[0] != "source004" {
|
||||
t.Fatalf("source ids = %#v, want [source004]", sourceIDs)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestListVideosKeywordMatchesFileName(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: "p115-115-sone-089-4k",
|
||||
DriveID: "drive",
|
||||
FileID: "file-sone-089-4k",
|
||||
FileName: "www.98T.la@sone-089-4k.mp4",
|
||||
Title: "www.98T.la@sone-089",
|
||||
Author: "4k",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{
|
||||
Keyword: "www.98T.la@sone-089-4k.mp4",
|
||||
Page: 1,
|
||||
PageSize: 10,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list videos: %v", err)
|
||||
}
|
||||
if total != 1 {
|
||||
t.Fatalf("total = %d, want 1", total)
|
||||
}
|
||||
if len(items) != 1 || items[0].ID != "p115-115-sone-089-4k" {
|
||||
t.Fatalf("items = %#v, want seeded video", items)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIncrementViewStoresLastViewedAt(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: "video-1",
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
Title: "Video 1",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
if _, err := cat.IncrementView(ctx, "video-1"); err != nil {
|
||||
t.Fatalf("increment view: %v", err)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, "video-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.Views != 1 {
|
||||
t.Fatalf("views = %d, want 1", got.Views)
|
||||
}
|
||||
if got.LastViewedAt.IsZero() {
|
||||
t.Fatal("last viewed time was not stored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestListVideosRecentSortUsesLastViewedAt(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "old-view", DriveID: "drive", FileID: "old-view", Title: "Old View", PublishedAt: now.Add(3 * time.Hour), CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "recent-view", DriveID: "drive", FileID: "recent-view", Title: "Recent View", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "unviewed", DriveID: "drive", FileID: "unviewed", Title: "Unviewed", PublishedAt: now.Add(4 * time.Hour), CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
if _, err := cat.db.ExecContext(ctx,
|
||||
`UPDATE videos SET last_viewed_at = CASE id
|
||||
WHEN 'old-view' THEN ?
|
||||
WHEN 'recent-view' THEN ?
|
||||
ELSE 0
|
||||
END`,
|
||||
now.Add(-time.Hour).UnixMilli(),
|
||||
now.Add(time.Hour).UnixMilli(),
|
||||
); err != nil {
|
||||
t.Fatalf("seed last_viewed_at: %v", err)
|
||||
}
|
||||
|
||||
items, _, err := cat.ListVideos(ctx, ListParams{Sort: "recent", Page: 1, PageSize: 3})
|
||||
if err != nil {
|
||||
t.Fatalf("list recent videos: %v", err)
|
||||
}
|
||||
if len(items) != 3 {
|
||||
t.Fatalf("items = %d, want 3", len(items))
|
||||
}
|
||||
got := []string{items[0].ID, items[1].ID, items[2].ID}
|
||||
want := []string{"recent-view", "old-view", "unviewed"}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("recent order = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -27,11 +27,11 @@ CREATE TABLE IF NOT EXISTS videos (
|
||||
transcoded_file_id TEXT DEFAULT '', -- 转码产物在同一 drive 上的 fileID,播放源优先用它
|
||||
transcoded_size INTEGER DEFAULT 0,
|
||||
views INTEGER DEFAULT 0,
|
||||
last_viewed_at INTEGER DEFAULT 0,
|
||||
favorites INTEGER DEFAULT 0,
|
||||
comments INTEGER DEFAULT 0,
|
||||
likes INTEGER DEFAULT 0,
|
||||
dislikes INTEGER DEFAULT 0,
|
||||
category TEXT,
|
||||
hidden INTEGER DEFAULT 0, -- 1 = hidden from public display
|
||||
tags_manual INTEGER DEFAULT 0, -- 1 = user explicitly curated tags
|
||||
badges TEXT, -- JSON array
|
||||
@@ -74,7 +74,7 @@ CREATE TABLE IF NOT EXISTS deleted_tags (
|
||||
deleted_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- 管理员显式删除过的视频。用于防止后续扫描 / spider91 爬虫把同一个源文件
|
||||
-- 管理员显式删除过的视频。用于防止后续扫描 / 爬虫把同一个源文件
|
||||
-- 再次入库;不代表原始云盘文件已被删除。
|
||||
CREATE TABLE IF NOT EXISTS deleted_videos (
|
||||
id TEXT PRIMARY KEY,
|
||||
@@ -83,6 +83,7 @@ CREATE TABLE IF NOT EXISTS deleted_videos (
|
||||
content_hash TEXT NOT NULL DEFAULT '',
|
||||
file_name TEXT NOT NULL DEFAULT '',
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
reason TEXT NOT NULL DEFAULT '',
|
||||
deleted_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
@@ -114,7 +115,7 @@ CREATE INDEX IF NOT EXISTS idx_crawler_seen_sources_drive
|
||||
-- 网盘账户
|
||||
CREATE TABLE IF NOT EXISTS drives (
|
||||
id TEXT PRIMARY KEY,
|
||||
kind TEXT NOT NULL, -- quark / p115 / p123 / pikpak / wopan / guangyapan / onedrive / googledrive / localstorage / spider91
|
||||
kind TEXT NOT NULL, -- quark / p115 / p123 / pikpak / wopan / guangyapan / onedrive / googledrive / localstorage / scriptcrawler
|
||||
name TEXT NOT NULL,
|
||||
root_id TEXT NOT NULL DEFAULT '0',
|
||||
scan_root_id TEXT, -- deprecated: 扫描起点固定等于 root_id
|
||||
|
||||
+193
-208
@@ -66,6 +66,9 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "thumbnail_failures", "INTEGER DEFAULT 0"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "last_viewed_at", "INTEGER DEFAULT 0"); err != nil {
|
||||
return err
|
||||
}
|
||||
// videos.transcode_*:浏览器兼容性转码状态。
|
||||
// status:''=未检测 / pending=已入队 / ready=已转码 / skipped=检测后无需转码 / failed=失败。
|
||||
// transcoded_file_id 指向转码产物在同一 drive 上的 fileID,播放源优先使用它。
|
||||
@@ -81,6 +84,12 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "transcoded_size", "INTEGER DEFAULT 0"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.dropColumnIfExists(ctx, "videos", "category"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.ensureBaseVideoIndexes(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
// drives.teaser_enabled:每盘预览视频开关,替代旧的全局 preview.enabled。
|
||||
// 升级路径:直接让 ALTER TABLE 的 DEFAULT 1 兜底 —— 每个现存 drive 都默认开启,
|
||||
// 不读旧的 settings.preview.enabled 字段。这样老用户即便之前关过全局开关,
|
||||
@@ -102,10 +111,14 @@ CREATE TABLE IF NOT EXISTS deleted_videos (
|
||||
content_hash TEXT NOT NULL DEFAULT '',
|
||||
file_name TEXT NOT NULL DEFAULT '',
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
reason TEXT NOT NULL DEFAULT '',
|
||||
deleted_at INTEGER NOT NULL
|
||||
)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "deleted_videos", "reason", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.syncDriveScanRootIDToRootID(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -145,6 +158,9 @@ CREATE TABLE IF NOT EXISTS deleted_videos (
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_visible_pub ON videos(COALESCE(hidden, 0), published_at DESC)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_last_viewed ON videos(last_viewed_at DESC)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_file_name_size ON videos(file_name, size_bytes)`); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -169,9 +185,6 @@ CREATE TABLE IF NOT EXISTS deleted_videos (
|
||||
if err := c.collapseAVCodeTags(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.createCollectionTagsFromCategories(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.classifySystemTags(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -181,7 +194,7 @@ CREATE TABLE IF NOT EXISTS deleted_videos (
|
||||
if err := c.clearRemoteP123ThumbnailsOnce(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.clearRemoteNonSpider91Thumbnails(ctx); err != nil {
|
||||
if err := c.clearRemoteThumbnails(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.hideZeroSizeVideosFromKnownDrives(ctx); err != nil {
|
||||
@@ -198,6 +211,172 @@ func (c *Catalog) addColumnIfMissing(ctx context.Context, table, column, definit
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) dropColumnIfExists(ctx context.Context, table, column string) error {
|
||||
rows, err := c.db.QueryContext(ctx, `PRAGMA table_info(`+table+`)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
found := false
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var name, typ string
|
||||
var notNull int
|
||||
var defaultValue any
|
||||
var pk int
|
||||
if err := rows.Scan(&cid, &name, &typ, ¬Null, &defaultValue, &pk); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.EqualFold(name, column) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return nil
|
||||
}
|
||||
if _, err = c.db.ExecContext(ctx, `ALTER TABLE `+table+` DROP COLUMN `+column); err == nil {
|
||||
return nil
|
||||
}
|
||||
if table == "videos" && strings.EqualFold(column, "category") {
|
||||
log.Printf("[catalog] native drop column videos.category failed, rebuilding videos table without category: %v", err)
|
||||
return c.rebuildVideosTableWithoutCategory(ctx)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) ensureBaseVideoIndexes(ctx context.Context) error {
|
||||
for _, stmt := range []string{
|
||||
`CREATE INDEX IF NOT EXISTS idx_videos_drive ON videos(drive_id, file_id)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_videos_pub ON videos(published_at DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_videos_views ON videos(views DESC)`,
|
||||
} {
|
||||
if _, err := c.db.ExecContext(ctx, stmt); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var currentVideoColumnNames = []string{
|
||||
"id",
|
||||
"drive_id",
|
||||
"file_id",
|
||||
"file_name",
|
||||
"content_hash",
|
||||
"sampled_sha256",
|
||||
"fingerprint_status",
|
||||
"fingerprint_error",
|
||||
"parent_id",
|
||||
"title",
|
||||
"author",
|
||||
"tags",
|
||||
"duration_seconds",
|
||||
"size_bytes",
|
||||
"ext",
|
||||
"quality",
|
||||
"thumbnail_url",
|
||||
"thumbnail_status",
|
||||
"thumbnail_failures",
|
||||
"preview_file_id",
|
||||
"preview_local",
|
||||
"preview_status",
|
||||
"transcode_status",
|
||||
"transcode_error",
|
||||
"transcoded_file_id",
|
||||
"transcoded_size",
|
||||
"views",
|
||||
"last_viewed_at",
|
||||
"favorites",
|
||||
"comments",
|
||||
"likes",
|
||||
"dislikes",
|
||||
"hidden",
|
||||
"tags_manual",
|
||||
"badges",
|
||||
"description",
|
||||
"published_at",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
}
|
||||
|
||||
const createVideosWithoutCategorySQL = `
|
||||
CREATE TABLE videos_category_drop_new (
|
||||
id TEXT PRIMARY KEY,
|
||||
drive_id TEXT NOT NULL,
|
||||
file_id TEXT NOT NULL,
|
||||
file_name TEXT DEFAULT '',
|
||||
content_hash TEXT DEFAULT '',
|
||||
sampled_sha256 TEXT DEFAULT '',
|
||||
fingerprint_status TEXT DEFAULT 'pending',
|
||||
fingerprint_error TEXT DEFAULT '',
|
||||
parent_id TEXT,
|
||||
title TEXT NOT NULL,
|
||||
author TEXT,
|
||||
tags TEXT,
|
||||
duration_seconds INTEGER DEFAULT 0,
|
||||
size_bytes INTEGER DEFAULT 0,
|
||||
ext TEXT,
|
||||
quality TEXT,
|
||||
thumbnail_url TEXT,
|
||||
thumbnail_status TEXT DEFAULT 'pending',
|
||||
thumbnail_failures INTEGER DEFAULT 0,
|
||||
preview_file_id TEXT,
|
||||
preview_local TEXT,
|
||||
preview_status TEXT DEFAULT 'pending',
|
||||
transcode_status TEXT DEFAULT '',
|
||||
transcode_error TEXT DEFAULT '',
|
||||
transcoded_file_id TEXT DEFAULT '',
|
||||
transcoded_size INTEGER DEFAULT 0,
|
||||
views INTEGER DEFAULT 0,
|
||||
last_viewed_at INTEGER DEFAULT 0,
|
||||
favorites INTEGER DEFAULT 0,
|
||||
comments INTEGER DEFAULT 0,
|
||||
likes INTEGER DEFAULT 0,
|
||||
dislikes INTEGER DEFAULT 0,
|
||||
hidden INTEGER DEFAULT 0,
|
||||
tags_manual INTEGER DEFAULT 0,
|
||||
badges TEXT,
|
||||
description TEXT,
|
||||
published_at INTEGER NOT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
)`
|
||||
|
||||
func (c *Catalog) rebuildVideosTableWithoutCategory(ctx context.Context) error {
|
||||
tx, err := c.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
if _, err := tx.ExecContext(ctx, `DROP TABLE IF EXISTS videos_category_drop_new`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, createVideosWithoutCategorySQL); err != nil {
|
||||
return err
|
||||
}
|
||||
cols := strings.Join(currentVideoColumnNames, ", ")
|
||||
if _, err := tx.ExecContext(ctx,
|
||||
`INSERT INTO videos_category_drop_new (`+cols+`) SELECT `+cols+` FROM videos`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DROP TABLE videos`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `ALTER TABLE videos_category_drop_new RENAME TO videos`); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// addColumnIfMissingReportNew 与 addColumnIfMissing 同步,但额外返回 added=true 表示
|
||||
// 本次确实创建了新列(即旧 schema 缺这列),方便调用方仅在迁移路径里补做一次性
|
||||
// 数据初始化(如把全局 setting 同步到新 per-drive 字段)。
|
||||
@@ -381,10 +560,9 @@ func (c *Catalog) clearRemoteP123ThumbnailsOnce(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Catalog) clearRemoteNonSpider91Thumbnails(ctx context.Context) error {
|
||||
// 非 91Spider 视频不再使用网盘侧返回的远程缩略图。清空历史 http/https
|
||||
// thumbnail_url 后,封面 worker 会重新从视频中间帧生成本地 /p/thumb/<id>。
|
||||
// 91Spider 的封面是爬虫下载后保存到本地 /p/thumb/<id>,不受这条规则影响。
|
||||
func (c *Catalog) clearRemoteThumbnails(ctx context.Context) error {
|
||||
// 不再使用网盘侧返回的远程缩略图。清空历史 http/https thumbnail_url 后,
|
||||
// 封面 worker 会重新从视频中间帧生成本地 /p/thumb/<id>。
|
||||
res, err := c.db.ExecContext(ctx, `
|
||||
UPDATE videos
|
||||
SET thumbnail_url = '',
|
||||
@@ -395,18 +573,12 @@ UPDATE videos
|
||||
lower(COALESCE(thumbnail_url, '')) LIKE 'http://%'
|
||||
OR lower(COALESCE(thumbnail_url, '')) LIKE 'https://%'
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM drives
|
||||
WHERE drives.id = videos.drive_id
|
||||
AND drives.kind = 'spider91'
|
||||
)
|
||||
`, time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected > 0 {
|
||||
log.Printf("[catalog] cleared %d remote non-91Spider thumbnail(s) for local regeneration", affected)
|
||||
log.Printf("[catalog] cleared %d remote thumbnail(s) for local regeneration", affected)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -494,61 +666,6 @@ WHERE COALESCE(tags, '') NOT IN ('', '[]', 'null')
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Catalog) createCollectionTagsFromCategories(ctx context.Context) error {
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT category, COUNT(*) FROM videos
|
||||
WHERE COALESCE(category, '') != ''
|
||||
GROUP BY category`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type categoryStat struct {
|
||||
category string
|
||||
count int
|
||||
}
|
||||
var categories []categoryStat
|
||||
for rows.Next() {
|
||||
var stat categoryStat
|
||||
if err := rows.Scan(&stat.category, &stat.count); err != nil {
|
||||
return err
|
||||
}
|
||||
categories = append(categories, stat)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, stat := range categories {
|
||||
if isAVCodePollutedLabel(stat.category) {
|
||||
if _, err := c.ensureTag(ctx, avTagLabel, fixedtags.AliasesFor(avTagLabel), "system"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addTagToVideosByCategory(ctx, stat.category, avTagLabel, "auto"); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if stat.count < 3 {
|
||||
continue
|
||||
}
|
||||
if !LooksLikeCollectionTag(stat.category) {
|
||||
continue
|
||||
}
|
||||
if c.tagDeleted(ctx, stat.category) {
|
||||
continue
|
||||
}
|
||||
if _, err := c.ensureTag(ctx, stat.category, nil, "collection"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addCollectionTagToVideos(ctx, stat.category); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Catalog) CreateTagAndClassify(ctx context.Context, label string, aliases []string, source string) (int, error) {
|
||||
tag, err := c.ensureTag(ctx, label, aliases, source)
|
||||
if err != nil {
|
||||
@@ -842,41 +959,6 @@ func (c *Catalog) MatchTags(ctx context.Context, text string) ([]string, error)
|
||||
return sortLabelsByTagOrder(tags, uniqueStrings(out)), nil
|
||||
}
|
||||
|
||||
func (c *Catalog) EnsureCollectionTag(ctx context.Context, label string) (string, bool, error) {
|
||||
label = cleanTagLabel(label)
|
||||
if isAVCodePollutedLabel(label) {
|
||||
if _, err := c.ensureTag(ctx, avTagLabel, fixedtags.AliasesFor(avTagLabel), "system"); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
if err := c.addTagToVideosByCategory(ctx, label, avTagLabel, "auto"); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return avTagLabel, true, nil
|
||||
}
|
||||
if !LooksLikeCollectionTag(label) {
|
||||
return "", false, nil
|
||||
}
|
||||
if c.tagDeleted(ctx, label) {
|
||||
return "", false, nil
|
||||
}
|
||||
if !c.tagExists(ctx, label) {
|
||||
count, err := c.categoryVideoCount(ctx, label)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
if count < 2 {
|
||||
return "", false, nil
|
||||
}
|
||||
}
|
||||
if _, err := c.ensureTag(ctx, label, nil, "collection"); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
if err := c.addCollectionTagToVideos(ctx, label); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return label, true, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) ensureTag(ctx context.Context, label string, aliases []string, source string) (Tag, error) {
|
||||
label = cleanTagLabel(label)
|
||||
if label == "" {
|
||||
@@ -929,7 +1011,7 @@ func (c *Catalog) classifyTag(ctx context.Context, tag Tag) (int, error) {
|
||||
return 0, err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT id, title, COALESCE(author, ''), COALESCE(category, ''), COALESCE(tags_manual, 0)
|
||||
SELECT id, title, COALESCE(author, ''), COALESCE(tags_manual, 0)
|
||||
FROM videos`)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@@ -938,15 +1020,15 @@ FROM videos`)
|
||||
|
||||
classified := 0
|
||||
for rows.Next() {
|
||||
var videoID, title, author, category string
|
||||
var videoID, title, author string
|
||||
var manual int
|
||||
if err := rows.Scan(&videoID, &title, &author, &category, &manual); err != nil {
|
||||
if err := rows.Scan(&videoID, &title, &author, &manual); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if manual == 1 {
|
||||
continue
|
||||
}
|
||||
matcher := normalizeTagText(title + " " + author + " " + category)
|
||||
matcher := normalizeTagText(title + " " + author)
|
||||
if !matcher.contains(tag.Label) {
|
||||
matchedAlias := false
|
||||
for _, alias := range tag.Aliases {
|
||||
@@ -1078,54 +1160,6 @@ func (c *Catalog) insertVideoTag(ctx context.Context, videoID string, tagID int6
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) addCollectionTagToVideos(ctx context.Context, category string) error {
|
||||
return c.addTagToVideosByCategory(ctx, category, category, "auto")
|
||||
}
|
||||
|
||||
func (c *Catalog) addTagToVideosByCategory(ctx context.Context, category, label, source string) error {
|
||||
tag, err := c.getTagByLabel(ctx, label)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT v.id
|
||||
FROM videos v
|
||||
WHERE v.category = ?
|
||||
AND COALESCE(v.tags_manual, 0) = 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
WHERE vt.video_id = v.id
|
||||
AND vt.tag_id = ?
|
||||
)`, category, tag.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var videoIDs []string
|
||||
for rows.Next() {
|
||||
var videoID string
|
||||
if err := rows.Scan(&videoID); err != nil {
|
||||
return err
|
||||
}
|
||||
videoIDs = append(videoIDs, videoID)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, videoID := range videoIDs {
|
||||
if err := c.insertVideoTag(ctx, videoID, tag.ID, source); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Catalog) collapseAVCodeTags(ctx context.Context) error {
|
||||
if _, err := c.ensureTag(ctx, avTagLabel, fixedtags.AliasesFor(avTagLabel), "system"); err != nil {
|
||||
return err
|
||||
@@ -1315,12 +1349,6 @@ func (c *Catalog) restoreDeletedTag(ctx context.Context, label string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) categoryVideoCount(ctx context.Context, category string) (int, error) {
|
||||
var count int
|
||||
err := c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM videos WHERE category = ?`, category).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
func (c *Catalog) getTagByLabelTx(ctx context.Context, tx *sql.Tx, label string) (Tag, error) {
|
||||
row := tx.QueryRowContext(ctx,
|
||||
`SELECT id, label, aliases, source, 0 FROM tags WHERE label = ? COLLATE NOCASE`,
|
||||
@@ -1470,46 +1498,6 @@ func isShortASCIIWord(s string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func LooksLikeCollectionTag(label string) bool {
|
||||
label = cleanTagLabel(label)
|
||||
if label == "" {
|
||||
return false
|
||||
}
|
||||
if isAVCodePollutedLabel(label) {
|
||||
return false
|
||||
}
|
||||
runes := []rune(label)
|
||||
if len(runes) < 2 || len(runes) > 24 {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(label)
|
||||
blocked := map[string]bool{
|
||||
"v": true, "pv": true, "my pack": true, "my upload": true,
|
||||
"视频": true, "视频1": true, "第一直播": true, "男人必备": true,
|
||||
"瑟女聚集地": true, "成人色游": true, "ai女友": true,
|
||||
}
|
||||
if blocked[lower] {
|
||||
return false
|
||||
}
|
||||
hasLetter := false
|
||||
for _, r := range label {
|
||||
if unicode.IsLetter(r) {
|
||||
hasLetter = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasLetter {
|
||||
return false
|
||||
}
|
||||
for _, r := range label {
|
||||
switch r {
|
||||
case ',', '。', '!', '?', ';', '、', ':', '~', '~':
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func IsAVCode(label string) bool {
|
||||
label = cleanTagLabel(label)
|
||||
if label == "" {
|
||||
@@ -1591,9 +1579,7 @@ func sortLabelsByTagOrder(tags []Tag, labels []string) []string {
|
||||
return labels
|
||||
}
|
||||
|
||||
// pruneOrphanCollectionTags 删除所有 source='collection' 且不再被任何 video_tags 引用的标签。
|
||||
// 在 migrate 末尾调用,相当于启动时自愈:之前 DeleteVideo 没顺带清理留下的孤儿,会在重启时被收回。
|
||||
// 只动 collection:system 是固定标签需保留;user 是管理员手动建的;auto/legacy 默认有视频在引用。
|
||||
// pruneOrphanCollectionTags 删除旧版本生成的 source='collection' 孤儿标签。
|
||||
func (c *Catalog) pruneOrphanCollectionTags(ctx context.Context) error {
|
||||
_, err := c.db.ExecContext(ctx, `
|
||||
DELETE FROM tags
|
||||
@@ -1602,8 +1588,7 @@ DELETE FROM tags
|
||||
return err
|
||||
}
|
||||
|
||||
// pruneOrphanCollectionTagsByID 在事务里检查一组候选 tag_id,删除其中
|
||||
// source='collection' 且已经没有视频引用的标签。供 DeleteVideo 调用。
|
||||
// pruneOrphanCollectionTagsByID 在事务里检查并删除旧版本生成的孤儿 collection 标签。
|
||||
func pruneOrphanCollectionTagsByID(ctx context.Context, tx *sql.Tx, tagIDs []int64) error {
|
||||
for _, tagID := range tagIDs {
|
||||
var src string
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -136,7 +137,6 @@ func TestCreateTagAndClassifyAddsTagToMatchingExistingVideos(t *testing.T) {
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
Title: "清纯短发合集",
|
||||
Category: "普通目录",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
@@ -148,7 +148,6 @@ func TestCreateTagAndClassifyAddsTagToMatchingExistingVideos(t *testing.T) {
|
||||
DriveID: "drive",
|
||||
FileID: "file-2",
|
||||
Title: "普通标题",
|
||||
Category: "普通目录",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
@@ -232,52 +231,6 @@ func TestDeleteTagRemovesTagFromVideos(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteTagSuppressesAutomaticCollectionRecreation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, id := range []string{"video-1", "video-2"} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: "合集视频",
|
||||
Category: "sunny",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
if label, ok, err := cat.EnsureCollectionTag(ctx, "sunny"); err != nil || !ok || label != "sunny" {
|
||||
t.Fatalf("ensure collection = %q, %v, %v; want sunny true nil", label, ok, err)
|
||||
}
|
||||
tag := mustTagByLabel(t, ctx, cat, "sunny")
|
||||
if _, err := cat.DeleteTag(ctx, tag.ID); err != nil {
|
||||
t.Fatalf("delete tag: %v", err)
|
||||
}
|
||||
|
||||
if label, ok, err := cat.EnsureCollectionTag(ctx, "sunny"); err != nil || ok || label != "" {
|
||||
t.Fatalf("ensure deleted collection = %q, %v, %v; want empty false nil", label, ok, err)
|
||||
}
|
||||
for _, tag := range mustListTags(t, ctx, cat) {
|
||||
if tag.Label == "sunny" {
|
||||
t.Fatal("deleted collection tag was recreated automatically")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateTagAndClassifyRestoresDeletedTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
@@ -343,13 +296,13 @@ func TestEnsureTagForVideoIDPrefixBackfillsSourceTag(t *testing.T) {
|
||||
id string
|
||||
manual bool
|
||||
}{
|
||||
{id: "spider91-91-spider-1200001"},
|
||||
{id: "spider91-91-spider-1200002", manual: true},
|
||||
{id: "spider91-other-1200003"},
|
||||
{id: "scriptcrawler-crawler-a-source001"},
|
||||
{id: "scriptcrawler-crawler-a-source002", manual: true},
|
||||
{id: "scriptcrawler-other-source003"},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: seed.id,
|
||||
DriveID: "91-spider",
|
||||
DriveID: "crawler-a",
|
||||
FileID: seed.id + ".mp4",
|
||||
Title: "legacy title without source text",
|
||||
PublishedAt: now,
|
||||
@@ -365,28 +318,28 @@ func TestEnsureTagForVideoIDPrefixBackfillsSourceTag(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
added, err := cat.EnsureTagForVideoIDPrefix(ctx, "spider91-91-spider-", "91porn", nil, "system")
|
||||
added, err := cat.EnsureTagForVideoIDPrefix(ctx, "scriptcrawler-crawler-a-", "crawler-tag", nil, "system")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure prefix tag: %v", err)
|
||||
}
|
||||
if added != 1 {
|
||||
t.Fatalf("added = %d, want 1", added)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, "spider91-91-spider-1200001")
|
||||
got, err := cat.GetVideo(ctx, "scriptcrawler-crawler-a-source001")
|
||||
if err != nil {
|
||||
t.Fatalf("get tagged video: %v", err)
|
||||
}
|
||||
if !sameStrings(got.Tags, []string{"91porn"}) {
|
||||
t.Fatalf("tagged video tags = %#v, want 91porn", got.Tags)
|
||||
if !sameStrings(got.Tags, []string{"crawler-tag"}) {
|
||||
t.Fatalf("tagged video tags = %#v, want crawler-tag", got.Tags)
|
||||
}
|
||||
manual, err := cat.GetVideo(ctx, "spider91-91-spider-1200002")
|
||||
manual, err := cat.GetVideo(ctx, "scriptcrawler-crawler-a-source002")
|
||||
if err != nil {
|
||||
t.Fatalf("get manual video: %v", err)
|
||||
}
|
||||
if len(manual.Tags) != 0 {
|
||||
t.Fatalf("manual video tags = %#v, want unchanged", manual.Tags)
|
||||
}
|
||||
other, err := cat.GetVideo(ctx, "spider91-other-1200003")
|
||||
other, err := cat.GetVideo(ctx, "scriptcrawler-other-source003")
|
||||
if err != nil {
|
||||
t.Fatalf("get other prefix video: %v", err)
|
||||
}
|
||||
@@ -486,7 +439,6 @@ func TestMigrateDoesNotRewriteAlreadySyncedVideoTags(t *testing.T) {
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: "巨乳后入合集",
|
||||
Category: "Better Call Saul S03",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
@@ -585,6 +537,25 @@ CREATE TABLE videos (
|
||||
)`); err != nil {
|
||||
t.Fatalf("create legacy videos table: %v", err)
|
||||
}
|
||||
nowMillis := time.Now().UnixMilli()
|
||||
if _, err := db.Exec(`
|
||||
INSERT INTO videos (
|
||||
id, drive_id, file_id, content_hash, parent_id, title, author, tags,
|
||||
duration_seconds, size_bytes, ext, quality, thumbnail_url, preview_file_id,
|
||||
preview_local, preview_status, views, favorites, comments, likes, dislikes,
|
||||
category, hidden, tags_manual, badges, description, published_at, created_at, updated_at
|
||||
) VALUES (
|
||||
'legacy-video', 'drive', 'file-legacy', 'hash-legacy', 'parent-1', 'Legacy Video', 'Legacy Author', '["旧标签"]',
|
||||
180, 1024, 'mp4', 'HD', '/thumb.jpg', 'preview-file',
|
||||
'/preview.mp4', 'ready', 7, 1, 2, 3, 4,
|
||||
'legacy-category', 0, 0, '["精选"]', 'legacy description', ?, ?, ?
|
||||
)`,
|
||||
nowMillis, nowMillis, nowMillis); err != nil {
|
||||
t.Fatalf("insert legacy video: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`CREATE INDEX idx_legacy_videos_category ON videos(category)`); err != nil {
|
||||
t.Fatalf("create legacy category index: %v", err)
|
||||
}
|
||||
if err := db.Close(); err != nil {
|
||||
t.Fatalf("close raw db: %v", err)
|
||||
}
|
||||
@@ -603,6 +574,45 @@ CREATE TABLE videos (
|
||||
if err := cat.db.QueryRow(`SELECT COALESCE(file_name, '') FROM videos LIMIT 1`).Scan(&fileNameDefault); err != nil && err != sql.ErrNoRows {
|
||||
t.Fatalf("query migrated file_name column: %v", err)
|
||||
}
|
||||
if fileNameDefault != "" {
|
||||
t.Fatalf("file_name default = %q, want empty", fileNameDefault)
|
||||
}
|
||||
if hasColumn(t, cat, "videos", "category") {
|
||||
t.Fatal("legacy category column was not dropped")
|
||||
}
|
||||
if indexExists(t, cat, "idx_legacy_videos_category") {
|
||||
t.Fatal("legacy category index was not dropped")
|
||||
}
|
||||
for _, index := range []string{"idx_videos_drive", "idx_videos_pub", "idx_videos_views"} {
|
||||
if !indexExists(t, cat, index) {
|
||||
t.Fatalf("base video index %s was not recreated", index)
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
got, err := cat.GetVideo(ctx, "legacy-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get migrated legacy video: %v", err)
|
||||
}
|
||||
if got.Title != "Legacy Video" || got.Author != "Legacy Author" || got.Views != 7 {
|
||||
t.Fatalf("migrated video lost data: %#v", got)
|
||||
}
|
||||
if !sameStrings(got.Tags, []string{"旧标签"}) {
|
||||
t.Fatalf("migrated video tags = %#v, want legacy tag preserved", got.Tags)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: "new-video",
|
||||
DriveID: "drive",
|
||||
FileID: "file-new",
|
||||
Title: "New Video",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert after migration: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetManualVideoTagsRejectsUnknownLabels(t *testing.T) {
|
||||
@@ -706,31 +716,6 @@ func TestCreateTagAndClassifyMapsAVCodeLabelToAV(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLooksLikeCollectionTagRejectsAVCodes(t *testing.T) {
|
||||
cases := []string{
|
||||
"DASS-499-C",
|
||||
"dass-499-c",
|
||||
"ADN-778",
|
||||
"SONE-247-C",
|
||||
"JUQ-502-UC",
|
||||
"ABF-032",
|
||||
"SSIS-233",
|
||||
"MIDA-607",
|
||||
"cc-1750027",
|
||||
"FC2-PPV-74663555",
|
||||
"ADN-778-FHD(1)",
|
||||
"ADN-778-中文字幕",
|
||||
"[44x.me]idbd-786",
|
||||
"NTRH-018_FHD_CH",
|
||||
"390JAC-233",
|
||||
}
|
||||
for _, label := range cases {
|
||||
if LooksLikeCollectionTag(label) {
|
||||
t.Fatalf("LooksLikeCollectionTag(%q) = true, want false", label)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateCollapsesAVCodeTagsIntoAV(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
@@ -759,7 +744,6 @@ func TestMigrateCollapsesAVCodeTagsIntoAV(t *testing.T) {
|
||||
FileID: seed.id,
|
||||
Title: seed.label + " sample",
|
||||
Tags: []string{seed.label},
|
||||
Category: seed.label,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
@@ -804,7 +788,7 @@ func TestMigrateCollapsesAVCodeTagsIntoAV(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateClearsRemoteNonSpiderThumbnailURLs(t *testing.T) {
|
||||
func TestMigrateClearsRemoteThumbnailURLs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -848,14 +832,14 @@ func TestMigrateClearsRemoteNonSpiderThumbnailURLs(t *testing.T) {
|
||||
t.Fatalf("seed pikpak: %v", err)
|
||||
}
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: "spider91-main",
|
||||
Kind: "spider91",
|
||||
Name: "91Spider",
|
||||
RootID: "root",
|
||||
ID: "crawler-main",
|
||||
Kind: "scriptcrawler",
|
||||
Name: "Crawler",
|
||||
RootID: "/",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed spider91: %v", err)
|
||||
t.Fatalf("seed crawler: %v", err)
|
||||
}
|
||||
|
||||
videos := []*Video{
|
||||
@@ -895,11 +879,18 @@ func TestMigrateClearsRemoteNonSpiderThumbnailURLs(t *testing.T) {
|
||||
ThumbnailURL: "/p/thumb/p123-local-thumb-video",
|
||||
},
|
||||
{
|
||||
ID: "spider91-local-thumb-video",
|
||||
DriveID: "spider91-main",
|
||||
ID: "scriptcrawler-crawler-main-local-thumb",
|
||||
DriveID: "crawler-main",
|
||||
FileID: "file-6",
|
||||
Title: "91Spider local thumb",
|
||||
ThumbnailURL: "/p/thumb/spider91-local-thumb-video",
|
||||
Title: "Crawler local thumb",
|
||||
ThumbnailURL: "/p/thumb/scriptcrawler-crawler-main-local-thumb",
|
||||
},
|
||||
{
|
||||
ID: "scriptcrawler-crawler-main-remote-thumb",
|
||||
DriveID: "crawler-main",
|
||||
FileID: "file-7",
|
||||
Title: "Crawler remote thumb",
|
||||
ThumbnailURL: "https://example.invalid/crawler-thumb.jpg",
|
||||
},
|
||||
}
|
||||
for _, v := range videos {
|
||||
@@ -962,12 +953,20 @@ func TestMigrateClearsRemoteNonSpiderThumbnailURLs(t *testing.T) {
|
||||
t.Fatalf("p123 local thumbnail = %q, want preserved", p123Local.ThumbnailURL)
|
||||
}
|
||||
|
||||
spider91Local, err := cat.GetVideo(ctx, "spider91-local-thumb-video")
|
||||
crawlerLocal, err := cat.GetVideo(ctx, "scriptcrawler-crawler-main-local-thumb")
|
||||
if err != nil {
|
||||
t.Fatalf("get spider91 local thumb video: %v", err)
|
||||
t.Fatalf("get crawler local thumb video: %v", err)
|
||||
}
|
||||
if spider91Local.ThumbnailURL != "/p/thumb/spider91-local-thumb-video" {
|
||||
t.Fatalf("spider91 local thumbnail = %q, want preserved", spider91Local.ThumbnailURL)
|
||||
if crawlerLocal.ThumbnailURL != "/p/thumb/scriptcrawler-crawler-main-local-thumb" {
|
||||
t.Fatalf("crawler local thumbnail = %q, want preserved", crawlerLocal.ThumbnailURL)
|
||||
}
|
||||
|
||||
crawlerRemote, err := cat.GetVideo(ctx, "scriptcrawler-crawler-main-remote-thumb")
|
||||
if err != nil {
|
||||
t.Fatalf("get crawler remote thumb video: %v", err)
|
||||
}
|
||||
if crawlerRemote.ThumbnailURL != "" {
|
||||
t.Fatalf("crawler remote thumbnail = %q, want cleared", crawlerRemote.ThumbnailURL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1113,33 +1112,33 @@ func TestTagFilterMatchesCanonicalDuplicateVideo(t *testing.T) {
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "spider91-dup-1",
|
||||
DriveID: "91-spider",
|
||||
ID: "scriptcrawler-crawler-a-dup-1",
|
||||
DriveID: "crawler-a",
|
||||
FileID: "dup-1.mp4",
|
||||
Title: "Spider duplicate 1",
|
||||
Tags: []string{"91porn"},
|
||||
Title: "Crawler duplicate 1",
|
||||
Tags: []string{"crawler-tag"},
|
||||
Size: 1024,
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
{
|
||||
ID: "spider91-dup-2",
|
||||
DriveID: "91-spider",
|
||||
ID: "scriptcrawler-crawler-a-dup-2",
|
||||
DriveID: "crawler-a",
|
||||
FileID: "dup-2.mp4",
|
||||
Title: "Spider duplicate 2",
|
||||
Tags: []string{"91porn"},
|
||||
Title: "Crawler duplicate 2",
|
||||
Tags: []string{"crawler-tag"},
|
||||
Size: 1024,
|
||||
PublishedAt: now.Add(2 * time.Second),
|
||||
CreatedAt: now.Add(2 * time.Second),
|
||||
UpdatedAt: now.Add(2 * time.Second),
|
||||
},
|
||||
{
|
||||
ID: "spider91-visible",
|
||||
DriveID: "91-spider",
|
||||
ID: "scriptcrawler-crawler-a-visible",
|
||||
DriveID: "crawler-a",
|
||||
FileID: "visible.mp4",
|
||||
Title: "Spider visible",
|
||||
Tags: []string{"91porn"},
|
||||
Title: "Crawler visible",
|
||||
Tags: []string{"crawler-tag"},
|
||||
Size: 2048,
|
||||
PublishedAt: now.Add(3 * time.Second),
|
||||
CreatedAt: now.Add(3 * time.Second),
|
||||
@@ -1150,16 +1149,16 @@ func TestTagFilterMatchesCanonicalDuplicateVideo(t *testing.T) {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"pikpak-canonical", "spider91-dup-1", "spider91-dup-2"} {
|
||||
for _, id := range []string{"pikpak-canonical", "scriptcrawler-crawler-a-dup-1", "scriptcrawler-crawler-a-dup-2"} {
|
||||
if err := cat.UpdateVideoFingerprint(ctx, id, "same-sampled-sha256", "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "spider91-visible", "unique-sampled-sha256", "ready", ""); err != nil {
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "scriptcrawler-crawler-a-visible", "unique-sampled-sha256", "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint visible: %v", err)
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{Tag: "91porn", Page: 1, PageSize: 10})
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{Tag: "crawler-tag", Page: 1, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list videos by tag: %v", err)
|
||||
}
|
||||
@@ -1170,13 +1169,13 @@ func TestTagFilterMatchesCanonicalDuplicateVideo(t *testing.T) {
|
||||
for _, item := range items {
|
||||
gotIDs[item.ID] = true
|
||||
}
|
||||
for _, want := range []string{"pikpak-canonical", "spider91-visible"} {
|
||||
for _, want := range []string{"pikpak-canonical", "scriptcrawler-crawler-a-visible"} {
|
||||
if !gotIDs[want] {
|
||||
t.Fatalf("tagged video ids = %#v, want %s", gotIDs, want)
|
||||
}
|
||||
}
|
||||
if got := mustTagByLabel(t, ctx, cat, "91porn").Count; got != 2 {
|
||||
t.Fatalf("91porn count = %d, want 2 visible canonical videos", got)
|
||||
if got := mustTagByLabel(t, ctx, cat, "crawler-tag").Count; got != 2 {
|
||||
t.Fatalf("crawler-tag count = %d, want 2 visible canonical videos", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1265,6 +1264,41 @@ func mustTagByLabel(t *testing.T, ctx context.Context, cat *Catalog, label strin
|
||||
return Tag{}
|
||||
}
|
||||
|
||||
func hasColumn(t *testing.T, cat *Catalog, table, column string) bool {
|
||||
t.Helper()
|
||||
rows, err := cat.db.Query(`PRAGMA table_info(` + table + `)`)
|
||||
if err != nil {
|
||||
t.Fatalf("query table info for %s: %v", table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var name, typ string
|
||||
var notNull int
|
||||
var defaultValue any
|
||||
var pk int
|
||||
if err := rows.Scan(&cid, &name, &typ, ¬Null, &defaultValue, &pk); err != nil {
|
||||
t.Fatalf("scan table info for %s: %v", table, err)
|
||||
}
|
||||
if strings.EqualFold(name, column) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatalf("iterate table info for %s: %v", table, err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func indexExists(t *testing.T, cat *Catalog, name string) bool {
|
||||
t.Helper()
|
||||
var count int
|
||||
if err := cat.db.QueryRow(`SELECT COUNT(*) FROM sqlite_schema WHERE type = 'index' AND name = ?`, name).Scan(&count); err != nil {
|
||||
t.Fatalf("query index %s: %v", name, err)
|
||||
}
|
||||
return count > 0
|
||||
}
|
||||
|
||||
func videoUpdatedAtByID(t *testing.T, ctx context.Context, cat *Catalog, ids ...string) map[string]int64 {
|
||||
t.Helper()
|
||||
out := make(map[string]int64, len(ids))
|
||||
@@ -1278,9 +1312,9 @@ func videoUpdatedAtByID(t *testing.T, ctx context.Context, cat *Catalog, ids ...
|
||||
return out
|
||||
}
|
||||
|
||||
// 删除 collection 标签的最后一个引用视频后,标签应当自动从 tags 表里消失。
|
||||
// 删除旧版本 collection 标签的最后一个引用视频后,标签应当自动从 tags 表里消失。
|
||||
// user/system 标签不受影响:用户/系统标签的语义由人维护,孤儿状态保留。
|
||||
func TestDeleteVideoPrunesOrphanCollectionTag(t *testing.T) {
|
||||
func TestDeleteVideoPrunesLegacyOrphanCollectionTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -1299,7 +1333,6 @@ func TestDeleteVideoPrunesOrphanCollectionTag(t *testing.T) {
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
Category: "Better Call Saul S02",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
@@ -1308,20 +1341,28 @@ func TestDeleteVideoPrunesOrphanCollectionTag(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
label, ok, err := cat.EnsureCollectionTag(ctx, "Better Call Saul S02")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure collection tag: %v", err)
|
||||
nowMillis := now.UnixMilli()
|
||||
if _, err := cat.db.ExecContext(ctx,
|
||||
`INSERT INTO tags (label, aliases, source, created_at, updated_at) VALUES (?, '[]', 'collection', ?, ?)`,
|
||||
"Better Call Saul S02", nowMillis, nowMillis); err != nil {
|
||||
t.Fatalf("insert legacy collection tag: %v", err)
|
||||
}
|
||||
if !ok || label != "Better Call Saul S02" {
|
||||
t.Fatalf("ensure collection tag = %q ok=%v, want collection tag created", label, ok)
|
||||
var collectionTagID int64
|
||||
if err := cat.db.QueryRowContext(ctx, `SELECT id FROM tags WHERE label = ?`, "Better Call Saul S02").Scan(&collectionTagID); err != nil {
|
||||
t.Fatalf("lookup legacy collection tag: %v", err)
|
||||
}
|
||||
for _, id := range []string{"video-a", "video-b"} {
|
||||
if _, err := cat.db.ExecContext(ctx,
|
||||
`INSERT INTO video_tags (video_id, tag_id, source, created_at) VALUES (?, ?, 'auto', ?)`,
|
||||
id, collectionTagID, nowMillis); err != nil {
|
||||
t.Fatalf("attach legacy collection tag to %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 用户标签:手动建出来,让它和 video-a 关联,验证 user 标签不会被孤儿清理流程误删。
|
||||
if _, err := cat.CreateTagAndClassify(ctx, "用户标签", nil, "user"); err != nil {
|
||||
t.Fatalf("create user tag: %v", err)
|
||||
}
|
||||
if err := cat.SetManualVideoTags(ctx, "video-a", []string{"用户标签"}); err != nil {
|
||||
t.Fatalf("attach user tag: %v", err)
|
||||
if _, err := cat.db.ExecContext(ctx,
|
||||
`INSERT INTO tags (label, aliases, source, created_at, updated_at) VALUES (?, '[]', 'user', ?, ?)`,
|
||||
"用户标签", nowMillis, nowMillis); err != nil {
|
||||
t.Fatalf("insert user orphan tag: %v", err)
|
||||
}
|
||||
|
||||
collectionExists := func() bool {
|
||||
@@ -1337,7 +1378,7 @@ func TestDeleteVideoPrunesOrphanCollectionTag(t *testing.T) {
|
||||
t.Fatal("collection tag missing right after creation")
|
||||
}
|
||||
|
||||
// 删第一个视频:还有 video-b 在引用 collection 标签,应保留。
|
||||
// 删第一个视频:还有 video-b 在引用旧 collection 标签,应保留。
|
||||
if err := cat.DeleteVideo(ctx, "video-a"); err != nil {
|
||||
t.Fatalf("delete video-a: %v", err)
|
||||
}
|
||||
@@ -1345,7 +1386,7 @@ func TestDeleteVideoPrunesOrphanCollectionTag(t *testing.T) {
|
||||
t.Fatal("collection tag was pruned while another video still references it")
|
||||
}
|
||||
|
||||
// 删最后一个引用视频,collection 标签应当被同步清掉。
|
||||
// 删最后一个引用视频,旧 collection 标签应当被同步清掉。
|
||||
if err := cat.DeleteVideo(ctx, "video-b"); err != nil {
|
||||
t.Fatalf("delete video-b: %v", err)
|
||||
}
|
||||
@@ -1353,7 +1394,7 @@ func TestDeleteVideoPrunesOrphanCollectionTag(t *testing.T) {
|
||||
t.Fatal("orphan collection tag was not pruned after deleting the last referencing video")
|
||||
}
|
||||
|
||||
// 用户手动建的标签即使变成孤儿(已经因为 video-a 删除而失去引用)也必须保留。
|
||||
// 用户标签即使是孤儿也必须保留。
|
||||
var userCount int
|
||||
if err := cat.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM tags WHERE label = ? AND source = 'user'`,
|
||||
|
||||
@@ -69,33 +69,49 @@ func TestBlacklistListAndRemove(t *testing.T) {
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
seed := []struct{ id, file string }{
|
||||
{"d1", "movie-alpha.avi"},
|
||||
{"d2", "movie-beta.mp4"},
|
||||
{"d3", "clip-gamma.wmv"},
|
||||
seed := []struct{ id, drive, file string }{
|
||||
{"d1", "drive", "movie-alpha.avi"},
|
||||
{"d2", "drive", "movie-beta.mp4"},
|
||||
{"d3", "archive", "clip-gamma.wmv"},
|
||||
}
|
||||
for _, s := range seed {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: s.id, DriveID: "drive", FileID: "f-" + s.id, FileName: s.file,
|
||||
ID: s.id, DriveID: s.drive, FileID: "f-" + s.id, FileName: s.file,
|
||||
Title: s.id, PublishedAt: now, CreatedAt: now, UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", s.id, err)
|
||||
}
|
||||
if err := cat.DeleteVideoWithTombstone(ctx, s.id); err != nil {
|
||||
var err error
|
||||
if s.id == "d2" {
|
||||
err = cat.DeleteVideoWithTombstoneReason(ctx, s.id, DeletedVideoReasonDuplicate)
|
||||
} else {
|
||||
err = cat.DeleteVideoWithTombstone(ctx, s.id)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("tombstone %s: %v", s.id, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, total, err := cat.ListDeletedVideos(ctx, "", 1, 50)
|
||||
items, total, err := cat.ListDeletedVideos(ctx, ListParams{Page: 1, PageSize: 50})
|
||||
if err != nil {
|
||||
t.Fatalf("list deleted: %v", err)
|
||||
}
|
||||
if total != 3 || len(items) != 3 {
|
||||
t.Fatalf("deleted total/len = %d/%d, want 3/3", total, len(items))
|
||||
}
|
||||
reasons := map[string]string{}
|
||||
for _, item := range items {
|
||||
reasons[item.ID] = item.Reason
|
||||
}
|
||||
if reasons["d1"] != "" || reasons["d3"] != "" {
|
||||
t.Fatalf("manual tombstone reasons = %#v, want empty", reasons)
|
||||
}
|
||||
if reasons["d2"] != DeletedVideoReasonDuplicate {
|
||||
t.Fatalf("duplicate tombstone reason = %q, want %q", reasons["d2"], DeletedVideoReasonDuplicate)
|
||||
}
|
||||
|
||||
// 关键字过滤
|
||||
filtered, ftotal, err := cat.ListDeletedVideos(ctx, "movie", 1, 50)
|
||||
filtered, ftotal, err := cat.ListDeletedVideos(ctx, ListParams{Keyword: "movie", Page: 1, PageSize: 50})
|
||||
if err != nil {
|
||||
t.Fatalf("list deleted filtered: %v", err)
|
||||
}
|
||||
@@ -103,6 +119,23 @@ func TestBlacklistListAndRemove(t *testing.T) {
|
||||
t.Fatalf("filtered total/len = %d/%d, want 2/2", ftotal, len(filtered))
|
||||
}
|
||||
|
||||
// 网盘过滤
|
||||
driveFiltered, driveTotal, err := cat.ListDeletedVideos(ctx, ListParams{DriveID: "archive", Page: 1, PageSize: 50})
|
||||
if err != nil {
|
||||
t.Fatalf("list deleted drive filtered: %v", err)
|
||||
}
|
||||
if driveTotal != 1 || len(driveFiltered) != 1 || driveFiltered[0].ID != "d3" {
|
||||
t.Fatalf("drive filtered = total %d items %#v, want only d3", driveTotal, driveFiltered)
|
||||
}
|
||||
|
||||
combined, combinedTotal, err := cat.ListDeletedVideos(ctx, ListParams{Keyword: "movie", DriveID: "archive", Page: 1, PageSize: 50})
|
||||
if err != nil {
|
||||
t.Fatalf("list deleted combined filtered: %v", err)
|
||||
}
|
||||
if combinedTotal != 0 || len(combined) != 0 {
|
||||
t.Fatalf("combined filtered total/len = %d/%d, want 0/0", combinedTotal, len(combined))
|
||||
}
|
||||
|
||||
// 移出黑名单
|
||||
if err := cat.RemoveDeletedVideo(ctx, "d1"); err != nil {
|
||||
t.Fatalf("remove d1: %v", err)
|
||||
@@ -110,7 +143,7 @@ func TestBlacklistListAndRemove(t *testing.T) {
|
||||
if deleted, err := cat.IsVideoDeleted(ctx, "d1"); err != nil || deleted {
|
||||
t.Fatalf("d1 should no longer be blacklisted (deleted=%v err=%v)", deleted, err)
|
||||
}
|
||||
_, total, err = cat.ListDeletedVideos(ctx, "", 1, 50)
|
||||
_, total, err = cat.ListDeletedVideos(ctx, ListParams{Page: 1, PageSize: 50})
|
||||
if err != nil {
|
||||
t.Fatalf("list deleted after remove: %v", err)
|
||||
}
|
||||
|
||||
+89
-262
@@ -1,18 +1,16 @@
|
||||
// Package spider91migrate 周期性把 spider91 drive 下载到本地的视频
|
||||
// 上传到一个指定的目标 drive 目录(PikPak、115、123、OneDrive、Google Drive、联通网盘或光鸭网盘),上传成功后:
|
||||
// Package crawlerupload uploads videos saved by script crawlers to a configured
|
||||
// target drive. Each crawler drive chooses its own upload target.
|
||||
//
|
||||
// - 改写 catalog 行:drive_id / file_id / content_hash 改成目标盘的;
|
||||
// 视频自身的 id 不变(仍是 spider91-<driveID>-<viewkey>),video_tags、
|
||||
// 收藏、点赞、views 等关联数据全部保留
|
||||
// - 删除本地 mp4(spider91/<id>/videos/<viewkey>.<ext>)和源 thumb
|
||||
// (spider91/<id>/thumbs/<viewkey>.jpg);公共 /p/thumb/<videoID> 副本会保留
|
||||
// 视频自身的 id 不变,video_tags、收藏、点赞、views 等关联数据全部保留
|
||||
// - 删除爬虫本地 mp4 和源 thumb;公共 /p/thumb/<videoID> 副本会保留
|
||||
//
|
||||
// 之后回放时,videoSource() 自动落到 /p/stream/<target>/<file_id>,
|
||||
// proxy 层走对应盘的直链 / 302 直连。
|
||||
//
|
||||
// 下次目标盘扫盘时,scanner 通过 (content_hash) / (file_name+size)
|
||||
// 已有的 findDuplicate 兜底逻辑,不会为同一物理文件再建一行。
|
||||
package spider91migrate
|
||||
package crawlerupload
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -37,12 +35,11 @@ import (
|
||||
"github.com/video-site/backend/internal/drives/p123"
|
||||
"github.com/video-site/backend/internal/drives/pikpak"
|
||||
"github.com/video-site/backend/internal/drives/scriptcrawler"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/drives/wopan"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
)
|
||||
|
||||
// uploadTarget 是 migrator 调用目标 drive 的最小接口。任何一种"接收 spider91 上传"的
|
||||
// uploadTarget 是 migrator 调用目标 drive 的最小接口。任何一种"接收爬虫上传"的
|
||||
// 网盘都要实现它;当前 PikPak、115、123、OneDrive、Google Drive、联通网盘和光鸭网盘各自通过适配器满足。
|
||||
//
|
||||
// 这一层抽象把"迁移调用方"和"具体盘的 SDK 协议"解耦:
|
||||
@@ -64,10 +61,10 @@ type uploadTarget interface {
|
||||
Rename(ctx context.Context, fileID, newName string) error
|
||||
}
|
||||
|
||||
// Spider91LocalSource is the local source interface used by the migration
|
||||
// worker. Legacy spider91.Driver and the new scriptcrawler.Driver both satisfy
|
||||
// it when they are mounted for the Spider91 built-in crawler.
|
||||
type Spider91LocalSource interface {
|
||||
// LocalSource is the local source interface used by the migration
|
||||
// worker. scriptcrawler.Driver satisfies it when mounted for a crawler that
|
||||
// keeps videos in local storage before uploading them to a target drive.
|
||||
type LocalSource interface {
|
||||
drives.Drive
|
||||
VideosDir() string
|
||||
ThumbsDir() string
|
||||
@@ -95,22 +92,17 @@ type UploadProgress struct {
|
||||
TotalCount int
|
||||
}
|
||||
|
||||
const (
|
||||
spider91UploadDirName = "91 Spider"
|
||||
scriptCrawlerUploadRootDirName = "Script Crawlers"
|
||||
)
|
||||
const scriptCrawlerUploadRootDirName = "Script Crawlers"
|
||||
|
||||
type migrationPlan struct {
|
||||
source Spider91LocalSource
|
||||
source LocalSource
|
||||
row *catalog.Drive
|
||||
sourceKinds []string
|
||||
targetDriveID string
|
||||
target uploadTarget
|
||||
uploadDir string
|
||||
keepLatestN int
|
||||
requireAssetsReady bool
|
||||
requirePreviewReady bool
|
||||
legacyBackfill bool
|
||||
}
|
||||
|
||||
// pikpakAdapter / p115Adapter / p123Adapter / onedriveAdapter / googledriveAdapter / wopanAdapter / guangyapanAdapter 把具体 driver 包装成 uploadTarget。
|
||||
@@ -119,7 +111,7 @@ type migrationPlan struct {
|
||||
//
|
||||
// 1. 各 driver 的 UploadAndReportXxx 返回的是各自包内的 UploadResult 类型,
|
||||
// 直接共用同名同签名方法会引入循环依赖;
|
||||
// 2. driver 包不应该感知 spider91migrate 这一层业务定义。
|
||||
// 2. driver 包不应该感知 crawlerupload 这一层业务定义。
|
||||
type pikpakAdapter struct {
|
||||
d *pikpak.Driver
|
||||
}
|
||||
@@ -289,7 +281,7 @@ func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
// 测试或自定义实现可以直接传入;优先使用具体类型分支以拿到适配器。
|
||||
return v, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("drive %q kind=%s does not support spider91 upload", d.ID(), d.Kind())
|
||||
return nil, fmt.Errorf("drive %q kind=%s does not support crawler upload", d.ID(), d.Kind())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -300,16 +292,15 @@ type Registry interface {
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Catalog *catalog.Catalog
|
||||
Registry Registry
|
||||
GetTargetDriveID func() string // 通常对应 App.Spider91UploadDriveID()
|
||||
Catalog *catalog.Catalog
|
||||
Registry Registry
|
||||
// Interval 已废弃 —— 旧版迁移 worker 是周期 ticker,新版只通过 nightly
|
||||
// pipeline 调用 RunOnce,不再有内置定时器。保留字段不删是为了兼容外
|
||||
// 部 yaml / 测试代码里仍传值的场景。
|
||||
Interval time.Duration
|
||||
BatchLimit int // 单轮最多迁多少个,0 时默认 50
|
||||
// KeepLatestN 是每个 spider91 drive 在本地保留的最新视频数。
|
||||
// 超过的部分中"已迁移"的会被清理;未迁移的不动。0 时默认 15;< 0 关闭清理。
|
||||
// KeepLatestN is deprecated. Script crawler uploads use 0 internally so all
|
||||
// local videos that satisfy asset requirements are eligible for upload.
|
||||
KeepLatestN int
|
||||
// CaptchaCooldown 是迁移 worker 在遇到 PikPak captcha 错误(error_code
|
||||
// 4002 / 9)后整体进入冷却的时长。冷却期间 runOnce 直接返回,不再发起任何
|
||||
@@ -401,9 +392,8 @@ func (m *Migrator) markCooldownLogged() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Trigger 安排一次"立即跑"。多次调用会被合并成一次(channel buffer=1)。
|
||||
// RunOnce 跑一次完整迁移:列出所有 spider91 drive,对每个超过 KeepLatestN 的旧
|
||||
// 视频上传到目标 drive,事务性改写 catalog 行,删本地文件。
|
||||
// RunOnce 跑一次完整迁移:列出所有配置了 upload_drive_id 的 scriptcrawler
|
||||
// drive,把本地视频上传到目标 drive,事务性改写 catalog 行,删本地文件。
|
||||
//
|
||||
// 这是上层 nightly 流水线 Phase 3 的入口;不再有周期 ticker / Trigger 通道。
|
||||
// captcha cooldown 状态在单次 RunOnce 内仍生效(多 drive 时遇到 4002 立即停整轮);
|
||||
@@ -417,7 +407,7 @@ func (m *Migrator) RunOnce(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// runOnce 单轮:扫所有 spider91 drive,对每条还有本地文件的视频做迁移。
|
||||
// runOnce 单轮:扫所有 scriptcrawler drive,对每条还有本地文件的视频做迁移。
|
||||
//
|
||||
// 互斥保证:同一 Migrator 内不会并发跑两轮(避免重复上传)。
|
||||
func (m *Migrator) runOnce(ctx context.Context) {
|
||||
@@ -439,11 +429,11 @@ func (m *Migrator) runOnce(ctx context.Context) {
|
||||
// 结束自然恢复。避免之前每秒一条 4002 的日志雪崩。
|
||||
if active, until, resumed := m.cooldownState(); active {
|
||||
if !m.markCooldownLogged() {
|
||||
log.Printf("[spider91migrate] captcha cooldown active until %s, skipping run", until.Format(time.RFC3339))
|
||||
log.Printf("[crawlerupload] captcha cooldown active until %s, skipping run", until.Format(time.RFC3339))
|
||||
}
|
||||
return
|
||||
} else if resumed {
|
||||
log.Printf("[spider91migrate] captcha cooldown ended at %s, resuming migration", until.Format(time.RFC3339))
|
||||
log.Printf("[crawlerupload] captcha cooldown ended at %s, resuming migration", until.Format(time.RFC3339))
|
||||
}
|
||||
|
||||
plans := m.migrationPlans(ctx)
|
||||
@@ -453,54 +443,39 @@ func (m *Migrator) runOnce(ctx context.Context) {
|
||||
}
|
||||
|
||||
migrated := 0
|
||||
backfillTargets := map[string]uploadTarget{}
|
||||
for _, plan := range plans {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return
|
||||
}
|
||||
n, err := m.migrateDrive(ctx, plan)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] drive=%s migrate batch error: %v", plan.source.ID(), err)
|
||||
log.Printf("[crawlerupload] drive=%s migrate batch error: %v", plan.source.ID(), err)
|
||||
}
|
||||
migrated += n
|
||||
if active, _ := m.inCooldown(); active {
|
||||
if migrated > 0 {
|
||||
log.Printf("[spider91migrate] migrated %d video(s)", migrated)
|
||||
log.Printf("[crawlerupload] migrated %d video(s)", migrated)
|
||||
}
|
||||
return
|
||||
}
|
||||
if plan.legacyBackfill {
|
||||
backfillTargets[plan.targetDriveID] = plan.target
|
||||
}
|
||||
}
|
||||
if migrated > 0 {
|
||||
log.Printf("[spider91migrate] migrated %d video(s)", migrated)
|
||||
log.Printf("[crawlerupload] migrated %d video(s)", migrated)
|
||||
}
|
||||
|
||||
// 收尾:扫每个本地爬虫 drive 的 videos 目录,把 catalog 已经迁到别处但本地
|
||||
// 仍有残留的孤儿文件清掉。这是纯防御性兜底——正常路径下 migrateDrive
|
||||
// 已经在迁移成功后立刻 CleanupSpider91Local,不会留孤儿。
|
||||
// 已经在迁移成功后立刻 CleanupLocal,不会留孤儿。
|
||||
for _, plan := range plans {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return
|
||||
}
|
||||
deleted, err := m.cleanupOldLocalVideos(ctx, plan)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] cleanup drive=%s: %v", plan.source.ID(), err)
|
||||
log.Printf("[crawlerupload] cleanup drive=%s: %v", plan.source.ID(), err)
|
||||
}
|
||||
if deleted > 0 {
|
||||
log.Printf("[spider91migrate] cleanup drive=%s deleted %d orphan local file(s)", plan.source.ID(), deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// 回填:把已迁移到 PikPak 的 spider91-* 视频里文件名仍是旧格式
|
||||
// (比如刚迁完没改、或人工导入)的统一改成方案 B 期望的格式。
|
||||
// 这一步幂等:已经是期望格式的不会再调 Rename。
|
||||
for targetDriveID, pp := range backfillTargets {
|
||||
if renamed, err := m.backfillFileNames(ctx, targetDriveID, pp); err != nil {
|
||||
log.Printf("[spider91migrate] backfill names: %v", err)
|
||||
} else if renamed > 0 {
|
||||
log.Printf("[spider91migrate] backfilled %d %s file name(s) to desired format", renamed, pp.Kind())
|
||||
log.Printf("[crawlerupload] cleanup drive=%s deleted %d orphan local file(s)", plan.source.ID(), deleted)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -519,33 +494,6 @@ func (m *Migrator) reportUploadProgress(progress UploadProgress) {
|
||||
m.cfg.OnUploadProgress(progress)
|
||||
}
|
||||
|
||||
// targetKindForLog 把当前目标盘 kind 转成对人友好的简称,用于日志。
|
||||
// 解析失败时回退 "target"。
|
||||
func (m *Migrator) targetKindForLog() string {
|
||||
if m.cfg.GetTargetDriveID == nil || m.cfg.Registry == nil {
|
||||
return "target"
|
||||
}
|
||||
id := m.cfg.GetTargetDriveID()
|
||||
if id == "" {
|
||||
return "target"
|
||||
}
|
||||
d, ok := m.cfg.Registry.Get(id)
|
||||
if !ok {
|
||||
return "target"
|
||||
}
|
||||
return d.Kind()
|
||||
}
|
||||
|
||||
// resolveTarget 返回 (target drive ID, target uploadTarget, err)。
|
||||
// 没设置、drive 找不到,或 drive 类型不支持上传时返回 err(调用方静默跳过)。
|
||||
func (m *Migrator) resolveTarget() (string, uploadTarget, error) {
|
||||
if m.cfg.GetTargetDriveID == nil {
|
||||
return "", nil, errors.New("no target getter")
|
||||
}
|
||||
id := m.cfg.GetTargetDriveID()
|
||||
return m.resolveTargetID(id)
|
||||
}
|
||||
|
||||
func (m *Migrator) resolveTargetID(id string) (string, uploadTarget, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
@@ -575,74 +523,37 @@ func (m *Migrator) migrationPlans(ctx context.Context) []migrationPlan {
|
||||
if d == nil {
|
||||
continue
|
||||
}
|
||||
src, ok := d.(Spider91LocalSource)
|
||||
src, ok := d.(LocalSource)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
row, err := m.cfg.Catalog.GetDrive(ctx, d.ID())
|
||||
if (err != nil || row == nil) && d.Kind() == spider91.Kind {
|
||||
row = &catalog.Drive{ID: d.ID(), Kind: spider91.Kind, RootID: "/"}
|
||||
}
|
||||
if row == nil {
|
||||
if err != nil || row == nil || row.Kind != scriptcrawler.Kind {
|
||||
continue
|
||||
}
|
||||
switch row.Kind {
|
||||
case scriptcrawler.Kind:
|
||||
targetID := strings.TrimSpace(row.Credentials["upload_drive_id"])
|
||||
if targetID == "" {
|
||||
continue
|
||||
}
|
||||
resolvedID, target, err := m.resolveTargetID(targetID)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] crawler=%s upload target=%q unavailable: %v", row.ID, targetID, err)
|
||||
continue
|
||||
}
|
||||
out = append(out, migrationPlan{
|
||||
source: src,
|
||||
row: row,
|
||||
sourceKinds: crawlerSourceKindsForRow(row),
|
||||
targetDriveID: resolvedID,
|
||||
target: target,
|
||||
uploadDir: scriptCrawlerUploadDir(row.ID),
|
||||
keepLatestN: 0,
|
||||
requireAssetsReady: true,
|
||||
requirePreviewReady: row.TeaserEnabled,
|
||||
})
|
||||
case spider91.Kind:
|
||||
if m.cfg.GetTargetDriveID == nil {
|
||||
continue
|
||||
}
|
||||
targetID := strings.TrimSpace(m.cfg.GetTargetDriveID())
|
||||
if targetID == "" {
|
||||
continue
|
||||
}
|
||||
resolvedID, target, err := m.resolveTargetID(targetID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, migrationPlan{
|
||||
source: src,
|
||||
row: row,
|
||||
sourceKinds: []string{spider91.Kind},
|
||||
targetDriveID: resolvedID,
|
||||
target: target,
|
||||
uploadDir: spider91UploadDirName,
|
||||
keepLatestN: m.cfg.KeepLatestN,
|
||||
legacyBackfill: true,
|
||||
})
|
||||
targetID := strings.TrimSpace(row.Credentials["upload_drive_id"])
|
||||
if targetID == "" {
|
||||
continue
|
||||
}
|
||||
resolvedID, target, err := m.resolveTargetID(targetID)
|
||||
if err != nil {
|
||||
log.Printf("[crawlerupload] crawler=%s upload target=%q unavailable: %v", row.ID, targetID, err)
|
||||
continue
|
||||
}
|
||||
out = append(out, migrationPlan{
|
||||
source: src,
|
||||
row: row,
|
||||
targetDriveID: resolvedID,
|
||||
target: target,
|
||||
uploadDir: scriptCrawlerUploadDir(row.ID),
|
||||
keepLatestN: 0,
|
||||
requireAssetsReady: true,
|
||||
requirePreviewReady: row.TeaserEnabled,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func crawlerSourceKindsForRow(d *catalog.Drive) []string {
|
||||
kinds := []string{scriptcrawler.Kind}
|
||||
if d != nil && strings.EqualFold(strings.TrimSpace(d.Credentials["builtin"]), spider91.Kind) {
|
||||
kinds = append(kinds, spider91.Kind)
|
||||
}
|
||||
return kinds
|
||||
}
|
||||
|
||||
func scriptCrawlerUploadDir(driveID string) string {
|
||||
driveID = sanitizeUploadDirSegment(driveID)
|
||||
if driveID == "" {
|
||||
@@ -660,41 +571,6 @@ func sanitizeUploadDirSegment(raw string) string {
|
||||
return clean
|
||||
}
|
||||
|
||||
// spider91Drives 返回当前注册的所有 Spider91 来源本地爬虫 driver。
|
||||
func (m *Migrator) spider91Drives(ctx context.Context) []Spider91LocalSource {
|
||||
all := m.cfg.Registry.All()
|
||||
out := make([]Spider91LocalSource, 0, len(all))
|
||||
for _, d := range all {
|
||||
if !m.isSpider91SourceDrive(ctx, d) {
|
||||
continue
|
||||
}
|
||||
if sd, ok := d.(Spider91LocalSource); ok {
|
||||
out = append(out, sd)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (m *Migrator) isSpider91SourceDrive(ctx context.Context, d drives.Drive) bool {
|
||||
if d == nil {
|
||||
return false
|
||||
}
|
||||
if d.Kind() == spider91.Kind {
|
||||
return true
|
||||
}
|
||||
if d.Kind() != scriptcrawler.Kind || m.cfg.Catalog == nil {
|
||||
return false
|
||||
}
|
||||
row, err := m.cfg.Catalog.GetDrive(ctx, d.ID())
|
||||
if err != nil || row == nil {
|
||||
return false
|
||||
}
|
||||
if row.Kind == spider91.Kind {
|
||||
return true
|
||||
}
|
||||
return row.Kind == scriptcrawler.Kind && strings.EqualFold(strings.TrimSpace(row.Credentials["builtin"]), spider91.Kind)
|
||||
}
|
||||
|
||||
// migrateDrive 对单个本地爬虫 drive 跑一批迁移;返回成功迁移的条数。
|
||||
func (m *Migrator) migrateDrive(ctx context.Context, plan migrationPlan) (int, error) {
|
||||
src := plan.source
|
||||
@@ -800,7 +676,7 @@ func (m *Migrator) migrateDrive(ctx context.Context, plan migrationPlan) (int, e
|
||||
})
|
||||
|
||||
if v.DriveID != src.ID() {
|
||||
CleanupSpider91Local(src, f.name)
|
||||
CleanupLocal(src, f.name)
|
||||
processed++
|
||||
m.reportUploadProgress(UploadProgress{
|
||||
DriveID: src.ID(),
|
||||
@@ -814,12 +690,12 @@ func (m *Migrator) migrateDrive(ctx context.Context, plan migrationPlan) (int, e
|
||||
|
||||
if targetDuplicate, err := m.cfg.Catalog.FindEquivalentVideoOnDrive(ctx, v, plan.targetDriveID); err != nil {
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
log.Printf("[spider91migrate] %s find target duplicate: %v", v.ID, err)
|
||||
log.Printf("[crawlerupload] %s find target duplicate: %v", v.ID, err)
|
||||
}
|
||||
} else if targetDuplicate != nil {
|
||||
ok, err := m.bindToExistingTarget(ctx, v, targetDuplicate, plan)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] %s: %v", v.ID, err)
|
||||
log.Printf("[crawlerupload] %s: %v", v.ID, err)
|
||||
continue
|
||||
}
|
||||
if ok {
|
||||
@@ -842,7 +718,7 @@ func (m *Migrator) migrateDrive(ctx context.Context, plan migrationPlan) (int, e
|
||||
if plan.requireAssetsReady {
|
||||
ready, err := m.crawlerVideoAssetsReady(ctx, v, plan.requirePreviewReady)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] %s check generated assets: %v", v.ID, err)
|
||||
log.Printf("[crawlerupload] %s check generated assets: %v", v.ID, err)
|
||||
continue
|
||||
}
|
||||
if !ready {
|
||||
@@ -860,14 +736,14 @@ func (m *Migrator) migrateDrive(ctx context.Context, plan migrationPlan) (int, e
|
||||
|
||||
ok, err := m.migrateOne(ctx, v, plan)
|
||||
if err != nil {
|
||||
log.Printf("[spider91migrate] %s: %v", v.ID, err)
|
||||
log.Printf("[crawlerupload] %s: %v", v.ID, err)
|
||||
// captcha 错误(4002 / 9)说明 PikPak 当前正拒绝我们;继续在
|
||||
// 同一轮里尝试其它文件大概率会拿到同样的 4002,并且每多一次
|
||||
// 失败就多一份"被风控加深"的风险。立即中止当前 batch 并
|
||||
// 打开冷却窗口,等 cfg.CaptchaCooldown 之后再重试。
|
||||
if pikpak.IsCaptchaError(err) {
|
||||
until := m.setCooldown()
|
||||
log.Printf("[spider91migrate] drive=%s captcha-blocked, cooling down until %s", src.ID(), until.Format(time.RFC3339))
|
||||
log.Printf("[crawlerupload] drive=%s captcha-blocked, cooling down until %s", src.ID(), until.Format(time.RFC3339))
|
||||
return migrated, nil
|
||||
}
|
||||
continue
|
||||
@@ -906,12 +782,10 @@ func (m *Migrator) findVideoForLocalFile(ctx context.Context, plan migrationPlan
|
||||
if plan.source != nil {
|
||||
driveID = plan.source.ID()
|
||||
}
|
||||
for _, kind := range plan.sourceKinds {
|
||||
id := scriptcrawler.BuildVideoIDForKind(kind, driveID, sourceID)
|
||||
v, err := m.cfg.Catalog.GetVideo(ctx, id)
|
||||
if err == nil && v != nil {
|
||||
return v
|
||||
}
|
||||
id := scriptcrawler.BuildVideoID(driveID, sourceID)
|
||||
v, err := m.cfg.Catalog.GetVideo(ctx, id)
|
||||
if err == nil && v != nil {
|
||||
return v
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -946,8 +820,8 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, plan migrat
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
// 本地文件被人手动删了,但 catalog 还显示 spider91 drive;
|
||||
// 这种状态没法迁移。跳过即可(保留行让管理员可见,避免数据丢失)。
|
||||
// 本地文件被人手动删了,但 catalog 还指向该爬虫;
|
||||
// 这种状态没法上传。跳过即可(保留行让管理员可见,避免数据丢失)。
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("stat local: %w", err)
|
||||
@@ -966,7 +840,7 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, plan migrat
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s ensure %q dir: %w", pp.Kind(), plan.uploadDir, err)
|
||||
}
|
||||
uploadName := desiredPikPakName(v.Title, sourceIDForUploadName(v, plan), v.Ext)
|
||||
uploadName := desiredUploadName(v.Title, sourceIDForUploadName(v, plan), v.Ext)
|
||||
res, err := pp.UploadAndReportHash(ctx, parent, uploadName, f, info.Size())
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s upload: %w", pp.Kind(), err)
|
||||
@@ -982,13 +856,13 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, plan migrat
|
||||
m.preserveCrawledThumbnail(ctx, src, v)
|
||||
// 同步 catalog 里的 file_name,让下次目标盘扫盘时 (file_name, size) 也能匹配上
|
||||
if err := m.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{FileName: uploadName}); err != nil {
|
||||
log.Printf("[spider91migrate] %s update file_name after migrate: %v", v.ID, err)
|
||||
log.Printf("[crawlerupload] %s update file_name after migrate: %v", v.ID, err)
|
||||
}
|
||||
|
||||
// 删除本地 mp4 和源 thumb(公共 /p/thumb 副本已在 preserveCrawledThumbnail 中保留)。
|
||||
CleanupSpider91Local(src, v.FileID)
|
||||
CleanupLocal(src, v.FileID)
|
||||
|
||||
log.Printf("[spider91migrate] %s migrated to drive=%s(kind=%s) file=%s name=%q", v.ID, plan.targetDriveID, pp.Kind(), res.FileID, uploadName)
|
||||
log.Printf("[crawlerupload] %s migrated to drive=%s(kind=%s) file=%s name=%q", v.ID, plan.targetDriveID, pp.Kind(), res.FileID, uploadName)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -1004,12 +878,12 @@ func (m *Migrator) bindToExistingTarget(ctx context.Context, v, target *catalog.
|
||||
}
|
||||
if target.FileName != "" {
|
||||
if err := m.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{FileName: target.FileName}); err != nil {
|
||||
log.Printf("[spider91migrate] %s update file_name after duplicate bind: %v", v.ID, err)
|
||||
log.Printf("[crawlerupload] %s update file_name after duplicate bind: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
m.preserveCrawledThumbnail(ctx, plan.source, v)
|
||||
CleanupSpider91Local(plan.source, v.FileID)
|
||||
log.Printf("[spider91migrate] %s bound to existing drive=%s(kind=%s) file=%s duplicate=%s", v.ID, plan.targetDriveID, plan.target.Kind(), target.FileID, target.ID)
|
||||
CleanupLocal(plan.source, v.FileID)
|
||||
log.Printf("[crawlerupload] %s bound to existing drive=%s(kind=%s) file=%s duplicate=%s", v.ID, plan.targetDriveID, plan.target.Kind(), target.FileID, target.ID)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -1026,22 +900,17 @@ func sourceIDForUploadName(v *catalog.Video, plan migrationPlan) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
if plan.legacyBackfill {
|
||||
return extractViewKey(v.ID)
|
||||
}
|
||||
for _, kind := range plan.sourceKinds {
|
||||
prefix := kind + "-" + plan.source.ID() + "-"
|
||||
if strings.HasPrefix(v.ID, prefix) {
|
||||
return strings.TrimPrefix(v.ID, prefix)
|
||||
}
|
||||
prefix := scriptcrawler.Kind + "-" + plan.source.ID() + "-"
|
||||
if strings.HasPrefix(v.ID, prefix) {
|
||||
return strings.TrimPrefix(v.ID, prefix)
|
||||
}
|
||||
if v.FileID != "" {
|
||||
return stripExt(v.FileID)
|
||||
}
|
||||
return extractViewKey(v.ID)
|
||||
return extractSourceID(v.ID)
|
||||
}
|
||||
|
||||
func (m *Migrator) preserveCrawledThumbnail(ctx context.Context, src Spider91LocalSource, v *catalog.Video) {
|
||||
func (m *Migrator) preserveCrawledThumbnail(ctx context.Context, src LocalSource, v *catalog.Video) {
|
||||
if m == nil || m.cfg.Catalog == nil || src == nil || v == nil || v.ID == "" || v.FileID == "" {
|
||||
return
|
||||
}
|
||||
@@ -1049,38 +918,38 @@ func (m *Migrator) preserveCrawledThumbnail(ctx context.Context, src Spider91Loc
|
||||
if commonDir == "" {
|
||||
return
|
||||
}
|
||||
thumbPath, ok := findSpider91ThumbPath(src, v.FileID)
|
||||
thumbPath, ok := findCrawlerThumbPath(src, v.FileID)
|
||||
if !ok {
|
||||
if v.ThumbnailURL == "" {
|
||||
log.Printf("[spider91migrate] %s crawled thumbnail missing before migration cleanup", v.ID)
|
||||
log.Printf("[crawlerupload] %s crawled thumbnail missing before migration cleanup", v.ID)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(commonDir, 0o755); err != nil {
|
||||
log.Printf("[spider91migrate] %s mkdir common thumbs: %v", v.ID, err)
|
||||
log.Printf("[crawlerupload] %s mkdir common thumbs: %v", v.ID, err)
|
||||
return
|
||||
}
|
||||
dst := mediaasset.ThumbnailPathInDir(commonDir, v.ID)
|
||||
if _, err := os.Stat(dst); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
log.Printf("[spider91migrate] %s stat common thumb: %v", v.ID, err)
|
||||
log.Printf("[crawlerupload] %s stat common thumb: %v", v.ID, err)
|
||||
return
|
||||
}
|
||||
if err := copyFileAtomic(thumbPath, dst); err != nil {
|
||||
log.Printf("[spider91migrate] %s preserve crawled thumbnail: %v", v.ID, err)
|
||||
log.Printf("[crawlerupload] %s preserve crawled thumbnail: %v", v.ID, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := m.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
ThumbnailURL: "/p/thumb/" + v.ID,
|
||||
}); err != nil {
|
||||
log.Printf("[spider91migrate] %s update crawled thumbnail url: %v", v.ID, err)
|
||||
log.Printf("[crawlerupload] %s update crawled thumbnail url: %v", v.ID, err)
|
||||
return
|
||||
}
|
||||
v.ThumbnailURL = "/p/thumb/" + v.ID
|
||||
}
|
||||
|
||||
func findSpider91ThumbPath(src Spider91LocalSource, fileID string) (string, bool) {
|
||||
func findCrawlerThumbPath(src LocalSource, fileID string) (string, bool) {
|
||||
thumbBase := stripExt(fileID)
|
||||
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
|
||||
thumbPath, err := src.ThumbPath(thumbBase + ext)
|
||||
@@ -1120,20 +989,19 @@ func copyFileAtomic(src, dst string) error {
|
||||
return os.Rename(tmp, dst)
|
||||
}
|
||||
|
||||
// CleanupSpider91Local 删除已迁移视频的本地 mp4 和 thumb。
|
||||
// CleanupLocal 删除已上传视频的本地 mp4 和 thumb。
|
||||
//
|
||||
// thumb 删除是 best-effort —— 找不到就算了(spider91 thumb 文件名带后缀,
|
||||
// 我们不知道具体是 .jpg 还是别的,逐个尝试常见后缀)。
|
||||
// thumb 删除是 best-effort —— 找不到就算了;逐个尝试常见后缀。
|
||||
//
|
||||
// 暴露成包级函数方便 cleanup 模块复用(任务 6)。
|
||||
func CleanupSpider91Local(src Spider91LocalSource, fileID string) {
|
||||
// 暴露成包级函数方便 cleanup 模块复用。
|
||||
func CleanupLocal(src LocalSource, fileID string) {
|
||||
videoPath, err := src.VideoPath(fileID)
|
||||
if err == nil {
|
||||
if err := os.Remove(videoPath); err != nil && !os.IsNotExist(err) {
|
||||
log.Printf("[spider91migrate] remove local mp4 %s: %v", videoPath, err)
|
||||
log.Printf("[crawlerupload] remove local mp4 %s: %v", videoPath, err)
|
||||
}
|
||||
}
|
||||
// thumb 文件名是 <viewkey>.<ext>;fileID 是 <viewkey>.<videoExt>,
|
||||
// thumb 文件名是 <sourceID>.<ext>;fileID 是 <sourceID>.<videoExt>,
|
||||
// 不一定相同。尝试用 fileID 去掉视频扩展名后拼 thumb 常见后缀。
|
||||
thumbBase := stripExt(fileID)
|
||||
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
|
||||
@@ -1150,7 +1018,7 @@ func stripExt(name string) string {
|
||||
return name[:len(name)-len(ext)]
|
||||
}
|
||||
|
||||
// cleanupOldLocalVideos 是防御性兜底:扫 spider91 drive 本地 videos/ 目录,
|
||||
// cleanupOldLocalVideos 是防御性兜底:扫爬虫本地 videos/ 目录,
|
||||
// 删除所有 catalog 中已经迁移到别处(drive_id != src.ID())的本地残留。
|
||||
//
|
||||
// 与 migrateDrive 的区别:
|
||||
@@ -1158,7 +1026,7 @@ func stripExt(name string) string {
|
||||
// - 不依赖 KeepLatestN —— 哪怕这个孤儿在"最新 N"窗口内,已迁移就该删
|
||||
// - 只看 catalog 状态,不看 mtime
|
||||
//
|
||||
// 正常路径下 migrateDrive 迁移成功后立刻 CleanupSpider91Local,所以这里
|
||||
// 正常路径下 migrateDrive 迁移成功后立刻 CleanupLocal,所以这里
|
||||
// 应该不会有任何工作。极端情况(手工改 catalog、迁移过程中 crash)才会
|
||||
// 找到孤儿。
|
||||
//
|
||||
@@ -1196,7 +1064,7 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, plan migrationPlan
|
||||
continue
|
||||
}
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
log.Printf("[spider91migrate] cleanup remove %s: %v", path, err)
|
||||
log.Printf("[crawlerupload] cleanup remove %s: %v", path, err)
|
||||
continue
|
||||
}
|
||||
// thumb 一并删(best-effort)
|
||||
@@ -1212,44 +1080,3 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, plan migrationPlan
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// backfillFileNames 扫描目标 drive(PikPak、115、123、OneDrive、Google Drive、联通网盘或光鸭网盘)下所有 spider91-* 起始 ID 的视频,
|
||||
// 对文件名不是 desiredPikPakName(...) 期望格式的,调 target.Rename 修正,
|
||||
// 并把 catalog.file_name 同步到新名字。
|
||||
//
|
||||
// 幂等:已经是期望格式的视频不会触发任何调用。
|
||||
//
|
||||
// 返回成功改名的条数。
|
||||
func (m *Migrator) backfillFileNames(ctx context.Context, targetDriveID string, pp uploadTarget) (int, error) {
|
||||
videos, err := m.cfg.Catalog.ListVideosByDriveID(ctx, targetDriveID, 10000)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("list videos: %w", err)
|
||||
}
|
||||
renamed := 0
|
||||
for _, v := range videos {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return renamed, err
|
||||
}
|
||||
if !strings.HasPrefix(v.ID, "spider91-") {
|
||||
continue
|
||||
}
|
||||
want := desiredPikPakName(v.Title, extractViewKey(v.ID), v.Ext)
|
||||
if v.FileName == want {
|
||||
continue
|
||||
}
|
||||
if v.FileID == "" {
|
||||
continue
|
||||
}
|
||||
if err := pp.Rename(ctx, v.FileID, want); err != nil {
|
||||
log.Printf("[spider91migrate] rename %s -> %q: %v", v.ID, want, err)
|
||||
continue
|
||||
}
|
||||
if err := m.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{FileName: want}); err != nil {
|
||||
log.Printf("[spider91migrate] %s update file_name after rename: %v", v.ID, err)
|
||||
// 目标盘已经改名成功,但 catalog 更新失败 —— 下轮会重试。继续。
|
||||
}
|
||||
log.Printf("[spider91migrate] renamed %s on %s: %q -> %q", v.ID, pp.Kind(), v.FileName, want)
|
||||
renamed++
|
||||
}
|
||||
return renamed, nil
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package crawlerupload
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/scriptcrawler"
|
||||
)
|
||||
|
||||
type fakeRegistry struct {
|
||||
byID map[string]drives.Drive
|
||||
}
|
||||
|
||||
func newFakeRegistry() *fakeRegistry {
|
||||
return &fakeRegistry{byID: make(map[string]drives.Drive)}
|
||||
}
|
||||
|
||||
func (r *fakeRegistry) Add(d drives.Drive) {
|
||||
r.byID[d.ID()] = d
|
||||
}
|
||||
|
||||
func (r *fakeRegistry) Get(id string) (drives.Drive, bool) {
|
||||
d, ok := r.byID[id]
|
||||
return d, ok
|
||||
}
|
||||
|
||||
func (r *fakeRegistry) All() []drives.Drive {
|
||||
out := make([]drives.Drive, 0, len(r.byID))
|
||||
for _, d := range r.byID {
|
||||
out = append(out, d)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type fakeUploadDrive struct {
|
||||
id string
|
||||
kind string
|
||||
rootID string
|
||||
mu sync.Mutex
|
||||
uploadCalls int
|
||||
gotBodies map[string][]byte
|
||||
gotParents map[string]string
|
||||
ensureCalls []string
|
||||
}
|
||||
|
||||
func newFakeUploadDrive(id, kind, rootID string) *fakeUploadDrive {
|
||||
return &fakeUploadDrive{
|
||||
id: id,
|
||||
kind: kind,
|
||||
rootID: rootID,
|
||||
gotBodies: make(map[string][]byte),
|
||||
gotParents: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
func (d *fakeUploadDrive) Kind() string { return d.kind }
|
||||
func (d *fakeUploadDrive) ID() string { return d.id }
|
||||
func (d *fakeUploadDrive) RootID() string {
|
||||
return d.rootID
|
||||
}
|
||||
func (d *fakeUploadDrive) Init(context.Context) error { return nil }
|
||||
func (d *fakeUploadDrive) List(context.Context, string) ([]drives.Entry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (d *fakeUploadDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeUploadDrive) StreamURL(context.Context, string) (*drives.StreamLink, error) {
|
||||
return nil, drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeUploadDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakeUploadDrive) EnsureDir(_ context.Context, pathFromRoot string) (string, error) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.ensureCalls = append(d.ensureCalls, pathFromRoot)
|
||||
return d.rootID + "/" + pathFromRoot, nil
|
||||
}
|
||||
func (d *fakeUploadDrive) Rename(context.Context, string, string) error {
|
||||
return nil
|
||||
}
|
||||
func (d *fakeUploadDrive) UploadAndReportHash(_ context.Context, parentID, name string, r io.Reader, _ int64) (UploadResult, error) {
|
||||
body, _ := io.ReadAll(r)
|
||||
d.mu.Lock()
|
||||
d.uploadCalls++
|
||||
d.gotBodies[name] = body
|
||||
d.gotParents[name] = parentID
|
||||
d.mu.Unlock()
|
||||
return UploadResult{FileID: "remote-" + name, Hash: strings.Repeat("a", 40), Size: int64(len(body))}, nil
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*fakeUploadDrive)(nil)
|
||||
var _ uploadTarget = (*fakeUploadDrive)(nil)
|
||||
|
||||
func TestRunOnceUploadsScriptCrawlerLocalVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat := setupCatalog(t)
|
||||
src := setupScriptCrawler(t, "crawler-one")
|
||||
target := newFakeUploadDrive("target-drive", "pikpak", "target-root")
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(target)
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: src.ID(),
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Example Crawler",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{"script_path": "/tmp/example.py", "upload_drive_id": target.ID()},
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert crawler drive: %v", err)
|
||||
}
|
||||
|
||||
videoID := writeCrawlerVideo(t, cat, src, "source-001", ".mp4", []byte("video payload"), true)
|
||||
commonThumbDir := filepath.Join(t.TempDir(), "thumbs")
|
||||
m := New(Config{Catalog: cat, Registry: reg, CommonThumbDir: commonThumbDir})
|
||||
|
||||
if err := m.RunOnce(ctx); err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
|
||||
wantName := desiredUploadName("Sample source-001", "source-001", "mp4")
|
||||
if target.uploadCalls != 1 {
|
||||
t.Fatalf("upload calls = %d, want 1", target.uploadCalls)
|
||||
}
|
||||
if got := string(target.gotBodies[wantName]); got != "video payload" {
|
||||
t.Fatalf("uploaded body = %q, want payload", got)
|
||||
}
|
||||
if got := target.gotParents[wantName]; got != "target-root/Script Crawlers/crawler-one" {
|
||||
t.Fatalf("upload parent = %q, want crawler folder", got)
|
||||
}
|
||||
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != "Script Crawlers/crawler-one" {
|
||||
t.Fatalf("ensure calls = %#v, want crawler upload folder", target.ensureCalls)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(ctx, videoID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DriveID != target.ID() || !strings.HasPrefix(got.FileID, "remote-") {
|
||||
t.Fatalf("catalog target = drive %q file %q, want target drive", got.DriveID, got.FileID)
|
||||
}
|
||||
if got.FileName != wantName {
|
||||
t.Fatalf("file_name = %q, want %q", got.FileName, wantName)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(src.VideosDir(), "source-001.mp4")); !os.IsNotExist(err) {
|
||||
t.Fatalf("local video still exists or stat failed: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(src.ThumbsDir(), "source-001.jpg")); !os.IsNotExist(err) {
|
||||
t.Fatalf("local thumb still exists or stat failed: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(commonThumbDir, videoID+".jpg")); err != nil {
|
||||
t.Fatalf("common thumbnail missing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnceRequiresPerCrawlerUploadTarget(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat := setupCatalog(t)
|
||||
src := setupScriptCrawler(t, "crawler-local-only")
|
||||
target := newFakeUploadDrive("target-drive", "pikpak", "target-root")
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(target)
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: src.ID(),
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Local Only",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{"script_path": "/tmp/example.py"},
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert crawler drive: %v", err)
|
||||
}
|
||||
videoID := writeCrawlerVideo(t, cat, src, "source-002", ".mp4", []byte("video payload"), true)
|
||||
|
||||
m := New(Config{Catalog: cat, Registry: reg})
|
||||
if err := m.RunOnce(ctx); err != nil {
|
||||
t.Fatalf("run once: %v", err)
|
||||
}
|
||||
if target.uploadCalls != 0 {
|
||||
t.Fatalf("upload calls = %d, want 0", target.uploadCalls)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, videoID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DriveID != src.ID() {
|
||||
t.Fatalf("drive_id = %q, want local crawler drive", got.DriveID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdaptUploadTargetRejectsUnsupportedTarget(t *testing.T) {
|
||||
src := scriptcrawler.New(scriptcrawler.Config{ID: "crawler", RootDir: t.TempDir()})
|
||||
_, err := adaptUploadTarget(src)
|
||||
if err == nil || !strings.Contains(err.Error(), "does not support crawler upload") {
|
||||
t.Fatalf("err = %v, want unsupported crawler upload target", err)
|
||||
}
|
||||
}
|
||||
|
||||
func setupCatalog(t *testing.T) *catalog.Catalog {
|
||||
t.Helper()
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "video-site.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
return cat
|
||||
}
|
||||
|
||||
func setupScriptCrawler(t *testing.T, id string) *scriptcrawler.Driver {
|
||||
t.Helper()
|
||||
d := scriptcrawler.New(scriptcrawler.Config{ID: id, RootDir: t.TempDir()})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("scriptcrawler init: %v", err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func writeCrawlerVideo(t *testing.T, cat *catalog.Catalog, d *scriptcrawler.Driver, sourceID, ext string, content []byte, readyAssets bool) string {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
fileID := sourceID + ext
|
||||
videoPath, err := d.VideoPath(fileID)
|
||||
if err != nil {
|
||||
t.Fatalf("video path: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(videoPath, content, 0o644); err != nil {
|
||||
t.Fatalf("write video: %v", err)
|
||||
}
|
||||
thumbPath, err := d.ThumbPath(sourceID + ".jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("thumb path: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(thumbPath, []byte("thumb"), 0o644); err != nil {
|
||||
t.Fatalf("write thumb: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
videoID := scriptcrawler.BuildVideoID(d.ID(), sourceID)
|
||||
previewStatus := "pending"
|
||||
fingerprintStatus := "pending"
|
||||
sampled := ""
|
||||
if readyAssets {
|
||||
previewStatus = "ready"
|
||||
fingerprintStatus = "ready"
|
||||
sampled = strings.Repeat("b", 64)
|
||||
}
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: videoID,
|
||||
DriveID: d.ID(),
|
||||
FileID: fileID,
|
||||
FileName: fileID,
|
||||
Title: "Sample " + sourceID,
|
||||
Author: "tester",
|
||||
Ext: strings.TrimPrefix(ext, "."),
|
||||
Quality: "HD",
|
||||
Size: int64(len(content)),
|
||||
PreviewStatus: previewStatus,
|
||||
FingerprintStatus: fingerprintStatus,
|
||||
SampledSHA256: sampled,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert video: %v", err)
|
||||
}
|
||||
return videoID
|
||||
}
|
||||
+21
-21
@@ -1,13 +1,13 @@
|
||||
package spider91migrate
|
||||
package crawlerupload
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// 期望的 PikPak 文件名格式(方案 B):
|
||||
// 期望的上传文件名格式:
|
||||
//
|
||||
// <sanitized-title>-<viewkey-后8位>.<ext>
|
||||
// <sanitized-title>-<sourceID-后8位>.<ext>
|
||||
//
|
||||
// 例如:
|
||||
//
|
||||
@@ -15,8 +15,8 @@ import (
|
||||
//
|
||||
// 设计目标:
|
||||
// - 文件名一眼能看出视频内容(用 catalog 里的 title)
|
||||
// - 后缀的 viewkey 8 字符保证同标题不会撞名
|
||||
// - 全部字符在常见文件系统、PikPak、HTTP/Aliyun OSS Key 编码里都安全
|
||||
// - 后缀的 sourceID 8 字符保证同标题不会撞名
|
||||
// - 全部字符在常见文件系统、网盘 API、HTTP/Aliyun OSS Key 编码里都安全
|
||||
//
|
||||
// 字符清洗规则(sanitizeTitle):
|
||||
// - 去除控制字符(< 0x20 或 0x7F)
|
||||
@@ -85,47 +85,47 @@ func truncateRunes(s string, maxRunes int) string {
|
||||
return s
|
||||
}
|
||||
|
||||
// extractViewKey 从 video.ID("spider91-<driveID>-<viewkey>")里
|
||||
// 取出最后一段 viewkey。
|
||||
// extractSourceID 从 video.ID("<kind>-<driveID>-<sourceID>")里
|
||||
// 取出最后一段 sourceID。
|
||||
//
|
||||
// driveID 中如果有 "-" 不影响(用 LastIndex),viewkey 本身(91 网站的
|
||||
// view 标识)目前都是纯 hex 或纯数字,不包含 "-"。
|
||||
func extractViewKey(videoID string) string {
|
||||
// driveID 中如果有 "-" 不影响(用 LastIndex)。爬虫脚本应提供不包含 "-"
|
||||
// 的稳定 source_id;如果包含 "-",这里会取最后一段作为文件名后缀。
|
||||
func extractSourceID(videoID string) string {
|
||||
if i := strings.LastIndex(videoID, "-"); i >= 0 {
|
||||
return videoID[i+1:]
|
||||
}
|
||||
return videoID
|
||||
}
|
||||
|
||||
// viewKeySuffix 取 viewkey 的最后 N 个字符;不足 N 返回原字符串。
|
||||
// sourceIDSuffix 取 sourceID 的最后 N 个字符;不足 N 返回原字符串。
|
||||
//
|
||||
// 默认 N=8(足够稀疏避免标题撞名时的同名冲突)。
|
||||
const viewKeySuffixLen = 8
|
||||
const sourceIDSuffixLen = 8
|
||||
|
||||
func viewKeySuffix(viewkey string) string {
|
||||
r := []rune(viewkey)
|
||||
if len(r) <= viewKeySuffixLen {
|
||||
func sourceIDSuffix(sourceID string) string {
|
||||
r := []rune(sourceID)
|
||||
if len(r) <= sourceIDSuffixLen {
|
||||
return string(r)
|
||||
}
|
||||
return string(r[len(r)-viewKeySuffixLen:])
|
||||
return string(r[len(r)-sourceIDSuffixLen:])
|
||||
}
|
||||
|
||||
// desiredPikPakName 构造 spider91 视频在 PikPak 上的期望文件名。
|
||||
// desiredUploadName 构造爬虫视频上传到目标网盘时的期望文件名。
|
||||
//
|
||||
// desiredPikPakName("超白大奶律师约炮", "476fa8bf4b47e672d2fa", "mp4")
|
||||
// desiredUploadName("超白大奶律师约炮", "476fa8bf4b47e672d2fa", "mp4")
|
||||
// → "超白大奶律师约炮-72d2fa.mp4" // 实际是 e672d2fa(取最后 8)
|
||||
//
|
||||
// ext 不带前导点;空时默认 mp4。
|
||||
func desiredPikPakName(title, viewkey, ext string) string {
|
||||
func desiredUploadName(title, sourceID, ext string) string {
|
||||
clean := sanitizeTitle(title)
|
||||
suffix := viewKeySuffix(strings.TrimSpace(viewkey))
|
||||
suffix := sourceIDSuffix(strings.TrimSpace(sourceID))
|
||||
ext = strings.TrimSpace(ext)
|
||||
ext = strings.TrimPrefix(ext, ".")
|
||||
if ext == "" {
|
||||
ext = "mp4"
|
||||
}
|
||||
if suffix == "" {
|
||||
// viewkey 缺失时退化成 "<title>.<ext>"
|
||||
// sourceID 缺失时退化成 "<title>.<ext>"
|
||||
return clean + "." + ext
|
||||
}
|
||||
return clean + "-" + suffix + "." + ext
|
||||
+18
-18
@@ -1,4 +1,4 @@
|
||||
package spider91migrate
|
||||
package crawlerupload
|
||||
|
||||
import (
|
||||
"strings"
|
||||
@@ -13,11 +13,11 @@ func TestSanitizeTitleHandlesCommonCases(t *testing.T) {
|
||||
{"hello", "hello"},
|
||||
{" hello ", "hello"},
|
||||
{"hello\nworld", "hello world"},
|
||||
{"hello / world", "hello world"}, // 单 forbidden 折叠成空格
|
||||
{"hello / world", "hello world"}, // 单 forbidden 折叠成空格
|
||||
{"a/b\\c:d*e?f\"g<h>i|j", "a b c d e f g h i j"},
|
||||
{"a b", "a b"}, // 多空格折叠
|
||||
{"a b", "a b"}, // 多空格折叠
|
||||
{"a\t\nb", "a b"},
|
||||
{"...trim.dots...", "trim.dots"}, // 首尾点号被 trim 掉
|
||||
{"...trim.dots...", "trim.dots"}, // 首尾点号被 trim 掉
|
||||
{"control\x01char\x1f\x7f", "controlchar"}, // 控制字符直接丢弃
|
||||
{"", "video"}, // 空串回退
|
||||
{" / ", "video"}, // 全是 forbidden+空白 → 回退
|
||||
@@ -51,22 +51,22 @@ func TestSanitizeTitleKeepsCJKAndUnicode(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractViewKey(t *testing.T) {
|
||||
func TestExtractSourceID(t *testing.T) {
|
||||
cases := []struct{ in, want string }{
|
||||
{"spider91-91Spider-476fa8bf4b47e672d2fa", "476fa8bf4b47e672d2fa"},
|
||||
{"spider91-91Spider-1587338723", "1587338723"},
|
||||
{"spider91-some-drive-with-dashes-vk001", "vk001"}, // LastIndex 拿尾段
|
||||
{"scriptcrawler-demo-476fa8bf4b47e672d2fa", "476fa8bf4b47e672d2fa"},
|
||||
{"scriptcrawler-demo-1587338723", "1587338723"},
|
||||
{"scriptcrawler-some-drive-with-dashes-vk001", "vk001"}, // LastIndex 拿尾段
|
||||
{"no-dashes-after-prefix", "prefix"},
|
||||
{"single", "single"}, // 没 dash → 原样返回
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := extractViewKey(c.in); got != c.want {
|
||||
t.Errorf("extractViewKey(%q) = %q, want %q", c.in, got, c.want)
|
||||
if got := extractSourceID(c.in); got != c.want {
|
||||
t.Errorf("extractSourceID(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestViewKeySuffix(t *testing.T) {
|
||||
func TestSourceIDSuffix(t *testing.T) {
|
||||
cases := []struct{ in, want string }{
|
||||
{"476fa8bf4b47e672d2fa", "e672d2fa"},
|
||||
{"1587338723", "87338723"},
|
||||
@@ -76,15 +76,15 @@ func TestViewKeySuffix(t *testing.T) {
|
||||
{"123456789", "23456789"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := viewKeySuffix(c.in); got != c.want {
|
||||
t.Errorf("viewKeySuffix(%q) = %q, want %q", c.in, got, c.want)
|
||||
if got := sourceIDSuffix(c.in); got != c.want {
|
||||
t.Errorf("sourceIDSuffix(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDesiredPikPakName(t *testing.T) {
|
||||
func TestDesiredUploadName(t *testing.T) {
|
||||
cases := []struct {
|
||||
title, viewkey, ext, want string
|
||||
title, sourceID, ext, want string
|
||||
}{
|
||||
{
|
||||
"超白大奶律师约炮第一季",
|
||||
@@ -112,7 +112,7 @@ func TestDesiredPikPakName(t *testing.T) {
|
||||
},
|
||||
{
|
||||
"title",
|
||||
"", // 空 viewkey → 退化成 "<title>.<ext>"
|
||||
"", // 空 sourceID → 退化成 "<title>.<ext>"
|
||||
"webm",
|
||||
"title.webm",
|
||||
},
|
||||
@@ -130,9 +130,9 @@ func TestDesiredPikPakName(t *testing.T) {
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := desiredPikPakName(c.title, c.viewkey, c.ext)
|
||||
got := desiredUploadName(c.title, c.sourceID, c.ext)
|
||||
if got != c.want {
|
||||
t.Errorf("desiredPikPakName(%q,%q,%q) = %q, want %q", c.title, c.viewkey, c.ext, got, c.want)
|
||||
t.Errorf("desiredUploadName(%q,%q,%q) = %q, want %q", c.title, c.sourceID, c.ext, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -227,10 +227,10 @@ func TestEnsureDirAndRenameUseGoogleDriveFileAPI(t *testing.T) {
|
||||
if err := json.NewDecoder(r.Body).Decode(&meta); err != nil {
|
||||
t.Fatalf("decode mkdir body: %v", err)
|
||||
}
|
||||
if meta.Name != "91 Spider" || len(meta.Parents) != 1 || meta.Parents[0] != "root" || meta.MimeType != "application/vnd.google-apps.folder" {
|
||||
if meta.Name != "Crawler Uploads" || len(meta.Parents) != 1 || meta.Parents[0] != "root" || meta.MimeType != "application/vnd.google-apps.folder" {
|
||||
t.Fatalf("mkdir body = %+v", meta)
|
||||
}
|
||||
writeTestJSON(w, driveFile{ID: "folder-91", Name: "91 Spider", MimeType: "application/vnd.google-apps.folder"})
|
||||
writeTestJSON(w, driveFile{ID: "folder-crawler", Name: "Crawler Uploads", MimeType: "application/vnd.google-apps.folder"})
|
||||
case r.Method == http.MethodPatch && r.URL.Path == "/drive/v3/files/file-1":
|
||||
renamed = true
|
||||
var body map[string]string
|
||||
@@ -251,12 +251,12 @@ func TestEnsureDirAndRenameUseGoogleDriveFileAPI(t *testing.T) {
|
||||
d.accessToken = "access"
|
||||
d.listInterval = -1
|
||||
|
||||
dirID, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
dirID, err := d.EnsureDir(context.Background(), "Crawler Uploads")
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureDir() error = %v", err)
|
||||
}
|
||||
if dirID != "folder-91" || !madeDir {
|
||||
t.Fatalf("dirID/madeDir = %q/%v, want folder-91/true", dirID, madeDir)
|
||||
if dirID != "folder-crawler" || !madeDir {
|
||||
t.Fatalf("dirID/madeDir = %q/%v, want folder-crawler/true", dirID, madeDir)
|
||||
}
|
||||
if err := d.Rename(context.Background(), "file-1", "new-name.mp4"); err != nil {
|
||||
t.Fatalf("Rename() error = %v", err)
|
||||
|
||||
@@ -327,8 +327,8 @@ func TestScannerPersistsLocalStorageSTRM(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.Ext != "strm" || got.FileID != fileID || got.Category != "collection" {
|
||||
t.Fatalf("video = %#v, want local strm video in collection", got)
|
||||
if got.Ext != "strm" || got.FileID != fileID || got.ParentID != encodeRel("collection") {
|
||||
t.Fatalf("video = %#v, want local strm video under collection", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -366,8 +366,8 @@ func TestScannerPersistsLocalStorageVideo(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DriveID != "local" || got.FileID != fileID || got.Category != "collection" {
|
||||
t.Fatalf("video = %#v, want local drive video in collection", got)
|
||||
if got.DriveID != "local" || got.FileID != fileID || got.ParentID != encodeRel("collection") {
|
||||
t.Fatalf("video = %#v, want local drive video under collection", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -20,11 +20,12 @@ import (
|
||||
)
|
||||
|
||||
type Driver struct {
|
||||
id string
|
||||
cookie string
|
||||
rootID string
|
||||
client *sdk.Pan115Client
|
||||
ua string
|
||||
id string
|
||||
cookie string
|
||||
rootID string
|
||||
client *sdk.Pan115Client
|
||||
ua string
|
||||
uploadTempDir string
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
@@ -32,10 +33,11 @@ type Driver struct {
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ID string
|
||||
Cookie string // 形如 "UID=xxx; CID=xxx; SEID=xxx; KID=xxx"
|
||||
RootID string // 默认 "0"
|
||||
UA string // 默认 UA115Browser
|
||||
ID string
|
||||
Cookie string // 形如 "UID=xxx; CID=xxx; SEID=xxx; KID=xxx"
|
||||
RootID string // 默认 "0"
|
||||
UA string // 默认 UA115Browser
|
||||
UploadTempDir string
|
||||
}
|
||||
|
||||
func New(c Config) *Driver {
|
||||
@@ -48,11 +50,12 @@ func New(c Config) *Driver {
|
||||
ua = sdk.UA115Browser
|
||||
}
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
cookie: c.Cookie,
|
||||
rootID: rootID,
|
||||
ua: ua,
|
||||
listInterval: 2 * time.Second,
|
||||
id: c.ID,
|
||||
cookie: c.Cookie,
|
||||
rootID: rootID,
|
||||
ua: ua,
|
||||
uploadTempDir: strings.TrimSpace(c.UploadTempDir),
|
||||
listInterval: 2 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -347,7 +350,7 @@ func (d *Driver) UploadAndReportSha1(ctx context.Context, parentID, name string,
|
||||
parentID = d.rootID
|
||||
}
|
||||
|
||||
tmp, sha1Hex, written, err := bufferAndHashSha1(r, size)
|
||||
tmp, sha1Hex, written, err := bufferAndHashSha1(d.uploadTempDir, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
@@ -472,8 +475,14 @@ func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
// 返回临时文件(位置在末尾,需调用方 Seek 回 0)、SHA1 hex 大写、实际字节数。
|
||||
//
|
||||
// 调用方负责 Close + Remove 临时文件。
|
||||
func bufferAndHashSha1(r io.Reader, declaredSize int64) (*os.File, string, int64, error) {
|
||||
tmp, err := os.CreateTemp("", "p115-upload-*.bin")
|
||||
func bufferAndHashSha1(tempDir string, r io.Reader, declaredSize int64) (*os.File, string, int64, error) {
|
||||
tempDir = strings.TrimSpace(tempDir)
|
||||
if tempDir != "" {
|
||||
if err := os.MkdirAll(tempDir, 0o755); err != nil {
|
||||
return nil, "", 0, fmt.Errorf("p115 upload: create tmp dir: %w", err)
|
||||
}
|
||||
}
|
||||
tmp, err := os.CreateTemp(tempDir, "p115-upload-*.bin")
|
||||
if err != nil {
|
||||
return nil, "", 0, fmt.Errorf("p115 upload: create tmp: %w", err)
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -86,7 +87,7 @@ func TestBufferAndHashSha1(t *testing.T) {
|
||||
wantHex := strings.ToUpper(hex.EncodeToString(want[:]))
|
||||
|
||||
t.Run("declared size matches", func(t *testing.T) {
|
||||
tmp, gotHex, n, err := bufferAndHashSha1(bytes.NewReader(body), int64(len(body)))
|
||||
tmp, gotHex, n, err := bufferAndHashSha1("", bytes.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("bufferAndHashSha1 returned error: %v", err)
|
||||
}
|
||||
@@ -111,14 +112,14 @@ func TestBufferAndHashSha1(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("declared size mismatch returns error", func(t *testing.T) {
|
||||
_, _, _, err := bufferAndHashSha1(bytes.NewReader(body), int64(len(body))+1)
|
||||
_, _, _, err := bufferAndHashSha1("", bytes.NewReader(body), int64(len(body))+1)
|
||||
if err == nil {
|
||||
t.Fatal("expected size mismatch error, got nil")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("declared size zero is unchecked", func(t *testing.T) {
|
||||
tmp, gotHex, n, err := bufferAndHashSha1(bytes.NewReader(body), 0)
|
||||
tmp, gotHex, n, err := bufferAndHashSha1("", bytes.NewReader(body), 0)
|
||||
if err != nil {
|
||||
t.Fatalf("bufferAndHashSha1 returned error: %v", err)
|
||||
}
|
||||
@@ -130,6 +131,18 @@ func TestBufferAndHashSha1(t *testing.T) {
|
||||
t.Errorf("written = %d, want %d", n, len(body))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("uses configured temp dir", func(t *testing.T) {
|
||||
tempDir := filepath.Join(t.TempDir(), "upload-tmp")
|
||||
tmp, _, _, err := bufferAndHashSha1(tempDir, bytes.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("bufferAndHashSha1 returned error: %v", err)
|
||||
}
|
||||
defer cleanup(tmp)
|
||||
if gotDir := filepath.Dir(tmp.Name()); gotDir != tempDir {
|
||||
t.Fatalf("tmp dir = %q, want %q", gotDir, tempDir)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestUploadAndReportSha1RejectsInvalidArgs 检查空 reader / 空 name / 负 size 在
|
||||
|
||||
@@ -70,6 +70,7 @@ type Driver struct {
|
||||
httpClient *http.Client
|
||||
|
||||
onTokenUpdate func(access string)
|
||||
uploadTempDir string
|
||||
|
||||
tokenMu sync.RWMutex
|
||||
|
||||
@@ -90,6 +91,7 @@ type Config struct {
|
||||
|
||||
MainAPIBaseURL string
|
||||
LoginAPIBaseURL string
|
||||
UploadTempDir string
|
||||
|
||||
OnTokenUpdate func(access string)
|
||||
}
|
||||
@@ -123,6 +125,7 @@ func New(c Config) *Driver {
|
||||
referer: defaultReferer,
|
||||
userAgent: defaultUserAgent,
|
||||
onTokenUpdate: c.OnTokenUpdate,
|
||||
uploadTempDir: strings.TrimSpace(c.UploadTempDir),
|
||||
client: resty.New().
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
@@ -289,7 +292,7 @@ func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string,
|
||||
parentID = d.rootID
|
||||
}
|
||||
|
||||
tmp, md5Hex, actualSize, err := bufferAndHashMD5(r, size)
|
||||
tmp, md5Hex, actualSize, err := bufferAndHashMD5(d.uploadTempDir, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
@@ -1058,8 +1061,14 @@ func splitPath(p string) []string {
|
||||
return strings.Split(p, "/")
|
||||
}
|
||||
|
||||
func bufferAndHashMD5(r io.Reader, declaredSize int64) (*os.File, string, int64, error) {
|
||||
tmp, err := os.CreateTemp("", "p123-upload-*.bin")
|
||||
func bufferAndHashMD5(tempDir string, r io.Reader, declaredSize int64) (*os.File, string, int64, error) {
|
||||
tempDir = strings.TrimSpace(tempDir)
|
||||
if tempDir != "" {
|
||||
if err := os.MkdirAll(tempDir, 0o755); err != nil {
|
||||
return nil, "", 0, fmt.Errorf("123pan upload: create tmp dir: %w", err)
|
||||
}
|
||||
}
|
||||
tmp, err := os.CreateTemp(tempDir, "p123-upload-*.bin")
|
||||
if err != nil {
|
||||
return nil, "", 0, fmt.Errorf("123pan upload: create tmp: %w", err)
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -458,6 +460,29 @@ func TestUploadPresignedPUT429ReturnsRateLimitError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBufferAndHashMD5UsesConfiguredTempDir(t *testing.T) {
|
||||
body := []byte("hello-123-upload-test")
|
||||
tempDir := filepath.Join(t.TempDir(), "upload-tmp")
|
||||
tmp, gotHex, n, err := bufferAndHashMD5(tempDir, bytes.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("bufferAndHashMD5 returned error: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = tmp.Close()
|
||||
_ = os.Remove(tmp.Name())
|
||||
}()
|
||||
if gotDir := filepath.Dir(tmp.Name()); gotDir != tempDir {
|
||||
t.Fatalf("tmp dir = %q, want %q", gotDir, tempDir)
|
||||
}
|
||||
want := md5.Sum(body)
|
||||
if gotHex != fmt.Sprintf("%x", want) {
|
||||
t.Fatalf("md5 = %s, want %x", gotHex, want)
|
||||
}
|
||||
if n != int64(len(body)) {
|
||||
t.Fatalf("written = %d, want %d", n, len(body))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameSendsExpectedBody(t *testing.T) {
|
||||
var renameRequest map[string]any
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -47,6 +47,7 @@ type Driver struct {
|
||||
client *resty.Client
|
||||
onTokenUpdate func(access, refresh, captcha, deviceID string)
|
||||
uploadToOSSFunc func(context.Context, *s3Params, io.Reader) error
|
||||
uploadTempDir string
|
||||
|
||||
// captchaMu serializes captcha-token refreshes triggered by 4002 / 9
|
||||
// recovery in requestOnce. Without it, N concurrent callers all hitting
|
||||
@@ -77,6 +78,7 @@ type Config struct {
|
||||
DeviceID string
|
||||
RootID string
|
||||
DisableMediaLink bool
|
||||
UploadTempDir string
|
||||
OnTokenUpdate func(access, refresh, captcha, deviceID string)
|
||||
}
|
||||
|
||||
@@ -109,6 +111,7 @@ func New(c Config) *Driver {
|
||||
deviceID: deviceID,
|
||||
disableMediaLink: c.DisableMediaLink,
|
||||
onTokenUpdate: c.OnTokenUpdate,
|
||||
uploadTempDir: strings.TrimSpace(c.UploadTempDir),
|
||||
client: resty.New().
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
|
||||
@@ -110,7 +110,7 @@ func TestEnsureDirReusesExistingFolder(t *testing.T) {
|
||||
"files": []map[string]any{{
|
||||
"id": "existing-folder-id",
|
||||
"kind": "drive#folder",
|
||||
"name": "91 Spider",
|
||||
"name": "Crawler Uploads",
|
||||
}},
|
||||
})
|
||||
case http.MethodPost:
|
||||
@@ -124,7 +124,7 @@ func TestEnsureDirReusesExistingFolder(t *testing.T) {
|
||||
defer srv.Close()
|
||||
|
||||
d := newTestDriver(t, srv)
|
||||
got, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
got, err := d.EnsureDir(context.Background(), "Crawler Uploads")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure dir: %v", err)
|
||||
}
|
||||
@@ -150,7 +150,7 @@ func TestEnsureDirCreatesMissingFolder(t *testing.T) {
|
||||
writePikPakJSON(t, w, map[string]any{
|
||||
"id": "new-folder-id",
|
||||
"kind": "drive#folder",
|
||||
"name": "91 Spider",
|
||||
"name": "Crawler Uploads",
|
||||
})
|
||||
default:
|
||||
t.Fatalf("unexpected method %s", r.Method)
|
||||
@@ -160,14 +160,14 @@ func TestEnsureDirCreatesMissingFolder(t *testing.T) {
|
||||
defer srv.Close()
|
||||
|
||||
d := newTestDriver(t, srv)
|
||||
id, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
id, err := d.EnsureDir(context.Background(), "Crawler Uploads")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure dir: %v", err)
|
||||
}
|
||||
if id != "new-folder-id" {
|
||||
t.Fatalf("dir id = %q, want new-folder-id", id)
|
||||
}
|
||||
if got.Kind != "drive#folder" || got.ParentID != "root-id" || got.Name != "91 Spider" {
|
||||
if got.Kind != "drive#folder" || got.ParentID != "root-id" || got.Name != "Crawler Uploads" {
|
||||
t.Fatalf("create folder body = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,7 +64,7 @@ func isCaptchaTokenRejectedCode(code int64) bool {
|
||||
}
|
||||
|
||||
// APIError is the public alias for the PikPak API error response. Callers
|
||||
// outside this package (e.g. the spider91→PikPak migrator, tests) can either
|
||||
// outside this package (e.g. crawler upload workers and tests) can either
|
||||
// construct it for fakes or unwrap it via errors.As. Prefer IsCaptchaError
|
||||
// over hard-coding the numeric error codes.
|
||||
type APIError = errResp
|
||||
|
||||
@@ -39,8 +39,7 @@ const (
|
||||
ossSecurityTokenHeaderName = "X-OSS-Security-Token"
|
||||
ossUserAgent = "aliyun-sdk-android/2.9.13(Linux/Android 14/M2004j7ac;UKQ1.231108.001)"
|
||||
// 单次 PutObject 的硬上限(OSS 文档限制 5GiB;保守用 5GiB-1)。
|
||||
// spider91 视频通常 ~100MiB,远低于该值。超过则需走 multipart,
|
||||
// 当前未实现,遇到会显式报错。
|
||||
// 超过该值需走 multipart;当前未实现,遇到会显式报错。
|
||||
maxSinglePutSize = 5*1024*1024*1024 - 1
|
||||
// 首次上传失败后最多再重试 3 次。每次重试都会重新申请 PikPak
|
||||
// upload session,以避开偶发不可解析/不可达的临时上传 endpoint。
|
||||
@@ -79,6 +78,20 @@ type UploadResult struct {
|
||||
Size int64
|
||||
}
|
||||
|
||||
type preparedUploadBody struct {
|
||||
reader io.ReadSeeker
|
||||
start int64
|
||||
cleanup func()
|
||||
}
|
||||
|
||||
func (b preparedUploadBody) rewind() error {
|
||||
if b.reader == nil {
|
||||
return errors.New("pikpak upload: nil upload body")
|
||||
}
|
||||
_, err := b.reader.Seek(b.start, io.SeekStart)
|
||||
return err
|
||||
}
|
||||
|
||||
// Upload 实现 drives.Drive 接口;只返回 fileID。
|
||||
// 完整上传元数据见 UploadAndReportHash。
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
@@ -91,7 +104,7 @@ func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader,
|
||||
|
||||
// UploadAndReportHash 上传并返回 file ID + GCID + 实际字节数。
|
||||
//
|
||||
// 用于 spider91 → PikPak 迁移 worker:上传完后直接把 hash 写回 catalog
|
||||
// 用于 crawler upload worker:上传完后直接把 hash 写回 catalog
|
||||
// 的 content_hash 字段,避免再读一次本地文件做 hash。
|
||||
//
|
||||
// 参数:
|
||||
@@ -104,8 +117,7 @@ func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader,
|
||||
// - 必须先算 GCID 再申请上传会话(PikPak API 要求 hash 字段),
|
||||
// 所以这里先 io.Copy 到临时文件并同步算 GCID。
|
||||
// - 命中秒传时不发任何字节;否则用 OSS PutObject 上传。
|
||||
// - 单次 PutObject 上限保守用 5GiB-1。spider91 视频远小于此值,
|
||||
// 超出该值会报错(暂不实现 multipart)。
|
||||
// - 单次 PutObject 上限保守用 5GiB-1,超出该值会报错(暂不实现 multipart)。
|
||||
func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
if r == nil {
|
||||
return UploadResult{}, errors.New("pikpak upload: nil reader")
|
||||
@@ -125,15 +137,15 @@ func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string,
|
||||
parentID = d.rootID
|
||||
}
|
||||
|
||||
// 1) 把 r 全量缓冲到临时文件,同时算 GCID。
|
||||
tmp, gcidHex, actualSize, err := bufferAndHashGCID(r, size)
|
||||
// 1) 算 GCID,并准备一个可重试读取的 body。爬虫迁移传入的是
|
||||
// *os.File,可直接复用原文件,避免再占用一份视频大小的临时空间。
|
||||
body, gcidHex, actualSize, err := d.prepareUploadBody(r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
defer func() {
|
||||
_ = tmp.Close()
|
||||
_ = os.Remove(tmp.Name())
|
||||
}()
|
||||
if body.cleanup != nil {
|
||||
defer body.cleanup()
|
||||
}
|
||||
|
||||
result := UploadResult{Hash: gcidHex, Size: actualSize}
|
||||
var lastErr error
|
||||
@@ -155,7 +167,7 @@ func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string,
|
||||
continue
|
||||
}
|
||||
|
||||
out, err := d.completeUploadAttempt(ctx, tmp, parentID, name, result, resp)
|
||||
out, err := d.completeUploadAttempt(ctx, body, parentID, name, result, resp)
|
||||
if err == nil {
|
||||
return out, nil
|
||||
}
|
||||
@@ -190,7 +202,7 @@ func (d *Driver) requestUploadSession(ctx context.Context, parentID, name string
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (d *Driver) completeUploadAttempt(ctx context.Context, tmp *os.File, parentID, name string, result UploadResult, resp uploadTaskData) (UploadResult, error) {
|
||||
func (d *Driver) completeUploadAttempt(ctx context.Context, body preparedUploadBody, parentID, name string, result UploadResult, resp uploadTaskData) (UploadResult, error) {
|
||||
// 命中秒传:服务端已经知道这个 hash,直接返回新文件 ID。
|
||||
if resp.Resumable == nil {
|
||||
if resp.File.ID != "" {
|
||||
@@ -207,10 +219,10 @@ func (d *Driver) completeUploadAttempt(ctx context.Context, tmp *os.File, parent
|
||||
}
|
||||
|
||||
// 未命中秒传:把字节传到 S3 兼容存储。
|
||||
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
|
||||
return UploadResult{}, fmt.Errorf("pikpak upload: seek tmp: %w", err)
|
||||
if err := body.rewind(); err != nil {
|
||||
return UploadResult{}, fmt.Errorf("pikpak upload: rewind body: %w", err)
|
||||
}
|
||||
if err := d.uploadToOSS(ctx, &resp.Resumable.Params, tmp); err != nil {
|
||||
if err := d.uploadToOSS(ctx, &resp.Resumable.Params, body.reader); err != nil {
|
||||
return UploadResult{}, fmt.Errorf("pikpak upload: oss put: %w", err)
|
||||
}
|
||||
|
||||
@@ -279,12 +291,62 @@ func isRetryablePikPakUploadError(err error) bool {
|
||||
strings.Contains(text, "service unavailable")
|
||||
}
|
||||
|
||||
func (d *Driver) prepareUploadBody(r io.Reader, size int64) (preparedUploadBody, string, int64, error) {
|
||||
if rs, ok := r.(io.ReadSeeker); ok {
|
||||
gcidHex, actualSize, start, err := hashGCIDFromReadSeeker(rs, size)
|
||||
if err != nil {
|
||||
return preparedUploadBody{}, "", 0, err
|
||||
}
|
||||
return preparedUploadBody{reader: rs, start: start, cleanup: func() {}}, gcidHex, actualSize, nil
|
||||
}
|
||||
|
||||
tmp, gcidHex, actualSize, err := bufferAndHashGCID(d.uploadTempDir, r, size)
|
||||
if err != nil {
|
||||
return preparedUploadBody{}, "", 0, err
|
||||
}
|
||||
return preparedUploadBody{
|
||||
reader: tmp,
|
||||
start: 0,
|
||||
cleanup: func() {
|
||||
_ = tmp.Close()
|
||||
_ = os.Remove(tmp.Name())
|
||||
},
|
||||
}, gcidHex, actualSize, nil
|
||||
}
|
||||
|
||||
func hashGCIDFromReadSeeker(r io.ReadSeeker, size int64) (string, int64, int64, error) {
|
||||
start, err := r.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return "", 0, 0, fmt.Errorf("pikpak upload: seek body: %w", err)
|
||||
}
|
||||
|
||||
h := NewGCID(size)
|
||||
written, copyErr := io.Copy(h, r)
|
||||
_, seekErr := r.Seek(start, io.SeekStart)
|
||||
if copyErr != nil {
|
||||
return "", 0, start, fmt.Errorf("pikpak upload: hash body: %w", copyErr)
|
||||
}
|
||||
if seekErr != nil {
|
||||
return "", 0, start, fmt.Errorf("pikpak upload: rewind body: %w", seekErr)
|
||||
}
|
||||
if size > 0 && written != size {
|
||||
return "", 0, start, fmt.Errorf("pikpak upload: size mismatch: declared %d, copied %d", size, written)
|
||||
}
|
||||
return strings.ToUpper(hex.EncodeToString(h.Sum(nil))), written, start, nil
|
||||
}
|
||||
|
||||
// bufferAndHashGCID 把 r 复制到一个临时文件,同时计算 GCID。
|
||||
// 返回临时文件(位置在末尾,需要调用方 Seek 回 0)、GCID hex 大写、实际写入字节数。
|
||||
// 返回临时文件(位置在末尾,需要调用方 Seek 回 start)、GCID hex 大写、实际写入字节数。
|
||||
//
|
||||
// 调用方负责 Close + Remove 临时文件。
|
||||
func bufferAndHashGCID(r io.Reader, size int64) (*os.File, string, int64, error) {
|
||||
tmp, err := os.CreateTemp("", "pikpak-upload-*.bin")
|
||||
func bufferAndHashGCID(tempDir string, r io.Reader, size int64) (*os.File, string, int64, error) {
|
||||
tempDir = strings.TrimSpace(tempDir)
|
||||
if tempDir != "" {
|
||||
if err := os.MkdirAll(tempDir, 0o755); err != nil {
|
||||
return nil, "", 0, fmt.Errorf("pikpak upload: create tmp dir: %w", err)
|
||||
}
|
||||
}
|
||||
tmp, err := os.CreateTemp(tempDir, "pikpak-upload-*.bin")
|
||||
if err != nil {
|
||||
return nil, "", 0, fmt.Errorf("pikpak upload: create tmp: %w", err)
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -142,6 +144,80 @@ func TestUploadInstantSuccessReturnsFileID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadUsesReadSeekerWithoutTempCopy(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"upload_type": "UPLOAD_TYPE_RESUMABLE",
|
||||
"resumable": null,
|
||||
"file": {"id": "instant-file-id", "name": "test.mp4", "kind": "drive#file"}
|
||||
}`))
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
uploadTempDir := filepath.Join(t.TempDir(), "upload-tmp")
|
||||
d.uploadTempDir = uploadTempDir
|
||||
|
||||
data := bytes.Repeat([]byte{0x31}, 1024)
|
||||
path := filepath.Join(t.TempDir(), "video.bin")
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatalf("write source: %v", err)
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open source: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
id, err := d.Upload(context.Background(), "parent-id", "test.mp4", f, int64(len(data)))
|
||||
if err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
if id != "instant-file-id" {
|
||||
t.Fatalf("file id = %q, want instant-file-id", id)
|
||||
}
|
||||
if _, err := os.Stat(uploadTempDir); !os.IsNotExist(err) {
|
||||
t.Fatalf("upload temp dir stat err = %v, want not created for read seeker input", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadBuffersNonSeekReaderInConfiguredTempDir(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"upload_type": "UPLOAD_TYPE_RESUMABLE",
|
||||
"resumable": null,
|
||||
"file": {"id": "instant-file-id", "name": "test.mp4", "kind": "drive#file"}
|
||||
}`))
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
uploadTempDir := filepath.Join(t.TempDir(), "upload-tmp")
|
||||
d.uploadTempDir = uploadTempDir
|
||||
|
||||
data := bytes.Repeat([]byte{0x42}, 1024)
|
||||
id, err := d.Upload(context.Background(), "parent-id", "test.mp4", bytes.NewBuffer(data), int64(len(data)))
|
||||
if err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
if id != "instant-file-id" {
|
||||
t.Fatalf("file id = %q, want instant-file-id", id)
|
||||
}
|
||||
entries, err := os.ReadDir(uploadTempDir)
|
||||
if err != nil {
|
||||
t.Fatalf("read upload temp dir: %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("upload temp dir entries = %d, want cleaned", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadInstantSuccessFallsBackToListWhenFileIDMissing(t *testing.T) {
|
||||
listCalled := false
|
||||
mux := http.NewServeMux()
|
||||
@@ -304,7 +380,7 @@ func TestUploadRejectsInvalidArguments(t *testing.T) {
|
||||
func TestBufferAndHashGCIDDetectsSizeMismatch(t *testing.T) {
|
||||
src := bytes.NewReader([]byte("hello"))
|
||||
// 声明 size=10 但实际只有 5 字节
|
||||
_, _, _, err := bufferAndHashGCID(src, 10)
|
||||
_, _, _, err := bufferAndHashGCID("", src, 10)
|
||||
if err == nil {
|
||||
t.Fatal("expected size mismatch error")
|
||||
}
|
||||
@@ -315,7 +391,7 @@ func TestBufferAndHashGCIDDetectsSizeMismatch(t *testing.T) {
|
||||
|
||||
func TestBufferAndHashGCIDComputesCorrectHash(t *testing.T) {
|
||||
data := bytes.Repeat([]byte{0x55}, 1024)
|
||||
tmp, hex, written, err := bufferAndHashGCID(bytes.NewReader(data), int64(len(data)))
|
||||
tmp, hex, written, err := bufferAndHashGCID("", bytes.NewReader(data), int64(len(data)))
|
||||
if err != nil {
|
||||
t.Fatalf("buffer: %v", err)
|
||||
}
|
||||
|
||||
@@ -41,7 +41,6 @@ type CrawlerConfig struct {
|
||||
Driver *Driver
|
||||
Catalog *catalog.Catalog
|
||||
CrawlerName string
|
||||
SourceKind string
|
||||
PythonPath string
|
||||
FFmpegPath string
|
||||
FFprobePath string
|
||||
@@ -145,7 +144,6 @@ type Event struct {
|
||||
DetailURL string `json:"detail_url,omitempty"`
|
||||
Author string `json:"author,omitempty"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Category string `json:"category,omitempty"`
|
||||
Quality string `json:"quality,omitempty"`
|
||||
DurationSeconds int `json:"duration_seconds,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
@@ -169,7 +167,6 @@ type Item struct {
|
||||
DetailURL string `json:"detail_url,omitempty"`
|
||||
Author string `json:"author,omitempty"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Category string `json:"category,omitempty"`
|
||||
Quality string `json:"quality,omitempty"`
|
||||
DurationSeconds int `json:"duration_seconds,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
@@ -216,9 +213,6 @@ func (e Event) normalizedItem() Item {
|
||||
if len(item.Tags) == 0 && len(e.Tags) > 0 {
|
||||
item.Tags = e.Tags
|
||||
}
|
||||
if strings.TrimSpace(item.Category) == "" {
|
||||
item.Category = e.Category
|
||||
}
|
||||
if strings.TrimSpace(item.Quality) == "" {
|
||||
item.Quality = e.Quality
|
||||
}
|
||||
@@ -393,7 +387,7 @@ func (c *Crawler) RunOnce(ctx context.Context, targetNew int) (*CrawlResult, err
|
||||
}
|
||||
|
||||
func (c *Crawler) writeSeenSourceIDs(ctx context.Context, path string) (int, error) {
|
||||
seenIDs, err := c.cfg.Catalog.ListCrawlerSourceIDs(ctx, c.sourceKind(), c.cfg.Driver.ID())
|
||||
seenIDs, err := c.cfg.Catalog.ListCrawlerSourceIDs(ctx, Kind, c.cfg.Driver.ID())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -514,8 +508,7 @@ func (c *Crawler) processItem(ctx context.Context, item Item) (bool, error) {
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
sourceKind := c.sourceKind()
|
||||
videoID := BuildVideoIDForKind(sourceKind, c.cfg.Driver.ID(), sourceID)
|
||||
videoID := BuildVideoID(c.cfg.Driver.ID(), sourceID)
|
||||
if deleted, err := c.cfg.Catalog.IsVideoDeleted(ctx, videoID); err != nil {
|
||||
return false, err
|
||||
} else if deleted {
|
||||
@@ -579,7 +572,6 @@ func (c *Crawler) processItem(ctx context.Context, item Item) (bool, error) {
|
||||
Size: size,
|
||||
Ext: strings.TrimPrefix(videoExt, "."),
|
||||
Quality: quality,
|
||||
Category: strings.TrimSpace(item.Category),
|
||||
Description: strings.TrimSpace(item.Description),
|
||||
PreviewStatus: previewStatus,
|
||||
PublishedAt: publishedAt,
|
||||
@@ -593,9 +585,9 @@ func (c *Crawler) processItem(ctx context.Context, item Item) (bool, error) {
|
||||
}
|
||||
v.SampledSHA256 = sampled
|
||||
v.FingerprintStatus = "ready"
|
||||
if duplicate, err := c.cfg.Catalog.FindEquivalentVideo(ctx, v); err == nil && duplicate != nil {
|
||||
if duplicate, err := c.cfg.Catalog.FindVideoBySampledFingerprint(ctx, v); err == nil && duplicate != nil {
|
||||
_ = os.Remove(videoPath)
|
||||
if markErr := c.cfg.Catalog.MarkCrawlerSourceSeen(ctx, sourceKind, c.cfg.Driver.ID(), sourceID, "duplicate", duplicate.ID, sampled, size); markErr != nil {
|
||||
if markErr := c.cfg.Catalog.MarkCrawlerSourceSeen(ctx, Kind, c.cfg.Driver.ID(), sourceID, "duplicate", duplicate.ID, sampled, size); markErr != nil {
|
||||
log.Printf("[scriptcrawler] drive=%s source_id=%s mark duplicate seen: %v", c.cfg.Driver.ID(), sourceID, markErr)
|
||||
}
|
||||
log.Printf("[scriptcrawler] drive=%s source_id=%s duplicate_of=%s title=%q size=%d", c.cfg.Driver.ID(), sourceID, duplicate.ID, title, size)
|
||||
@@ -606,19 +598,25 @@ func (c *Crawler) processItem(ctx context.Context, item Item) (bool, error) {
|
||||
}
|
||||
|
||||
thumbReady := false
|
||||
thumbPath := ""
|
||||
commonThumbPath := ""
|
||||
if item.Thumbnail.URL != "" || item.Thumbnail.LocalFile != "" {
|
||||
thumbFile := sourceID + detectThumbExt(item.Thumbnail.URL, item.Thumbnail.LocalFile)
|
||||
thumbPath, err := c.cfg.Driver.ThumbPath(thumbFile)
|
||||
thumbPath, err = c.cfg.Driver.ThumbPath(thumbFile)
|
||||
if err == nil {
|
||||
if _, err := c.materializeMedia(ctx, item.Thumbnail, thumbPath, item.DetailURL, false); err != nil {
|
||||
log.Printf("[scriptcrawler] drive=%s source_id=%s thumbnail failed: %v", c.cfg.Driver.ID(), sourceID, err)
|
||||
} else if c.cfg.CommonThumbDir != "" {
|
||||
if err := os.MkdirAll(c.cfg.CommonThumbDir, 0o755); err != nil {
|
||||
log.Printf("[scriptcrawler] drive=%s common thumbs mkdir: %v", c.cfg.Driver.ID(), err)
|
||||
} else if err := copyFileAtomic(thumbPath, mediaasset.ThumbnailPathInDir(c.cfg.CommonThumbDir, videoID)); err != nil {
|
||||
log.Printf("[scriptcrawler] drive=%s source_id=%s copy thumbnail: %v", c.cfg.Driver.ID(), sourceID, err)
|
||||
} else {
|
||||
thumbReady = true
|
||||
dst := mediaasset.ThumbnailPathInDir(c.cfg.CommonThumbDir, videoID)
|
||||
if err := copyFileAtomic(thumbPath, dst); err != nil {
|
||||
log.Printf("[scriptcrawler] drive=%s source_id=%s copy thumbnail: %v", c.cfg.Driver.ID(), sourceID, err)
|
||||
} else {
|
||||
commonThumbPath = dst
|
||||
thumbReady = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -626,11 +624,48 @@ func (c *Crawler) processItem(ctx context.Context, item Item) (bool, error) {
|
||||
if thumbReady {
|
||||
v.ThumbnailURL = "/p/thumb/" + v.ID
|
||||
}
|
||||
if duplicate, err := c.findNearDuplicateVideo(ctx, v, commonThumbPath); err != nil {
|
||||
_ = os.Remove(videoPath)
|
||||
if thumbPath != "" {
|
||||
_ = os.Remove(thumbPath)
|
||||
}
|
||||
if commonThumbPath != "" {
|
||||
_ = os.Remove(commonThumbPath)
|
||||
}
|
||||
return false, fmt.Errorf("near duplicate lookup: %w", err)
|
||||
} else if duplicate != nil && duplicate.video != nil {
|
||||
if v.Size > duplicate.video.Size {
|
||||
if err := c.cfg.Catalog.DeleteVideoWithTombstoneReason(ctx, duplicate.video.ID, catalog.DeletedVideoReasonDuplicate); err != nil {
|
||||
_ = os.Remove(videoPath)
|
||||
if thumbPath != "" {
|
||||
_ = os.Remove(thumbPath)
|
||||
}
|
||||
if commonThumbPath != "" {
|
||||
_ = os.Remove(commonThumbPath)
|
||||
}
|
||||
return false, fmt.Errorf("delete smaller near duplicate %s: %w", duplicate.video.ID, err)
|
||||
}
|
||||
log.Printf("[scriptcrawler] drive=%s source_id=%s replacing_smaller_near_duplicate=%s old_size=%d new_size=%d title_similarity=%.3f thumbnail_ssim=%.3f title=%q duration=%d", c.cfg.Driver.ID(), sourceID, duplicate.video.ID, duplicate.video.Size, v.Size, duplicate.titleSimilarity, duplicate.thumbnailSSIM, title, v.DurationSeconds)
|
||||
} else {
|
||||
_ = os.Remove(videoPath)
|
||||
if thumbPath != "" {
|
||||
_ = os.Remove(thumbPath)
|
||||
}
|
||||
if commonThumbPath != "" {
|
||||
_ = os.Remove(commonThumbPath)
|
||||
}
|
||||
if markErr := c.cfg.Catalog.MarkCrawlerSourceSeen(ctx, Kind, c.cfg.Driver.ID(), sourceID, "duplicate", duplicate.video.ID, sampled, size); markErr != nil {
|
||||
log.Printf("[scriptcrawler] drive=%s source_id=%s mark near duplicate seen: %v", c.cfg.Driver.ID(), sourceID, markErr)
|
||||
}
|
||||
log.Printf("[scriptcrawler] drive=%s source_id=%s near_duplicate_of=%s old_size=%d new_size=%d title_similarity=%.3f thumbnail_ssim=%.3f title=%q duration=%d", c.cfg.Driver.ID(), sourceID, duplicate.video.ID, duplicate.video.Size, v.Size, duplicate.titleSimilarity, duplicate.thumbnailSSIM, title, v.DurationSeconds)
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
if err := c.cfg.Catalog.UpsertVideo(ctx, v); err != nil {
|
||||
_ = os.Remove(videoPath)
|
||||
return false, err
|
||||
}
|
||||
if err := c.cfg.Catalog.MarkCrawlerSourceSeen(ctx, sourceKind, c.cfg.Driver.ID(), sourceID, "imported", v.ID, sampled, size); err != nil {
|
||||
if err := c.cfg.Catalog.MarkCrawlerSourceSeen(ctx, Kind, c.cfg.Driver.ID(), sourceID, "imported", v.ID, sampled, size); err != nil {
|
||||
log.Printf("[scriptcrawler] drive=%s source_id=%s mark imported seen: %v", c.cfg.Driver.ID(), sourceID, err)
|
||||
}
|
||||
log.Printf("[scriptcrawler] drive=%s source_id=%s ok title=%q size=%d", c.cfg.Driver.ID(), sourceID, title, size)
|
||||
@@ -800,6 +835,10 @@ func (c *Crawler) downloadHLSAtomic(ctx context.Context, ref MediaRef, dst, refe
|
||||
args = append(args, "-headers", h)
|
||||
}
|
||||
args = append(args,
|
||||
"-protocol_whitelist", "http,https,tcp,tls,crypto",
|
||||
"-allowed_extensions", "ALL",
|
||||
"-allowed_segment_extensions", "ALL",
|
||||
"-extension_picky", "0",
|
||||
"-i", src,
|
||||
"-c", "copy",
|
||||
"-bsf:a", "aac_adtstoasc",
|
||||
@@ -996,7 +1035,6 @@ func normalizeItemForImport(item Item) (Item, string, error) {
|
||||
}
|
||||
item.DetailURL = strings.TrimSpace(item.DetailURL)
|
||||
item.Author = strings.TrimSpace(item.Author)
|
||||
item.Category = strings.TrimSpace(item.Category)
|
||||
item.Quality = strings.TrimSpace(item.Quality)
|
||||
item.Description = strings.TrimSpace(item.Description)
|
||||
item.PublishedAt = strings.TrimSpace(item.PublishedAt)
|
||||
@@ -1101,16 +1139,6 @@ func stableURLKey(raw string) string {
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func (c *Crawler) sourceKind() string {
|
||||
if c == nil {
|
||||
return Kind
|
||||
}
|
||||
if v := strings.TrimSpace(c.cfg.SourceKind); v != "" {
|
||||
return v
|
||||
}
|
||||
return Kind
|
||||
}
|
||||
|
||||
func (c *Crawler) crawlerTagName() string {
|
||||
if c == nil {
|
||||
return ""
|
||||
@@ -1142,14 +1170,7 @@ func candidateBudgetForTarget(targetNew int) int {
|
||||
}
|
||||
|
||||
func BuildVideoID(driveID, sourceID string) string {
|
||||
return BuildVideoIDForKind(Kind, driveID, sourceID)
|
||||
}
|
||||
|
||||
func BuildVideoIDForKind(kind, driveID, sourceID string) string {
|
||||
if kind = strings.TrimSpace(kind); kind == "" {
|
||||
kind = Kind
|
||||
}
|
||||
return kind + "-" + driveID + "-" + sourceID
|
||||
return Kind + "-" + driveID + "-" + sourceID
|
||||
}
|
||||
|
||||
func detectVideoExt(rawURL, localFile string) string {
|
||||
|
||||
@@ -4,6 +4,9 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
@@ -14,6 +17,7 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/fingerprint"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -39,13 +43,31 @@ func writeScriptCrawlerFFprobeStub(t *testing.T, dir string, ok bool) string {
|
||||
func writeScriptCrawlerFFmpegStub(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, "ffmpeg-hls.sh")
|
||||
body := "#!/bin/sh\nout=\"\"\nfor arg do out=\"$arg\"; done\nprintf 'hls-video-bytes' > \"$out\"\n"
|
||||
body := "#!/bin/sh\nif [ -n \"$GO_SCRIPTCRAWLER_FFMPEG_ARGS_FILE\" ]; then printf '%s\\n' \"$@\" > \"$GO_SCRIPTCRAWLER_FFMPEG_ARGS_FILE\"; fi\nout=\"\"\nfor arg do out=\"$arg\"; done\nprintf 'hls-video-bytes' > \"$out\"\n"
|
||||
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
|
||||
t.Fatalf("write ffmpeg stub: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func writeScriptCrawlerJPEG(t *testing.T, path string, c color.RGBA) {
|
||||
t.Helper()
|
||||
img := image.NewRGBA(image.Rect(0, 0, 48, 48))
|
||||
for y := 0; y < 48; y++ {
|
||||
for x := 0; x < 48; x++ {
|
||||
img.SetRGBA(x, y, c)
|
||||
}
|
||||
}
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
t.Fatalf("create jpeg: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
if err := jpeg.Encode(f, img, &jpeg.Options{Quality: 95}); err != nil {
|
||||
t.Fatalf("encode jpeg: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceImportsLocalFileAndSkipsExisting(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
@@ -236,7 +258,7 @@ func TestCrawlerRunOnceUsesCurrentDrivePreviewSwitch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceUsesSourceKindNamespace(t *testing.T) {
|
||||
func TestCrawlerRunOnceUsesDefaultCrawlerNamespace(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
@@ -266,7 +288,6 @@ func TestCrawlerRunOnceUsesSourceKindNamespace(t *testing.T) {
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
SourceKind: "spider91",
|
||||
PythonPath: wrapper,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
ScriptPath: dummyScript,
|
||||
@@ -278,12 +299,9 @@ func TestCrawlerRunOnceUsesSourceKindNamespace(t *testing.T) {
|
||||
if res.NewVideos != 1 || res.SeenSnapshot != 0 {
|
||||
t.Fatalf("result = new:%d seen:%d, want 1/0", res.NewVideos, res.SeenSnapshot)
|
||||
}
|
||||
videoID := BuildVideoIDForKind("spider91", "demo", "abc-123")
|
||||
videoID := BuildVideoID("demo", "abc-123")
|
||||
if _, err := cat.GetVideo(ctx, videoID); err != nil {
|
||||
t.Fatalf("get source-kind video: %v", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, BuildVideoID("demo", "abc-123")); err == nil {
|
||||
t.Fatalf("default namespace video unexpectedly exists")
|
||||
t.Fatalf("get crawler video: %v", err)
|
||||
}
|
||||
|
||||
res, err = c.RunOnce(ctx, 1)
|
||||
@@ -537,6 +555,182 @@ func TestCrawlerRunOnceSkipsFingerprintDuplicateAndContinues(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerProcessItemSkipsNearDuplicateByTitleDurationAndThumbnail(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
commonThumbDir := filepath.Join(tmp, "common-thumbs")
|
||||
if err := os.MkdirAll(commonThumbDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir common thumbs: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
canonicalID := "existing-canonical"
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: canonicalID,
|
||||
DriveID: "other-drive",
|
||||
FileID: "existing.mp4",
|
||||
FileName: "existing.mp4",
|
||||
Title: "91 Test Similar Title 1215516",
|
||||
DurationSeconds: 257,
|
||||
Size: 12345,
|
||||
Ext: "mp4",
|
||||
ThumbnailURL: "/p/thumb/" + canonicalID,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed canonical video: %v", err)
|
||||
}
|
||||
writeScriptCrawlerJPEG(t, mediaasset.ThumbnailPathInDir(commonThumbDir, canonicalID), color.RGBA{R: 210, G: 40, B: 40, A: 255})
|
||||
|
||||
outputDir := drv.OutputDir()
|
||||
mediaPath := filepath.Join(outputDir, "near-video.mp4")
|
||||
if err := os.WriteFile(mediaPath, []byte("near-duplicate-but-different-bytes"), 0o644); err != nil {
|
||||
t.Fatalf("write media: %v", err)
|
||||
}
|
||||
thumbPath := filepath.Join(outputDir, "near-thumb.jpg")
|
||||
writeScriptCrawlerJPEG(t, thumbPath, color.RGBA{R: 211, G: 41, B: 41, A: 255})
|
||||
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
CommonThumbDir: commonThumbDir,
|
||||
})
|
||||
imported, err := c.processItem(ctx, Item{
|
||||
SourceID: "near-source",
|
||||
Title: "91 Test Similar Title 1215516 - source suffix",
|
||||
Author: "helper",
|
||||
DurationSeconds: 257,
|
||||
Media: MediaRef{LocalFile: mediaPath},
|
||||
Thumbnail: MediaRef{LocalFile: thumbPath},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("process item: %v", err)
|
||||
}
|
||||
if imported {
|
||||
t.Fatal("near duplicate imported, want skipped")
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, BuildVideoID("demo", "near-source")); err == nil {
|
||||
t.Fatal("near duplicate should not be inserted into catalog")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(drv.VideosDir(), "near-source.mp4")); !os.IsNotExist(err) {
|
||||
t.Fatalf("near duplicate video stat = %v, want removed", err)
|
||||
}
|
||||
if sourceThumb, err := drv.ThumbPath("near-source.jpg"); err != nil {
|
||||
t.Fatalf("source thumb path: %v", err)
|
||||
} else if _, err := os.Stat(sourceThumb); !os.IsNotExist(err) {
|
||||
t.Fatalf("source thumb stat = %v, want removed", err)
|
||||
}
|
||||
if _, err := os.Stat(mediaasset.ThumbnailPathInDir(commonThumbDir, BuildVideoID("demo", "near-source"))); !os.IsNotExist(err) {
|
||||
t.Fatalf("common thumb stat = %v, want removed", err)
|
||||
}
|
||||
seen, err := cat.ListCrawlerSourceIDs(ctx, Kind, "demo")
|
||||
if err != nil {
|
||||
t.Fatalf("list seen source ids: %v", err)
|
||||
}
|
||||
if !hasString(seen, "near-source") {
|
||||
t.Fatalf("seen ids = %#v, want near-source", seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerProcessItemKeepsLargerNearDuplicate(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
|
||||
if err := drv.Init(ctx); err != nil {
|
||||
t.Fatalf("driver init: %v", err)
|
||||
}
|
||||
commonThumbDir := filepath.Join(tmp, "common-thumbs")
|
||||
if err := os.MkdirAll(commonThumbDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir common thumbs: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
smallerID := "smaller-canonical"
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: smallerID,
|
||||
DriveID: "other-drive",
|
||||
FileID: "smaller.mp4",
|
||||
FileName: "smaller.mp4",
|
||||
Title: "91 Test Larger Candidate 1215516",
|
||||
DurationSeconds: 257,
|
||||
Size: 5,
|
||||
Ext: "mp4",
|
||||
ThumbnailURL: "/p/thumb/" + smallerID,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed smaller video: %v", err)
|
||||
}
|
||||
writeScriptCrawlerJPEG(t, mediaasset.ThumbnailPathInDir(commonThumbDir, smallerID), color.RGBA{R: 80, G: 160, B: 80, A: 255})
|
||||
|
||||
outputDir := drv.OutputDir()
|
||||
mediaPath := filepath.Join(outputDir, "larger-video.mp4")
|
||||
if err := os.WriteFile(mediaPath, []byte("near-duplicate-larger-candidate-bytes"), 0o644); err != nil {
|
||||
t.Fatalf("write media: %v", err)
|
||||
}
|
||||
thumbPath := filepath.Join(outputDir, "larger-thumb.jpg")
|
||||
writeScriptCrawlerJPEG(t, thumbPath, color.RGBA{R: 81, G: 161, B: 81, A: 255})
|
||||
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
|
||||
CommonThumbDir: commonThumbDir,
|
||||
})
|
||||
imported, err := c.processItem(ctx, Item{
|
||||
SourceID: "larger-source",
|
||||
Title: "91 Test Larger Candidate 1215516 - source suffix",
|
||||
Author: "helper",
|
||||
DurationSeconds: 257,
|
||||
Media: MediaRef{LocalFile: mediaPath},
|
||||
Thumbnail: MediaRef{LocalFile: thumbPath},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("process item: %v", err)
|
||||
}
|
||||
if !imported {
|
||||
t.Fatal("larger near duplicate was skipped, want imported")
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, smallerID); err == nil {
|
||||
t.Fatal("smaller near duplicate should be deleted from catalog")
|
||||
}
|
||||
if deleted, err := cat.IsVideoDeleted(ctx, smallerID); err != nil || !deleted {
|
||||
t.Fatalf("smaller tombstone = %v, %v; want deleted tombstone", deleted, err)
|
||||
}
|
||||
larger, err := cat.GetVideo(ctx, BuildVideoID("demo", "larger-source"))
|
||||
if err != nil {
|
||||
t.Fatalf("larger video should be imported: %v", err)
|
||||
}
|
||||
if larger.Size <= 5 {
|
||||
t.Fatalf("larger size = %d, want > 5", larger.Size)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRunOnceRejectsInvalidDownloadedVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tmp := t.TempDir()
|
||||
@@ -622,6 +816,8 @@ func TestCrawlerRunOnceDownloadsHLSMediaURL(t *testing.T) {
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
|
||||
t.Setenv("GO_WANT_SCRIPTCRAWLER_HLS", "1")
|
||||
ffmpegArgsFile := filepath.Join(tmp, "ffmpeg-args.txt")
|
||||
t.Setenv("GO_SCRIPTCRAWLER_FFMPEG_ARGS_FILE", ffmpegArgsFile)
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
@@ -652,6 +848,21 @@ func TestCrawlerRunOnceDownloadsHLSMediaURL(t *testing.T) {
|
||||
if string(data) != "hls-video-bytes" {
|
||||
t.Fatalf("hls output = %q", string(data))
|
||||
}
|
||||
argsData, err := os.ReadFile(ffmpegArgsFile)
|
||||
if err != nil {
|
||||
t.Fatalf("read ffmpeg args: %v", err)
|
||||
}
|
||||
argsText := "\n" + string(argsData) + "\n"
|
||||
for _, want := range []string{
|
||||
"\n-protocol_whitelist\nhttp,https,tcp,tls,crypto\n",
|
||||
"\n-allowed_extensions\nALL\n",
|
||||
"\n-allowed_segment_extensions\nALL\n",
|
||||
"\n-extension_picky\n0\n",
|
||||
} {
|
||||
if !strings.Contains(argsText, want) {
|
||||
t.Fatalf("ffmpeg args missing %q in:\n%s", strings.TrimSpace(want), string(argsData))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptCrawlerHelperProcess(t *testing.T) {
|
||||
|
||||
@@ -67,6 +67,55 @@ type DryRunResult struct {
|
||||
DurationMs int64 `json:"durationMs"`
|
||||
}
|
||||
|
||||
type dryRunLogTail struct {
|
||||
mu sync.Mutex
|
||||
lines []string
|
||||
partial string
|
||||
}
|
||||
|
||||
func newDryRunLogTail() *dryRunLogTail {
|
||||
return &dryRunLogTail{lines: make([]string, 0, dryRunLogTailLines)}
|
||||
}
|
||||
|
||||
func (t *dryRunLogTail) Write(p []byte) (int, error) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
chunk := strings.ReplaceAll(string(p), "\r\n", "\n")
|
||||
parts := strings.Split(t.partial+chunk, "\n")
|
||||
t.partial = parts[len(parts)-1]
|
||||
for _, line := range parts[:len(parts)-1] {
|
||||
t.appendLocked(line)
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (t *dryRunLogTail) snapshot() []string {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
lines := append([]string{}, t.lines...)
|
||||
if partial := strings.TrimSpace(t.partial); partial != "" {
|
||||
lines = appendDryRunLogLine(lines, partial)
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func (t *dryRunLogTail) appendLocked(line string) {
|
||||
t.lines = appendDryRunLogLine(t.lines, line)
|
||||
}
|
||||
|
||||
func appendDryRunLogLine(lines []string, line string) []string {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
return lines
|
||||
}
|
||||
if len(lines) >= dryRunLogTailLines {
|
||||
lines = lines[1:]
|
||||
}
|
||||
return append(lines, line)
|
||||
}
|
||||
|
||||
func DryRun(ctx context.Context, cfg DryRunConfig) *DryRunResult {
|
||||
started := time.Now()
|
||||
result := &DryRunResult{Items: []DryRunItem{}}
|
||||
@@ -169,41 +218,14 @@ func DryRun(ctx context.Context, cfg DryRunConfig) *DryRunResult {
|
||||
result.Error = fmt.Sprintf("启动脚本失败: %v", err)
|
||||
return result
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
_ = stdout.Close()
|
||||
result.Error = fmt.Sprintf("启动脚本失败: %v", err)
|
||||
return result
|
||||
}
|
||||
logTail := newDryRunLogTail()
|
||||
cmd.Stderr = logTail
|
||||
if err := cmd.Start(); err != nil {
|
||||
_ = stdout.Close()
|
||||
_ = stderr.Close()
|
||||
result.Error = fmt.Sprintf("启动脚本失败: %v", err)
|
||||
return result
|
||||
}
|
||||
|
||||
// stderr 是脚本日志,保留尾部若干行用于排错回显。
|
||||
var logMu sync.Mutex
|
||||
logTail := make([]string, 0, dryRunLogTailLines)
|
||||
stderrDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(stderrDone)
|
||||
scanner := bufio.NewScanner(stderr)
|
||||
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
logMu.Lock()
|
||||
if len(logTail) >= dryRunLogTailLines {
|
||||
logTail = logTail[1:]
|
||||
}
|
||||
logTail = append(logTail, line)
|
||||
logMu.Unlock()
|
||||
}
|
||||
}()
|
||||
|
||||
items := []DryRunItem{}
|
||||
var firstMediaHeaders map[string]string
|
||||
parseFailures := 0
|
||||
@@ -264,11 +286,8 @@ func DryRun(ctx context.Context, cfg DryRunConfig) *DryRunResult {
|
||||
_ = killDryRunProcess(cmd)
|
||||
<-waitDone
|
||||
}
|
||||
<-stderrDone
|
||||
|
||||
logMu.Lock()
|
||||
result.Log = append([]string{}, logTail...)
|
||||
logMu.Unlock()
|
||||
result.Log = logTail.snapshot()
|
||||
result.Items = items
|
||||
|
||||
if len(items) == 0 {
|
||||
|
||||
@@ -48,6 +48,29 @@ echo '{"type":"done","stats":{"emitted":1}}'
|
||||
}
|
||||
}
|
||||
|
||||
func TestDryRunCapturesStderrWhenStoppingAfterFirstItem(t *testing.T) {
|
||||
script := writeDryRunScript(t, `
|
||||
echo '[log] first item ready' >&2
|
||||
echo '{"type":"item","item":{"title":"Early Stop Video","media_url":"https://cdn.example.test/v.mp4","source_id":"early-stop"}}'
|
||||
sleep 30
|
||||
`)
|
||||
start := time.Now()
|
||||
result := DryRun(context.Background(), DryRunConfig{
|
||||
PythonPath: "/bin/sh",
|
||||
ScriptPath: script,
|
||||
SkipMediaProbe: true,
|
||||
})
|
||||
if !result.OK {
|
||||
t.Fatalf("ok = false, error = %q, log = %v", result.Error, result.Log)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed > 5*time.Second {
|
||||
t.Fatalf("dry run took %s, script was not stopped after first item", elapsed)
|
||||
}
|
||||
if len(result.Log) == 0 || !strings.Contains(result.Log[0], "first item ready") {
|
||||
t.Fatalf("log tail = %v, want stderr captured before early stop", result.Log)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDryRunProbesMediaURL(t *testing.T) {
|
||||
var gotRange, gotReferer string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package scriptcrawler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/mediaasset"
|
||||
"github.com/video-site/backend/internal/mediasim"
|
||||
)
|
||||
|
||||
const (
|
||||
nearDuplicateTitleThreshold = 0.90
|
||||
nearDuplicateSSIMThreshold = 0.95
|
||||
nearDuplicateDurationToleranceSeconds = 2
|
||||
nearDuplicateCandidateLimit = 200
|
||||
)
|
||||
|
||||
type nearDuplicateMatch struct {
|
||||
video *catalog.Video
|
||||
titleSimilarity float64
|
||||
thumbnailSSIM float64
|
||||
}
|
||||
|
||||
func (c *Crawler) findNearDuplicateVideo(ctx context.Context, source *catalog.Video, sourceThumbPath string) (*nearDuplicateMatch, error) {
|
||||
if c == nil || c.cfg.Catalog == nil || source == nil {
|
||||
return nil, nil
|
||||
}
|
||||
sourceThumbPath = strings.TrimSpace(sourceThumbPath)
|
||||
commonThumbDir := strings.TrimSpace(c.cfg.CommonThumbDir)
|
||||
if sourceThumbPath == "" || commonThumbDir == "" || strings.TrimSpace(source.Title) == "" || source.DurationSeconds <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if _, err := os.Stat(sourceThumbPath); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
candidates, err := c.cfg.Catalog.ListNearDuplicateVideoCandidates(ctx, source, nearDuplicateDurationToleranceSeconds, nearDuplicateCandidateLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
if candidate == nil || candidate.ID == source.ID {
|
||||
continue
|
||||
}
|
||||
titleScore := mediasim.TitleSimilarity(source.Title, candidate.Title)
|
||||
if titleScore < nearDuplicateTitleThreshold {
|
||||
continue
|
||||
}
|
||||
candidateThumbPath := mediaasset.ThumbnailPathInDir(commonThumbDir, candidate.ID)
|
||||
if _, err := os.Stat(candidateThumbPath); err != nil {
|
||||
continue
|
||||
}
|
||||
ssimScore, err := mediasim.ImageSSIM(sourceThumbPath, candidateThumbPath)
|
||||
if err != nil {
|
||||
log.Printf("[scriptcrawler] drive=%s source_id=%s candidate=%s thumbnail ssim failed: %v", c.cfg.Driver.ID(), source.ID, candidate.ID, err)
|
||||
continue
|
||||
}
|
||||
if ssimScore >= nearDuplicateSSIMThreshold {
|
||||
return &nearDuplicateMatch{
|
||||
video: candidate,
|
||||
titleSimilarity: titleScore,
|
||||
thumbnailSSIM: ssimScore,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,799 +0,0 @@
|
||||
package spider91
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
)
|
||||
|
||||
// TestCrawlerRunOnceFullFlow 用一个伪 python 脚本 + httptest 服务器
|
||||
// 把 Crawler.RunOnce 的完整流程跑一遍:脚本生成 JSON、下载视频和封面、入库、
|
||||
// 重复运行跳过已存在的 91 源视频 ID。
|
||||
func TestCrawlerRunOnceFullFlow(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell-based fake script only on unix")
|
||||
}
|
||||
|
||||
tmp := t.TempDir()
|
||||
|
||||
// 1. 假 HTTP 服务器:根据路径返回视频数据或封面数据
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.Contains(r.URL.Path, "120001.mp4"):
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write([]byte("FAKEVIDEO1"))
|
||||
case strings.Contains(r.URL.Path, "120002.mp4"):
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write([]byte("FAKEVIDEO2BYTES"))
|
||||
case strings.Contains(r.URL.Path, "/thumb/120001.jpg"):
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write([]byte("\xff\xd8\xff\xe0fakejpg1"))
|
||||
case strings.Contains(r.URL.Path, "/thumb/120002.jpg"):
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write([]byte("\xff\xd8\xff\xe0fakejpg2"))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
// 2. 假 python 脚本:解析 --output / --stream-output 参数,
|
||||
// 在 stream 模式下逐行 echo 每条视频的 JSON 到 stdout(模拟 Python 端 stream),
|
||||
// 同时仍写 --output 文件作归档。
|
||||
videoEntries := []map[string]string{
|
||||
{
|
||||
"title": "Video One 口交",
|
||||
"thumb_url": srv.URL + "/thumb/not-120001.jpg",
|
||||
"video_url": srv.URL + "/videos/120001.mp4",
|
||||
"viewkey": "vk-001",
|
||||
"detail_url": srv.URL + "/v.php?viewkey=vk-001",
|
||||
},
|
||||
{
|
||||
"title": "Video Two",
|
||||
"thumb_url": srv.URL + "/thumb/not-120002.jpg",
|
||||
"video_url": srv.URL + "/videos/120002.mp4",
|
||||
"viewkey": "vk-002",
|
||||
"detail_url": srv.URL + "/v.php?viewkey=vk-002",
|
||||
},
|
||||
}
|
||||
scriptPath := filepath.Join(tmp, "fake_spider.sh")
|
||||
scriptBody := buildFakeSpiderScript(videoEntries)
|
||||
if err := os.WriteFile(scriptPath, []byte(scriptBody), 0o755); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
|
||||
// 3. 准备 catalog + driver + crawler
|
||||
dbPath := filepath.Join(tmp, "test.db")
|
||||
cat, err := catalog.Open(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
defer cat.Close()
|
||||
|
||||
driveID := "spider91-test"
|
||||
rootDir := filepath.Join(tmp, "spider91", driveID)
|
||||
commonThumbs := filepath.Join(tmp, "previews", "thumbs")
|
||||
drv := New(Config{ID: driveID, RootDir: rootDir})
|
||||
|
||||
// 把 drive 也写入 catalog(Crawler 不直接读,但 main 真实流程会写)
|
||||
if err := cat.UpsertDrive(context.Background(), &catalog.Drive{
|
||||
ID: driveID,
|
||||
Kind: Kind,
|
||||
Name: "test crawler",
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert drive: %v", err)
|
||||
}
|
||||
if _, err := cat.CreateTagAndClassify(context.Background(), "Video One", nil, "user"); err != nil {
|
||||
t.Fatalf("create user tag: %v", err)
|
||||
}
|
||||
|
||||
var newVideos []*catalog.Video
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
PythonPath: "sh",
|
||||
ScriptPath: scriptPath,
|
||||
CommonThumbDir: commonThumbs,
|
||||
SpiderTimeout: 10 * time.Second,
|
||||
DownloadTimeout: 10 * time.Second,
|
||||
OnNewVideo: func(v *catalog.Video) {
|
||||
newVideos = append(newVideos, v)
|
||||
},
|
||||
})
|
||||
|
||||
// 4. 第一次 RunOnce:应该新入库 2 条
|
||||
res, err := c.RunOnce(context.Background(), 15)
|
||||
if err != nil {
|
||||
t.Fatalf("RunOnce: %v", err)
|
||||
}
|
||||
if res.NewVideos != 2 || res.Skipped != 0 || res.Failed != 0 {
|
||||
t.Fatalf("first run result: new=%d skipped=%d failed=%d, want 2/0/0",
|
||||
res.NewVideos, res.Skipped, res.Failed)
|
||||
}
|
||||
if res.TargetNew != 15 {
|
||||
t.Fatalf("first run TargetNew = %d, want 15", res.TargetNew)
|
||||
}
|
||||
if res.SeenSnapshot != 0 {
|
||||
t.Fatalf("first run SeenSnapshot = %d, want 0 (catalog empty before first run)", res.SeenSnapshot)
|
||||
}
|
||||
if len(newVideos) != 2 {
|
||||
t.Fatalf("OnNewVideo called %d times, want 2", len(newVideos))
|
||||
}
|
||||
|
||||
// 5. 检查文件落盘
|
||||
for _, item := range []struct {
|
||||
sourceID string
|
||||
size int64
|
||||
}{
|
||||
{"120001", 10},
|
||||
{"120002", 15},
|
||||
} {
|
||||
videoPath := filepath.Join(rootDir, "videos", item.sourceID+".mp4")
|
||||
info, err := os.Stat(videoPath)
|
||||
if err != nil {
|
||||
t.Fatalf("video %s missing: %v", item.sourceID, err)
|
||||
}
|
||||
if info.Size() != item.size {
|
||||
t.Fatalf("video %s size = %d, want %d", item.sourceID, info.Size(), item.size)
|
||||
}
|
||||
|
||||
thumbPath := filepath.Join(rootDir, "thumbs", item.sourceID+".jpg")
|
||||
if _, err := os.Stat(thumbPath); err != nil {
|
||||
t.Fatalf("thumb %s missing: %v", item.sourceID, err)
|
||||
}
|
||||
|
||||
// 复制到 common thumbs 目录的副本,名字按 videoID 来
|
||||
videoID := BuildVideoID(driveID, item.sourceID)
|
||||
commonThumb := filepath.Join(commonThumbs, videoID+".jpg")
|
||||
if _, err := os.Stat(commonThumb); err != nil {
|
||||
t.Fatalf("common thumb %s missing: %v", commonThumb, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 6. 检查 catalog 入库
|
||||
for _, sourceID := range []string{"120001", "120002"} {
|
||||
videoID := BuildVideoID(driveID, sourceID)
|
||||
v, err := cat.GetVideo(context.Background(), videoID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetVideo %s: %v", videoID, err)
|
||||
}
|
||||
if v.DriveID != driveID {
|
||||
t.Fatalf("video %s drive_id = %q want %q", videoID, v.DriveID, driveID)
|
||||
}
|
||||
if v.FileID != sourceID+".mp4" {
|
||||
t.Fatalf("video %s file_id = %q want %q", videoID, v.FileID, sourceID+".mp4")
|
||||
}
|
||||
if v.ThumbnailURL == "" {
|
||||
t.Fatalf("video %s ThumbnailURL empty (cover should be ready)", videoID)
|
||||
}
|
||||
if v.Author != DefaultAuthor {
|
||||
t.Fatalf("video %s author = %q want %q", videoID, v.Author, DefaultAuthor)
|
||||
}
|
||||
// 每条视频都应该带 "91porn" 标签(UpsertVideo 路径自动同步 tags 表)
|
||||
hasDefaultTag := false
|
||||
for _, tag := range v.Tags {
|
||||
if tag == DefaultTag {
|
||||
hasDefaultTag = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasDefaultTag {
|
||||
t.Fatalf("video %s tags = %v, want contain %q", videoID, v.Tags, DefaultTag)
|
||||
}
|
||||
if sourceID == "120001" {
|
||||
if !containsString(v.Tags, "口交") {
|
||||
t.Fatalf("video %s tags = %v, want contain built-in tag 口交", videoID, v.Tags)
|
||||
}
|
||||
if !containsString(v.Tags, "Video One") {
|
||||
t.Fatalf("video %s tags = %v, want contain user tag Video One", videoID, v.Tags)
|
||||
}
|
||||
}
|
||||
if sourceID == "120002" && (containsString(v.Tags, "口交") || containsString(v.Tags, "Video One")) {
|
||||
t.Fatalf("video %s tags = %v, should not inherit tags from other spider91 videos", videoID, v.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
// 7. 第二次 RunOnce:源视频 ID 已存在 → 全部 skipped,无新文件下载
|
||||
newVideos = nil
|
||||
res2, err := c.RunOnce(context.Background(), 15)
|
||||
if err != nil {
|
||||
t.Fatalf("second RunOnce: %v", err)
|
||||
}
|
||||
if res2.NewVideos != 0 {
|
||||
t.Fatalf("second run NewVideos = %d, want 0", res2.NewVideos)
|
||||
}
|
||||
if res2.Skipped != 2 {
|
||||
t.Fatalf("second run Skipped = %d, want 2", res2.Skipped)
|
||||
}
|
||||
// 第二次运行时 catalog 里已经有 2 条,seen snapshot 应该写出 2 个源视频 ID
|
||||
if res2.SeenSnapshot != 2 {
|
||||
t.Fatalf("second run SeenSnapshot = %d, want 2", res2.SeenSnapshot)
|
||||
}
|
||||
if len(newVideos) != 0 {
|
||||
t.Fatalf("second run OnNewVideo fired %d times, want 0", len(newVideos))
|
||||
}
|
||||
}
|
||||
|
||||
// TestCrawlerRunOnceMissingScript 报错而不是 panic。
|
||||
func TestCrawlerRunOnceMissingScript(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "x.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("catalog: %v", err)
|
||||
}
|
||||
defer cat.Close()
|
||||
drv := New(Config{ID: "x", RootDir: filepath.Join(tmp, "x")})
|
||||
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
PythonPath: "python3",
|
||||
ScriptPath: filepath.Join(tmp, "does-not-exist.py"),
|
||||
})
|
||||
|
||||
if _, err := c.RunOnce(context.Background(), 1); err == nil {
|
||||
t.Fatalf("expected error for missing script")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerPassesProxyToSpiderProcess(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell-based fake script only on unix")
|
||||
}
|
||||
|
||||
tmp := t.TempDir()
|
||||
scriptPath := filepath.Join(tmp, "print_proxy_env.sh")
|
||||
script := `#!/bin/sh
|
||||
printf 'HTTP_PROXY=%s\n' "$HTTP_PROXY"
|
||||
printf 'HTTPS_PROXY=%s\n' "$HTTPS_PROXY"
|
||||
printf 'http_proxy=%s\n' "$http_proxy"
|
||||
printf 'https_proxy=%s\n' "$https_proxy"
|
||||
printf 'NO_PROXY=%s\n' "$NO_PROXY"
|
||||
printf 'no_proxy=%s\n' "$no_proxy"
|
||||
`
|
||||
if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
|
||||
proxyURL := "socks5h://proxy.local:1080"
|
||||
drv := New(Config{ID: "proxy-drive", RootDir: filepath.Join(tmp, "proxy-drive")})
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
PythonPath: "sh",
|
||||
ScriptPath: scriptPath,
|
||||
ProxyURL: proxyURL,
|
||||
})
|
||||
cmd, stdout, err := c.startSpiderTargetNew(
|
||||
context.Background(),
|
||||
1,
|
||||
filepath.Join(tmp, "seen.txt"),
|
||||
filepath.Join(tmp, "out.json"),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("startSpiderTargetNew: %v", err)
|
||||
}
|
||||
raw, err := io.ReadAll(stdout)
|
||||
if err != nil {
|
||||
t.Fatalf("read stdout: %v", err)
|
||||
}
|
||||
if err := cmd.Wait(); err != nil {
|
||||
t.Fatalf("wait: %v", err)
|
||||
}
|
||||
|
||||
want := strings.Join([]string{
|
||||
"HTTP_PROXY=" + proxyURL,
|
||||
"HTTPS_PROXY=" + proxyURL,
|
||||
"http_proxy=" + proxyURL,
|
||||
"https_proxy=" + proxyURL,
|
||||
"NO_PROXY=",
|
||||
"no_proxy=",
|
||||
}, "\n") + "\n"
|
||||
if string(raw) != want {
|
||||
t.Fatalf("proxy env = %q, want %q", string(raw), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigureExplicitProxySupportsSocksSchemes(t *testing.T) {
|
||||
for _, raw := range []string{
|
||||
"socks5://127.0.0.1:1080",
|
||||
"socks5h://proxy-user:proxy-pass@127.0.0.1:1080",
|
||||
} {
|
||||
t.Run(raw, func(t *testing.T) {
|
||||
transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
|
||||
if err := configureExplicitProxy(transport, raw); err != nil {
|
||||
t.Fatalf("configureExplicitProxy: %v", err)
|
||||
}
|
||||
if transport.Proxy != nil {
|
||||
t.Fatalf("Transport.Proxy should be nil for SOCKS proxy")
|
||||
}
|
||||
if transport.DialContext == nil {
|
||||
t.Fatalf("Transport.DialContext should be set for SOCKS proxy")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
|
||||
if err := configureExplicitProxy(transport, "http://127.0.0.1:7890"); err != nil {
|
||||
t.Fatalf("configureExplicitProxy http: %v", err)
|
||||
}
|
||||
if transport.Proxy == nil {
|
||||
t.Fatalf("Transport.Proxy should be set for HTTP proxy")
|
||||
}
|
||||
if transport.DialContext != nil {
|
||||
t.Fatalf("Transport.DialContext should not be set for HTTP proxy")
|
||||
}
|
||||
|
||||
if err := configureExplicitProxy(&http.Transport{}, "ftp://127.0.0.1:21"); err == nil {
|
||||
t.Fatalf("expected unsupported proxy scheme error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSocksTargetIPPrefersIPv4(t *testing.T) {
|
||||
got := selectSocksTargetIP([]net.IPAddr{
|
||||
{IP: net.ParseIP("2606:4700:20::681a:229")},
|
||||
{IP: net.ParseIP("104.26.3.41")},
|
||||
})
|
||||
if got == nil || got.String() != "104.26.3.41" {
|
||||
t.Fatalf("selectSocksTargetIP = %v, want IPv4 104.26.3.41", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCrawlerThumbDownloadFailureMarksStatusFailed 验证:网站封面下载失败时
|
||||
// crawler 把 thumbnail_status 显式标 'failed',避免后续封面补队列一直重复
|
||||
// 捞到这条 spider91 视频。
|
||||
//
|
||||
// 历史 bug:之前 thumb 下载失败仅打 log,url=”, status 走 schema DEFAULT 'pending'。
|
||||
// CountVideosNeedingThumbnail 条件是 url=” AND status != 'failed' → count=1。
|
||||
// spider91 drive 的 thumb worker 按设计不处理 spider91 视频 → 没人会改 status,
|
||||
// 后续补队列会一直认为它还缺封面。
|
||||
func TestCrawlerThumbDownloadFailureMarksStatusFailed(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell-based fake script only on unix")
|
||||
}
|
||||
tmp := t.TempDir()
|
||||
|
||||
// 假 HTTP 服务器:thumb 路径返回 500,video 正常返回字节。
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.Contains(r.URL.Path, "120101.mp4"):
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write([]byte("FAKEVIDEO"))
|
||||
case strings.Contains(r.URL.Path, "120101.jpg"):
|
||||
http.Error(w, "broken", http.StatusInternalServerError)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
videoEntries := []map[string]string{
|
||||
{
|
||||
"title": "Thumb Failure Video",
|
||||
"thumb_url": srv.URL + "/thumb/120101.jpg",
|
||||
"video_url": srv.URL + "/videos/120101.mp4",
|
||||
"viewkey": "vk-thumb-fail",
|
||||
"detail_url": srv.URL + "/v.php?viewkey=vk-thumb-fail",
|
||||
},
|
||||
}
|
||||
scriptPath := filepath.Join(tmp, "fake.sh")
|
||||
if err := os.WriteFile(scriptPath, []byte(buildFakeSpiderScript(videoEntries)), 0o755); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
|
||||
cat, err := catalog.Open(filepath.Join(tmp, "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("catalog: %v", err)
|
||||
}
|
||||
defer cat.Close()
|
||||
|
||||
driveID := "thumbfail-drive"
|
||||
drv := New(Config{ID: driveID, RootDir: filepath.Join(tmp, "spider91", driveID)})
|
||||
if err := cat.UpsertDrive(context.Background(), &catalog.Drive{
|
||||
ID: driveID, Kind: Kind, Name: "thumbfail",
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert drive: %v", err)
|
||||
}
|
||||
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
PythonPath: "sh",
|
||||
ScriptPath: scriptPath,
|
||||
CommonThumbDir: filepath.Join(tmp, "previews", "thumbs"),
|
||||
SpiderTimeout: 10 * time.Second,
|
||||
DownloadTimeout: 10 * time.Second,
|
||||
})
|
||||
|
||||
res, err := c.RunOnce(context.Background(), 5)
|
||||
if err != nil {
|
||||
t.Fatalf("RunOnce: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 {
|
||||
t.Fatalf("expected 1 new video, got %d (failed=%d)", res.NewVideos, res.Failed)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(context.Background(), "spider91-"+driveID+"-120101")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Errorf("ThumbnailURL = %q, want empty (download failed)", got.ThumbnailURL)
|
||||
}
|
||||
|
||||
// 关键断言:CountVideosNeedingThumbnail 应该返回 0。
|
||||
// 该函数的 SQL 条件是 `url = '' AND status != 'failed'`;如果 crawler 没把
|
||||
// status 标 'failed'(schema DEFAULT 'pending'),count 就会是 1。
|
||||
count, err := cat.CountVideosNeedingThumbnail(context.Background(), driveID)
|
||||
if err != nil {
|
||||
t.Fatalf("count: %v", err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Fatalf("CountVideosNeedingThumbnail = %d, want 0 (status should be 'failed' to unblock teaser worker)", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerUsesCrawlerVideoURLForFirstDownload(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell-based fake script only on unix")
|
||||
}
|
||||
tmp := t.TempDir()
|
||||
|
||||
var detailRequests int32
|
||||
var originalRequests int32
|
||||
var wrongRequests int32
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/v.php":
|
||||
atomic.AddInt32(&detailRequests, 1)
|
||||
_, _ = w.Write([]byte(spider91DetailHTML(srv.URL + "/videos/856305.mp4?token=wrong")))
|
||||
case r.URL.Path == "/videos/120201.mp4" && r.URL.Query().Get("token") == "original":
|
||||
atomic.AddInt32(&originalRequests, 1)
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write([]byte("ORIGINALVIDEO"))
|
||||
case r.URL.Path == "/videos/856305.mp4":
|
||||
atomic.AddInt32(&wrongRequests, 1)
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write([]byte("WRONGVIDEO"))
|
||||
case r.URL.Path == "/thumb/120201.jpg":
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write([]byte("\xff\xd8\xff\xe0thumb"))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
entry := map[string]string{
|
||||
"title": "Use Original URL First",
|
||||
"thumb_url": srv.URL + "/thumb/wrong-thumb.jpg",
|
||||
"video_url": srv.URL + "/videos/120201.mp4?token=original",
|
||||
"viewkey": "vk-use-original",
|
||||
"detail_url": srv.URL + "/v.php?viewkey=vk-use-original",
|
||||
}
|
||||
cat, drv, scriptPath := seedCrawlerTestDeps(t, tmp, "use-original-drive", []map[string]string{entry})
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
PythonPath: "sh",
|
||||
ScriptPath: scriptPath,
|
||||
CommonThumbDir: filepath.Join(tmp, "previews", "thumbs"),
|
||||
SpiderTimeout: 10 * time.Second,
|
||||
DownloadTimeout: 10 * time.Second,
|
||||
})
|
||||
|
||||
res, err := c.RunOnce(context.Background(), 1)
|
||||
if err != nil {
|
||||
t.Fatalf("RunOnce: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 || res.Failed != 0 {
|
||||
t.Fatalf("result new=%d failed=%d, want 1/0", res.NewVideos, res.Failed)
|
||||
}
|
||||
if got := atomic.LoadInt32(&detailRequests); got != 0 {
|
||||
t.Fatalf("detail requests = %d, want 0 (first download should use crawler URL)", got)
|
||||
}
|
||||
if got := atomic.LoadInt32(&originalRequests); got != 1 {
|
||||
t.Fatalf("original URL requests = %d, want 1", got)
|
||||
}
|
||||
if got := atomic.LoadInt32(&wrongRequests); got != 0 {
|
||||
t.Fatalf("wrong source URL requests = %d, want 0", got)
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(drv.RootDir(), "videos", "120201.mp4"))
|
||||
if err != nil {
|
||||
t.Fatalf("original video missing: %v", err)
|
||||
}
|
||||
if info.Size() != int64(len("ORIGINALVIDEO")) {
|
||||
t.Fatalf("original video size = %d, want %d", info.Size(), len("ORIGINALVIDEO"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRefreshesVideoURLAfterExpiredDownload(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell-based fake script only on unix")
|
||||
}
|
||||
tmp := t.TempDir()
|
||||
|
||||
var detailRequests int32
|
||||
var staleRequests int32
|
||||
var freshRequests int32
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/v.php":
|
||||
n := atomic.AddInt32(&detailRequests, 1)
|
||||
videoURL := srv.URL + "/videos/120202.mp4?token=stale"
|
||||
if n > 1 {
|
||||
videoURL = srv.URL + "/videos/120202.mp4?token=fresh"
|
||||
}
|
||||
_, _ = w.Write([]byte(spider91DetailHTML(videoURL)))
|
||||
case r.URL.Path == "/videos/120202.mp4" && r.URL.Query().Get("token") == "stale":
|
||||
atomic.AddInt32(&staleRequests, 1)
|
||||
http.Error(w, "expired", http.StatusForbidden)
|
||||
case r.URL.Path == "/videos/120202.mp4" && r.URL.Query().Get("token") == "fresh":
|
||||
atomic.AddInt32(&freshRequests, 1)
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write([]byte("REFRESHEDVIDEO"))
|
||||
case r.URL.Path == "/thumb/120202.jpg":
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write([]byte("\xff\xd8\xff\xe0thumb"))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
entry := map[string]string{
|
||||
"title": "Refresh After Expired Download",
|
||||
"thumb_url": srv.URL + "/thumb/wrong-thumb.jpg",
|
||||
"video_url": srv.URL + "/videos/120202.mp4?token=old",
|
||||
"viewkey": "vk-refresh-after",
|
||||
"detail_url": srv.URL + "/v.php?viewkey=vk-refresh-after",
|
||||
}
|
||||
cat, drv, scriptPath := seedCrawlerTestDeps(t, tmp, "refresh-after-drive", []map[string]string{entry})
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
PythonPath: "sh",
|
||||
ScriptPath: scriptPath,
|
||||
CommonThumbDir: filepath.Join(tmp, "previews", "thumbs"),
|
||||
SpiderTimeout: 10 * time.Second,
|
||||
DownloadTimeout: 10 * time.Second,
|
||||
})
|
||||
|
||||
res, err := c.RunOnce(context.Background(), 1)
|
||||
if err != nil {
|
||||
t.Fatalf("RunOnce: %v", err)
|
||||
}
|
||||
if res.NewVideos != 1 || res.Failed != 0 {
|
||||
t.Fatalf("result new=%d failed=%d, want 1/0", res.NewVideos, res.Failed)
|
||||
}
|
||||
if got := atomic.LoadInt32(&detailRequests); got < 2 {
|
||||
t.Fatalf("detail requests = %d, want at least 2 (initial refresh + retry refresh)", got)
|
||||
}
|
||||
if got := atomic.LoadInt32(&staleRequests); got != 1 {
|
||||
t.Fatalf("stale URL requests = %d, want 1", got)
|
||||
}
|
||||
if got := atomic.LoadInt32(&freshRequests); got != 1 {
|
||||
t.Fatalf("fresh URL requests = %d, want 1", got)
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(drv.RootDir(), "videos", "120202.mp4"))
|
||||
if err != nil {
|
||||
t.Fatalf("refreshed video missing: %v", err)
|
||||
}
|
||||
if info.Size() != int64(len("REFRESHEDVIDEO")) {
|
||||
t.Fatalf("refreshed video size = %d, want %d", info.Size(), len("REFRESHEDVIDEO"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerRejectsRefreshedSourceIDMismatch(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell-based fake script only on unix")
|
||||
}
|
||||
tmp := t.TempDir()
|
||||
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/v.php":
|
||||
_, _ = w.Write([]byte(spider91DetailHTML(srv.URL + "/videos/856305.mp4?token=fresh")))
|
||||
case r.URL.Path == "/videos/1203058.mp4":
|
||||
http.Error(w, "expired", http.StatusForbidden)
|
||||
case r.URL.Path == "/videos/856305.mp4":
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write([]byte("WRONGVIDEO"))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
entry := map[string]string{
|
||||
"title": "Source ID Mismatch",
|
||||
"thumb_url": srv.URL + "/thumb/1203058.jpg",
|
||||
"video_url": srv.URL + "/videos/1203058.mp4?token=old",
|
||||
"viewkey": "86fd91cce1f2e1a154cc",
|
||||
"source_id": "1203058",
|
||||
"detail_url": srv.URL + "/v.php?viewkey=86fd91cce1f2e1a154cc",
|
||||
}
|
||||
cat, drv, scriptPath := seedCrawlerTestDeps(t, tmp, "mismatch-drive", []map[string]string{entry})
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
Catalog: cat,
|
||||
PythonPath: "sh",
|
||||
ScriptPath: scriptPath,
|
||||
CommonThumbDir: filepath.Join(tmp, "previews", "thumbs"),
|
||||
SpiderTimeout: 10 * time.Second,
|
||||
DownloadTimeout: 10 * time.Second,
|
||||
})
|
||||
|
||||
res, err := c.RunOnce(context.Background(), 1)
|
||||
if err != nil {
|
||||
t.Fatalf("RunOnce: %v", err)
|
||||
}
|
||||
if res.NewVideos != 0 || res.Failed != 1 {
|
||||
t.Fatalf("result new=%d failed=%d, want 0/1", res.NewVideos, res.Failed)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(drv.RootDir(), "videos", "1203058.mp4")); !os.IsNotExist(err) {
|
||||
t.Fatalf("mismatched source file should not be written, stat err=%v", err)
|
||||
}
|
||||
if v, _ := cat.GetVideo(context.Background(), BuildVideoID(drv.ID(), "1203058")); v != nil {
|
||||
t.Fatalf("mismatched video should not be inserted: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceIDForItemRequiresNumericSourceID(t *testing.T) {
|
||||
if got := sourceIDForItem(spiderVideoEntry{
|
||||
Viewkey: "86fd91cce1f2e1a154cc",
|
||||
VideoURL: "https://cdn.example/videos/1203058.mp4?token=x",
|
||||
}); got != "1203058" {
|
||||
t.Fatalf("sourceIDForItem(video url) = %q, want 1203058", got)
|
||||
}
|
||||
if got := sourceIDForItem(spiderVideoEntry{
|
||||
Viewkey: "86fd91cce1f2e1a154cc",
|
||||
ThumbURL: "https://img.example/thumb/1203058.jpg",
|
||||
}); got != "1203058" {
|
||||
t.Fatalf("sourceIDForItem(thumb url) = %q, want 1203058", got)
|
||||
}
|
||||
if got := sourceIDForItem(spiderVideoEntry{
|
||||
Viewkey: "86fd91cce1f2e1a154cc",
|
||||
SourceID: "not-numeric",
|
||||
VideoURL: "https://cdn.example/videos/video.mp4",
|
||||
}); got != "" {
|
||||
t.Fatalf("sourceIDForItem(non numeric) = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeThumbURLForSource(t *testing.T) {
|
||||
got := normalizeThumbURLForSource("https://img.example/thumb/856305.jpg?x=1#frag", "1203058")
|
||||
want := "https://img.example/thumb/1203058.jpg"
|
||||
if got != want {
|
||||
t.Fatalf("normalizeThumbURLForSource = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpider91ListURLForDetail(t *testing.T) {
|
||||
got := spider91ListURLForDetail("https://www.91porn.com/view_video.php?viewkey=abc&page=5&c=furum&viewtype=basic&category=top")
|
||||
want := "https://www.91porn.com/v.php?category=top&page=5&viewtype=basic"
|
||||
if got != want {
|
||||
t.Fatalf("spider91ListURLForDetail = %q, want %q", got, want)
|
||||
}
|
||||
if got := spider91ListURLForDetail("http://127.0.0.1/v.php?viewkey=abc&page=5&viewtype=basic&category=top"); got != "" {
|
||||
t.Fatalf("spider91ListURLForDetail(localhost) = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpider91CookieHeader(t *testing.T) {
|
||||
got := spider91CookieHeader([]*http.Cookie{
|
||||
{Name: "CLIPSHARE", Value: "abc"},
|
||||
{Name: "ga", Value: "def"},
|
||||
{Name: "mode", Value: "m"},
|
||||
})
|
||||
want := "mode=d; CLIPSHARE=abc; ga=def"
|
||||
if got != want {
|
||||
t.Fatalf("spider91CookieHeader = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpider91ProgressLogLineClassifiers(t *testing.T) {
|
||||
if !isSpider91CheckedVideoLogLine("[2026-06-08 16:49:17] 处理视频 3/24: 标题") {
|
||||
t.Fatal("checked video log line was not recognized")
|
||||
}
|
||||
if isSpider91CheckedVideoLogLine("[2026-06-08 16:49:17] [页 2] 发现 24 个视频") {
|
||||
t.Fatal("page summary log line should not count as checked video")
|
||||
}
|
||||
if !isSpider91ExtractedVideoLogLine("[2026-06-08 16:49:39] [OK] 成功提取视频直链") {
|
||||
t.Fatal("extracted video log line was not recognized")
|
||||
}
|
||||
}
|
||||
|
||||
func spider91DetailHTML(videoURL string) string {
|
||||
fragment := `<video><source src="` + videoURL + `" type="video/mp4"></video>`
|
||||
return `document.write(strencode2("` + url.PathEscape(fragment) + `"));`
|
||||
}
|
||||
|
||||
func seedCrawlerTestDeps(t *testing.T, tmp, driveID string, entries []map[string]string) (*catalog.Catalog, *Driver, string) {
|
||||
t.Helper()
|
||||
scriptPath := filepath.Join(tmp, driveID+"-fake.sh")
|
||||
if err := os.WriteFile(scriptPath, []byte(buildFakeSpiderScript(entries)), 0o755); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
cat, err := catalog.Open(filepath.Join(tmp, driveID+".db"))
|
||||
if err != nil {
|
||||
t.Fatalf("catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
drv := New(Config{ID: driveID, RootDir: filepath.Join(tmp, "spider91", driveID)})
|
||||
if err := cat.UpsertDrive(context.Background(), &catalog.Drive{
|
||||
ID: driveID, Kind: Kind, Name: driveID,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert drive: %v", err)
|
||||
}
|
||||
return cat, drv, scriptPath
|
||||
}
|
||||
|
||||
// buildFakeSpiderScript 生成一个伪 python 脚本(其实是 sh)。
|
||||
//
|
||||
// 行为:
|
||||
// - 解析 --output FILE / --stream-output 两个 flag
|
||||
// - --stream-output 时:逐行输出每个 entry 的 JSON 到 stdout 并 flush
|
||||
// - --output 时:把完整 JSON 数据写到 FILE(向后兼容,且作归档)
|
||||
//
|
||||
// 用 sh 来写是为了避免 Python 依赖。每条 entry 的 JSON 用 Go marshal 出来后嵌入。
|
||||
func buildFakeSpiderScript(entries []map[string]string) string {
|
||||
var sb strings.Builder
|
||||
sb.WriteString("#!/bin/sh\n")
|
||||
sb.WriteString("out=\"\"; stream=0\n")
|
||||
sb.WriteString("while [ $# -gt 0 ]; do case \"$1\" in --output) out=\"$2\"; shift 2;; --stream-output) stream=1; shift;; *) shift;; esac; done\n")
|
||||
|
||||
// stream 模式:逐行 echo
|
||||
sb.WriteString("if [ \"$stream\" = \"1\" ]; then\n")
|
||||
for _, e := range entries {
|
||||
raw, _ := json.Marshal(e)
|
||||
// 用单引号 here-string 形式确保 JSON 中的双引号原样出来
|
||||
sb.WriteString(" cat <<'STREAM_EOF'\n")
|
||||
sb.Write(raw)
|
||||
sb.WriteString("\nSTREAM_EOF\n")
|
||||
}
|
||||
sb.WriteString("fi\n")
|
||||
|
||||
// 写 --output 文件(带完整 wrapper)
|
||||
sb.WriteString("if [ -n \"$out\" ]; then\n")
|
||||
sb.WriteString(" mkdir -p \"$(dirname \"$out\")\" 2>/dev/null\n")
|
||||
sb.WriteString(" cat > \"$out\" <<'OUT_EOF'\n")
|
||||
wrapper := map[string]any{
|
||||
"crawl_time": "2026-01-01T00:00:00",
|
||||
"total_videos": len(entries),
|
||||
"videos": entries,
|
||||
}
|
||||
wrapped, _ := json.MarshalIndent(wrapper, "", " ")
|
||||
sb.Write(wrapped)
|
||||
sb.WriteString("\nOUT_EOF\n")
|
||||
sb.WriteString("fi\n")
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func containsString(values []string, want string) bool {
|
||||
for _, value := range values {
|
||||
if value == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,235 +0,0 @@
|
||||
// Package spider91 把 91porn 爬虫的产物(本地下载好的视频和封面)
|
||||
// 包装成一个 drives.Drive 实现,让它跟其它网盘一样可以挂载到 catalog 上。
|
||||
//
|
||||
// 与其它 drive 不同的是:
|
||||
// - 数据来源不是云盘 API,而是 Python 子进程跑 spider_91porn.py 后下载到本地
|
||||
// - StreamURL 直接返回本地文件路径,由 api.handleSpider91Video 用 http.ServeFile 服务
|
||||
// - List/Stat 用于 GC 兜底(按本地文件名列出 videos/ 目录)
|
||||
package spider91
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
// Kind 是该 drive 的类型代号,写到 catalog.drives.kind。
|
||||
const Kind = "spider91"
|
||||
|
||||
// Config 创建 Driver 所需的配置。
|
||||
type Config struct {
|
||||
// ID 是 catalog 中的 drive id,driver 用它隔离每个 spider91 实例的本地目录。
|
||||
ID string
|
||||
// RootDir 是该 drive 在磁盘上的根目录,driver 会在下面创建 videos/ 和 thumbs/。
|
||||
// 一般由 backend 拼成 <data_dir>/spider91/<driveID>/。
|
||||
RootDir string
|
||||
}
|
||||
|
||||
// Driver 实现 drives.Drive。
|
||||
type Driver struct {
|
||||
id string
|
||||
rootDir string
|
||||
}
|
||||
|
||||
// New 构造一个 Driver。
|
||||
func New(c Config) *Driver {
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
rootDir: c.RootDir,
|
||||
}
|
||||
}
|
||||
|
||||
// Kind 返回 "spider91"。
|
||||
func (d *Driver) Kind() string { return Kind }
|
||||
|
||||
// ID 返回 catalog 中的 drive id。
|
||||
func (d *Driver) ID() string { return d.id }
|
||||
|
||||
// RootID 返回根目录的逻辑 ID。spider91 没有真正的目录结构,
|
||||
// 这里固定返回 "/" 占位,调用方实际不会用它去 List 子目录。
|
||||
func (d *Driver) RootID() string { return "/" }
|
||||
|
||||
// Init 确保 rootDir/videos 和 rootDir/thumbs 存在。
|
||||
func (d *Driver) Init(ctx context.Context) error {
|
||||
if strings.TrimSpace(d.rootDir) == "" {
|
||||
return errors.New("spider91: empty rootDir")
|
||||
}
|
||||
for _, sub := range []string{"videos", "thumbs"} {
|
||||
if err := os.MkdirAll(filepath.Join(d.rootDir, sub), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// VideosDir 返回视频文件存放目录的绝对路径。
|
||||
func (d *Driver) VideosDir() string { return filepath.Join(d.rootDir, "videos") }
|
||||
|
||||
// ThumbsDir 返回封面文件存放目录的绝对路径。
|
||||
func (d *Driver) ThumbsDir() string { return filepath.Join(d.rootDir, "thumbs") }
|
||||
|
||||
// RootDir 返回 driver 的存储根。
|
||||
func (d *Driver) RootDir() string { return d.rootDir }
|
||||
|
||||
// VideoPath 返回某个视频文件的绝对路径,并校验路径不会逃出 videos/ 目录。
|
||||
func (d *Driver) VideoPath(fileID string) (string, error) {
|
||||
return safeJoin(d.VideosDir(), fileID)
|
||||
}
|
||||
|
||||
// ThumbPath 返回某个封面文件的绝对路径。
|
||||
func (d *Driver) ThumbPath(fileID string) (string, error) {
|
||||
return safeJoin(d.ThumbsDir(), fileID)
|
||||
}
|
||||
|
||||
// List 列出 videos/ 目录下的视频文件,便于上层做 GC 兜底;
|
||||
// dirID 当前会被忽略,spider91 没有目录树。
|
||||
func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error) {
|
||||
entries, err := os.ReadDir(d.VideosDir())
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
out := make([]drives.Entry, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, drives.Entry{
|
||||
ID: e.Name(),
|
||||
Name: e.Name(),
|
||||
Size: info.Size(),
|
||||
IsDir: false,
|
||||
ModTime: info.ModTime(),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Stat 查询单个视频文件的元数据。
|
||||
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
|
||||
path, err := d.VideoPath(fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &drives.Entry{
|
||||
ID: fileID,
|
||||
Name: fileID,
|
||||
Size: info.Size(),
|
||||
IsDir: info.IsDir(),
|
||||
ModTime: info.ModTime(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// StreamURL 返回本地视频文件路径,给 ffmpeg / 上层服务使用。
|
||||
// 注意:proxy.serve 不能直接处理本地路径,回放要走 api.handleSpider91Video。
|
||||
// 预览视频/封面 worker 通过 localPreviewLink 兜底走本地文件,刚好兼容 path 形式的 URL。
|
||||
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
path, err := d.VideoPath(fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.IsDir() || info.Size() == 0 {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return &drives.StreamLink{
|
||||
URL: path,
|
||||
Expires: time.Now().Add(24 * time.Hour),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Upload 不支持:上传由 crawler 自己完成,不通过 Drive 接口。
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
// EnsureDir 不支持。
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
videoPath, err := d.VideoPath(fileID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Stat(videoPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
removeThumbCandidates(d.ThumbPath, strings.TrimSuffix(fileID, filepath.Ext(fileID)))
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return errors.New("spider91: refusing to remove directory")
|
||||
}
|
||||
if err := os.Remove(videoPath); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
removeThumbCandidates(d.ThumbPath, strings.TrimSuffix(fileID, filepath.Ext(fileID)))
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeThumbCandidates(pathFor func(string) (string, error), stem string) {
|
||||
stem = strings.TrimSpace(stem)
|
||||
if stem == "" {
|
||||
return
|
||||
}
|
||||
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
|
||||
path, err := pathFor(stem + ext)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}
|
||||
|
||||
// safeJoin 把 fileID 拼到 root 下,保证最终路径不会逃出 root。
|
||||
// fileID 必须是单纯的文件名(不含 / 或 .. 等组件)。
|
||||
func safeJoin(root, fileID string) (string, error) {
|
||||
id := strings.TrimSpace(fileID)
|
||||
if id == "" || filepath.Base(id) != id {
|
||||
return "", errors.New("spider91: invalid file id")
|
||||
}
|
||||
if root == "" {
|
||||
return "", errors.New("spider91: empty root dir")
|
||||
}
|
||||
rootAbs, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
pathAbs, err := filepath.Abs(filepath.Join(rootAbs, id))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if pathAbs != rootAbs && !strings.HasPrefix(pathAbs, rootAbs+string(os.PathSeparator)) {
|
||||
return "", errors.New("spider91: file id escapes root")
|
||||
}
|
||||
return pathAbs, nil
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
var _ drives.Remover = (*Driver)(nil)
|
||||
@@ -1,149 +0,0 @@
|
||||
package spider91
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDriverInitCreatesSubdirs(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
d := New(Config{ID: "test", RootDir: filepath.Join(dir, "drive1")})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
for _, sub := range []string{"videos", "thumbs"} {
|
||||
info, err := os.Stat(filepath.Join(dir, "drive1", sub))
|
||||
if err != nil {
|
||||
t.Fatalf("stat %s: %v", sub, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
t.Fatalf("%s is not a dir", sub)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriverInitRejectsEmptyRoot(t *testing.T) {
|
||||
d := New(Config{ID: "test", RootDir: ""})
|
||||
if err := d.Init(context.Background()); err == nil {
|
||||
t.Fatalf("expected error for empty root")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoPathRejectsTraversal(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
d := New(Config{ID: "test", RootDir: dir})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
cases := []string{
|
||||
"",
|
||||
" ",
|
||||
"../etc/passwd",
|
||||
"sub/dir.mp4",
|
||||
"./abc.mp4",
|
||||
}
|
||||
for _, c := range cases {
|
||||
if _, err := d.VideoPath(c); err == nil {
|
||||
t.Fatalf("VideoPath(%q) accepted, want error", c)
|
||||
}
|
||||
if _, err := d.ThumbPath(c); err == nil {
|
||||
t.Fatalf("ThumbPath(%q) accepted, want error", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoPathHappy(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
d := New(Config{ID: "test", RootDir: dir})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
got, err := d.VideoPath("abc.mp4")
|
||||
if err != nil {
|
||||
t.Fatalf("VideoPath: %v", err)
|
||||
}
|
||||
want := filepath.Join(dir, "videos", "abc.mp4")
|
||||
wantAbs, _ := filepath.Abs(want)
|
||||
if got != wantAbs {
|
||||
t.Fatalf("VideoPath: got %q want %q", got, wantAbs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListReturnsFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
d := New(Config{ID: "test", RootDir: dir})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
mustWrite(t, filepath.Join(d.VideosDir(), "abc.mp4"), "data")
|
||||
mustWrite(t, filepath.Join(d.VideosDir(), "def.mp4"), "x")
|
||||
|
||||
entries, err := d.List(context.Background(), "/")
|
||||
if err != nil {
|
||||
t.Fatalf("List: %v", err)
|
||||
}
|
||||
if len(entries) != 2 {
|
||||
t.Fatalf("List len = %d, want 2", len(entries))
|
||||
}
|
||||
names := map[string]int64{}
|
||||
for _, e := range entries {
|
||||
names[e.Name] = e.Size
|
||||
}
|
||||
if names["abc.mp4"] != 4 || names["def.mp4"] != 1 {
|
||||
t.Fatalf("unexpected entries: %+v", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLReturnsLocalPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
d := New(Config{ID: "test", RootDir: dir})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
mustWrite(t, filepath.Join(d.VideosDir(), "abc.mp4"), "videodata")
|
||||
|
||||
link, err := d.StreamURL(context.Background(), "abc.mp4")
|
||||
if err != nil {
|
||||
t.Fatalf("StreamURL: %v", err)
|
||||
}
|
||||
if !strings.HasSuffix(link.URL, "videos/abc.mp4") {
|
||||
t.Fatalf("StreamURL.URL = %q, want suffix videos/abc.mp4", link.URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamURLEmptyFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
d := New(Config{ID: "test", RootDir: dir})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
mustWrite(t, filepath.Join(d.VideosDir(), "abc.mp4"), "")
|
||||
if _, err := d.StreamURL(context.Background(), "abc.mp4"); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("empty file should return os.ErrNotExist, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildVideoIDStable(t *testing.T) {
|
||||
id1 := BuildVideoID("crawler1", "abc")
|
||||
id2 := BuildVideoID("crawler1", "abc")
|
||||
if id1 != id2 {
|
||||
t.Fatalf("BuildVideoID not deterministic")
|
||||
}
|
||||
if id1 != "spider91-crawler1-abc" {
|
||||
t.Fatalf("BuildVideoID format unexpected: %q", id1)
|
||||
}
|
||||
}
|
||||
|
||||
func mustWrite(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
package spider91
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestDetectVideoExt(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
url string
|
||||
want string
|
||||
}{
|
||||
{"mp4 with token", "https://cdn.example.com/mp43/abc.mp4?st=xyz&e=12345", ".mp4"},
|
||||
{"webm", "https://cdn.example.com/path/video.webm?token=1", ".webm"},
|
||||
{"mkv", "https://cdn.example.com/path/foo.mkv", ".mkv"},
|
||||
{"mov", "https://cdn.example.com/path/foo.mov?x=1", ".mov"},
|
||||
{"flv", "https://cdn.example.com/path/foo.flv", ".flv"},
|
||||
{"m4v", "https://cdn.example.com/path/foo.m4v", ".m4v"},
|
||||
{"avi", "https://cdn.example.com/path/foo.avi", ".avi"},
|
||||
{"m3u8 fallback to mp4", "https://cdn.example.com/path/playlist.m3u8", ".mp4"},
|
||||
{"ts fallback to mp4", "https://cdn.example.com/path/seg001.ts", ".mp4"},
|
||||
{"unknown ext fallback", "https://cdn.example.com/path/foo.weird", ".mp4"},
|
||||
{"no ext fallback", "https://cdn.example.com/v.php?id=12345", ".mp4"},
|
||||
{"empty url", "", ".mp4"},
|
||||
{"uppercase", "https://cdn.example.com/path/FOO.MP4?token=1", ".mp4"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := detectVideoExt(tc.url)
|
||||
if got != tc.want {
|
||||
t.Fatalf("detectVideoExt(%q) = %q, want %q", tc.url, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectThumbExt(t *testing.T) {
|
||||
tests := []struct {
|
||||
url string
|
||||
want string
|
||||
}{
|
||||
{"https://cdn.example.com/thumb/foo.jpg", ".jpg"},
|
||||
{"https://cdn.example.com/thumb/foo.jpeg", ".jpeg"},
|
||||
{"https://cdn.example.com/thumb/foo.png", ".png"},
|
||||
{"https://cdn.example.com/thumb/foo.webp", ".webp"},
|
||||
{"https://cdn.example.com/thumb/foo.gif", ".gif"},
|
||||
{"https://cdn.example.com/thumb/foo.svg", ".jpg"}, // not in whitelist
|
||||
{"https://cdn.example.com/thumb/no-ext", ".jpg"},
|
||||
{"", ".jpg"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
got := detectThumbExt(tc.url)
|
||||
if got != tc.want {
|
||||
t.Fatalf("detectThumbExt(%q) = %q, want %q", tc.url, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,7 @@ type Driver struct {
|
||||
refreshToken string
|
||||
client *sdk.WoClient
|
||||
onTokenUpdate func(access, refresh string)
|
||||
uploadTempDir string
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
@@ -38,11 +39,12 @@ type Driver struct {
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ID string
|
||||
AccessToken string
|
||||
RefreshToken string
|
||||
FamilyID string // 空则走个人空间,有值则走家庭空间
|
||||
RootID string // 根目录 ID,默认 "0"
|
||||
ID string
|
||||
AccessToken string
|
||||
RefreshToken string
|
||||
FamilyID string // 空则走个人空间,有值则走家庭空间
|
||||
RootID string // 根目录 ID,默认 "0"
|
||||
UploadTempDir string
|
||||
// 当 SDK 刷新 token 时回调,便于持久化
|
||||
OnTokenUpdate func(access, refresh string)
|
||||
}
|
||||
@@ -59,6 +61,7 @@ func New(c Config) *Driver {
|
||||
accessToken: c.AccessToken,
|
||||
refreshToken: c.RefreshToken,
|
||||
onTokenUpdate: c.OnTokenUpdate,
|
||||
uploadTempDir: strings.TrimSpace(c.UploadTempDir),
|
||||
listInterval: 800 * time.Millisecond,
|
||||
listCooldown: 5 * time.Minute,
|
||||
fidToID: make(map[string]string),
|
||||
@@ -162,7 +165,12 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
// wopan SDK 要求 *os.File,先把流落到临时文件再上传
|
||||
tmp, err := os.CreateTemp("", "wopan-upload-*.tmp")
|
||||
if d.uploadTempDir != "" {
|
||||
if err := os.MkdirAll(d.uploadTempDir, 0o755); err != nil {
|
||||
return "", fmt.Errorf("wopan upload: create tmp dir: %w", err)
|
||||
}
|
||||
}
|
||||
tmp, err := os.CreateTemp(d.uploadTempDir, "wopan-upload-*.tmp")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
package mediasim
|
||||
|
||||
import (
|
||||
"image"
|
||||
_ "image/gif"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"math"
|
||||
"os"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
const (
|
||||
ssimSampleSize = 96
|
||||
minCoreTitleRunes = 12
|
||||
)
|
||||
|
||||
var titleCoreSeparators = []string{
|
||||
" - ",
|
||||
" -- ",
|
||||
" — ",
|
||||
" – ",
|
||||
" | ",
|
||||
" | ",
|
||||
"_",
|
||||
"_",
|
||||
"-",
|
||||
"—",
|
||||
"–",
|
||||
"-",
|
||||
"|",
|
||||
}
|
||||
|
||||
// TitleSimilarity returns the best normalized Levenshtein similarity in [0, 1]
|
||||
// between the full titles and their leading core title segments.
|
||||
func TitleSimilarity(a, b string) float64 {
|
||||
leftVariants := titleVariants(a)
|
||||
rightVariants := titleVariants(b)
|
||||
if len(leftVariants) == 0 && len(rightVariants) == 0 {
|
||||
return 1
|
||||
}
|
||||
if len(leftVariants) == 0 || len(rightVariants) == 0 {
|
||||
return 0
|
||||
}
|
||||
best := 0.0
|
||||
for _, left := range leftVariants {
|
||||
for _, right := range rightVariants {
|
||||
score := normalizedLevenshteinSimilarity(left, right)
|
||||
if score > best {
|
||||
best = score
|
||||
}
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// TitleKeys returns the normalized full title and core-title variants used by
|
||||
// TitleSimilarity. It is intended for cheap caller-side prefiltering before
|
||||
// running the heavier Levenshtein comparison.
|
||||
func TitleKeys(value string) []string {
|
||||
return append([]string(nil), titleVariants(value)...)
|
||||
}
|
||||
|
||||
func normalizedLevenshteinSimilarity(left, right string) float64 {
|
||||
leftRunes := []rune(left)
|
||||
rightRunes := []rune(right)
|
||||
if len(leftRunes) == 0 && len(rightRunes) == 0 {
|
||||
return 1
|
||||
}
|
||||
if len(leftRunes) == 0 || len(rightRunes) == 0 {
|
||||
return 0
|
||||
}
|
||||
maxLen := len(leftRunes)
|
||||
if len(rightRunes) > maxLen {
|
||||
maxLen = len(rightRunes)
|
||||
}
|
||||
return 1 - float64(levenshtein(leftRunes, rightRunes))/float64(maxLen)
|
||||
}
|
||||
|
||||
func titleVariants(value string) []string {
|
||||
full := normalizeTitle(value)
|
||||
if full == "" {
|
||||
return nil
|
||||
}
|
||||
out := appendTitleVariant(nil, full)
|
||||
if core := normalizeTitleCore(value); core != "" && core != full {
|
||||
out = appendTitleVariant(out, core)
|
||||
}
|
||||
for _, tail := range titleTailVariants(value) {
|
||||
normalized := normalizeTitle(tail)
|
||||
if len([]rune(normalized)) >= minCoreTitleRunes {
|
||||
out = appendTitleVariant(out, normalized)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func appendTitleVariant(out []string, value string) []string {
|
||||
for _, existing := range out {
|
||||
if existing == value {
|
||||
return out
|
||||
}
|
||||
}
|
||||
return append(out, value)
|
||||
}
|
||||
|
||||
func titleTailVariants(value string) []string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, sep := range []string{"@", "@"} {
|
||||
if idx := strings.LastIndex(value, sep); idx >= 0 && idx+len(sep) < len(value) {
|
||||
out = append(out, strings.TrimSpace(value[idx+len(sep):]))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normalizeTitleCore(value string) string {
|
||||
head := strings.TrimSpace(value)
|
||||
for _, sep := range titleCoreSeparators {
|
||||
if idx := strings.Index(head, sep); idx > 0 {
|
||||
head = strings.TrimSpace(head[:idx])
|
||||
break
|
||||
}
|
||||
}
|
||||
normalized := normalizeTitle(head)
|
||||
if len([]rune(normalized)) < minCoreTitleRunes {
|
||||
return ""
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
func normalizeTitle(value string) string {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
for _, ext := range []string{".mp4", ".m4v", ".mkv", ".mov", ".avi", ".webm", ".ts", ".m3u8"} {
|
||||
if strings.HasSuffix(value, ext) {
|
||||
value = strings.TrimSuffix(value, ext)
|
||||
break
|
||||
}
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range value {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
if b.Len() > 0 {
|
||||
return b.String()
|
||||
}
|
||||
return strings.Join(strings.Fields(value), "")
|
||||
}
|
||||
|
||||
func levenshtein(a, b []rune) int {
|
||||
if len(a) < len(b) {
|
||||
a, b = b, a
|
||||
}
|
||||
previous := make([]int, len(b)+1)
|
||||
current := make([]int, len(b)+1)
|
||||
for j := range previous {
|
||||
previous[j] = j
|
||||
}
|
||||
for i := 1; i <= len(a); i++ {
|
||||
current[0] = i
|
||||
for j := 1; j <= len(b); j++ {
|
||||
cost := 0
|
||||
if a[i-1] != b[j-1] {
|
||||
cost = 1
|
||||
}
|
||||
current[j] = minInt(
|
||||
previous[j]+1,
|
||||
current[j-1]+1,
|
||||
previous[j-1]+cost,
|
||||
)
|
||||
}
|
||||
previous, current = current, previous
|
||||
}
|
||||
return previous[len(b)]
|
||||
}
|
||||
|
||||
func minInt(values ...int) int {
|
||||
min := values[0]
|
||||
for _, value := range values[1:] {
|
||||
if value < min {
|
||||
min = value
|
||||
}
|
||||
}
|
||||
return min
|
||||
}
|
||||
|
||||
// ImageSSIM compares two local images using luminance SSIM over a fixed grid.
|
||||
func ImageSSIM(leftPath, rightPath string) (float64, error) {
|
||||
left, err := decodeImage(leftPath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
right, err := decodeImage(rightPath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return SSIM(left, right), nil
|
||||
}
|
||||
|
||||
func decodeImage(path string) (image.Image, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
img, _, err := image.Decode(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return img, nil
|
||||
}
|
||||
|
||||
// SSIM compares two images after nearest-neighbor sampling onto the same grid.
|
||||
func SSIM(left, right image.Image) float64 {
|
||||
if left == nil || right == nil {
|
||||
return 0
|
||||
}
|
||||
leftSamples := grayscaleSamples(left, ssimSampleSize, ssimSampleSize)
|
||||
rightSamples := grayscaleSamples(right, ssimSampleSize, ssimSampleSize)
|
||||
if len(leftSamples) == 0 || len(leftSamples) != len(rightSamples) {
|
||||
return 0
|
||||
}
|
||||
|
||||
var leftMean, rightMean float64
|
||||
for i := range leftSamples {
|
||||
leftMean += leftSamples[i]
|
||||
rightMean += rightSamples[i]
|
||||
}
|
||||
n := float64(len(leftSamples))
|
||||
leftMean /= n
|
||||
rightMean /= n
|
||||
|
||||
var leftVariance, rightVariance, covariance float64
|
||||
for i := range leftSamples {
|
||||
leftDelta := leftSamples[i] - leftMean
|
||||
rightDelta := rightSamples[i] - rightMean
|
||||
leftVariance += leftDelta * leftDelta
|
||||
rightVariance += rightDelta * rightDelta
|
||||
covariance += leftDelta * rightDelta
|
||||
}
|
||||
leftVariance /= n
|
||||
rightVariance /= n
|
||||
covariance /= n
|
||||
|
||||
const c1 = 6.5025 // (0.01 * 255)^2
|
||||
const c2 = 58.5225 // (0.03 * 255)^2
|
||||
denominator := (leftMean*leftMean + rightMean*rightMean + c1) * (leftVariance + rightVariance + c2)
|
||||
if denominator == 0 {
|
||||
return 0
|
||||
}
|
||||
score := ((2*leftMean*rightMean + c1) * (2*covariance + c2)) / denominator
|
||||
if math.IsNaN(score) || math.IsInf(score, 0) {
|
||||
return 0
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
func grayscaleSamples(img image.Image, width, height int) []float64 {
|
||||
bounds := img.Bounds()
|
||||
if bounds.Dx() <= 0 || bounds.Dy() <= 0 || width <= 0 || height <= 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]float64, 0, width*height)
|
||||
for y := 0; y < height; y++ {
|
||||
sourceY := bounds.Min.Y + y*bounds.Dy()/height
|
||||
for x := 0; x < width; x++ {
|
||||
sourceX := bounds.Min.X + x*bounds.Dx()/width
|
||||
r, g, b, _ := img.At(sourceX, sourceY).RGBA()
|
||||
out = append(out, 0.299*float64(r>>8)+0.587*float64(g>>8)+0.114*float64(b>>8))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package mediasim
|
||||
|
||||
import (
|
||||
"image"
|
||||
"image/color"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTitleSimilarityNormalizesPunctuationAndWhitespace(t *testing.T) {
|
||||
score := TitleSimilarity("AB-123 测试视频.mp4", "ab123测试视频")
|
||||
if score < 0.90 {
|
||||
t.Fatalf("similarity = %.3f, want >= 0.90", score)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTitleSimilarityUsesLeadingCoreTitle(t *testing.T) {
|
||||
score := TitleSimilarity(
|
||||
"反差极品大二女友,叫声可射~,“射进小骚逼里面~” - 性感小皮鞭",
|
||||
"反差极品大二女友,叫声可射~,“射进小骚逼里面~”",
|
||||
)
|
||||
if score < 0.99 {
|
||||
t.Fatalf("similarity = %.3f, want core-title match", score)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTitleSimilarityDoesNotMatchBySharedSuffixOnly(t *testing.T) {
|
||||
score := TitleSimilarity(
|
||||
"高颜值大学生宿舍自拍视频完整流出 - 同一个来源",
|
||||
"户外旅行风景记录城市夜景合集 - 同一个来源",
|
||||
)
|
||||
if score >= 0.90 {
|
||||
t.Fatalf("similarity = %.3f, want < 0.90", score)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTitleSimilarityRejectsDifferentTitles(t *testing.T) {
|
||||
score := TitleSimilarity("完全不同的视频标题", "another unrelated movie")
|
||||
if score >= 0.90 {
|
||||
t.Fatalf("similarity = %.3f, want < 0.90", score)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSIMScoresIdenticalAndDifferentImages(t *testing.T) {
|
||||
red := solidImage(color.RGBA{R: 220, G: 20, B: 20, A: 255})
|
||||
redAgain := solidImage(color.RGBA{R: 220, G: 20, B: 20, A: 255})
|
||||
blue := solidImage(color.RGBA{R: 20, G: 20, B: 220, A: 255})
|
||||
|
||||
if score := SSIM(red, redAgain); score < 0.999 {
|
||||
t.Fatalf("identical SSIM = %.6f, want close to 1", score)
|
||||
}
|
||||
if score := SSIM(red, blue); score >= 0.95 {
|
||||
t.Fatalf("different SSIM = %.6f, want < 0.95", score)
|
||||
}
|
||||
}
|
||||
|
||||
func solidImage(c color.RGBA) image.Image {
|
||||
img := image.NewRGBA(image.Rect(0, 0, 32, 32))
|
||||
for y := 0; y < 32; y++ {
|
||||
for x := 0; x < 32; x++ {
|
||||
img.SetRGBA(x, y, c)
|
||||
}
|
||||
}
|
||||
return img
|
||||
}
|
||||
@@ -1,19 +1,19 @@
|
||||
// Package nightly orchestrates the single nightly maintenance pipeline that
|
||||
// replaces the legacy scanLoop / crawlerLoop / spider91 migrator periodic loop.
|
||||
// replaces the legacy scanLoop / crawlerLoop / crawler upload periodic loop.
|
||||
//
|
||||
// Pipeline (fired once per day at cron_hour, also via TriggerNow for admin
|
||||
// "扫描所有网盘"):
|
||||
//
|
||||
// Phase 1: for each non-spider91 cloud drive
|
||||
// Phase 1: for each non-crawler cloud drive
|
||||
// scan + delete-detection + enqueue thumb + enqueue preview video
|
||||
// wait until all thumb / preview-video queues are idle
|
||||
// Phase 2: if any spider91 drive configured
|
||||
// Phase 2: if any script crawler configured
|
||||
// crawl + enqueue preview video for new videos
|
||||
// wait until preview-video queues are idle
|
||||
// Phase 3: spider91 → cloud migration (single sweep, captcha cooldown still
|
||||
// Phase 3: crawler local video → cloud upload (single sweep, captcha cooldown still
|
||||
// honored within this call)
|
||||
// Phase 4: cleanup duplicate local preview/thumbnail assets after sampled
|
||||
// fingerprints have identified canonical videos
|
||||
// Phase 4: full-library duplicate video maintenance:
|
||||
// exact size+sampled_sha256 dedupe, then title/duration/thumbnail dedupe
|
||||
//
|
||||
// A 6h soft deadline guards each pipeline run; phases check deadline at their
|
||||
// boundaries and exit cleanly if exceeded (no in-flight ffmpeg / upload is
|
||||
@@ -64,32 +64,32 @@ type Config struct {
|
||||
MaxDuration time.Duration
|
||||
|
||||
// ListScanTargets returns the drive IDs to run Phase 1 on, in deterministic
|
||||
// order. Should exclude spider91 and localupload drives.
|
||||
// order. Should exclude crawler and localupload drives.
|
||||
ListScanTargets func(ctx context.Context) []string
|
||||
|
||||
// RunScan synchronously runs scan + cleanup + enqueueDriveGeneration for
|
||||
// one drive. Errors are expected to be logged inside, not surfaced.
|
||||
RunScan func(ctx context.Context, driveID string)
|
||||
|
||||
// ListSpider91Drives returns spider91 drive IDs to crawl in Phase 2.
|
||||
// Returns empty slice when no spider91 drive is configured.
|
||||
ListSpider91Drives func(ctx context.Context) []string
|
||||
// ListCrawlerDrives returns script crawler drive IDs to crawl in Phase 2.
|
||||
// Returns empty slice when no crawler is configured.
|
||||
ListCrawlerDrives func(ctx context.Context) []string
|
||||
|
||||
// RunSpider91Crawl synchronously runs one crawl cycle (downloads + thumbs +
|
||||
// preview-video enqueue) for a single spider91 drive.
|
||||
RunSpider91Crawl func(ctx context.Context, driveID string)
|
||||
// RunCrawlerCrawl synchronously runs one crawl cycle (downloads + thumbs +
|
||||
// preview-video enqueue) for a single crawler drive.
|
||||
RunCrawlerCrawl func(ctx context.Context, driveID string)
|
||||
|
||||
// WaitPreviewQueuesIdle blocks until both the thumbnail and preview-video queues
|
||||
// across all drives are drained (queue empty + no in-flight task). It must
|
||||
// honor ctx cancellation.
|
||||
WaitPreviewQueuesIdle func(ctx context.Context) error
|
||||
|
||||
// RunMigration runs spider91migrate.Migrator.RunOnce for Phase 3.
|
||||
// RunMigration runs crawlerupload.Migrator.RunOnce for Phase 3.
|
||||
RunMigration func(ctx context.Context) error
|
||||
|
||||
// RunDedupeAssetCleanup removes generated local assets from non-canonical
|
||||
// videos in size+sampled_sha256 duplicate groups. It must not delete cloud
|
||||
// files or catalog rows.
|
||||
// RunDedupeAssetCleanup runs full-library duplicate video maintenance. It
|
||||
// removes duplicate catalog rows and local generated assets, but never
|
||||
// deletes cloud source files.
|
||||
RunDedupeAssetCleanup func(ctx context.Context) error
|
||||
|
||||
// Now is injected for tests; nil → time.Now.
|
||||
@@ -351,23 +351,23 @@ func (r *Runner) runPipeline(ctx context.Context) {
|
||||
if r.checkDeadline(ctx, "phase 2") {
|
||||
return
|
||||
}
|
||||
spiderIDs := []string{}
|
||||
if r.cfg.ListSpider91Drives != nil {
|
||||
spiderIDs = r.cfg.ListSpider91Drives(ctx)
|
||||
crawlerIDs := []string{}
|
||||
if r.cfg.ListCrawlerDrives != nil {
|
||||
crawlerIDs = r.cfg.ListCrawlerDrives(ctx)
|
||||
}
|
||||
if len(spiderIDs) == 0 {
|
||||
log.Printf("[nightly] phase 2/3 skipped: no spider91 drive configured")
|
||||
if len(crawlerIDs) == 0 {
|
||||
log.Printf("[nightly] phase 2/3 skipped: no crawler configured")
|
||||
r.runDedupeAssetCleanupPhase(ctx)
|
||||
return
|
||||
}
|
||||
log.Printf("[nightly] phase 2: crawling %d spider91 drive(s)", len(spiderIDs))
|
||||
for _, id := range spiderIDs {
|
||||
log.Printf("[nightly] phase 2: crawling %d crawler drive(s)", len(crawlerIDs))
|
||||
for _, id := range crawlerIDs {
|
||||
if ctx.Err() != nil {
|
||||
log.Printf("[nightly] phase 2 aborted by ctx: %v", ctx.Err())
|
||||
return
|
||||
}
|
||||
log.Printf("[nightly] phase 2: crawling drive=%s", id)
|
||||
r.cfg.RunSpider91Crawl(ctx, id)
|
||||
r.cfg.RunCrawlerCrawl(ctx, id)
|
||||
}
|
||||
log.Printf("[nightly] phase 2: waiting for teaser queue to drain")
|
||||
if err := r.waitIdle(ctx, "phase 2"); err != nil {
|
||||
@@ -378,7 +378,7 @@ func (r *Runner) runPipeline(ctx context.Context) {
|
||||
if r.checkDeadline(ctx, "phase 3") {
|
||||
return
|
||||
}
|
||||
log.Printf("[nightly] phase 3: spider91 migration")
|
||||
log.Printf("[nightly] phase 3: crawler upload")
|
||||
if r.cfg.RunMigration != nil {
|
||||
if err := r.cfg.RunMigration(ctx); err != nil {
|
||||
log.Printf("[nightly] phase 3 migration: %v", err)
|
||||
@@ -418,9 +418,9 @@ func (r *Runner) runDedupeAssetCleanupPhase(ctx context.Context) {
|
||||
if r.cfg.RunDedupeAssetCleanup == nil {
|
||||
return
|
||||
}
|
||||
log.Printf("[nightly] phase 4: duplicate asset cleanup")
|
||||
log.Printf("[nightly] phase 4: duplicate video maintenance")
|
||||
if err := r.cfg.RunDedupeAssetCleanup(ctx); err != nil {
|
||||
log.Printf("[nightly] phase 4 duplicate asset cleanup: %v", err)
|
||||
log.Printf("[nightly] phase 4 duplicate video maintenance: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -99,11 +99,11 @@ func TestRunPipelineHonoursPhaseOrder(t *testing.T) {
|
||||
RunScan: func(_ context.Context, id string) {
|
||||
rec.push("scan:" + id)
|
||||
},
|
||||
ListSpider91Drives: func(context.Context) []string {
|
||||
rec.push("list-spider")
|
||||
ListCrawlerDrives: func(context.Context) []string {
|
||||
rec.push("list-crawler")
|
||||
return []string{"sp-1"}
|
||||
},
|
||||
RunSpider91Crawl: func(_ context.Context, id string) {
|
||||
RunCrawlerCrawl: func(_ context.Context, id string) {
|
||||
rec.push("crawl:" + id)
|
||||
},
|
||||
WaitPreviewQueuesIdle: func(context.Context) error {
|
||||
@@ -128,7 +128,7 @@ func TestRunPipelineHonoursPhaseOrder(t *testing.T) {
|
||||
"scan:drive-a",
|
||||
"scan:drive-b",
|
||||
"wait-idle", // after phase 1
|
||||
"list-spider",
|
||||
"list-crawler",
|
||||
"crawl:sp-1",
|
||||
"wait-idle", // after phase 2
|
||||
"migrate",
|
||||
@@ -144,15 +144,15 @@ func TestRunPipelineHonoursPhaseOrder(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineSkipsMigrationWhenNoSpider91(t *testing.T) {
|
||||
func TestRunPipelineSkipsMigrationWhenNoCrawler(t *testing.T) {
|
||||
rec := &recorder{}
|
||||
|
||||
r := New(Config{
|
||||
Settings: newStubSettings(),
|
||||
ListScanTargets: func(context.Context) []string { return []string{"drive-a"} },
|
||||
RunScan: func(_ context.Context, id string) { rec.push("scan:" + id) },
|
||||
ListSpider91Drives: func(context.Context) []string { return nil },
|
||||
RunSpider91Crawl: func(_ context.Context, id string) { rec.push("crawl:" + id) },
|
||||
Settings: newStubSettings(),
|
||||
ListScanTargets: func(context.Context) []string { return []string{"drive-a"} },
|
||||
RunScan: func(_ context.Context, id string) { rec.push("scan:" + id) },
|
||||
ListCrawlerDrives: func(context.Context) []string { return nil },
|
||||
RunCrawlerCrawl: func(_ context.Context, id string) { rec.push("crawl:" + id) },
|
||||
WaitPreviewQueuesIdle: func(context.Context) error {
|
||||
rec.push("wait-idle")
|
||||
return nil
|
||||
@@ -171,7 +171,7 @@ func TestRunPipelineSkipsMigrationWhenNoSpider91(t *testing.T) {
|
||||
|
||||
for _, c := range rec.snapshot() {
|
||||
if c == "migrate" || c == "crawl:sp-1" {
|
||||
t.Fatalf("phase 2/3 should be skipped when no spider91 drive, got call %q", c)
|
||||
t.Fatalf("phase 2/3 should be skipped when no crawler, got call %q", c)
|
||||
}
|
||||
}
|
||||
foundCleanup := false
|
||||
@@ -181,7 +181,7 @@ func TestRunPipelineSkipsMigrationWhenNoSpider91(t *testing.T) {
|
||||
}
|
||||
}
|
||||
if !foundCleanup {
|
||||
t.Fatalf("dedupe cleanup should still run when spider91 is absent; calls=%v", rec.snapshot())
|
||||
t.Fatalf("dedupe cleanup should still run when crawler is absent; calls=%v", rec.snapshot())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -200,8 +200,8 @@ func TestRunPipelineExitsWhenContextCancelledMidPhase(t *testing.T) {
|
||||
cancel()
|
||||
}
|
||||
},
|
||||
ListSpider91Drives: func(context.Context) []string { return []string{"x"} },
|
||||
RunSpider91Crawl: func(context.Context, string) { rec.push("crawl") },
|
||||
ListCrawlerDrives: func(context.Context) []string { return []string{"x"} },
|
||||
RunCrawlerCrawl: func(context.Context, string) { rec.push("crawl") },
|
||||
WaitPreviewQueuesIdle: func(context.Context) error { rec.push("wait-idle"); return nil },
|
||||
RunMigration: func(context.Context) error { rec.push("migrate"); return nil },
|
||||
RunDedupeAssetCleanup: func(context.Context) error { rec.push("dedupe-cleanup"); return nil },
|
||||
@@ -289,12 +289,12 @@ func TestCtxCancelPreventsLaterPhases(t *testing.T) {
|
||||
WaitPreviewQueuesIdle: func(ctx context.Context) error {
|
||||
return ctx.Err()
|
||||
},
|
||||
ListSpider91Drives: func(context.Context) []string {
|
||||
rec.push("list-spider")
|
||||
ListCrawlerDrives: func(context.Context) []string {
|
||||
rec.push("list-crawler")
|
||||
return []string{"x"}
|
||||
},
|
||||
RunSpider91Crawl: func(context.Context, string) { rec.push("crawl") },
|
||||
RunMigration: func(context.Context) error { rec.push("migrate"); return nil },
|
||||
RunCrawlerCrawl: func(context.Context, string) { rec.push("crawl") },
|
||||
RunMigration: func(context.Context) error { rec.push("migrate"); return nil },
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
@@ -303,7 +303,7 @@ func TestCtxCancelPreventsLaterPhases(t *testing.T) {
|
||||
r.runPipeline(ctx)
|
||||
|
||||
for _, c := range rec.snapshot() {
|
||||
if c == "crawl" || c == "migrate" || c == "list-spider" {
|
||||
if c == "crawl" || c == "migrate" || c == "list-crawler" {
|
||||
t.Fatalf("later phase should not run after ctx done; got %q", c)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1589,11 +1589,6 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) bool {
|
||||
return false
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "pending"})
|
||||
if isSpider91OriginVideo(v) {
|
||||
log.Printf("[thumb] skip %s: spider91-origin video must use crawled thumbnail", v.Title)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
|
||||
return false
|
||||
}
|
||||
link, err := w.streamLink(ctx, v)
|
||||
if err != nil {
|
||||
if w.pauseForRecoverableError(ctx, v, err, "streamURL") {
|
||||
@@ -1675,10 +1670,6 @@ func (w *ThumbWorker) generateThumbnailFromLink(ctx context.Context, v *catalog.
|
||||
return nil
|
||||
}
|
||||
|
||||
func isSpider91OriginVideo(v *catalog.Video) bool {
|
||||
return v != nil && strings.HasPrefix(v.ID, "spider91-")
|
||||
}
|
||||
|
||||
func localPreviewLink(v *catalog.Video) (*drives.StreamLink, bool) {
|
||||
if v.PreviewLocal == "" {
|
||||
return nil, false
|
||||
|
||||
@@ -89,9 +89,9 @@ func TestThumbWorkerBackfillsDurationWhenThumbnailAlreadyExists(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerDoesNotGenerateThumbnailForSpider91OriginVideo(t *testing.T) {
|
||||
func TestThumbWorkerGeneratesThumbnailForCrawlerLikeVideoID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "spider91-91-spider-1200001")
|
||||
cat, video := seedPreviewTestVideo(t, "scriptcrawler-crawler-main-source001")
|
||||
|
||||
gen := &fakeThumbGenerator{probeDuration: 42}
|
||||
drv := &previewFakeDrive{kind: "pikpak"}
|
||||
@@ -103,18 +103,18 @@ func TestThumbWorkerDoesNotGenerateThumbnailForSpider91OriginVideo(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want empty when crawled spider91 thumbnail is missing", got.ThumbnailURL)
|
||||
if got.ThumbnailURL != "/p/thumb/"+video.ID {
|
||||
t.Fatalf("thumbnail = %q, want generated thumb URL", got.ThumbnailURL)
|
||||
}
|
||||
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
|
||||
ready, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "ready", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list failed thumbnails: %v", err)
|
||||
t.Fatalf("list ready thumbnails: %v", err)
|
||||
}
|
||||
if len(failed) != 1 || failed[0].ID != video.ID {
|
||||
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
|
||||
if len(ready) != 1 || ready[0].ID != video.ID {
|
||||
t.Fatalf("ready thumbnails = %#v, want only %s", ready, video.ID)
|
||||
}
|
||||
if gen.probeCalls != 0 || gen.generateCalls != 0 {
|
||||
t.Fatalf("generator calls probe=%d generate=%d, want no ffmpeg work for spider91-origin thumbnail", gen.probeCalls, gen.generateCalls)
|
||||
if gen.probeCalls != 1 || gen.generateCalls != 1 {
|
||||
t.Fatalf("generator calls probe=%d generate=%d, want one thumbnail generation", gen.probeCalls, gen.generateCalls)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -189,12 +189,6 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if label, ok, err := s.Catalog.EnsureCollectionTag(ctx, dirName); err == nil && ok {
|
||||
tags = mergeTags(tags, []string{label})
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
existing, _ := s.Catalog.GetVideo(ctx, id)
|
||||
if err := ctx.Err(); err != nil {
|
||||
@@ -206,15 +200,15 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
patch.ContentHash = e.Hash
|
||||
existing.ContentHash = e.Hash
|
||||
}
|
||||
if e.Name != "" && existing.FileName == "" {
|
||||
if e.Name != "" && existing.FileName != e.Name {
|
||||
patch.FileName = e.Name
|
||||
existing.FileName = e.Name
|
||||
patch.Title = parsed.Title
|
||||
patch.TitleSet = true
|
||||
patch.Author = parsed.Author
|
||||
patch.AuthorSet = true
|
||||
}
|
||||
// 已存在但轻量元数据空缺时,顺便补齐。
|
||||
if existing.Category == "" && dirName != "" {
|
||||
patch.Category = dirName
|
||||
}
|
||||
if patch.Category != "" || patch.ContentHash != "" || patch.FileName != "" {
|
||||
if patch.ContentHash != "" || patch.FileName != "" || patch.TitleSet || patch.AuthorSet {
|
||||
_ = s.Catalog.UpdateVideoMeta(ctx, id, patch)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
@@ -257,7 +251,6 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
Quality: "HD",
|
||||
Size: e.Size,
|
||||
PreviewStatus: "pending",
|
||||
Category: dirName,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
|
||||
@@ -323,6 +323,67 @@ func TestRunDoesNotBackfillRemoteThumbnailForExistingVideo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSyncsRenamedExistingVideoMetadata(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "fake-drive-file-1",
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
FileName: "old-name - Old Author.mp4",
|
||||
Title: "old-name",
|
||||
Author: "Old Author",
|
||||
PreviewStatus: "pending",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
drv := &scannerFakeDrive{
|
||||
entries: []drives.Entry{{
|
||||
ID: "file-1",
|
||||
Name: "[4K] renamed clip.mp4",
|
||||
Size: 123,
|
||||
ModTime: now,
|
||||
}},
|
||||
}
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
|
||||
stats, err := sc.Run(ctx, "")
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Added != 0 {
|
||||
t.Fatalf("added = %d, want existing video to be updated in place", stats.Added)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(ctx, "fake-drive-file-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.FileName != "[4K] renamed clip.mp4" {
|
||||
t.Fatalf("file_name = %q, want remote name", got.FileName)
|
||||
}
|
||||
if got.Title != "renamed clip" {
|
||||
t.Fatalf("title = %q, want parsed title from remote name", got.Title)
|
||||
}
|
||||
if got.Author != "" {
|
||||
t.Fatalf("author = %q, want cleared author from remote name without author suffix", got.Author)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReplacesExistingVideoTagsWithFixedFilenameTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -374,7 +435,7 @@ func TestRunReplacesExistingVideoTagsWithFixedFilenameTags(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunAddsShortCollectionDirectoryAsTag(t *testing.T) {
|
||||
func TestRunDoesNotCreateTagFromDirectoryName(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -392,7 +453,6 @@ func TestRunAddsShortCollectionDirectoryAsTag(t *testing.T) {
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: "Existing",
|
||||
Category: "sunny",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
@@ -423,84 +483,6 @@ func TestRunAddsShortCollectionDirectoryAsTag(t *testing.T) {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(ctx, "fake-drive-file-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if !sameStrings(got.Tags, []string{"sunny"}) {
|
||||
t.Fatalf("tags = %#v, want sunny", got.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDoesNotRecreateDeletedCollectionDirectoryTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
now := time.Now()
|
||||
for _, id := range []string{"existing-1", "existing-2"} {
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: "Existing",
|
||||
Category: "sunny",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed existing sunny video: %v", err)
|
||||
}
|
||||
}
|
||||
if label, ok, err := cat.EnsureCollectionTag(ctx, "sunny"); err != nil || !ok || label != "sunny" {
|
||||
t.Fatalf("ensure collection = %q, %v, %v; want sunny true nil", label, ok, err)
|
||||
}
|
||||
tags, err := cat.ListTags(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list tags: %v", err)
|
||||
}
|
||||
var tagID int64
|
||||
for _, tag := range tags {
|
||||
if tag.Label == "sunny" {
|
||||
tagID = tag.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
if tagID == 0 {
|
||||
t.Fatal("sunny tag not found before delete")
|
||||
}
|
||||
if _, err := cat.DeleteTag(ctx, tagID); err != nil {
|
||||
t.Fatalf("delete tag: %v", err)
|
||||
}
|
||||
|
||||
drv := &scannerTreeFakeDrive{
|
||||
entries: map[string][]drives.Entry{
|
||||
"root": {{
|
||||
ID: "dir-1",
|
||||
Name: "sunny",
|
||||
IsDir: true,
|
||||
}},
|
||||
"dir-1": {{
|
||||
ID: "file-1",
|
||||
ParentID: "dir-1",
|
||||
Name: "clip.mp4",
|
||||
Size: 123,
|
||||
ModTime: now,
|
||||
}},
|
||||
},
|
||||
}
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
|
||||
if _, err := sc.Run(ctx, ""); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(ctx, "fake-drive-file-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
@@ -508,15 +490,6 @@ func TestRunDoesNotRecreateDeletedCollectionDirectoryTag(t *testing.T) {
|
||||
if len(got.Tags) != 0 {
|
||||
t.Fatalf("tags = %#v, want none", got.Tags)
|
||||
}
|
||||
tags, err = cat.ListTags(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list tags after scan: %v", err)
|
||||
}
|
||||
for _, tag := range tags {
|
||||
if tag.Label == "sunny" {
|
||||
t.Fatal("deleted collection tag was recreated during scan")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMapsAVCodeDirectoryToAVTag(t *testing.T) {
|
||||
@@ -537,7 +510,6 @@ func TestRunMapsAVCodeDirectoryToAVTag(t *testing.T) {
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: "Existing",
|
||||
Category: "cc-1750027",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
-283
@@ -1,283 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
光鸭网盘 - 扫码登录脚本
|
||||
========================
|
||||
1. 调用 API 获取登录二维码
|
||||
2. 保存二维码图片,等待用户扫描
|
||||
3. 扫描成功后保存用户凭证信息
|
||||
"""
|
||||
|
||||
import io
|
||||
import sys
|
||||
|
||||
# 修复 Windows 终端 GBK 编码问题
|
||||
if sys.platform == 'win32':
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace')
|
||||
|
||||
import requests
|
||||
import json
|
||||
import time
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
# ========== 配置 ==========
|
||||
API_ORIGIN = "https://account.guangyapan.com"
|
||||
CLIENT_ID = "aMe-8VSlkrbQXpUR"
|
||||
SCOPE = "user"
|
||||
QR_IMAGE_PATH = "login_qr.png"
|
||||
CREDENTIALS_PATH = "credentials.json"
|
||||
|
||||
# ========== 可选依赖 ==========
|
||||
try:
|
||||
import qrcode
|
||||
HAS_QRCODE = True
|
||||
except ImportError:
|
||||
HAS_QRCODE = False
|
||||
|
||||
try:
|
||||
from PIL import Image
|
||||
HAS_PIL = True
|
||||
except ImportError:
|
||||
HAS_PIL = False
|
||||
|
||||
|
||||
def generate_qr_image(url: str, path: str):
|
||||
"""生成二维码图片"""
|
||||
if HAS_QRCODE:
|
||||
qr = qrcode.QRCode(
|
||||
version=1,
|
||||
error_correction=qrcode.constants.ERROR_CORRECT_M,
|
||||
box_size=10,
|
||||
border=4,
|
||||
)
|
||||
qr.add_data(url)
|
||||
qr.make(fit=True)
|
||||
img = qr.make_image(fill_color="black", back_color="white")
|
||||
img.save(path)
|
||||
print(f"[✓] 二维码已保存到: {path}")
|
||||
else:
|
||||
# Fallback: 使用 qrencode 命令行工具
|
||||
import subprocess
|
||||
try:
|
||||
subprocess.run(["qrencode", "-o", path, url], check=True)
|
||||
print(f"[✓] 二维码已保存到: {path}")
|
||||
except FileNotFoundError:
|
||||
print("[✗] 需要安装 qrcode 库: pip install qrcode[pil]")
|
||||
print(f"[!] 请手动访问以下链接扫码:")
|
||||
print(f" {url}")
|
||||
return
|
||||
|
||||
# 尝试直接显示二维码到终端
|
||||
try:
|
||||
if HAS_PIL:
|
||||
img = Image.open(path)
|
||||
img.show()
|
||||
print("[✓] 二维码已在图片查看器中打开")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 终端内显示小二维码
|
||||
if HAS_QRCODE:
|
||||
try:
|
||||
qr.print_ascii(invert=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
session = requests.Session()
|
||||
session.headers.update({
|
||||
"User-Agent": "GuangYaPan-Login/1.0",
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
})
|
||||
|
||||
# ====== Step 1: 获取设备码和二维码链接 ======
|
||||
print("=" * 60)
|
||||
print("Step 1: 请求登录二维码...")
|
||||
print("=" * 60)
|
||||
|
||||
device_code_url = f"{API_ORIGIN}/v1/auth/device/code"
|
||||
device_payload = {
|
||||
"client_id": CLIENT_ID,
|
||||
"scope": SCOPE,
|
||||
}
|
||||
|
||||
try:
|
||||
resp = session.post(device_code_url, json=device_payload, timeout=30)
|
||||
resp.raise_for_status()
|
||||
device_data = resp.json()
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"[✗] 请求失败: {e}")
|
||||
if hasattr(e, 'response') and e.response is not None:
|
||||
print(f" 响应内容: {e.response.text[:500]}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"[✓] 设备码获取成功")
|
||||
print(f" device_code: {device_data.get('device_code', 'N/A')[:30]}...")
|
||||
print(f" interval: {device_data.get('interval', 5)} 秒")
|
||||
print(f" expires_in: {device_data.get('expires_in', 'N/A')} 秒")
|
||||
|
||||
device_code = device_data["device_code"]
|
||||
interval = int(device_data.get("interval", 5))
|
||||
expires_in = int(device_data.get("expires_in", 300))
|
||||
|
||||
# 二维码链接
|
||||
qr_url = device_data.get("verification_uri_complete") or device_data.get("short_uri_complete")
|
||||
if not qr_url:
|
||||
print("[✗] 响应中没有找到二维码链接")
|
||||
print(f" 完整响应: {json.dumps(device_data, indent=2, ensure_ascii=False)}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f" qr_url: {qr_url}")
|
||||
print()
|
||||
|
||||
# ====== Step 2: 生成并保存二维码 ======
|
||||
print("=" * 60)
|
||||
print("Step 2: 生成二维码图片...")
|
||||
print("=" * 60)
|
||||
|
||||
generate_qr_image(qr_url, QR_IMAGE_PATH)
|
||||
|
||||
print()
|
||||
print("!" * 60)
|
||||
print("! 请使用「光鸭APP」扫描二维码登录")
|
||||
print("!" * 60)
|
||||
print()
|
||||
|
||||
# ====== Step 3: 轮询等待用户扫描 ======
|
||||
print("=" * 60)
|
||||
print("Step 3: 等待扫码授权...")
|
||||
print("=" * 60)
|
||||
|
||||
token_url = f"{API_ORIGIN}/v1/auth/token"
|
||||
token_payload = {
|
||||
"client_id": CLIENT_ID,
|
||||
"grant_type": "urn:ietf:params:oauth:grant-type:device_code",
|
||||
"device_code": device_code,
|
||||
}
|
||||
|
||||
start_time = time.time()
|
||||
attempt = 0
|
||||
|
||||
while True:
|
||||
attempt += 1
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
# 检查是否超时
|
||||
if elapsed > expires_in:
|
||||
print(f"\n[✗] 二维码已过期({expires_in}秒),请重新运行脚本")
|
||||
sys.exit(1)
|
||||
|
||||
time.sleep(interval)
|
||||
|
||||
try:
|
||||
resp = session.post(token_url, json=token_payload, timeout=30)
|
||||
token_data = resp.json()
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"\n[!] 网络错误: {e},重试中...")
|
||||
continue
|
||||
|
||||
if "error" in token_data:
|
||||
error = token_data["error"]
|
||||
if error in ("authorization_pending", "slow_down"):
|
||||
# 用户还未扫描或确认
|
||||
dots = "." * ((attempt % 10) + 1)
|
||||
print(f"\r 等待中{dots:<10} ({int(elapsed)}s / {expires_in}s)", end="", flush=True)
|
||||
|
||||
if error == "slow_down":
|
||||
interval = min(interval * 2, 60)
|
||||
continue
|
||||
|
||||
elif error == "expired_token":
|
||||
print(f"\n[✗] 二维码已过期,请重新运行脚本")
|
||||
sys.exit(1)
|
||||
|
||||
elif error == "access_denied":
|
||||
print(f"\n[✗] 用户拒绝了授权")
|
||||
sys.exit(1)
|
||||
|
||||
else:
|
||||
print(f"\n[✗] 未知错误: {error}")
|
||||
print(f" 完整响应: {json.dumps(token_data, indent=2, ensure_ascii=False)}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
# 成功!
|
||||
print(f"\n[✓] 扫码授权成功!({int(elapsed)}s)")
|
||||
break
|
||||
|
||||
# ====== Step 4: 保存凭证 ======
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("Step 4: 保存用户凭证...")
|
||||
print("=" * 60)
|
||||
|
||||
# 保存完整 token 响应
|
||||
credentials = {
|
||||
"saved_at": datetime.now().isoformat(),
|
||||
"api_origin": API_ORIGIN,
|
||||
"client_id": CLIENT_ID,
|
||||
"token_response": token_data,
|
||||
"cookies": dict(session.cookies),
|
||||
}
|
||||
|
||||
with open(CREDENTIALS_PATH, "w", encoding="utf-8") as f:
|
||||
json.dump(credentials, f, indent=2, ensure_ascii=False)
|
||||
print(f"[✓] 完整凭证已保存到: {CREDENTIALS_PATH}")
|
||||
|
||||
# 提取关键信息
|
||||
access_token = token_data.get("access_token", "")
|
||||
refresh_token = token_data.get("refresh_token", "")
|
||||
id_token = token_data.get("id_token", "")
|
||||
token_type = token_data.get("token_type", "Bearer")
|
||||
expires_in = token_data.get("expires_in", 0)
|
||||
|
||||
print()
|
||||
print("-" * 60)
|
||||
print("凭证摘要:")
|
||||
print("-" * 60)
|
||||
print(f" access_token: {access_token[:50]}..." if access_token else " access_token: (无)")
|
||||
print(f" refresh_token: {refresh_token[:50]}..." if refresh_token else " refresh_token: (无)")
|
||||
print(f" id_token: {id_token[:50]}..." if id_token else " id_token: (无)")
|
||||
print(f" token_type: {token_type}")
|
||||
print(f" expires_in: {expires_in} 秒")
|
||||
print(f" scope: {token_data.get('scope', SCOPE)}")
|
||||
print("-" * 60)
|
||||
|
||||
# 尝试获取用户信息
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("Step 5: 获取用户信息...")
|
||||
print("=" * 60)
|
||||
|
||||
user_info_url = f"{API_ORIGIN}/v1/user/me"
|
||||
try:
|
||||
user_headers = {
|
||||
"Authorization": f"{token_type} {access_token}",
|
||||
}
|
||||
user_resp = requests.get(user_info_url, headers=user_headers, timeout=15)
|
||||
if user_resp.status_code == 200:
|
||||
user_data = user_resp.json()
|
||||
print("[✓] 用户信息获取成功:")
|
||||
print(json.dumps(user_data, indent=2, ensure_ascii=False))
|
||||
|
||||
# 追加用户信息到凭证文件
|
||||
credentials["user_info"] = user_data
|
||||
with open(CREDENTIALS_PATH, "w", encoding="utf-8") as f:
|
||||
json.dump(credentials, f, indent=2, ensure_ascii=False)
|
||||
else:
|
||||
print(f"[!] 获取用户信息返回 {user_resp.status_code}: {user_resp.text[:200]}")
|
||||
except Exception as e:
|
||||
print(f"[!] 获取用户信息失败: {e}")
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("完成!凭证文件: " + CREDENTIALS_PATH)
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+1
-1
@@ -4,7 +4,7 @@
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="referrer" content="no-referrer" />
|
||||
<link rel="icon" type="image/png" href="/icon.png" />
|
||||
<link rel="apple-touch-icon" href="/icon.png" />
|
||||
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png" />
|
||||
<link rel="manifest" href="/manifest.webmanifest" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
|
||||
<meta name="mobile-web-app-capable" content="yes" />
|
||||
|
||||
+2
-7
@@ -194,7 +194,7 @@ backup_install_files() {
|
||||
local backup="$1"
|
||||
mkdir -p "$backup"
|
||||
cp -a "$INSTALL_PATH/server" "$backup/server"
|
||||
for item in dist config.example.yaml 91VideoSpider config.yaml .version; do
|
||||
for item in dist config.example.yaml config.yaml .version; do
|
||||
if [[ -e "$INSTALL_PATH/$item" ]]; then
|
||||
cp -a "$INSTALL_PATH/$item" "$backup/$item"
|
||||
fi
|
||||
@@ -205,7 +205,7 @@ restore_install_files() {
|
||||
local backup="$1"
|
||||
mkdir -p "$INSTALL_PATH"
|
||||
cp -a "$backup/server" "$INSTALL_PATH/server"
|
||||
for item in dist config.example.yaml 91VideoSpider config.yaml .version; do
|
||||
for item in dist config.example.yaml config.yaml .version; do
|
||||
rm -rf "${INSTALL_PATH:?}/$item"
|
||||
if [[ -e "$backup/$item" ]]; then
|
||||
cp -a "$backup/$item" "$INSTALL_PATH/$item"
|
||||
@@ -441,7 +441,6 @@ process_looks_like_app() {
|
||||
[[ "$cmd" == *"VIDEO_FRONTEND_DIR=$INSTALL_PATH/dist"* ]] && return 0
|
||||
[[ "$cmd" == *"VIDEO_CONFIG=$INSTALL_PATH/config.yaml"* ]] && return 0
|
||||
[[ "$cmd" == *"video-site-91"* ]] && return 0
|
||||
[[ "$cmd" == *"91VideoSpider"* ]] && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -595,10 +594,6 @@ fetch_and_unpack() {
|
||||
rm -rf "$INSTALL_PATH/dist"
|
||||
cp -R "$root/dist" "$INSTALL_PATH/dist"
|
||||
cp "$root/config.example.yaml" "$INSTALL_PATH/config.example.yaml"
|
||||
if [[ -d "$root/91VideoSpider" ]]; then
|
||||
rm -rf "$INSTALL_PATH/91VideoSpider"
|
||||
cp -R "$root/91VideoSpider" "$INSTALL_PATH/91VideoSpider"
|
||||
fi
|
||||
chmod +x "$INSTALL_PATH/server"
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "video-site",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.2",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "video-site",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.2",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"artplayer": "^5.4.0",
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
"name": "video-site",
|
||||
"private": true,
|
||||
"license": "MIT",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.2",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 21 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 136 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 114 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 19 KiB |
@@ -1,28 +0,0 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32">
|
||||
<defs>
|
||||
<!-- Background Gradient: Warm Orange to Sakura Pink, representing both themes -->
|
||||
<linearGradient id="bg-grad" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||
<stop offset="0%" stop-color="#FF7E40" />
|
||||
<stop offset="100%" stop-color="#FF4B91" />
|
||||
</linearGradient>
|
||||
|
||||
<!-- Subtle drop shadow for the play button to give it depth -->
|
||||
<filter id="shadow" x="-20%" y="-20%" width="140%" height="140%">
|
||||
<feDropShadow dx="0" dy="1.5" stdDeviation="1" flood-opacity="0.25" />
|
||||
</filter>
|
||||
</defs>
|
||||
|
||||
<!-- Main Squircle Background -->
|
||||
<rect x="2" y="2" width="28" height="28" rx="8" fill="url(#bg-grad)" />
|
||||
|
||||
<!-- Inner border for a premium, glassmorphic feel -->
|
||||
<rect x="3" y="3" width="26" height="26" rx="7" fill="none" stroke="#ffffff" stroke-width="1" opacity="0.2" />
|
||||
|
||||
<!-- Stylized Play Button Icon, perfectly centered with rounded corners and drop shadow -->
|
||||
<path d="M13 10.5 L21.5 16 L13 21.5 Z"
|
||||
fill="#ffffff"
|
||||
stroke="#ffffff"
|
||||
stroke-width="2.5"
|
||||
stroke-linejoin="round"
|
||||
filter="url(#shadow)" />
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 1.1 KiB |
@@ -9,10 +9,22 @@
|
||||
"theme_color": "#000000",
|
||||
"icons": [
|
||||
{
|
||||
"src": "/icon.png",
|
||||
"src": "/app-icon-192.png",
|
||||
"sizes": "192x192",
|
||||
"type": "image/png",
|
||||
"purpose": "any"
|
||||
},
|
||||
{
|
||||
"src": "/app-icon-512.png",
|
||||
"sizes": "512x512",
|
||||
"type": "image/png",
|
||||
"purpose": "any maskable"
|
||||
"purpose": "any"
|
||||
},
|
||||
{
|
||||
"src": "/app-icon-maskable-512.png",
|
||||
"sizes": "512x512",
|
||||
"type": "image/png",
|
||||
"purpose": "maskable"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
+108
-23
@@ -1,33 +1,75 @@
|
||||
import { Navigate, Route, Routes } from "react-router-dom";
|
||||
import { Suspense, lazy, useEffect, type ReactNode } from "react";
|
||||
import { Navigate, Route, Routes, useLocation } from "react-router-dom";
|
||||
import { SkyStarfield } from "@/components/SkyStarfield";
|
||||
import HomePage from "@/pages/HomePage";
|
||||
import ListingPage from "@/pages/ListingPage";
|
||||
import ShortsPage from "@/pages/ShortsPage";
|
||||
import UploadPage from "@/pages/UploadPage";
|
||||
import VideoDetailPage from "@/pages/VideoDetailPage";
|
||||
import { AdminLayout } from "@/admin/AdminLayout";
|
||||
import { LoginPage } from "@/admin/LoginPage";
|
||||
import { RequireAuth } from "@/admin/RequireAuth";
|
||||
import { DrivesPage } from "@/admin/DrivesPage";
|
||||
import { CrawlersPage } from "@/admin/CrawlersPage";
|
||||
import { VideosPage } from "@/admin/VideosPage";
|
||||
import { TagsPage } from "@/admin/TagsPage";
|
||||
import { ThemePage } from "@/admin/ThemePage";
|
||||
import { rememberVideoReturnPath, routeToPath } from "@/lib/videoReturnPath";
|
||||
|
||||
const HomePage = lazy(() => import("@/pages/HomePage"));
|
||||
const ListingPage = lazy(() => import("@/pages/ListingPage"));
|
||||
const ShortsPage = lazy(() => import("@/pages/ShortsPage"));
|
||||
const UploadPage = lazy(() => import("@/pages/UploadPage"));
|
||||
const VideoDetailPage = lazy(() => import("@/pages/VideoDetailPage"));
|
||||
|
||||
const LoginPage = lazy(() =>
|
||||
import("@/admin/LoginPage").then((module) => ({ default: module.LoginPage }))
|
||||
);
|
||||
const DrivesPage = lazy(() =>
|
||||
import("@/admin/DrivesPage").then((module) => ({ default: module.DrivesPage }))
|
||||
);
|
||||
const CrawlersPage = lazy(() =>
|
||||
import("@/admin/CrawlersPage").then((module) => ({
|
||||
default: module.CrawlersPage,
|
||||
}))
|
||||
);
|
||||
const VideosPage = lazy(() =>
|
||||
import("@/admin/VideosPage").then((module) => ({ default: module.VideosPage }))
|
||||
);
|
||||
const TagsPage = lazy(() =>
|
||||
import("@/admin/TagsPage").then((module) => ({ default: module.TagsPage }))
|
||||
);
|
||||
const ThemePage = lazy(() =>
|
||||
import("@/admin/ThemePage").then((module) => ({ default: module.ThemePage }))
|
||||
);
|
||||
|
||||
function PageSuspense({ children }: { children: ReactNode }) {
|
||||
return <Suspense fallback={null}>{children}</Suspense>;
|
||||
}
|
||||
|
||||
function VideoReturnPathRecorder() {
|
||||
const location = useLocation();
|
||||
|
||||
useEffect(() => {
|
||||
rememberVideoReturnPath(routeToPath(location));
|
||||
}, [location.pathname, location.search, location.hash]);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export default function App() {
|
||||
return (
|
||||
<>
|
||||
{/* 星空蓝主题的固定位置星星层,仅在 data-theme="sky" 下可见 */}
|
||||
<SkyStarfield />
|
||||
<VideoReturnPathRecorder />
|
||||
<Routes>
|
||||
<Route path="/login" element={<LoginPage />} />
|
||||
<Route
|
||||
path="/login"
|
||||
element={
|
||||
<PageSuspense>
|
||||
<LoginPage />
|
||||
</PageSuspense>
|
||||
}
|
||||
/>
|
||||
|
||||
{/* 主站需要登录 */}
|
||||
<Route
|
||||
path="/"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<HomePage />
|
||||
<PageSuspense>
|
||||
<HomePage />
|
||||
</PageSuspense>
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
@@ -35,7 +77,9 @@ export default function App() {
|
||||
path="/list"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<ListingPage />
|
||||
<PageSuspense>
|
||||
<ListingPage />
|
||||
</PageSuspense>
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
@@ -43,7 +87,9 @@ export default function App() {
|
||||
path="/shorts"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<ShortsPage />
|
||||
<PageSuspense>
|
||||
<ShortsPage />
|
||||
</PageSuspense>
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
@@ -51,7 +97,9 @@ export default function App() {
|
||||
path="/upload"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<UploadPage />
|
||||
<PageSuspense>
|
||||
<UploadPage />
|
||||
</PageSuspense>
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
@@ -59,7 +107,9 @@ export default function App() {
|
||||
path="/video/:id"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<VideoDetailPage />
|
||||
<PageSuspense>
|
||||
<VideoDetailPage />
|
||||
</PageSuspense>
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
@@ -74,11 +124,46 @@ export default function App() {
|
||||
}
|
||||
>
|
||||
<Route index element={<Navigate to="/admin/drives" replace />} />
|
||||
<Route path="drives" element={<DrivesPage />} />
|
||||
<Route path="crawlers" element={<CrawlersPage />} />
|
||||
<Route path="videos" element={<VideosPage />} />
|
||||
<Route path="tags" element={<TagsPage />} />
|
||||
<Route path="theme" element={<ThemePage />} />
|
||||
<Route
|
||||
path="drives"
|
||||
element={
|
||||
<PageSuspense>
|
||||
<DrivesPage />
|
||||
</PageSuspense>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="crawlers"
|
||||
element={
|
||||
<PageSuspense>
|
||||
<CrawlersPage />
|
||||
</PageSuspense>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="videos"
|
||||
element={
|
||||
<PageSuspense>
|
||||
<VideosPage />
|
||||
</PageSuspense>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="tags"
|
||||
element={
|
||||
<PageSuspense>
|
||||
<TagsPage />
|
||||
</PageSuspense>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="theme"
|
||||
element={
|
||||
<PageSuspense>
|
||||
<ThemePage />
|
||||
</PageSuspense>
|
||||
}
|
||||
/>
|
||||
</Route>
|
||||
|
||||
<Route path="*" element={<Navigate to="/" replace />} />
|
||||
|
||||
+36
-52
@@ -30,7 +30,7 @@ import { Modal } from "./Modal";
|
||||
import { ConfirmModal } from "./ConfirmModal";
|
||||
import { useToast } from "./ToastContext";
|
||||
import { generationStateClass, generationStateLabel } from "./drive/constants";
|
||||
import { Spider91UploadTargetField } from "./drive/Spider91UploadTargetField";
|
||||
import { CrawlerUploadTargetField } from "./drive/CrawlerUploadTargetField";
|
||||
import { SpiderIcon } from "./icons/SpiderIcon";
|
||||
|
||||
const BUSY_STATES = new Set(["scanning", "generating", "uploading", "queued"]);
|
||||
@@ -57,6 +57,7 @@ export function CrawlersPage() {
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [expandedId, setExpandedId] = useState("");
|
||||
const [runningId, setRunningId] = useState("");
|
||||
const [uploadingId, setUploadingId] = useState("");
|
||||
const [stoppingId, setStoppingId] = useState("");
|
||||
const [togglingTeaserId, setTogglingTeaserId] = useState("");
|
||||
// undefined = 编辑器关闭;null = 新建;其余 = 编辑已有爬虫
|
||||
@@ -126,6 +127,23 @@ export function CrawlersPage() {
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadVideos(crawler: api.AdminCrawler) {
|
||||
setUploadingId(crawler.id);
|
||||
try {
|
||||
const resp = await api.uploadCrawlerVideos(crawler.id);
|
||||
if (!resp.accepted) {
|
||||
show(resp.message || "当前爬虫暂不满足上传条件", "info");
|
||||
return;
|
||||
}
|
||||
show("已触发上传任务", "success");
|
||||
await refresh(true);
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "触发上传失败", "error");
|
||||
} finally {
|
||||
setUploadingId("");
|
||||
}
|
||||
}
|
||||
|
||||
async function stop(crawler: api.AdminCrawler) {
|
||||
setStoppingId(crawler.id);
|
||||
try {
|
||||
@@ -233,10 +251,12 @@ export function CrawlersPage() {
|
||||
crawler={crawler}
|
||||
expanded={expandedId === crawler.id}
|
||||
running={runningId === crawler.id}
|
||||
uploading={uploadingId === crawler.id}
|
||||
stopping={stoppingId === crawler.id}
|
||||
togglingTeaser={togglingTeaserId === crawler.id}
|
||||
onToggle={() => setExpandedId(expandedId === crawler.id ? "" : crawler.id)}
|
||||
onRun={() => run(crawler)}
|
||||
onUpload={() => uploadVideos(crawler)}
|
||||
onStop={() => stop(crawler)}
|
||||
onToggleTeaser={() => toggleTeaser(crawler)}
|
||||
onEdit={() => setEditorTarget(crawler)}
|
||||
@@ -284,37 +304,16 @@ function CrawlerMetric({ label, value, icon, tone }: { label: string; value: num
|
||||
);
|
||||
}
|
||||
|
||||
type StageInfo = {
|
||||
key: string;
|
||||
label: string;
|
||||
status?: api.DriveGenerationStatus;
|
||||
};
|
||||
|
||||
function crawlerStages(crawler: api.AdminCrawler): StageInfo[] {
|
||||
return [
|
||||
{ key: "scan", label: "抓取", status: crawler.scanGenerationStatus },
|
||||
{ key: "thumbnail", label: "封面", status: crawler.thumbnailGenerationStatus },
|
||||
{ key: "preview", label: "预览", status: crawler.previewGenerationStatus },
|
||||
{ key: "fingerprint", label: "指纹", status: crawler.fingerprintGenerationStatus },
|
||||
{ key: "upload", label: "上传", status: crawler.uploadGenerationStatus },
|
||||
];
|
||||
}
|
||||
|
||||
function stageStateLabel(stage: StageInfo): string {
|
||||
const state = stage.status?.state || "idle";
|
||||
if (stage.key === "scan" && state === "scanning") return "抓取中";
|
||||
if (stage.key === "upload" && state === "uploading") return "上传中";
|
||||
return generationStateLabel(state);
|
||||
}
|
||||
|
||||
function CrawlerRow({
|
||||
crawler,
|
||||
expanded,
|
||||
running,
|
||||
uploading,
|
||||
stopping,
|
||||
togglingTeaser,
|
||||
onToggle,
|
||||
onRun,
|
||||
onUpload,
|
||||
onStop,
|
||||
onToggleTeaser,
|
||||
onEdit,
|
||||
@@ -323,16 +322,19 @@ function CrawlerRow({
|
||||
crawler: api.AdminCrawler;
|
||||
expanded: boolean;
|
||||
running: boolean;
|
||||
uploading: boolean;
|
||||
stopping: boolean;
|
||||
togglingTeaser: boolean;
|
||||
onToggle: () => void;
|
||||
onRun: () => void;
|
||||
onUpload: () => void;
|
||||
onStop: () => void;
|
||||
onToggleTeaser: () => void;
|
||||
onEdit: () => void;
|
||||
onDelete: () => void;
|
||||
}) {
|
||||
const busy = crawlerBusy(crawler);
|
||||
const uploadButtonTitle = uploading ? "上传请求处理中" : "上传本地爬虫视频到已配置的上传网盘";
|
||||
return (
|
||||
<div className={`admin-crawler-row ${expanded ? "is-expanded" : ""}`}>
|
||||
<div className="admin-crawler-row__line">
|
||||
@@ -346,31 +348,11 @@ function CrawlerRow({
|
||||
上次抓取 {formatLastCrawl(crawler.lastCrawlAt)} · 每次新增 {crawler.targetNew || "10"} 条 · 累计爬取 {crawler.totalCrawledCount ?? 0} 条
|
||||
</span>
|
||||
</span>
|
||||
<span className="admin-crawler-pipeline">
|
||||
{crawlerStages(crawler).map((stage) => {
|
||||
const state = stage.status?.state || "idle";
|
||||
const active = BUSY_STATES.has(state) || state === "cooling";
|
||||
return (
|
||||
<span
|
||||
key={stage.key}
|
||||
className={`admin-crawler-stage is-${generationStateClass(state)}`}
|
||||
title={`${stage.label}:${stageStateLabel(stage)}`}
|
||||
>
|
||||
<span className="admin-crawler-stage__dot" />
|
||||
{stage.label}
|
||||
{active && <em>{stageStateLabel(stage)}</em>}
|
||||
</span>
|
||||
);
|
||||
})}
|
||||
</span>
|
||||
<span className={`admin-status is-${crawler.status === "ok" ? "ok" : crawler.status === "error" ? "error" : "pending"}`}>
|
||||
{crawlerStatusLabel(crawler)}
|
||||
</span>
|
||||
<ChevronDown size={16} className="admin-crawler-row__chevron" />
|
||||
</button>
|
||||
<div className="admin-crawler-row__actions">
|
||||
<button
|
||||
className={`admin-btn admin-crawler-preview-card-toggle ${crawler.teaserEnabled ? "is-on" : ""}`}
|
||||
className="admin-btn admin-crawler-preview-card-toggle"
|
||||
type="button"
|
||||
onClick={onToggleTeaser}
|
||||
disabled={togglingTeaser}
|
||||
@@ -389,6 +371,14 @@ function CrawlerRow({
|
||||
<Download size={13} /> {running ? "触发中..." : "立即抓取"}
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
className="admin-btn"
|
||||
type="button"
|
||||
onClick={onUpload}
|
||||
title={uploadButtonTitle}
|
||||
>
|
||||
<Upload size={13} /> {uploading ? "上传中..." : "上传视频"}
|
||||
</button>
|
||||
<button className="admin-btn" type="button" onClick={onEdit}>
|
||||
<Pencil size={13} /> 编辑
|
||||
</button>
|
||||
@@ -980,7 +970,7 @@ function CrawlerEditorModal({
|
||||
placeholder="http://127.0.0.1:7890"
|
||||
/>
|
||||
</div>
|
||||
<Spider91UploadTargetField
|
||||
<CrawlerUploadTargetField
|
||||
value={form.uploadDriveId}
|
||||
onChange={(value) => set("uploadDriveId", value)}
|
||||
uploadTargets={uploadTargets}
|
||||
@@ -1075,12 +1065,6 @@ function crawlerTestFailure(result: api.CrawlerDryRunResult) {
|
||||
return result.error || result.mediaCheck?.error || "";
|
||||
}
|
||||
|
||||
function crawlerStatusLabel(crawler: api.AdminCrawler) {
|
||||
if (crawler.status === "ok") return "已就绪";
|
||||
if (crawler.status === "error") return "错误";
|
||||
return "未连接";
|
||||
}
|
||||
|
||||
function formatLastCrawl(ts?: number) {
|
||||
if (!ts) return "从未";
|
||||
return new Date(ts * 1000).toLocaleString("zh-CN", {
|
||||
|
||||
+30
-102
@@ -4,7 +4,6 @@ import {
|
||||
ArrowLeft,
|
||||
ChevronRight,
|
||||
CircleStop,
|
||||
Download,
|
||||
FolderTree,
|
||||
HardDrive,
|
||||
PlayCircle,
|
||||
@@ -58,7 +57,6 @@ function isDriveBusy(d: api.AdminDrive) {
|
||||
export function DrivesPage() {
|
||||
const [list, setList] = useState<api.AdminDrive[]>([]);
|
||||
const [storage, setStorage] = useState<api.AdminDriveStorage | null>(null);
|
||||
const [settings, setSettings] = useState<api.Settings | null>(null);
|
||||
const [nightlyStatus, setNightlyStatus] =
|
||||
useState<api.NightlyJobStatus>(idleNightlyStatus);
|
||||
const [loading, setLoading] = useState(true);
|
||||
@@ -91,22 +89,7 @@ export function DrivesPage() {
|
||||
const nameError = nameTouched && nameMissing ? "请填写网盘名称" : "";
|
||||
const formDirty = form.id
|
||||
? !sameForm(form, initialForm)
|
||||
: hasCreateFormChanges(form, initialForm);
|
||||
|
||||
const uploadTargets = useMemo(
|
||||
() =>
|
||||
list.filter(
|
||||
(d) =>
|
||||
d.kind === "pikpak" ||
|
||||
d.kind === "p115" ||
|
||||
d.kind === "p123" ||
|
||||
d.kind === "onedrive" ||
|
||||
d.kind === "googledrive" ||
|
||||
d.kind === "wopan" ||
|
||||
d.kind === "guangyapan"
|
||||
),
|
||||
[list]
|
||||
);
|
||||
: hasCreateFormChanges(form);
|
||||
|
||||
function openDriveDetail(id: string) {
|
||||
setSearchParams((prev) => {
|
||||
@@ -128,15 +111,13 @@ export function DrivesPage() {
|
||||
setLoading(true);
|
||||
setLoadError("");
|
||||
try {
|
||||
const [data, storageData, settingsData, jobStatus] = await Promise.all([
|
||||
const [data, storageData, jobStatus] = await Promise.all([
|
||||
api.listDrives(),
|
||||
api.getDriveStorage(),
|
||||
api.getSettings().catch(() => null),
|
||||
api.getNightlyJobStatus().catch(() => null),
|
||||
]);
|
||||
setList(data ?? []);
|
||||
setStorage(storageData);
|
||||
if (settingsData) setSettings(settingsData);
|
||||
if (jobStatus) setNightlyStatus(jobStatus);
|
||||
} catch (e) {
|
||||
const message = e instanceof Error ? e.message : "加载失败";
|
||||
@@ -197,10 +178,7 @@ export function DrivesPage() {
|
||||
}, [trackingNightly]);
|
||||
|
||||
function openCreate() {
|
||||
const nextForm = {
|
||||
...emptyForm,
|
||||
spider91UploadDriveId: settings?.spider91UploadDriveId ?? "",
|
||||
};
|
||||
const nextForm = { ...emptyForm };
|
||||
setForm(nextForm);
|
||||
setInitialForm(nextForm);
|
||||
setNameTouched(false);
|
||||
@@ -214,14 +192,14 @@ export function DrivesPage() {
|
||||
name: d.name,
|
||||
rootId: d.rootId,
|
||||
creds:
|
||||
d.kind === "spider91"
|
||||
? { proxy: d.spider91Proxy ?? "" }
|
||||
: d.kind === "googledrive"
|
||||
? { use_online_api: (d.googleDriveUseOnlineAPI ?? true) ? "true" : "false" }
|
||||
d.kind === "googledrive"
|
||||
? {
|
||||
use_online_api: (d.googleDriveUseOnlineAPI ?? true) ? "true" : "false",
|
||||
api_url_address: d.googleDriveOpenListApiUrl ?? "",
|
||||
}
|
||||
: d.kind === "localstorage"
|
||||
? { strm_allow_outside_root: (d.strmAllowOutsideRoot ?? false) ? "true" : "false" }
|
||||
: {},
|
||||
spider91UploadDriveId: settings?.spider91UploadDriveId ?? "",
|
||||
};
|
||||
setForm(nextForm);
|
||||
setInitialForm(nextForm);
|
||||
@@ -247,7 +225,7 @@ export function DrivesPage() {
|
||||
|
||||
function handleCreateFormChange(nextForm: FormState) {
|
||||
setForm(nextForm);
|
||||
if (!nextForm.id && !hasCreateFormChanges(nextForm, initialForm)) {
|
||||
if (!nextForm.id && !hasCreateFormChanges(nextForm)) {
|
||||
setInitialForm(nextForm);
|
||||
}
|
||||
}
|
||||
@@ -276,26 +254,6 @@ export function DrivesPage() {
|
||||
credentials: form.creds,
|
||||
});
|
||||
|
||||
if (form.kind === "spider91" && form.spider91UploadDriveId !== (settings?.spider91UploadDriveId ?? "")) {
|
||||
try {
|
||||
const updated = await api.updateSettings({
|
||||
spider91UploadDriveId: form.spider91UploadDriveId,
|
||||
});
|
||||
setSettings(updated);
|
||||
} catch (settingsErr) {
|
||||
show(
|
||||
settingsErr instanceof Error
|
||||
? `Drive 已保存,但上传目标设置失败:${settingsErr.message}`
|
||||
: "上传目标设置失败",
|
||||
"error"
|
||||
);
|
||||
setModalOpen(false);
|
||||
setInitialForm(form);
|
||||
refresh();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (resp.warning) {
|
||||
show(`已保存,但 driver 初始化失败:${resp.warning}`, "error");
|
||||
} else {
|
||||
@@ -331,10 +289,6 @@ export function DrivesPage() {
|
||||
}
|
||||
|
||||
async function handleRescan(d: api.AdminDrive) {
|
||||
if (d.kind === "spider91") {
|
||||
show("91Spider 不再支持通过网盘运行,请到爬虫管理添加爬虫脚本", "info");
|
||||
return;
|
||||
}
|
||||
if (nightlyBusy) {
|
||||
show(nightlyBusyText(nightlyStatus) || NIGHTLY_BUSY_MESSAGE, "info");
|
||||
return;
|
||||
@@ -564,7 +518,7 @@ export function DrivesPage() {
|
||||
</div>
|
||||
<div className="admin-drive-detail__header-right">
|
||||
<span className="admin-drive-detail__kind-chip">{kindLabel[d.kind] ?? d.kind}</span>
|
||||
<StatusTag kind={d.kind} status={d.status} error={d.lastError} hasCred={d.hasCredential} />
|
||||
<StatusTag status={d.status} error={d.lastError} hasCred={d.hasCredential} />
|
||||
</div>
|
||||
</header>
|
||||
|
||||
@@ -589,12 +543,6 @@ export function DrivesPage() {
|
||||
<span className="admin-detail-value admin-mono-cell">{d.rootId}</span>
|
||||
</div>
|
||||
)}
|
||||
{d.kind === "spider91" && (
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">配置状态</span>
|
||||
<span className="admin-detail-value">已废弃,请到爬虫管理添加</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{d.lastError && (
|
||||
<div className="admin-detail-error">{d.lastError}</div>
|
||||
@@ -606,29 +554,17 @@ export function DrivesPage() {
|
||||
type="button"
|
||||
className="admin-btn is-primary"
|
||||
onClick={() => handleRescan(d)}
|
||||
disabled={d.kind === "spider91"}
|
||||
aria-disabled={d.kind === "spider91" || nightlyBusy || isDriveBusy(d) || !!scanningDriveIds[d.id]}
|
||||
aria-disabled={nightlyBusy || isDriveBusy(d) || !!scanningDriveIds[d.id]}
|
||||
title={
|
||||
d.kind === "spider91"
|
||||
? "91Spider 不再支持通过网盘运行,请到爬虫管理添加爬虫脚本"
|
||||
: nightlyBusy
|
||||
nightlyBusy
|
||||
? nightlyBusyText(nightlyStatus) || NIGHTLY_BUSY_MESSAGE
|
||||
: isDriveBusy(d) || scanningDriveIds[d.id]
|
||||
? DRIVE_BUSY_MESSAGE
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
{d.kind === "spider91" ? (
|
||||
<>
|
||||
<Download size={13} className={scanningDriveIds[d.id] ? "admin-spin" : undefined} />
|
||||
已废弃
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<RefreshCw size={13} className={scanningDriveIds[d.id] ? "admin-spin" : undefined} />
|
||||
{scanningDriveIds[d.id] ? "触发中..." : "立即重扫"}
|
||||
</>
|
||||
)}
|
||||
<RefreshCw size={13} className={scanningDriveIds[d.id] ? "admin-spin" : undefined} />
|
||||
{scanningDriveIds[d.id] ? "触发中..." : "立即重扫"}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
@@ -641,30 +577,26 @@ export function DrivesPage() {
|
||||
{stoppingDriveId === d.id ? "停止中..." : "停止所有任务"}
|
||||
</button>
|
||||
</div>
|
||||
{d.kind !== "spider91" && (
|
||||
<button type="button" className="admin-btn is-primary" onClick={() => openEdit(d)}>
|
||||
编辑配置凭证
|
||||
</button>
|
||||
)}
|
||||
<button type="button" className="admin-btn is-primary" onClick={() => openEdit(d)}>
|
||||
编辑配置凭证
|
||||
</button>
|
||||
<button type="button" className="admin-btn is-danger admin-detail-actions__danger" onClick={() => setDeleteTarget(d)}>
|
||||
<Trash2 size={13} /> 删除网盘
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{d.kind !== "spider91" && (
|
||||
<SkipDirsPanel
|
||||
drive={d}
|
||||
onSaved={(saved) => {
|
||||
setList((prev) =>
|
||||
prev.map((item) =>
|
||||
item.id === saved.id ? { ...item, skipDirIds: saved.skipDirIds } : item
|
||||
)
|
||||
);
|
||||
refreshDriveList();
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
<SkipDirsPanel
|
||||
drive={d}
|
||||
onSaved={(saved) => {
|
||||
setList((prev) =>
|
||||
prev.map((item) =>
|
||||
item.id === saved.id ? { ...item, skipDirIds: saved.skipDirIds } : item
|
||||
)
|
||||
);
|
||||
refreshDriveList();
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
@@ -732,7 +664,6 @@ export function DrivesPage() {
|
||||
form={form}
|
||||
onChange={setForm}
|
||||
isEdit={true}
|
||||
uploadTargets={uploadTargets}
|
||||
nameError={nameError}
|
||||
onNameBlur={() => setNameTouched(true)}
|
||||
/>
|
||||
@@ -830,7 +761,7 @@ export function DrivesPage() {
|
||||
</span>
|
||||
<span>{d.name || d.id}</span>
|
||||
</div>
|
||||
<StatusTag kind={d.kind} status={d.status} error={d.lastError} hasCred={d.hasCredential} />
|
||||
<StatusTag status={d.status} error={d.lastError} hasCred={d.hasCredential} />
|
||||
</div>
|
||||
|
||||
<DriveCardMetrics d={d} />
|
||||
@@ -870,7 +801,6 @@ export function DrivesPage() {
|
||||
form={form}
|
||||
onChange={handleCreateFormChange}
|
||||
isEdit={!!list.find((x) => x.id === form.id)}
|
||||
uploadTargets={uploadTargets}
|
||||
nameError={nameError}
|
||||
onNameBlur={() => setNameTouched(true)}
|
||||
onBack={() => setNameTouched(false)}
|
||||
@@ -907,7 +837,6 @@ function sameForm(a: FormState, b: FormState): boolean {
|
||||
a.kind === b.kind &&
|
||||
a.name === b.name &&
|
||||
a.rootId === b.rootId &&
|
||||
a.spider91UploadDriveId === b.spider91UploadDriveId &&
|
||||
sameRecord(a.creds, b.creds)
|
||||
);
|
||||
}
|
||||
@@ -920,9 +849,8 @@ function sameRecord(a: Record<string, string>, b: Record<string, string>): boole
|
||||
return true;
|
||||
}
|
||||
|
||||
function hasCreateFormChanges(form: FormState, initial: FormState): boolean {
|
||||
function hasCreateFormChanges(form: FormState): boolean {
|
||||
if (form.name.trim() !== "") return true;
|
||||
if (form.rootId.trim() !== "") return true;
|
||||
if (form.spider91UploadDriveId !== initial.spider91UploadDriveId) return true;
|
||||
return Object.values(form.creds).some((value) => value.trim() !== "");
|
||||
}
|
||||
|
||||
+11
-7
@@ -1,16 +1,18 @@
|
||||
import { useEffect, useId, useRef, ReactNode } from "react";
|
||||
import { createPortal } from "react-dom";
|
||||
import { X } from "lucide-react";
|
||||
|
||||
type Props = {
|
||||
open: boolean;
|
||||
title: string;
|
||||
title?: string;
|
||||
ariaLabel?: string;
|
||||
onClose: () => void;
|
||||
children: ReactNode;
|
||||
footer?: ReactNode;
|
||||
className?: string;
|
||||
};
|
||||
|
||||
export function Modal({ open, title, onClose, children, footer, className = "" }: Props) {
|
||||
export function Modal({ open, title, ariaLabel, onClose, children, footer, className = "" }: Props) {
|
||||
const dialogRef = useRef<HTMLDivElement>(null);
|
||||
const onCloseRef = useRef(onClose);
|
||||
const titleId = useId();
|
||||
@@ -74,18 +76,19 @@ export function Modal({ open, title, onClose, children, footer, className = "" }
|
||||
}, [open]);
|
||||
|
||||
if (!open) return null;
|
||||
return (
|
||||
return createPortal(
|
||||
<div className="admin-modal-backdrop">
|
||||
<div
|
||||
ref={dialogRef}
|
||||
className={`admin-modal${className ? ` ${className}` : ""}`}
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
aria-labelledby={titleId}
|
||||
aria-labelledby={title ? titleId : undefined}
|
||||
aria-label={title ? undefined : ariaLabel ?? "对话框"}
|
||||
tabIndex={-1}
|
||||
>
|
||||
<div className="admin-modal__header">
|
||||
<span id={titleId}>{title}</span>
|
||||
<div className={`admin-modal__header${title ? "" : " is-titleless"}`}>
|
||||
{title && <span id={titleId}>{title}</span>}
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
@@ -99,7 +102,8 @@ export function Modal({ open, title, onClose, children, footer, className = "" }
|
||||
<div className="admin-modal__body">{children}</div>
|
||||
{footer && <div className="admin-modal__footer">{footer}</div>}
|
||||
</div>
|
||||
</div>
|
||||
</div>,
|
||||
document.body
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
+3
-14
@@ -55,7 +55,7 @@ export function TagsPage() {
|
||||
setSaving(true);
|
||||
try {
|
||||
const r = await api.createTag(cleanLabel, splitList(aliases));
|
||||
show(`已添加标签,自动归类 ${r.classified} 个视频`, "success");
|
||||
show(`已添加标签,自动匹配 ${r.classified} 个视频`, "success");
|
||||
setLabel("");
|
||||
setAliases("");
|
||||
await refresh();
|
||||
@@ -131,14 +131,12 @@ export function TagsPage() {
|
||||
let totalVideos = 0;
|
||||
let systemCount = 0;
|
||||
let userCount = 0;
|
||||
let collectionCount = 0;
|
||||
let legacyCount = 0;
|
||||
|
||||
tags.forEach((t) => {
|
||||
totalVideos += t.count ?? 0;
|
||||
if (t.source === "system") systemCount++;
|
||||
else if (t.source === "user") userCount++;
|
||||
else if (t.source === "collection") collectionCount++;
|
||||
else if (t.source === "legacy") legacyCount++;
|
||||
});
|
||||
|
||||
@@ -147,7 +145,6 @@ export function TagsPage() {
|
||||
totalVideos,
|
||||
systemCount,
|
||||
userCount,
|
||||
collectionCount,
|
||||
legacyCount,
|
||||
};
|
||||
}, [tags]);
|
||||
@@ -213,7 +210,7 @@ export function TagsPage() {
|
||||
<div>
|
||||
<div className="admin-card">
|
||||
<div className="admin-card__title">
|
||||
<Plus size={15} /> 新增分类标签
|
||||
<Plus size={15} /> 新增标签
|
||||
</div>
|
||||
<form
|
||||
className="admin-form"
|
||||
@@ -245,7 +242,7 @@ export function TagsPage() {
|
||||
className="admin-btn is-primary"
|
||||
disabled={saving || !label.trim()}
|
||||
>
|
||||
<Plus size={13} /> {saving ? "添加中..." : "添加并自动归类"}
|
||||
<Plus size={13} /> {saving ? "添加中..." : "添加并自动匹配"}
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
@@ -303,13 +300,6 @@ export function TagsPage() {
|
||||
>
|
||||
用户 ({stats.userCount})
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`admin-tags-filter-tab ${filterSource === "collection" ? "is-active" : ""}`}
|
||||
onClick={() => setFilterSource("collection")}
|
||||
>
|
||||
合集 ({stats.collectionCount})
|
||||
</button>
|
||||
{stats.legacyCount > 0 && (
|
||||
<button
|
||||
type="button"
|
||||
@@ -538,7 +528,6 @@ function splitList(s: string): string[] {
|
||||
|
||||
function sourceLabel(source: string): string {
|
||||
if (source === "system") return "系统";
|
||||
if (source === "collection") return "合集";
|
||||
if (source === "legacy") return "旧数据";
|
||||
return "用户";
|
||||
}
|
||||
|
||||
+128
-20
@@ -9,35 +9,125 @@ import {
|
||||
} from "react";
|
||||
|
||||
type ToastKind = "info" | "success" | "error";
|
||||
type Toast = { id: number; kind: ToastKind; text: string };
|
||||
type Toast = { id: number; kind: ToastKind; text: string; copyable: boolean };
|
||||
|
||||
type Ctx = {
|
||||
show: (text: string, kind?: ToastKind) => void;
|
||||
};
|
||||
|
||||
const ToastCtx = createContext<Ctx | null>(null);
|
||||
const TOAST_DISMISS_MS = 2500;
|
||||
const TOAST_COPY_SUCCESS_TEXT = "已复制到剪贴板";
|
||||
const TOAST_COPY_ERROR_TEXT = "复制失败,请手动复制";
|
||||
|
||||
async function copyTextToClipboard(text: string) {
|
||||
try {
|
||||
if (navigator.clipboard?.writeText) {
|
||||
await navigator.clipboard.writeText(text);
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// Fall back to the legacy copy command below.
|
||||
}
|
||||
if (!fallbackCopyText(text)) {
|
||||
throw new Error("copy failed");
|
||||
}
|
||||
}
|
||||
|
||||
function fallbackCopyText(text: string) {
|
||||
if (!document.body) return false;
|
||||
const textarea = document.createElement("textarea");
|
||||
textarea.value = text;
|
||||
textarea.setAttribute("readonly", "");
|
||||
textarea.style.position = "fixed";
|
||||
textarea.style.left = "-9999px";
|
||||
textarea.style.top = "0";
|
||||
document.body.appendChild(textarea);
|
||||
textarea.select();
|
||||
try {
|
||||
return document.execCommand("copy");
|
||||
} finally {
|
||||
document.body.removeChild(textarea);
|
||||
}
|
||||
}
|
||||
|
||||
export function ToastProvider({ children }: { children: ReactNode }) {
|
||||
const [items, setItems] = useState<Toast[]>([]);
|
||||
const timers = useRef<Record<string, ReturnType<typeof window.setTimeout>>>({});
|
||||
const timers = useRef(new Map<number, ReturnType<typeof window.setTimeout>>());
|
||||
const idsByText = useRef(new Map<string, number>());
|
||||
|
||||
const clearDismissTimer = useCallback((id: number) => {
|
||||
const timer = timers.current.get(id);
|
||||
if (!timer) return;
|
||||
window.clearTimeout(timer);
|
||||
timers.current.delete(id);
|
||||
}, []);
|
||||
|
||||
const removeToast = useCallback(
|
||||
(id: number, text: string) => {
|
||||
clearDismissTimer(id);
|
||||
if (idsByText.current.get(text) === id) {
|
||||
idsByText.current.delete(text);
|
||||
}
|
||||
setItems((list) => list.filter((t) => t.id !== id));
|
||||
},
|
||||
[clearDismissTimer]
|
||||
);
|
||||
|
||||
const scheduleDismiss = useCallback(
|
||||
(id: number, text: string) => {
|
||||
clearDismissTimer(id);
|
||||
timers.current.set(
|
||||
id,
|
||||
window.setTimeout(() => removeToast(id, text), TOAST_DISMISS_MS)
|
||||
);
|
||||
},
|
||||
[clearDismissTimer, removeToast]
|
||||
);
|
||||
|
||||
const addToast = useCallback(
|
||||
(text: string, kind: ToastKind = "info", copyable = true) => {
|
||||
const existingID = idsByText.current.get(text);
|
||||
if (existingID !== undefined) {
|
||||
setItems((list) =>
|
||||
list.map((t) => (t.id === existingID ? { ...t, kind, copyable } : t))
|
||||
);
|
||||
scheduleDismiss(existingID, text);
|
||||
return;
|
||||
}
|
||||
const id = Date.now() + Math.random();
|
||||
idsByText.current.set(text, id);
|
||||
setItems((list) => [...list, { id, kind, text, copyable }]);
|
||||
scheduleDismiss(id, text);
|
||||
},
|
||||
[scheduleDismiss]
|
||||
);
|
||||
|
||||
// Deduplicate: same text won't stack, just resets the dismiss timer
|
||||
const show = useCallback((text: string, kind: ToastKind = "info") => {
|
||||
// Reset timer if duplicate
|
||||
if (timers.current[text]) {
|
||||
window.clearTimeout(timers.current[text]);
|
||||
timers.current[text] = window.setTimeout(() => {
|
||||
setItems((list) => list.filter((t) => t.text !== text));
|
||||
delete timers.current[text];
|
||||
}, 2600);
|
||||
return;
|
||||
}
|
||||
const id = Date.now() + Math.random();
|
||||
timers.current[text] = window.setTimeout(() => {
|
||||
setItems((list) => list.filter((t) => t.id !== id));
|
||||
delete timers.current[text];
|
||||
}, 2600);
|
||||
setItems((list) => [...list, { id, kind, text }]);
|
||||
const show = useCallback(
|
||||
(text: string, kind: ToastKind = "info") => {
|
||||
addToast(text, kind, true);
|
||||
},
|
||||
[addToast]
|
||||
);
|
||||
|
||||
const copyToastText = useCallback(
|
||||
(text: string) => {
|
||||
void copyTextToClipboard(text)
|
||||
.then(() => addToast(TOAST_COPY_SUCCESS_TEXT, "success", false))
|
||||
.catch(() => addToast(TOAST_COPY_ERROR_TEXT, "error", false));
|
||||
},
|
||||
[addToast]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
for (const timer of timers.current.values()) {
|
||||
window.clearTimeout(timer);
|
||||
}
|
||||
timers.current.clear();
|
||||
idsByText.current.clear();
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
@@ -45,8 +135,26 @@ export function ToastProvider({ children }: { children: ReactNode }) {
|
||||
{children}
|
||||
<div className="admin-toast-stack" role="status" aria-live="polite">
|
||||
{items.map((t) => (
|
||||
<div key={t.id} className={`admin-toast is-${t.kind}`}>
|
||||
{t.text}
|
||||
<div
|
||||
key={t.id}
|
||||
className={`admin-toast is-${t.kind}${
|
||||
t.copyable ? " is-copyable" : ""
|
||||
}`}
|
||||
role={t.copyable ? "button" : undefined}
|
||||
tabIndex={t.copyable ? 0 : undefined}
|
||||
aria-label={t.copyable ? `复制提示:${t.text}` : undefined}
|
||||
onClick={t.copyable ? () => copyToastText(t.text) : undefined}
|
||||
onKeyDown={
|
||||
t.copyable
|
||||
? (event) => {
|
||||
if (event.key !== "Enter" && event.key !== " ") return;
|
||||
event.preventDefault();
|
||||
copyToastText(t.text);
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
<span className="admin-toast__text">{t.text}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
+96
-147
@@ -82,7 +82,7 @@ export function VideosPage() {
|
||||
<h1 className="admin-page__title">视频管理</h1>
|
||||
</header>
|
||||
|
||||
<div className="admin-video-tabs" role="tablist" aria-label="视频管理分类">
|
||||
<div className="admin-video-tabs" role="tablist" aria-label="视频管理标签页">
|
||||
{TABS.map((t) => (
|
||||
<button
|
||||
key={t.key}
|
||||
@@ -217,11 +217,6 @@ function CurrentVideosTab({ onStatsChanged }: { onStatsChanged: () => void }) {
|
||||
|
||||
const listItems = list;
|
||||
const totalPages = Math.max(1, Math.ceil(total / pageSize));
|
||||
const pageStart = total === 0 ? 0 : (page - 1) * pageSize + 1;
|
||||
const pageEnd = Math.min(total, page * pageSize);
|
||||
const listSummary = driveId
|
||||
? `${driveNameMap.get(driveId) ?? driveId}:共 ${total} 个视频,第 ${page} / ${totalPages} 页,显示 ${pageStart}-${pageEnd}`
|
||||
: `全部网盘:共 ${total} 个视频,第 ${page} / ${totalPages} 页,显示 ${pageStart}-${pageEnd}`;
|
||||
|
||||
async function handleRegen(v: api.AdminVideo) {
|
||||
try {
|
||||
@@ -379,29 +374,27 @@ function CurrentVideosTab({ onStatsChanged }: { onStatsChanged: () => void }) {
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="admin-page__actions admin-videos-filter">
|
||||
<DriveFilter drives={drives} driveId={driveId} onChange={(id) => { setDriveId(id); setPage(1); }} withCounts />
|
||||
<div className={`admin-videos-current${selectedIds.size > 0 ? " has-bulk-actions" : ""}`}>
|
||||
<div className="admin-page__actions admin-videos-filter admin-videos-filter--current">
|
||||
<DriveFilter drives={drives} driveId={driveId} onChange={(id) => { setDriveId(id); setPage(1); }} />
|
||||
<SearchBox keyword={keyword} onChange={setKeyword} onSubmit={handleSearchSubmit} />
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 刷新
|
||||
<button type="button" className="admin-btn admin-videos-filter__refresh" onClick={refresh} aria-label="刷新当前视频">
|
||||
<RefreshCw size={13} />
|
||||
<span className="admin-videos-filter__refresh-text">刷新</span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{!loading && (
|
||||
{!loading && selectedIds.size > 0 && (
|
||||
<div className="admin-videos-list-toolbar">
|
||||
<div className="admin-videos-summary">{listSummary}</div>
|
||||
{selectedIds.size > 0 && (
|
||||
<div className="admin-videos-bulk-actions">
|
||||
<span className="admin-videos-bulk-actions__count">已选择 {selectedIds.size} 项</span>
|
||||
<button type="button" className="admin-btn is-primary admin-videos-bulk-actions__btn" onClick={handleBatchRegen}>
|
||||
<RefreshCw size={13} /> 批量重生预览视频
|
||||
</button>
|
||||
<button type="button" className="admin-btn is-danger admin-videos-bulk-actions__btn" onClick={handleBatchDelete}>
|
||||
<Trash2 size={13} /> 批量删除
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
<div className="admin-videos-bulk-actions">
|
||||
<span className="admin-videos-bulk-actions__count">已选择 {selectedIds.size} 项</span>
|
||||
<button type="button" className="admin-btn is-primary admin-videos-bulk-actions__btn" onClick={handleBatchRegen}>
|
||||
<RefreshCw size={13} /> 批量重生预览视频
|
||||
</button>
|
||||
<button type="button" className="admin-btn is-danger admin-videos-bulk-actions__btn" onClick={handleBatchDelete}>
|
||||
<Trash2 size={13} /> 批量删除
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -452,60 +445,64 @@ function CurrentVideosTab({ onStatsChanged }: { onStatsChanged: () => void }) {
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{listItems.map((v) => (
|
||||
<tr key={v.id} className={selectedIds.has(v.id) ? "is-selected" : ""}>
|
||||
<td className="is-checkbox">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-table-checkbox-btn"
|
||||
onClick={() => toggleSelect(v.id)}
|
||||
aria-label={`${selectedIds.has(v.id) ? "取消选择" : "选择"}视频 ${v.title}`}
|
||||
>
|
||||
{selectedIds.has(v.id) ? (
|
||||
<CheckSquare size={16} color="var(--accent)" />
|
||||
) : (
|
||||
<Square size={16} color="var(--border-strong)" />
|
||||
)}
|
||||
</button>
|
||||
</td>
|
||||
<td data-label="标题">
|
||||
<VideoTitleCell video={v} />
|
||||
</td>
|
||||
<td data-label="作者">{v.author || <span className="admin-text-faint">—</span>}</td>
|
||||
<td data-label="时长">{formatDur(v.durationSeconds)}</td>
|
||||
<td data-label="预览视频">
|
||||
<PreviewStatus s={isPreviewGenerating(v) ? REGEN_PREVIEW_STATUS : v.previewStatus} />
|
||||
</td>
|
||||
<td data-label="来源" className="admin-mono-cell">
|
||||
{driveNameMap.get(v.driveId) ?? v.driveId}
|
||||
</td>
|
||||
<td className="is-actions" data-label="操作">
|
||||
<button type="button" className="admin-btn" onClick={() => setEditing(v)} title="编辑视频">
|
||||
<Edit size={13} />
|
||||
</button>{" "}
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => handleRegen(v)}
|
||||
disabled={isPreviewGenerating(v)}
|
||||
title={isPreviewGenerating(v) ? "预览视频正在生成" : "重生预览视频"}
|
||||
>
|
||||
<RefreshCw size={13} className={isPreviewGenerating(v) ? "admin-spin" : undefined} />
|
||||
</button>{" "}
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn is-danger"
|
||||
onClick={() => {
|
||||
setDeleteSource(false);
|
||||
setDeleteTarget(v);
|
||||
}}
|
||||
title="删除视频"
|
||||
>
|
||||
<Trash2 size={13} />
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
{listItems.map((v) => {
|
||||
const isSelected = selectedIds.has(v.id);
|
||||
|
||||
return (
|
||||
<tr key={v.id} className={isSelected ? "is-selected" : ""}>
|
||||
<td className="is-checkbox">
|
||||
<button
|
||||
type="button"
|
||||
className={`admin-table-checkbox-btn ${isSelected ? "is-selected" : ""}`}
|
||||
onClick={() => toggleSelect(v.id)}
|
||||
aria-label={`${isSelected ? "取消选择" : "选择"}视频 ${v.title}`}
|
||||
>
|
||||
{isSelected ? (
|
||||
<CheckSquare size={16} color="var(--accent)" />
|
||||
) : (
|
||||
<Square size={16} color="var(--border-strong)" />
|
||||
)}
|
||||
</button>
|
||||
</td>
|
||||
<td data-label="标题">
|
||||
<VideoTitleCell video={v} />
|
||||
</td>
|
||||
<td data-label="作者">{v.author || <span className="admin-text-faint">—</span>}</td>
|
||||
<td data-label="时长">{formatDur(v.durationSeconds)}</td>
|
||||
<td data-label="预览视频">
|
||||
<PreviewStatus s={isPreviewGenerating(v) ? REGEN_PREVIEW_STATUS : v.previewStatus} />
|
||||
</td>
|
||||
<td data-label="来源" className="admin-mono-cell">
|
||||
{driveNameMap.get(v.driveId) ?? v.driveId}
|
||||
</td>
|
||||
<td className="is-actions" data-label="操作">
|
||||
<button type="button" className="admin-btn" onClick={() => setEditing(v)} title="编辑视频">
|
||||
<Edit size={13} />
|
||||
</button>{" "}
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={() => handleRegen(v)}
|
||||
disabled={isPreviewGenerating(v)}
|
||||
title={isPreviewGenerating(v) ? "预览视频正在生成" : "重生预览视频"}
|
||||
>
|
||||
<RefreshCw size={13} className={isPreviewGenerating(v) ? "admin-spin" : undefined} />
|
||||
</button>{" "}
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn is-danger"
|
||||
onClick={() => {
|
||||
setDeleteSource(false);
|
||||
setDeleteTarget(v);
|
||||
}}
|
||||
title="删除视频"
|
||||
>
|
||||
<Trash2 size={13} />
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
<Pagination page={page} totalPages={totalPages} pageSize={pageSize} onPage={setPage} />
|
||||
@@ -572,7 +569,7 @@ function CurrentVideosTab({ onStatsChanged }: { onStatsChanged: () => void }) {
|
||||
>
|
||||
<DeleteSourceOption checked={batchDeleteSource} disabled={batchDeleting} onChange={setBatchDeleteSource} note="开启后会先删除源文件,失败的视频会保留管理库记录。" />
|
||||
</ConfirmModal>
|
||||
</>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -585,6 +582,7 @@ function BlacklistTab({ onStatsChanged }: { onStatsChanged: () => void }) {
|
||||
const [loadError, setLoadError] = useState("");
|
||||
const [keyword, setKeyword] = useState("");
|
||||
const [searchKeyword, setSearchKeyword] = useState("");
|
||||
const [driveId, setDriveId] = useState("");
|
||||
const [page, setPage] = useState(1);
|
||||
const [total, setTotal] = useState(0);
|
||||
const [removeTarget, setRemoveTarget] = useState<api.AdminDeletedVideo | null>(null);
|
||||
@@ -597,7 +595,7 @@ function BlacklistTab({ onStatsChanged }: { onStatsChanged: () => void }) {
|
||||
setLoadError("");
|
||||
try {
|
||||
const [r, driveList] = await Promise.all([
|
||||
api.listBlacklist({ page, size: pageSize, keyword: searchKeyword }),
|
||||
api.listBlacklist({ driveId, page, size: pageSize, keyword: searchKeyword }),
|
||||
api.listDrives(),
|
||||
]);
|
||||
setList(r.items ?? []);
|
||||
@@ -614,7 +612,7 @@ function BlacklistTab({ onStatsChanged }: { onStatsChanged: () => void }) {
|
||||
|
||||
useEffect(() => {
|
||||
refresh();
|
||||
}, [page, searchKeyword, pageSize]);
|
||||
}, [driveId, page, searchKeyword, pageSize]);
|
||||
|
||||
useEffect(() => {
|
||||
setPage(1);
|
||||
@@ -661,13 +659,12 @@ function BlacklistTab({ onStatsChanged }: { onStatsChanged: () => void }) {
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="admin-tab-intro">
|
||||
被删除和被隐藏的视频会进入黑名单,扫盘时不再重新入库。这里只保留文件名等基本信息(原始记录、封面、预览已删除)。移出黑名单后,视频会在下次扫盘时被重新发现并入库
|
||||
</div>
|
||||
<div className="admin-page__actions admin-videos-filter">
|
||||
<div className="admin-page__actions admin-videos-filter admin-videos-filter--blacklist">
|
||||
<DriveFilter drives={drives} driveId={driveId} onChange={(id) => { setDriveId(id); setPage(1); }} />
|
||||
<SearchBox keyword={keyword} onChange={setKeyword} onSubmit={handleSearchSubmit} placeholder="搜索文件名" />
|
||||
<button type="button" className="admin-btn" onClick={refresh}>
|
||||
<RefreshCw size={13} /> 刷新
|
||||
<button type="button" className="admin-btn admin-videos-filter__refresh" onClick={refresh} aria-label="刷新拉黑视频">
|
||||
<RefreshCw size={13} />
|
||||
<span className="admin-videos-filter__refresh-text">刷新</span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
@@ -701,7 +698,10 @@ function BlacklistTab({ onStatsChanged }: { onStatsChanged: () => void }) {
|
||||
{list.map((v) => (
|
||||
<tr key={v.id}>
|
||||
<td data-label="文件名">
|
||||
<span className="admin-blacklist-filename">{v.fileName || <span className="admin-text-faint">(无文件名)</span>}</span>
|
||||
<div className="admin-blacklist-filecell">
|
||||
<span className="admin-blacklist-filename">{v.fileName || <span className="admin-text-faint">(无文件名)</span>}</span>
|
||||
{v.reason === "duplicate" && <span className="admin-blacklist-reason-pill">重复文件</span>}
|
||||
</div>
|
||||
</td>
|
||||
<td data-label="来源" className="admin-mono-cell">
|
||||
{driveNameMap.get(v.driveId) ?? v.driveId}
|
||||
@@ -752,12 +752,10 @@ function DriveFilter({
|
||||
drives,
|
||||
driveId,
|
||||
onChange,
|
||||
withCounts = false,
|
||||
}: {
|
||||
drives: api.AdminDrive[];
|
||||
driveId: string;
|
||||
onChange: (id: string) => void;
|
||||
withCounts?: boolean;
|
||||
}) {
|
||||
return (
|
||||
<div className="admin-videos-filter__select-wrap">
|
||||
@@ -770,7 +768,6 @@ function DriveFilter({
|
||||
{drives.map((d) => (
|
||||
<option key={d.id} value={d.id}>
|
||||
{d.name || d.id}
|
||||
{withCounts ? `(已生成 ${d.teaserReadyCount ?? 0},待生成 ${d.teaserPendingCount ?? 0})` : ""}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
@@ -888,7 +885,7 @@ function VideoTitleCell({ video: v }: { video: api.AdminVideo }) {
|
||||
<div className="admin-video-title-cell">
|
||||
<div className="admin-video-thumb-wrap" aria-hidden="true">
|
||||
{v.thumbnailUrl ? (
|
||||
<img className="admin-video-thumb" src={v.thumbnailUrl} alt="" />
|
||||
<img className="admin-video-thumb" src={v.thumbnailUrl} alt="" loading="lazy" decoding="async" />
|
||||
) : (
|
||||
<div className="admin-video-thumb-placeholder">
|
||||
<Image size={14} />
|
||||
@@ -896,7 +893,7 @@ function VideoTitleCell({ video: v }: { video: api.AdminVideo }) {
|
||||
)}
|
||||
</div>
|
||||
<div className="admin-video-title-body">
|
||||
<div className="admin-video-title">{v.title}</div>
|
||||
<div className="admin-video-title" title={v.title}>{v.title}</div>
|
||||
{fileMeta(v) && <div className="admin-video-filemeta">{fileMeta(v)}</div>}
|
||||
{(v.tags ?? []).length > 0 && (
|
||||
<div className="admin-pills admin-video-title-tags">
|
||||
@@ -924,8 +921,7 @@ function PreviewStatus({ s }: { s: string }) {
|
||||
|
||||
function VideoFileMetaPills({ video }: { video: api.AdminVideo }) {
|
||||
const parts = fileMetaParts(video);
|
||||
const category = (video.category ?? "").trim();
|
||||
if (parts.length === 0 && !category) return null;
|
||||
if (parts.length === 0) return null;
|
||||
|
||||
return (
|
||||
<div className="admin-video-filemeta-pills" aria-label="视频文件信息">
|
||||
@@ -934,7 +930,6 @@ function VideoFileMetaPills({ video }: { video: api.AdminVideo }) {
|
||||
{part}
|
||||
</span>
|
||||
))}
|
||||
{category && <span className="admin-video-filemeta-pill is-category">{category}</span>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -993,11 +988,7 @@ function EditVideoModal({
|
||||
const [title, setTitle] = useState(video.title);
|
||||
const [author, setAuthor] = useState(video.author ?? "");
|
||||
const [selectedTags, setSelectedTags] = useState(video.tags ?? []);
|
||||
const [category, setCategory] = useState(video.category ?? "");
|
||||
const [badges, setBadges] = useState((video.badges ?? []).join(", "));
|
||||
const [description, setDescription] = useState(video.description ?? "");
|
||||
const [thumbnail, setThumbnail] = useState(video.thumbnailUrl ?? "");
|
||||
const [quality, setQuality] = useState(video.quality ?? "");
|
||||
const [durationSec, setDurationSec] = useState(String(video.durationSeconds || 0));
|
||||
const [saving, setSaving] = useState(false);
|
||||
const { show } = useToast();
|
||||
@@ -1009,11 +1000,7 @@ function EditVideoModal({
|
||||
title: title.trim(),
|
||||
author: author.trim(),
|
||||
tags: selectedTags,
|
||||
category: category.trim(),
|
||||
badges: splitList(badges),
|
||||
description,
|
||||
thumbnail: thumbnail.trim(),
|
||||
quality: quality.trim(),
|
||||
durationSeconds: Number(durationSec) || 0,
|
||||
});
|
||||
show("已保存", "success");
|
||||
@@ -1028,7 +1015,7 @@ function EditVideoModal({
|
||||
return (
|
||||
<Modal
|
||||
open
|
||||
title={`编辑视频 · ${video.title}`}
|
||||
ariaLabel="编辑视频"
|
||||
onClose={onClose}
|
||||
footer={
|
||||
<>
|
||||
@@ -1052,36 +1039,20 @@ function EditVideoModal({
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<div className="admin-form__label">标签</div>
|
||||
<div className="admin-tag-picker">
|
||||
<div className="admin-tag-picker admin-video-tag-picker">
|
||||
{availableTags.map((tag) => (
|
||||
<label key={tag.id} className="admin-check">
|
||||
<label key={tag.id} className="admin-check admin-video-tag-option">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={selectedTags.includes(tag.label)}
|
||||
onChange={() => setSelectedTags(toggleTag(selectedTags, tag.label))}
|
||||
/>
|
||||
<span>{tag.label}</span>
|
||||
<em>{tag.count}</em>
|
||||
<span className="admin-video-tag-option__label" title={tag.label}>{tag.label}</span>
|
||||
<em className="admin-video-tag-option__count">{tag.count}</em>
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label htmlFor={`${idPrefix}-video-category`}>分类</label>
|
||||
<input id={`${idPrefix}-video-category`} value={category} onChange={(e) => setCategory(e.target.value)} />
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label htmlFor={`${idPrefix}-video-badges`}>徽标(逗号分隔,例如 精选, 原创)</label>
|
||||
<input id={`${idPrefix}-video-badges`} value={badges} onChange={(e) => setBadges(e.target.value)} />
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label htmlFor={`${idPrefix}-video-quality`}>质量</label>
|
||||
<select id={`${idPrefix}-video-quality`} value={quality} onChange={(e) => setQuality(e.target.value)}>
|
||||
<option value="">未设置</option>
|
||||
<option value="HD">HD</option>
|
||||
<option value="SD">SD</option>
|
||||
</select>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label htmlFor={`${idPrefix}-video-duration`}>时长(秒)</label>
|
||||
<input
|
||||
@@ -1091,21 +1062,6 @@ function EditVideoModal({
|
||||
inputMode="numeric"
|
||||
/>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label htmlFor={`${idPrefix}-video-thumbnail`}>封面 URL</label>
|
||||
<div className="admin-thumbnail-preview">
|
||||
<input id={`${idPrefix}-video-thumbnail`} value={thumbnail} onChange={(e) => setThumbnail(e.target.value)} />
|
||||
{thumbnail && (
|
||||
<img
|
||||
src={thumbnail}
|
||||
alt="封面预览"
|
||||
className="admin-thumbnail-img"
|
||||
onError={(e) => (e.currentTarget.style.display = "none")}
|
||||
onLoad={(e) => (e.currentTarget.style.display = "block")}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label htmlFor={`${idPrefix}-video-description`}>描述</label>
|
||||
<textarea
|
||||
@@ -1151,13 +1107,6 @@ function normalizeExt(ext: string): string {
|
||||
return value ? value.toUpperCase() : "";
|
||||
}
|
||||
|
||||
function splitList(s: string): string[] {
|
||||
return s
|
||||
.split(/[,,、\s]+/)
|
||||
.map((x) => x.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function toggleTag(tags: string[], label: string): string[] {
|
||||
return tags.includes(label) ? tags.filter((tag) => tag !== label) : [...tags, label];
|
||||
}
|
||||
|
||||
+17
-16
@@ -78,7 +78,7 @@ export function checkUpdate() {
|
||||
|
||||
export type AdminDrive = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "p123" | "pikpak" | "wopan" | "guangyapan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
kind: "quark" | "p115" | "p123" | "pikpak" | "wopan" | "guangyapan" | "onedrive" | "googledrive" | "localstorage";
|
||||
name: string;
|
||||
rootId: string;
|
||||
status: string;
|
||||
@@ -92,12 +92,10 @@ export type AdminDrive = {
|
||||
* 替代旧版硬编码 p115 "影视" 目录例外分支。
|
||||
*/
|
||||
skipDirIds: string[];
|
||||
// spider91 上次成功爬取时间(unix 秒);其它 kind 留空。
|
||||
lastCrawlAt?: number;
|
||||
// spider91 专用代理地址;仅后台管理接口返回,用于编辑表单回显。
|
||||
spider91Proxy?: string;
|
||||
// Google Drive 是否使用 OpenList 在线续期 API;未配置时后端按 true 返回。
|
||||
googleDriveUseOnlineAPI?: boolean;
|
||||
// Google Drive OpenList 在线续期 API 地址;为空时后端使用驱动默认值。
|
||||
googleDriveOpenListApiUrl?: string;
|
||||
// localstorage 的 .strm 是否允许指向存储根目录之外;未配置时后端按 false 返回。
|
||||
strmAllowOutsideRoot?: boolean;
|
||||
scanGenerationStatus?: DriveGenerationStatus;
|
||||
@@ -155,7 +153,7 @@ export function getDriveStorage() {
|
||||
|
||||
export type UpsertDriveInput = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "p123" | "pikpak" | "wopan" | "guangyapan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
kind: "quark" | "p115" | "p123" | "pikpak" | "wopan" | "guangyapan" | "onedrive" | "googledrive" | "localstorage";
|
||||
name: string;
|
||||
rootId: string;
|
||||
credentials: Record<string, string>;
|
||||
@@ -204,7 +202,7 @@ export function stopDriveTasks(id: string) {
|
||||
export type AdminCrawler = {
|
||||
id: string;
|
||||
name: string;
|
||||
kind: "scriptcrawler" | "spider91";
|
||||
kind: "scriptcrawler";
|
||||
status: string;
|
||||
lastError?: string;
|
||||
scriptPath: string;
|
||||
@@ -315,6 +313,13 @@ export function runCrawler(id: string) {
|
||||
);
|
||||
}
|
||||
|
||||
export function uploadCrawlerVideos(id: string) {
|
||||
return request<{ ok: boolean; accepted: boolean; message?: string; status?: NightlyJobStatus }>(
|
||||
`/crawlers/${encodeURIComponent(id)}/upload`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
export function stopCrawlerTasks(id: string) {
|
||||
return request<{ ok: boolean; stopped: boolean }>(
|
||||
`/crawlers/${encodeURIComponent(id)}/tasks/stop`,
|
||||
@@ -524,7 +529,6 @@ export type AdminVideo = {
|
||||
favorites: number;
|
||||
comments: number;
|
||||
likes: number;
|
||||
category: string;
|
||||
badges: string[];
|
||||
description: string;
|
||||
publishedAt: string;
|
||||
@@ -568,6 +572,7 @@ export type AdminDeletedVideo = {
|
||||
fileId: string;
|
||||
fileName: string;
|
||||
size: number;
|
||||
reason?: string;
|
||||
deletedAt: number;
|
||||
};
|
||||
|
||||
@@ -578,8 +583,11 @@ export type AdminBlacklistList = {
|
||||
size: number;
|
||||
};
|
||||
|
||||
export function listBlacklist(params: { page?: number; size?: number; keyword?: string } = {}) {
|
||||
export function listBlacklist(
|
||||
params: { driveId?: string; page?: number; size?: number; keyword?: string } = {}
|
||||
) {
|
||||
const qs = new URLSearchParams();
|
||||
if (params.driveId) qs.set("driveId", params.driveId);
|
||||
if (params.page) qs.set("page", String(params.page));
|
||||
if (params.size) qs.set("size", String(params.size));
|
||||
if (params.keyword) qs.set("keyword", params.keyword);
|
||||
@@ -598,7 +606,6 @@ export type UpdateVideoInput = Partial<{
|
||||
title: string;
|
||||
author: string;
|
||||
tags: string[];
|
||||
category: string;
|
||||
badges: string[];
|
||||
description: string;
|
||||
thumbnail: string;
|
||||
@@ -664,12 +671,6 @@ export type Theme = "dark" | "pink" | "sky";
|
||||
|
||||
export type Settings = {
|
||||
theme: Theme;
|
||||
/**
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115、p123、onedrive、googledrive 或 wopan drive)。
|
||||
* - 空字符串:本地保存,不上传到云盘。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, p123, onedrive, googledrive, wopan}。
|
||||
*/
|
||||
spider91UploadDriveId: string;
|
||||
};
|
||||
|
||||
export function getSettings() {
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ import { ChevronDown } from "lucide-react";
|
||||
import { kindLabel } from "./constants";
|
||||
import * as api from "../api";
|
||||
|
||||
export function Spider91UploadTargetField({
|
||||
export function CrawlerUploadTargetField({
|
||||
value,
|
||||
onChange,
|
||||
uploadTargets,
|
||||
@@ -14,8 +14,7 @@ export function DeleteDriveModal({
|
||||
onConfirm: () => void;
|
||||
}) {
|
||||
const name = drive?.name || drive?.id || "";
|
||||
const isSpider91 = drive?.kind === "spider91";
|
||||
const title = isSpider91 ? "删除 91Spider" : "删除存储";
|
||||
const title = "删除存储";
|
||||
const primaryText = deleting ? "删除中..." : "确认删除";
|
||||
|
||||
return (
|
||||
|
||||
@@ -91,24 +91,15 @@ export function GenerationStatusLine({
|
||||
}
|
||||
|
||||
export function StatusTag({
|
||||
kind,
|
||||
status,
|
||||
error,
|
||||
hasCred,
|
||||
}: {
|
||||
kind: string;
|
||||
status: string;
|
||||
error?: string;
|
||||
hasCred: boolean;
|
||||
}) {
|
||||
if (kind === "spider91") {
|
||||
return (
|
||||
<span className="admin-status is-error" title={error || "请到爬虫管理添加爬虫脚本"}>
|
||||
已废弃
|
||||
</span>
|
||||
);
|
||||
}
|
||||
if (kind !== "spider91" && !hasCred) {
|
||||
if (!hasCred) {
|
||||
return <span className="admin-status is-pending">未配置凭证</span>;
|
||||
}
|
||||
if (status === "ok") {
|
||||
@@ -221,7 +212,7 @@ export function DriveGenerationPanel({
|
||||
|
||||
<div className="admin-gen-columns">
|
||||
<DriveGenCol
|
||||
label={d.kind === "spider91" ? "已废弃" : "扫盘"}
|
||||
label="扫盘"
|
||||
status={d.scanGenerationStatus}
|
||||
showCounts={false}
|
||||
/>
|
||||
|
||||
@@ -3,7 +3,6 @@ import { ArrowLeft, ChevronDown } from "lucide-react";
|
||||
import { P123QRCodeLogin } from "./P123QRCodeLogin";
|
||||
import { WopanQRCodeLogin } from "./WopanQRCodeLogin";
|
||||
import { GuangYaPanQRCodeLogin } from "./GuangYaPanQRCodeLogin";
|
||||
import { Spider91UploadTargetField } from "./Spider91UploadTargetField";
|
||||
import {
|
||||
FormState,
|
||||
Kind,
|
||||
@@ -12,7 +11,6 @@ import {
|
||||
usesRootDirectoryID,
|
||||
rootIdPlaceholder,
|
||||
} from "./constants";
|
||||
import * as api from "../api";
|
||||
|
||||
type DriveOption = {
|
||||
kind: Kind;
|
||||
@@ -37,7 +35,6 @@ export function DriveForm({
|
||||
form,
|
||||
onChange,
|
||||
isEdit,
|
||||
uploadTargets,
|
||||
nameError,
|
||||
onNameBlur,
|
||||
onBack,
|
||||
@@ -45,7 +42,6 @@ export function DriveForm({
|
||||
form: FormState;
|
||||
onChange: (f: FormState) => void;
|
||||
isEdit: boolean;
|
||||
uploadTargets: api.AdminDrive[];
|
||||
nameError?: string;
|
||||
onNameBlur?: () => void;
|
||||
onBack?: () => void;
|
||||
@@ -266,17 +262,6 @@ export function DriveForm({
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{form.kind === "spider91" && (
|
||||
<div className="admin-form__section">
|
||||
<h3 className="admin-form__section-label">上传设置</h3>
|
||||
<Spider91UploadTargetField
|
||||
value={form.spider91UploadDriveId}
|
||||
onChange={(v) => set("spider91UploadDriveId", v)}
|
||||
uploadTargets={uploadTargets}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export type Kind = "quark" | "p115" | "p123" | "pikpak" | "wopan" | "guangyapan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
export type Kind = "quark" | "p115" | "p123" | "pikpak" | "wopan" | "guangyapan" | "onedrive" | "googledrive" | "localstorage";
|
||||
|
||||
export const kindAbbr: Record<string, string> = {
|
||||
quark: "Qk",
|
||||
@@ -10,7 +10,6 @@ export const kindAbbr: Record<string, string> = {
|
||||
onedrive: "OD",
|
||||
googledrive: "GD",
|
||||
localstorage: "Lo",
|
||||
spider91: "91",
|
||||
};
|
||||
|
||||
export function driveKindAbbr(kind: string): string {
|
||||
@@ -33,7 +32,6 @@ export const kindLabel: Record<string, string> = {
|
||||
onedrive: "OneDrive",
|
||||
googledrive: "Google Drive",
|
||||
localstorage: "本地存储",
|
||||
spider91: "91 爬虫",
|
||||
};
|
||||
|
||||
export type FormState = {
|
||||
@@ -42,7 +40,6 @@ export type FormState = {
|
||||
name: string;
|
||||
rootId: string;
|
||||
creds: Record<string, string>;
|
||||
spider91UploadDriveId: string;
|
||||
};
|
||||
|
||||
export const emptyForm: FormState = {
|
||||
@@ -51,7 +48,6 @@ export const emptyForm: FormState = {
|
||||
name: "",
|
||||
rootId: "",
|
||||
creds: {},
|
||||
spider91UploadDriveId: "",
|
||||
};
|
||||
|
||||
export const idleNightlyStatus = {
|
||||
@@ -132,12 +128,11 @@ export function defaultRootId(kind: Kind): string {
|
||||
if (kind === "onedrive") return "root";
|
||||
if (kind === "googledrive") return "root";
|
||||
if (kind === "localstorage") return "/";
|
||||
if (kind === "spider91") return "/";
|
||||
return "0";
|
||||
}
|
||||
|
||||
export function usesRootDirectoryID(kind: Kind): boolean {
|
||||
return kind !== "localstorage" && kind !== "spider91";
|
||||
return kind !== "localstorage";
|
||||
}
|
||||
|
||||
export function rootIdPlaceholder(kind: Kind): string {
|
||||
@@ -168,8 +163,6 @@ export function credentialHelp(kind: Kind, isEdit: boolean): string {
|
||||
: "请参考OpenList文档中关于谷歌云盘的配置方法";
|
||||
case "localstorage":
|
||||
return `填写服务器可访问的本地目录绝对路径,例如 /mnt/videos。系统会扫描该目录及子目录中的视频文件和 .strm 文件;.strm 可指向 HTTP/HTTPS 直链或本地视频路径(指向目录外需开启下方开关)。Docker 部署时请填写容器内路径。${note}`;
|
||||
case "spider91":
|
||||
return "91Spider 不再支持通过网盘添加或编辑。请到后台爬虫管理页面添加爬虫脚本。";
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
@@ -323,15 +316,15 @@ export function credentialFields(kind: Kind, creds: Record<string, string> = {})
|
||||
{ value: "false", label: "自建 Google OAuth 客户端" },
|
||||
],
|
||||
},
|
||||
{
|
||||
key: "refresh_token",
|
||||
label: "refresh_token",
|
||||
placeholder: "OpenList Google Drive refresh_token",
|
||||
multiline: true,
|
||||
required: true,
|
||||
},
|
||||
...(googleDriveUsesOnlineAPI(creds)
|
||||
? []
|
||||
? [
|
||||
{
|
||||
key: "api_url_address",
|
||||
label: "OpenList 在线 API URL",
|
||||
placeholder: "默认:https://api.oplist.org/googleui/renewapi",
|
||||
help: "留空时使用 OpenList 官方在线 API,填写后会使用自定义续期 API。",
|
||||
},
|
||||
]
|
||||
: [
|
||||
{
|
||||
key: "client_id",
|
||||
@@ -348,6 +341,13 @@ export function credentialFields(kind: Kind, creds: Record<string, string> = {})
|
||||
help: "Google Cloud Console 中同一个 OAuth 客户端的 Client Secret",
|
||||
},
|
||||
]),
|
||||
{
|
||||
key: "refresh_token",
|
||||
label: "refresh_token",
|
||||
placeholder: "OpenList Google Drive refresh_token",
|
||||
multiline: true,
|
||||
required: true,
|
||||
},
|
||||
];
|
||||
case "localstorage":
|
||||
return [
|
||||
@@ -371,14 +371,5 @@ export function credentialFields(kind: Kind, creds: Record<string, string> = {})
|
||||
help: "开启后 .strm 可指向本目录之外的本地文件(如 rclone 挂载点)。注意:等于允许通过 .strm 读取服务器上任意文件,请只在自己完全掌控媒体目录时开启。Docker 部署时路径必须是容器内路径。",
|
||||
},
|
||||
];
|
||||
case "spider91":
|
||||
return [
|
||||
{
|
||||
key: "proxy",
|
||||
label: "代理地址(可选)",
|
||||
placeholder: "http://127.0.0.1:7890",
|
||||
help: "支持 http://、https://、socks5://、socks5h://代理",
|
||||
},
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { promoItems } from "@/data/categories";
|
||||
import { promoItems } from "@/data/promos";
|
||||
|
||||
const kindLabel: Record<string, string> = {
|
||||
channel: "频道",
|
||||
collection: "合集",
|
||||
topic: "专题",
|
||||
event: "活动",
|
||||
};
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useEffect, useRef, useState, useSyncExternalStore } from "react";
|
||||
import { Link } from "react-router-dom";
|
||||
import { Link, useLocation } from "react-router-dom";
|
||||
import type { PreviewState, VideoItem } from "@/types";
|
||||
import { formatCount } from "@/lib/format";
|
||||
import { previewController } from "@/lib/previewController";
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
shouldStartInstantPreview,
|
||||
} from "@/lib/previewIntent";
|
||||
import { useInViewport } from "@/lib/useInViewport";
|
||||
import { resolveVideoReturnPath, routeToPath } from "@/lib/videoReturnPath";
|
||||
import { PreviewVideo } from "./PreviewVideo";
|
||||
|
||||
type Props = {
|
||||
@@ -66,6 +67,12 @@ function RecommendedItem({ video }: { video: VideoItem }) {
|
||||
|
||||
const activeId = useActivePreviewId();
|
||||
const inView = useInViewport(rootRef);
|
||||
const location = useLocation();
|
||||
const locationState = location.state as { from?: unknown } | null;
|
||||
const returnPath =
|
||||
typeof locationState?.from === "string"
|
||||
? resolveVideoReturnPath(locationState.from)
|
||||
: resolveVideoReturnPath(routeToPath(location));
|
||||
|
||||
// 全局预览换卡时立即清理
|
||||
useEffect(() => {
|
||||
@@ -196,6 +203,7 @@ function RecommendedItem({ video }: { video: VideoItem }) {
|
||||
>
|
||||
<Link
|
||||
to={video.href}
|
||||
state={{ from: returnPath }}
|
||||
className="vd-rail__link"
|
||||
onClickCapture={handleClickCapture}
|
||||
>
|
||||
|
||||
@@ -13,10 +13,7 @@ type Props = {
|
||||
const sortOptions: { key: SortKey; label: string }[] = [
|
||||
{ key: "latest", label: "最新" },
|
||||
{ key: "hot", label: "最热" },
|
||||
{ key: "week", label: "本周" },
|
||||
{ key: "long", label: "最长" },
|
||||
{ key: "hd", label: "高清" },
|
||||
{ key: "featured", label: "精选" },
|
||||
{ key: "recent", label: "最近观看" },
|
||||
];
|
||||
|
||||
export function SortToolbar({ sort, view, onSortChange, onViewChange }: Props) {
|
||||
|
||||
@@ -114,7 +114,7 @@ export function TagCloud() {
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="tag-cloud-container" aria-label="热门分类">
|
||||
<div className="tag-cloud-container" aria-label="热门标签">
|
||||
<div className="tag-cloud__grid" ref={containerRef}>
|
||||
<div className="tag-cloud__row">
|
||||
{row1.map(renderTag)}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useEffect, useRef, useState, useSyncExternalStore } from "react";
|
||||
import { Link } from "react-router-dom";
|
||||
import { Link, useLocation } from "react-router-dom";
|
||||
import type { PreviewState, VideoItem } from "@/types";
|
||||
import { previewController } from "@/lib/previewController";
|
||||
import {
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
} from "@/lib/previewIntent";
|
||||
import { useInViewport } from "@/lib/useInViewport";
|
||||
import { formatCount } from "@/lib/format";
|
||||
import { isVideoReturnPath, routeToPath } from "@/lib/videoReturnPath";
|
||||
import { PreviewVideo } from "./PreviewVideo";
|
||||
|
||||
type Props = {
|
||||
@@ -30,6 +31,12 @@ export function VideoCard({ video, priority = false }: Props) {
|
||||
const [shouldRenderPreview, setShouldRenderPreview] = useState(false);
|
||||
const [progress, setProgress] = useState(0); // 0~1
|
||||
const [thumbnailRetry, setThumbnailRetry] = useState(0);
|
||||
const author = video.author.trim();
|
||||
const location = useLocation();
|
||||
const currentPath = routeToPath(location);
|
||||
const linkState = isVideoReturnPath(currentPath)
|
||||
? { from: currentPath }
|
||||
: undefined;
|
||||
|
||||
const rootRef = useRef<HTMLElement | null>(null);
|
||||
const hoverTimerRef = useRef<number | null>(null);
|
||||
@@ -196,6 +203,7 @@ export function VideoCard({ video, priority = false }: Props) {
|
||||
>
|
||||
<Link
|
||||
to={video.href}
|
||||
state={linkState}
|
||||
className="video-card__link"
|
||||
tabIndex={0}
|
||||
onClickCapture={handleClickCapture}
|
||||
@@ -271,9 +279,13 @@ export function VideoCard({ video, priority = false }: Props) {
|
||||
</h3>
|
||||
|
||||
<div className="video-meta">
|
||||
<span className="video-meta__author">{video.author}</span>
|
||||
<span>{formatCount(video.views)} 观看</span>
|
||||
<span>{video.publishedAt}</span>
|
||||
{author && (
|
||||
<span className="video-meta__author" title={author}>
|
||||
{author}
|
||||
</span>
|
||||
)}
|
||||
<span className="video-meta__views">{formatCount(video.views)} 观看</span>
|
||||
<span className="video-meta__date">{video.publishedAt}</span>
|
||||
</div>
|
||||
</Link>
|
||||
</article>
|
||||
|
||||
@@ -288,11 +288,10 @@ function inferSourceType(src: string) {
|
||||
|
||||
function isBackendNativeVideoRoute(cleanPath: string) {
|
||||
const pathname = sourcePathname(cleanPath);
|
||||
return (
|
||||
pathname.startsWith("/p/stream/") ||
|
||||
pathname.startsWith("/p/upload/") ||
|
||||
pathname.startsWith("/p/spider91/")
|
||||
);
|
||||
return (
|
||||
pathname.startsWith("/p/stream/") ||
|
||||
pathname.startsWith("/p/upload/")
|
||||
);
|
||||
}
|
||||
|
||||
function sourcePathname(src: string) {
|
||||
@@ -986,7 +985,7 @@ function createHlsSourceLoader(
|
||||
destroyHls(target);
|
||||
onError(null);
|
||||
|
||||
void import("hls.js")
|
||||
void import("hls.js/light")
|
||||
.then((hlsModule) => {
|
||||
if (art.isDestroy || !video.isConnected) return;
|
||||
loadHlsSourceWith(video, url, art, hlsModule.default, onError);
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
import type { PromoItem } from "@/types";
|
||||
|
||||
// 第一版不再预置横幅。真实素材来自后续的"合集/专题"接口,这里先留空。
|
||||
export const promoItems: PromoItem[] = [];
|
||||
@@ -0,0 +1,4 @@
|
||||
import type { PromoItem } from "@/types";
|
||||
|
||||
// 第一版不再预置横幅。真实素材来自后续推荐接口,这里先留空。
|
||||
export const promoItems: PromoItem[] = [];
|
||||
+1
-2
@@ -13,7 +13,7 @@ export function fetchHomeVideos(excludeIds?: string[]): Promise<VideoItem[]> {
|
||||
export function fetchListing(
|
||||
page: number,
|
||||
pageSize: number,
|
||||
params?: { q?: string; tag?: string; cat?: string; sort?: string; includeTotal?: boolean }
|
||||
params?: { q?: string; tag?: string; sort?: string; includeTotal?: boolean }
|
||||
): Promise<{ items: VideoItem[]; total: number }> {
|
||||
const qs = new URLSearchParams({
|
||||
page: String(page),
|
||||
@@ -21,7 +21,6 @@ export function fetchListing(
|
||||
});
|
||||
if (params?.q) qs.set("q", params.q);
|
||||
if (params?.tag) qs.set("tag", params.tag);
|
||||
if (params?.cat) qs.set("cat", params.cat);
|
||||
if (params?.sort) qs.set("sort", params.sort);
|
||||
if (params?.includeTotal === false) qs.set("count", "false");
|
||||
return apiGet<{ items: VideoItem[]; total: number }>(
|
||||
|
||||
Vendored
+4
@@ -0,0 +1,4 @@
|
||||
declare module "hls.js/light" {
|
||||
export { default } from "hls.js";
|
||||
export * from "hls.js";
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
export const VIDEO_RETURN_PATH_STORAGE_KEY = "video-site:video-return-path";
|
||||
|
||||
type RouteLike = {
|
||||
pathname: string;
|
||||
search?: string;
|
||||
hash?: string;
|
||||
};
|
||||
|
||||
export function routeToPath(route: RouteLike): string {
|
||||
return `${route.pathname}${route.search ?? ""}${route.hash ?? ""}`;
|
||||
}
|
||||
|
||||
export function normalizeVideoReturnPath(path: string, origin = browserOrigin()): string | null {
|
||||
const raw = path.trim();
|
||||
if (!raw) return null;
|
||||
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(raw, origin ?? "http://localhost");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (origin && url.origin !== origin) return null;
|
||||
if (!url.pathname.startsWith("/")) return null;
|
||||
if (url.pathname === "/login") return null;
|
||||
if (url.pathname === "/video" || url.pathname.startsWith("/video/")) return null;
|
||||
|
||||
return `${url.pathname}${url.search}${url.hash}` || "/";
|
||||
}
|
||||
|
||||
export function isVideoReturnPath(path: string, origin = browserOrigin()): boolean {
|
||||
return normalizeVideoReturnPath(path, origin) !== null;
|
||||
}
|
||||
|
||||
export function rememberVideoReturnPath(path: string) {
|
||||
const normalized = normalizeVideoReturnPath(path);
|
||||
if (!normalized || typeof window === "undefined") return;
|
||||
|
||||
try {
|
||||
window.sessionStorage.setItem(VIDEO_RETURN_PATH_STORAGE_KEY, normalized);
|
||||
} catch {
|
||||
// sessionStorage 不可用时退回默认首页,不影响播放和删除流程。
|
||||
}
|
||||
}
|
||||
|
||||
export function readVideoReturnPath(): string | null {
|
||||
if (typeof window === "undefined") return null;
|
||||
|
||||
try {
|
||||
const saved = window.sessionStorage.getItem(VIDEO_RETURN_PATH_STORAGE_KEY);
|
||||
return saved ? normalizeVideoReturnPath(saved) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveVideoReturnPath(candidate?: string | null): string {
|
||||
if (candidate) {
|
||||
const normalized = normalizeVideoReturnPath(candidate);
|
||||
if (normalized) return normalized;
|
||||
}
|
||||
return readVideoReturnPath() ?? "/";
|
||||
}
|
||||
|
||||
function browserOrigin(): string | undefined {
|
||||
return typeof window === "undefined" ? undefined : window.location.origin;
|
||||
}
|
||||
@@ -26,10 +26,9 @@ export default function ListingPage() {
|
||||
const [params] = useSearchParams();
|
||||
const keyword = params.get("q") ?? "";
|
||||
const tag = params.get("tag") ?? "";
|
||||
const cat = params.get("cat") ?? "";
|
||||
const listKey = useMemo(
|
||||
() => listingStateKey({ keyword, tag, cat }),
|
||||
[keyword, tag, cat]
|
||||
() => listingStateKey({ keyword, tag }),
|
||||
[keyword, tag]
|
||||
);
|
||||
const initialState = useMemo(() => readListingState(listKey), [listKey]);
|
||||
const activeListKeyRef = useRef(listKey);
|
||||
@@ -62,8 +61,6 @@ export default function ListingPage() {
|
||||
? `搜索 "${keyword}" · 91`
|
||||
: tag
|
||||
? `标签 ${tag} · 91`
|
||||
: cat
|
||||
? `分类 ${cat} · 91`
|
||||
: "视频列表 · 91";
|
||||
|
||||
let active = true;
|
||||
@@ -73,7 +70,7 @@ export default function ListingPage() {
|
||||
} else {
|
||||
setRefreshing(true);
|
||||
}
|
||||
fetchListing(page, tag ? PAGE_SIZE_TAG : PAGE_SIZE_DEFAULT, { q: keyword, tag, cat, sort }).then((r) => {
|
||||
fetchListing(page, tag ? PAGE_SIZE_TAG : PAGE_SIZE_DEFAULT, { q: keyword, tag, sort }).then((r) => {
|
||||
if (!active) return;
|
||||
setItems(r.items ?? []);
|
||||
setTotal(r.total ?? 0);
|
||||
@@ -84,7 +81,7 @@ export default function ListingPage() {
|
||||
return () => {
|
||||
active = false;
|
||||
};
|
||||
}, [keyword, tag, cat, sort, page]);
|
||||
}, [keyword, tag, sort, page]);
|
||||
|
||||
useEffect(() => {
|
||||
const previous = window.history.scrollRestoration;
|
||||
@@ -134,8 +131,6 @@ export default function ListingPage() {
|
||||
? `搜索结果:${keyword}`
|
||||
: tag
|
||||
? `标签:${tag}`
|
||||
: cat && cat !== "all"
|
||||
? `分类:${cat}`
|
||||
: "全部视频";
|
||||
|
||||
return (
|
||||
@@ -186,12 +181,10 @@ export default function ListingPage() {
|
||||
function listingStateKey(filters: {
|
||||
keyword: string;
|
||||
tag: string;
|
||||
cat: string;
|
||||
}): string {
|
||||
const params = new URLSearchParams();
|
||||
if (filters.keyword) params.set("q", filters.keyword);
|
||||
if (filters.tag) params.set("tag", filters.tag);
|
||||
if (filters.cat) params.set("cat", filters.cat);
|
||||
return `${LISTING_STATE_PREFIX}${params.toString()}`;
|
||||
}
|
||||
|
||||
@@ -226,9 +219,6 @@ function isSortKey(value: unknown): value is SortKey {
|
||||
return (
|
||||
value === "latest" ||
|
||||
value === "hot" ||
|
||||
value === "week" ||
|
||||
value === "long" ||
|
||||
value === "hd" ||
|
||||
value === "featured"
|
||||
value === "recent"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useEffect, useLayoutEffect, useRef, useState } from "react";
|
||||
import { useNavigate, useParams } from "react-router-dom";
|
||||
import { useLocation, useNavigate, useParams } from "react-router-dom";
|
||||
import { AppShell } from "@/components/AppShell";
|
||||
import { VideoPlayer } from "@/components/VideoPlayer";
|
||||
import { VideoActions } from "@/components/VideoActions";
|
||||
@@ -13,11 +13,14 @@ import {
|
||||
recordView,
|
||||
updateVideoTags,
|
||||
} from "@/data/videos";
|
||||
import { resolveVideoReturnPath } from "@/lib/videoReturnPath";
|
||||
import type { TagItem, VideoDetail } from "@/types";
|
||||
|
||||
export default function VideoDetailPage() {
|
||||
const { id } = useParams<{ id: string }>();
|
||||
const navigate = useNavigate();
|
||||
const location = useLocation();
|
||||
const locationState = location.state as { from?: unknown } | null;
|
||||
const [detail, setDetail] = useState<VideoDetail | null>(null);
|
||||
const [tags, setTags] = useState<TagItem[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
@@ -85,7 +88,8 @@ export default function VideoDetailPage() {
|
||||
setDeleteError("");
|
||||
try {
|
||||
await deleteVideo(detail.id, { deleteSource });
|
||||
navigate("/list", { replace: true });
|
||||
const from = typeof locationState?.from === "string" ? locationState.from : null;
|
||||
navigate(resolveVideoReturnPath(from), { replace: true });
|
||||
} catch {
|
||||
setDeleteError(
|
||||
deleteSource
|
||||
|
||||
+430
-178
@@ -626,7 +626,7 @@
|
||||
width: 100%;
|
||||
min-width: 0;
|
||||
display: grid;
|
||||
grid-template-columns: 38px minmax(160px, 1.1fr) minmax(0, 1.4fr) auto 18px;
|
||||
grid-template-columns: 38px minmax(160px, 1fr) 18px;
|
||||
align-items: center;
|
||||
gap: var(--space-3);
|
||||
padding: 0;
|
||||
@@ -676,77 +676,6 @@
|
||||
font-size: var(--font-xs);
|
||||
}
|
||||
|
||||
.admin-crawler-pipeline {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.admin-crawler-stage {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
min-height: 24px;
|
||||
padding: 3px 9px;
|
||||
border-radius: var(--radius-pill);
|
||||
border: 1px solid var(--border-subtle);
|
||||
background: var(--bg-sunken);
|
||||
color: var(--text-muted);
|
||||
font-size: var(--font-xs);
|
||||
font-weight: var(--weight-medium);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-crawler-stage em {
|
||||
font-style: normal;
|
||||
color: inherit;
|
||||
}
|
||||
|
||||
.admin-crawler-stage__dot {
|
||||
width: 7px;
|
||||
height: 7px;
|
||||
border-radius: 50%;
|
||||
background: var(--text-faint);
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
.admin-crawler-stage.is-generating {
|
||||
color: var(--info);
|
||||
border-color: transparent;
|
||||
background: var(--info-soft);
|
||||
}
|
||||
|
||||
.admin-crawler-stage.is-generating .admin-crawler-stage__dot {
|
||||
background: var(--info);
|
||||
animation: admin-crawler-stage-pulse 1.2s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.admin-crawler-stage.is-cooling {
|
||||
color: var(--warning);
|
||||
border-color: transparent;
|
||||
background: var(--warning-soft);
|
||||
}
|
||||
|
||||
.admin-crawler-stage.is-cooling .admin-crawler-stage__dot {
|
||||
background: var(--warning);
|
||||
}
|
||||
|
||||
.admin-crawler-stage.is-queued {
|
||||
color: var(--text-muted);
|
||||
background: rgba(255, 255, 255, 0.06);
|
||||
}
|
||||
|
||||
.admin-crawler-stage.is-queued .admin-crawler-stage__dot {
|
||||
background: var(--text-muted);
|
||||
}
|
||||
|
||||
@keyframes admin-crawler-stage-pulse {
|
||||
0%, 100% { opacity: 1; transform: scale(1); }
|
||||
50% { opacity: 0.4; transform: scale(0.75); }
|
||||
}
|
||||
|
||||
.admin-crawler-row__chevron {
|
||||
color: var(--text-faint);
|
||||
transition: transform var(--transition-fast), color var(--transition-fast);
|
||||
@@ -773,20 +702,6 @@
|
||||
padding-inline: 10px;
|
||||
}
|
||||
|
||||
.admin-crawler-preview-card-toggle.is-on {
|
||||
border-color: var(--accent);
|
||||
background: var(--accent);
|
||||
color: var(--text-on-accent);
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
.admin-crawler-preview-card-toggle.is-on:hover:not(:disabled) {
|
||||
border-color: var(--accent-hover);
|
||||
background: var(--accent-hover);
|
||||
color: var(--text-on-accent);
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
.admin-crawler-row__delete {
|
||||
padding-inline: 10px;
|
||||
}
|
||||
@@ -1214,13 +1129,7 @@
|
||||
|
||||
@media (max-width: 1180px) {
|
||||
.admin-crawler-row__main {
|
||||
grid-template-columns: 38px minmax(0, 1fr) auto 18px;
|
||||
row-gap: var(--space-2);
|
||||
}
|
||||
|
||||
.admin-crawler-pipeline {
|
||||
grid-column: 2 / 4;
|
||||
grid-row: 2;
|
||||
grid-template-columns: 38px minmax(0, 1fr) 18px;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2004,6 +1913,13 @@
|
||||
background: var(--bg-elevated);
|
||||
}
|
||||
|
||||
.admin-modal__header.is-titleless {
|
||||
justify-content: flex-end;
|
||||
padding: var(--space-3) var(--space-4) var(--space-2);
|
||||
border-bottom: 0;
|
||||
background: var(--bg-surface);
|
||||
}
|
||||
|
||||
.admin-modal__body {
|
||||
padding: var(--space-5);
|
||||
}
|
||||
@@ -2194,18 +2110,32 @@
|
||||
}
|
||||
|
||||
.admin-toast {
|
||||
padding: 12px 18px;
|
||||
max-width: min(520px, calc(100vw - 48px));
|
||||
padding: 14px 18px;
|
||||
background: var(--bg-elevated);
|
||||
color: var(--text-strong);
|
||||
border: 1px solid var(--border-default);
|
||||
border-radius: var(--radius-sm);
|
||||
position: relative;
|
||||
font-size: var(--font-sm);
|
||||
font-weight: var(--weight-medium);
|
||||
line-height: 1.5;
|
||||
overflow-wrap: anywhere;
|
||||
touch-action: manipulation;
|
||||
box-shadow: var(--shadow-lg);
|
||||
animation: toast-in var(--duration-normal) var(--ease-out);
|
||||
pointer-events: auto;
|
||||
}
|
||||
|
||||
.admin-toast.is-copyable {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.admin-toast__text {
|
||||
display: block;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
@keyframes toast-in {
|
||||
from { opacity: 0; transform: translateY(10px); }
|
||||
to { opacity: 1; transform: none; }
|
||||
@@ -2344,6 +2274,49 @@
|
||||
font-size: 10px;
|
||||
}
|
||||
|
||||
.admin-video-tag-option {
|
||||
max-width: 100%;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.admin-video-tag-option input {
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
.admin-video-tag-option__label {
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
word-break: keep-all;
|
||||
}
|
||||
|
||||
.admin-video-tag-option__count {
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
@media (max-width: 640px) {
|
||||
.admin-video-tag-picker {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 8px;
|
||||
align-items: stretch;
|
||||
}
|
||||
|
||||
.admin-video-tag-option {
|
||||
display: grid;
|
||||
grid-template-columns: 18px minmax(0, 1fr) auto;
|
||||
align-items: center;
|
||||
min-height: 44px;
|
||||
padding: 6px 10px;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.admin-video-tag-option__label {
|
||||
line-height: 1.2;
|
||||
}
|
||||
}
|
||||
|
||||
.admin-kv {
|
||||
display: grid;
|
||||
grid-template-columns: 110px 1fr;
|
||||
@@ -2542,9 +2515,9 @@
|
||||
min-height: 48px;
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
overflow-x: auto;
|
||||
overflow-x: hidden;
|
||||
overflow-y: hidden;
|
||||
padding: 6px var(--space-2);
|
||||
padding: 6px calc(var(--space-2) + 42px) 6px var(--space-2);
|
||||
background: var(--glass-nav);
|
||||
backdrop-filter: blur(16px);
|
||||
-webkit-backdrop-filter: blur(16px);
|
||||
@@ -2564,8 +2537,16 @@
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
margin-left: auto;
|
||||
flex-shrink: 0;
|
||||
position: absolute;
|
||||
top: 50%;
|
||||
right: var(--space-2);
|
||||
width: 36px;
|
||||
height: 36px;
|
||||
transform: translateY(-50%);
|
||||
z-index: 1;
|
||||
border-radius: var(--radius-pill);
|
||||
background: var(--glass-nav);
|
||||
box-shadow: -10px 0 14px var(--bg-page);
|
||||
}
|
||||
|
||||
.admin-sidebar__mobile-overlay {
|
||||
@@ -2598,16 +2579,22 @@
|
||||
}
|
||||
|
||||
.admin-nav {
|
||||
flex: 0 0 auto;
|
||||
flex: 1 1 auto;
|
||||
flex-direction: row;
|
||||
flex-wrap: nowrap;
|
||||
width: max-content;
|
||||
width: 100%;
|
||||
max-width: 100%;
|
||||
overflow-x: auto;
|
||||
overflow-y: hidden;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
gap: 6px;
|
||||
align-items: center;
|
||||
scrollbar-width: none;
|
||||
}
|
||||
|
||||
.admin-nav::-webkit-scrollbar { display: none; }
|
||||
|
||||
.admin-nav__group {
|
||||
flex: 0 0 auto;
|
||||
flex-direction: row;
|
||||
@@ -2848,12 +2835,18 @@
|
||||
}
|
||||
|
||||
.admin-modal-backdrop {
|
||||
align-items: stretch;
|
||||
padding: var(--space-2);
|
||||
place-items: center;
|
||||
padding:
|
||||
calc(var(--space-2) + env(safe-area-inset-top))
|
||||
var(--space-2)
|
||||
calc(var(--space-2) + env(safe-area-inset-bottom));
|
||||
}
|
||||
|
||||
.admin-modal {
|
||||
max-height: calc(100vh - 16px);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
max-height: calc(100dvh - 16px - env(safe-area-inset-top) - env(safe-area-inset-bottom));
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.admin-modal--delete-confirm {
|
||||
@@ -2867,6 +2860,12 @@
|
||||
padding: var(--space-4);
|
||||
}
|
||||
|
||||
.admin-modal__body {
|
||||
flex: 1 1 auto;
|
||||
min-height: 0;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.admin-modal__footer {
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
@@ -2883,7 +2882,8 @@
|
||||
}
|
||||
|
||||
.admin-toast {
|
||||
text-align: center;
|
||||
max-width: 100%;
|
||||
text-align: left;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2979,17 +2979,14 @@
|
||||
--admin-video-card-pill-bg: var(--bg-elevated);
|
||||
--admin-video-card-pill-border: var(--border-subtle);
|
||||
--admin-video-card-pill-text: var(--text-muted);
|
||||
--admin-video-card-category-bg: var(--accent-soft);
|
||||
--admin-video-card-category-border: var(--border-accent);
|
||||
--admin-video-card-category-text: var(--accent);
|
||||
--admin-video-card-button-bg: var(--bg-elevated);
|
||||
--admin-video-card-button-hover-bg: var(--bg-surface);
|
||||
--admin-video-card-button-text: var(--text-default);
|
||||
--admin-video-card-danger: var(--danger);
|
||||
--admin-video-card-danger-border: rgba(241, 85, 108, 0.4);
|
||||
position: relative;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 0;
|
||||
grid-template-columns: repeat(12, minmax(0, 1fr));
|
||||
gap: 0 10px;
|
||||
padding: 12px 14px;
|
||||
background: var(--admin-video-card-bg);
|
||||
border: 1px solid var(--admin-video-card-border);
|
||||
@@ -3013,9 +3010,6 @@
|
||||
--admin-video-card-pill-bg: rgba(255, 255, 255, 0.06);
|
||||
--admin-video-card-pill-border: rgba(255, 255, 255, 0.05);
|
||||
--admin-video-card-pill-text: rgba(255, 255, 255, 0.62);
|
||||
--admin-video-card-category-bg: rgba(255, 255, 255, 0.11);
|
||||
--admin-video-card-category-border: rgba(255, 255, 255, 0.14);
|
||||
--admin-video-card-category-text: #f0f0f0;
|
||||
--admin-video-card-button-bg: rgba(255, 255, 255, 0.06);
|
||||
--admin-video-card-button-hover-bg: rgba(255, 255, 255, 0.1);
|
||||
--admin-video-card-button-text: #e0e0e0;
|
||||
@@ -3050,13 +3044,14 @@
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) td.is-checkbox {
|
||||
position: absolute;
|
||||
top: 12px;
|
||||
left: 14px;
|
||||
z-index: 1;
|
||||
grid-column: 1 / 4;
|
||||
grid-row: 3;
|
||||
display: flex;
|
||||
width: 28px;
|
||||
height: 28px;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
margin-top: 8px;
|
||||
padding-top: 8px;
|
||||
border-top: 1px solid var(--admin-video-card-line);
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) td.is-checkbox::before,
|
||||
@@ -3066,10 +3061,22 @@
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) .admin-table-checkbox-btn {
|
||||
width: 100%;
|
||||
min-width: 0;
|
||||
height: 32px;
|
||||
min-height: 32px;
|
||||
border: 1px solid transparent;
|
||||
border-radius: 8px;
|
||||
background: var(--admin-video-card-button-bg);
|
||||
box-shadow: none;
|
||||
color: var(--admin-video-card-muted);
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) .admin-table-checkbox-btn svg {
|
||||
color: var(--admin-video-card-button-text);
|
||||
stroke: currentColor;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) .admin-table-checkbox-btn:hover,
|
||||
.admin-videos-table:not(.admin-drives-table) .admin-table-checkbox-btn:focus-visible {
|
||||
background: var(--admin-video-card-button-hover-bg);
|
||||
@@ -3079,28 +3086,48 @@
|
||||
.admin-videos-table:not(.admin-drives-table) td[data-label="标题"] {
|
||||
grid-column: 1 / -1;
|
||||
grid-row: 1;
|
||||
min-height: 28px;
|
||||
padding-left: 36px;
|
||||
gap: 6px;
|
||||
align-content: center;
|
||||
min-height: 72px;
|
||||
padding-left: 0;
|
||||
gap: 8px;
|
||||
align-content: stretch;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) .admin-video-title-cell {
|
||||
gap: 8px;
|
||||
display: grid;
|
||||
grid-template-columns: clamp(104px, 32vw, 156px) minmax(0, 1fr);
|
||||
align-items: stretch;
|
||||
gap: 10px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) .admin-video-thumb-wrap {
|
||||
flex: 0 0 54px;
|
||||
width: 54px;
|
||||
height: 34px;
|
||||
border-radius: 4px;
|
||||
width: 100%;
|
||||
height: auto;
|
||||
aspect-ratio: 16 / 9;
|
||||
border: 1px solid var(--admin-video-card-line);
|
||||
border-radius: 8px;
|
||||
background: var(--bg-sunken);
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) .admin-video-title-body {
|
||||
display: flex;
|
||||
min-width: 0;
|
||||
flex-direction: column;
|
||||
justify-content: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) .admin-video-title {
|
||||
display: -webkit-box;
|
||||
overflow: hidden;
|
||||
-webkit-box-orient: vertical;
|
||||
-webkit-line-clamp: 2;
|
||||
color: var(--admin-video-card-main);
|
||||
font-size: 14px;
|
||||
font-weight: var(--weight-bold);
|
||||
line-height: 1.35;
|
||||
overflow-wrap: anywhere;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) .admin-video-filemeta {
|
||||
@@ -3132,48 +3159,48 @@
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) .admin-video-filemeta-pill.is-category {
|
||||
border-color: var(--admin-video-card-category-border);
|
||||
background: var(--admin-video-card-category-bg);
|
||||
color: var(--admin-video-card-category-text);
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) td[data-label="作者"],
|
||||
.admin-videos-table:not(.admin-drives-table) td[data-label="来源"],
|
||||
.admin-videos-table:not(.admin-drives-table) td[data-label="时长"],
|
||||
.admin-videos-table:not(.admin-drives-table) td[data-label="预览视频"] {
|
||||
grid-row: 2;
|
||||
min-width: 0;
|
||||
margin-top: 8px;
|
||||
padding-top: 8px;
|
||||
border-top: 1px solid var(--admin-video-card-line);
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) td[data-label="作者"],
|
||||
.admin-videos-table:not(.admin-drives-table) td[data-label="时长"] {
|
||||
grid-column: 1;
|
||||
padding-right: 12px;
|
||||
.admin-videos-table:not(.admin-drives-table) td[data-label="作者"] {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) td[data-label="来源"],
|
||||
.admin-videos-table:not(.admin-drives-table) td[data-label="预览视频"] {
|
||||
grid-column: 2;
|
||||
padding-left: 12px;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) td[data-label="作者"],
|
||||
.admin-videos-table:not(.admin-drives-table) td[data-label="来源"] {
|
||||
grid-row: 2;
|
||||
grid-column: 1 / 5;
|
||||
justify-items: start;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) td[data-label="时长"] {
|
||||
grid-column: 5 / 9;
|
||||
justify-items: center;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) td[data-label="时长"],
|
||||
.admin-videos-table:not(.admin-drives-table) td[data-label="预览视频"] {
|
||||
grid-row: 3;
|
||||
grid-column: 9 / -1;
|
||||
justify-items: end;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) .admin-mono-cell {
|
||||
color: var(--admin-video-card-text);
|
||||
font-family: inherit;
|
||||
font-size: 12px;
|
||||
word-break: break-word;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
word-break: normal;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) .admin-text-faint {
|
||||
@@ -3183,12 +3210,18 @@
|
||||
.admin-videos-table:not(.admin-drives-table) .admin-status {
|
||||
width: max-content;
|
||||
min-height: 22px;
|
||||
gap: 0;
|
||||
padding: 3px 8px;
|
||||
border-radius: var(--radius-pill);
|
||||
font-size: 11px;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) .admin-status::before {
|
||||
content: none;
|
||||
display: none;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) .admin-status.is-ok {
|
||||
background: var(--success-soft);
|
||||
color: var(--success);
|
||||
@@ -3200,28 +3233,29 @@
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) td.is-actions {
|
||||
grid-column: 1 / -1;
|
||||
grid-row: 4;
|
||||
display: flex;
|
||||
flex-wrap: nowrap;
|
||||
grid-column: 4 / -1;
|
||||
grid-row: 3;
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
gap: 6px;
|
||||
gap: 10px;
|
||||
margin-top: 8px;
|
||||
padding-top: 8px;
|
||||
border-top: 1px solid var(--admin-video-card-line);
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) td.is-actions::before {
|
||||
flex: 0 0 auto;
|
||||
margin-right: auto;
|
||||
content: none;
|
||||
display: none;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) td.is-actions .admin-btn {
|
||||
min-width: 28px;
|
||||
height: 28px;
|
||||
min-height: 28px;
|
||||
padding: 0 9px;
|
||||
justify-content: center;
|
||||
width: 100%;
|
||||
min-width: 0;
|
||||
height: 32px;
|
||||
min-height: 32px;
|
||||
padding: 0;
|
||||
border-color: transparent;
|
||||
border-radius: 8px;
|
||||
background: var(--admin-video-card-button-bg);
|
||||
@@ -3235,11 +3269,6 @@
|
||||
margin-left: 0;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) td.is-actions .admin-btn:not(:first-of-type) {
|
||||
width: 28px;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.admin-videos-table:not(.admin-drives-table) td.is-actions .admin-btn:hover:not(:disabled) {
|
||||
background: var(--admin-video-card-button-hover-bg);
|
||||
border-color: transparent;
|
||||
@@ -3282,6 +3311,43 @@
|
||||
|
||||
.admin-videos-filter {
|
||||
flex-wrap: wrap;
|
||||
margin-bottom: var(--space-4);
|
||||
}
|
||||
|
||||
.admin-videos-filter--current,
|
||||
.admin-videos-filter--blacklist {
|
||||
display: grid;
|
||||
grid-template-columns: 72px minmax(0, 1fr) auto;
|
||||
align-items: center;
|
||||
width: 100%;
|
||||
flex-wrap: nowrap;
|
||||
gap: var(--space-2);
|
||||
}
|
||||
|
||||
.admin-videos-filter--current .admin-videos-filter__select-wrap,
|
||||
.admin-videos-filter--blacklist .admin-videos-filter__select-wrap,
|
||||
.admin-videos-filter--current .admin-videos-filter__search,
|
||||
.admin-videos-filter--blacklist .admin-videos-filter__search {
|
||||
width: 100%;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.admin-videos-filter__refresh {
|
||||
justify-content: center;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-videos-filter--current .admin-videos-filter__select,
|
||||
.admin-videos-filter--blacklist .admin-videos-filter__select {
|
||||
padding: 0 4px;
|
||||
text-align: center;
|
||||
text-align-last: center;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.admin-videos-filter--current .admin-videos-filter__select-icon,
|
||||
.admin-videos-filter--blacklist .admin-videos-filter__select-icon {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.admin-videos-filter__select-wrap {
|
||||
@@ -3446,6 +3512,29 @@
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.admin-blacklist-filecell {
|
||||
display: flex;
|
||||
min-width: 0;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.admin-blacklist-reason-pill {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
min-height: 22px;
|
||||
padding: 2px 8px;
|
||||
border: 1px solid var(--warning-border, rgba(205, 132, 38, 0.38));
|
||||
border-radius: 999px;
|
||||
background: var(--warning-soft, rgba(245, 158, 11, 0.12));
|
||||
color: var(--warning-strong, #9a5b00);
|
||||
font-size: var(--font-xs);
|
||||
font-weight: 700;
|
||||
line-height: 1.2;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-blacklist-restore-btn {
|
||||
border-color: var(--border-accent);
|
||||
background: var(--accent-softer);
|
||||
@@ -3468,6 +3557,26 @@
|
||||
margin: var(--space-2) 0 var(--space-4);
|
||||
}
|
||||
|
||||
.admin-videos-current .admin-videos-list-toolbar {
|
||||
position: fixed;
|
||||
left: auto;
|
||||
right: var(--space-7);
|
||||
bottom: var(--space-5);
|
||||
z-index: calc(var(--z-nav) + 3);
|
||||
width: max-content;
|
||||
max-width: calc(100vw - 288px - (var(--space-7) * 2));
|
||||
margin: 0;
|
||||
padding: 10px 12px;
|
||||
border: 1px solid var(--border-subtle);
|
||||
border-radius: 12px;
|
||||
background: var(--bg-surface);
|
||||
box-shadow: 0 12px 32px rgba(0, 0, 0, 0.34);
|
||||
}
|
||||
|
||||
.admin-videos-current.has-bulk-actions {
|
||||
padding-bottom: 72px;
|
||||
}
|
||||
|
||||
.admin-videos-summary {
|
||||
font-size: var(--font-xs);
|
||||
color: var(--text-faint);
|
||||
@@ -3558,6 +3667,61 @@
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.admin-videos-filter--blacklist {
|
||||
display: grid;
|
||||
grid-template-columns: 72px minmax(0, 1fr) auto;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.admin-videos-filter--current {
|
||||
display: grid;
|
||||
grid-template-columns: 72px minmax(0, 1fr) 38px;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.admin-videos-filter--blacklist .admin-videos-filter__select-wrap,
|
||||
.admin-videos-filter--blacklist .admin-videos-filter__search,
|
||||
.admin-videos-filter--current .admin-videos-filter__select-wrap,
|
||||
.admin-videos-filter--current .admin-videos-filter__search {
|
||||
width: 100%;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.admin-videos-filter--blacklist .admin-videos-filter__select,
|
||||
.admin-videos-filter--current .admin-videos-filter__select {
|
||||
padding: 0 4px;
|
||||
text-align: center;
|
||||
text-align-last: center;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.admin-videos-filter--blacklist .admin-videos-filter__select-icon,
|
||||
.admin-videos-filter--current .admin-videos-filter__select-icon {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.admin-videos-filter--current .admin-videos-filter__refresh {
|
||||
justify-content: center;
|
||||
width: 38px;
|
||||
min-width: 38px;
|
||||
height: 38px;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.admin-videos-filter--current .admin-videos-filter__refresh-text {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.admin-videos-filter--blacklist .admin-btn {
|
||||
width: auto;
|
||||
min-width: 54px;
|
||||
height: 38px;
|
||||
padding: 0 10px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-table-pagination {
|
||||
justify-content: center;
|
||||
}
|
||||
@@ -3568,24 +3732,38 @@
|
||||
margin: 0 0 2px;
|
||||
}
|
||||
|
||||
.admin-videos-list-toolbar {
|
||||
.admin-videos-current .admin-videos-list-toolbar {
|
||||
align-items: stretch;
|
||||
flex-direction: column;
|
||||
position: fixed;
|
||||
left: var(--space-3);
|
||||
right: var(--space-3);
|
||||
bottom: calc(var(--space-3) + env(safe-area-inset-bottom));
|
||||
width: auto;
|
||||
max-width: none;
|
||||
margin: 0;
|
||||
padding: 10px;
|
||||
}
|
||||
|
||||
.admin-videos-current.has-bulk-actions {
|
||||
padding-bottom: calc(104px + env(safe-area-inset-bottom));
|
||||
}
|
||||
|
||||
.admin-videos-bulk-actions {
|
||||
flex-wrap: wrap;
|
||||
justify-content: flex-start;
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
align-items: center;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.admin-videos-bulk-actions__count {
|
||||
flex: 1 0 100%;
|
||||
grid-column: 1 / -1;
|
||||
}
|
||||
|
||||
.admin-videos-bulk-actions__btn {
|
||||
flex: 1 1 136px;
|
||||
justify-content: center;
|
||||
width: 100%;
|
||||
min-height: 40px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
@@ -3681,7 +3859,6 @@
|
||||
.admin-drive-type-card[data-kind="onedrive"]:hover { border-color: var(--drive-onedrive); box-shadow: 0 4px 18px rgba(76,171,234,.2); }
|
||||
.admin-drive-type-card[data-kind="googledrive"]:hover { border-color: #4285f4; box-shadow: 0 4px 18px rgba(66,133,244,.2); }
|
||||
.admin-drive-type-card[data-kind="localstorage"]:hover { border-color: var(--drive-localstorage); box-shadow: 0 4px 18px rgba(53,184,143,.2); }
|
||||
.admin-drive-type-card[data-kind="spider91"]:hover { border-color: var(--accent); box-shadow: 0 4px 18px var(--accent-glow); }
|
||||
.admin-drive-type-card[data-kind="quark"]:hover { border-color: var(--drive-quark); box-shadow: 0 4px 18px rgba(91,141,239,.2); }
|
||||
.admin-drive-type-card[data-kind="wopan"]:hover { border-color: var(--drive-wopan); box-shadow: 0 4px 18px rgba(255,138,60,.2); }
|
||||
.admin-drive-type-card[data-kind="guangyapan"]:hover { border-color: var(--drive-guangyapan); box-shadow: 0 4px 18px rgba(48,195,168,.2); }
|
||||
@@ -3707,7 +3884,6 @@
|
||||
.admin-drive-type-card__icon[data-kind="onedrive"] { background: rgba(76,171,234,.14); color: var(--drive-onedrive); }
|
||||
.admin-drive-type-card__icon[data-kind="googledrive"] { background: rgba(66,133,244,.14); color: #4285f4; }
|
||||
.admin-drive-type-card__icon[data-kind="localstorage"]{ background: rgba(53,184,143,.14); color: var(--drive-localstorage); }
|
||||
.admin-drive-type-card__icon[data-kind="spider91"] { background: var(--accent-soft); color: var(--accent); }
|
||||
.admin-drive-type-card__icon[data-kind="quark"] { background: rgba(91,141,239,.14); color: var(--drive-quark); }
|
||||
.admin-drive-type-card__icon[data-kind="wopan"] { background: rgba(255,138,60,.14); color: var(--drive-wopan); }
|
||||
.admin-drive-type-card__icon[data-kind="guangyapan"] { background: rgba(48,195,168,.14); color: var(--drive-guangyapan); }
|
||||
@@ -3727,7 +3903,29 @@
|
||||
|
||||
@media (max-width: 520px) {
|
||||
.admin-drive-type-grid {
|
||||
grid-template-columns: repeat(2, 1fr);
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: var(--space-2);
|
||||
}
|
||||
|
||||
.admin-drive-type-card {
|
||||
min-height: 94px;
|
||||
gap: 6px;
|
||||
padding: 10px 6px 8px;
|
||||
border-radius: var(--radius-sm);
|
||||
}
|
||||
|
||||
.admin-drive-type-card__icon {
|
||||
width: 38px;
|
||||
height: 38px;
|
||||
border-radius: var(--radius-sm);
|
||||
font-size: var(--font-sm);
|
||||
letter-spacing: 0;
|
||||
}
|
||||
|
||||
.admin-drive-type-card__label {
|
||||
font-size: var(--font-sm);
|
||||
line-height: 1.2;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3766,7 +3964,6 @@
|
||||
.admin-drive-selected-bar__icon[data-kind="onedrive"] { background: rgba(76,171,234,.14); color: var(--drive-onedrive); }
|
||||
.admin-drive-selected-bar__icon[data-kind="googledrive"] { background: rgba(66,133,244,.14); color: #4285f4; }
|
||||
.admin-drive-selected-bar__icon[data-kind="localstorage"]{ background: rgba(53,184,143,.14); color: var(--drive-localstorage); }
|
||||
.admin-drive-selected-bar__icon[data-kind="spider91"] { background: var(--accent-soft); color: var(--accent); }
|
||||
.admin-drive-selected-bar__icon[data-kind="quark"] { background: rgba(91,141,239,.14); color: var(--drive-quark); }
|
||||
.admin-drive-selected-bar__icon[data-kind="wopan"] { background: rgba(255,138,60,.14); color: var(--drive-wopan); }
|
||||
.admin-drive-selected-bar__icon[data-kind="guangyapan"] { background: rgba(48,195,168,.14); color: var(--drive-guangyapan); }
|
||||
@@ -4358,7 +4555,6 @@
|
||||
.admin-drive-card__brand-icon[data-kind="onedrive"] { background: var(--drive-onedrive); }
|
||||
.admin-drive-card__brand-icon[data-kind="googledrive"] { background: #4285f4; }
|
||||
.admin-drive-card__brand-icon[data-kind="localstorage"] { background: var(--drive-localstorage); }
|
||||
.admin-drive-card__brand-icon[data-kind="spider91"] { background: var(--accent); }
|
||||
|
||||
.admin-drive-card__info {
|
||||
display: grid;
|
||||
@@ -4721,6 +4917,10 @@
|
||||
align-items: start;
|
||||
}
|
||||
|
||||
.admin-tags-layout > * {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.admin-crawler-layout,
|
||||
.admin-tags-layout {
|
||||
@@ -4764,6 +4964,7 @@
|
||||
gap: var(--space-3);
|
||||
margin-bottom: var(--space-4);
|
||||
flex-wrap: wrap;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.admin-tags-search {
|
||||
@@ -4838,6 +5039,7 @@
|
||||
grid-template-columns: repeat(auto-fill, minmax(240px, 1fr));
|
||||
gap: var(--space-3);
|
||||
align-items: stretch;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.admin-tags-bulkbar {
|
||||
@@ -4884,6 +5086,7 @@
|
||||
.admin-tag-card {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
min-width: 0;
|
||||
background: var(--bg-surface);
|
||||
border: 1px solid var(--border-subtle);
|
||||
border-radius: var(--radius-sm);
|
||||
@@ -4896,6 +5099,57 @@
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
@media (max-width: 640px) {
|
||||
.admin-tags-layout {
|
||||
width: 100%;
|
||||
max-width: 100%;
|
||||
overflow-x: clip;
|
||||
}
|
||||
|
||||
.admin-tags-toolbar {
|
||||
align-items: stretch;
|
||||
gap: var(--space-2);
|
||||
width: 100%;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
.admin-tags-search {
|
||||
flex: 1 1 100%;
|
||||
min-width: 0;
|
||||
max-width: none;
|
||||
}
|
||||
|
||||
.admin-tags-filter-tabs {
|
||||
flex: 1 1 100%;
|
||||
width: 100%;
|
||||
min-width: 0;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
.admin-tags-bulkbar,
|
||||
.admin-tags-pagination {
|
||||
width: 100%;
|
||||
min-width: 0;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
.admin-tags-pagination .admin-table-pagination__info {
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.admin-tags-grid {
|
||||
width: 100%;
|
||||
max-width: 100%;
|
||||
grid-template-columns: minmax(0, 1fr);
|
||||
justify-items: stretch;
|
||||
}
|
||||
|
||||
.admin-tag-card {
|
||||
width: 100%;
|
||||
max-width: 100%;
|
||||
}
|
||||
}
|
||||
|
||||
.admin-tag-card:hover {
|
||||
border-color: var(--border-accent);
|
||||
box-shadow: var(--shadow-md);
|
||||
@@ -5058,8 +5312,6 @@
|
||||
.admin-empty-state { display: flex; flex-direction: column; align-items: center; justify-content: center; padding: 64px 16px; color: var(--text-faint); text-align: center; border: 1px dashed var(--border-default); border-radius: 8px; }
|
||||
.admin-empty-state__icon { color: var(--border-default); margin-bottom: 16px; }
|
||||
.admin-empty-state h3 { margin: 0 0 8px 0; color: var(--text-strong); font-size: 16px; }
|
||||
.admin-thumbnail-preview { display: flex; flex-direction: column; gap: 8px; }
|
||||
.admin-thumbnail-img { max-width: 200px; max-height: 120px; object-fit: contain; border-radius: 4px; border: 1px solid var(--border-default); background: var(--bg-sunken); }
|
||||
.admin-table { overflow: visible !important; border-collapse: separate !important; border-spacing: 0 !important; }
|
||||
.admin-table th { position: sticky; top: 0; background: var(--bg-elevated) !important; z-index: 10; }
|
||||
@media (hover: hover) and (pointer: fine) {
|
||||
|
||||
+22
-10
@@ -417,28 +417,40 @@
|
||||
|
||||
.video-meta {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 2px var(--space-2);
|
||||
align-items: center;
|
||||
flex-wrap: nowrap;
|
||||
gap: 0;
|
||||
margin-top: 4px;
|
||||
padding: 0 4px;
|
||||
font-size: var(--font-xs);
|
||||
color: var(--text-faint);
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.video-meta__author {
|
||||
color: var(--text-muted);
|
||||
font-weight: var(--weight-medium);
|
||||
.video-meta > span {
|
||||
min-width: 0;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.video-meta__author::after {
|
||||
.video-meta > span + span::before {
|
||||
content: "·";
|
||||
color: var(--text-disabled);
|
||||
margin: 0 6px;
|
||||
}
|
||||
|
||||
.video-meta > span:last-child::after {
|
||||
content: "";
|
||||
margin: 0;
|
||||
.video-meta__author {
|
||||
flex: 0 1 auto;
|
||||
max-width: min(100%, 18em);
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
color: var(--text-muted);
|
||||
font-weight: var(--weight-medium);
|
||||
}
|
||||
|
||||
.video-meta__views,
|
||||
.video-meta__date {
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
/* 卡片底部源信息(compact 视图用) */
|
||||
@@ -647,7 +659,7 @@
|
||||
}
|
||||
|
||||
/* 移动端隐藏发布时间,保持卡片更紧凑 */
|
||||
.video-meta span:nth-child(n + 3) {
|
||||
.video-meta__date {
|
||||
display: none;
|
||||
}
|
||||
|
||||
|
||||
+2
-9
@@ -19,7 +19,6 @@ export type VideoItem = {
|
||||
publishedAt: string;
|
||||
rating?: number;
|
||||
tags?: string[];
|
||||
category?: string;
|
||||
};
|
||||
|
||||
export type AuthorProfile = {
|
||||
@@ -57,7 +56,7 @@ export type VideoDetail = VideoItem & {
|
||||
|
||||
export type PreviewState = "idle" | "intent" | "loading" | "playing" | "error";
|
||||
|
||||
export type SortKey = "latest" | "hot" | "week" | "long" | "hd" | "featured";
|
||||
export type SortKey = "latest" | "hot" | "recent";
|
||||
|
||||
export type TagItem = {
|
||||
id: string;
|
||||
@@ -65,15 +64,9 @@ export type TagItem = {
|
||||
count?: number;
|
||||
};
|
||||
|
||||
export type CategoryItem = {
|
||||
id: string;
|
||||
label: string;
|
||||
href: string;
|
||||
};
|
||||
|
||||
export type PromoItem = {
|
||||
id: string;
|
||||
kind: "channel" | "collection" | "event";
|
||||
kind: "channel" | "topic" | "event";
|
||||
label: string;
|
||||
title: string;
|
||||
meta?: string;
|
||||
|
||||
@@ -22,8 +22,8 @@ const appSource = readFileSync(
|
||||
new URL("../src/App.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
const spider91UploadTargetSource = readFileSync(
|
||||
new URL("../src/admin/drive/Spider91UploadTargetField.tsx", import.meta.url),
|
||||
const crawlerUploadTargetSource = readFileSync(
|
||||
new URL("../src/admin/drive/CrawlerUploadTargetField.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
const driveFormSource = readFileSync(
|
||||
@@ -43,7 +43,7 @@ const constantsSource = readFileSync(
|
||||
"utf8"
|
||||
);
|
||||
|
||||
const combinedSource = drivesPageSource + "\n" + driveFormSource + "\n" + constantsSource + "\n" + spider91UploadTargetSource;
|
||||
const combinedSource = drivesPageSource + "\n" + driveFormSource + "\n" + constantsSource + "\n" + crawlerUploadTargetSource;
|
||||
|
||||
function driveTypeOptions() {
|
||||
const match = /const DRIVE_OPTIONS:\s*DriveOption\[]\s*=\s*\[([\s\S]*?)\];/.exec(
|
||||
@@ -74,22 +74,22 @@ test("crawler sources are not selectable as storage drives", () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("spider91 upload target uses explicit local-save option instead of auto target", () => {
|
||||
test("crawler upload target uses explicit local-save option instead of auto target", () => {
|
||||
assert.match(combinedSource, /本地保存,不上传/);
|
||||
assert.match(
|
||||
combinedSource,
|
||||
/d\.kind === "pikpak"[\s\S]*d\.kind === "p115"[\s\S]*d\.kind === "p123"[\s\S]*d\.kind === "onedrive"[\s\S]*d\.kind === "googledrive"[\s\S]*d\.kind === "wopan"[\s\S]*d\.kind === "guangyapan"/
|
||||
crawlerPageSource,
|
||||
/UPLOAD_TARGET_KINDS\s*=\s*new Set\(\["p115", "pikpak", "p123", "googledrive", "onedrive", "wopan", "guangyapan"\]\)/
|
||||
);
|
||||
assert.match(crawlerPageSource, /UPLOAD_TARGET_KINDS[\s\S]*"wopan"[\s\S]*"guangyapan"/);
|
||||
assert.match(crawlerPageSource, /drives\.filter\(\(d\) => UPLOAD_TARGET_KINDS\.has\(d\.kind\)\)/);
|
||||
assert.doesNotMatch(combinedSource, /自动:唯一/);
|
||||
assert.doesNotMatch(combinedSource, /自动模式/);
|
||||
assert.doesNotMatch(combinedSource, /较早的视频会上传到该云盘根目录下的 91 Spider 文件夹/);
|
||||
assert.doesNotMatch(combinedSource, /较早的视频会上传到该云盘根目录下/);
|
||||
});
|
||||
|
||||
test("spider91 upload target select uses an aligned custom arrow", () => {
|
||||
assert.match(spider91UploadTargetSource, /className="admin-form-select-wrap"/);
|
||||
assert.match(spider91UploadTargetSource, /className="admin-form-select"/);
|
||||
assert.match(spider91UploadTargetSource, /className="admin-form-select__icon"/);
|
||||
test("crawler upload target select uses an aligned custom arrow", () => {
|
||||
assert.match(crawlerUploadTargetSource, /className="admin-form-select-wrap"/);
|
||||
assert.match(crawlerUploadTargetSource, /className="admin-form-select"/);
|
||||
assert.match(crawlerUploadTargetSource, /className="admin-form-select__icon"/);
|
||||
assert.match(adminCss, /\.admin-form__row \.admin-form-select\s*\{[^}]*appearance\s*:\s*none/s);
|
||||
assert.match(
|
||||
adminCss,
|
||||
@@ -97,11 +97,11 @@ test("spider91 upload target select uses an aligned custom arrow", () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("drive form hides root directory id for localstorage and spider91", () => {
|
||||
test("drive form hides root directory id for localstorage", () => {
|
||||
assert.match(combinedSource, /<label[^>]*>根目录 ID<\/label>/);
|
||||
assert.match(
|
||||
combinedSource,
|
||||
/usesRootDirectoryID\(kind:\s*Kind\):\s*boolean\s*\{\s*return kind !== "localstorage" && kind !== "spider91";\s*\}/
|
||||
/usesRootDirectoryID\(kind:\s*Kind\):\s*boolean\s*\{\s*return kind !== "localstorage";\s*\}/
|
||||
);
|
||||
assert.match(combinedSource, /\{usesRootDirectoryID\(form\.kind\) && \(/);
|
||||
assert.match(combinedSource, /\{usesRootDirectoryID\(d\.kind\) && \(/);
|
||||
@@ -145,7 +145,8 @@ test("googledrive drive form supports online API and custom OAuth client modes",
|
||||
assert.match(fields, /key: "client_id"/);
|
||||
assert.match(fields, /key: "client_secret"/);
|
||||
assert.match(fields, /googleDriveUsesOnlineAPI\(creds\)/);
|
||||
assert.doesNotMatch(fields, /key: "api_url_address"/);
|
||||
assert.match(fields, /key: "api_url_address"/);
|
||||
assert.match(fields, /OpenList 在线 API URL/);
|
||||
assert.doesNotMatch(fields, /在线 API 模式填写 OpenList 获取的 refresh_token/);
|
||||
assert.doesNotMatch(constantsSource, /请参考OpenList文档中关于谷歌云盘的配置方法。/);
|
||||
assert.doesNotMatch(constantsSource, /选择自建 Google OAuth 客户端后,服务端会直接请求 Google OAuth token 接口续期。/);
|
||||
@@ -154,7 +155,9 @@ test("googledrive drive form supports online API and custom OAuth client modes",
|
||||
assert.match(driveFormSource, /className="admin-form-select"/);
|
||||
assert.match(driveFormSource, /ChevronDown/);
|
||||
assert.match(drivesPageSource, /googleDriveUseOnlineAPI/);
|
||||
assert.match(drivesPageSource, /googleDriveOpenListApiUrl/);
|
||||
assert.match(apiSource, /googleDriveUseOnlineAPI\?: boolean/);
|
||||
assert.match(apiSource, /googleDriveOpenListApiUrl\?: string/);
|
||||
assert.doesNotMatch(fields, /key: "access_token"/);
|
||||
});
|
||||
|
||||
@@ -205,7 +208,7 @@ test("localstorage drive form asks for a server directory path", () => {
|
||||
assertDriveTypeOption("localstorage", "本地存储");
|
||||
|
||||
const match =
|
||||
/case "localstorage":\s*return \[([\s\S]*?)\];\s*case "spider91":/.exec(
|
||||
/case "localstorage":\s*return \[([\s\S]*?)\];\s*\}\s*\}/.exec(
|
||||
combinedSource
|
||||
);
|
||||
assert.ok(match, "localstorage credential field block should be present");
|
||||
@@ -214,7 +217,8 @@ test("localstorage drive form asks for a server directory path", () => {
|
||||
assert.match(fields, /key: "path"/);
|
||||
assert.match(fields, /label: "本地目录路径"/);
|
||||
assert.match(combinedSource, /if \(kind === "localstorage"\) return "\/"/);
|
||||
assert.match(combinedSource, /kind !== "localstorage" && kind !== "spider91"/);
|
||||
assert.match(combinedSource, /kind !== "localstorage"/);
|
||||
assert.doesNotMatch(combinedSource, /spider91/);
|
||||
});
|
||||
|
||||
test("drive type selector keeps primary source order", () => {
|
||||
@@ -235,7 +239,10 @@ test("crawler management is a separate admin section", () => {
|
||||
assert.match(adminLayoutSource, /to="\/admin\/crawlers"/);
|
||||
assert.match(adminLayoutSource, /admin-nav__title">爬虫管理/);
|
||||
assert.match(adminLayoutSource, /admin-nav__icon"><SpiderIcon size=\{16\} \/>/);
|
||||
assert.match(appSource, /path="crawlers" element=\{<CrawlersPage \/>/);
|
||||
assert.match(
|
||||
appSource,
|
||||
/path="crawlers"[\s\S]*<PageSuspense>[\s\S]*<CrawlersPage \/>[\s\S]*<\/PageSuspense>/
|
||||
);
|
||||
assert.match(crawlerPageSource, /export function CrawlersPage/);
|
||||
assert.match(crawlerPageSource, /SpiderIcon/);
|
||||
assert.match(crawlerPageSource, /添加爬虫/);
|
||||
@@ -248,6 +255,7 @@ test("crawler management is a separate admin section", () => {
|
||||
assert.match(crawlerPageSource, /api\.listDrives/);
|
||||
assert.match(crawlerPageSource, /api\.upsertCrawler/);
|
||||
assert.match(crawlerPageSource, /api\.runCrawler/);
|
||||
assert.match(crawlerPageSource, /api\.uploadCrawlerVideos/);
|
||||
assert.match(crawlerPageSource, /api\.stopCrawlerTasks/);
|
||||
assert.match(crawlerPageSource, /api\.deleteCrawler/);
|
||||
assert.match(crawlerPageSource, /api\.importCrawlerScriptFile/);
|
||||
@@ -257,13 +265,21 @@ test("crawler management is a separate admin section", () => {
|
||||
assert.match(crawlerPageSource, /链接导入/);
|
||||
assert.match(crawlerPageSource, /测试脚本/);
|
||||
assert.match(crawlerPageSource, /测试通过/);
|
||||
assert.match(crawlerPageSource, /Spider91UploadTargetField/);
|
||||
assert.match(crawlerPageSource, /CrawlerUploadTargetField/);
|
||||
assert.match(crawlerPageSource, /uploadDriveId/);
|
||||
assert.match(crawlerPageSource, /api\.setDriveTeaserEnabled/);
|
||||
assert.match(crawlerPageSource, /admin-crawler-preview-card-toggle/);
|
||||
assert.match(crawlerPageSource, /预览:开/);
|
||||
assert.match(crawlerPageSource, /预览:关/);
|
||||
assert.match(crawlerPageSource, /上传视频/);
|
||||
assert.match(crawlerPageSource, /aria-pressed=\{crawler\.teaserEnabled\}/);
|
||||
assert.doesNotMatch(crawlerPageSource, /crawlerUploadBlockedReason/);
|
||||
assert.doesNotMatch(crawlerPageSource, /disabled=\{uploading/);
|
||||
assert.doesNotMatch(crawlerPageSource, /crawlerStatusLabel/);
|
||||
assert.doesNotMatch(crawlerPageSource, /admin-crawler-preview-card-toggle \$\{crawler\.teaserEnabled/);
|
||||
assert.doesNotMatch(adminCss, /admin-crawler-preview-card-toggle\.is-on/);
|
||||
assert.doesNotMatch(crawlerPageSource, /admin-crawler-pipeline/);
|
||||
assert.doesNotMatch(adminCss, /admin-crawler-(pipeline|stage)/);
|
||||
assert.doesNotMatch(crawlerPageSource, /teaserEnabled: form\.teaserEnabled/);
|
||||
assert.doesNotMatch(crawlerPageSource, /aria-pressed=\{form\.teaserEnabled\}/);
|
||||
assert.match(crawlerPageSource, /UPLOAD_TARGET_KINDS/);
|
||||
@@ -284,6 +300,7 @@ test("crawler management is a separate admin section", () => {
|
||||
assert.match(apiSource, /teaserEnabled: boolean/);
|
||||
assert.doesNotMatch(apiSource, /teaserEnabled\?: boolean/);
|
||||
assert.match(apiSource, /"\/crawlers"/);
|
||||
assert.match(apiSource, /\/crawlers\/\$\{encodeURIComponent\(id\)\}\/upload/);
|
||||
assert.match(apiSource, /"\/crawlers\/import-file"/);
|
||||
assert.match(apiSource, /"\/crawlers\/import-url"/);
|
||||
assert.match(apiSource, /"\/crawlers\/test-script"/);
|
||||
@@ -293,6 +310,18 @@ test("crawler management is a separate admin section", () => {
|
||||
assert.doesNotMatch(driveFormSource, /scriptcrawler/);
|
||||
});
|
||||
|
||||
test("admin shell stays mounted while lazy admin pages load", () => {
|
||||
assert.match(appSource, /import \{ AdminLayout \} from "@\/admin\/AdminLayout";/);
|
||||
assert.doesNotMatch(appSource, /const AdminLayout\s*=\s*lazy/);
|
||||
assert.doesNotMatch(appSource, /<Suspense fallback=\{null\}>\s*<Routes>/);
|
||||
assert.match(appSource, /function PageSuspense\(\{ children \}: \{ children: ReactNode \}\)/);
|
||||
assert.match(appSource, /path="\/admin"[\s\S]*<AdminLayout \/>/);
|
||||
assert.match(
|
||||
appSource,
|
||||
/path="drives"[\s\S]*<PageSuspense>[\s\S]*<DrivesPage \/>[\s\S]*<\/PageSuspense>/
|
||||
);
|
||||
});
|
||||
|
||||
test("drive cards use configured abbreviations and visible fallback icon colors", () => {
|
||||
assert.match(constantsSource, /googledrive:\s*"GD"/);
|
||||
assert.match(constantsSource, /function driveKindAbbr\(kind: string\)/);
|
||||
@@ -348,7 +377,7 @@ test("nightly scan duplicate trigger uses full-scan busy message", () => {
|
||||
});
|
||||
|
||||
test("drive generation panel shows scan or crawler status first", () => {
|
||||
assert.match(driveComponentsSource, /label=\{d\.kind === "spider91" \? "已废弃" : "扫盘"\}/);
|
||||
assert.match(driveComponentsSource, /label="扫盘"/);
|
||||
assert.match(driveComponentsSource, /status=\{d\.scanGenerationStatus\}/);
|
||||
assert.match(driveComponentsSource, /showCounts=\{false\}/);
|
||||
assert.match(driveComponentsSource, /status\?\.scannedCount/);
|
||||
@@ -358,11 +387,10 @@ test("drive generation panel shows scan or crawler status first", () => {
|
||||
assert.match(constantsSource, /if \(state === "scanning"\) return "扫盘中"/);
|
||||
});
|
||||
|
||||
test("legacy spider91 storage is disabled in drive management", () => {
|
||||
assert.match(drivesPageSource, /91Spider 不再支持通过网盘运行,请到爬虫管理添加爬虫脚本/);
|
||||
assert.match(drivesPageSource, /disabled=\{d\.kind === "spider91"\}/);
|
||||
assert.match(drivesPageSource, /已废弃,请到爬虫管理添加/);
|
||||
assert.match(constantsSource, /91Spider 不再支持通过网盘添加或编辑/);
|
||||
test("drive management has no spider91 storage branch", () => {
|
||||
assert.doesNotMatch(drivesPageSource, /spider91|91Spider/);
|
||||
assert.doesNotMatch(constantsSource, /spider91|91Spider/);
|
||||
assert.doesNotMatch(driveComponentsSource, /spider91|91Spider/);
|
||||
});
|
||||
|
||||
test("drive detail selection is stored in the URL history", () => {
|
||||
@@ -392,16 +420,16 @@ test("drive discard confirmation matches delete confirmation modal styling", ()
|
||||
test("new drive type selection alone is not treated as unsaved config", () => {
|
||||
assert.match(
|
||||
drivesPageSource,
|
||||
/const formDirty = form\.id\s*\?\s*!sameForm\(form, initialForm\)\s*:\s*hasCreateFormChanges\(form, initialForm\);/
|
||||
/const formDirty = form\.id\s*\?\s*!sameForm\(form, initialForm\)\s*:\s*hasCreateFormChanges\(form\);/
|
||||
);
|
||||
assert.match(drivesPageSource, /function handleCreateFormChange\(nextForm: FormState\)/);
|
||||
assert.match(
|
||||
drivesPageSource,
|
||||
/if \(!nextForm\.id && !hasCreateFormChanges\(nextForm, initialForm\)\) \{\s*setInitialForm\(nextForm\);/
|
||||
/if \(!nextForm\.id && !hasCreateFormChanges\(nextForm\)\) \{\s*setInitialForm\(nextForm\);/
|
||||
);
|
||||
assert.match(drivesPageSource, /onChange=\{handleCreateFormChange\}/);
|
||||
|
||||
const match = /function hasCreateFormChanges\(form: FormState, initial: FormState\): boolean \{([\s\S]*?)\n\}/.exec(
|
||||
const match = /function hasCreateFormChanges\(form: FormState\): boolean \{([\s\S]*?)\n\}/.exec(
|
||||
drivesPageSource
|
||||
);
|
||||
assert.ok(match, "create form dirty helper should be present");
|
||||
@@ -409,7 +437,6 @@ test("new drive type selection alone is not treated as unsaved config", () => {
|
||||
|
||||
assert.match(helper, /form\.name\.trim\(\) !== ""/);
|
||||
assert.match(helper, /form\.rootId\.trim\(\) !== ""/);
|
||||
assert.match(helper, /form\.spider91UploadDriveId !== initial\.spider91UploadDriveId/);
|
||||
assert.match(helper, /Object\.values\(form\.creds\)\.some/);
|
||||
assert.doesNotMatch(helper, /form\.kind/);
|
||||
});
|
||||
|
||||
@@ -17,7 +17,19 @@ test("admin modal does not reset focus when close handler identity changes", ()
|
||||
});
|
||||
|
||||
test("admin modal backdrop clicks do not close dialogs", () => {
|
||||
assert.match(modalSource, /import \{ createPortal \} from "react-dom";/);
|
||||
assert.match(modalSource, /createPortal\(/);
|
||||
assert.match(modalSource, /document\.body/);
|
||||
assert.match(modalSource, /className="admin-modal-backdrop"/);
|
||||
assert.doesNotMatch(modalSource, /onMouseDown=\{\(e\) =>/);
|
||||
assert.doesNotMatch(modalSource, /e\.target === e\.currentTarget/);
|
||||
});
|
||||
|
||||
test("admin modal supports titleless dialogs with aria labels", () => {
|
||||
assert.match(modalSource, /title\?: string;/);
|
||||
assert.match(modalSource, /ariaLabel\?: string;/);
|
||||
assert.match(modalSource, /aria-labelledby=\{title \? titleId : undefined\}/);
|
||||
assert.match(modalSource, /aria-label=\{title \? undefined : ariaLabel \?\? "对话框"\}/);
|
||||
assert.match(modalSource, /admin-modal__header\$\{title \? "" : " is-titleless"\}/);
|
||||
assert.match(modalSource, /\{title && <span id=\{titleId\}>\{title\}<\/span>\}/);
|
||||
});
|
||||
|
||||
@@ -10,6 +10,10 @@ const videosPageSource = readFileSync(
|
||||
new URL("../src/admin/VideosPage.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
const apiSource = readFileSync(
|
||||
new URL("../src/admin/api.ts", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
|
||||
function ruleBody(css: string, selector: string): string {
|
||||
const escapedSelector = selector.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
@@ -71,8 +75,9 @@ test("admin tables scroll inside the mobile viewport", () => {
|
||||
});
|
||||
|
||||
test("admin video filter select uses an aligned custom arrow", () => {
|
||||
const select = ruleBody(adminCss, ".admin-videos-filter__select");
|
||||
const icon = ruleBody(adminCss, ".admin-videos-filter__select-icon");
|
||||
const select = allRuleBodies(adminCss, ".admin-videos-filter__select");
|
||||
const icon = allRuleBodies(adminCss, ".admin-videos-filter__select-icon");
|
||||
const focus = ruleBody(adminCss, ".admin-videos-filter__select:focus");
|
||||
const mobileWrap = ruleBodyByContains(mobileCss(), ".admin-videos-filter__select-wrap");
|
||||
|
||||
assert.match(select, /appearance\s*:\s*none/);
|
||||
@@ -80,6 +85,7 @@ test("admin video filter select uses an aligned custom arrow", () => {
|
||||
assert.match(icon, /top\s*:\s*50%/);
|
||||
assert.match(icon, /right\s*:\s*12px/);
|
||||
assert.match(icon, /transform\s*:\s*translateY\(-50%\)/);
|
||||
assert.match(focus, /box-shadow\s*:\s*0\s+0\s+0\s+3px\s+var\(--accent-soft\)/);
|
||||
assert.match(mobileWrap, /flex\s*:\s*1\s+1\s+100%/);
|
||||
});
|
||||
|
||||
@@ -100,11 +106,66 @@ test("admin video bulk actions use semantic theme colors", () => {
|
||||
assert.doesNotMatch(bulkBodies, /#ff5b8a|#fff6f9|rgba\(255,\s*91,\s*138/);
|
||||
});
|
||||
|
||||
test("admin video list summary stays below filter controls", () => {
|
||||
test("current video list does not render the drive summary under filters", () => {
|
||||
const filter = ruleBody(adminCss, ".admin-videos-filter");
|
||||
const toolbar = ruleBody(adminCss, ".admin-videos-list-toolbar");
|
||||
const currentToolbar = ruleBody(adminCss, ".admin-videos-current .admin-videos-list-toolbar");
|
||||
const currentWithBulk = ruleBody(adminCss, ".admin-videos-current.has-bulk-actions");
|
||||
|
||||
assert.doesNotMatch(videosPageSource, /listSummary/);
|
||||
assert.doesNotMatch(videosPageSource, /全部网盘:共/);
|
||||
assert.doesNotMatch(videosPageSource, /withCounts/);
|
||||
assert.doesNotMatch(videosPageSource, /teaserReadyCount|teaserPendingCount/);
|
||||
assert.match(videosPageSource, /admin-videos-filter admin-videos-filter--current/);
|
||||
assert.match(videosPageSource, /className="admin-btn admin-videos-filter__refresh"[\s\S]*aria-label="刷新当前视频"/);
|
||||
assert.match(videosPageSource, /className="admin-videos-filter__refresh-text">刷新/);
|
||||
assert.match(videosPageSource, /admin-videos-current\$\{selectedIds\.size > 0 \? " has-bulk-actions" : ""\}/);
|
||||
assert.match(videosPageSource, /\{!loading && selectedIds\.size > 0 && \(/);
|
||||
assert.match(filter, /margin-bottom\s*:\s*var\(--space-4\)/);
|
||||
assert.match(toolbar, /margin\s*:\s*var\(--space-2\)\s+0\s+var\(--space-4\)/);
|
||||
assert.doesNotMatch(toolbar, /margin\s*:\s*-/);
|
||||
assert.match(currentToolbar, /position\s*:\s*fixed/);
|
||||
assert.match(currentToolbar, /left\s*:\s*auto/);
|
||||
assert.match(currentToolbar, /right\s*:\s*var\(--space-7\)/);
|
||||
assert.match(currentToolbar, /bottom\s*:\s*var\(--space-5\)/);
|
||||
assert.match(currentToolbar, /max-width\s*:\s*calc\(100vw\s*-\s*288px\s*-\s*\(var\(--space-7\)\s*\*\s*2\)\)/);
|
||||
assert.match(currentToolbar, /margin\s*:\s*0/);
|
||||
assert.match(currentWithBulk, /padding-bottom\s*:\s*72px/);
|
||||
});
|
||||
|
||||
test("desktop video management filters use a stable three-column toolbar", () => {
|
||||
const css = adminCss;
|
||||
const currentFilter = ruleBodyByContains(css, ".admin-videos-filter--current");
|
||||
const blacklistFilter = ruleBodyByContains(css, ".admin-videos-filter--blacklist");
|
||||
const currentFilterSelect = ruleBodyByContains(css, ".admin-videos-filter--current .admin-videos-filter__select-wrap");
|
||||
const blacklistFilterSelect = ruleBodyByContains(css, ".admin-videos-filter--blacklist .admin-videos-filter__select-wrap");
|
||||
const currentSelect = ruleBodyByContains(css, ".admin-videos-filter--current .admin-videos-filter__select");
|
||||
const blacklistSelect = ruleBodyByContains(css, ".admin-videos-filter--blacklist .admin-videos-filter__select");
|
||||
const currentIcon = ruleBodyByContains(css, ".admin-videos-filter--current .admin-videos-filter__select-icon");
|
||||
const blacklistIcon = ruleBodyByContains(css, ".admin-videos-filter--blacklist .admin-videos-filter__select-icon");
|
||||
const currentFilterSearch = ruleBodyByContains(css, ".admin-videos-filter--current .admin-videos-filter__search");
|
||||
const blacklistFilterSearch = ruleBodyByContains(css, ".admin-videos-filter--blacklist .admin-videos-filter__search");
|
||||
const refresh = ruleBody(css, ".admin-videos-filter__refresh");
|
||||
|
||||
assert.match(videosPageSource, /className="admin-btn admin-videos-filter__refresh"[\s\S]*aria-label="刷新当前视频"/);
|
||||
assert.match(videosPageSource, /className="admin-btn admin-videos-filter__refresh"[\s\S]*aria-label="刷新拉黑视频"/);
|
||||
assert.match(currentFilter, /display\s*:\s*grid/);
|
||||
assert.match(currentFilter, /grid-template-columns\s*:\s*72px\s+minmax\(0,\s*1fr\)\s+auto/);
|
||||
assert.match(currentFilter, /width\s*:\s*100%/);
|
||||
assert.match(blacklistFilter, /display\s*:\s*grid/);
|
||||
assert.match(blacklistFilter, /grid-template-columns\s*:\s*72px\s+minmax\(0,\s*1fr\)\s+auto/);
|
||||
assert.match(blacklistFilter, /width\s*:\s*100%/);
|
||||
assert.match(currentFilterSelect, /min-width\s*:\s*0/);
|
||||
assert.match(blacklistFilterSelect, /min-width\s*:\s*0/);
|
||||
assert.match(currentSelect, /padding\s*:\s*0\s+4px/);
|
||||
assert.match(blacklistSelect, /padding\s*:\s*0\s+4px/);
|
||||
assert.match(currentSelect, /text-align\s*:\s*center/);
|
||||
assert.match(blacklistSelect, /text-align\s*:\s*center/);
|
||||
assert.match(currentIcon, /display\s*:\s*none/);
|
||||
assert.match(blacklistIcon, /display\s*:\s*none/);
|
||||
assert.match(currentFilterSearch, /min-width\s*:\s*0/);
|
||||
assert.match(blacklistFilterSearch, /min-width\s*:\s*0/);
|
||||
assert.match(refresh, /white-space\s*:\s*nowrap/);
|
||||
assert.doesNotMatch(refresh, /display\s*:\s*none/);
|
||||
});
|
||||
|
||||
test("admin table action headers center-align with action buttons", () => {
|
||||
@@ -118,15 +179,45 @@ test("admin table action headers center-align with action buttons", () => {
|
||||
test("blacklist restore action uses a light button style", () => {
|
||||
const restoreButton = ruleBody(adminCss, ".admin-blacklist-restore-btn");
|
||||
|
||||
assert.match(videosPageSource, /const \[driveId, setDriveId\] = useState\(""\);/);
|
||||
assert.match(videosPageSource, /api\.listBlacklist\(\{ driveId, page, size: pageSize, keyword: searchKeyword \}\)/);
|
||||
assert.match(videosPageSource, /admin-videos-filter admin-videos-filter--blacklist/);
|
||||
assert.match(videosPageSource, /<DriveFilter drives=\{drives\} driveId=\{driveId\}/);
|
||||
assert.match(apiSource, /listBlacklist\(\s*params: \{ driveId\?: string; page\?: number; size\?: number; keyword\?: string \}/);
|
||||
assert.match(apiSource, /if \(params\.driveId\) qs\.set\("driveId", params\.driveId\);/);
|
||||
assert.match(videosPageSource, /className="admin-btn admin-blacklist-restore-btn"/);
|
||||
assert.doesNotMatch(videosPageSource, /被删除和被隐藏的视频会进入黑名单/);
|
||||
assert.doesNotMatch(videosPageSource, /原始记录、封面、预览已删除/);
|
||||
assert.match(restoreButton, /background\s*:\s*var\(--accent-softer\)/);
|
||||
assert.match(restoreButton, /color\s*:\s*var\(--accent\)/);
|
||||
assert.doesNotMatch(restoreButton, /background\s*:\s*var\(--accent\)/);
|
||||
});
|
||||
|
||||
test("blacklist duplicate reason renders as a compact pill", () => {
|
||||
const pill = ruleBody(adminCss, ".admin-blacklist-reason-pill");
|
||||
|
||||
assert.match(videosPageSource, /admin-blacklist-reason-pill/);
|
||||
assert.match(videosPageSource, /重复文件/);
|
||||
assert.match(pill, /border-radius\s*:\s*999px/);
|
||||
assert.match(pill, /white-space\s*:\s*nowrap/);
|
||||
});
|
||||
|
||||
test("admin video management controls wrap instead of covering text on mobile", () => {
|
||||
const css = mobileCss();
|
||||
const paginationInfo = allRuleBodies(css, ".admin-table-pagination__info");
|
||||
const currentFilter = ruleBody(css, ".admin-videos-filter--current");
|
||||
const currentFilterField = ruleBodyByContains(css, ".admin-videos-filter--current .admin-videos-filter__search");
|
||||
const currentFilterSelect = ruleBody(css, ".admin-videos-filter--current .admin-videos-filter__select");
|
||||
const currentFilterIcon = ruleBody(css, ".admin-videos-filter--current .admin-videos-filter__select-icon");
|
||||
const currentFilterRefresh = ruleBody(css, ".admin-videos-filter--current .admin-videos-filter__refresh");
|
||||
const currentFilterRefreshText = ruleBody(css, ".admin-videos-filter--current .admin-videos-filter__refresh-text");
|
||||
const blacklistFilter = allRuleBodies(css, ".admin-videos-filter--blacklist");
|
||||
const blacklistFilterField = ruleBodyByContains(css, ".admin-videos-filter--blacklist .admin-videos-filter__search");
|
||||
const blacklistFilterSelect = ruleBodyByContains(css, ".admin-videos-filter--blacklist .admin-videos-filter__select");
|
||||
const blacklistFilterIcon = ruleBodyByContains(css, ".admin-videos-filter--blacklist .admin-videos-filter__select-icon");
|
||||
const blacklistFilterButton = ruleBody(css, ".admin-videos-filter--blacklist .admin-btn");
|
||||
const bulkToolbar = allRuleBodies(css, ".admin-videos-current .admin-videos-list-toolbar");
|
||||
const currentWithBulk = allRuleBodies(css, ".admin-videos-current.has-bulk-actions");
|
||||
const bulkActions = allRuleBodies(css, ".admin-videos-bulk-actions");
|
||||
const bulkCount = allRuleBodies(css, ".admin-videos-bulk-actions__count");
|
||||
const bulkButton = allRuleBodies(css, ".admin-videos-bulk-actions__btn");
|
||||
@@ -152,8 +243,33 @@ test("admin video management controls wrap instead of covering text on mobile",
|
||||
);
|
||||
|
||||
assert.match(paginationInfo, /flex\s*:\s*1\s+0\s+100%/);
|
||||
assert.match(bulkActions, /flex-wrap\s*:\s*wrap/);
|
||||
assert.match(bulkCount, /flex\s*:\s*1\s+0\s+100%/);
|
||||
assert.match(currentFilter, /display\s*:\s*grid/);
|
||||
assert.match(currentFilter, /grid-template-columns\s*:\s*72px\s+minmax\(0,\s*1fr\)\s+38px/);
|
||||
assert.match(currentFilterField, /min-width\s*:\s*0/);
|
||||
assert.match(currentFilterSelect, /padding\s*:\s*0\s+4px/);
|
||||
assert.match(currentFilterSelect, /text-align\s*:\s*center/);
|
||||
assert.match(currentFilterSelect, /text-align-last\s*:\s*center/);
|
||||
assert.match(currentFilterSelect, /text-overflow\s*:\s*ellipsis/);
|
||||
assert.match(currentFilterIcon, /display\s*:\s*none/);
|
||||
assert.match(currentFilterRefresh, /width\s*:\s*38px/);
|
||||
assert.match(currentFilterRefresh, /min-width\s*:\s*38px/);
|
||||
assert.match(currentFilterRefreshText, /display\s*:\s*none/);
|
||||
assert.match(blacklistFilter, /display\s*:\s*grid/);
|
||||
assert.match(blacklistFilter, /grid-template-columns\s*:\s*72px\s+minmax\(0,\s*1fr\)\s+auto/);
|
||||
assert.match(blacklistFilterField, /min-width\s*:\s*0/);
|
||||
assert.match(blacklistFilterSelect, /padding\s*:\s*0\s+4px/);
|
||||
assert.match(blacklistFilterSelect, /text-align\s*:\s*center/);
|
||||
assert.match(blacklistFilterSelect, /text-align-last\s*:\s*center/);
|
||||
assert.match(blacklistFilterIcon, /display\s*:\s*none/);
|
||||
assert.match(blacklistFilterButton, /white-space\s*:\s*nowrap/);
|
||||
assert.match(bulkToolbar, /position\s*:\s*fixed/);
|
||||
assert.match(bulkToolbar, /bottom\s*:\s*calc\(var\(--space-3\)\s*\+\s*env\(safe-area-inset-bottom\)\)/);
|
||||
assert.match(bulkToolbar, /margin\s*:\s*0/);
|
||||
assert.match(currentWithBulk, /padding-bottom\s*:\s*calc\(104px\s*\+\s*env\(safe-area-inset-bottom\)\)/);
|
||||
assert.match(bulkActions, /display\s*:\s*grid/);
|
||||
assert.match(bulkActions, /grid-template-columns\s*:\s*repeat\(2,\s*minmax\(0,\s*1fr\)\)/);
|
||||
assert.match(bulkCount, /grid-column\s*:\s*1\s*\/\s*-1/);
|
||||
assert.match(bulkButton, /min-height\s*:\s*40px/);
|
||||
assert.match(bulkButton, /min-width\s*:\s*0/);
|
||||
assert.match(blacklistName, /grid-column\s*:\s*1\s*\/\s*-1/);
|
||||
assert.match(blacklistTime, /grid-column\s*:\s*1/);
|
||||
@@ -178,9 +294,26 @@ test("mobile video management uses compact theme-aware video cards", () => {
|
||||
const css = mobileCss();
|
||||
const card = ruleBody(css, ".admin-videos-table:not(.admin-drives-table) tr");
|
||||
const title = ruleBody(css, ".admin-videos-table:not(.admin-drives-table) td[data-label=\"标题\"]");
|
||||
const checkbox = ruleBody(css, ".admin-videos-table:not(.admin-drives-table) td.is-checkbox");
|
||||
const label = ruleBody(css, ".admin-videos-table:not(.admin-drives-table) td::before");
|
||||
const titleCell = ruleBody(css, ".admin-videos-table:not(.admin-drives-table) .admin-video-title-cell");
|
||||
const thumb = ruleBody(css, ".admin-videos-table:not(.admin-drives-table) .admin-video-thumb-wrap");
|
||||
const titleText = ruleBody(css, ".admin-videos-table:not(.admin-drives-table) .admin-video-title");
|
||||
const pills = ruleBody(css, ".admin-videos-table:not(.admin-drives-table) .admin-video-filemeta-pills");
|
||||
const authorColumn = ruleBodyByContains(css, ".admin-videos-table:not(.admin-drives-table) td[data-label=\"作者\"]");
|
||||
const sourceColumn = ruleBodyByContains(css, ".admin-videos-table:not(.admin-drives-table) td[data-label=\"来源\"]");
|
||||
const durationColumn = ruleBodyByContains(css, ".admin-videos-table:not(.admin-drives-table) td[data-label=\"时长\"]");
|
||||
const previewColumn = ruleBodyByContains(css, ".admin-videos-table:not(.admin-drives-table) td[data-label=\"预览视频\"]");
|
||||
const actions = ruleBody(css, ".admin-videos-table:not(.admin-drives-table) td.is-actions");
|
||||
const actionsLabel = ruleBody(css, ".admin-videos-table:not(.admin-drives-table) td.is-actions::before");
|
||||
const checkboxLabel = ruleBodyByContains(css, ".admin-videos-table:not(.admin-drives-table) td.is-checkbox::before");
|
||||
const checkboxButton = ruleBody(css, ".admin-videos-table:not(.admin-drives-table) .admin-table-checkbox-btn");
|
||||
const checkboxIcon = ruleBody(
|
||||
css,
|
||||
".admin-videos-table:not(.admin-drives-table) .admin-table-checkbox-btn svg"
|
||||
);
|
||||
const status = ruleBody(css, ".admin-videos-table:not(.admin-drives-table) .admin-status");
|
||||
const statusDot = ruleBody(css, ".admin-videos-table:not(.admin-drives-table) .admin-status::before");
|
||||
const actionButton = ruleBody(css, ".admin-videos-table:not(.admin-drives-table) td.is-actions .admin-btn");
|
||||
const dangerButton = ruleBody(css, ".admin-videos-table:not(.admin-drives-table) td.is-actions .admin-btn.is-danger");
|
||||
|
||||
@@ -188,19 +321,78 @@ test("mobile video management uses compact theme-aware video cards", () => {
|
||||
assert.match(card, /background\s*:\s*var\(--admin-video-card-bg\)/);
|
||||
assert.match(card, /border-radius\s*:\s*14px/);
|
||||
assert.match(card, /padding\s*:\s*12px\s+14px/);
|
||||
assert.match(card, /grid-template-columns\s*:\s*repeat\(12,\s*minmax\(0,\s*1fr\)\)/);
|
||||
assert.match(card, /gap\s*:\s*0\s+10px/);
|
||||
assert.match(css, /:root:not\(\[data-theme="pink"\]\)\s+\.admin-videos-table:not\(\.admin-drives-table\)\s+tr\s*\{[^}]*--admin-video-card-bg\s*:\s*#1e1e1e/s);
|
||||
assert.match(css, /:root\[data-theme="pink"\]\s+\.admin-videos-table:not\(\.admin-drives-table\)\s+tr\s*\{/);
|
||||
assert.match(title, /padding-left\s*:\s*36px/);
|
||||
assert.match(checkbox, /grid-column\s*:\s*1\s*\/\s*4/);
|
||||
assert.match(checkbox, /grid-row\s*:\s*3/);
|
||||
assert.match(checkbox, /display\s*:\s*flex/);
|
||||
assert.match(checkboxLabel, /content\s*:\s*none/);
|
||||
assert.match(checkboxButton, /width\s*:\s*100%/);
|
||||
assert.match(checkboxButton, /height\s*:\s*32px/);
|
||||
assert.match(videosPageSource, /admin-table-checkbox-btn \$\{isSelected \? "is-selected" : ""\}/);
|
||||
assert.match(checkboxIcon, /color\s*:\s*var\(--admin-video-card-button-text\)/);
|
||||
assert.match(checkboxIcon, /stroke\s*:\s*currentColor/);
|
||||
assert.match(title, /padding-left\s*:\s*0/);
|
||||
assert.match(title, /min-height\s*:\s*72px/);
|
||||
assert.match(label, /font-size\s*:\s*10px/);
|
||||
assert.match(label, /letter-spacing\s*:\s*0\.06em/);
|
||||
assert.match(titleCell, /grid-template-columns\s*:\s*clamp\(104px,\s*32vw,\s*156px\)\s+minmax\(0,\s*1fr\)/);
|
||||
assert.match(thumb, /aspect-ratio\s*:\s*16\s*\/\s*9/);
|
||||
assert.match(thumb, /border-radius\s*:\s*8px/);
|
||||
assert.match(titleText, /-webkit-line-clamp\s*:\s*2/);
|
||||
assert.match(titleText, /overflow-wrap\s*:\s*anywhere/);
|
||||
assert.match(videosPageSource, /loading="lazy"\s+decoding="async"/);
|
||||
assert.match(videosPageSource, /className="admin-video-title" title=\{v\.title\}/);
|
||||
assert.match(pills, /display\s*:\s*flex/);
|
||||
assert.doesNotMatch(videosPageSource, /admin-video-filemeta-pill is-category/);
|
||||
assert.doesNotMatch(css, /admin-video-card-category/);
|
||||
assert.match(authorColumn, /display\s*:\s*none/);
|
||||
assert.match(sourceColumn, /grid-row\s*:\s*2/);
|
||||
assert.match(sourceColumn, /grid-column\s*:\s*1\s*\/\s*5/);
|
||||
assert.match(sourceColumn, /justify-items\s*:\s*start/);
|
||||
assert.match(sourceColumn, /text-overflow\s*:\s*ellipsis/);
|
||||
assert.match(durationColumn, /grid-row\s*:\s*2/);
|
||||
assert.match(durationColumn, /grid-column\s*:\s*5\s*\/\s*9/);
|
||||
assert.match(durationColumn, /justify-items\s*:\s*center/);
|
||||
assert.match(previewColumn, /grid-row\s*:\s*2/);
|
||||
assert.match(previewColumn, /grid-column\s*:\s*9\s*\/\s*-1/);
|
||||
assert.match(previewColumn, /justify-items\s*:\s*end/);
|
||||
assert.match(actions, /grid-column\s*:\s*4\s*\/\s*-1/);
|
||||
assert.match(actions, /grid-row\s*:\s*3/);
|
||||
assert.match(actions, /display\s*:\s*grid/);
|
||||
assert.match(actions, /grid-template-columns\s*:\s*repeat\(3,\s*minmax\(0,\s*1fr\)\)/);
|
||||
assert.match(actions, /gap\s*:\s*10px/);
|
||||
assert.match(actionsLabel, /content\s*:\s*none/);
|
||||
assert.match(status, /gap\s*:\s*0/);
|
||||
assert.match(statusDot, /content\s*:\s*none/);
|
||||
assert.doesNotMatch(sourceColumn, /border-left/);
|
||||
assert.match(actionButton, /height\s*:\s*28px/);
|
||||
assert.match(actionButton, /width\s*:\s*100%/);
|
||||
assert.match(actionButton, /height\s*:\s*32px/);
|
||||
assert.match(actionButton, /justify-content\s*:\s*center/);
|
||||
assert.match(actionButton, /border-radius\s*:\s*8px/);
|
||||
assert.match(dangerButton, /border-color\s*:\s*var\(--admin-video-card-danger-border\)/);
|
||||
assert.match(dangerButton, /color\s*:\s*var\(--admin-video-card-danger\)/);
|
||||
});
|
||||
|
||||
test("video edit modal stays focused on common metadata", () => {
|
||||
assert.match(videosPageSource, /ariaLabel="编辑视频"/);
|
||||
assert.doesNotMatch(videosPageSource, /title=\{`编辑视频 ·/);
|
||||
assert.doesNotMatch(videosPageSource, /const \[badges, setBadges\]/);
|
||||
assert.doesNotMatch(videosPageSource, /const \[thumbnail, setThumbnail\]/);
|
||||
assert.doesNotMatch(videosPageSource, /const \[quality, setQuality\]/);
|
||||
assert.doesNotMatch(videosPageSource, /video-badges/);
|
||||
assert.doesNotMatch(videosPageSource, /video-quality/);
|
||||
assert.doesNotMatch(videosPageSource, /video-thumbnail/);
|
||||
assert.doesNotMatch(videosPageSource, /徽标(/);
|
||||
assert.doesNotMatch(videosPageSource, /封面 URL/);
|
||||
assert.doesNotMatch(videosPageSource, /封面预览/);
|
||||
assert.doesNotMatch(videosPageSource, /badges:\s*splitList\(badges\)/);
|
||||
assert.doesNotMatch(videosPageSource, /thumbnail:\s*thumbnail\.trim\(\)/);
|
||||
assert.doesNotMatch(videosPageSource, /quality:\s*quality\.trim\(\)/);
|
||||
});
|
||||
|
||||
test("admin modals and action footers adapt on mobile", () => {
|
||||
const css = mobileCss();
|
||||
|
||||
@@ -208,17 +400,62 @@ test("admin modals and action footers adapt on mobile", () => {
|
||||
// 只重写 max-height,所以这里断桌面规则即可。
|
||||
assert.match(ruleBody(adminCss, ".admin-modal"), /width\s*:\s*min\(\d+px,\s*100%\)/);
|
||||
assert.match(ruleBody(adminCss, ".admin-modal.admin-modal--crawler"), /width\s*:\s*min\(1080px,\s*100%\)/);
|
||||
assert.match(allRuleBodies(css, ".admin-modal"), /display\s*:\s*flex/);
|
||||
assert.match(allRuleBodies(css, ".admin-modal"), /overflow\s*:\s*hidden/);
|
||||
assert.match(allRuleBodies(css, ".admin-modal__body"), /overflow-y\s*:\s*auto/);
|
||||
assert.match(allRuleBodies(css, ".admin-modal-backdrop"), /safe-area-inset-top/);
|
||||
assert.match(allRuleBodies(css, ".admin-modal-backdrop"), /place-items\s*:\s*center/);
|
||||
assert.doesNotMatch(allRuleBodies(css, ".admin-modal-backdrop"), /align-items\s*:\s*stretch/);
|
||||
// 多按钮 footer 在 mobile 下要换行避免溢出。
|
||||
assert.match(allRuleBodies(css, ".admin-modal__footer"), /flex-wrap\s*:\s*wrap/);
|
||||
// 删除/放弃类确认弹窗在 mobile 下不能跟随通用 modal stretch 到顶部。
|
||||
const confirmModal = ruleBody(css, ".admin-modal--delete-confirm");
|
||||
assert.match(confirmModal, /align-self\s*:\s*center/);
|
||||
assert.match(confirmModal, /justify-self\s*:\s*center/);
|
||||
assert.match(ruleBody(adminCss, ".admin-modal__header.is-titleless"), /justify-content\s*:\s*flex-end/);
|
||||
// 表单 input/select/textarea 在 mobile 下铺满。规则用逗号合并写法(多 selector
|
||||
// 共享 body),所以走 ruleBodyByContains 而不是简单正则。
|
||||
assert.match(ruleBodyByContains(css, ".admin-form__row input"), /width\s*:\s*100%/);
|
||||
});
|
||||
|
||||
test("mobile drive type picker uses compact three-column cards", () => {
|
||||
const driveTypeGridBodies = allRuleBodies(adminCss, ".admin-drive-type-grid");
|
||||
const driveTypeCardBodies = allRuleBodies(adminCss, ".admin-drive-type-card");
|
||||
const driveTypeIconBodies = allRuleBodies(adminCss, ".admin-drive-type-card__icon");
|
||||
|
||||
assert.match(driveTypeGridBodies, /grid-template-columns\s*:\s*repeat\(3,\s*minmax\(0,\s*1fr\)\)/);
|
||||
assert.doesNotMatch(driveTypeGridBodies, /grid-template-columns\s*:\s*repeat\(2,\s*1fr\)/);
|
||||
assert.match(driveTypeCardBodies, /min-height\s*:\s*94px/);
|
||||
assert.match(driveTypeIconBodies, /width\s*:\s*38px/);
|
||||
assert.match(driveTypeIconBodies, /height\s*:\s*38px/);
|
||||
});
|
||||
|
||||
test("mobile tags management does not create horizontal page overflow", () => {
|
||||
const css = mobileCss();
|
||||
const layout = allRuleBodies(css, ".admin-tags-layout");
|
||||
const toolbar = allRuleBodies(css, ".admin-tags-toolbar");
|
||||
const search = allRuleBodies(css, ".admin-tags-search");
|
||||
const filters = allRuleBodies(css, ".admin-tags-filter-tabs");
|
||||
const grid = allRuleBodies(css, ".admin-tags-grid");
|
||||
const card = allRuleBodies(css, ".admin-tag-card");
|
||||
const pagination = allRuleBodies(css, ".admin-tags-pagination");
|
||||
const paginationInfo = allRuleBodies(css, ".admin-tags-pagination .admin-table-pagination__info");
|
||||
|
||||
assert.match(layout, /width\s*:\s*100%/);
|
||||
assert.match(layout, /max-width\s*:\s*100%/);
|
||||
assert.match(layout, /overflow-x\s*:\s*clip/);
|
||||
assert.match(toolbar, /max-width\s*:\s*100%/);
|
||||
assert.match(search, /min-width\s*:\s*0/);
|
||||
assert.match(filters, /width\s*:\s*100%/);
|
||||
assert.match(filters, /min-width\s*:\s*0/);
|
||||
assert.match(filters, /max-width\s*:\s*100%/);
|
||||
assert.match(grid, /grid-template-columns\s*:\s*minmax\(0,\s*1fr\)/);
|
||||
assert.match(grid, /max-width\s*:\s*100%/);
|
||||
assert.match(card, /max-width\s*:\s*100%/);
|
||||
assert.match(pagination, /min-width\s*:\s*0/);
|
||||
assert.match(paginationInfo, /overflow-wrap\s*:\s*anywhere/);
|
||||
});
|
||||
|
||||
test("mobile admin top navigation stays compact", () => {
|
||||
const css = mobileCss();
|
||||
|
||||
@@ -226,7 +463,12 @@ test("mobile admin top navigation stays compact", () => {
|
||||
assert.match(ruleBody(css, ".admin-shell"), /flex-direction\s*:\s*column/);
|
||||
assert.match(ruleBody(css, ".admin-sidebar"), /height\s*:\s*48px/);
|
||||
assert.match(ruleBody(css, ".admin-sidebar"), /min-height\s*:\s*48px/);
|
||||
assert.match(ruleBody(css, ".admin-sidebar"), /overflow-x\s*:\s*hidden/);
|
||||
assert.match(ruleBody(css, ".admin-sidebar__mobile-menu"), /position\s*:\s*absolute/);
|
||||
assert.match(ruleBody(css, ".admin-sidebar__mobile-menu"), /right\s*:\s*var\(--space-2\)/);
|
||||
assert.match(ruleBody(css, ".admin-sidebar__mobile-menu"), /transform\s*:\s*translateY\(-50%\)/);
|
||||
assert.match(ruleBody(css, ".admin-nav"), /align-items\s*:\s*center/);
|
||||
assert.match(ruleBody(css, ".admin-nav"), /overflow-x\s*:\s*auto/);
|
||||
assert.match(ruleBody(css, ".admin-nav__link"), /height\s*:\s*34px/);
|
||||
assert.match(ruleBody(css, ".admin-nav__link"), /line-height\s*:\s*1/);
|
||||
assert.match(ruleBody(css, ".admin-nav__link"), /flex\s*:\s*0\s+0\s+auto/);
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import test from "node:test";
|
||||
|
||||
const toastSource = readFileSync(
|
||||
new URL("../src/admin/ToastContext.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
const adminCss = readFileSync(
|
||||
new URL("../src/styles/admin.css", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
|
||||
function ruleBody(css: string, selector: string): string {
|
||||
const escapedSelector = selector.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const match = css.match(new RegExp(`${escapedSelector}\\s*\\{([^}]*)\\}`));
|
||||
assert.ok(match, `Expected CSS rule for ${selector}`);
|
||||
return match[1];
|
||||
}
|
||||
|
||||
function mobileCss(): string {
|
||||
const marker = "@media (max-width: 768px)";
|
||||
const start = adminCss.indexOf(marker);
|
||||
assert.notEqual(start, -1, "Expected mobile admin media query");
|
||||
return adminCss.slice(start);
|
||||
}
|
||||
|
||||
test("admin toasts auto-dismiss and copy their text when clicked", () => {
|
||||
assert.match(toastSource, /const TOAST_DISMISS_MS = 2500/);
|
||||
assert.match(toastSource, /const TOAST_COPY_SUCCESS_TEXT = "已复制到剪贴板"/);
|
||||
assert.match(toastSource, /const TOAST_COPY_ERROR_TEXT = "复制失败,请手动复制"/);
|
||||
assert.match(toastSource, /navigator\.clipboard\?\.writeText/);
|
||||
assert.match(toastSource, /fallbackCopyText\(text\)/);
|
||||
assert.match(toastSource, /document\.execCommand\("copy"\)/);
|
||||
assert.match(toastSource, /addToast\(TOAST_COPY_SUCCESS_TEXT,\s*"success",\s*false\)/);
|
||||
assert.match(toastSource, /addToast\(TOAST_COPY_ERROR_TEXT,\s*"error",\s*false\)/);
|
||||
assert.match(toastSource, /t\.copyable\s*\?\s*" is-copyable"\s*:\s*""/);
|
||||
assert.match(toastSource, /onClick=\{t\.copyable \? \(\) => copyToastText\(t\.text\) : undefined\}/);
|
||||
assert.match(toastSource, /aria-label=\{t\.copyable \? `复制提示:\$\{t\.text\}` : undefined\}/);
|
||||
assert.match(toastSource, /event\.key !== "Enter" && event\.key !== " "/);
|
||||
assert.doesNotMatch(toastSource, /onClick=\{\(\) => scheduleDismiss/);
|
||||
assert.doesNotMatch(toastSource, /pinnedToastIDs/);
|
||||
assert.doesNotMatch(toastSource, /isDismissPaused/);
|
||||
assert.doesNotMatch(toastSource, /pinDismiss/);
|
||||
assert.doesNotMatch(toastSource, /className="admin-toast__close"/);
|
||||
assert.doesNotMatch(toastSource, /aria-label="关闭提示"/);
|
||||
assert.doesNotMatch(toastSource, /<X size=/);
|
||||
assert.doesNotMatch(toastSource, /event\.stopPropagation\(\)/);
|
||||
assert.doesNotMatch(toastSource, /onPointerEnter/);
|
||||
assert.doesNotMatch(toastSource, /onPointerLeave/);
|
||||
});
|
||||
|
||||
test("admin toasts show long messages without internal scrolling", () => {
|
||||
const baseToast = ruleBody(adminCss, ".admin-toast");
|
||||
const baseText = ruleBody(adminCss, ".admin-toast__text");
|
||||
const mobileToast = ruleBody(mobileCss(), ".admin-toast");
|
||||
|
||||
assert.match(baseToast, /max-width\s*:\s*min\(520px,\s*calc\(100vw - 48px\)\)/);
|
||||
assert.match(baseToast, /padding\s*:\s*14px\s+18px/);
|
||||
assert.match(baseToast, /position\s*:\s*relative/);
|
||||
assert.match(baseToast, /overflow-wrap\s*:\s*anywhere/);
|
||||
assert.match(baseToast, /touch-action\s*:\s*manipulation/);
|
||||
assert.doesNotMatch(baseToast, /cursor\s*:\s*pointer/);
|
||||
assert.match(ruleBody(adminCss, ".admin-toast.is-copyable"), /cursor\s*:\s*pointer/);
|
||||
assert.match(baseText, /display\s*:\s*block/);
|
||||
assert.doesNotMatch(adminCss, /\.admin-toast__close/);
|
||||
assert.match(mobileToast, /max-width\s*:\s*100%/);
|
||||
assert.match(mobileToast, /text-align\s*:\s*left/);
|
||||
assert.doesNotMatch(baseToast, /max-height/);
|
||||
assert.doesNotMatch(baseText, /max-height/);
|
||||
assert.doesNotMatch(baseText, /overflow-y\s*:\s*auto/);
|
||||
assert.doesNotMatch(mobileToast, /max-height/);
|
||||
assert.doesNotMatch(mobileCss(), /\.admin-toast__text\s*\{/);
|
||||
});
|
||||
@@ -0,0 +1,20 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import test from "node:test";
|
||||
|
||||
const sortToolbarSource = readFileSync(
|
||||
new URL("../src/components/SortToolbar.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
const typesSource = readFileSync(new URL("../src/types.ts", import.meta.url), "utf8");
|
||||
|
||||
test("list page sort toolbar only exposes active sort options", () => {
|
||||
assert.match(sortToolbarSource, /\{ key: "latest", label: "最新" \}/);
|
||||
assert.match(sortToolbarSource, /\{ key: "hot", label: "最热" \}/);
|
||||
assert.match(sortToolbarSource, /\{ key: "recent", label: "最近观看" \}/);
|
||||
|
||||
for (const removed of ["本周", "最长", "高清", "精选"]) {
|
||||
assert.doesNotMatch(sortToolbarSource, new RegExp(removed));
|
||||
}
|
||||
assert.match(typesSource, /export type SortKey = "latest" \| "hot" \| "recent";/);
|
||||
});
|
||||
@@ -14,6 +14,9 @@ const indexHtml = readFileSync(
|
||||
new URL("../index.html", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
const manifest = JSON.parse(
|
||||
readFileSync(new URL("../public/manifest.webmanifest", import.meta.url), "utf8")
|
||||
) as { icons: Array<{ src: string; sizes: string; purpose: string }> };
|
||||
|
||||
// iOS Safari/WebKit does not composite an inline <video> nested inside a
|
||||
// `position: fixed` ancestor — the video decodes and plays but never paints
|
||||
@@ -46,6 +49,33 @@ test("iPhone browser uses document scrolling and only explicit fullscreen", () =
|
||||
|
||||
test("app has standalone display metadata for iPhone home-screen launch", () => {
|
||||
assert.match(indexHtml, /<link rel="manifest" href="\/manifest\.webmanifest" \/>/);
|
||||
assert.match(
|
||||
indexHtml,
|
||||
/<link rel="apple-touch-icon" sizes="180x180" href="\/apple-touch-icon\.png" \/>/
|
||||
);
|
||||
assert.match(indexHtml, /<meta name="apple-mobile-web-app-capable" content="yes" \/>/);
|
||||
assert.match(indexHtml, /<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" \/>/);
|
||||
});
|
||||
|
||||
test("home-screen icons use safe-area assets instead of the in-app logo", () => {
|
||||
assert.ok(
|
||||
manifest.icons.some(
|
||||
(icon) =>
|
||||
icon.src === "/app-icon-512.png" &&
|
||||
icon.sizes === "512x512" &&
|
||||
icon.purpose === "any"
|
||||
)
|
||||
);
|
||||
assert.ok(
|
||||
manifest.icons.some(
|
||||
(icon) =>
|
||||
icon.src === "/app-icon-maskable-512.png" &&
|
||||
icon.sizes === "512x512" &&
|
||||
icon.purpose === "maskable"
|
||||
)
|
||||
);
|
||||
assert.equal(
|
||||
manifest.icons.some((icon) => icon.src === "/icon.png" && icon.purpose.includes("maskable")),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user