mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-22 19:52:39 +08:00
Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 00aaeed736 | |||
| 5efbceb205 | |||
| 0faeaf408f | |||
| 1b5eda92b0 | |||
| 840a858dbd | |||
| 1ee5ee35be | |||
| 12b737b6fe | |||
| bd33d26a1f | |||
| 36fe32cb84 | |||
| 194d98895a |
@@ -30,9 +30,6 @@ tmp/
|
||||
|
||||
# 91 爬虫脚本独立运行时的默认输出文件(backend 跑时会显式 --output 到 backend/data/spider91/,所以不会落在这里)
|
||||
91porn_videos.json
|
||||
91VideoSpider/91porn_videos.json
|
||||
91VideoSpider/data/
|
||||
91VideoSpider/__pycache__/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
|
||||
@@ -1,988 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
91porn 视频爬虫脚本
|
||||
===================
|
||||
爬取 https://www.91porn.com/v.php?category=top&viewtype=basic 下的所有视频信息:
|
||||
- 视频名称
|
||||
- 封面图直链
|
||||
- 视频直链 (MP4)
|
||||
|
||||
依赖安装:
|
||||
pip install requests beautifulsoup4 lxml PySocks
|
||||
|
||||
使用方法:
|
||||
# 作为 video-site-91 通用爬虫脚本运行(后台会自动这样调用)
|
||||
python spider_91porn.py --job /path/to/job.json
|
||||
|
||||
# 全量爬取(默认行为,从 page=1 一直爬到末尾,写到 OUTPUT_FILE)
|
||||
python spider_91porn.py
|
||||
|
||||
# 只爬指定页(单页模式,手动调试用)
|
||||
python spider_91porn.py --page 1 --output /tmp/spider91_page1.json
|
||||
|
||||
# 凑够 N 个新视频模式(backend 凌晨任务用)
|
||||
python spider_91porn.py --target-new 15 --seen-viewkeys-file /tmp/seen.txt --output /tmp/new.json
|
||||
|
||||
CLI 参数:
|
||||
--job FILE crawler.v1 job JSON 路径;后台爬虫管理会使用此模式
|
||||
--page N 只爬第 N 页,配合 --output 用于手动调试
|
||||
--target-new N 从 page 1 起翻页直到凑够 N 个新视频(不在 seen 列表里的)
|
||||
--seen-viewkeys-file FILE 每行一个已知 viewkey 或 mp4 源 ID,命中即跳过;与 --target-new 配合使用
|
||||
--output FILE 输出 JSON 路径,覆盖默认的 OUTPUT_FILE
|
||||
--no-resume 禁用断点续爬(单页/target-new 模式下自动禁用)
|
||||
--quiet 压缩日志,每条视频只输出一行
|
||||
-h / --help 帮助
|
||||
|
||||
配置说明 (编辑脚本内 "配置区域"):
|
||||
- MIN_PAGE_DELAY / MAX_PAGE_DELAY : 列表页请求间隔 (默认 3-6 秒)
|
||||
- MIN_DETAIL_DELAY / MAX_DETAIL_DELAY : 详情页请求间隔 (默认 2-5 秒)
|
||||
- MAX_PAGES : 限制最大爬取页数 (None=不限, 如 5=只爬前5页)
|
||||
- OUTPUT_FILE : 输出文件名
|
||||
|
||||
输出格式 (JSON):
|
||||
--job 模式下 stdout 输出 crawler.v1 JSON Lines,日志全部写到 stderr。
|
||||
手动运行模式仍会写传统 JSON 文件:
|
||||
{
|
||||
"videos": [
|
||||
{
|
||||
"title": "视频标题",
|
||||
"thumb_url": "https://...thumb/xxxx.jpg",
|
||||
"video_url": "https://...mp43/xxxx.mp4?st=...",
|
||||
"viewkey": "abc123...",
|
||||
"source_id": "xxxx",
|
||||
"detail_url": "https://...view_video.php?viewkey=..."
|
||||
},
|
||||
...
|
||||
]
|
||||
}
|
||||
|
||||
注意:
|
||||
1. 视频直链包含时效性token (e参数为过期时间戳),会过期,需定期重新爬取
|
||||
2. 脚本已内置随机延时,请勿移除,避免对服务器造成压力
|
||||
3. 网站有Cloudflare保护,如遇到403/5xx错误,可能需要使用带cookie的session
|
||||
4. 本脚本仅供学习交流,请遵守当地法律法规
|
||||
|
||||
作者: OpenCode
|
||||
日期: 2026-05-22
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import requests
|
||||
import re
|
||||
import time
|
||||
import random
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import html
|
||||
from urllib.parse import urljoin, unquote, urlparse
|
||||
from datetime import datetime
|
||||
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except ImportError:
|
||||
print("错误: 缺少依赖库 beautifulsoup4", file=sys.stderr)
|
||||
print("请运行: pip install beautifulsoup4 lxml", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def prefer_ipv4_for_plain_socks5_proxy():
|
||||
"""PySocks may pick IPv6 first for socks5://; some SOCKS5 servers only accept IPv4."""
|
||||
proxy_envs = (
|
||||
os.environ.get("HTTPS_PROXY", ""),
|
||||
os.environ.get("HTTP_PROXY", ""),
|
||||
os.environ.get("https_proxy", ""),
|
||||
os.environ.get("http_proxy", ""),
|
||||
)
|
||||
uses_plain_socks5 = any(v.strip().lower().startswith("socks5://") for v in proxy_envs)
|
||||
if not uses_plain_socks5 or getattr(socket, "_spider91_ipv4_first", False):
|
||||
return
|
||||
|
||||
original_getaddrinfo = socket.getaddrinfo
|
||||
|
||||
def getaddrinfo_ipv4_first(*args, **kwargs):
|
||||
infos = original_getaddrinfo(*args, **kwargs)
|
||||
return sorted(infos, key=lambda info: 0 if info[0] == socket.AF_INET else 1)
|
||||
|
||||
socket.getaddrinfo = getaddrinfo_ipv4_first
|
||||
socket._spider91_ipv4_first = True
|
||||
|
||||
# ===================== 配置区域 =====================
|
||||
BASE_URL = "https://www.91porn.com/v.php"
|
||||
LIST_PARAMS = {
|
||||
"category": "top",
|
||||
"viewtype": "basic"
|
||||
}
|
||||
|
||||
# 请求头 (模拟真实浏览器)
|
||||
HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/125.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": (
|
||||
"text/html,application/xhtml+xml,application/xml;"
|
||||
"q=0.9,image/avif,image/webp,image/apng,*/*;"
|
||||
"q=0.8,application/signed-exchange;v=b3;q=0.7"
|
||||
),
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
# 注意: 不要包含 "br" (brotli),除非安装了 brotli 库
|
||||
# "Accept-Encoding": "gzip, deflate, br",
|
||||
"Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
}
|
||||
|
||||
# 延时配置 (秒) - 控制爬取频率,避免被封
|
||||
MIN_PAGE_DELAY = 3.0 # 列表页之间最小延时
|
||||
MAX_PAGE_DELAY = 6.0 # 列表页之间最大延时
|
||||
MIN_DETAIL_DELAY = 2.0 # 详情页之间最小延时
|
||||
MAX_DETAIL_DELAY = 5.0 # 详情页之间最大延时
|
||||
|
||||
# 重试配置
|
||||
MAX_RETRIES = 3
|
||||
RETRY_DELAY = 5.0
|
||||
|
||||
# 输出配置
|
||||
OUTPUT_FILE = "91porn_videos.json"
|
||||
MAX_PAGES = None # 设置为 None 爬取所有页,或设置整数如 5 只爬前5页
|
||||
RESUME = True # 是否跳过输出文件中已存在的 viewkey (断点续爬)
|
||||
MAX_EMPTY_PAGES = 2 # 连续空页数达到此值时停止爬取
|
||||
CRAWLER_NAME = "91Porn"
|
||||
CRAWLER_PROTOCOL = "crawler.v1"
|
||||
# ===================================================
|
||||
|
||||
|
||||
def crawler_source_id(raw: str) -> str:
|
||||
"""Return a backend-safe source_id, preserving existing numeric 91 IDs."""
|
||||
value = str(raw or "").strip()
|
||||
if not value:
|
||||
return ""
|
||||
safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", value).strip("._-")
|
||||
return safe[:160]
|
||||
|
||||
|
||||
def write_jsonl(event: dict):
|
||||
print(json.dumps(event, ensure_ascii=False), flush=True)
|
||||
|
||||
|
||||
class Porn91Spider:
|
||||
def __init__(
|
||||
self,
|
||||
output_file: str = None,
|
||||
start_page: int = 1,
|
||||
max_pages: int = None,
|
||||
resume: bool = None,
|
||||
max_empty_pages: int = None,
|
||||
quiet: bool = False,
|
||||
target_new: int = None,
|
||||
seen_viewkeys: list = None,
|
||||
stream_output: bool = False,
|
||||
stream_protocol: str = "legacy",
|
||||
):
|
||||
"""
|
||||
构造函数。所有参数都有默认值,等同于使用脚本顶部的全局配置。
|
||||
backend 调用时会传 output_file/seen_viewkeys/target_new,等价于:
|
||||
"从第 1 页开始爬,跳过 seen_viewkeys 里的视频,凑够 target_new 个新视频后停止"
|
||||
|
||||
stream_output=True 时(backend 流水线用):
|
||||
- 每凑齐一个 video 直链就把该 entry 作为一行 JSON 写到 stdout 并 flush,
|
||||
便于上层(Go crawler)边读边下载,不再等所有详情页处理完。
|
||||
- 所有日志改走 stderr,避免与 stdout JSONL 流混合。
|
||||
- --output 仍生效,作为离线归档用(脚本退出时一次性写完整 JSON)。
|
||||
"""
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(HEADERS)
|
||||
# 91porn 没有固定 mode cookie 时,详情页首次请求可能返回与列表卡片
|
||||
# 不一致的视频源;固定桌面模式让列表页和详情页解析保持一致。
|
||||
self.session.cookies.set("mode", "d")
|
||||
|
||||
# 解析后的实际配置;优先使用构造参数,回退到模块级配置
|
||||
self.output_file = output_file if output_file is not None else OUTPUT_FILE
|
||||
self.start_page = max(1, int(start_page or 1))
|
||||
# max_pages=None 表示不限制;max_pages=N 表示从 start_page 起爬 N 页
|
||||
self.max_pages = max_pages if max_pages is None or max_pages > 0 else None
|
||||
# resume 默认跟模块配置;单页模式下调用方应该显式传 False
|
||||
self.resume = RESUME if resume is None else bool(resume)
|
||||
self.max_empty_pages = (
|
||||
MAX_EMPTY_PAGES if max_empty_pages is None else int(max_empty_pages)
|
||||
)
|
||||
# target_new 是 backend 触发时的核心模式:累计处理这么多新源视频后退出。
|
||||
self.target_new = target_new if target_new and target_new > 0 else None
|
||||
self.quiet = bool(quiet)
|
||||
# stream_output:每解析出一个 video 直链立即输出一行 JSON 到 stdout
|
||||
# (配合 backend Go 端 bufio.Scanner 实时消费,下载一个就开始下一个)。
|
||||
# 开启后所有 log 都走 stderr。
|
||||
self.stream_output = bool(stream_output)
|
||||
self.stream_protocol = stream_protocol or "legacy"
|
||||
|
||||
# 添加重试适配器
|
||||
try:
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
retry_strategy = Retry(
|
||||
total=MAX_RETRIES,
|
||||
backoff_factor=1,
|
||||
status_forcelist=[429, 500, 502, 503, 504],
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry_strategy)
|
||||
self.session.mount("https://", adapter)
|
||||
self.session.mount("http://", adapter)
|
||||
except ImportError:
|
||||
pass # urllib3 版本可能较低
|
||||
|
||||
self.results = []
|
||||
self.pages_crawled = 0
|
||||
self.processed_videos = 0
|
||||
self.skipped_videos = 0
|
||||
self.failed_videos = 0
|
||||
self.skip_viewkeys = set()
|
||||
|
||||
# backend 通过 --seen-viewkeys-file 传进来一批已入库的历史 ID。
|
||||
# 兼容旧名:文件里可能是 viewkey,也可能是新逻辑使用的 mp4 源 ID。
|
||||
if seen_viewkeys:
|
||||
for vk in seen_viewkeys:
|
||||
if not vk:
|
||||
continue
|
||||
vk = vk.strip()
|
||||
if vk:
|
||||
self.skip_viewkeys.add(vk)
|
||||
|
||||
# 断点续爬:加载已有结果,跳过已处理的 viewkey
|
||||
if self.resume and os.path.exists(self.output_file):
|
||||
try:
|
||||
with open(self.output_file, 'r', encoding='utf-8') as f:
|
||||
existing_data = json.load(f)
|
||||
existing_videos = existing_data.get('videos', [])
|
||||
self.results = existing_videos
|
||||
for v in existing_videos:
|
||||
vk = v.get('viewkey', '')
|
||||
if vk:
|
||||
self.skip_viewkeys.add(vk)
|
||||
self.processed_videos = existing_data.get('successful', 0)
|
||||
self.failed_videos = existing_data.get('failed', 0)
|
||||
self.log(f"加载已有数据: {len(self.results)} 个视频, 将跳过已处理项")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def log(self, message: str):
|
||||
"""带时间戳的日志输出。stream_output 模式下走 stderr,避免污染 stdout JSONL。"""
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
line = f"[{timestamp}] {message}"
|
||||
if self.stream_output:
|
||||
print(line, file=sys.stderr, flush=True)
|
||||
else:
|
||||
print(line)
|
||||
|
||||
def emit_stream_video(self, video: dict):
|
||||
"""stream_output 模式下把单条 video entry 作为一行 JSON 写到 stdout 并立即刷盘。
|
||||
Go 端 bufio.Scanner 按行读取,每收到一行就立即下载视频和封面。"""
|
||||
if not self.stream_output:
|
||||
return
|
||||
try:
|
||||
if self.stream_protocol == "crawler.v1":
|
||||
source_id = crawler_source_id(video.get("source_id") or video.get("viewkey") or "")
|
||||
item = {
|
||||
"title": video.get("title") or "",
|
||||
"detail_url": video.get("detail_url") or "",
|
||||
"author": "91porn",
|
||||
"tags": ["91porn"],
|
||||
"media_url": video.get("video_url") or "",
|
||||
"thumbnail_url": video.get("thumb_url") or "",
|
||||
"headers": {
|
||||
"Referer": video.get("detail_url") or BASE_URL,
|
||||
},
|
||||
}
|
||||
if source_id:
|
||||
item["source_id"] = source_id
|
||||
event = {
|
||||
"type": "item",
|
||||
"item": item,
|
||||
}
|
||||
write_jsonl(event)
|
||||
else:
|
||||
print(json.dumps(video, ensure_ascii=False), flush=True)
|
||||
except Exception as e:
|
||||
# stdout 异常基本只在管道断开时发生(消费方进程死了);
|
||||
# 写到 stderr 让 backend 看到,然后让 crawl 循环自己 break。
|
||||
print(f"[stream] emit failed: {e}", file=sys.stderr, flush=True)
|
||||
|
||||
def random_sleep(self, min_sec: float, max_sec: float):
|
||||
"""随机延时,模拟人类行为"""
|
||||
delay = random.uniform(min_sec, max_sec)
|
||||
if not self.quiet:
|
||||
self.log(f" 随机延时 {delay:.2f} 秒...")
|
||||
time.sleep(delay)
|
||||
|
||||
def fetch_page(self, url: str, description: str = "", referer: str = "") -> str:
|
||||
"""
|
||||
获取页面HTML内容,带错误处理和重试
|
||||
"""
|
||||
headers_extra = {}
|
||||
if referer:
|
||||
headers_extra["Referer"] = referer
|
||||
|
||||
for attempt in range(1, MAX_RETRIES + 1):
|
||||
try:
|
||||
self.log(f"正在请求: {description or url} (尝试 {attempt}/{MAX_RETRIES})")
|
||||
response = self.session.get(url, timeout=30, headers=headers_extra)
|
||||
|
||||
# 检查是否被Cloudflare拦截 (需在 raise_for_status 之前)
|
||||
if response.status_code == 403:
|
||||
self.log("警告: 收到 403 Forbidden,可能被拦截")
|
||||
if attempt < MAX_RETRIES:
|
||||
self.random_sleep(RETRY_DELAY, RETRY_DELAY + 3)
|
||||
continue
|
||||
return ""
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
# 优先使用 content.decode('utf-8'),避免 requests 编码检测问题
|
||||
try:
|
||||
html_content = response.content.decode('utf-8', errors='replace')
|
||||
except Exception:
|
||||
html_content = response.text
|
||||
|
||||
# Cloudflare 挑战检测:如果页面主要内容只有挑战页面,而非正常内容
|
||||
# 注意:网站本身会加载 challenge-platform 脚本,所以不能仅凭此判断
|
||||
is_cf_challenge = (
|
||||
"Just a moment" in html_content and
|
||||
len(html_content) < 8000
|
||||
)
|
||||
if is_cf_challenge:
|
||||
self.log("警告: 页面被Cloudflare挑战拦截,需要浏览器环境或正确cookie")
|
||||
if attempt < MAX_RETRIES:
|
||||
self.random_sleep(RETRY_DELAY, RETRY_DELAY + 5)
|
||||
continue
|
||||
return ""
|
||||
|
||||
return html_content
|
||||
except requests.exceptions.HTTPError as e:
|
||||
self.log(f"HTTP错误: {e}")
|
||||
if attempt < MAX_RETRIES:
|
||||
self.random_sleep(RETRY_DELAY, RETRY_DELAY + 3)
|
||||
else:
|
||||
return ""
|
||||
except requests.exceptions.RequestException as e:
|
||||
self.log(f"请求失败: {e}")
|
||||
if attempt < MAX_RETRIES:
|
||||
self.random_sleep(RETRY_DELAY, RETRY_DELAY + 3)
|
||||
else:
|
||||
self.log(f"达到最大重试次数,放弃: {url}")
|
||||
return ""
|
||||
return ""
|
||||
|
||||
def parse_list_page(self, html: str) -> list:
|
||||
"""
|
||||
解析列表页,提取视频基本信息
|
||||
返回: [{title, detail_url, thumb_url, viewkey}, ...]
|
||||
"""
|
||||
videos = []
|
||||
soup = BeautifulSoup(html, 'lxml')
|
||||
|
||||
# 只解析正常视频卡片。页面中还混有 col-lg-8 的异常大卡片,里面的标题、
|
||||
# thumb、detail URL 会串到其它视频,不能作为入库来源。
|
||||
video_cards = soup.select('div.col-xs-12.col-sm-4.col-md-3.col-lg-3')
|
||||
|
||||
seen_cards = set()
|
||||
|
||||
for card in video_cards:
|
||||
link = card.find('a', href=re.compile(r'view_video\.php\?viewkey='))
|
||||
if not link:
|
||||
continue
|
||||
href = link.get('href', '')
|
||||
if not href:
|
||||
continue
|
||||
|
||||
# 提取 viewkey
|
||||
match = re.search(r'viewkey=([^&]+)', href)
|
||||
if not match:
|
||||
continue
|
||||
viewkey = match.group(1)
|
||||
|
||||
detail_url = urljoin(BASE_URL, href)
|
||||
|
||||
# 提取标题
|
||||
title = self._extract_title(link)
|
||||
|
||||
# 提取列表卡片来源 ID 和封面图 URL
|
||||
thumb_url = ""
|
||||
source_id = ""
|
||||
overlay = link.find(id=re.compile(r'^playvthumb_\d+$'))
|
||||
if overlay:
|
||||
source_id = overlay.get('id', '').rsplit('_', 1)[-1]
|
||||
img = link.find('img', class_=re.compile(r'img-responsive'))
|
||||
if img:
|
||||
thumb_url = img.get('src', '') or img.get('data-original', '')
|
||||
if thumb_url:
|
||||
thumb_url = urljoin(BASE_URL, thumb_url)
|
||||
if not source_id and thumb_url:
|
||||
source_id = self._extract_thumb_source_id(thumb_url)
|
||||
|
||||
card_key = source_id or detail_url
|
||||
if card_key in seen_cards:
|
||||
continue
|
||||
seen_cards.add(card_key)
|
||||
|
||||
videos.append({
|
||||
"title": title,
|
||||
"detail_url": detail_url,
|
||||
"thumb_url": thumb_url,
|
||||
"viewkey": viewkey,
|
||||
"source_id": source_id
|
||||
})
|
||||
|
||||
return videos
|
||||
|
||||
def _extract_title(self, link) -> str:
|
||||
"""
|
||||
从视频链接标签中提取并清理标题
|
||||
"""
|
||||
# 优先从 span.video-title 获取 (已渲染的干净标题)
|
||||
title_el = link.find('span', class_=re.compile(r'video-title'))
|
||||
if title_el:
|
||||
title = title_el.get_text(strip=True)
|
||||
if title:
|
||||
return html.unescape(title)
|
||||
|
||||
# 备用: 从 link 的 title 属性提取
|
||||
title = link.get('title', '').strip()
|
||||
if title:
|
||||
return html.unescape(title)
|
||||
|
||||
# 最后手段: 从链接文本提取并清理前缀
|
||||
text = link.get_text(separator=' ', strip=True)
|
||||
# 去掉前缀: "HD" / "91" / 时间戳 "HH:MM:SS"
|
||||
text = re.sub(r'^(HD\s+|91\s+)?\d{2}:\d{2}:\d{2}\s*', '', text)
|
||||
text = re.sub(r'\s+', ' ', text).strip()
|
||||
return html.unescape(text)[:120]
|
||||
|
||||
def parse_detail_page(self, html: str) -> dict:
|
||||
"""
|
||||
解析详情页,提取视频直链
|
||||
返回: {"video_url": "...", "source_id": "...", "title": "..."} 或空字典
|
||||
"""
|
||||
result = {}
|
||||
|
||||
if not html:
|
||||
return result
|
||||
|
||||
title = self._extract_detail_title(html)
|
||||
if title:
|
||||
result["title"] = title
|
||||
|
||||
# 方法1: 解码 strencode2 (主要方式, 页面通过 document.write 动态写入 video 标签)
|
||||
# 格式: document.write(strencode2("%3c%73%6f..."));
|
||||
strencode_match = re.search(r'strencode2\(["\']([^"\']+)["\']\)', html)
|
||||
if strencode_match:
|
||||
encoded = strencode_match.group(1)
|
||||
try:
|
||||
# strencode2 在JS中等价于 unescape / decodeURIComponent
|
||||
decoded = unquote(encoded)
|
||||
|
||||
# 从解码后的 HTML 片段中提取 src
|
||||
src_match = re.search(r"src=['\"]([^'\"]+)['\"]", decoded)
|
||||
if src_match:
|
||||
video_url = src_match.group(1)
|
||||
# 规范化双斜杠 (如 https://host//path -> https://host/path)
|
||||
video_url = re.sub(r'(https?://[^/]+)//+', r'\1/', video_url)
|
||||
result["video_url"] = video_url
|
||||
result["source_id"] = self._extract_source_id(video_url)
|
||||
return result
|
||||
except Exception as e:
|
||||
self.log(f" 解码 strencode2 失败: {e}")
|
||||
|
||||
# 方法2: 通用正则匹配页面中的 mp4 链接 (备用, 过滤广告)
|
||||
mp4_match = re.search(
|
||||
r'https?://[^\s"\'<>]+\.mp4[^\s"\'<>]*',
|
||||
html
|
||||
)
|
||||
if mp4_match:
|
||||
url = mp4_match.group(0)
|
||||
if 'kwai' not in url and 'ad-' not in url.lower():
|
||||
result["video_url"] = url
|
||||
result["source_id"] = self._extract_source_id(url)
|
||||
return result
|
||||
|
||||
return result
|
||||
|
||||
def _extract_detail_title(self, html_text: str) -> str:
|
||||
soup = BeautifulSoup(html_text, 'lxml')
|
||||
title_el = soup.find('title')
|
||||
if not title_el:
|
||||
return ""
|
||||
title = title_el.get_text(" ", strip=True)
|
||||
title = re.sub(r'\s*-\s*91porn.*$', '', title, flags=re.IGNORECASE).strip()
|
||||
return html.unescape(title)[:160]
|
||||
|
||||
def _extract_source_id(self, video_url: str) -> str:
|
||||
path = urlparse(video_url or "").path
|
||||
name = os.path.basename(path)
|
||||
stem, ext = os.path.splitext(name)
|
||||
if ext.lower() not in {".mp4", ".m4v", ".mov", ".webm", ".mkv", ".avi"}:
|
||||
return ""
|
||||
source_id = re.sub(r'[^0-9]+', '', stem)
|
||||
if not source_id or source_id != stem:
|
||||
return ""
|
||||
return source_id
|
||||
|
||||
def _extract_thumb_source_id(self, thumb_url: str) -> str:
|
||||
path = urlparse(thumb_url or "").path
|
||||
match = re.search(r'/thumb/(\d+)\.[A-Za-z0-9]+$', path)
|
||||
return match.group(1) if match else ""
|
||||
|
||||
def _thumb_url_for_source(self, thumb_url: str, source_id: str) -> str:
|
||||
if not thumb_url or not source_id:
|
||||
return thumb_url
|
||||
parsed = urlparse(thumb_url)
|
||||
match = re.search(r'/thumb/([^/?#]+)\.[A-Za-z0-9]+$', parsed.path)
|
||||
if not match:
|
||||
return thumb_url
|
||||
current = match.group(1)
|
||||
if current == source_id:
|
||||
return thumb_url
|
||||
path = re.sub(
|
||||
r'/thumb/[^/?#]+\.[A-Za-z0-9]+$',
|
||||
f'/thumb/{source_id}.jpg',
|
||||
parsed.path,
|
||||
)
|
||||
return parsed._replace(path=path, query="", fragment="").geturl()
|
||||
|
||||
def crawl(self):
|
||||
"""
|
||||
主爬取流程。停止条件(任一满足即停):
|
||||
- 达到 max_pages 配置
|
||||
- 连续 max_empty_pages 页都没有视频
|
||||
- target_new 模式下,已经累计处理 target_new 个新视频
|
||||
"""
|
||||
self.log("=" * 60)
|
||||
self.log("91porn 视频爬虫启动")
|
||||
self.log("=" * 60)
|
||||
self.log(f"配置: 列表页延时 {MIN_PAGE_DELAY}-{MAX_PAGE_DELAY}s, 详情页延时 {MIN_DETAIL_DELAY}-{MAX_DETAIL_DELAY}s")
|
||||
self.log(f"配置: 最大重试 {MAX_RETRIES} 次, 连续空页上限 {self.max_empty_pages}")
|
||||
self.log(f"配置: 起始页 {self.start_page}, 最大爬取页数 {self.max_pages if self.max_pages else '不限'}")
|
||||
if self.target_new:
|
||||
self.log(f"配置: 目标新增视频数 {self.target_new}")
|
||||
self.log(f"配置: 输出文件 {os.path.abspath(self.output_file)}")
|
||||
if self.skip_viewkeys:
|
||||
self.log(f"配置: 已跳过 {len(self.skip_viewkeys)} 个已知 viewkey")
|
||||
self.log("")
|
||||
|
||||
page_num = self.start_page
|
||||
consecutive_empty = 0
|
||||
crawled_in_session = 0
|
||||
|
||||
while True:
|
||||
if self.max_pages is not None and crawled_in_session >= self.max_pages:
|
||||
self.log(f"达到配置的页数上限 {self.max_pages},停止")
|
||||
break
|
||||
if consecutive_empty >= self.max_empty_pages:
|
||||
self.log(f"连续 {self.max_empty_pages} 页无结果,已达到末尾")
|
||||
break
|
||||
if self.target_new is not None and self.processed_videos >= self.target_new:
|
||||
self.log(f"已累计 {self.processed_videos} 个新视频,达到目标 {self.target_new},停止")
|
||||
break
|
||||
|
||||
if page_num == 1:
|
||||
page_url = f"{BASE_URL}?category=top&viewtype=basic"
|
||||
else:
|
||||
page_url = f"{BASE_URL}?category=top&viewtype=basic&page={page_num}"
|
||||
|
||||
if crawled_in_session > 0:
|
||||
self.log("")
|
||||
self.random_sleep(MIN_PAGE_DELAY, MAX_PAGE_DELAY)
|
||||
|
||||
self.log(f"[页 {page_num}] 请求: {page_url}")
|
||||
page_html = self.fetch_page(page_url, f"列表页 第{page_num}页")
|
||||
|
||||
if not page_html:
|
||||
self.log(f"[页 {page_num}] 获取失败,跳过")
|
||||
consecutive_empty += 1
|
||||
page_num += 1
|
||||
crawled_in_session += 1
|
||||
continue
|
||||
|
||||
page_videos = self.parse_list_page(page_html)
|
||||
|
||||
# 判断页面是否真的没有视频(而非全部已处理)
|
||||
if not page_videos:
|
||||
self.log(f"[页 {page_num}] 页面无视频,可能已到末尾")
|
||||
consecutive_empty += 1
|
||||
page_num += 1
|
||||
crawled_in_session += 1
|
||||
continue
|
||||
|
||||
consecutive_empty = 0
|
||||
|
||||
# 过滤已处理的 viewkey,只保留新视频
|
||||
new_videos = [v for v in page_videos if v['viewkey'] not in self.skip_viewkeys]
|
||||
skipped_on_page = len(page_videos) - len(new_videos)
|
||||
|
||||
if skipped_on_page > 0:
|
||||
self.log(f"[页 {page_num}] 发现 {len(page_videos)} 个链接, 其中 {skipped_on_page} 个已处理, {len(new_videos)} 个新视频")
|
||||
else:
|
||||
self.log(f"[页 {page_num}] 发现 {len(new_videos)} 个视频")
|
||||
|
||||
if new_videos:
|
||||
self._process_video_list(new_videos, referer=page_url)
|
||||
self.pages_crawled += 1
|
||||
page_num += 1
|
||||
crawled_in_session += 1
|
||||
|
||||
self._save_results()
|
||||
self._print_summary()
|
||||
|
||||
def _process_video_list(self, videos: list, referer: str = ""):
|
||||
"""
|
||||
处理一批视频列表,逐个获取详情页
|
||||
"""
|
||||
for idx, video in enumerate(videos, 1):
|
||||
# target_new 模式下,凑够后立即停止,不再请求详情页
|
||||
if self.target_new is not None and self.processed_videos >= self.target_new:
|
||||
return
|
||||
# 跳过已处理的 viewkey (断点续爬)
|
||||
if video['viewkey'] in self.skip_viewkeys:
|
||||
self.log(f" [SKIP] 已处理过: {video['viewkey']}")
|
||||
self.skipped_videos += 1
|
||||
continue
|
||||
|
||||
self.log(f" 处理视频 {idx}/{len(videos)}: {video['title'][:40]}...")
|
||||
|
||||
# 延时控制 (同一批次内第一个视频不延时)
|
||||
if idx > 1:
|
||||
self.random_sleep(MIN_DETAIL_DELAY, MAX_DETAIL_DELAY)
|
||||
|
||||
# 获取详情页
|
||||
detail_html = self.fetch_page(video['detail_url'], f"详情页 viewkey={video['viewkey']}", referer=referer)
|
||||
|
||||
if not detail_html:
|
||||
self.log(f" [FAIL] 详情页获取失败: {video['viewkey']}")
|
||||
video["video_url"] = ""
|
||||
self.results.append(video)
|
||||
self.skip_viewkeys.add(video['viewkey'])
|
||||
self.failed_videos += 1
|
||||
continue
|
||||
|
||||
# 解析视频直链
|
||||
detail_info = self.parse_detail_page(detail_html)
|
||||
|
||||
if detail_info.get("video_url"):
|
||||
video["video_url"] = detail_info["video_url"]
|
||||
if detail_info.get("title"):
|
||||
video["title"] = detail_info["title"]
|
||||
list_source_id = video.get("source_id", "")
|
||||
detail_source_id = detail_info.get("source_id", "")
|
||||
if list_source_id and detail_source_id and list_source_id != detail_source_id:
|
||||
self.log(
|
||||
f" [FAIL] 详情页视频源不匹配: list_source_id={list_source_id} "
|
||||
f"detail_source_id={detail_source_id} viewkey={video['viewkey']}"
|
||||
)
|
||||
self.failed_videos += 1
|
||||
self.skip_viewkeys.add(video['viewkey'])
|
||||
continue
|
||||
if not list_source_id and detail_source_id:
|
||||
video["source_id"] = detail_source_id
|
||||
if video.get("source_id"):
|
||||
video["thumb_url"] = self._thumb_url_for_source(
|
||||
video.get("thumb_url", ""),
|
||||
video["source_id"],
|
||||
)
|
||||
if video["source_id"] in self.skip_viewkeys:
|
||||
self.log(f" [SKIP] 已处理过 source_id: {video['source_id']}")
|
||||
self.skipped_videos += 1
|
||||
continue
|
||||
self.results.append(video)
|
||||
self.skip_viewkeys.add(video['viewkey'])
|
||||
if video.get("source_id"):
|
||||
self.skip_viewkeys.add(video["source_id"])
|
||||
self.processed_videos += 1
|
||||
self.log(f" [OK] 成功提取视频直链")
|
||||
# 流式:立刻把这条 entry 交给 Go 端开始下载,不等本批余下视频
|
||||
self.emit_stream_video(video)
|
||||
else:
|
||||
self.log(f" [FAIL] 未找到视频直链: {video['viewkey']}")
|
||||
video["video_url"] = ""
|
||||
self.results.append(video)
|
||||
self.skip_viewkeys.add(video['viewkey'])
|
||||
self.failed_videos += 1
|
||||
|
||||
def _save_results(self):
|
||||
"""
|
||||
保存结果到JSON文件
|
||||
"""
|
||||
output_data = {
|
||||
"crawl_time": datetime.now().isoformat(),
|
||||
"source_url": BASE_URL,
|
||||
"pages_crawled": self.pages_crawled,
|
||||
"total_videos": len(self.results),
|
||||
"successful": self.processed_videos,
|
||||
"skipped": self.skipped_videos,
|
||||
"failed": self.failed_videos,
|
||||
"videos": self.results
|
||||
}
|
||||
|
||||
try:
|
||||
# 保证父目录存在;写入临时文件后原子 rename,避免读到半截 JSON
|
||||
out_path = self.output_file
|
||||
parent = os.path.dirname(os.path.abspath(out_path))
|
||||
if parent:
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
tmp_path = out_path + ".part"
|
||||
with open(tmp_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(output_data, f, ensure_ascii=False, indent=2)
|
||||
os.replace(tmp_path, out_path)
|
||||
self.log(f"结果已保存到: {os.path.abspath(out_path)}")
|
||||
except Exception as e:
|
||||
self.log(f"保存文件失败: {e}")
|
||||
# 尝试输出到控制台作为备份
|
||||
backup_out = sys.stderr if self.stream_output else sys.stdout
|
||||
print("\n--- 备份输出 ---", file=backup_out, flush=True)
|
||||
print(json.dumps(output_data, ensure_ascii=False, indent=2), file=backup_out, flush=True)
|
||||
|
||||
def _print_summary(self):
|
||||
"""
|
||||
打印爬取摘要
|
||||
"""
|
||||
self.log("")
|
||||
self.log("=" * 60)
|
||||
self.log("爬取完成!")
|
||||
self.log("=" * 60)
|
||||
self.log(f"爬取页数: {self.pages_crawled}")
|
||||
self.log(f"总视频数: {len(self.results)}")
|
||||
self.log(f"成功提取直链: {self.processed_videos}")
|
||||
self.log(f"跳过(已处理): {self.skipped_videos}")
|
||||
self.log(f"失败/缺失直链: {self.failed_videos}")
|
||||
self.log(f"输出文件: {os.path.abspath(self.output_file)}")
|
||||
self.log("=" * 60)
|
||||
|
||||
|
||||
def print_help():
|
||||
print("""
|
||||
================================================
|
||||
91porn 视频爬虫 v1.0
|
||||
================================================
|
||||
|
||||
本脚本将爬取 91porn "本月最热" 分类下的所有视频信息:
|
||||
- 视频名称
|
||||
- 封面图直链
|
||||
- 视频直链 (MP4)
|
||||
|
||||
依赖安装:
|
||||
pip install requests beautifulsoup4 lxml PySocks
|
||||
|
||||
使用方法:
|
||||
python spider_91porn.py
|
||||
|
||||
配置说明 (编辑脚本内 "配置区域"):
|
||||
MIN_PAGE_DELAY / MAX_PAGE_DELAY : 列表页请求间隔 (默认 3-6 秒)
|
||||
MIN_DETAIL_DELAY / MAX_DETAIL_DELAY : 详情页请求间隔 (默认 2-5 秒)
|
||||
MAX_PAGES : 限制最大爬取页数 (None=不限, 如 5=只爬前5页)
|
||||
OUTPUT_FILE : 输出文件名 (默认 91porn_videos.json)
|
||||
|
||||
按 Ctrl+C 可随时中断并保存已爬取的数据
|
||||
|
||||
注意:
|
||||
1. 视频直链包含时效性token,会过期,需定期重新爬取
|
||||
2. 脚本已内置随机延时,请勿移除,避免对服务器造成压力
|
||||
3. 如遇到Cloudflare拦截,需要先通过浏览器获取Cookie
|
||||
4. 本脚本仅供学习交流,请遵守当地法律法规
|
||||
================================================
|
||||
""")
|
||||
|
||||
|
||||
def run_job(job_path: str):
|
||||
"""Run as a crawler.v1 script plugin.
|
||||
|
||||
The Go host passes a job JSON file and expects stdout JSONL events. Logs go
|
||||
to stderr so stdout stays machine-readable.
|
||||
"""
|
||||
with open(job_path, "r", encoding="utf-8") as f:
|
||||
job = json.load(f)
|
||||
|
||||
if job.get("protocol") != CRAWLER_PROTOCOL:
|
||||
raise ValueError(f"unsupported crawler protocol: {job.get('protocol')!r}")
|
||||
if job.get("mode") not in ("", None, "crawl"):
|
||||
raise ValueError(f"unsupported crawler mode: {job.get('mode')!r}")
|
||||
|
||||
try:
|
||||
target_new = int(job.get("target_new") or 15)
|
||||
except (TypeError, ValueError):
|
||||
target_new = 15
|
||||
if target_new <= 0:
|
||||
target_new = 15
|
||||
seen_file = job.get("seen_source_ids_file") or ""
|
||||
output_dir = job.get("output_dir") or os.getcwd()
|
||||
run_id = job.get("run_id") or datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
output_file = os.path.join(output_dir, f"spider91-{run_id}.json")
|
||||
|
||||
network = job.get("network") if isinstance(job.get("network"), dict) else {}
|
||||
proxy_url = str(network.get("proxy_url") or "").strip()
|
||||
if proxy_url:
|
||||
os.environ["HTTP_PROXY"] = proxy_url
|
||||
os.environ["HTTPS_PROXY"] = proxy_url
|
||||
os.environ["http_proxy"] = proxy_url
|
||||
os.environ["https_proxy"] = proxy_url
|
||||
os.environ["NO_PROXY"] = ""
|
||||
os.environ["no_proxy"] = ""
|
||||
|
||||
seen_viewkeys = []
|
||||
if seen_file:
|
||||
try:
|
||||
with open(seen_file, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
seen_viewkeys.append(line)
|
||||
except FileNotFoundError:
|
||||
print(f"警告: seen_source_ids_file 不存在: {seen_file}", file=sys.stderr, flush=True)
|
||||
except Exception as e:
|
||||
print(f"警告: 读取 seen_source_ids_file 失败: {e}", file=sys.stderr, flush=True)
|
||||
|
||||
prefer_ipv4_for_plain_socks5_proxy()
|
||||
spider = Porn91Spider(
|
||||
output_file=output_file,
|
||||
start_page=1,
|
||||
max_pages=None,
|
||||
resume=False,
|
||||
quiet=True,
|
||||
target_new=target_new,
|
||||
seen_viewkeys=seen_viewkeys,
|
||||
stream_output=True,
|
||||
stream_protocol="crawler.v1",
|
||||
)
|
||||
try:
|
||||
spider.crawl()
|
||||
done = {
|
||||
"type": "done",
|
||||
"stats": {
|
||||
"emitted": spider.processed_videos,
|
||||
"failed": spider.failed_videos,
|
||||
"skipped": spider.skipped_videos,
|
||||
},
|
||||
}
|
||||
write_jsonl(done)
|
||||
except KeyboardInterrupt:
|
||||
spider.log("\n用户中断,正在保存已爬取的数据...")
|
||||
spider._save_results()
|
||||
raise
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) > 1 and sys.argv[1] in ('-h', '--help', 'help'):
|
||||
print_help()
|
||||
return
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="spider_91porn.py",
|
||||
description="91porn 视频元数据爬虫",
|
||||
add_help=False, # 让 -h/--help 走 print_help() 中文版本
|
||||
)
|
||||
parser.add_argument("--page", type=int, default=None,
|
||||
help="只爬指定页(单页模式,配合 --output 用于定时任务)")
|
||||
parser.add_argument("--output", type=str, default=None,
|
||||
help="输出 JSON 路径,覆盖默认 OUTPUT_FILE")
|
||||
parser.add_argument("--max-pages", type=int, default=None,
|
||||
help="单页模式下,从 --page 起最多再爬几页(默认 1)")
|
||||
parser.add_argument("--no-resume", action="store_true",
|
||||
help="禁用断点续爬(单页模式默认禁用)")
|
||||
parser.add_argument("--quiet", action="store_true",
|
||||
help="压缩日志,每条视频只输出关键事件")
|
||||
parser.add_argument("--target-new", type=int, default=None,
|
||||
help="目标新增模式:从 page 1 起翻页直到累计处理这么多新源视频后停止(backend 凌晨任务用)")
|
||||
parser.add_argument("--seen-viewkeys-file", type=str, default=None,
|
||||
help="文件路径,每行一个已处理过的 viewkey 或 mp4 源 ID;脚本会跳过这些视频")
|
||||
parser.add_argument("--stream-output", action="store_true",
|
||||
help="流式模式:每解析一条视频直链就立即把它作为一行 JSON 写到 stdout 并 flush;"
|
||||
"日志改走 stderr。配合 backend 边读边下载使用。")
|
||||
parser.add_argument("--job", type=str, default=None,
|
||||
help="crawler.v1 job JSON 路径;作为通用脚本爬虫运行。")
|
||||
|
||||
args, _ = parser.parse_known_args()
|
||||
if args.job:
|
||||
run_job(args.job)
|
||||
return
|
||||
|
||||
cli_out = sys.stderr if args.stream_output else sys.stdout
|
||||
prefer_ipv4_for_plain_socks5_proxy()
|
||||
|
||||
print("""
|
||||
================================================
|
||||
91porn 视频爬虫启动中...
|
||||
================================================
|
||||
按 Ctrl+C 可随时中断并保存进度
|
||||
""", file=cli_out)
|
||||
|
||||
# 加载已知 ID(来自 backend 的 catalog 已入库列表;兼容旧参数名)
|
||||
seen_viewkeys = []
|
||||
if args.seen_viewkeys_file:
|
||||
try:
|
||||
with open(args.seen_viewkeys_file, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
seen_viewkeys.append(line)
|
||||
except FileNotFoundError:
|
||||
print(f"警告: --seen-viewkeys-file 不存在: {args.seen_viewkeys_file}", file=cli_out)
|
||||
except Exception as e:
|
||||
print(f"警告: 读取 --seen-viewkeys-file 失败: {e}", file=cli_out)
|
||||
|
||||
# 决定运行模式
|
||||
if args.target_new is not None:
|
||||
# 凑够 N 个新视频模式:从 page 1 起翻页,直到累计 target_new 个新视频
|
||||
spider = Porn91Spider(
|
||||
output_file=args.output,
|
||||
start_page=1,
|
||||
max_pages=None,
|
||||
resume=False, # 凑够 N 模式靠 seen_viewkeys 去重,不读 OUTPUT_FILE
|
||||
quiet=args.quiet,
|
||||
target_new=args.target_new,
|
||||
seen_viewkeys=seen_viewkeys,
|
||||
stream_output=args.stream_output,
|
||||
)
|
||||
elif args.page is not None:
|
||||
# 单页模式(保留作手动调试用):start_page=N, max_pages=1
|
||||
start_page = max(1, args.page)
|
||||
max_pages = args.max_pages if args.max_pages and args.max_pages > 0 else 1
|
||||
spider = Porn91Spider(
|
||||
output_file=args.output,
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
resume=False,
|
||||
quiet=args.quiet,
|
||||
seen_viewkeys=seen_viewkeys,
|
||||
stream_output=args.stream_output,
|
||||
)
|
||||
else:
|
||||
# 全量模式(向后兼容):从 page 1 起爬到末尾
|
||||
spider = Porn91Spider(
|
||||
output_file=args.output,
|
||||
resume=False if args.no_resume else None,
|
||||
quiet=args.quiet,
|
||||
seen_viewkeys=seen_viewkeys,
|
||||
stream_output=args.stream_output,
|
||||
)
|
||||
|
||||
try:
|
||||
spider.crawl()
|
||||
except KeyboardInterrupt:
|
||||
spider.log("\n用户中断,正在保存已爬取的数据...")
|
||||
spider._save_results()
|
||||
spider._print_summary()
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
spider.log(f"发生未预料的错误: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
spider._save_results()
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+134
-14
@@ -214,6 +214,9 @@ func main() {
|
||||
}
|
||||
return app.scheduleScan(ctx, driveID)
|
||||
},
|
||||
OnCrawlerUploadRequested: func(driveID string) (bool, string) {
|
||||
return app.scheduleManualCrawlerUploadMigration(ctx, driveID)
|
||||
},
|
||||
OnStopDriveTasks: func(driveID string) bool {
|
||||
return app.stopDriveTasks(ctx, driveID)
|
||||
},
|
||||
@@ -938,18 +941,20 @@ func (a *App) attachDriveUnlocked(ctx context.Context, d *catalog.Drive) error {
|
||||
})
|
||||
case "p115":
|
||||
drv = p115.New(p115.Config{
|
||||
ID: d.ID,
|
||||
Cookie: d.Credentials["cookie"],
|
||||
RootID: d.RootID,
|
||||
ID: d.ID,
|
||||
Cookie: d.Credentials["cookie"],
|
||||
RootID: d.RootID,
|
||||
UploadTempDir: a.uploadWorkDir("p115"),
|
||||
})
|
||||
case p123.Kind:
|
||||
drv = p123.New(p123.Config{
|
||||
ID: d.ID,
|
||||
Username: d.Credentials["username"],
|
||||
Password: d.Credentials["password"],
|
||||
AccessToken: d.Credentials["access_token"],
|
||||
Platform: d.Credentials["platform"],
|
||||
RootID: d.RootID,
|
||||
ID: d.ID,
|
||||
Username: d.Credentials["username"],
|
||||
Password: d.Credentials["password"],
|
||||
AccessToken: d.Credentials["access_token"],
|
||||
Platform: d.Credentials["platform"],
|
||||
RootID: d.RootID,
|
||||
UploadTempDir: a.uploadWorkDir(p123.Kind),
|
||||
OnTokenUpdate: func(access string) {
|
||||
if d.Credentials == nil {
|
||||
d.Credentials = make(map[string]string)
|
||||
@@ -970,6 +975,7 @@ func (a *App) attachDriveUnlocked(ctx context.Context, d *catalog.Drive) error {
|
||||
DeviceID: d.Credentials["device_id"],
|
||||
RootID: d.RootID,
|
||||
DisableMediaLink: pikpak.ParseBoolDefault(d.Credentials["disable_media_link"], true),
|
||||
UploadTempDir: a.uploadWorkDir("pikpak"),
|
||||
OnTokenUpdate: func(access, refresh, captcha, deviceID string) {
|
||||
d.Credentials["access_token"] = access
|
||||
d.Credentials["refresh_token"] = refresh
|
||||
@@ -980,11 +986,12 @@ func (a *App) attachDriveUnlocked(ctx context.Context, d *catalog.Drive) error {
|
||||
})
|
||||
case "wopan":
|
||||
drv = wopan.New(wopan.Config{
|
||||
ID: d.ID,
|
||||
AccessToken: d.Credentials["access_token"],
|
||||
RefreshToken: d.Credentials["refresh_token"],
|
||||
FamilyID: d.Credentials["family_id"],
|
||||
RootID: d.RootID,
|
||||
ID: d.ID,
|
||||
AccessToken: d.Credentials["access_token"],
|
||||
RefreshToken: d.Credentials["refresh_token"],
|
||||
FamilyID: d.Credentials["family_id"],
|
||||
RootID: d.RootID,
|
||||
UploadTempDir: a.uploadWorkDir("wopan"),
|
||||
OnTokenUpdate: func(access, refresh string) {
|
||||
d.Credentials["access_token"] = access
|
||||
d.Credentials["refresh_token"] = refresh
|
||||
@@ -1158,6 +1165,17 @@ func (a *App) localUploadDir() string {
|
||||
return filepath.Join(filepath.Dir(a.cfg.Storage.LocalPreviewDir), "uploads")
|
||||
}
|
||||
|
||||
func (a *App) uploadWorkDir(kind string) string {
|
||||
if a == nil || a.cfg == nil || strings.TrimSpace(a.cfg.Storage.LocalPreviewDir) == "" {
|
||||
return ""
|
||||
}
|
||||
kind = strings.Trim(strings.ToLower(strings.TrimSpace(kind)), string(filepath.Separator))
|
||||
if kind == "" {
|
||||
kind = "generic"
|
||||
}
|
||||
return filepath.Join(filepath.Dir(a.cfg.Storage.LocalPreviewDir), "upload-tmp", kind)
|
||||
}
|
||||
|
||||
func fingerprintConfigForDrive(drv drives.Drive) fingerprint.Config {
|
||||
cfg := fingerprint.Config{RateLimitCooldown: 5 * time.Minute}
|
||||
if drv == nil {
|
||||
@@ -3285,6 +3303,108 @@ func (a *App) runCrawlerUploadMigrationAfterSave(ctx context.Context, driveID st
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) scheduleManualCrawlerUploadMigration(ctx context.Context, driveID string) (bool, string) {
|
||||
driveID = strings.TrimSpace(driveID)
|
||||
if driveID == "" || a == nil || a.cat == nil {
|
||||
return false, "爬虫不存在"
|
||||
}
|
||||
if a.spider91Migrator == nil {
|
||||
return false, "上传迁移器未初始化"
|
||||
}
|
||||
if a.driveHasActiveWork(driveID) {
|
||||
return false, "当前爬虫有正在进行的任务,请稍后重试"
|
||||
}
|
||||
d, err := a.cat.GetDrive(ctx, driveID)
|
||||
if err != nil || d == nil || d.Kind != scriptcrawler.Kind {
|
||||
return false, "爬虫不存在"
|
||||
}
|
||||
targetDriveID := strings.TrimSpace(d.Credentials["upload_drive_id"])
|
||||
if targetDriveID == "" {
|
||||
return false, "请先配置上传网盘"
|
||||
}
|
||||
assets, err := a.cat.CountCrawlerAssets(ctx, driveID, crawlerCatalogVideoIDPrefixes(d))
|
||||
if err != nil {
|
||||
log.Printf("[scriptcrawler] drive=%s manual upload count assets: %v", driveID, err)
|
||||
return false, "读取待上传视频失败"
|
||||
}
|
||||
if reason := crawlerUploadAssetBlockReason(d, assets); reason != "" {
|
||||
return false, reason
|
||||
}
|
||||
if err := a.ensureDriveAttached(ctx, driveID); err != nil {
|
||||
log.Printf("[scriptcrawler] drive=%s manual upload source attach: %v", driveID, err)
|
||||
return false, "爬虫本地存储不可用"
|
||||
}
|
||||
if err := a.ensureDriveAttached(ctx, targetDriveID); err != nil {
|
||||
log.Printf("[scriptcrawler] drive=%s manual upload target=%s attach: %v", driveID, targetDriveID, err)
|
||||
return false, "上传网盘不可用:" + err.Error()
|
||||
}
|
||||
|
||||
a.crawlerUploadMu.Lock()
|
||||
if a.crawlerUploadRunning == nil {
|
||||
a.crawlerUploadRunning = make(map[string]bool)
|
||||
}
|
||||
if a.crawlerUploadRunning[driveID] {
|
||||
a.crawlerUploadMu.Unlock()
|
||||
return false, "当前爬虫已有上传任务正在运行"
|
||||
}
|
||||
a.crawlerUploadRunning[driveID] = true
|
||||
a.crawlerUploadMu.Unlock()
|
||||
|
||||
taskCtx, done := a.registerDriveTaskContext(ctx, driveID)
|
||||
go func() {
|
||||
defer func() {
|
||||
done()
|
||||
a.crawlerUploadMu.Lock()
|
||||
delete(a.crawlerUploadRunning, driveID)
|
||||
a.crawlerUploadMu.Unlock()
|
||||
}()
|
||||
a.runManualCrawlerUploadMigration(taskCtx, driveID, targetDriveID)
|
||||
}()
|
||||
return true, ""
|
||||
}
|
||||
|
||||
func crawlerUploadAssetBlockReason(d *catalog.Drive, assets catalog.CrawlerAssetCounts) string {
|
||||
if assets.Local <= 0 {
|
||||
return "没有待上传的本地视频"
|
||||
}
|
||||
if assets.Fingerprint.Pending > 0 {
|
||||
return "还有待生成的视频指纹"
|
||||
}
|
||||
if assets.Fingerprint.Failed > 0 {
|
||||
return "存在指纹生成失败的视频,请先重试或处理失败项"
|
||||
}
|
||||
if d != nil && d.TeaserEnabled {
|
||||
if assets.Teaser.Pending > 0 {
|
||||
return "还有待生成的预览视频"
|
||||
}
|
||||
if assets.Teaser.Failed > 0 {
|
||||
return "存在预览视频生成失败的视频,请先重试或处理失败项"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func crawlerCatalogVideoIDPrefixes(d *catalog.Drive) []string {
|
||||
if d == nil {
|
||||
return nil
|
||||
}
|
||||
return []string{
|
||||
scriptcrawler.Kind + "-" + d.ID + "-",
|
||||
spider91.Kind + "-" + d.ID + "-",
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) runManualCrawlerUploadMigration(ctx context.Context, driveID, targetDriveID string) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
log.Printf("[scriptcrawler] drive=%s skip manual upload migration: %v", driveID, err)
|
||||
return
|
||||
}
|
||||
log.Printf("[scriptcrawler] drive=%s running manual upload migration target=%s", driveID, targetDriveID)
|
||||
if err := a.spider91Migrator.RunOnce(ctx); err != nil {
|
||||
log.Printf("[scriptcrawler] drive=%s manual upload migration: %v", driveID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) runCrawlerMigrationAfterManualCrawl(ctx context.Context, driveID string) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
log.Printf("[scriptcrawler] drive=%s skip post-crawl migration: %v", driveID, err)
|
||||
|
||||
@@ -578,6 +578,128 @@ func TestScheduleCrawlerUploadMigrationSkipsWithoutUploadTarget(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleManualCrawlerUploadMigrationRunsWhenAssetsReady(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "crawler-ready",
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Ready Crawler",
|
||||
RootID: "/",
|
||||
TeaserEnabled: true,
|
||||
Credentials: map[string]string{
|
||||
"script_path": "/tmp/ready.py",
|
||||
"upload_drive_id": "pikpak-target",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed crawler: %v", err)
|
||||
}
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: scriptcrawler.BuildVideoID("crawler-ready", "source-1"),
|
||||
DriveID: "crawler-ready",
|
||||
FileID: "source-1.mp4",
|
||||
FileName: "source-1.mp4",
|
||||
Title: "Source 1",
|
||||
Size: 123,
|
||||
Ext: "mp4",
|
||||
SampledSHA256: "sampled-source-1",
|
||||
FingerprintStatus: "ready",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: time.Now(),
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
registry := proxy.NewRegistry()
|
||||
registry.Set("crawler-ready", &serverFakeKindDrive{id: "crawler-ready", kind: scriptcrawler.Kind})
|
||||
registry.Set("pikpak-target", &serverFakeKindDrive{id: "pikpak-target", kind: "pikpak"})
|
||||
migrator := &serverFakeSpider91MigrationRunner{}
|
||||
app := &App{
|
||||
cat: cat,
|
||||
registry: registry,
|
||||
spider91Migrator: migrator,
|
||||
workers: map[string]*preview.Worker{},
|
||||
thumbWorkers: map[string]*preview.ThumbWorker{},
|
||||
fingerprintWorkers: map[string]*fingerprint.Worker{},
|
||||
}
|
||||
|
||||
accepted, message := app.scheduleManualCrawlerUploadMigration(ctx, "crawler-ready")
|
||||
if !accepted {
|
||||
t.Fatalf("accepted = false, message = %q", message)
|
||||
}
|
||||
deadline := time.After(time.Second)
|
||||
for migrator.called == 0 {
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatalf("migration calls = %d, want 1", migrator.called)
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleManualCrawlerUploadMigrationRejectsPendingFingerprint(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "crawler-pending",
|
||||
Kind: scriptcrawler.Kind,
|
||||
Name: "Pending Crawler",
|
||||
RootID: "/",
|
||||
TeaserEnabled: true,
|
||||
Credentials: map[string]string{
|
||||
"script_path": "/tmp/pending.py",
|
||||
"upload_drive_id": "pikpak-target",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed crawler: %v", err)
|
||||
}
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: scriptcrawler.BuildVideoID("crawler-pending", "source-1"),
|
||||
DriveID: "crawler-pending",
|
||||
FileID: "source-1.mp4",
|
||||
FileName: "source-1.mp4",
|
||||
Title: "Source 1",
|
||||
Size: 123,
|
||||
Ext: "mp4",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: time.Now(),
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
migrator := &serverFakeSpider91MigrationRunner{}
|
||||
app := &App{cat: cat, registry: proxy.NewRegistry(), spider91Migrator: migrator}
|
||||
|
||||
accepted, message := app.scheduleManualCrawlerUploadMigration(ctx, "crawler-pending")
|
||||
if accepted {
|
||||
t.Fatal("accepted = true, want false")
|
||||
}
|
||||
if !strings.Contains(message, "指纹") {
|
||||
t.Fatalf("message = %q, want fingerprint reason", message)
|
||||
}
|
||||
if migrator.called != 0 {
|
||||
t.Fatalf("migration calls = %d, want 0", migrator.called)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriveGenerationStatusUsesWorkerQueueNotPendingCatalogRows(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
@@ -53,6 +53,7 @@ type AdminServer struct {
|
||||
OnDriveDeleteCleanup func(ctx context.Context, driveID string) (int, error)
|
||||
OnDriveRemoved func(driveID string)
|
||||
OnScanRequested func(driveID string) bool
|
||||
OnCrawlerUploadRequested func(driveID string) (bool, string)
|
||||
OnStopDriveTasks func(driveID string) bool
|
||||
OnStopAllTasks func() int
|
||||
OnRegenPreview func(videoID string)
|
||||
@@ -194,6 +195,7 @@ func (a *AdminServer) Register(r chi.Router) {
|
||||
r.Post("/crawlers/test-script", a.handleTestCrawlerScript)
|
||||
r.Delete("/crawlers/{id}", a.handleDeleteCrawler)
|
||||
r.Post("/crawlers/{id}/run", a.handleRunCrawler)
|
||||
r.Post("/crawlers/{id}/upload", a.handleUploadCrawlerVideos)
|
||||
r.Post("/crawlers/{id}/tasks/stop", a.handleStopCrawlerTasks)
|
||||
|
||||
// 视频
|
||||
@@ -479,9 +481,10 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
SkipDirIDs []string `json:"skipDirIds"`
|
||||
// LastCrawlAt 是 spider91 上次成功爬取的 unix 秒(来自 credentials.last_crawl_at)。
|
||||
// 其它 kind 留 0;前端用它显示"上次抓取: N 小时前"。
|
||||
Spider91Proxy string `json:"spider91Proxy,omitempty"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
GoogleDriveUseOnlineAPI *bool `json:"googleDriveUseOnlineAPI,omitempty"`
|
||||
Spider91Proxy string `json:"spider91Proxy,omitempty"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
GoogleDriveUseOnlineAPI *bool `json:"googleDriveUseOnlineAPI,omitempty"`
|
||||
GoogleDriveOpenListAPIURL string `json:"googleDriveOpenListApiUrl,omitempty"`
|
||||
// STRMAllowOutsideRoot 是 localstorage 的 .strm 越root开关;其它 kind 省略。
|
||||
STRMAllowOutsideRoot *bool `json:"strmAllowOutsideRoot,omitempty"`
|
||||
ScanGenerationStatus GenerationStatus `json:"scanGenerationStatus"`
|
||||
@@ -560,6 +563,7 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
Spider91Proxy: spider91ProxyForDrive(d),
|
||||
LastCrawlAt: lastCrawlAt,
|
||||
GoogleDriveUseOnlineAPI: googleDriveUseOnlineAPIForDrive(d),
|
||||
GoogleDriveOpenListAPIURL: googleDriveOpenListAPIURLForDrive(d),
|
||||
STRMAllowOutsideRoot: strmAllowOutsideRootForDrive(d),
|
||||
ScanGenerationStatus: generation.Scan,
|
||||
ThumbnailGenerationStatus: generation.Thumbnail,
|
||||
@@ -628,7 +632,9 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
body.Credentials = credentials
|
||||
} else if body.Kind == "googledrive" || body.Kind == "localstorage" || body.Kind == "guangyapan" {
|
||||
} else if body.Kind == "googledrive" {
|
||||
body.Credentials = mergeGoogleDriveCredentials(existing, body.Credentials)
|
||||
} else if body.Kind == "localstorage" || body.Kind == "guangyapan" {
|
||||
// 按键合并、空值沿用旧值:这些网盘的编辑表单允许只改某几个字段,
|
||||
// 其它 token / 路径 / 开关字段应保留旧值。
|
||||
body.Credentials = mergeNonEmptyCredentials(existing, body.Credentials)
|
||||
@@ -956,7 +962,7 @@ func (a *AdminServer) validateCrawlerUploadDrive(ctx context.Context, driveID st
|
||||
|
||||
func isCrawlerUploadTargetKind(kind string) bool {
|
||||
switch strings.TrimSpace(kind) {
|
||||
case "p115", "pikpak", "p123", "googledrive", "onedrive", "wopan":
|
||||
case "p115", "pikpak", "p123", "googledrive", "onedrive", "wopan", "guangyapan":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
@@ -1285,6 +1291,104 @@ func (a *AdminServer) handleRunCrawler(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusAccepted, resp)
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleUploadCrawlerVideos(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
d, err := a.Catalog.GetDrive(r.Context(), id)
|
||||
if err != nil || d == nil || !isConfiguredCrawlerDrive(d) {
|
||||
http.Error(w, "crawler not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
status := a.nightlyJobStatus()
|
||||
if status.Running || status.Queued {
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"ok": true,
|
||||
"accepted": false,
|
||||
"message": fullScanBusyMessage,
|
||||
"status": status,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
assets, err := a.Catalog.CountCrawlerAssets(r.Context(), d.ID, crawlerVideoIDPrefixes(d))
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
generation := DriveGenerationStatuses{}
|
||||
if a.GetDriveGenerationStatuses != nil {
|
||||
generation = a.GetDriveGenerationStatuses()[d.ID]
|
||||
}
|
||||
if reason := crawlerUploadBlockedReason(d, assets, generation); reason != "" {
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"ok": true,
|
||||
"accepted": false,
|
||||
"message": reason,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
accepted := true
|
||||
message := ""
|
||||
if a.OnCrawlerUploadRequested != nil {
|
||||
accepted, message = a.OnCrawlerUploadRequested(id)
|
||||
}
|
||||
resp := map[string]any{"ok": true, "accepted": accepted}
|
||||
if !accepted {
|
||||
if strings.TrimSpace(message) == "" {
|
||||
message = driveTaskBusyMessage
|
||||
}
|
||||
resp["message"] = message
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, resp)
|
||||
}
|
||||
|
||||
func crawlerUploadBlockedReason(d *catalog.Drive, assets catalog.CrawlerAssetCounts, generation DriveGenerationStatuses) string {
|
||||
if d == nil || !isConfiguredCrawlerDrive(d) {
|
||||
return "爬虫不存在"
|
||||
}
|
||||
if strings.TrimSpace(d.Credentials["upload_drive_id"]) == "" {
|
||||
return "请先配置上传网盘"
|
||||
}
|
||||
if assets.Local <= 0 {
|
||||
return "没有待上传的本地视频"
|
||||
}
|
||||
if crawlerGenerationBusy(generation) {
|
||||
return "当前爬虫有正在进行的任务,请稍后重试"
|
||||
}
|
||||
if assets.Fingerprint.Pending > 0 {
|
||||
return "还有待生成的视频指纹"
|
||||
}
|
||||
if assets.Fingerprint.Failed > 0 {
|
||||
return "存在指纹生成失败的视频,请先重试或处理失败项"
|
||||
}
|
||||
if d.TeaserEnabled {
|
||||
if assets.Teaser.Pending > 0 {
|
||||
return "还有待生成的预览视频"
|
||||
}
|
||||
if assets.Teaser.Failed > 0 {
|
||||
return "存在预览视频生成失败的视频,请先重试或处理失败项"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func crawlerGenerationBusy(g DriveGenerationStatuses) bool {
|
||||
return generationBusy(g.Scan) ||
|
||||
generationBusy(g.Thumbnail) ||
|
||||
generationBusy(g.Preview) ||
|
||||
generationBusy(g.Fingerprint) ||
|
||||
generationBusy(g.Upload)
|
||||
}
|
||||
|
||||
func generationBusy(g GenerationStatus) bool {
|
||||
switch strings.TrimSpace(g.State) {
|
||||
case "", "idle":
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleStopCrawlerTasks(w http.ResponseWriter, r *http.Request) {
|
||||
a.handleStopDriveTasks(w, r)
|
||||
}
|
||||
@@ -1412,6 +1516,21 @@ func googleDriveUseOnlineAPIForDrive(d *catalog.Drive) *bool {
|
||||
return &result
|
||||
}
|
||||
|
||||
func googleDriveOpenListAPIURLForDrive(d *catalog.Drive) string {
|
||||
if d == nil || d.Kind != "googledrive" || d.Credentials == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(d.Credentials["api_url_address"])
|
||||
}
|
||||
|
||||
func mergeGoogleDriveCredentials(existing *catalog.Drive, incoming map[string]string) map[string]string {
|
||||
merged := mergeNonEmptyCredentials(existing, incoming)
|
||||
if _, ok := incoming["api_url_address"]; ok && strings.TrimSpace(incoming["api_url_address"]) == "" {
|
||||
delete(merged, "api_url_address")
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
// mergeNonEmptyCredentials 逐键合并凭证:incoming 里非空的键覆盖旧值,
|
||||
// 空值/缺失的键沿用旧值。googledrive、localstorage 和 guangyapan 的编辑表单都依赖
|
||||
// 这个语义(留空 = 不修改)。
|
||||
|
||||
@@ -732,6 +732,31 @@ func TestHandleUpsertGoogleDriveMergesOAuthCredentials(t *testing.T) {
|
||||
if got.Credentials["client_id"] != "google-client-id" || got.Credentials["client_secret"] != "google-client-secret" {
|
||||
t.Fatalf("oauth client credentials = %#v, want saved", got.Credentials)
|
||||
}
|
||||
if got.Credentials["api_url_address"] != "https://api.oplist.org/googleui/renewapi" {
|
||||
t.Fatalf("api_url_address = %q, want preserved", got.Credentials["api_url_address"])
|
||||
}
|
||||
|
||||
clearReq := httptest.NewRequest(http.MethodPost, "/admin/api/drives", bytes.NewBufferString(`{
|
||||
"id": "google-main",
|
||||
"kind": "googledrive",
|
||||
"name": "Google Drive",
|
||||
"rootId": "root",
|
||||
"credentials": {
|
||||
"api_url_address": ""
|
||||
}
|
||||
}`))
|
||||
clearRR := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(clearRR, clearReq)
|
||||
if clearRR.Code != http.StatusOK {
|
||||
t.Fatalf("clear status = %d, body = %s", clearRR.Code, clearRR.Body.String())
|
||||
}
|
||||
cleared, err := cat.GetDrive(ctx, "google-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get cleared drive: %v", err)
|
||||
}
|
||||
if _, ok := cleared.Credentials["api_url_address"]; ok {
|
||||
t.Fatalf("api_url_address was not cleared: %#v", cleared.Credentials)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertSpider91DriveIsRejected(t *testing.T) {
|
||||
@@ -754,7 +779,7 @@ func TestHandleUpsertSpider91DriveIsRejected(t *testing.T) {
|
||||
Credentials: map[string]string{
|
||||
"last_crawl_at": "1800000000",
|
||||
"proxy": "http://old-proxy.local:7890",
|
||||
"script_path": "/opt/video-site-91/91VideoSpider/spider_91porn.py",
|
||||
"script_path": "/opt/video-site-91/data/crawler-scripts/legacy-spider.py",
|
||||
},
|
||||
Status: "ok",
|
||||
}); err != nil {
|
||||
@@ -1271,6 +1296,7 @@ func TestHandleUpsertCrawlerPersistsAndValidatesUploadDrive(t *testing.T) {
|
||||
for _, d := range []*catalog.Drive{
|
||||
{ID: "p115-target", Kind: "p115", Name: "115", RootID: "0", Credentials: map[string]string{"cookie": "x"}},
|
||||
{ID: "wopan-target", Kind: "wopan", Name: "沃盘", RootID: "0", Credentials: map[string]string{"access_token": "a", "refresh_token": "r"}},
|
||||
{ID: "guangyapan-target", Kind: "guangyapan", Name: "光鸭", RootID: "", Credentials: map[string]string{"access_token": "a", "refresh_token": "r"}},
|
||||
{ID: "local-target", Kind: "localstorage", Name: "Local", RootID: "/", Credentials: map[string]string{"path": tmp}},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
@@ -1336,6 +1362,24 @@ func TestHandleUpsertCrawlerPersistsAndValidatesUploadDrive(t *testing.T) {
|
||||
t.Fatalf("teaser callback after preserved edit = %q, want none", teaserCallbackID)
|
||||
}
|
||||
|
||||
req = httptest.NewRequest(http.MethodPost, "/admin/api/crawlers", strings.NewReader(`{
|
||||
"id": "crawler-upload",
|
||||
"scriptPath": "`+scriptPath+`",
|
||||
"uploadDriveId": "guangyapan-target"
|
||||
}`))
|
||||
rr = httptest.NewRecorder()
|
||||
srv.handleUpsertCrawler(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("guangyapan target status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
got, err = cat.GetDrive(ctx, "crawler-upload")
|
||||
if err != nil {
|
||||
t.Fatalf("get crawler after guangyapan target: %v", err)
|
||||
}
|
||||
if got.Credentials["upload_drive_id"] != "guangyapan-target" {
|
||||
t.Fatalf("upload_drive_id = %q, want guangyapan-target", got.Credentials["upload_drive_id"])
|
||||
}
|
||||
|
||||
req = httptest.NewRequest(http.MethodPost, "/admin/api/crawlers", strings.NewReader(`{
|
||||
"id": "crawler-upload",
|
||||
"scriptPath": "`+scriptPath+`",
|
||||
@@ -1929,7 +1973,8 @@ func TestHandleListDrivesIncludesGoogleDriveOnlineAPIMode(t *testing.T) {
|
||||
Name: "Google Legacy",
|
||||
RootID: "root",
|
||||
Credentials: map[string]string{
|
||||
"refresh_token": "legacy-refresh",
|
||||
"refresh_token": "legacy-refresh",
|
||||
"api_url_address": "https://openlist-api.example/googleui/renewapi",
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
@@ -1960,15 +2005,18 @@ func TestHandleListDrivesIncludesGoogleDriveOnlineAPIMode(t *testing.T) {
|
||||
}
|
||||
|
||||
var got []struct {
|
||||
ID string `json:"id"`
|
||||
GoogleDriveUseOnlineAPI bool `json:"googleDriveUseOnlineAPI"`
|
||||
ID string `json:"id"`
|
||||
GoogleDriveUseOnlineAPI bool `json:"googleDriveUseOnlineAPI"`
|
||||
GoogleDriveOpenListAPIURL string `json:"googleDriveOpenListApiUrl"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
byID := map[string]bool{}
|
||||
byAPIURL := map[string]string{}
|
||||
for _, d := range got {
|
||||
byID[d.ID] = d.GoogleDriveUseOnlineAPI
|
||||
byAPIURL[d.ID] = d.GoogleDriveOpenListAPIURL
|
||||
}
|
||||
if !byID["google-legacy"] {
|
||||
t.Fatalf("legacy google drive use_online_api = false, want true")
|
||||
@@ -1976,6 +2024,9 @@ func TestHandleListDrivesIncludesGoogleDriveOnlineAPIMode(t *testing.T) {
|
||||
if byID["google-oauth"] {
|
||||
t.Fatalf("oauth google drive use_online_api = true, want false")
|
||||
}
|
||||
if byAPIURL["google-legacy"] != "https://openlist-api.example/googleui/renewapi" {
|
||||
t.Fatalf("legacy google drive openlist api url = %q, want custom URL", byAPIURL["google-legacy"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
|
||||
@@ -78,6 +78,7 @@ type Video struct {
|
||||
TranscodedFileID string `json:"transcodedFileId"`
|
||||
TranscodedSize int64 `json:"transcodedSize"`
|
||||
Views int `json:"views"`
|
||||
LastViewedAt time.Time `json:"lastViewedAt"`
|
||||
Favorites int `json:"favorites"`
|
||||
Comments int `json:"comments"`
|
||||
Likes int `json:"likes"`
|
||||
@@ -112,13 +113,13 @@ INSERT INTO videos (
|
||||
id, drive_id, file_id, file_name, content_hash, sampled_sha256, fingerprint_status, fingerprint_error, parent_id, title, author, tags,
|
||||
duration_seconds, size_bytes, ext, quality, thumbnail_url, thumbnail_status,
|
||||
preview_file_id, preview_local, preview_status,
|
||||
views, favorites, comments, likes, dislikes,
|
||||
views, last_viewed_at, favorites, comments, likes, dislikes,
|
||||
category, hidden, badges, description, published_at, created_at, updated_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, CASE WHEN COALESCE(?, '') != '' THEN 'ready' ELSE 'pending' END,
|
||||
?, ?, ?,
|
||||
?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?
|
||||
)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
@@ -169,7 +170,7 @@ ON CONFLICT(id) DO UPDATE SET
|
||||
v.ID, v.DriveID, v.FileID, v.FileName, v.ContentHash, v.SampledSHA256, fingerprintStatus, v.FingerprintError, v.ParentID, v.Title, v.Author, string(tagsJSON),
|
||||
v.DurationSeconds, v.Size, v.Ext, v.Quality, v.ThumbnailURL, v.ThumbnailURL,
|
||||
v.PreviewFileID, v.PreviewLocal, nullableStatus(v.PreviewStatus),
|
||||
v.Views, v.Favorites, v.Comments, v.Likes, v.Dislikes,
|
||||
v.Views, unixMilliOrZero(v.LastViewedAt), v.Favorites, v.Comments, v.Likes, v.Dislikes,
|
||||
v.Category, boolToInt(v.Hidden), string(badgesJSON), v.Description,
|
||||
v.PublishedAt.UnixMilli(), v.CreatedAt.UnixMilli(), v.UpdatedAt.UnixMilli(),
|
||||
)
|
||||
@@ -423,9 +424,10 @@ func (c *Catalog) IncrementView(ctx context.Context, id string) (int, error) {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
now := time.Now().UnixMilli()
|
||||
res, err := tx.ExecContext(ctx,
|
||||
`UPDATE videos SET views = views + 1, updated_at = ? WHERE id = ?`,
|
||||
time.Now().UnixMilli(), id)
|
||||
`UPDATE videos SET views = views + 1, last_viewed_at = ?, updated_at = ? WHERE id = ?`,
|
||||
now, now, id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -451,6 +453,10 @@ type VideoMetaPatch struct {
|
||||
Category string
|
||||
ContentHash string
|
||||
FileName string
|
||||
Title string
|
||||
TitleSet bool
|
||||
Author string
|
||||
AuthorSet bool
|
||||
Tags []string
|
||||
TagsSet bool
|
||||
}
|
||||
@@ -500,6 +506,14 @@ func (c *Catalog) UpdateVideoMeta(ctx context.Context, id string, p VideoMetaPat
|
||||
parts = append(parts, "file_name = ?")
|
||||
args = append(args, p.FileName)
|
||||
}
|
||||
if p.TitleSet {
|
||||
parts = append(parts, "title = ?")
|
||||
args = append(args, p.Title)
|
||||
}
|
||||
if p.AuthorSet {
|
||||
parts = append(parts, "author = ?")
|
||||
args = append(args, p.Author)
|
||||
}
|
||||
if p.TagsSet {
|
||||
tagsJSON, _ := json.Marshal(p.Tags)
|
||||
parts = append(parts, "tags = ?")
|
||||
@@ -1352,7 +1366,7 @@ type ListParams struct {
|
||||
DriveID string
|
||||
Tag string
|
||||
Category string
|
||||
Sort string // latest | hot | week | long
|
||||
Sort string // latest | hot | recent
|
||||
ThumbnailReadyOnly bool
|
||||
PreferReadyThumbnails bool
|
||||
SkipTotal bool
|
||||
@@ -1407,10 +1421,8 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
|
||||
case "hot":
|
||||
// 热度 = 点赞数,点赞相同按最新
|
||||
orderBy = " ORDER BY " + readyOrderPrefix + "likes DESC, published_at DESC"
|
||||
case "week":
|
||||
orderBy = " ORDER BY " + readyOrderPrefix + "likes DESC"
|
||||
case "long":
|
||||
orderBy = " ORDER BY " + readyOrderPrefix + "duration_seconds DESC"
|
||||
case "recent":
|
||||
orderBy = " ORDER BY " + readyOrderPrefix + "COALESCE(last_viewed_at, 0) DESC, published_at DESC"
|
||||
}
|
||||
|
||||
var total int
|
||||
@@ -2203,7 +2215,7 @@ COALESCE(parent_id, ''), title, COALESCE(author, ''), COALESCE(tags, '[]'),
|
||||
duration_seconds, size_bytes, COALESCE(ext, ''), COALESCE(quality, ''), COALESCE(thumbnail_url, ''),
|
||||
COALESCE(preview_file_id, ''), COALESCE(preview_local, ''), COALESCE(preview_status, 'pending'),
|
||||
COALESCE(transcode_status, ''), COALESCE(transcode_error, ''), COALESCE(transcoded_file_id, ''), COALESCE(transcoded_size, 0),
|
||||
views, favorites, comments, likes, dislikes,
|
||||
views, COALESCE(last_viewed_at, 0), favorites, comments, likes, dislikes,
|
||||
COALESCE(category, ''), COALESCE(hidden, 0), COALESCE(badges, '[]'), COALESCE(description, ''),
|
||||
published_at, created_at, updated_at
|
||||
`
|
||||
@@ -2266,7 +2278,7 @@ type rowScanner interface {
|
||||
func scanVideo(row rowScanner) (*Video, error) {
|
||||
v := &Video{}
|
||||
var tagsJSON, badgesJSON string
|
||||
var publishedAt, createdAt, updatedAt int64
|
||||
var publishedAt, createdAt, updatedAt, lastViewedAt int64
|
||||
var hidden int
|
||||
err := row.Scan(
|
||||
&v.ID, &v.DriveID, &v.FileID, &v.FileName, &v.ContentHash,
|
||||
@@ -2275,7 +2287,7 @@ func scanVideo(row rowScanner) (*Video, error) {
|
||||
&v.DurationSeconds, &v.Size, &v.Ext, &v.Quality, &v.ThumbnailURL,
|
||||
&v.PreviewFileID, &v.PreviewLocal, &v.PreviewStatus,
|
||||
&v.TranscodeStatus, &v.TranscodeError, &v.TranscodedFileID, &v.TranscodedSize,
|
||||
&v.Views, &v.Favorites, &v.Comments, &v.Likes, &v.Dislikes,
|
||||
&v.Views, &lastViewedAt, &v.Favorites, &v.Comments, &v.Likes, &v.Dislikes,
|
||||
&v.Category, &hidden, &badgesJSON, &v.Description,
|
||||
&publishedAt, &createdAt, &updatedAt,
|
||||
)
|
||||
@@ -2288,6 +2300,9 @@ func scanVideo(row rowScanner) (*Video, error) {
|
||||
v.PublishedAt = time.UnixMilli(publishedAt)
|
||||
v.CreatedAt = time.UnixMilli(createdAt)
|
||||
v.UpdatedAt = time.UnixMilli(updatedAt)
|
||||
if lastViewedAt > 0 {
|
||||
v.LastViewedAt = time.UnixMilli(lastViewedAt)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
@@ -2295,6 +2310,13 @@ func normalizeContentHash(hash string) string {
|
||||
return strings.ToLower(strings.TrimSpace(hash))
|
||||
}
|
||||
|
||||
func unixMilliOrZero(t time.Time) int64 {
|
||||
if t.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return t.UnixMilli()
|
||||
}
|
||||
|
||||
func boolToInt(v bool) int {
|
||||
if v {
|
||||
return 1
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIncrementViewStoresLastViewedAt(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: "video-1",
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
Title: "Video 1",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
if _, err := cat.IncrementView(ctx, "video-1"); err != nil {
|
||||
t.Fatalf("increment view: %v", err)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, "video-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.Views != 1 {
|
||||
t.Fatalf("views = %d, want 1", got.Views)
|
||||
}
|
||||
if got.LastViewedAt.IsZero() {
|
||||
t.Fatal("last viewed time was not stored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestListVideosRecentSortUsesLastViewedAt(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{ID: "old-view", DriveID: "drive", FileID: "old-view", Title: "Old View", PublishedAt: now.Add(3 * time.Hour), CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "recent-view", DriveID: "drive", FileID: "recent-view", Title: "Recent View", PublishedAt: now, CreatedAt: now, UpdatedAt: now},
|
||||
{ID: "unviewed", DriveID: "drive", FileID: "unviewed", Title: "Unviewed", PublishedAt: now.Add(4 * time.Hour), CreatedAt: now, UpdatedAt: now},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
if _, err := cat.db.ExecContext(ctx,
|
||||
`UPDATE videos SET last_viewed_at = CASE id
|
||||
WHEN 'old-view' THEN ?
|
||||
WHEN 'recent-view' THEN ?
|
||||
ELSE 0
|
||||
END`,
|
||||
now.Add(-time.Hour).UnixMilli(),
|
||||
now.Add(time.Hour).UnixMilli(),
|
||||
); err != nil {
|
||||
t.Fatalf("seed last_viewed_at: %v", err)
|
||||
}
|
||||
|
||||
items, _, err := cat.ListVideos(ctx, ListParams{Sort: "recent", Page: 1, PageSize: 3})
|
||||
if err != nil {
|
||||
t.Fatalf("list recent videos: %v", err)
|
||||
}
|
||||
if len(items) != 3 {
|
||||
t.Fatalf("items = %d, want 3", len(items))
|
||||
}
|
||||
got := []string{items[0].ID, items[1].ID, items[2].ID}
|
||||
want := []string{"recent-view", "old-view", "unviewed"}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("recent order = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,7 @@ CREATE TABLE IF NOT EXISTS videos (
|
||||
transcoded_file_id TEXT DEFAULT '', -- 转码产物在同一 drive 上的 fileID,播放源优先用它
|
||||
transcoded_size INTEGER DEFAULT 0,
|
||||
views INTEGER DEFAULT 0,
|
||||
last_viewed_at INTEGER DEFAULT 0,
|
||||
favorites INTEGER DEFAULT 0,
|
||||
comments INTEGER DEFAULT 0,
|
||||
likes INTEGER DEFAULT 0,
|
||||
|
||||
@@ -66,6 +66,9 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "thumbnail_failures", "INTEGER DEFAULT 0"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.addColumnIfMissing(ctx, "videos", "last_viewed_at", "INTEGER DEFAULT 0"); err != nil {
|
||||
return err
|
||||
}
|
||||
// videos.transcode_*:浏览器兼容性转码状态。
|
||||
// status:''=未检测 / pending=已入队 / ready=已转码 / skipped=检测后无需转码 / failed=失败。
|
||||
// transcoded_file_id 指向转码产物在同一 drive 上的 fileID,播放源优先使用它。
|
||||
@@ -145,6 +148,9 @@ CREATE TABLE IF NOT EXISTS deleted_videos (
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_visible_pub ON videos(COALESCE(hidden, 0), published_at DESC)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_last_viewed ON videos(last_viewed_at DESC)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_file_name_size ON videos(file_name, size_bytes)`); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -20,11 +20,12 @@ import (
|
||||
)
|
||||
|
||||
type Driver struct {
|
||||
id string
|
||||
cookie string
|
||||
rootID string
|
||||
client *sdk.Pan115Client
|
||||
ua string
|
||||
id string
|
||||
cookie string
|
||||
rootID string
|
||||
client *sdk.Pan115Client
|
||||
ua string
|
||||
uploadTempDir string
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
@@ -32,10 +33,11 @@ type Driver struct {
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ID string
|
||||
Cookie string // 形如 "UID=xxx; CID=xxx; SEID=xxx; KID=xxx"
|
||||
RootID string // 默认 "0"
|
||||
UA string // 默认 UA115Browser
|
||||
ID string
|
||||
Cookie string // 形如 "UID=xxx; CID=xxx; SEID=xxx; KID=xxx"
|
||||
RootID string // 默认 "0"
|
||||
UA string // 默认 UA115Browser
|
||||
UploadTempDir string
|
||||
}
|
||||
|
||||
func New(c Config) *Driver {
|
||||
@@ -48,11 +50,12 @@ func New(c Config) *Driver {
|
||||
ua = sdk.UA115Browser
|
||||
}
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
cookie: c.Cookie,
|
||||
rootID: rootID,
|
||||
ua: ua,
|
||||
listInterval: 2 * time.Second,
|
||||
id: c.ID,
|
||||
cookie: c.Cookie,
|
||||
rootID: rootID,
|
||||
ua: ua,
|
||||
uploadTempDir: strings.TrimSpace(c.UploadTempDir),
|
||||
listInterval: 2 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -347,7 +350,7 @@ func (d *Driver) UploadAndReportSha1(ctx context.Context, parentID, name string,
|
||||
parentID = d.rootID
|
||||
}
|
||||
|
||||
tmp, sha1Hex, written, err := bufferAndHashSha1(r, size)
|
||||
tmp, sha1Hex, written, err := bufferAndHashSha1(d.uploadTempDir, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
@@ -472,8 +475,14 @@ func (d *Driver) Remove(ctx context.Context, fileID string) error {
|
||||
// 返回临时文件(位置在末尾,需调用方 Seek 回 0)、SHA1 hex 大写、实际字节数。
|
||||
//
|
||||
// 调用方负责 Close + Remove 临时文件。
|
||||
func bufferAndHashSha1(r io.Reader, declaredSize int64) (*os.File, string, int64, error) {
|
||||
tmp, err := os.CreateTemp("", "p115-upload-*.bin")
|
||||
func bufferAndHashSha1(tempDir string, r io.Reader, declaredSize int64) (*os.File, string, int64, error) {
|
||||
tempDir = strings.TrimSpace(tempDir)
|
||||
if tempDir != "" {
|
||||
if err := os.MkdirAll(tempDir, 0o755); err != nil {
|
||||
return nil, "", 0, fmt.Errorf("p115 upload: create tmp dir: %w", err)
|
||||
}
|
||||
}
|
||||
tmp, err := os.CreateTemp(tempDir, "p115-upload-*.bin")
|
||||
if err != nil {
|
||||
return nil, "", 0, fmt.Errorf("p115 upload: create tmp: %w", err)
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -86,7 +87,7 @@ func TestBufferAndHashSha1(t *testing.T) {
|
||||
wantHex := strings.ToUpper(hex.EncodeToString(want[:]))
|
||||
|
||||
t.Run("declared size matches", func(t *testing.T) {
|
||||
tmp, gotHex, n, err := bufferAndHashSha1(bytes.NewReader(body), int64(len(body)))
|
||||
tmp, gotHex, n, err := bufferAndHashSha1("", bytes.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("bufferAndHashSha1 returned error: %v", err)
|
||||
}
|
||||
@@ -111,14 +112,14 @@ func TestBufferAndHashSha1(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("declared size mismatch returns error", func(t *testing.T) {
|
||||
_, _, _, err := bufferAndHashSha1(bytes.NewReader(body), int64(len(body))+1)
|
||||
_, _, _, err := bufferAndHashSha1("", bytes.NewReader(body), int64(len(body))+1)
|
||||
if err == nil {
|
||||
t.Fatal("expected size mismatch error, got nil")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("declared size zero is unchecked", func(t *testing.T) {
|
||||
tmp, gotHex, n, err := bufferAndHashSha1(bytes.NewReader(body), 0)
|
||||
tmp, gotHex, n, err := bufferAndHashSha1("", bytes.NewReader(body), 0)
|
||||
if err != nil {
|
||||
t.Fatalf("bufferAndHashSha1 returned error: %v", err)
|
||||
}
|
||||
@@ -130,6 +131,18 @@ func TestBufferAndHashSha1(t *testing.T) {
|
||||
t.Errorf("written = %d, want %d", n, len(body))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("uses configured temp dir", func(t *testing.T) {
|
||||
tempDir := filepath.Join(t.TempDir(), "upload-tmp")
|
||||
tmp, _, _, err := bufferAndHashSha1(tempDir, bytes.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("bufferAndHashSha1 returned error: %v", err)
|
||||
}
|
||||
defer cleanup(tmp)
|
||||
if gotDir := filepath.Dir(tmp.Name()); gotDir != tempDir {
|
||||
t.Fatalf("tmp dir = %q, want %q", gotDir, tempDir)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestUploadAndReportSha1RejectsInvalidArgs 检查空 reader / 空 name / 负 size 在
|
||||
|
||||
@@ -70,6 +70,7 @@ type Driver struct {
|
||||
httpClient *http.Client
|
||||
|
||||
onTokenUpdate func(access string)
|
||||
uploadTempDir string
|
||||
|
||||
tokenMu sync.RWMutex
|
||||
|
||||
@@ -90,6 +91,7 @@ type Config struct {
|
||||
|
||||
MainAPIBaseURL string
|
||||
LoginAPIBaseURL string
|
||||
UploadTempDir string
|
||||
|
||||
OnTokenUpdate func(access string)
|
||||
}
|
||||
@@ -123,6 +125,7 @@ func New(c Config) *Driver {
|
||||
referer: defaultReferer,
|
||||
userAgent: defaultUserAgent,
|
||||
onTokenUpdate: c.OnTokenUpdate,
|
||||
uploadTempDir: strings.TrimSpace(c.UploadTempDir),
|
||||
client: resty.New().
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
@@ -289,7 +292,7 @@ func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string,
|
||||
parentID = d.rootID
|
||||
}
|
||||
|
||||
tmp, md5Hex, actualSize, err := bufferAndHashMD5(r, size)
|
||||
tmp, md5Hex, actualSize, err := bufferAndHashMD5(d.uploadTempDir, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
@@ -1058,8 +1061,14 @@ func splitPath(p string) []string {
|
||||
return strings.Split(p, "/")
|
||||
}
|
||||
|
||||
func bufferAndHashMD5(r io.Reader, declaredSize int64) (*os.File, string, int64, error) {
|
||||
tmp, err := os.CreateTemp("", "p123-upload-*.bin")
|
||||
func bufferAndHashMD5(tempDir string, r io.Reader, declaredSize int64) (*os.File, string, int64, error) {
|
||||
tempDir = strings.TrimSpace(tempDir)
|
||||
if tempDir != "" {
|
||||
if err := os.MkdirAll(tempDir, 0o755); err != nil {
|
||||
return nil, "", 0, fmt.Errorf("123pan upload: create tmp dir: %w", err)
|
||||
}
|
||||
}
|
||||
tmp, err := os.CreateTemp(tempDir, "p123-upload-*.bin")
|
||||
if err != nil {
|
||||
return nil, "", 0, fmt.Errorf("123pan upload: create tmp: %w", err)
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -458,6 +460,29 @@ func TestUploadPresignedPUT429ReturnsRateLimitError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBufferAndHashMD5UsesConfiguredTempDir(t *testing.T) {
|
||||
body := []byte("hello-123-upload-test")
|
||||
tempDir := filepath.Join(t.TempDir(), "upload-tmp")
|
||||
tmp, gotHex, n, err := bufferAndHashMD5(tempDir, bytes.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("bufferAndHashMD5 returned error: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = tmp.Close()
|
||||
_ = os.Remove(tmp.Name())
|
||||
}()
|
||||
if gotDir := filepath.Dir(tmp.Name()); gotDir != tempDir {
|
||||
t.Fatalf("tmp dir = %q, want %q", gotDir, tempDir)
|
||||
}
|
||||
want := md5.Sum(body)
|
||||
if gotHex != fmt.Sprintf("%x", want) {
|
||||
t.Fatalf("md5 = %s, want %x", gotHex, want)
|
||||
}
|
||||
if n != int64(len(body)) {
|
||||
t.Fatalf("written = %d, want %d", n, len(body))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameSendsExpectedBody(t *testing.T) {
|
||||
var renameRequest map[string]any
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -47,6 +47,7 @@ type Driver struct {
|
||||
client *resty.Client
|
||||
onTokenUpdate func(access, refresh, captcha, deviceID string)
|
||||
uploadToOSSFunc func(context.Context, *s3Params, io.Reader) error
|
||||
uploadTempDir string
|
||||
|
||||
// captchaMu serializes captcha-token refreshes triggered by 4002 / 9
|
||||
// recovery in requestOnce. Without it, N concurrent callers all hitting
|
||||
@@ -77,6 +78,7 @@ type Config struct {
|
||||
DeviceID string
|
||||
RootID string
|
||||
DisableMediaLink bool
|
||||
UploadTempDir string
|
||||
OnTokenUpdate func(access, refresh, captcha, deviceID string)
|
||||
}
|
||||
|
||||
@@ -109,6 +111,7 @@ func New(c Config) *Driver {
|
||||
deviceID: deviceID,
|
||||
disableMediaLink: c.DisableMediaLink,
|
||||
onTokenUpdate: c.OnTokenUpdate,
|
||||
uploadTempDir: strings.TrimSpace(c.UploadTempDir),
|
||||
client: resty.New().
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
|
||||
@@ -79,6 +79,20 @@ type UploadResult struct {
|
||||
Size int64
|
||||
}
|
||||
|
||||
type preparedUploadBody struct {
|
||||
reader io.ReadSeeker
|
||||
start int64
|
||||
cleanup func()
|
||||
}
|
||||
|
||||
func (b preparedUploadBody) rewind() error {
|
||||
if b.reader == nil {
|
||||
return errors.New("pikpak upload: nil upload body")
|
||||
}
|
||||
_, err := b.reader.Seek(b.start, io.SeekStart)
|
||||
return err
|
||||
}
|
||||
|
||||
// Upload 实现 drives.Drive 接口;只返回 fileID。
|
||||
// 完整上传元数据见 UploadAndReportHash。
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
@@ -125,15 +139,15 @@ func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string,
|
||||
parentID = d.rootID
|
||||
}
|
||||
|
||||
// 1) 把 r 全量缓冲到临时文件,同时算 GCID。
|
||||
tmp, gcidHex, actualSize, err := bufferAndHashGCID(r, size)
|
||||
// 1) 算 GCID,并准备一个可重试读取的 body。爬虫迁移传入的是
|
||||
// *os.File,可直接复用原文件,避免再占用一份视频大小的临时空间。
|
||||
body, gcidHex, actualSize, err := d.prepareUploadBody(r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
defer func() {
|
||||
_ = tmp.Close()
|
||||
_ = os.Remove(tmp.Name())
|
||||
}()
|
||||
if body.cleanup != nil {
|
||||
defer body.cleanup()
|
||||
}
|
||||
|
||||
result := UploadResult{Hash: gcidHex, Size: actualSize}
|
||||
var lastErr error
|
||||
@@ -155,7 +169,7 @@ func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string,
|
||||
continue
|
||||
}
|
||||
|
||||
out, err := d.completeUploadAttempt(ctx, tmp, parentID, name, result, resp)
|
||||
out, err := d.completeUploadAttempt(ctx, body, parentID, name, result, resp)
|
||||
if err == nil {
|
||||
return out, nil
|
||||
}
|
||||
@@ -190,7 +204,7 @@ func (d *Driver) requestUploadSession(ctx context.Context, parentID, name string
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (d *Driver) completeUploadAttempt(ctx context.Context, tmp *os.File, parentID, name string, result UploadResult, resp uploadTaskData) (UploadResult, error) {
|
||||
func (d *Driver) completeUploadAttempt(ctx context.Context, body preparedUploadBody, parentID, name string, result UploadResult, resp uploadTaskData) (UploadResult, error) {
|
||||
// 命中秒传:服务端已经知道这个 hash,直接返回新文件 ID。
|
||||
if resp.Resumable == nil {
|
||||
if resp.File.ID != "" {
|
||||
@@ -207,10 +221,10 @@ func (d *Driver) completeUploadAttempt(ctx context.Context, tmp *os.File, parent
|
||||
}
|
||||
|
||||
// 未命中秒传:把字节传到 S3 兼容存储。
|
||||
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
|
||||
return UploadResult{}, fmt.Errorf("pikpak upload: seek tmp: %w", err)
|
||||
if err := body.rewind(); err != nil {
|
||||
return UploadResult{}, fmt.Errorf("pikpak upload: rewind body: %w", err)
|
||||
}
|
||||
if err := d.uploadToOSS(ctx, &resp.Resumable.Params, tmp); err != nil {
|
||||
if err := d.uploadToOSS(ctx, &resp.Resumable.Params, body.reader); err != nil {
|
||||
return UploadResult{}, fmt.Errorf("pikpak upload: oss put: %w", err)
|
||||
}
|
||||
|
||||
@@ -279,12 +293,62 @@ func isRetryablePikPakUploadError(err error) bool {
|
||||
strings.Contains(text, "service unavailable")
|
||||
}
|
||||
|
||||
func (d *Driver) prepareUploadBody(r io.Reader, size int64) (preparedUploadBody, string, int64, error) {
|
||||
if rs, ok := r.(io.ReadSeeker); ok {
|
||||
gcidHex, actualSize, start, err := hashGCIDFromReadSeeker(rs, size)
|
||||
if err != nil {
|
||||
return preparedUploadBody{}, "", 0, err
|
||||
}
|
||||
return preparedUploadBody{reader: rs, start: start, cleanup: func() {}}, gcidHex, actualSize, nil
|
||||
}
|
||||
|
||||
tmp, gcidHex, actualSize, err := bufferAndHashGCID(d.uploadTempDir, r, size)
|
||||
if err != nil {
|
||||
return preparedUploadBody{}, "", 0, err
|
||||
}
|
||||
return preparedUploadBody{
|
||||
reader: tmp,
|
||||
start: 0,
|
||||
cleanup: func() {
|
||||
_ = tmp.Close()
|
||||
_ = os.Remove(tmp.Name())
|
||||
},
|
||||
}, gcidHex, actualSize, nil
|
||||
}
|
||||
|
||||
func hashGCIDFromReadSeeker(r io.ReadSeeker, size int64) (string, int64, int64, error) {
|
||||
start, err := r.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return "", 0, 0, fmt.Errorf("pikpak upload: seek body: %w", err)
|
||||
}
|
||||
|
||||
h := NewGCID(size)
|
||||
written, copyErr := io.Copy(h, r)
|
||||
_, seekErr := r.Seek(start, io.SeekStart)
|
||||
if copyErr != nil {
|
||||
return "", 0, start, fmt.Errorf("pikpak upload: hash body: %w", copyErr)
|
||||
}
|
||||
if seekErr != nil {
|
||||
return "", 0, start, fmt.Errorf("pikpak upload: rewind body: %w", seekErr)
|
||||
}
|
||||
if size > 0 && written != size {
|
||||
return "", 0, start, fmt.Errorf("pikpak upload: size mismatch: declared %d, copied %d", size, written)
|
||||
}
|
||||
return strings.ToUpper(hex.EncodeToString(h.Sum(nil))), written, start, nil
|
||||
}
|
||||
|
||||
// bufferAndHashGCID 把 r 复制到一个临时文件,同时计算 GCID。
|
||||
// 返回临时文件(位置在末尾,需要调用方 Seek 回 0)、GCID hex 大写、实际写入字节数。
|
||||
// 返回临时文件(位置在末尾,需要调用方 Seek 回 start)、GCID hex 大写、实际写入字节数。
|
||||
//
|
||||
// 调用方负责 Close + Remove 临时文件。
|
||||
func bufferAndHashGCID(r io.Reader, size int64) (*os.File, string, int64, error) {
|
||||
tmp, err := os.CreateTemp("", "pikpak-upload-*.bin")
|
||||
func bufferAndHashGCID(tempDir string, r io.Reader, size int64) (*os.File, string, int64, error) {
|
||||
tempDir = strings.TrimSpace(tempDir)
|
||||
if tempDir != "" {
|
||||
if err := os.MkdirAll(tempDir, 0o755); err != nil {
|
||||
return nil, "", 0, fmt.Errorf("pikpak upload: create tmp dir: %w", err)
|
||||
}
|
||||
}
|
||||
tmp, err := os.CreateTemp(tempDir, "pikpak-upload-*.bin")
|
||||
if err != nil {
|
||||
return nil, "", 0, fmt.Errorf("pikpak upload: create tmp: %w", err)
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -142,6 +144,80 @@ func TestUploadInstantSuccessReturnsFileID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadUsesReadSeekerWithoutTempCopy(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"upload_type": "UPLOAD_TYPE_RESUMABLE",
|
||||
"resumable": null,
|
||||
"file": {"id": "instant-file-id", "name": "test.mp4", "kind": "drive#file"}
|
||||
}`))
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
uploadTempDir := filepath.Join(t.TempDir(), "upload-tmp")
|
||||
d.uploadTempDir = uploadTempDir
|
||||
|
||||
data := bytes.Repeat([]byte{0x31}, 1024)
|
||||
path := filepath.Join(t.TempDir(), "video.bin")
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatalf("write source: %v", err)
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open source: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
id, err := d.Upload(context.Background(), "parent-id", "test.mp4", f, int64(len(data)))
|
||||
if err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
if id != "instant-file-id" {
|
||||
t.Fatalf("file id = %q, want instant-file-id", id)
|
||||
}
|
||||
if _, err := os.Stat(uploadTempDir); !os.IsNotExist(err) {
|
||||
t.Fatalf("upload temp dir stat err = %v, want not created for read seeker input", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadBuffersNonSeekReaderInConfiguredTempDir(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"upload_type": "UPLOAD_TYPE_RESUMABLE",
|
||||
"resumable": null,
|
||||
"file": {"id": "instant-file-id", "name": "test.mp4", "kind": "drive#file"}
|
||||
}`))
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
d := newTestDriver(t, server)
|
||||
uploadTempDir := filepath.Join(t.TempDir(), "upload-tmp")
|
||||
d.uploadTempDir = uploadTempDir
|
||||
|
||||
data := bytes.Repeat([]byte{0x42}, 1024)
|
||||
id, err := d.Upload(context.Background(), "parent-id", "test.mp4", bytes.NewBuffer(data), int64(len(data)))
|
||||
if err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
if id != "instant-file-id" {
|
||||
t.Fatalf("file id = %q, want instant-file-id", id)
|
||||
}
|
||||
entries, err := os.ReadDir(uploadTempDir)
|
||||
if err != nil {
|
||||
t.Fatalf("read upload temp dir: %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("upload temp dir entries = %d, want cleaned", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadInstantSuccessFallsBackToListWhenFileIDMissing(t *testing.T) {
|
||||
listCalled := false
|
||||
mux := http.NewServeMux()
|
||||
@@ -304,7 +380,7 @@ func TestUploadRejectsInvalidArguments(t *testing.T) {
|
||||
func TestBufferAndHashGCIDDetectsSizeMismatch(t *testing.T) {
|
||||
src := bytes.NewReader([]byte("hello"))
|
||||
// 声明 size=10 但实际只有 5 字节
|
||||
_, _, _, err := bufferAndHashGCID(src, 10)
|
||||
_, _, _, err := bufferAndHashGCID("", src, 10)
|
||||
if err == nil {
|
||||
t.Fatal("expected size mismatch error")
|
||||
}
|
||||
@@ -315,7 +391,7 @@ func TestBufferAndHashGCIDDetectsSizeMismatch(t *testing.T) {
|
||||
|
||||
func TestBufferAndHashGCIDComputesCorrectHash(t *testing.T) {
|
||||
data := bytes.Repeat([]byte{0x55}, 1024)
|
||||
tmp, hex, written, err := bufferAndHashGCID(bytes.NewReader(data), int64(len(data)))
|
||||
tmp, hex, written, err := bufferAndHashGCID("", bytes.NewReader(data), int64(len(data)))
|
||||
if err != nil {
|
||||
t.Fatalf("buffer: %v", err)
|
||||
}
|
||||
|
||||
@@ -67,6 +67,55 @@ type DryRunResult struct {
|
||||
DurationMs int64 `json:"durationMs"`
|
||||
}
|
||||
|
||||
type dryRunLogTail struct {
|
||||
mu sync.Mutex
|
||||
lines []string
|
||||
partial string
|
||||
}
|
||||
|
||||
func newDryRunLogTail() *dryRunLogTail {
|
||||
return &dryRunLogTail{lines: make([]string, 0, dryRunLogTailLines)}
|
||||
}
|
||||
|
||||
func (t *dryRunLogTail) Write(p []byte) (int, error) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
chunk := strings.ReplaceAll(string(p), "\r\n", "\n")
|
||||
parts := strings.Split(t.partial+chunk, "\n")
|
||||
t.partial = parts[len(parts)-1]
|
||||
for _, line := range parts[:len(parts)-1] {
|
||||
t.appendLocked(line)
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (t *dryRunLogTail) snapshot() []string {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
lines := append([]string{}, t.lines...)
|
||||
if partial := strings.TrimSpace(t.partial); partial != "" {
|
||||
lines = appendDryRunLogLine(lines, partial)
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func (t *dryRunLogTail) appendLocked(line string) {
|
||||
t.lines = appendDryRunLogLine(t.lines, line)
|
||||
}
|
||||
|
||||
func appendDryRunLogLine(lines []string, line string) []string {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
return lines
|
||||
}
|
||||
if len(lines) >= dryRunLogTailLines {
|
||||
lines = lines[1:]
|
||||
}
|
||||
return append(lines, line)
|
||||
}
|
||||
|
||||
func DryRun(ctx context.Context, cfg DryRunConfig) *DryRunResult {
|
||||
started := time.Now()
|
||||
result := &DryRunResult{Items: []DryRunItem{}}
|
||||
@@ -169,41 +218,14 @@ func DryRun(ctx context.Context, cfg DryRunConfig) *DryRunResult {
|
||||
result.Error = fmt.Sprintf("启动脚本失败: %v", err)
|
||||
return result
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
_ = stdout.Close()
|
||||
result.Error = fmt.Sprintf("启动脚本失败: %v", err)
|
||||
return result
|
||||
}
|
||||
logTail := newDryRunLogTail()
|
||||
cmd.Stderr = logTail
|
||||
if err := cmd.Start(); err != nil {
|
||||
_ = stdout.Close()
|
||||
_ = stderr.Close()
|
||||
result.Error = fmt.Sprintf("启动脚本失败: %v", err)
|
||||
return result
|
||||
}
|
||||
|
||||
// stderr 是脚本日志,保留尾部若干行用于排错回显。
|
||||
var logMu sync.Mutex
|
||||
logTail := make([]string, 0, dryRunLogTailLines)
|
||||
stderrDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(stderrDone)
|
||||
scanner := bufio.NewScanner(stderr)
|
||||
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
logMu.Lock()
|
||||
if len(logTail) >= dryRunLogTailLines {
|
||||
logTail = logTail[1:]
|
||||
}
|
||||
logTail = append(logTail, line)
|
||||
logMu.Unlock()
|
||||
}
|
||||
}()
|
||||
|
||||
items := []DryRunItem{}
|
||||
var firstMediaHeaders map[string]string
|
||||
parseFailures := 0
|
||||
@@ -264,11 +286,8 @@ func DryRun(ctx context.Context, cfg DryRunConfig) *DryRunResult {
|
||||
_ = killDryRunProcess(cmd)
|
||||
<-waitDone
|
||||
}
|
||||
<-stderrDone
|
||||
|
||||
logMu.Lock()
|
||||
result.Log = append([]string{}, logTail...)
|
||||
logMu.Unlock()
|
||||
result.Log = logTail.snapshot()
|
||||
result.Items = items
|
||||
|
||||
if len(items) == 0 {
|
||||
|
||||
@@ -48,6 +48,29 @@ echo '{"type":"done","stats":{"emitted":1}}'
|
||||
}
|
||||
}
|
||||
|
||||
func TestDryRunCapturesStderrWhenStoppingAfterFirstItem(t *testing.T) {
|
||||
script := writeDryRunScript(t, `
|
||||
echo '[log] first item ready' >&2
|
||||
echo '{"type":"item","item":{"title":"Early Stop Video","media_url":"https://cdn.example.test/v.mp4","source_id":"early-stop"}}'
|
||||
sleep 30
|
||||
`)
|
||||
start := time.Now()
|
||||
result := DryRun(context.Background(), DryRunConfig{
|
||||
PythonPath: "/bin/sh",
|
||||
ScriptPath: script,
|
||||
SkipMediaProbe: true,
|
||||
})
|
||||
if !result.OK {
|
||||
t.Fatalf("ok = false, error = %q, log = %v", result.Error, result.Log)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed > 5*time.Second {
|
||||
t.Fatalf("dry run took %s, script was not stopped after first item", elapsed)
|
||||
}
|
||||
if len(result.Log) == 0 || !strings.Contains(result.Log[0], "first item ready") {
|
||||
t.Fatalf("log tail = %v, want stderr captured before early stop", result.Log)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDryRunProbesMediaURL(t *testing.T) {
|
||||
var gotRange, gotReferer string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -27,6 +27,7 @@ type Driver struct {
|
||||
refreshToken string
|
||||
client *sdk.WoClient
|
||||
onTokenUpdate func(access, refresh string)
|
||||
uploadTempDir string
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
@@ -38,11 +39,12 @@ type Driver struct {
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ID string
|
||||
AccessToken string
|
||||
RefreshToken string
|
||||
FamilyID string // 空则走个人空间,有值则走家庭空间
|
||||
RootID string // 根目录 ID,默认 "0"
|
||||
ID string
|
||||
AccessToken string
|
||||
RefreshToken string
|
||||
FamilyID string // 空则走个人空间,有值则走家庭空间
|
||||
RootID string // 根目录 ID,默认 "0"
|
||||
UploadTempDir string
|
||||
// 当 SDK 刷新 token 时回调,便于持久化
|
||||
OnTokenUpdate func(access, refresh string)
|
||||
}
|
||||
@@ -59,6 +61,7 @@ func New(c Config) *Driver {
|
||||
accessToken: c.AccessToken,
|
||||
refreshToken: c.RefreshToken,
|
||||
onTokenUpdate: c.OnTokenUpdate,
|
||||
uploadTempDir: strings.TrimSpace(c.UploadTempDir),
|
||||
listInterval: 800 * time.Millisecond,
|
||||
listCooldown: 5 * time.Minute,
|
||||
fidToID: make(map[string]string),
|
||||
@@ -162,7 +165,12 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
// wopan SDK 要求 *os.File,先把流落到临时文件再上传
|
||||
tmp, err := os.CreateTemp("", "wopan-upload-*.tmp")
|
||||
if d.uploadTempDir != "" {
|
||||
if err := os.MkdirAll(d.uploadTempDir, 0o755); err != nil {
|
||||
return "", fmt.Errorf("wopan upload: create tmp dir: %w", err)
|
||||
}
|
||||
}
|
||||
tmp, err := os.CreateTemp(d.uploadTempDir, "wopan-upload-*.tmp")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -206,15 +206,19 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
patch.ContentHash = e.Hash
|
||||
existing.ContentHash = e.Hash
|
||||
}
|
||||
if e.Name != "" && existing.FileName == "" {
|
||||
if e.Name != "" && existing.FileName != e.Name {
|
||||
patch.FileName = e.Name
|
||||
existing.FileName = e.Name
|
||||
patch.Title = parsed.Title
|
||||
patch.TitleSet = true
|
||||
patch.Author = parsed.Author
|
||||
patch.AuthorSet = true
|
||||
}
|
||||
// 已存在但轻量元数据空缺时,顺便补齐。
|
||||
if existing.Category == "" && dirName != "" {
|
||||
patch.Category = dirName
|
||||
}
|
||||
if patch.Category != "" || patch.ContentHash != "" || patch.FileName != "" {
|
||||
if patch.Category != "" || patch.ContentHash != "" || patch.FileName != "" || patch.TitleSet || patch.AuthorSet {
|
||||
_ = s.Catalog.UpdateVideoMeta(ctx, id, patch)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
|
||||
@@ -323,6 +323,67 @@ func TestRunDoesNotBackfillRemoteThumbnailForExistingVideo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSyncsRenamedExistingVideoMetadata(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "fake-drive-file-1",
|
||||
DriveID: "drive",
|
||||
FileID: "file-1",
|
||||
FileName: "old-name - Old Author.mp4",
|
||||
Title: "old-name",
|
||||
Author: "Old Author",
|
||||
PreviewStatus: "pending",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed video: %v", err)
|
||||
}
|
||||
|
||||
drv := &scannerFakeDrive{
|
||||
entries: []drives.Entry{{
|
||||
ID: "file-1",
|
||||
Name: "[4K] renamed clip.mp4",
|
||||
Size: 123,
|
||||
ModTime: now,
|
||||
}},
|
||||
}
|
||||
sc := New(cat, drv, []string{".mp4"}, nil, nil)
|
||||
|
||||
stats, err := sc.Run(ctx, "")
|
||||
if err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if stats.Added != 0 {
|
||||
t.Fatalf("added = %d, want existing video to be updated in place", stats.Added)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(ctx, "fake-drive-file-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.FileName != "[4K] renamed clip.mp4" {
|
||||
t.Fatalf("file_name = %q, want remote name", got.FileName)
|
||||
}
|
||||
if got.Title != "renamed clip" {
|
||||
t.Fatalf("title = %q, want parsed title from remote name", got.Title)
|
||||
}
|
||||
if got.Author != "" {
|
||||
t.Fatalf("author = %q, want cleared author from remote name without author suffix", got.Author)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReplacesExistingVideoTagsWithFixedFilenameTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="referrer" content="no-referrer" />
|
||||
<link rel="icon" type="image/png" href="/icon.png" />
|
||||
<link rel="apple-touch-icon" href="/icon.png" />
|
||||
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png" />
|
||||
<link rel="manifest" href="/manifest.webmanifest" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
|
||||
<meta name="mobile-web-app-capable" content="yes" />
|
||||
|
||||
+2
-7
@@ -194,7 +194,7 @@ backup_install_files() {
|
||||
local backup="$1"
|
||||
mkdir -p "$backup"
|
||||
cp -a "$INSTALL_PATH/server" "$backup/server"
|
||||
for item in dist config.example.yaml 91VideoSpider config.yaml .version; do
|
||||
for item in dist config.example.yaml config.yaml .version; do
|
||||
if [[ -e "$INSTALL_PATH/$item" ]]; then
|
||||
cp -a "$INSTALL_PATH/$item" "$backup/$item"
|
||||
fi
|
||||
@@ -205,7 +205,7 @@ restore_install_files() {
|
||||
local backup="$1"
|
||||
mkdir -p "$INSTALL_PATH"
|
||||
cp -a "$backup/server" "$INSTALL_PATH/server"
|
||||
for item in dist config.example.yaml 91VideoSpider config.yaml .version; do
|
||||
for item in dist config.example.yaml config.yaml .version; do
|
||||
rm -rf "${INSTALL_PATH:?}/$item"
|
||||
if [[ -e "$backup/$item" ]]; then
|
||||
cp -a "$backup/$item" "$INSTALL_PATH/$item"
|
||||
@@ -441,7 +441,6 @@ process_looks_like_app() {
|
||||
[[ "$cmd" == *"VIDEO_FRONTEND_DIR=$INSTALL_PATH/dist"* ]] && return 0
|
||||
[[ "$cmd" == *"VIDEO_CONFIG=$INSTALL_PATH/config.yaml"* ]] && return 0
|
||||
[[ "$cmd" == *"video-site-91"* ]] && return 0
|
||||
[[ "$cmd" == *"91VideoSpider"* ]] && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -595,10 +594,6 @@ fetch_and_unpack() {
|
||||
rm -rf "$INSTALL_PATH/dist"
|
||||
cp -R "$root/dist" "$INSTALL_PATH/dist"
|
||||
cp "$root/config.example.yaml" "$INSTALL_PATH/config.example.yaml"
|
||||
if [[ -d "$root/91VideoSpider" ]]; then
|
||||
rm -rf "$INSTALL_PATH/91VideoSpider"
|
||||
cp -R "$root/91VideoSpider" "$INSTALL_PATH/91VideoSpider"
|
||||
fi
|
||||
chmod +x "$INSTALL_PATH/server"
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "video-site",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "video-site",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.1",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"artplayer": "^5.4.0",
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
"name": "video-site",
|
||||
"private": true,
|
||||
"license": "MIT",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.1",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 21 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 136 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 114 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 19 KiB |
@@ -1,28 +0,0 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32">
|
||||
<defs>
|
||||
<!-- Background Gradient: Warm Orange to Sakura Pink, representing both themes -->
|
||||
<linearGradient id="bg-grad" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||
<stop offset="0%" stop-color="#FF7E40" />
|
||||
<stop offset="100%" stop-color="#FF4B91" />
|
||||
</linearGradient>
|
||||
|
||||
<!-- Subtle drop shadow for the play button to give it depth -->
|
||||
<filter id="shadow" x="-20%" y="-20%" width="140%" height="140%">
|
||||
<feDropShadow dx="0" dy="1.5" stdDeviation="1" flood-opacity="0.25" />
|
||||
</filter>
|
||||
</defs>
|
||||
|
||||
<!-- Main Squircle Background -->
|
||||
<rect x="2" y="2" width="28" height="28" rx="8" fill="url(#bg-grad)" />
|
||||
|
||||
<!-- Inner border for a premium, glassmorphic feel -->
|
||||
<rect x="3" y="3" width="26" height="26" rx="7" fill="none" stroke="#ffffff" stroke-width="1" opacity="0.2" />
|
||||
|
||||
<!-- Stylized Play Button Icon, perfectly centered with rounded corners and drop shadow -->
|
||||
<path d="M13 10.5 L21.5 16 L13 21.5 Z"
|
||||
fill="#ffffff"
|
||||
stroke="#ffffff"
|
||||
stroke-width="2.5"
|
||||
stroke-linejoin="round"
|
||||
filter="url(#shadow)" />
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 1.1 KiB |
@@ -9,10 +9,22 @@
|
||||
"theme_color": "#000000",
|
||||
"icons": [
|
||||
{
|
||||
"src": "/icon.png",
|
||||
"src": "/app-icon-192.png",
|
||||
"sizes": "192x192",
|
||||
"type": "image/png",
|
||||
"purpose": "any"
|
||||
},
|
||||
{
|
||||
"src": "/app-icon-512.png",
|
||||
"sizes": "512x512",
|
||||
"type": "image/png",
|
||||
"purpose": "any maskable"
|
||||
"purpose": "any"
|
||||
},
|
||||
{
|
||||
"src": "/app-icon-maskable-512.png",
|
||||
"sizes": "512x512",
|
||||
"type": "image/png",
|
||||
"purpose": "maskable"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
+96
-73
@@ -1,88 +1,111 @@
|
||||
import { Suspense, lazy } from "react";
|
||||
import { Navigate, Route, Routes } from "react-router-dom";
|
||||
import { SkyStarfield } from "@/components/SkyStarfield";
|
||||
import HomePage from "@/pages/HomePage";
|
||||
import ListingPage from "@/pages/ListingPage";
|
||||
import ShortsPage from "@/pages/ShortsPage";
|
||||
import UploadPage from "@/pages/UploadPage";
|
||||
import VideoDetailPage from "@/pages/VideoDetailPage";
|
||||
import { AdminLayout } from "@/admin/AdminLayout";
|
||||
import { LoginPage } from "@/admin/LoginPage";
|
||||
import { RequireAuth } from "@/admin/RequireAuth";
|
||||
import { DrivesPage } from "@/admin/DrivesPage";
|
||||
import { CrawlersPage } from "@/admin/CrawlersPage";
|
||||
import { VideosPage } from "@/admin/VideosPage";
|
||||
import { TagsPage } from "@/admin/TagsPage";
|
||||
import { ThemePage } from "@/admin/ThemePage";
|
||||
|
||||
const HomePage = lazy(() => import("@/pages/HomePage"));
|
||||
const ListingPage = lazy(() => import("@/pages/ListingPage"));
|
||||
const ShortsPage = lazy(() => import("@/pages/ShortsPage"));
|
||||
const UploadPage = lazy(() => import("@/pages/UploadPage"));
|
||||
const VideoDetailPage = lazy(() => import("@/pages/VideoDetailPage"));
|
||||
|
||||
const LoginPage = lazy(() =>
|
||||
import("@/admin/LoginPage").then((module) => ({ default: module.LoginPage }))
|
||||
);
|
||||
const AdminLayout = lazy(() =>
|
||||
import("@/admin/AdminLayout").then((module) => ({
|
||||
default: module.AdminLayout,
|
||||
}))
|
||||
);
|
||||
const DrivesPage = lazy(() =>
|
||||
import("@/admin/DrivesPage").then((module) => ({ default: module.DrivesPage }))
|
||||
);
|
||||
const CrawlersPage = lazy(() =>
|
||||
import("@/admin/CrawlersPage").then((module) => ({
|
||||
default: module.CrawlersPage,
|
||||
}))
|
||||
);
|
||||
const VideosPage = lazy(() =>
|
||||
import("@/admin/VideosPage").then((module) => ({ default: module.VideosPage }))
|
||||
);
|
||||
const TagsPage = lazy(() =>
|
||||
import("@/admin/TagsPage").then((module) => ({ default: module.TagsPage }))
|
||||
);
|
||||
const ThemePage = lazy(() =>
|
||||
import("@/admin/ThemePage").then((module) => ({ default: module.ThemePage }))
|
||||
);
|
||||
|
||||
export default function App() {
|
||||
return (
|
||||
<>
|
||||
{/* 星空蓝主题的固定位置星星层,仅在 data-theme="sky" 下可见 */}
|
||||
<SkyStarfield />
|
||||
<Routes>
|
||||
<Route path="/login" element={<LoginPage />} />
|
||||
<Suspense fallback={null}>
|
||||
<Routes>
|
||||
<Route path="/login" element={<LoginPage />} />
|
||||
|
||||
{/* 主站需要登录 */}
|
||||
<Route
|
||||
path="/"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<HomePage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/list"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<ListingPage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/shorts"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<ShortsPage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/upload"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<UploadPage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/video/:id"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<VideoDetailPage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
{/* 主站需要登录 */}
|
||||
<Route
|
||||
path="/"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<HomePage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/list"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<ListingPage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/shorts"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<ShortsPage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/upload"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<UploadPage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/video/:id"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<VideoDetailPage />
|
||||
</RequireAuth>
|
||||
}
|
||||
/>
|
||||
|
||||
{/* 管理后台也需要登录 */}
|
||||
<Route
|
||||
path="/admin"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<AdminLayout />
|
||||
</RequireAuth>
|
||||
}
|
||||
>
|
||||
<Route index element={<Navigate to="/admin/drives" replace />} />
|
||||
<Route path="drives" element={<DrivesPage />} />
|
||||
<Route path="crawlers" element={<CrawlersPage />} />
|
||||
<Route path="videos" element={<VideosPage />} />
|
||||
<Route path="tags" element={<TagsPage />} />
|
||||
<Route path="theme" element={<ThemePage />} />
|
||||
</Route>
|
||||
{/* 管理后台也需要登录 */}
|
||||
<Route
|
||||
path="/admin"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<AdminLayout />
|
||||
</RequireAuth>
|
||||
}
|
||||
>
|
||||
<Route index element={<Navigate to="/admin/drives" replace />} />
|
||||
<Route path="drives" element={<DrivesPage />} />
|
||||
<Route path="crawlers" element={<CrawlersPage />} />
|
||||
<Route path="videos" element={<VideosPage />} />
|
||||
<Route path="tags" element={<TagsPage />} />
|
||||
<Route path="theme" element={<ThemePage />} />
|
||||
</Route>
|
||||
|
||||
<Route path="*" element={<Navigate to="/" replace />} />
|
||||
</Routes>
|
||||
<Route path="*" element={<Navigate to="/" replace />} />
|
||||
</Routes>
|
||||
</Suspense>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
+34
-50
@@ -57,6 +57,7 @@ export function CrawlersPage() {
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [expandedId, setExpandedId] = useState("");
|
||||
const [runningId, setRunningId] = useState("");
|
||||
const [uploadingId, setUploadingId] = useState("");
|
||||
const [stoppingId, setStoppingId] = useState("");
|
||||
const [togglingTeaserId, setTogglingTeaserId] = useState("");
|
||||
// undefined = 编辑器关闭;null = 新建;其余 = 编辑已有爬虫
|
||||
@@ -126,6 +127,23 @@ export function CrawlersPage() {
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadVideos(crawler: api.AdminCrawler) {
|
||||
setUploadingId(crawler.id);
|
||||
try {
|
||||
const resp = await api.uploadCrawlerVideos(crawler.id);
|
||||
if (!resp.accepted) {
|
||||
show(resp.message || "当前爬虫暂不满足上传条件", "info");
|
||||
return;
|
||||
}
|
||||
show("已触发上传任务", "success");
|
||||
await refresh(true);
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "触发上传失败", "error");
|
||||
} finally {
|
||||
setUploadingId("");
|
||||
}
|
||||
}
|
||||
|
||||
async function stop(crawler: api.AdminCrawler) {
|
||||
setStoppingId(crawler.id);
|
||||
try {
|
||||
@@ -233,10 +251,12 @@ export function CrawlersPage() {
|
||||
crawler={crawler}
|
||||
expanded={expandedId === crawler.id}
|
||||
running={runningId === crawler.id}
|
||||
uploading={uploadingId === crawler.id}
|
||||
stopping={stoppingId === crawler.id}
|
||||
togglingTeaser={togglingTeaserId === crawler.id}
|
||||
onToggle={() => setExpandedId(expandedId === crawler.id ? "" : crawler.id)}
|
||||
onRun={() => run(crawler)}
|
||||
onUpload={() => uploadVideos(crawler)}
|
||||
onStop={() => stop(crawler)}
|
||||
onToggleTeaser={() => toggleTeaser(crawler)}
|
||||
onEdit={() => setEditorTarget(crawler)}
|
||||
@@ -284,37 +304,16 @@ function CrawlerMetric({ label, value, icon, tone }: { label: string; value: num
|
||||
);
|
||||
}
|
||||
|
||||
type StageInfo = {
|
||||
key: string;
|
||||
label: string;
|
||||
status?: api.DriveGenerationStatus;
|
||||
};
|
||||
|
||||
function crawlerStages(crawler: api.AdminCrawler): StageInfo[] {
|
||||
return [
|
||||
{ key: "scan", label: "抓取", status: crawler.scanGenerationStatus },
|
||||
{ key: "thumbnail", label: "封面", status: crawler.thumbnailGenerationStatus },
|
||||
{ key: "preview", label: "预览", status: crawler.previewGenerationStatus },
|
||||
{ key: "fingerprint", label: "指纹", status: crawler.fingerprintGenerationStatus },
|
||||
{ key: "upload", label: "上传", status: crawler.uploadGenerationStatus },
|
||||
];
|
||||
}
|
||||
|
||||
function stageStateLabel(stage: StageInfo): string {
|
||||
const state = stage.status?.state || "idle";
|
||||
if (stage.key === "scan" && state === "scanning") return "抓取中";
|
||||
if (stage.key === "upload" && state === "uploading") return "上传中";
|
||||
return generationStateLabel(state);
|
||||
}
|
||||
|
||||
function CrawlerRow({
|
||||
crawler,
|
||||
expanded,
|
||||
running,
|
||||
uploading,
|
||||
stopping,
|
||||
togglingTeaser,
|
||||
onToggle,
|
||||
onRun,
|
||||
onUpload,
|
||||
onStop,
|
||||
onToggleTeaser,
|
||||
onEdit,
|
||||
@@ -323,16 +322,19 @@ function CrawlerRow({
|
||||
crawler: api.AdminCrawler;
|
||||
expanded: boolean;
|
||||
running: boolean;
|
||||
uploading: boolean;
|
||||
stopping: boolean;
|
||||
togglingTeaser: boolean;
|
||||
onToggle: () => void;
|
||||
onRun: () => void;
|
||||
onUpload: () => void;
|
||||
onStop: () => void;
|
||||
onToggleTeaser: () => void;
|
||||
onEdit: () => void;
|
||||
onDelete: () => void;
|
||||
}) {
|
||||
const busy = crawlerBusy(crawler);
|
||||
const uploadButtonTitle = uploading ? "上传请求处理中" : "上传本地爬虫视频到已配置的上传网盘";
|
||||
return (
|
||||
<div className={`admin-crawler-row ${expanded ? "is-expanded" : ""}`}>
|
||||
<div className="admin-crawler-row__line">
|
||||
@@ -346,31 +348,11 @@ function CrawlerRow({
|
||||
上次抓取 {formatLastCrawl(crawler.lastCrawlAt)} · 每次新增 {crawler.targetNew || "10"} 条 · 累计爬取 {crawler.totalCrawledCount ?? 0} 条
|
||||
</span>
|
||||
</span>
|
||||
<span className="admin-crawler-pipeline">
|
||||
{crawlerStages(crawler).map((stage) => {
|
||||
const state = stage.status?.state || "idle";
|
||||
const active = BUSY_STATES.has(state) || state === "cooling";
|
||||
return (
|
||||
<span
|
||||
key={stage.key}
|
||||
className={`admin-crawler-stage is-${generationStateClass(state)}`}
|
||||
title={`${stage.label}:${stageStateLabel(stage)}`}
|
||||
>
|
||||
<span className="admin-crawler-stage__dot" />
|
||||
{stage.label}
|
||||
{active && <em>{stageStateLabel(stage)}</em>}
|
||||
</span>
|
||||
);
|
||||
})}
|
||||
</span>
|
||||
<span className={`admin-status is-${crawler.status === "ok" ? "ok" : crawler.status === "error" ? "error" : "pending"}`}>
|
||||
{crawlerStatusLabel(crawler)}
|
||||
</span>
|
||||
<ChevronDown size={16} className="admin-crawler-row__chevron" />
|
||||
</button>
|
||||
<div className="admin-crawler-row__actions">
|
||||
<button
|
||||
className={`admin-btn admin-crawler-preview-card-toggle ${crawler.teaserEnabled ? "is-on" : ""}`}
|
||||
className="admin-btn admin-crawler-preview-card-toggle"
|
||||
type="button"
|
||||
onClick={onToggleTeaser}
|
||||
disabled={togglingTeaser}
|
||||
@@ -389,6 +371,14 @@ function CrawlerRow({
|
||||
<Download size={13} /> {running ? "触发中..." : "立即抓取"}
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
className="admin-btn"
|
||||
type="button"
|
||||
onClick={onUpload}
|
||||
title={uploadButtonTitle}
|
||||
>
|
||||
<Upload size={13} /> {uploading ? "上传中..." : "上传视频"}
|
||||
</button>
|
||||
<button className="admin-btn" type="button" onClick={onEdit}>
|
||||
<Pencil size={13} /> 编辑
|
||||
</button>
|
||||
@@ -1075,12 +1065,6 @@ function crawlerTestFailure(result: api.CrawlerDryRunResult) {
|
||||
return result.error || result.mediaCheck?.error || "";
|
||||
}
|
||||
|
||||
function crawlerStatusLabel(crawler: api.AdminCrawler) {
|
||||
if (crawler.status === "ok") return "已就绪";
|
||||
if (crawler.status === "error") return "错误";
|
||||
return "未连接";
|
||||
}
|
||||
|
||||
function formatLastCrawl(ts?: number) {
|
||||
if (!ts) return "从未";
|
||||
return new Date(ts * 1000).toLocaleString("zh-CN", {
|
||||
|
||||
@@ -217,7 +217,10 @@ export function DrivesPage() {
|
||||
d.kind === "spider91"
|
||||
? { proxy: d.spider91Proxy ?? "" }
|
||||
: d.kind === "googledrive"
|
||||
? { use_online_api: (d.googleDriveUseOnlineAPI ?? true) ? "true" : "false" }
|
||||
? {
|
||||
use_online_api: (d.googleDriveUseOnlineAPI ?? true) ? "true" : "false",
|
||||
api_url_address: d.googleDriveOpenListApiUrl ?? "",
|
||||
}
|
||||
: d.kind === "localstorage"
|
||||
? { strm_allow_outside_root: (d.strmAllowOutsideRoot ?? false) ? "true" : "false" }
|
||||
: {},
|
||||
|
||||
+128
-20
@@ -9,35 +9,125 @@ import {
|
||||
} from "react";
|
||||
|
||||
type ToastKind = "info" | "success" | "error";
|
||||
type Toast = { id: number; kind: ToastKind; text: string };
|
||||
type Toast = { id: number; kind: ToastKind; text: string; copyable: boolean };
|
||||
|
||||
type Ctx = {
|
||||
show: (text: string, kind?: ToastKind) => void;
|
||||
};
|
||||
|
||||
const ToastCtx = createContext<Ctx | null>(null);
|
||||
const TOAST_DISMISS_MS = 2500;
|
||||
const TOAST_COPY_SUCCESS_TEXT = "已复制到剪贴板";
|
||||
const TOAST_COPY_ERROR_TEXT = "复制失败,请手动复制";
|
||||
|
||||
async function copyTextToClipboard(text: string) {
|
||||
try {
|
||||
if (navigator.clipboard?.writeText) {
|
||||
await navigator.clipboard.writeText(text);
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// Fall back to the legacy copy command below.
|
||||
}
|
||||
if (!fallbackCopyText(text)) {
|
||||
throw new Error("copy failed");
|
||||
}
|
||||
}
|
||||
|
||||
function fallbackCopyText(text: string) {
|
||||
if (!document.body) return false;
|
||||
const textarea = document.createElement("textarea");
|
||||
textarea.value = text;
|
||||
textarea.setAttribute("readonly", "");
|
||||
textarea.style.position = "fixed";
|
||||
textarea.style.left = "-9999px";
|
||||
textarea.style.top = "0";
|
||||
document.body.appendChild(textarea);
|
||||
textarea.select();
|
||||
try {
|
||||
return document.execCommand("copy");
|
||||
} finally {
|
||||
document.body.removeChild(textarea);
|
||||
}
|
||||
}
|
||||
|
||||
export function ToastProvider({ children }: { children: ReactNode }) {
|
||||
const [items, setItems] = useState<Toast[]>([]);
|
||||
const timers = useRef<Record<string, ReturnType<typeof window.setTimeout>>>({});
|
||||
const timers = useRef(new Map<number, ReturnType<typeof window.setTimeout>>());
|
||||
const idsByText = useRef(new Map<string, number>());
|
||||
|
||||
const clearDismissTimer = useCallback((id: number) => {
|
||||
const timer = timers.current.get(id);
|
||||
if (!timer) return;
|
||||
window.clearTimeout(timer);
|
||||
timers.current.delete(id);
|
||||
}, []);
|
||||
|
||||
const removeToast = useCallback(
|
||||
(id: number, text: string) => {
|
||||
clearDismissTimer(id);
|
||||
if (idsByText.current.get(text) === id) {
|
||||
idsByText.current.delete(text);
|
||||
}
|
||||
setItems((list) => list.filter((t) => t.id !== id));
|
||||
},
|
||||
[clearDismissTimer]
|
||||
);
|
||||
|
||||
const scheduleDismiss = useCallback(
|
||||
(id: number, text: string) => {
|
||||
clearDismissTimer(id);
|
||||
timers.current.set(
|
||||
id,
|
||||
window.setTimeout(() => removeToast(id, text), TOAST_DISMISS_MS)
|
||||
);
|
||||
},
|
||||
[clearDismissTimer, removeToast]
|
||||
);
|
||||
|
||||
const addToast = useCallback(
|
||||
(text: string, kind: ToastKind = "info", copyable = true) => {
|
||||
const existingID = idsByText.current.get(text);
|
||||
if (existingID !== undefined) {
|
||||
setItems((list) =>
|
||||
list.map((t) => (t.id === existingID ? { ...t, kind, copyable } : t))
|
||||
);
|
||||
scheduleDismiss(existingID, text);
|
||||
return;
|
||||
}
|
||||
const id = Date.now() + Math.random();
|
||||
idsByText.current.set(text, id);
|
||||
setItems((list) => [...list, { id, kind, text, copyable }]);
|
||||
scheduleDismiss(id, text);
|
||||
},
|
||||
[scheduleDismiss]
|
||||
);
|
||||
|
||||
// Deduplicate: same text won't stack, just resets the dismiss timer
|
||||
const show = useCallback((text: string, kind: ToastKind = "info") => {
|
||||
// Reset timer if duplicate
|
||||
if (timers.current[text]) {
|
||||
window.clearTimeout(timers.current[text]);
|
||||
timers.current[text] = window.setTimeout(() => {
|
||||
setItems((list) => list.filter((t) => t.text !== text));
|
||||
delete timers.current[text];
|
||||
}, 2600);
|
||||
return;
|
||||
}
|
||||
const id = Date.now() + Math.random();
|
||||
timers.current[text] = window.setTimeout(() => {
|
||||
setItems((list) => list.filter((t) => t.id !== id));
|
||||
delete timers.current[text];
|
||||
}, 2600);
|
||||
setItems((list) => [...list, { id, kind, text }]);
|
||||
const show = useCallback(
|
||||
(text: string, kind: ToastKind = "info") => {
|
||||
addToast(text, kind, true);
|
||||
},
|
||||
[addToast]
|
||||
);
|
||||
|
||||
const copyToastText = useCallback(
|
||||
(text: string) => {
|
||||
void copyTextToClipboard(text)
|
||||
.then(() => addToast(TOAST_COPY_SUCCESS_TEXT, "success", false))
|
||||
.catch(() => addToast(TOAST_COPY_ERROR_TEXT, "error", false));
|
||||
},
|
||||
[addToast]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
for (const timer of timers.current.values()) {
|
||||
window.clearTimeout(timer);
|
||||
}
|
||||
timers.current.clear();
|
||||
idsByText.current.clear();
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
@@ -45,8 +135,26 @@ export function ToastProvider({ children }: { children: ReactNode }) {
|
||||
{children}
|
||||
<div className="admin-toast-stack" role="status" aria-live="polite">
|
||||
{items.map((t) => (
|
||||
<div key={t.id} className={`admin-toast is-${t.kind}`}>
|
||||
{t.text}
|
||||
<div
|
||||
key={t.id}
|
||||
className={`admin-toast is-${t.kind}${
|
||||
t.copyable ? " is-copyable" : ""
|
||||
}`}
|
||||
role={t.copyable ? "button" : undefined}
|
||||
tabIndex={t.copyable ? 0 : undefined}
|
||||
aria-label={t.copyable ? `复制提示:${t.text}` : undefined}
|
||||
onClick={t.copyable ? () => copyToastText(t.text) : undefined}
|
||||
onKeyDown={
|
||||
t.copyable
|
||||
? (event) => {
|
||||
if (event.key !== "Enter" && event.key !== " ") return;
|
||||
event.preventDefault();
|
||||
copyToastText(t.text);
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
<span className="admin-toast__text">{t.text}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
@@ -98,6 +98,8 @@ export type AdminDrive = {
|
||||
spider91Proxy?: string;
|
||||
// Google Drive 是否使用 OpenList 在线续期 API;未配置时后端按 true 返回。
|
||||
googleDriveUseOnlineAPI?: boolean;
|
||||
// Google Drive OpenList 在线续期 API 地址;为空时后端使用驱动默认值。
|
||||
googleDriveOpenListApiUrl?: string;
|
||||
// localstorage 的 .strm 是否允许指向存储根目录之外;未配置时后端按 false 返回。
|
||||
strmAllowOutsideRoot?: boolean;
|
||||
scanGenerationStatus?: DriveGenerationStatus;
|
||||
@@ -315,6 +317,13 @@ export function runCrawler(id: string) {
|
||||
);
|
||||
}
|
||||
|
||||
export function uploadCrawlerVideos(id: string) {
|
||||
return request<{ ok: boolean; accepted: boolean; message?: string; status?: NightlyJobStatus }>(
|
||||
`/crawlers/${encodeURIComponent(id)}/upload`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
export function stopCrawlerTasks(id: string) {
|
||||
return request<{ ok: boolean; stopped: boolean }>(
|
||||
`/crawlers/${encodeURIComponent(id)}/tasks/stop`,
|
||||
|
||||
@@ -323,15 +323,15 @@ export function credentialFields(kind: Kind, creds: Record<string, string> = {})
|
||||
{ value: "false", label: "自建 Google OAuth 客户端" },
|
||||
],
|
||||
},
|
||||
{
|
||||
key: "refresh_token",
|
||||
label: "refresh_token",
|
||||
placeholder: "OpenList Google Drive refresh_token",
|
||||
multiline: true,
|
||||
required: true,
|
||||
},
|
||||
...(googleDriveUsesOnlineAPI(creds)
|
||||
? []
|
||||
? [
|
||||
{
|
||||
key: "api_url_address",
|
||||
label: "OpenList 在线 API URL",
|
||||
placeholder: "默认:https://api.oplist.org/googleui/renewapi",
|
||||
help: "留空时使用 OpenList 官方在线 API,填写后会使用自定义续期 API。",
|
||||
},
|
||||
]
|
||||
: [
|
||||
{
|
||||
key: "client_id",
|
||||
@@ -348,6 +348,13 @@ export function credentialFields(kind: Kind, creds: Record<string, string> = {})
|
||||
help: "Google Cloud Console 中同一个 OAuth 客户端的 Client Secret",
|
||||
},
|
||||
]),
|
||||
{
|
||||
key: "refresh_token",
|
||||
label: "refresh_token",
|
||||
placeholder: "OpenList Google Drive refresh_token",
|
||||
multiline: true,
|
||||
required: true,
|
||||
},
|
||||
];
|
||||
case "localstorage":
|
||||
return [
|
||||
|
||||
@@ -13,10 +13,7 @@ type Props = {
|
||||
const sortOptions: { key: SortKey; label: string }[] = [
|
||||
{ key: "latest", label: "最新" },
|
||||
{ key: "hot", label: "最热" },
|
||||
{ key: "week", label: "本周" },
|
||||
{ key: "long", label: "最长" },
|
||||
{ key: "hd", label: "高清" },
|
||||
{ key: "featured", label: "精选" },
|
||||
{ key: "recent", label: "最近观看" },
|
||||
];
|
||||
|
||||
export function SortToolbar({ sort, view, onSortChange, onViewChange }: Props) {
|
||||
|
||||
@@ -986,7 +986,7 @@ function createHlsSourceLoader(
|
||||
destroyHls(target);
|
||||
onError(null);
|
||||
|
||||
void import("hls.js")
|
||||
void import("hls.js/light")
|
||||
.then((hlsModule) => {
|
||||
if (art.isDestroy || !video.isConnected) return;
|
||||
loadHlsSourceWith(video, url, art, hlsModule.default, onError);
|
||||
|
||||
Vendored
+4
@@ -0,0 +1,4 @@
|
||||
declare module "hls.js/light" {
|
||||
export { default } from "hls.js";
|
||||
export * from "hls.js";
|
||||
}
|
||||
@@ -226,9 +226,6 @@ function isSortKey(value: unknown): value is SortKey {
|
||||
return (
|
||||
value === "latest" ||
|
||||
value === "hot" ||
|
||||
value === "week" ||
|
||||
value === "long" ||
|
||||
value === "hd" ||
|
||||
value === "featured"
|
||||
value === "recent"
|
||||
);
|
||||
}
|
||||
|
||||
+19
-95
@@ -626,7 +626,7 @@
|
||||
width: 100%;
|
||||
min-width: 0;
|
||||
display: grid;
|
||||
grid-template-columns: 38px minmax(160px, 1.1fr) minmax(0, 1.4fr) auto 18px;
|
||||
grid-template-columns: 38px minmax(160px, 1fr) 18px;
|
||||
align-items: center;
|
||||
gap: var(--space-3);
|
||||
padding: 0;
|
||||
@@ -676,77 +676,6 @@
|
||||
font-size: var(--font-xs);
|
||||
}
|
||||
|
||||
.admin-crawler-pipeline {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.admin-crawler-stage {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
min-height: 24px;
|
||||
padding: 3px 9px;
|
||||
border-radius: var(--radius-pill);
|
||||
border: 1px solid var(--border-subtle);
|
||||
background: var(--bg-sunken);
|
||||
color: var(--text-muted);
|
||||
font-size: var(--font-xs);
|
||||
font-weight: var(--weight-medium);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-crawler-stage em {
|
||||
font-style: normal;
|
||||
color: inherit;
|
||||
}
|
||||
|
||||
.admin-crawler-stage__dot {
|
||||
width: 7px;
|
||||
height: 7px;
|
||||
border-radius: 50%;
|
||||
background: var(--text-faint);
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
.admin-crawler-stage.is-generating {
|
||||
color: var(--info);
|
||||
border-color: transparent;
|
||||
background: var(--info-soft);
|
||||
}
|
||||
|
||||
.admin-crawler-stage.is-generating .admin-crawler-stage__dot {
|
||||
background: var(--info);
|
||||
animation: admin-crawler-stage-pulse 1.2s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.admin-crawler-stage.is-cooling {
|
||||
color: var(--warning);
|
||||
border-color: transparent;
|
||||
background: var(--warning-soft);
|
||||
}
|
||||
|
||||
.admin-crawler-stage.is-cooling .admin-crawler-stage__dot {
|
||||
background: var(--warning);
|
||||
}
|
||||
|
||||
.admin-crawler-stage.is-queued {
|
||||
color: var(--text-muted);
|
||||
background: rgba(255, 255, 255, 0.06);
|
||||
}
|
||||
|
||||
.admin-crawler-stage.is-queued .admin-crawler-stage__dot {
|
||||
background: var(--text-muted);
|
||||
}
|
||||
|
||||
@keyframes admin-crawler-stage-pulse {
|
||||
0%, 100% { opacity: 1; transform: scale(1); }
|
||||
50% { opacity: 0.4; transform: scale(0.75); }
|
||||
}
|
||||
|
||||
.admin-crawler-row__chevron {
|
||||
color: var(--text-faint);
|
||||
transition: transform var(--transition-fast), color var(--transition-fast);
|
||||
@@ -773,20 +702,6 @@
|
||||
padding-inline: 10px;
|
||||
}
|
||||
|
||||
.admin-crawler-preview-card-toggle.is-on {
|
||||
border-color: var(--accent);
|
||||
background: var(--accent);
|
||||
color: var(--text-on-accent);
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
.admin-crawler-preview-card-toggle.is-on:hover:not(:disabled) {
|
||||
border-color: var(--accent-hover);
|
||||
background: var(--accent-hover);
|
||||
color: var(--text-on-accent);
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
.admin-crawler-row__delete {
|
||||
padding-inline: 10px;
|
||||
}
|
||||
@@ -1214,13 +1129,7 @@
|
||||
|
||||
@media (max-width: 1180px) {
|
||||
.admin-crawler-row__main {
|
||||
grid-template-columns: 38px minmax(0, 1fr) auto 18px;
|
||||
row-gap: var(--space-2);
|
||||
}
|
||||
|
||||
.admin-crawler-pipeline {
|
||||
grid-column: 2 / 4;
|
||||
grid-row: 2;
|
||||
grid-template-columns: 38px minmax(0, 1fr) 18px;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2194,18 +2103,32 @@
|
||||
}
|
||||
|
||||
.admin-toast {
|
||||
padding: 12px 18px;
|
||||
max-width: min(520px, calc(100vw - 48px));
|
||||
padding: 14px 18px;
|
||||
background: var(--bg-elevated);
|
||||
color: var(--text-strong);
|
||||
border: 1px solid var(--border-default);
|
||||
border-radius: var(--radius-sm);
|
||||
position: relative;
|
||||
font-size: var(--font-sm);
|
||||
font-weight: var(--weight-medium);
|
||||
line-height: 1.5;
|
||||
overflow-wrap: anywhere;
|
||||
touch-action: manipulation;
|
||||
box-shadow: var(--shadow-lg);
|
||||
animation: toast-in var(--duration-normal) var(--ease-out);
|
||||
pointer-events: auto;
|
||||
}
|
||||
|
||||
.admin-toast.is-copyable {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.admin-toast__text {
|
||||
display: block;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
@keyframes toast-in {
|
||||
from { opacity: 0; transform: translateY(10px); }
|
||||
to { opacity: 1; transform: none; }
|
||||
@@ -2883,7 +2806,8 @@
|
||||
}
|
||||
|
||||
.admin-toast {
|
||||
text-align: center;
|
||||
max-width: 100%;
|
||||
text-align: left;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -57,7 +57,7 @@ export type VideoDetail = VideoItem & {
|
||||
|
||||
export type PreviewState = "idle" | "intent" | "loading" | "playing" | "error";
|
||||
|
||||
export type SortKey = "latest" | "hot" | "week" | "long" | "hd" | "featured";
|
||||
export type SortKey = "latest" | "hot" | "recent";
|
||||
|
||||
export type TagItem = {
|
||||
id: string;
|
||||
|
||||
@@ -145,7 +145,8 @@ test("googledrive drive form supports online API and custom OAuth client modes",
|
||||
assert.match(fields, /key: "client_id"/);
|
||||
assert.match(fields, /key: "client_secret"/);
|
||||
assert.match(fields, /googleDriveUsesOnlineAPI\(creds\)/);
|
||||
assert.doesNotMatch(fields, /key: "api_url_address"/);
|
||||
assert.match(fields, /key: "api_url_address"/);
|
||||
assert.match(fields, /OpenList 在线 API URL/);
|
||||
assert.doesNotMatch(fields, /在线 API 模式填写 OpenList 获取的 refresh_token/);
|
||||
assert.doesNotMatch(constantsSource, /请参考OpenList文档中关于谷歌云盘的配置方法。/);
|
||||
assert.doesNotMatch(constantsSource, /选择自建 Google OAuth 客户端后,服务端会直接请求 Google OAuth token 接口续期。/);
|
||||
@@ -154,7 +155,9 @@ test("googledrive drive form supports online API and custom OAuth client modes",
|
||||
assert.match(driveFormSource, /className="admin-form-select"/);
|
||||
assert.match(driveFormSource, /ChevronDown/);
|
||||
assert.match(drivesPageSource, /googleDriveUseOnlineAPI/);
|
||||
assert.match(drivesPageSource, /googleDriveOpenListApiUrl/);
|
||||
assert.match(apiSource, /googleDriveUseOnlineAPI\?: boolean/);
|
||||
assert.match(apiSource, /googleDriveOpenListApiUrl\?: string/);
|
||||
assert.doesNotMatch(fields, /key: "access_token"/);
|
||||
});
|
||||
|
||||
@@ -248,6 +251,7 @@ test("crawler management is a separate admin section", () => {
|
||||
assert.match(crawlerPageSource, /api\.listDrives/);
|
||||
assert.match(crawlerPageSource, /api\.upsertCrawler/);
|
||||
assert.match(crawlerPageSource, /api\.runCrawler/);
|
||||
assert.match(crawlerPageSource, /api\.uploadCrawlerVideos/);
|
||||
assert.match(crawlerPageSource, /api\.stopCrawlerTasks/);
|
||||
assert.match(crawlerPageSource, /api\.deleteCrawler/);
|
||||
assert.match(crawlerPageSource, /api\.importCrawlerScriptFile/);
|
||||
@@ -263,7 +267,15 @@ test("crawler management is a separate admin section", () => {
|
||||
assert.match(crawlerPageSource, /admin-crawler-preview-card-toggle/);
|
||||
assert.match(crawlerPageSource, /预览:开/);
|
||||
assert.match(crawlerPageSource, /预览:关/);
|
||||
assert.match(crawlerPageSource, /上传视频/);
|
||||
assert.match(crawlerPageSource, /aria-pressed=\{crawler\.teaserEnabled\}/);
|
||||
assert.doesNotMatch(crawlerPageSource, /crawlerUploadBlockedReason/);
|
||||
assert.doesNotMatch(crawlerPageSource, /disabled=\{uploading/);
|
||||
assert.doesNotMatch(crawlerPageSource, /crawlerStatusLabel/);
|
||||
assert.doesNotMatch(crawlerPageSource, /admin-crawler-preview-card-toggle \$\{crawler\.teaserEnabled/);
|
||||
assert.doesNotMatch(adminCss, /admin-crawler-preview-card-toggle\.is-on/);
|
||||
assert.doesNotMatch(crawlerPageSource, /admin-crawler-pipeline/);
|
||||
assert.doesNotMatch(adminCss, /admin-crawler-(pipeline|stage)/);
|
||||
assert.doesNotMatch(crawlerPageSource, /teaserEnabled: form\.teaserEnabled/);
|
||||
assert.doesNotMatch(crawlerPageSource, /aria-pressed=\{form\.teaserEnabled\}/);
|
||||
assert.match(crawlerPageSource, /UPLOAD_TARGET_KINDS/);
|
||||
@@ -284,6 +296,7 @@ test("crawler management is a separate admin section", () => {
|
||||
assert.match(apiSource, /teaserEnabled: boolean/);
|
||||
assert.doesNotMatch(apiSource, /teaserEnabled\?: boolean/);
|
||||
assert.match(apiSource, /"\/crawlers"/);
|
||||
assert.match(apiSource, /\/crawlers\/\$\{encodeURIComponent\(id\)\}\/upload/);
|
||||
assert.match(apiSource, /"\/crawlers\/import-file"/);
|
||||
assert.match(apiSource, /"\/crawlers\/import-url"/);
|
||||
assert.match(apiSource, /"\/crawlers\/test-script"/);
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import test from "node:test";
|
||||
|
||||
const toastSource = readFileSync(
|
||||
new URL("../src/admin/ToastContext.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
const adminCss = readFileSync(
|
||||
new URL("../src/styles/admin.css", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
|
||||
function ruleBody(css: string, selector: string): string {
|
||||
const escapedSelector = selector.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const match = css.match(new RegExp(`${escapedSelector}\\s*\\{([^}]*)\\}`));
|
||||
assert.ok(match, `Expected CSS rule for ${selector}`);
|
||||
return match[1];
|
||||
}
|
||||
|
||||
function mobileCss(): string {
|
||||
const marker = "@media (max-width: 768px)";
|
||||
const start = adminCss.indexOf(marker);
|
||||
assert.notEqual(start, -1, "Expected mobile admin media query");
|
||||
return adminCss.slice(start);
|
||||
}
|
||||
|
||||
test("admin toasts auto-dismiss and copy their text when clicked", () => {
|
||||
assert.match(toastSource, /const TOAST_DISMISS_MS = 2500/);
|
||||
assert.match(toastSource, /const TOAST_COPY_SUCCESS_TEXT = "已复制到剪贴板"/);
|
||||
assert.match(toastSource, /const TOAST_COPY_ERROR_TEXT = "复制失败,请手动复制"/);
|
||||
assert.match(toastSource, /navigator\.clipboard\?\.writeText/);
|
||||
assert.match(toastSource, /fallbackCopyText\(text\)/);
|
||||
assert.match(toastSource, /document\.execCommand\("copy"\)/);
|
||||
assert.match(toastSource, /addToast\(TOAST_COPY_SUCCESS_TEXT,\s*"success",\s*false\)/);
|
||||
assert.match(toastSource, /addToast\(TOAST_COPY_ERROR_TEXT,\s*"error",\s*false\)/);
|
||||
assert.match(toastSource, /t\.copyable\s*\?\s*" is-copyable"\s*:\s*""/);
|
||||
assert.match(toastSource, /onClick=\{t\.copyable \? \(\) => copyToastText\(t\.text\) : undefined\}/);
|
||||
assert.match(toastSource, /aria-label=\{t\.copyable \? `复制提示:\$\{t\.text\}` : undefined\}/);
|
||||
assert.match(toastSource, /event\.key !== "Enter" && event\.key !== " "/);
|
||||
assert.doesNotMatch(toastSource, /onClick=\{\(\) => scheduleDismiss/);
|
||||
assert.doesNotMatch(toastSource, /pinnedToastIDs/);
|
||||
assert.doesNotMatch(toastSource, /isDismissPaused/);
|
||||
assert.doesNotMatch(toastSource, /pinDismiss/);
|
||||
assert.doesNotMatch(toastSource, /className="admin-toast__close"/);
|
||||
assert.doesNotMatch(toastSource, /aria-label="关闭提示"/);
|
||||
assert.doesNotMatch(toastSource, /<X size=/);
|
||||
assert.doesNotMatch(toastSource, /event\.stopPropagation\(\)/);
|
||||
assert.doesNotMatch(toastSource, /onPointerEnter/);
|
||||
assert.doesNotMatch(toastSource, /onPointerLeave/);
|
||||
});
|
||||
|
||||
test("admin toasts show long messages without internal scrolling", () => {
|
||||
const baseToast = ruleBody(adminCss, ".admin-toast");
|
||||
const baseText = ruleBody(adminCss, ".admin-toast__text");
|
||||
const mobileToast = ruleBody(mobileCss(), ".admin-toast");
|
||||
|
||||
assert.match(baseToast, /max-width\s*:\s*min\(520px,\s*calc\(100vw - 48px\)\)/);
|
||||
assert.match(baseToast, /padding\s*:\s*14px\s+18px/);
|
||||
assert.match(baseToast, /position\s*:\s*relative/);
|
||||
assert.match(baseToast, /overflow-wrap\s*:\s*anywhere/);
|
||||
assert.match(baseToast, /touch-action\s*:\s*manipulation/);
|
||||
assert.doesNotMatch(baseToast, /cursor\s*:\s*pointer/);
|
||||
assert.match(ruleBody(adminCss, ".admin-toast.is-copyable"), /cursor\s*:\s*pointer/);
|
||||
assert.match(baseText, /display\s*:\s*block/);
|
||||
assert.doesNotMatch(adminCss, /\.admin-toast__close/);
|
||||
assert.match(mobileToast, /max-width\s*:\s*100%/);
|
||||
assert.match(mobileToast, /text-align\s*:\s*left/);
|
||||
assert.doesNotMatch(baseToast, /max-height/);
|
||||
assert.doesNotMatch(baseText, /max-height/);
|
||||
assert.doesNotMatch(baseText, /overflow-y\s*:\s*auto/);
|
||||
assert.doesNotMatch(mobileToast, /max-height/);
|
||||
assert.doesNotMatch(mobileCss(), /\.admin-toast__text\s*\{/);
|
||||
});
|
||||
@@ -0,0 +1,20 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import test from "node:test";
|
||||
|
||||
const sortToolbarSource = readFileSync(
|
||||
new URL("../src/components/SortToolbar.tsx", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
const typesSource = readFileSync(new URL("../src/types.ts", import.meta.url), "utf8");
|
||||
|
||||
test("list page sort toolbar only exposes active sort options", () => {
|
||||
assert.match(sortToolbarSource, /\{ key: "latest", label: "最新" \}/);
|
||||
assert.match(sortToolbarSource, /\{ key: "hot", label: "最热" \}/);
|
||||
assert.match(sortToolbarSource, /\{ key: "recent", label: "最近观看" \}/);
|
||||
|
||||
for (const removed of ["本周", "最长", "高清", "精选"]) {
|
||||
assert.doesNotMatch(sortToolbarSource, new RegExp(removed));
|
||||
}
|
||||
assert.match(typesSource, /export type SortKey = "latest" \| "hot" \| "recent";/);
|
||||
});
|
||||
@@ -14,6 +14,9 @@ const indexHtml = readFileSync(
|
||||
new URL("../index.html", import.meta.url),
|
||||
"utf8"
|
||||
);
|
||||
const manifest = JSON.parse(
|
||||
readFileSync(new URL("../public/manifest.webmanifest", import.meta.url), "utf8")
|
||||
) as { icons: Array<{ src: string; sizes: string; purpose: string }> };
|
||||
|
||||
// iOS Safari/WebKit does not composite an inline <video> nested inside a
|
||||
// `position: fixed` ancestor — the video decodes and plays but never paints
|
||||
@@ -46,6 +49,33 @@ test("iPhone browser uses document scrolling and only explicit fullscreen", () =
|
||||
|
||||
test("app has standalone display metadata for iPhone home-screen launch", () => {
|
||||
assert.match(indexHtml, /<link rel="manifest" href="\/manifest\.webmanifest" \/>/);
|
||||
assert.match(
|
||||
indexHtml,
|
||||
/<link rel="apple-touch-icon" sizes="180x180" href="\/apple-touch-icon\.png" \/>/
|
||||
);
|
||||
assert.match(indexHtml, /<meta name="apple-mobile-web-app-capable" content="yes" \/>/);
|
||||
assert.match(indexHtml, /<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" \/>/);
|
||||
});
|
||||
|
||||
test("home-screen icons use safe-area assets instead of the in-app logo", () => {
|
||||
assert.ok(
|
||||
manifest.icons.some(
|
||||
(icon) =>
|
||||
icon.src === "/app-icon-512.png" &&
|
||||
icon.sizes === "512x512" &&
|
||||
icon.purpose === "any"
|
||||
)
|
||||
);
|
||||
assert.ok(
|
||||
manifest.icons.some(
|
||||
(icon) =>
|
||||
icon.src === "/app-icon-maskable-512.png" &&
|
||||
icon.sizes === "512x512" &&
|
||||
icon.purpose === "maskable"
|
||||
)
|
||||
);
|
||||
assert.equal(
|
||||
manifest.icons.some((icon) => icon.src === "/icon.png" && icon.purpose.includes("maskable")),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user