15 Commits

Author SHA1 Message Date
nianzhibai 5cd5e8a126 fix: rename failed teaser retry action 2026-06-02 15:54:37 +08:00
nianzhibai 233b55c0f8 chore: verify runtime dependencies during install 2026-06-02 15:50:19 +08:00
nianzhibai 11c8585ea9 fix: prevent empty listing layout flicker 2026-06-02 15:39:21 +08:00
nianzhibai c3ada441c1 fix: prevent hover animation flicker 2026-06-02 15:30:13 +08:00
nianzhibai 2250d3ca4c docs: add MIT license 2026-06-02 15:16:35 +08:00
nianzhibai e465eac568 fix: improve local storage path diagnostics 2026-06-02 15:11:53 +08:00
nianzhibai 667a0c4e1c perf: optimize home page loading 2026-06-02 15:04:12 +08:00
nianzhibai 33ff07314a 123云盘支持,删除存储逻辑优化 2026-06-02 14:30:16 +08:00
nianzhibai 858abd7532 fix: install socks support for 91Spider proxy 2026-06-01 20:31:33 +08:00
nianzhibai 91352ac681 fix: reduce duplicate home recommendations 2026-06-01 19:02:41 +08:00
nianzhibai d70cea7320 fix: improve 91Spider tagging and deduped tag filters 2026-06-01 18:51:56 +08:00
nianzhibai 6654a1b730 perf: speed up catalog startup migrations 2026-06-01 18:03:21 +08:00
nianzhibai bcc887e088 feat: add 91Spider proxy support and drive improvements 2026-06-01 17:41:20 +08:00
nianzhibai 2c0cfe1d15 Add failed fingerprint retry controls 2026-06-01 13:42:32 +08:00
nianzhibai a276e6b32d 网盘 302 支持说明 2026-05-31 19:42:10 +08:00
68 changed files with 6060 additions and 260 deletions
+30 -5
View File
@@ -9,7 +9,7 @@
- 视频直链 (MP4)
依赖安装:
pip install requests beautifulsoup4 lxml
pip install requests beautifulsoup4 lxml PySocks
使用方法:
# 全量爬取(默认行为,从 page=1 一直爬到末尾,写到 OUTPUT_FILE
@@ -68,6 +68,7 @@ import time
import random
import json
import os
import socket
import sys
import html
from urllib.parse import urljoin, unquote, urlparse
@@ -80,6 +81,28 @@ except ImportError:
print("请运行: pip install beautifulsoup4 lxml")
sys.exit(1)
def prefer_ipv4_for_plain_socks5_proxy():
"""PySocks may pick IPv6 first for socks5://; some SOCKS5 servers only accept IPv4."""
proxy_envs = (
os.environ.get("HTTPS_PROXY", ""),
os.environ.get("HTTP_PROXY", ""),
os.environ.get("https_proxy", ""),
os.environ.get("http_proxy", ""),
)
uses_plain_socks5 = any(v.strip().lower().startswith("socks5://") for v in proxy_envs)
if not uses_plain_socks5 or getattr(socket, "_spider91_ipv4_first", False):
return
original_getaddrinfo = socket.getaddrinfo
def getaddrinfo_ipv4_first(*args, **kwargs):
infos = original_getaddrinfo(*args, **kwargs)
return sorted(infos, key=lambda info: 0 if info[0] == socket.AF_INET else 1)
socket.getaddrinfo = getaddrinfo_ipv4_first
socket._spider91_ipv4_first = True
# ===================== 配置区域 =====================
BASE_URL = "https://www.91porn.com/v.php"
LIST_PARAMS = {
@@ -706,7 +729,7 @@ def print_help():
- 视频直链 (MP4)
依赖安装:
pip install requests beautifulsoup4 lxml
pip install requests beautifulsoup4 lxml PySocks
使用方法:
python spider_91porn.py
@@ -757,13 +780,15 @@ def main():
"日志改走 stderr。配合 backend 边读边下载使用。")
args, _ = parser.parse_known_args()
cli_out = sys.stderr if args.stream_output else sys.stdout
prefer_ipv4_for_plain_socks5_proxy()
print("""
================================================
91porn 视频爬虫启动中...
================================================
按 Ctrl+C 可随时中断并保存进度
""")
""", file=cli_out)
# 加载已知 ID(来自 backend 的 catalog 已入库列表;兼容旧参数名)
seen_viewkeys = []
@@ -775,9 +800,9 @@ def main():
if line:
seen_viewkeys.append(line)
except FileNotFoundError:
print(f"警告: --seen-viewkeys-file 不存在: {args.seen_viewkeys_file}")
print(f"警告: --seen-viewkeys-file 不存在: {args.seen_viewkeys_file}", file=cli_out)
except Exception as e:
print(f"警告: 读取 --seen-viewkeys-file 失败: {e}")
print(f"警告: 读取 --seen-viewkeys-file 失败: {e}", file=cli_out)
# 决定运行模式
if args.target_new is not None:
+3
View File
@@ -36,10 +36,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
python3-bs4 \
python3-lxml \
python3-requests \
python3-socks \
tar \
tzdata \
&& rm -rf /var/lib/apt/lists/*
RUN python3 -c "import requests, bs4, lxml, socks"
WORKDIR /opt/video-site-91
COPY --from=backend /out/server ./server
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2026 nianzhibai
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+9 -2
View File
@@ -12,7 +12,8 @@
<a href="#快速开始">快速开始</a> ·
<a href="#功能特性">功能特性</a> ·
<a href="#预览图">预览图</a> ·
<a href="#数据存放位置">数据目录</a>
<a href="#数据存放位置">数据目录</a> ·
<a href="#许可证">许可证</a>
</p>
---
@@ -20,7 +21,7 @@
## 功能特性
- **多后端支持** — 兼容 115 云盘、PikPak 云盘、OneDrive、Google Drive 和本地存储
- **低带宽播放** — 支持 302 的云盘可直连播放;Google Drive 等需鉴权直链的来源走后端代理
- **低带宽播放** — 115 云盘、PikPak 云盘、OneDrive 都支持302模式,在线播放视频时,不占用服务器带宽,播放体验不受服务器带宽影响;Google Drive 不支持302模式,走服务器中转,观看体验会受服务器带宽影响
- **封面 & 预览片段** — 自动为每个视频生成封面图和预览片段,首页快速选片
- **91 爬虫** — 内置爬虫,支持抓取 91 本月最热视频
- **双主题** — 黑黄经典主题 / 粉白清新主题,随时切换
@@ -196,6 +197,12 @@ docker compose up -d # 更新并重启
---
## 许可证
本项目基于 [MIT License](LICENSE) 开源。
---
## 致谢
- [OpenList](https://github.com/OpenListTeam/OpenList) — 优秀的开源项目
+225 -5
View File
@@ -30,6 +30,7 @@ import (
"github.com/video-site/backend/internal/drives/localupload"
"github.com/video-site/backend/internal/drives/onedrive"
"github.com/video-site/backend/internal/drives/p115"
"github.com/video-site/backend/internal/drives/p123"
"github.com/video-site/backend/internal/drives/pikpak"
"github.com/video-site/backend/internal/drives/quark"
"github.com/video-site/backend/internal/drives/spider91"
@@ -81,6 +82,7 @@ func main() {
Catalog: cat,
Registry: app.registry,
GetTargetDriveID: func() string { return app.Spider91UploadDriveID() },
CommonThumbDir: app.commonThumbsDir(),
})
// 初始化本地内置盘;外部云盘放到 HTTP 服务启动后异步挂载,避免上游
@@ -90,6 +92,11 @@ func main() {
app.loadTheme(ctx)
app.loadSpider91UploadDriveID(ctx)
if removed, err := app.cleanupOrphanDriveVideos(ctx); err != nil {
log.Printf("[cleanup] orphan drive videos: %v", err)
} else if removed > 0 {
log.Printf("[cleanup] removed %d orphan drive videos", removed)
}
if err := app.attachLocalUpload(ctx); err != nil {
log.Printf("[local-upload] attach failed: %v", err)
}
@@ -156,6 +163,9 @@ func main() {
}
return app.attachDrive(ctx, d)
},
OnDriveDeleteCleanup: func(cleanupCtx context.Context, driveID string) (int, error) {
return app.cleanupDriveVideosForDelete(cleanupCtx, driveID)
},
OnDriveRemoved: func(driveID string) {
app.detachDrive(driveID)
},
@@ -182,6 +192,9 @@ func main() {
OnRegenFailedThumbnails: func(driveID string) {
go app.regenFailedThumbnails(ctx, driveID)
},
OnRegenFailedFingerprints: func(driveID string) {
go app.regenFailedFingerprints(ctx, driveID)
},
GetDriveGenerationStatuses: func() map[string]api.DriveGenerationStatuses {
return app.driveGenerationStatuses()
},
@@ -608,6 +621,22 @@ func (a *App) attachDriveUnlocked(ctx context.Context, d *catalog.Drive) error {
Cookie: d.Credentials["cookie"],
RootID: d.RootID,
})
case p123.Kind:
drv = p123.New(p123.Config{
ID: d.ID,
Username: d.Credentials["username"],
Password: d.Credentials["password"],
AccessToken: d.Credentials["access_token"],
Platform: d.Credentials["platform"],
RootID: d.RootID,
OnTokenUpdate: func(access string) {
if d.Credentials == nil {
d.Credentials = make(map[string]string)
}
d.Credentials["access_token"] = access
_ = a.cat.UpsertDrive(ctx, d)
},
})
case "pikpak":
drv = pikpak.New(pikpak.Config{
ID: d.ID,
@@ -774,7 +803,7 @@ func fingerprintConfigForDrive(drv drives.Drive) fingerprint.Config {
return cfg
}
switch strings.ToLower(drv.Kind()) {
case "p115", "onedrive":
case "p115", "p123", "onedrive":
cfg.RateLimitCooldown = 10 * time.Minute
case "pikpak":
cfg.RateLimitCooldown = 5 * time.Minute
@@ -854,13 +883,14 @@ func (a *App) attachSpider91Crawler(d *catalog.Drive, drv *spider91.Driver) {
a.spider91Crawlers[driveID] = c
a.mu.Unlock()
// 确保 "91porn" 系统标签存在,并把已入库的 spider91 视频按 author 字段
// 匹配补打这个标签(CreateTagAndClassify 内部对所有视频走一遍 classify)。
// 重复调用是幂等的:tags 用 INSERT OR IGNOREvideo_tags 也是 INSERT OR IGNORE
// 确保 "91porn" 系统标签存在,并 spider91 来源前缀给历史视频补打。
// 不能只靠文本匹配:老版本入库的视频可能没有 author/tags 字段,但 id 前缀
// "spider91-<driveID>-" 会一直保留,即使后续迁移到 PikPak/115 也不变
bgCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
go func() {
defer cancel()
if _, err := a.cat.CreateTagAndClassify(bgCtx, spider91.DefaultTag, nil, "system"); err != nil {
prefix := "spider91-" + driveID + "-"
if _, err := a.cat.EnsureTagForVideoIDPrefix(bgCtx, prefix, spider91.DefaultTag, nil, "system"); err != nil {
log.Printf("[spider91] ensure %q tag: %v", spider91.DefaultTag, err)
}
}()
@@ -1231,6 +1261,160 @@ func (a *App) cleanupMissingDriveVideos(ctx context.Context, driveID string, liv
return removed, nil
}
func (a *App) cleanupDriveVideosForDelete(ctx context.Context, driveID string) (int, error) {
if a == nil || a.cat == nil {
return 0, nil
}
d, err := a.cat.GetDrive(ctx, driveID)
if err != nil {
return 0, err
}
// Stop generation/crawl workers before deleting assets so they do not keep
// writing files for a drive that is being removed.
a.detachDrive(driveID)
items, err := a.videosForDriveDelete(ctx, d)
if err != nil {
return 0, err
}
localDir := ""
if a.cfg != nil {
localDir = a.cfg.Storage.LocalPreviewDir
}
for _, v := range items {
if err := ctx.Err(); err != nil {
return 0, err
}
if err := removeLocalVideoAssets(localDir, v); err != nil {
return 0, fmt.Errorf("remove local assets for %s: %w", v.ID, err)
}
}
if strings.EqualFold(d.Kind, spider91.Kind) {
if err := a.removeSpider91DriveDir(driveID); err != nil {
return 0, err
}
}
removed := 0
for _, v := range items {
if err := ctx.Err(); err != nil {
return removed, err
}
if err := a.cat.DeleteVideo(ctx, v.ID); err != nil {
if errors.Is(err, sql.ErrNoRows) {
continue
}
return removed, fmt.Errorf("delete catalog video %s: %w", v.ID, err)
}
removed++
}
return removed, nil
}
func (a *App) cleanupOrphanDriveVideos(ctx context.Context) (int, error) {
if a == nil || a.cat == nil {
return 0, nil
}
items, err := a.cat.ListVideosWithMissingDrive(ctx)
if err != nil {
return 0, err
}
if len(items) == 0 {
return 0, nil
}
localDir := ""
if a.cfg != nil {
localDir = a.cfg.Storage.LocalPreviewDir
}
spider91Dirs := map[string]struct{}{}
for _, v := range items {
if err := ctx.Err(); err != nil {
return 0, err
}
if err := removeLocalVideoAssets(localDir, v); err != nil {
return 0, fmt.Errorf("remove local assets for orphan %s: %w", v.ID, err)
}
if strings.HasPrefix(v.ID, "spider91-"+v.DriveID+"-") {
spider91Dirs[v.DriveID] = struct{}{}
}
}
for driveID := range spider91Dirs {
if err := a.removeSpider91DriveDir(driveID); err != nil {
return 0, err
}
}
removed := 0
for _, v := range items {
if err := ctx.Err(); err != nil {
return removed, err
}
if err := a.cat.DeleteVideo(ctx, v.ID); err != nil {
if errors.Is(err, sql.ErrNoRows) {
continue
}
return removed, fmt.Errorf("delete orphan catalog video %s: %w", v.ID, err)
}
removed++
}
return removed, nil
}
func (a *App) videosForDriveDelete(ctx context.Context, d *catalog.Drive) ([]*catalog.Video, error) {
if d == nil {
return nil, nil
}
items, err := a.cat.ListVideosByDrive(ctx, d.ID)
if err != nil {
return nil, err
}
byID := make(map[string]*catalog.Video, len(items))
for _, v := range items {
byID[v.ID] = v
}
if strings.EqualFold(d.Kind, spider91.Kind) {
prefix := "spider91-" + d.ID + "-"
originItems, err := a.cat.ListVideosByIDPrefix(ctx, prefix)
if err != nil {
return nil, err
}
for _, v := range originItems {
byID[v.ID] = v
}
}
out := make([]*catalog.Video, 0, len(byID))
for _, v := range byID {
out = append(out, v)
}
return out, nil
}
func (a *App) removeSpider91DriveDir(driveID string) error {
if strings.TrimSpace(driveID) == "" {
return errors.New("remove spider91 drive dir: empty drive id")
}
root := a.spider91RootDir()
dir := a.spider91DriveDir(driveID)
clean, ok := localPathWithin(root, dir)
if !ok {
return fmt.Errorf("remove spider91 drive dir: unsafe path %s", dir)
}
rootClean, ok := localPathWithin(root, root)
if !ok || clean == rootClean {
return fmt.Errorf("remove spider91 drive dir: refusing to remove root %s", root)
}
if err := os.RemoveAll(clean); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("remove spider91 drive dir %s: %w", clean, err)
}
return nil
}
func removeLocalVideoAssets(localDir string, v *catalog.Video) error {
if localDir == "" || v == nil || v.ID == "" {
return nil
@@ -1551,6 +1735,42 @@ func (a *App) regenFailedThumbnails(ctx context.Context, driveID string) {
log.Printf("[thumb] enqueued failed thumbnails for regen drive=%s queued=%d", driveID, queued)
}
func (a *App) regenFailedFingerprints(ctx context.Context, driveID string) {
items, err := a.cat.ListVideosByFingerprintStatus(ctx, driveID, "failed", 0)
if err != nil {
log.Printf("[fingerprint] list failed videos for regen drive=%s: %v", driveID, err)
return
}
a.mu.Lock()
fingerprintWorker := a.fingerprintWorkers[driveID]
a.mu.Unlock()
if fingerprintWorker == nil {
log.Printf("[fingerprint] regen failed drive=%s skipped: fingerprint worker not found", driveID)
return
}
log.Printf("[fingerprint] enqueue failed videos for regen drive=%s count=%d", driveID, len(items))
queued := 0
for _, v := range items {
if err := ctx.Err(); err != nil {
log.Printf("[fingerprint] enqueue failed canceled drive=%s queued=%d: %v", driveID, queued, err)
return
}
if err := a.cat.UpdateVideoFingerprint(ctx, v.ID, "", "pending", ""); err != nil {
log.Printf("[fingerprint] reset failed video %s drive=%s: %v", v.ID, driveID, err)
continue
}
v.SampledSHA256 = ""
v.FingerprintStatus = "pending"
v.FingerprintError = ""
if !fingerprintWorker.EnqueueBlocking(ctx, v) {
log.Printf("[fingerprint] enqueue failed canceled drive=%s queued=%d", driveID, queued)
return
}
queued++
}
log.Printf("[fingerprint] enqueued failed videos for regen drive=%s queued=%d", driveID, queued)
}
// listScanTargetIDs 返回 nightly Phase 1 应扫描的所有 drive ID
// (非 spider91、非 localupload)。它直接读 catalog,而不是 registry,这样
// 进程刚启动、云盘还在后台挂载时,nightly 也不会漏掉配置过的 drive。
+304
View File
@@ -13,6 +13,7 @@ import (
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/config"
"github.com/video-site/backend/internal/drives"
"github.com/video-site/backend/internal/drives/spider91"
"github.com/video-site/backend/internal/fingerprint"
"github.com/video-site/backend/internal/preview"
"github.com/video-site/backend/internal/proxy"
@@ -671,6 +672,309 @@ func TestCleanupMissingPikPakVideosRemovesDatabaseRowsAndLocalAssets(t *testing.
}
}
func TestCleanupDriveVideosForDeleteRemovesRowsAndGeneratedAssetsOnly(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
localDir := filepath.Join(root, "previews")
originalDir := filepath.Join(root, "local-videos")
originalVideo := filepath.Join(originalDir, "clip.mp4")
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
for _, path := range []string{originalVideo} {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", path, err)
}
if err := os.WriteFile(path, []byte("original"), 0o644); err != nil {
t.Fatalf("write %s: %v", path, err)
}
}
if err := cat.UpsertDrive(ctx, &catalog.Drive{
ID: "local-main",
Kind: "localstorage",
Name: "Local",
RootID: "/",
Credentials: map[string]string{"path": originalDir},
TeaserEnabled: true,
}); err != nil {
t.Fatalf("seed drive: %v", err)
}
previewPath := filepath.Join(localDir, "localstorage-local-main-file.mp4")
thumbPath := filepath.Join(localDir, "thumbs", "localstorage-local-main-file.jpg")
for _, path := range []string{previewPath, thumbPath} {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", path, err)
}
if err := os.WriteFile(path, []byte("generated"), 0o644); err != nil {
t.Fatalf("write %s: %v", path, err)
}
}
now := time.Now()
if err := cat.UpsertVideo(ctx, &catalog.Video{
ID: "localstorage-local-main-file",
DriveID: "local-main",
FileID: "encoded-local-file",
Title: "Local File",
PreviewLocal: previewPath,
PreviewStatus: "ready",
ThumbnailURL: "/p/thumb/localstorage-local-main-file",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
}); err != nil {
t.Fatalf("seed local video: %v", err)
}
if err := cat.UpsertVideo(ctx, &catalog.Video{
ID: "pikpak-other",
DriveID: "PikPak",
FileID: "other",
Title: "Other",
PreviewStatus: "ready",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
}); err != nil {
t.Fatalf("seed other video: %v", err)
}
app := &App{
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
cat: cat,
registry: proxy.NewRegistry(),
workers: make(map[string]*preview.Worker),
thumbWorkers: make(map[string]*preview.ThumbWorker),
fingerprintWorkers: make(map[string]*fingerprint.Worker),
spider91Crawlers: make(map[string]*spider91.Crawler),
}
removed, err := app.cleanupDriveVideosForDelete(ctx, "local-main")
if err != nil {
t.Fatalf("cleanup drive videos: %v", err)
}
if removed != 1 {
t.Fatalf("removed = %d, want 1", removed)
}
if _, err := cat.GetVideo(ctx, "localstorage-local-main-file"); err != sql.ErrNoRows {
t.Fatalf("deleted video lookup error = %v, want sql.ErrNoRows", err)
}
if _, err := cat.GetVideo(ctx, "pikpak-other"); err != nil {
t.Fatalf("other drive video missing: %v", err)
}
for _, path := range []string{previewPath, thumbPath} {
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("generated asset %s still exists, stat err=%v", path, err)
}
}
if _, err := os.Stat(originalVideo); err != nil {
t.Fatalf("original local video should remain, stat err=%v", err)
}
}
func TestCleanupDriveVideosForDeleteSpider91RemovesCrawledDirAndOriginRecords(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
localDir := filepath.Join(root, "previews")
driveID := "spider-main"
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
if err := cat.UpsertDrive(ctx, &catalog.Drive{
ID: driveID,
Kind: "spider91",
Name: "91 Spider",
RootID: "/",
TeaserEnabled: true,
}); err != nil {
t.Fatalf("seed spider91 drive: %v", err)
}
spiderDriveDir := filepath.Join(root, "spider91", driveID)
sourceVideo := filepath.Join(spiderDriveDir, "videos", "source.mp4")
sourceThumb := filepath.Join(spiderDriveDir, "thumbs", "source.jpg")
localPreview := filepath.Join(localDir, "spider91-spider-main-source.mp4")
localThumb := filepath.Join(localDir, "thumbs", "spider91-spider-main-source.jpg")
migratedPreview := filepath.Join(localDir, "spider91-spider-main-migrated.mp4")
migratedThumb := filepath.Join(localDir, "thumbs", "spider91-spider-main-migrated.jpg")
for _, path := range []string{sourceVideo, sourceThumb, localPreview, localThumb, migratedPreview, migratedThumb} {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", path, err)
}
if err := os.WriteFile(path, []byte("asset"), 0o644); err != nil {
t.Fatalf("write %s: %v", path, err)
}
}
now := time.Now()
for _, v := range []*catalog.Video{
{
ID: "spider91-spider-main-source",
DriveID: driveID,
FileID: "source.mp4",
Title: "Source",
PreviewLocal: localPreview,
PreviewStatus: "ready",
ThumbnailURL: "/p/thumb/spider91-spider-main-source",
},
{
ID: "spider91-spider-main-migrated",
DriveID: "PikPak",
FileID: "pikpak-file-id",
Title: "Migrated",
PreviewLocal: migratedPreview,
PreviewStatus: "ready",
ThumbnailURL: "/p/thumb/spider91-spider-main-migrated",
},
{
ID: "pikpak-PikPak-other",
DriveID: "PikPak",
FileID: "other",
Title: "Other",
PreviewStatus: "ready",
},
} {
v.PublishedAt = now
v.CreatedAt = now
v.UpdatedAt = now
if err := cat.UpsertVideo(ctx, v); err != nil {
t.Fatalf("seed video %s: %v", v.ID, err)
}
}
app := &App{
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
cat: cat,
registry: proxy.NewRegistry(),
workers: make(map[string]*preview.Worker),
thumbWorkers: make(map[string]*preview.ThumbWorker),
fingerprintWorkers: make(map[string]*fingerprint.Worker),
spider91Crawlers: make(map[string]*spider91.Crawler),
}
removed, err := app.cleanupDriveVideosForDelete(ctx, driveID)
if err != nil {
t.Fatalf("cleanup spider91 videos: %v", err)
}
if removed != 2 {
t.Fatalf("removed = %d, want 2", removed)
}
for _, id := range []string{"spider91-spider-main-source", "spider91-spider-main-migrated"} {
if _, err := cat.GetVideo(ctx, id); err != sql.ErrNoRows {
t.Fatalf("%s lookup error = %v, want sql.ErrNoRows", id, err)
}
}
if _, err := cat.GetVideo(ctx, "pikpak-PikPak-other"); err != nil {
t.Fatalf("unrelated pikpak video missing: %v", err)
}
for _, path := range []string{spiderDriveDir, localPreview, localThumb, migratedPreview, migratedThumb} {
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("%s still exists, stat err=%v", path, err)
}
}
}
func TestCleanupOrphanDriveVideosRemovesRowsAndGeneratedAssets(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
localDir := filepath.Join(root, "previews")
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
if err := cat.UpsertDrive(ctx, &catalog.Drive{
ID: "active-drive",
Kind: "pikpak",
Name: "Active",
RootID: "root",
TeaserEnabled: true,
}); err != nil {
t.Fatalf("seed active drive: %v", err)
}
previewPath := filepath.Join(localDir, "p123-123-orphan.mp4")
thumbPath := filepath.Join(localDir, "thumbs", "p123-123-orphan.jpg")
for _, path := range []string{previewPath, thumbPath} {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", path, err)
}
if err := os.WriteFile(path, []byte("generated"), 0o644); err != nil {
t.Fatalf("write %s: %v", path, err)
}
}
now := time.Now()
for _, v := range []*catalog.Video{
{
ID: "p123-123-orphan",
DriveID: "123",
FileID: "orphan-file",
Title: "Orphan",
PreviewLocal: previewPath,
PreviewStatus: "ready",
ThumbnailURL: "/p/thumb/p123-123-orphan",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
},
{
ID: "pikpak-active",
DriveID: "active-drive",
FileID: "active-file",
Title: "Active",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
},
} {
if err := cat.UpsertVideo(ctx, v); err != nil {
t.Fatalf("seed video %s: %v", v.ID, err)
}
}
app := &App{
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
cat: cat,
}
removed, err := app.cleanupOrphanDriveVideos(ctx)
if err != nil {
t.Fatalf("cleanup orphan videos: %v", err)
}
if removed != 1 {
t.Fatalf("removed = %d, want 1", removed)
}
if _, err := cat.GetVideo(ctx, "p123-123-orphan"); err != sql.ErrNoRows {
t.Fatalf("orphan video lookup error = %v, want sql.ErrNoRows", err)
}
if _, err := cat.GetVideo(ctx, "pikpak-active"); err != nil {
t.Fatalf("active video missing: %v", err)
}
for _, path := range []string{previewPath, thumbPath} {
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("orphan asset %s still exists, stat err=%v", path, err)
}
}
}
func TestCleanupDuplicateVideoAssetsRemovesOnlyDuplicateLocalAssets(t *testing.T) {
ctx := context.Background()
localDir := t.TempDir()
+1 -1
View File
@@ -59,7 +59,7 @@ preview:
width: 480
# 盘列表。上线后请通过管理后台添加,本文件可留空。
# kind 支持 quark / p115 / pikpak / wopan / onedrive / googledrive / localstorage。
# kind 支持 quark / p115 / p123 / pikpak / wopan / onedrive / googledrive / localstorage。
# OneDrive 示例:
# - id: "my-onedrive"
# kind: "onedrive"
+4 -4
View File
@@ -7,15 +7,18 @@ toolchain go1.23.4
require (
github.com/OpenListTeam/wopan-sdk-go v0.2.0
github.com/SheltonZhu/115driver v1.3.2
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
github.com/go-chi/chi/v5 v5.1.0
github.com/go-resty/resty/v2 v2.14.0
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
golang.org/x/net v0.27.0
golang.org/x/sys v0.30.0
gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.33.1
)
require (
github.com/aead/ecdh v0.2.0 // indirect
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect
github.com/andreburgaud/crypt2go v1.1.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/google/uuid v1.6.0 // indirect
@@ -26,10 +29,7 @@ require (
github.com/pierrec/lz4/v4 v4.1.17 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
golang.org/x/crypto v0.25.0 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/sys v0.30.0 // indirect
golang.org/x/time v0.8.0 // indirect
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
modernc.org/libc v1.55.3 // indirect
+143 -2
View File
@@ -6,7 +6,9 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strconv"
"strings"
@@ -16,6 +18,7 @@ import (
"github.com/video-site/backend/internal/auth"
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/drives/p123"
)
type AdminServer struct {
@@ -40,12 +43,14 @@ type AdminServer struct {
LocalPreviewDir string
// Hooks:外层注入实际执行者
OnDriveSaved func(driveID string) error
OnDriveDeleteCleanup func(ctx context.Context, driveID string) (int, error)
OnDriveRemoved func(driveID string)
OnScanRequested func(driveID string)
OnRegenPreview func(videoID string)
OnRegenAllPreviews func()
OnRegenFailedPreviews func(driveID string)
OnRegenFailedThumbnails func(driveID string)
OnRegenFailedFingerprints func(driveID string)
GetDriveGenerationStatuses func() map[string]DriveGenerationStatuses
// OnTeaserEnabledChanged 在 per-drive teaser 开关被切换后调用。
// enabled=true 时上层应该重新把 pending teaser 入队(类似旧的全局开关从关到开);
@@ -68,6 +73,9 @@ type AdminServer struct {
// 用于"设置跳过目录"弹窗按需展开浏览网盘目录树;只返回目录条目,文件忽略。
// 调用方应当处理 error 并以 5xx 返回前端。
ListDriveDirChildren func(ctx context.Context, driveID, parentID string) ([]DriveDirEntry, error)
// 123 云盘扫码登录接口测试注入;生产留空走官方 user.123pan.cn。
P123UserAPIBaseURL string
P123HTTPClient *http.Client
}
// DriveDirEntry 是 dirtree 接口的一条返回项:网盘上的一个目录节点。
@@ -114,6 +122,8 @@ func (a *AdminServer) Register(r chi.Router) {
r.Get("/drives", a.handleListDrives)
r.Get("/drives/storage", a.handleDriveStorage)
r.Post("/drives", a.handleUpsertDrive)
r.Post("/drives/p123/qr", a.handleP123QRStart)
r.Get("/drives/p123/qr/{uniID}", a.handleP123QRStatus)
r.Delete("/drives/{id}", a.handleDeleteDrive)
r.Post("/drives/{id}/rescan", a.handleRescan)
r.Post("/drives/{id}/teaser-enabled", a.handleSetDriveTeaserEnabled)
@@ -121,6 +131,7 @@ func (a *AdminServer) Register(r chi.Router) {
r.Get("/drives/{id}/dirtree", a.handleListDriveDirTree)
r.Post("/drives/{id}/previews/failed/regenerate", a.handleRegenFailedPreviews)
r.Post("/drives/{id}/thumbnails/failed/regenerate", a.handleRegenFailedThumbnails)
r.Post("/drives/{id}/fingerprints/failed/regenerate", a.handleRegenFailedFingerprints)
// 视频
r.Get("/videos", a.handleAdminListVideos)
@@ -393,6 +404,7 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
SkipDirIDs []string `json:"skipDirIds"`
// LastCrawlAt 是 spider91 上次成功爬取的 unix 秒(来自 credentials.last_crawl_at)。
// 其它 kind 留 0;前端用它显示"上次抓取: N 小时前"。
Spider91Proxy string `json:"spider91Proxy,omitempty"`
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
@@ -451,6 +463,7 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
HasCredential: hasCred,
TeaserEnabled: d.TeaserEnabled,
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
Spider91Proxy: spider91ProxyForDrive(d),
LastCrawlAt: lastCrawlAt,
ThumbnailGenerationStatus: generation.Thumbnail,
PreviewGenerationStatus: generation.Preview,
@@ -503,7 +516,14 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
if existingDrive, err := a.Catalog.GetDrive(r.Context(), body.ID); err == nil {
existing = existingDrive
}
if len(body.Credentials) == 0 && existing != nil && len(existing.Credentials) > 0 {
if body.Kind == "spider91" {
credentials, err := mergeSpider91Credentials(existing, body.Credentials)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
body.Credentials = credentials
} else if len(body.Credentials) == 0 && existing != nil && len(existing.Credentials) > 0 {
body.Credentials = existing.Credentials
}
@@ -552,8 +572,82 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
}
func spider91ProxyForDrive(d *catalog.Drive) string {
if d == nil || d.Kind != "spider91" || d.Credentials == nil {
return ""
}
return strings.TrimSpace(d.Credentials["proxy"])
}
func mergeSpider91Credentials(existing *catalog.Drive, incoming map[string]string) (map[string]string, error) {
merged := map[string]string{}
if existing != nil {
for k, v := range existing.Credentials {
merged[k] = v
}
}
for k, v := range incoming {
if strings.TrimSpace(k) == "" {
continue
}
if k == "proxy" {
proxy, err := normalizeSpider91ProxyURL(v)
if err != nil {
return nil, err
}
if proxy == "" {
delete(merged, "proxy")
} else {
merged["proxy"] = proxy
}
continue
}
merged[k] = v
}
return merged, nil
}
func normalizeSpider91ProxyURL(raw string) (string, error) {
proxy := strings.TrimSpace(raw)
if proxy == "" {
return "", nil
}
u, err := url.Parse(proxy)
if err != nil || u.Scheme == "" || u.Host == "" {
return "", fmt.Errorf("91Spider 代理地址格式无效,请填写类似 http://127.0.0.1:7890 的地址")
}
switch strings.ToLower(u.Scheme) {
case "http", "https", "socks5", "socks5h":
return proxy, nil
default:
return "", fmt.Errorf("91Spider 代理地址仅支持 http://、https://、socks5:// 或 socks5h://")
}
}
func (a *AdminServer) handleDeleteDrive(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
var body deleteDriveReq
if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
writeErr(w, http.StatusBadRequest, err)
return
}
if !body.DeleteVideos {
http.Error(w, "deleteVideos=true is required when deleting a drive", http.StatusBadRequest)
return
}
deletedVideos := 0
if a.OnDriveDeleteCleanup == nil {
http.Error(w, "drive video cleanup is not available", http.StatusInternalServerError)
return
}
removed, err := a.OnDriveDeleteCleanup(r.Context(), id)
if err != nil {
writeErr(w, http.StatusInternalServerError, err)
return
}
deletedVideos = removed
if err := a.Catalog.DeleteDrive(r.Context(), id); err != nil {
writeErr(w, http.StatusInternalServerError, err)
return
@@ -561,7 +655,11 @@ func (a *AdminServer) handleDeleteDrive(w http.ResponseWriter, r *http.Request)
if a.OnDriveRemoved != nil {
a.OnDriveRemoved(id)
}
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "deletedVideos": deletedVideos})
}
type deleteDriveReq struct {
DeleteVideos bool `json:"deleteVideos"`
}
func (a *AdminServer) handleRescan(w http.ResponseWriter, r *http.Request) {
@@ -572,6 +670,39 @@ func (a *AdminServer) handleRescan(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
}
func (a *AdminServer) p123QRClient() *p123.QRClient {
return p123.NewQRClient(p123.QRConfig{
UserAPIBaseURL: a.P123UserAPIBaseURL,
HTTPClient: a.P123HTTPClient,
})
}
func (a *AdminServer) handleP123QRStart(w http.ResponseWriter, r *http.Request) {
session, err := a.p123QRClient().Generate(r.Context())
if err != nil {
writeErr(w, http.StatusBadGateway, err)
return
}
w.Header().Set("Cache-Control", "no-store")
writeJSON(w, http.StatusOK, session)
}
func (a *AdminServer) handleP123QRStatus(w http.ResponseWriter, r *http.Request) {
uniID := chi.URLParam(r, "uniID")
loginUUID := r.URL.Query().Get("loginUuid")
if strings.TrimSpace(uniID) == "" || strings.TrimSpace(loginUUID) == "" {
http.Error(w, "uniID and loginUuid are required", http.StatusBadRequest)
return
}
status, err := a.p123QRClient().Poll(r.Context(), loginUUID, uniID)
if err != nil {
writeErr(w, http.StatusBadGateway, err)
return
}
w.Header().Set("Cache-Control", "no-store")
writeJSON(w, http.StatusOK, status)
}
// handleRunNightlyJob 触发一次完整的凌晨流水线(不论当前时间,不论今日是否已跑)。
// 立即返回 202;进度通过 backend 日志和下次 GET /admin/api/drives 的状态变化观察。
// 流水线已在跑或已排队时,Runner 会拒绝重复触发。
@@ -907,6 +1038,16 @@ func (a *AdminServer) handleRegenFailedThumbnails(w http.ResponseWriter, r *http
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
}
// handleRegenFailedFingerprints triggers regeneration for all failed sampled
// fingerprints on a drive. It mirrors the failed teaser/thumbnail retry endpoints.
func (a *AdminServer) handleRegenFailedFingerprints(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
if a.OnRegenFailedFingerprints != nil {
a.OnRegenFailedFingerprints(id)
}
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
}
// ---------- Settings ----------
// settingsDTO 是 GET/PUT /admin/api/settings 的入参/出参。
+317
View File
@@ -3,6 +3,7 @@ package api
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"net/http"
"net/http/httptest"
@@ -439,6 +440,300 @@ func TestHandleUpsertDriveReplacesExistingCredentialsWhenProvided(t *testing.T)
}
}
func TestHandleUpsertSpider91ProxyPreservesRuntimeCredentials(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
if err := cat.UpsertDrive(ctx, &catalog.Drive{
ID: "spider91-main",
Kind: "spider91",
Name: "91 Spider",
RootID: "/",
Credentials: map[string]string{
"last_crawl_at": "1800000000",
"proxy": "http://old-proxy.local:7890",
"script_path": "/opt/video-site-91/91VideoSpider/spider_91porn.py",
},
Status: "ok",
}); err != nil {
t.Fatalf("seed drive: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
"id": "spider91-main",
"kind": "spider91",
"name": "91 Spider",
"rootId": "/",
"credentials": {"proxy": " socks5h://proxy-user:proxy-pass@127.0.0.1:7891 "}
}`))
rr := httptest.NewRecorder()
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
got, err := cat.GetDrive(ctx, "spider91-main")
if err != nil {
t.Fatalf("get drive: %v", err)
}
if got.Credentials["proxy"] != "socks5h://proxy-user:proxy-pass@127.0.0.1:7891" {
t.Fatalf("proxy = %q, want trimmed new proxy", got.Credentials["proxy"])
}
if got.Credentials["last_crawl_at"] != "1800000000" {
t.Fatalf("last_crawl_at = %q, want preserved", got.Credentials["last_crawl_at"])
}
if got.Credentials["script_path"] == "" {
t.Fatalf("script_path should be preserved")
}
req = httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
"id": "spider91-main",
"kind": "spider91",
"name": "91 Spider",
"rootId": "/",
"credentials": {"proxy": " "}
}`))
rr = httptest.NewRecorder()
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("clear status = %d, body = %s", rr.Code, rr.Body.String())
}
got, err = cat.GetDrive(ctx, "spider91-main")
if err != nil {
t.Fatalf("get cleared drive: %v", err)
}
if _, ok := got.Credentials["proxy"]; ok {
t.Fatalf("proxy should be removed after empty save, got %q", got.Credentials["proxy"])
}
if got.Credentials["last_crawl_at"] != "1800000000" {
t.Fatalf("last_crawl_at after clear = %q, want preserved", got.Credentials["last_crawl_at"])
}
}
func TestHandleUpsertSpider91RejectsUnsupportedProxyScheme(t *testing.T) {
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
"id": "spider91-main",
"kind": "spider91",
"name": "91 Spider",
"rootId": "/",
"credentials": {"proxy": "ftp://127.0.0.1:21"}
}`))
rr := httptest.NewRecorder()
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
if rr.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want 400; body = %s", rr.Code, rr.Body.String())
}
if !strings.Contains(rr.Body.String(), "socks5:// 或 socks5h://") {
t.Fatalf("body = %q, want supported schemes message", rr.Body.String())
}
}
func TestHandleDeleteDriveRunsRequestedCleanupBeforeDeletingDrive(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
if err := cat.UpsertDrive(ctx, &catalog.Drive{
ID: "drive-one",
Kind: "pikpak",
Name: "Drive One",
RootID: "root",
TeaserEnabled: true,
}); err != nil {
t.Fatalf("seed drive: %v", err)
}
cleanupCalled := ""
removedCalled := ""
req := httptest.NewRequest(http.MethodDelete, "/admin/api/drives/drive-one", strings.NewReader(`{"deleteVideos":true}`))
rctx := chi.NewRouteContext()
rctx.URLParams.Add("id", "drive-one")
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
rr := httptest.NewRecorder()
(&AdminServer{
Catalog: cat,
OnDriveDeleteCleanup: func(cleanupCtx context.Context, driveID string) (int, error) {
cleanupCalled = driveID
if _, err := cat.GetDrive(cleanupCtx, driveID); err != nil {
t.Fatalf("drive should still exist during cleanup: %v", err)
}
return 3, nil
},
OnDriveRemoved: func(driveID string) {
removedCalled = driveID
},
}).handleDeleteDrive(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
if cleanupCalled != "drive-one" {
t.Fatalf("cleanup called with %q, want drive-one", cleanupCalled)
}
if removedCalled != "drive-one" {
t.Fatalf("removed hook called with %q, want drive-one", removedCalled)
}
if _, err := cat.GetDrive(ctx, "drive-one"); err != sql.ErrNoRows {
t.Fatalf("drive lookup error = %v, want sql.ErrNoRows", err)
}
var got struct {
OK bool `json:"ok"`
DeletedVideos int `json:"deletedVideos"`
}
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
t.Fatalf("decode: %v", err)
}
if !got.OK || got.DeletedVideos != 3 {
t.Fatalf("response = %#v, want ok with deletedVideos=3", got)
}
}
func TestHandleDeleteDriveRequiresCleanupConfirmation(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
if err := cat.UpsertDrive(ctx, &catalog.Drive{
ID: "drive-one",
Kind: "pikpak",
Name: "Drive One",
RootID: "root",
TeaserEnabled: true,
}); err != nil {
t.Fatalf("seed drive: %v", err)
}
req := httptest.NewRequest(http.MethodDelete, "/admin/api/drives/drive-one", strings.NewReader(`{"deleteVideos":false}`))
rctx := chi.NewRouteContext()
rctx.URLParams.Add("id", "drive-one")
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
rr := httptest.NewRecorder()
(&AdminServer{
Catalog: cat,
OnDriveDeleteCleanup: func(context.Context, string) (int, error) {
t.Fatal("cleanup hook should not be called without confirmation")
return 0, nil
},
}).handleDeleteDrive(rr, req)
if rr.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want 400; body = %s", rr.Code, rr.Body.String())
}
if _, err := cat.GetDrive(ctx, "drive-one"); err != nil {
t.Fatalf("drive should remain after rejected delete: %v", err)
}
}
func TestHandleListDrivesIncludesSpider91Proxy(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
for _, d := range []*catalog.Drive{
{
ID: "spider91-main",
Kind: "spider91",
Name: "91 Spider",
RootID: "/",
Credentials: map[string]string{
"last_crawl_at": "1800000000",
"proxy": " http://127.0.0.1:7890 ",
},
Status: "ok",
},
{
ID: "onedrive-main",
Kind: "onedrive",
Name: "OneDrive",
RootID: "root",
Credentials: map[string]string{
"proxy": "http://should-not-leak.local:7890",
},
Status: "ok",
},
} {
if err := cat.UpsertDrive(ctx, d); err != nil {
t.Fatalf("seed drive %s: %v", d.ID, err)
}
}
req := httptest.NewRequest(http.MethodGet, "/admin/api/drives", nil)
rr := httptest.NewRecorder()
(&AdminServer{Catalog: cat}).handleListDrives(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
var got []struct {
ID string `json:"id"`
Spider91Proxy string `json:"spider91Proxy"`
LastCrawlAt int64 `json:"lastCrawlAt"`
}
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
t.Fatalf("decode: %v", err)
}
byID := map[string]struct {
Spider91Proxy string
LastCrawlAt int64
}{}
for _, d := range got {
byID[d.ID] = struct {
Spider91Proxy string
LastCrawlAt int64
}{Spider91Proxy: d.Spider91Proxy, LastCrawlAt: d.LastCrawlAt}
}
if byID["spider91-main"].Spider91Proxy != "http://127.0.0.1:7890" {
t.Fatalf("spider91 proxy = %q, want trimmed proxy", byID["spider91-main"].Spider91Proxy)
}
if byID["spider91-main"].LastCrawlAt != 1800000000 {
t.Fatalf("lastCrawlAt = %d, want 1800000000", byID["spider91-main"].LastCrawlAt)
}
if byID["onedrive-main"].Spider91Proxy != "" {
t.Fatalf("onedrive spider91Proxy = %q, want empty", byID["onedrive-main"].Spider91Proxy)
}
}
func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
@@ -598,6 +893,28 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
}
}
func TestHandleRegenFailedFingerprintsInvokesHook(t *testing.T) {
called := ""
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives/drive-one/fingerprints/failed/regenerate", nil)
rctx := chi.NewRouteContext()
rctx.URLParams.Add("id", "drive-one")
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
rr := httptest.NewRecorder()
(&AdminServer{
OnRegenFailedFingerprints: func(driveID string) {
called = driveID
},
}).handleRegenFailedFingerprints(rr, req)
if rr.Code != http.StatusAccepted {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
if called != "drive-one" {
t.Fatalf("called drive = %q, want drive-one", called)
}
}
func TestHandleDriveStorageReportsLocalMediaUsage(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
+93 -35
View File
@@ -15,6 +15,7 @@ import (
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/go-chi/chi/v5"
@@ -53,6 +54,10 @@ type Server struct {
UploadDir string
OnVideoUploaded func(*catalog.Video)
tagCacheMu sync.Mutex
tagCacheUntil time.Time
tagCache []TagDTO
// GetTheme 返回当前生效的主题("dark" | "pink")。前台 /api/settings/theme 用,
// 不需要登录。无注入时返回 "dark"。
GetTheme func() string
@@ -86,6 +91,12 @@ type VideoDTO struct {
Category string `json:"category,omitempty"`
}
type TagDTO struct {
ID string `json:"id"`
Label string `json:"label"`
Count int `json:"count"`
}
type VideoDetailDTO struct {
VideoDTO
VideoSrc string `json:"videoSrc"`
@@ -156,41 +167,60 @@ func (s *Server) handleGetTheme(w http.ResponseWriter, r *http.Request) {
}
func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) {
// 首页优先展示封面已经生成好的视频,避免新盘扫盘时大量黑封面占满首页
// 候选仍按发布时间覆盖最近 200 个,随后随机洗牌;封面不足时再用普通可见视频补齐。
const candidatePool = 200
readyItems, _, err := s.Catalog.ListVideos(r.Context(), catalog.ListParams{
Sort: "latest", Page: 1, PageSize: candidatePool, ThumbnailReadyOnly: true,
})
// 首页优先从全量已有封面的视频里随机抽取,避免只在最近一小段候选中反复出现
excludeIDs := parseVideoIDQuery(r, "exclude", 120)
items, err := s.Catalog.RandomVideosWithReadyThumbnailsExcluding(r.Context(), excludeIDs, homePageSize)
if err != nil {
writeErr(w, http.StatusInternalServerError, err)
return
}
rand.Shuffle(len(readyItems), func(i, j int) {
readyItems[i], readyItems[j] = readyItems[j], readyItems[i]
})
items := appendUniqueVideos(nil, readyItems, homePageSize)
if len(items) > homePageSize {
items = items[:homePageSize]
}
if len(items) < homePageSize {
fallback, _, err := s.Catalog.ListVideos(r.Context(), catalog.ListParams{
Sort: "latest", Page: 1, PageSize: candidatePool,
})
fallbackExclude := append([]string{}, excludeIDs...)
for _, item := range items {
if item != nil {
fallbackExclude = append(fallbackExclude, item.ID)
}
}
fallback, err := s.Catalog.RandomVideosExcluding(r.Context(), fallbackExclude, homePageSize-len(items))
if err != nil {
writeErr(w, http.StatusInternalServerError, err)
return
}
rand.Shuffle(len(fallback), func(i, j int) {
fallback[i], fallback[j] = fallback[j], fallback[i]
})
items = appendUniqueVideos(items, fallback, homePageSize)
}
w.Header().Set("Cache-Control", "no-store")
writeJSON(w, http.StatusOK, mapVideos(items))
}
func parseVideoIDQuery(r *http.Request, key string, limit int) []string {
if r == nil {
return nil
}
values := r.URL.Query()[key]
if len(values) == 0 {
return nil
}
seen := map[string]struct{}{}
out := make([]string, 0, len(values))
for _, value := range values {
for _, id := range strings.Split(value, ",") {
id = strings.TrimSpace(id)
if id == "" {
continue
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
out = append(out, id)
if limit > 0 && len(out) >= limit {
return out
}
}
}
return out
}
func appendUniqueVideos(dst []*catalog.Video, candidates []*catalog.Video, limit int) []*catalog.Video {
if len(dst) >= limit {
return dst[:limit]
@@ -226,12 +256,13 @@ func (s *Server) handleList(w http.ResponseWriter, r *http.Request) {
}
sort := q.Get("sort")
params := catalog.ListParams{
Keyword: q.Get("q"),
Tag: q.Get("tag"),
Category: q.Get("cat"),
Sort: sort,
Page: page,
PageSize: size,
Keyword: q.Get("q"),
Tag: q.Get("tag"),
Category: q.Get("cat"),
Sort: sort,
Page: page,
PageSize: size,
SkipTotal: strings.EqualFold(q.Get("count"), "false"),
}
if sort == "" || sort == "latest" {
params.PreferReadyThumbnails = true
@@ -260,6 +291,15 @@ func (s *Server) handleVideoDetail(w http.ResponseWriter, r *http.Request) {
writeErr(w, http.StatusNotFound, sql.ErrNoRows)
return
}
if v.DriveID != localUploadDriveID {
if _, err := s.Catalog.GetDrive(r.Context(), v.DriveID); err != nil {
drives, listErr := s.Catalog.ListDrives(r.Context())
if listErr != nil || len(drives) > 0 {
writeErr(w, http.StatusNotFound, sql.ErrNoRows)
return
}
}
}
related := s.pickRelatedVideos(r.Context(), v, 6)
dto := mapVideo(v)
if d, err := s.Catalog.GetDrive(r.Context(), v.DriveID); err == nil {
@@ -423,20 +463,32 @@ func appendRandomRelated(picked []*catalog.Video, pool []*catalog.Video, targetL
}
func (s *Server) handleTags(w http.ResponseWriter, r *http.Request) {
now := time.Now()
s.tagCacheMu.Lock()
if s.tagCache != nil && now.Before(s.tagCacheUntil) {
out := append([]TagDTO(nil), s.tagCache...)
s.tagCacheMu.Unlock()
w.Header().Set("Cache-Control", "private, max-age=15")
writeJSON(w, http.StatusOK, out)
return
}
s.tagCacheMu.Unlock()
stats, err := s.Catalog.ListTags(r.Context())
if err != nil {
writeErr(w, http.StatusInternalServerError, err)
return
}
type tag struct {
ID string `json:"id"`
Label string `json:"label"`
Count int `json:"count"`
}
out := make([]tag, 0, len(stats))
out := make([]TagDTO, 0, len(stats))
for _, stat := range stats {
out = append(out, tag{ID: stat.Label, Label: stat.Label, Count: stat.Count})
out = append(out, TagDTO{ID: stat.Label, Label: stat.Label, Count: stat.Count})
}
s.tagCacheMu.Lock()
s.tagCache = append([]TagDTO(nil), out...)
s.tagCacheUntil = now.Add(30 * time.Second)
s.tagCacheMu.Unlock()
w.Header().Set("Cache-Control", "private, max-age=15")
writeJSON(w, http.StatusOK, out)
}
@@ -867,10 +919,14 @@ func previewURL(v *catalog.Video) string {
}
func thumbnailURL(v *catalog.Video) string {
base := "/p/thumb/" + v.ID
if v.ThumbnailURL != "" {
return v.ThumbnailURL
base = v.ThumbnailURL
}
return "/p/thumb/" + v.ID
if !strings.HasPrefix(base, "/p/thumb/") || v.UpdatedAt.IsZero() {
return base
}
return base + "?v=" + strconv.FormatInt(v.UpdatedAt.UnixMilli(), 10)
}
func (s *Server) videoSource(v *catalog.Video) string {
@@ -900,6 +956,8 @@ func driveKindLabel(kind string) string {
return "夸克网盘"
case "p115":
return "115 网盘"
case "p123":
return "123 云盘"
case "pikpak":
return "PikPak"
case "wopan":
+94
View File
@@ -99,6 +99,27 @@ func TestPreviewURLFallsBackWithoutUpdatedAt(t *testing.T) {
}
}
func TestThumbnailURLVersionsLocalGeneratedThumbnails(t *testing.T) {
got := thumbnailURL(&catalog.Video{
ID: "video-1",
ThumbnailURL: "/p/thumb/video-1",
UpdatedAt: time.UnixMilli(1778863000123),
})
if got != "/p/thumb/video-1?v=1778863000123" {
t.Fatalf("thumbnail URL = %q, want versioned local URL", got)
}
remote := "https://thumb.example/video-1.jpg"
got = thumbnailURL(&catalog.Video{
ID: "video-1",
ThumbnailURL: remote,
UpdatedAt: time.UnixMilli(1778863000123),
})
if got != remote {
t.Fatalf("remote thumbnail URL = %q, want unchanged %q", got, remote)
}
}
func TestHandleHomePrioritizesVideosWithReadyThumbnails(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
@@ -166,6 +187,59 @@ func TestHandleHomePrioritizesVideosWithReadyThumbnails(t *testing.T) {
}
}
func TestHandleHomeExcludesRecentlyShownVideos(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
now := time.Now()
for i := 0; i < homePageSize+4; i++ {
id := "ready-video-" + strconv.Itoa(i)
if err := cat.UpsertVideo(ctx, &catalog.Video{
ID: id,
DriveID: "drive",
FileID: id,
Title: id,
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
PublishedAt: now.Add(time.Duration(i) * time.Minute),
CreatedAt: now.Add(time.Duration(i) * time.Minute),
UpdatedAt: now.Add(time.Duration(i) * time.Minute),
}); err != nil {
t.Fatalf("seed ready video %s: %v", id, err)
}
}
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/home?exclude=ready-video-0&exclude=ready-video-1", nil)
(&Server{Catalog: cat}).handleHome(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
var got []VideoDTO
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
t.Fatalf("decode response: %v", err)
}
if len(got) != homePageSize {
t.Fatalf("home items = %d, want %d", len(got), homePageSize)
}
for _, item := range got {
if item.ID == "ready-video-0" || item.ID == "ready-video-1" {
t.Fatalf("home returned excluded video %q; items=%#v", item.ID, got)
}
if !strings.HasPrefix(item.ID, "ready-video-") {
t.Fatalf("home returned %q without a ready thumbnail; items=%#v", item.ID, got)
}
}
}
func TestHandleListLatestPrefersReadyThumbnails(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
@@ -237,6 +311,26 @@ func TestHandleListLatestPrefersReadyThumbnails(t *testing.T) {
t.Fatalf("thumbnail for %q = %q, want ready thumbnail URL", item.ID, item.Thumbnail)
}
}
rr = httptest.NewRecorder()
req = httptest.NewRequest(http.MethodGet, "/api/list?page=1&size=12&sort=latest&count=false", nil)
(&Server{Catalog: cat}).handleList(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("count=false status = %d, body = %s", rr.Code, rr.Body.String())
}
got = struct {
Items []VideoDTO `json:"items"`
Total int `json:"total"`
}{}
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
t.Fatalf("decode count=false response: %v", err)
}
if got.Total != 0 {
t.Fatalf("count=false total = %d, want 0", got.Total)
}
if len(got.Items) != 12 {
t.Fatalf("count=false items = %d, want 12", len(got.Items))
}
}
func TestHandleUploadVideoSavesFileVideoTagsAndQueuesPreview(t *testing.T) {
+115 -13
View File
@@ -622,6 +622,56 @@ func (c *Catalog) ListVideosByDrive(ctx context.Context, driveID string) ([]*Vid
return out, rows.Err()
}
func (c *Catalog) ListVideosByIDPrefix(ctx context.Context, prefix string) ([]*Video, error) {
prefix = strings.TrimSpace(prefix)
if prefix == "" {
return nil, fmt.Errorf("catalog: list videos by id prefix: empty prefix")
}
rows, err := c.db.QueryContext(ctx,
`SELECT `+allVideoCols+` FROM videos
WHERE SUBSTR(id, 1, LENGTH(?)) = ?
ORDER BY created_at ASC, id ASC`,
prefix, prefix)
if err != nil {
return nil, err
}
defer rows.Close()
var out []*Video
for rows.Next() {
v, err := scanVideo(rows)
if err != nil {
return nil, err
}
out = append(out, v)
}
return out, rows.Err()
}
func (c *Catalog) ListVideosWithMissingDrive(ctx context.Context) ([]*Video, error) {
rows, err := c.db.QueryContext(ctx,
`SELECT `+allVideoCols+` FROM videos
WHERE drive_id != 'local-upload'
AND NOT EXISTS (
SELECT 1
FROM drives
WHERE drives.id = videos.drive_id
)
ORDER BY drive_id ASC, id ASC`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []*Video
for rows.Next() {
v, err := scanVideo(rows)
if err != nil {
return nil, err
}
out = append(out, v)
}
return out, rows.Err()
}
// ListVideoFileIDsByDrive 只返回某 drive 下所有视频的 file_id 集合,
// 比 ListVideosByDrive 轻量。
func (c *Catalog) ListVideoFileIDsByDrive(ctx context.Context, driveID string) ([]string, error) {
@@ -765,6 +815,37 @@ func (c *Catalog) ListVideosNeedingFingerprint(ctx context.Context, driveID stri
return out, rows.Err()
}
// ListVideosByFingerprintStatus lists visible videos on a drive by fingerprint status.
// It is used by the admin "retry failed fingerprints" action to reset failed rows
// back to pending and enqueue them again.
func (c *Catalog) ListVideosByFingerprintStatus(ctx context.Context, driveID, status string, limit int) ([]*Video, error) {
if limit <= 0 {
limit = 10000
}
rows, err := c.db.QueryContext(ctx,
`SELECT `+allVideoCols+` FROM videos
WHERE drive_id = ?
AND COALESCE(sampled_sha256, '') = ''
AND COALESCE(fingerprint_status, 'pending') = ?
AND COALESCE(hidden, 0) = 0
ORDER BY created_at ASC, id ASC
LIMIT ?`,
driveID, status, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []*Video
for rows.Next() {
v, err := scanVideo(rows)
if err != nil {
return nil, err
}
out = append(out, v)
}
return out, rows.Err()
}
func (c *Catalog) UpdateVideoFingerprint(ctx context.Context, id, sampledSHA256, status, errText string) error {
sampledSHA256 = normalizeContentHash(sampledSHA256)
if status == "" {
@@ -798,6 +879,7 @@ type ListParams struct {
Sort string // latest | hot | week | long
ThumbnailReadyOnly bool
PreferReadyThumbnails bool
SkipTotal bool
Page int
PageSize int
}
@@ -822,12 +904,7 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
args = append(args, p.DriveID)
}
if p.Tag != "" {
where = append(where, `EXISTS (
SELECT 1
FROM video_tags vt
JOIN tags t ON t.id = vt.tag_id
WHERE vt.video_id = videos.id AND t.label = ? COLLATE NOCASE
)`)
where = append(where, videoMatchesTagLabelSQL("videos"))
args = append(args, p.Tag)
}
if p.Category != "" && p.Category != "all" {
@@ -838,6 +915,7 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
where = append(where, "COALESCE(thumbnail_url, '') != ''")
}
where = append(where, "COALESCE(hidden, 0) = 0")
where = append(where, activeDriveWhereSQL)
where = append(where, uniqueVideoWhereSQL)
whereSQL := ""
@@ -859,10 +937,11 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
orderBy = " ORDER BY " + readyOrderPrefix + "duration_seconds DESC"
}
// count
var total int
if err := c.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM videos"+whereSQL, args...).Scan(&total); err != nil {
return nil, 0, err
if !p.SkipTotal {
if err := c.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM videos"+whereSQL, args...).Scan(&total); err != nil {
return nil, 0, err
}
}
// list
@@ -893,6 +972,7 @@ func (c *Catalog) CountVisibleVideos(ctx context.Context) (int, error) {
err := c.db.QueryRowContext(ctx,
`SELECT COUNT(*) FROM videos
WHERE COALESCE(hidden, 0) = 0
AND `+activeDriveWhereSQL+`
AND `+uniqueVideoWhereSQL,
).Scan(&total)
if err != nil {
@@ -906,6 +986,14 @@ func (c *Catalog) CountVisibleVideos(ctx context.Context) (int, error) {
// 如果剩余可选数量 < limit,就返回所有可选项;调用方负责判断是否需要开新一轮。
// limit <= 0 时返回 nil, nil。
func (c *Catalog) RandomVideosExcluding(ctx context.Context, excludeIDs []string, limit int) ([]*Video, error) {
return c.randomVideosExcluding(ctx, excludeIDs, limit, false)
}
func (c *Catalog) RandomVideosWithReadyThumbnailsExcluding(ctx context.Context, excludeIDs []string, limit int) ([]*Video, error) {
return c.randomVideosExcluding(ctx, excludeIDs, limit, true)
}
func (c *Catalog) randomVideosExcluding(ctx context.Context, excludeIDs []string, limit int, thumbnailReadyOnly bool) ([]*Video, error) {
if limit <= 0 {
return nil, nil
}
@@ -913,7 +1001,11 @@ func (c *Catalog) RandomVideosExcluding(ctx context.Context, excludeIDs []string
cleaned := cleanVideoIDs(excludeIDs)
args := make([]any, 0, len(cleaned)+1)
whereSQL := `WHERE COALESCE(hidden, 0) = 0
AND ` + activeDriveWhereSQL + `
AND ` + uniqueVideoWhereSQL
if thumbnailReadyOnly {
whereSQL += " AND COALESCE(thumbnail_url, '') != ''"
}
if len(cleaned) > 0 {
placeholders := strings.Repeat("?,", len(cleaned))
placeholders = placeholders[:len(placeholders)-1]
@@ -993,6 +1085,7 @@ func (c *Catalog) LeastPopulatedVisibleUniqueTag(ctx context.Context, labels []s
`SELECT COUNT(*)
FROM videos
WHERE COALESCE(hidden, 0) = 0
AND `+activeDriveWhereSQL+`
AND `+uniqueVideoWhereSQL+`
AND EXISTS (
SELECT 1
@@ -1029,6 +1122,7 @@ func (c *Catalog) RandomVideosByTagExcluding(ctx context.Context, tag string, ex
args := make([]any, 0, len(cleaned)+2)
args = append(args, tag)
whereSQL := `WHERE COALESCE(hidden, 0) = 0
AND ` + activeDriveWhereSQL + `
AND ` + uniqueVideoWhereSQL + `
AND EXISTS (
SELECT 1
@@ -1472,10 +1566,7 @@ func normalizeDriveRootID(kind, rootID string) string {
}
return rootID
case "localstorage", "spider91":
if rootID == "" {
return "/"
}
return rootID
return "/"
default:
if rootID == "" {
return "0"
@@ -1679,6 +1770,17 @@ COALESCE(category, ''), COALESCE(hidden, 0), COALESCE(badges, '[]'), COALESCE(de
published_at, created_at, updated_at
`
const activeDriveWhereSQL = `(videos.drive_id = 'local-upload'
OR EXISTS (
SELECT 1
FROM drives
WHERE drives.id = videos.drive_id
)
OR NOT EXISTS (
SELECT 1
FROM drives
))`
const uniqueVideoWhereSQL = `((COALESCE(videos.content_hash, '') = ''
OR NOT EXISTS (
SELECT 1
+42
View File
@@ -61,6 +61,7 @@ func TestUpsertDriveDefaultsRootIDByKind(t *testing.T) {
{id: "onedrive", kind: "onedrive", want: "root"},
{id: "googledrive", kind: "googledrive", want: "root"},
{id: "localstorage", kind: "localstorage", want: "/"},
{id: "spider91", kind: "spider91", want: "/"},
}
for _, tc := range cases {
if err := cat.UpsertDrive(ctx, &Drive{
@@ -82,3 +83,44 @@ func TestUpsertDriveDefaultsRootIDByKind(t *testing.T) {
}
}
}
func TestUpsertDriveIgnoresRootIDForLocalStorageAndSpider91(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
for _, tc := range []struct {
id string
kind string
}{
{id: "localstorage", kind: "localstorage"},
{id: "spider91", kind: "spider91"},
} {
if err := cat.UpsertDrive(ctx, &Drive{
ID: tc.id,
Kind: tc.kind,
Name: tc.kind,
RootID: "manual-root",
ScanRootID: "manual-scan-root",
}); err != nil {
t.Fatalf("upsert %s: %v", tc.kind, err)
}
got, err := cat.GetDrive(ctx, tc.id)
if err != nil {
t.Fatalf("get %s: %v", tc.kind, err)
}
if got.RootID != "/" {
t.Fatalf("%s rootId = %q, want /", tc.kind, got.RootID)
}
if got.ScanRootID != "/" {
t.Fatalf("%s scanRootId = %q, want /", tc.kind, got.ScanRootID)
}
}
}
@@ -0,0 +1,64 @@
package catalog
import (
"context"
"testing"
"time"
)
func TestListVideosHidesMissingDriveVideosWhenDrivesExist(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
if err := cat.UpsertDrive(ctx, &Drive{
ID: "active-drive",
Kind: "pikpak",
Name: "Active",
RootID: "root",
TeaserEnabled: true,
}); err != nil {
t.Fatalf("seed drive: %v", err)
}
now := time.Now()
for _, v := range []*Video{
{
ID: "visible-video",
DriveID: "active-drive",
FileID: "visible-file",
Title: "Visible",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
},
{
ID: "orphan-video",
DriveID: "deleted-drive",
FileID: "orphan-file",
Title: "Orphan",
PublishedAt: now.Add(time.Second),
CreatedAt: now.Add(time.Second),
UpdatedAt: now.Add(time.Second),
},
} {
if err := cat.UpsertVideo(ctx, v); err != nil {
t.Fatalf("seed video %s: %v", v.ID, err)
}
}
items, total, err := cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 10, Sort: "latest"})
if err != nil {
t.Fatalf("list videos: %v", err)
}
if total != 1 || len(items) != 1 || items[0].ID != "visible-video" {
t.Fatalf("items total=%d items=%v, want only visible-video", total, items)
}
}
+1 -1
View File
@@ -73,7 +73,7 @@ CREATE TABLE IF NOT EXISTS deleted_tags (
-- 网盘账户
CREATE TABLE IF NOT EXISTS drives (
id TEXT PRIMARY KEY,
kind TEXT NOT NULL, -- quark / p115 / pikpak / wopan / onedrive / googledrive / localstorage / spider91
kind TEXT NOT NULL, -- quark / p115 / p123 / pikpak / wopan / onedrive / googledrive / localstorage / spider91
name TEXT NOT NULL,
root_id TEXT NOT NULL DEFAULT '0',
scan_root_id TEXT, -- deprecated: 扫描起点固定等于 root_id
+56
View File
@@ -110,6 +110,62 @@ func TestRandomVideosExcluding(t *testing.T) {
}
}
func TestRandomVideosWithReadyThumbnailsExcluding(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() { _ = cat.Close() })
now := time.Now()
for i := 0; i < 4; i++ {
id := "ready-" + string(rune('a'+i))
if err := cat.UpsertVideo(ctx, &Video{
ID: id,
DriveID: "drive",
FileID: "f-" + id,
Title: id,
ThumbnailURL: "/p/thumb/" + id,
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
}); err != nil {
t.Fatalf("seed %s: %v", id, err)
}
}
for i := 0; i < 4; i++ {
id := "pending-" + string(rune('a'+i))
if err := cat.UpsertVideo(ctx, &Video{
ID: id,
DriveID: "drive",
FileID: "f-" + id,
Title: id,
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
}); err != nil {
t.Fatalf("seed %s: %v", id, err)
}
}
got, err := cat.RandomVideosWithReadyThumbnailsExcluding(ctx, []string{"ready-a"}, 10)
if err != nil {
t.Fatalf("random ready excluding: %v", err)
}
if len(got) != 3 {
t.Fatalf("ready random count = %d, want 3", len(got))
}
for _, v := range got {
if v.ID == "ready-a" {
t.Fatal("excluded ready video was returned")
}
if v.ThumbnailURL == "" {
t.Fatalf("pending video %q was returned", v.ID)
}
}
}
func TestRandomVideosForPreferredVideoChoosesLeastPopulatedTag(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
+341 -18
View File
@@ -100,15 +100,27 @@ func (c *Catalog) migrate(ctx context.Context) error {
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_content_hash ON videos(content_hash)`); err != nil {
return err
}
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_content_hash_created ON videos(content_hash, created_at, id)`); err != nil {
return err
}
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_sampled_sha256 ON videos(size_bytes, sampled_sha256)`); err != nil {
return err
}
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_sampled_sha256_created ON videos(size_bytes, sampled_sha256, created_at, id)`); err != nil {
return err
}
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_hidden ON videos(hidden)`); err != nil {
return err
}
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_visible_pub ON videos(COALESCE(hidden, 0), published_at DESC)`); err != nil {
return err
}
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_file_name_size ON videos(file_name, size_bytes)`); err != nil {
return err
}
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_file_name_size_created ON videos(file_name, size_bytes, created_at, id)`); err != nil {
return err
}
if err := c.seedSystemTags(ctx); err != nil {
return err
}
@@ -127,6 +139,12 @@ func (c *Catalog) migrate(ctx context.Context) error {
if err := c.clearVolatileOneDriveThumbnails(ctx); err != nil {
return err
}
if err := c.clearRemoteP123ThumbnailsOnce(ctx); err != nil {
return err
}
if err := c.clearRemoteNonSpider91Thumbnails(ctx); err != nil {
return err
}
if err := c.hideZeroSizeVideosFromKnownDrives(ctx); err != nil {
return err
}
@@ -257,6 +275,85 @@ UPDATE videos
return err
}
func (c *Catalog) clearRemoteP123ThumbnailsOnce(ctx context.Context) error {
// 123 云盘列表返回的缩略图尺寸和稳定性都不适合作为站内封面;清空历史写入的
// 远程 URL,让封面 worker 统一从视频直链抽帧生成本地 /p/thumb/<id>。
const markerKey = "videos.p123.remote_thumbnails_cleared"
marker, err := c.GetSetting(ctx, markerKey, "")
if err != nil {
return fmt.Errorf("read %s marker: %w", markerKey, err)
}
if strings.TrimSpace(marker) == "1" {
return nil
}
var p123Drives int
if err := c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM drives WHERE kind = 'p123'`).Scan(&p123Drives); err != nil {
return fmt.Errorf("count p123 drives: %w", err)
}
if p123Drives == 0 {
return nil
}
res, err := c.db.ExecContext(ctx, `
UPDATE videos
SET thumbnail_url = '',
thumbnail_status = 'pending',
thumbnail_failures = 0,
updated_at = ?
WHERE EXISTS (
SELECT 1
FROM drives
WHERE drives.id = videos.drive_id
AND drives.kind = 'p123'
)
AND (
lower(COALESCE(thumbnail_url, '')) LIKE 'http://%'
OR lower(COALESCE(thumbnail_url, '')) LIKE 'https://%'
)
`, time.Now().UnixMilli())
if err != nil {
return err
}
if affected, err := res.RowsAffected(); err == nil && affected > 0 {
log.Printf("[catalog] cleared %d remote 123pan thumbnail(s) for local regeneration", affected)
}
if err := c.SetSetting(ctx, markerKey, "1"); err != nil {
return fmt.Errorf("write %s marker: %w", markerKey, err)
}
return nil
}
func (c *Catalog) clearRemoteNonSpider91Thumbnails(ctx context.Context) error {
// 非 91Spider 视频不再使用网盘侧返回的远程缩略图。清空历史 http/https
// thumbnail_url 后,封面 worker 会重新从视频中间帧生成本地 /p/thumb/<id>。
// 91Spider 的封面是爬虫下载后保存到本地 /p/thumb/<id>,不受这条规则影响。
res, err := c.db.ExecContext(ctx, `
UPDATE videos
SET thumbnail_url = '',
thumbnail_status = 'pending',
thumbnail_failures = 0,
updated_at = ?
WHERE (
lower(COALESCE(thumbnail_url, '')) LIKE 'http://%'
OR lower(COALESCE(thumbnail_url, '')) LIKE 'https://%'
)
AND NOT EXISTS (
SELECT 1
FROM drives
WHERE drives.id = videos.drive_id
AND drives.kind = 'spider91'
)
`, time.Now().UnixMilli())
if err != nil {
return err
}
if affected, err := res.RowsAffected(); err == nil && affected > 0 {
log.Printf("[catalog] cleared %d remote non-91Spider thumbnail(s) for local regeneration", affected)
}
return nil
}
func (c *Catalog) hideZeroSizeVideosFromKnownDrives(ctx context.Context) error {
_, err := c.db.ExecContext(ctx, `
UPDATE videos
@@ -302,7 +399,15 @@ func (c *Catalog) classifySystemTags(ctx context.Context) error {
}
func (c *Catalog) backfillVideoTags(ctx context.Context) error {
rows, err := c.db.QueryContext(ctx, `SELECT id, COALESCE(tags, '[]') FROM videos`)
rows, err := c.db.QueryContext(ctx, `
SELECT id, COALESCE(tags, '[]')
FROM videos
WHERE COALESCE(tags, '') NOT IN ('', '[]', 'null')
AND NOT EXISTS (
SELECT 1
FROM video_tags vt
WHERE vt.video_id = videos.id
)`)
if err != nil {
return err
}
@@ -319,11 +424,14 @@ func (c *Catalog) backfillVideoTags(ctx context.Context) error {
if len(labels) == 0 {
continue
}
if err := c.addVideoTags(ctx, videoID, labels, "legacy", true); err != nil {
added, err := c.addVideoTags(ctx, videoID, labels, "legacy", true)
if err != nil {
return err
}
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
return err
if added {
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
return err
}
}
}
return nil
@@ -392,6 +500,57 @@ func (c *Catalog) CreateTagAndClassify(ctx context.Context, label string, aliase
return c.classifyTag(ctx, tag)
}
func (c *Catalog) EnsureTagForVideoIDPrefix(ctx context.Context, prefix, label string, aliases []string, source string) (int, error) {
prefix = strings.TrimSpace(prefix)
if prefix == "" {
return 0, errors.New("video id prefix is required")
}
tag, err := c.ensureTag(ctx, label, aliases, source)
if err != nil {
return 0, err
}
rows, err := c.db.QueryContext(ctx, `
SELECT v.id
FROM videos v
WHERE v.id LIKE ? || '%'
AND COALESCE(v.tags_manual, 0) = 0
AND NOT EXISTS (
SELECT 1
FROM video_tags vt
WHERE vt.video_id = v.id
AND vt.tag_id = ?
)
ORDER BY v.id ASC`, prefix, tag.ID)
if err != nil {
return 0, err
}
var videoIDs []string
for rows.Next() {
var videoID string
if err := rows.Scan(&videoID); err != nil {
rows.Close()
return 0, err
}
videoIDs = append(videoIDs, videoID)
}
if err := rows.Err(); err != nil {
rows.Close()
return 0, err
}
if err := rows.Close(); err != nil {
return 0, err
}
for _, videoID := range videoIDs {
if err := c.insertVideoTag(ctx, videoID, tag.ID, "auto"); err != nil {
return 0, err
}
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
return 0, err
}
}
return len(videoIDs), nil
}
func (c *Catalog) DeleteTag(ctx context.Context, tagID int64) (int, error) {
tx, err := c.db.BeginTx(ctx, nil)
if err != nil {
@@ -453,10 +612,66 @@ func (c *Catalog) DeleteTag(ctx context.Context, tagID int64) (int, error) {
func (c *Catalog) ListTags(ctx context.Context) ([]Tag, error) {
rows, err := c.db.QueryContext(ctx, `
SELECT t.id, t.label, t.aliases, t.source, COUNT(v.id) AS cnt
WITH tagged_tags AS (
SELECT vt.tag_id,
tagged.id,
COALESCE(tagged.content_hash, '') AS content_hash,
COALESCE(tagged.sampled_sha256, '') AS sampled_sha256,
tagged.size_bytes,
COALESCE(tagged.file_name, '') AS file_name
FROM video_tags vt
JOIN videos tagged ON tagged.id = vt.video_id
WHERE COALESCE(tagged.hidden, 0) = 0
),
tag_candidates AS (
SELECT tag_id, id AS video_id
FROM tagged_tags
UNION ALL
SELECT tag_id,
(SELECT canonical.id
FROM videos canonical
WHERE tagged_tags.content_hash != ''
AND canonical.content_hash = tagged_tags.content_hash
AND COALESCE(canonical.content_hash, '') != ''
ORDER BY canonical.created_at ASC, canonical.id ASC
LIMIT 1) AS video_id
FROM tagged_tags
WHERE content_hash != ''
UNION ALL
SELECT tag_id,
(SELECT canonical.id
FROM videos canonical
WHERE tagged_tags.sampled_sha256 != ''
AND tagged_tags.size_bytes > 0
AND canonical.sampled_sha256 = tagged_tags.sampled_sha256
AND canonical.size_bytes = tagged_tags.size_bytes
AND COALESCE(canonical.sampled_sha256, '') != ''
AND canonical.size_bytes > 0
ORDER BY canonical.created_at ASC, canonical.id ASC
LIMIT 1) AS video_id
FROM tagged_tags
WHERE sampled_sha256 != '' AND size_bytes > 0
UNION ALL
SELECT tag_id,
(SELECT canonical.id
FROM videos canonical
WHERE tagged_tags.file_name != ''
AND tagged_tags.size_bytes > 0
AND canonical.file_name = tagged_tags.file_name
AND canonical.size_bytes = tagged_tags.size_bytes
AND COALESCE(canonical.file_name, '') != ''
AND canonical.size_bytes > 0
ORDER BY canonical.created_at ASC, canonical.id ASC
LIMIT 1) AS video_id
FROM tagged_tags
WHERE file_name != '' AND size_bytes > 0
)
SELECT t.id, t.label, t.aliases, t.source, COUNT(DISTINCT videos.id) AS cnt
FROM tags t
LEFT JOIN video_tags vt ON vt.tag_id = t.id
LEFT JOIN videos v ON v.id = vt.video_id AND COALESCE(v.hidden, 0) = 0
LEFT JOIN tag_candidates tc ON tc.tag_id = t.id AND tc.video_id IS NOT NULL
LEFT JOIN videos ON videos.id = tc.video_id
AND COALESCE(videos.hidden, 0) = 0
AND `+uniqueVideoWhereSQL+`
GROUP BY t.id, t.label, t.aliases, t.source
ORDER BY cnt DESC, t.label ASC`)
if err != nil {
@@ -474,6 +689,66 @@ ORDER BY cnt DESC, t.label ASC`)
return out, nil
}
func videoMatchesTagLabelSQL(videoAlias string) string {
return fmt.Sprintf(`%s.id IN (
WITH tagged_videos AS (
SELECT tagged.id,
COALESCE(tagged.content_hash, '') AS content_hash,
COALESCE(tagged.sampled_sha256, '') AS sampled_sha256,
tagged.size_bytes,
COALESCE(tagged.file_name, '') AS file_name
FROM video_tags vt
JOIN tags tag_filter ON tag_filter.id = vt.tag_id
JOIN videos tagged ON tagged.id = vt.video_id
WHERE tag_filter.label = ? COLLATE NOCASE
AND COALESCE(tagged.hidden, 0) = 0
),
tag_candidates AS (
SELECT id AS video_id
FROM tagged_videos
UNION ALL
SELECT (SELECT canonical.id
FROM videos canonical
WHERE tagged_videos.content_hash != ''
AND canonical.content_hash = tagged_videos.content_hash
AND COALESCE(canonical.content_hash, '') != ''
ORDER BY canonical.created_at ASC, canonical.id ASC
LIMIT 1) AS video_id
FROM tagged_videos
WHERE content_hash != ''
UNION ALL
SELECT (SELECT canonical.id
FROM videos canonical
WHERE tagged_videos.sampled_sha256 != ''
AND tagged_videos.size_bytes > 0
AND canonical.sampled_sha256 = tagged_videos.sampled_sha256
AND canonical.size_bytes = tagged_videos.size_bytes
AND COALESCE(canonical.sampled_sha256, '') != ''
AND canonical.size_bytes > 0
ORDER BY canonical.created_at ASC, canonical.id ASC
LIMIT 1) AS video_id
FROM tagged_videos
WHERE sampled_sha256 != '' AND size_bytes > 0
UNION ALL
SELECT (SELECT canonical.id
FROM videos canonical
WHERE tagged_videos.file_name != ''
AND tagged_videos.size_bytes > 0
AND canonical.file_name = tagged_videos.file_name
AND canonical.size_bytes = tagged_videos.size_bytes
AND COALESCE(canonical.file_name, '') != ''
AND canonical.size_bytes > 0
ORDER BY canonical.created_at ASC, canonical.id ASC
LIMIT 1) AS video_id
FROM tagged_videos
WHERE file_name != '' AND size_bytes > 0
)
SELECT video_id
FROM tag_candidates
WHERE video_id IS NOT NULL
)`, videoAlias)
}
func (c *Catalog) SetManualVideoTags(ctx context.Context, videoID string, labels []string) error {
if _, err := c.GetVideo(ctx, videoID); err != nil {
return err
@@ -592,6 +867,10 @@ func (c *Catalog) getTagByLabel(ctx context.Context, label string) (Tag, error)
}
func (c *Catalog) classifyTag(ctx context.Context, tag Tag) (int, error) {
existingIDs, err := c.videoIDSetForTagID(ctx, tag.ID)
if err != nil {
return 0, err
}
rows, err := c.db.QueryContext(ctx, `
SELECT id, title, COALESCE(author, ''), COALESCE(category, ''), COALESCE(tags_manual, 0)
FROM videos`)
@@ -623,13 +902,14 @@ FROM videos`)
continue
}
}
added, err := c.addVideoTag(ctx, videoID, tag.Label, "auto", false)
if err != nil {
if existingIDs[videoID] {
continue
}
if err := c.insertVideoTag(ctx, videoID, tag.ID, "auto"); err != nil {
return 0, err
}
if added {
classified++
}
existingIDs[videoID] = true
classified++
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
return 0, err
}
@@ -689,17 +969,22 @@ func (c *Catalog) replaceVideoTags(ctx context.Context, videoID string, labels [
return c.syncVideoTagsJSON(ctx, videoID, manual)
}
func (c *Catalog) addVideoTags(ctx context.Context, videoID string, labels []string, source string, createMissing bool) error {
func (c *Catalog) addVideoTags(ctx context.Context, videoID string, labels []string, source string, createMissing bool) (bool, error) {
labels = uniqueStrings(cleanLabels(labels))
if source != "manual" {
labels = c.filterDeletedTagLabels(ctx, labels)
}
changed := false
for _, label := range labels {
if _, err := c.addVideoTag(ctx, videoID, label, source, createMissing); err != nil {
return err
added, err := c.addVideoTag(ctx, videoID, label, source, createMissing)
if err != nil {
return false, err
}
if added {
changed = true
}
}
return nil
return changed, nil
}
func (c *Catalog) addVideoTag(ctx context.Context, videoID, label, source string, createMissing bool) (bool, error) {
@@ -729,12 +1014,33 @@ func (c *Catalog) addVideoTag(ctx context.Context, videoID, label, source string
return n > 0, nil
}
func (c *Catalog) insertVideoTag(ctx context.Context, videoID string, tagID int64, source string) error {
_, err := c.db.ExecContext(ctx,
`INSERT OR IGNORE INTO video_tags (video_id, tag_id, source, created_at) VALUES (?, ?, ?, ?)`,
videoID, tagID, source, time.Now().UnixMilli())
return err
}
func (c *Catalog) addCollectionTagToVideos(ctx context.Context, category string) error {
return c.addTagToVideosByCategory(ctx, category, category, "auto")
}
func (c *Catalog) addTagToVideosByCategory(ctx context.Context, category, label, source string) error {
rows, err := c.db.QueryContext(ctx, `SELECT id FROM videos WHERE category = ? AND COALESCE(tags_manual, 0) = 0`, category)
tag, err := c.getTagByLabel(ctx, label)
if err != nil {
return err
}
rows, err := c.db.QueryContext(ctx, `
SELECT v.id
FROM videos v
WHERE v.category = ?
AND COALESCE(v.tags_manual, 0) = 0
AND NOT EXISTS (
SELECT 1
FROM video_tags vt
WHERE vt.video_id = v.id
AND vt.tag_id = ?
)`, category, tag.ID)
if err != nil {
return err
}
@@ -753,7 +1059,7 @@ func (c *Catalog) addTagToVideosByCategory(ctx context.Context, category, label,
return err
}
for _, videoID := range videoIDs {
if _, err := c.addVideoTag(ctx, videoID, label, source, false); err != nil {
if err := c.insertVideoTag(ctx, videoID, tag.ID, source); err != nil {
return err
}
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
@@ -837,6 +1143,23 @@ func (c *Catalog) videoIDsForTagID(ctx context.Context, tagID int64) ([]string,
return videoIDs, rows.Err()
}
func (c *Catalog) videoIDSetForTagID(ctx context.Context, tagID int64) (map[string]bool, error) {
rows, err := c.db.QueryContext(ctx, `SELECT video_id FROM video_tags WHERE tag_id = ?`, tagID)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]bool{}
for rows.Next() {
var videoID string
if err := rows.Scan(&videoID); err != nil {
return nil, err
}
out[videoID] = true
}
return out, rows.Err()
}
func (c *Catalog) validateTagsExist(ctx context.Context, labels []string) error {
for _, label := range labels {
if _, err := c.getTagByLabel(ctx, label); err != nil {
+337 -3
View File
@@ -326,6 +326,75 @@ func TestCreateTagAndClassifyRestoresDeletedTag(t *testing.T) {
}
}
func TestEnsureTagForVideoIDPrefixBackfillsSourceTag(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
now := time.Now()
for _, seed := range []struct {
id string
manual bool
}{
{id: "spider91-91-spider-1200001"},
{id: "spider91-91-spider-1200002", manual: true},
{id: "spider91-other-1200003"},
} {
if err := cat.UpsertVideo(ctx, &Video{
ID: seed.id,
DriveID: "91-spider",
FileID: seed.id + ".mp4",
Title: "legacy title without source text",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
}); err != nil {
t.Fatalf("seed %s: %v", seed.id, err)
}
if seed.manual {
if err := cat.SetManualVideoTags(ctx, seed.id, nil); err != nil {
t.Fatalf("mark %s manual: %v", seed.id, err)
}
}
}
added, err := cat.EnsureTagForVideoIDPrefix(ctx, "spider91-91-spider-", "91porn", nil, "system")
if err != nil {
t.Fatalf("ensure prefix tag: %v", err)
}
if added != 1 {
t.Fatalf("added = %d, want 1", added)
}
got, err := cat.GetVideo(ctx, "spider91-91-spider-1200001")
if err != nil {
t.Fatalf("get tagged video: %v", err)
}
if !sameStrings(got.Tags, []string{"91porn"}) {
t.Fatalf("tagged video tags = %#v, want 91porn", got.Tags)
}
manual, err := cat.GetVideo(ctx, "spider91-91-spider-1200002")
if err != nil {
t.Fatalf("get manual video: %v", err)
}
if len(manual.Tags) != 0 {
t.Fatalf("manual video tags = %#v, want unchanged", manual.Tags)
}
other, err := cat.GetVideo(ctx, "spider91-other-1200003")
if err != nil {
t.Fatalf("get other prefix video: %v", err)
}
if len(other.Tags) != 0 {
t.Fatalf("other prefix video tags = %#v, want unchanged", other.Tags)
}
}
func TestDeleteTagRejectsSystemTags(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
@@ -398,6 +467,84 @@ VALUES
}
}
func TestMigrateDoesNotRewriteAlreadySyncedVideoTags(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
now := time.Now()
for _, id := range []string{"video-1", "video-2", "video-3"} {
if err := cat.UpsertVideo(ctx, &Video{
ID: id,
DriveID: "drive",
FileID: id,
Title: "巨乳后入合集",
Category: "Better Call Saul S03",
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
}); err != nil {
t.Fatalf("seed %s: %v", id, err)
}
}
if err := cat.migrate(ctx); err != nil {
t.Fatalf("first migrate: %v", err)
}
before := videoUpdatedAtByID(t, ctx, cat, "video-1", "video-2", "video-3")
time.Sleep(5 * time.Millisecond)
if err := cat.migrate(ctx); err != nil {
t.Fatalf("second migrate: %v", err)
}
after := videoUpdatedAtByID(t, ctx, cat, "video-1", "video-2", "video-3")
for id, want := range before {
if got := after[id]; got != want {
t.Fatalf("%s updated_at changed on no-op migrate: got %d, want %d", id, got, want)
}
}
}
func TestMigrateBackfillsLegacyTagsWithoutRelations(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
now := time.Now().UnixMilli()
if _, err := cat.db.ExecContext(ctx, `
INSERT INTO videos (id, drive_id, file_id, title, tags, tags_manual, published_at, created_at, updated_at)
VALUES ('legacy-video', 'drive', 'file-legacy', 'legacy title', '["legacy-tag"]', 0, ?, ?, ?)`,
now, now, now); err != nil {
t.Fatalf("seed legacy video: %v", err)
}
if err := cat.migrate(ctx); err != nil {
t.Fatalf("migrate: %v", err)
}
tag := mustTagByLabel(t, ctx, cat, "legacy-tag")
var count int
if err := cat.db.QueryRowContext(ctx,
`SELECT COUNT(*) FROM video_tags WHERE video_id = 'legacy-video' AND tag_id = ?`, tag.ID).Scan(&count); err != nil {
t.Fatalf("count video tag: %v", err)
}
if count != 1 {
t.Fatalf("legacy video tag relation count = %d, want 1", count)
}
}
func TestOpenMigratesLegacyVideosWithoutFileName(t *testing.T) {
path := t.TempDir() + "/catalog.db"
db, err := sql.Open("sqlite", path)
@@ -657,7 +804,7 @@ func TestMigrateCollapsesAVCodeTagsIntoAV(t *testing.T) {
}
}
func TestMigrateClearsVolatileOneDriveThumbnailURLs(t *testing.T) {
func TestMigrateClearsRemoteNonSpiderThumbnailURLs(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
if err != nil {
@@ -680,6 +827,36 @@ func TestMigrateClearsVolatileOneDriveThumbnailURLs(t *testing.T) {
}); err != nil {
t.Fatalf("seed onedrive: %v", err)
}
if err := cat.UpsertDrive(ctx, &Drive{
ID: "p123-main",
Kind: "p123",
Name: "123Pan",
RootID: "root",
CreatedAt: now,
UpdatedAt: now,
}); err != nil {
t.Fatalf("seed p123: %v", err)
}
if err := cat.UpsertDrive(ctx, &Drive{
ID: "pikpak-main",
Kind: "pikpak",
Name: "PikPak",
RootID: "root",
CreatedAt: now,
UpdatedAt: now,
}); err != nil {
t.Fatalf("seed pikpak: %v", err)
}
if err := cat.UpsertDrive(ctx, &Drive{
ID: "spider91-main",
Kind: "spider91",
Name: "91Spider",
RootID: "root",
CreatedAt: now,
UpdatedAt: now,
}); err != nil {
t.Fatalf("seed spider91: %v", err)
}
videos := []*Video{
{
@@ -703,6 +880,27 @@ func TestMigrateClearsVolatileOneDriveThumbnailURLs(t *testing.T) {
Title: "PikPak",
ThumbnailURL: "https://sg-thumbnail-drive.mypikpak.net/v0/screenshot-thumbnails/demo",
},
{
ID: "p123-remote-thumb-video",
DriveID: "p123-main",
FileID: "file-4",
Title: "123Pan remote thumb",
ThumbnailURL: "https://download.123pan.com/thumb/file_70_70?w=70&h=70",
},
{
ID: "p123-local-thumb-video",
DriveID: "p123-main",
FileID: "file-5",
Title: "123Pan local thumb",
ThumbnailURL: "/p/thumb/p123-local-thumb-video",
},
{
ID: "spider91-local-thumb-video",
DriveID: "spider91-main",
FileID: "file-6",
Title: "91Spider local thumb",
ThumbnailURL: "/p/thumb/spider91-local-thumb-video",
},
}
for _, v := range videos {
v.PublishedAt = now
@@ -737,8 +935,39 @@ func TestMigrateClearsVolatileOneDriveThumbnailURLs(t *testing.T) {
if err != nil {
t.Fatalf("get pikpak video: %v", err)
}
if pikpak.ThumbnailURL == "" {
t.Fatal("pikpak thumbnail was cleared")
if pikpak.ThumbnailURL != "" {
t.Fatalf("pikpak thumbnail = %q, want cleared", pikpak.ThumbnailURL)
}
p123Remote, err := cat.GetVideo(ctx, "p123-remote-thumb-video")
if err != nil {
t.Fatalf("get p123 remote thumb video: %v", err)
}
if p123Remote.ThumbnailURL != "" {
t.Fatalf("p123 remote thumbnail = %q, want cleared", p123Remote.ThumbnailURL)
}
var p123Status string
if err := cat.db.QueryRowContext(ctx, `SELECT thumbnail_status FROM videos WHERE id = ?`, "p123-remote-thumb-video").Scan(&p123Status); err != nil {
t.Fatalf("read p123 thumbnail status: %v", err)
}
if p123Status != "pending" {
t.Fatalf("p123 remote thumbnail_status = %q, want pending", p123Status)
}
p123Local, err := cat.GetVideo(ctx, "p123-local-thumb-video")
if err != nil {
t.Fatalf("get p123 local thumb video: %v", err)
}
if p123Local.ThumbnailURL != "/p/thumb/p123-local-thumb-video" {
t.Fatalf("p123 local thumbnail = %q, want preserved", p123Local.ThumbnailURL)
}
spider91Local, err := cat.GetVideo(ctx, "spider91-local-thumb-video")
if err != nil {
t.Fatalf("get spider91 local thumb video: %v", err)
}
if spider91Local.ThumbnailURL != "/p/thumb/spider91-local-thumb-video" {
t.Fatalf("spider91 local thumbnail = %q, want preserved", spider91Local.ThumbnailURL)
}
}
@@ -859,6 +1088,98 @@ func TestListVideosHidesDuplicateContentHashes(t *testing.T) {
}
}
func TestTagFilterMatchesCanonicalDuplicateVideo(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
now := time.Now()
for _, v := range []*Video{
{
ID: "pikpak-canonical",
DriveID: "pikpak",
FileID: "canonical.mp4",
Title: "Canonical",
Size: 1024,
PublishedAt: now,
CreatedAt: now,
UpdatedAt: now,
},
{
ID: "spider91-dup-1",
DriveID: "91-spider",
FileID: "dup-1.mp4",
Title: "Spider duplicate 1",
Tags: []string{"91porn"},
Size: 1024,
PublishedAt: now.Add(time.Second),
CreatedAt: now.Add(time.Second),
UpdatedAt: now.Add(time.Second),
},
{
ID: "spider91-dup-2",
DriveID: "91-spider",
FileID: "dup-2.mp4",
Title: "Spider duplicate 2",
Tags: []string{"91porn"},
Size: 1024,
PublishedAt: now.Add(2 * time.Second),
CreatedAt: now.Add(2 * time.Second),
UpdatedAt: now.Add(2 * time.Second),
},
{
ID: "spider91-visible",
DriveID: "91-spider",
FileID: "visible.mp4",
Title: "Spider visible",
Tags: []string{"91porn"},
Size: 2048,
PublishedAt: now.Add(3 * time.Second),
CreatedAt: now.Add(3 * time.Second),
UpdatedAt: now.Add(3 * time.Second),
},
} {
if err := cat.UpsertVideo(ctx, v); err != nil {
t.Fatalf("seed %s: %v", v.ID, err)
}
}
for _, id := range []string{"pikpak-canonical", "spider91-dup-1", "spider91-dup-2"} {
if err := cat.UpdateVideoFingerprint(ctx, id, "same-sampled-sha256", "ready", ""); err != nil {
t.Fatalf("fingerprint %s: %v", id, err)
}
}
if err := cat.UpdateVideoFingerprint(ctx, "spider91-visible", "unique-sampled-sha256", "ready", ""); err != nil {
t.Fatalf("fingerprint visible: %v", err)
}
items, total, err := cat.ListVideos(ctx, ListParams{Tag: "91porn", Page: 1, PageSize: 10})
if err != nil {
t.Fatalf("list videos by tag: %v", err)
}
if total != 2 || len(items) != 2 {
t.Fatalf("tagged videos total=%d len=%d, want 2", total, len(items))
}
gotIDs := map[string]bool{}
for _, item := range items {
gotIDs[item.ID] = true
}
for _, want := range []string{"pikpak-canonical", "spider91-visible"} {
if !gotIDs[want] {
t.Fatalf("tagged video ids = %#v, want %s", gotIDs, want)
}
}
if got := mustTagByLabel(t, ctx, cat, "91porn").Count; got != 2 {
t.Fatalf("91porn count = %d, want 2 visible canonical videos", got)
}
}
func TestListVideosCanFilterReadyThumbnails(t *testing.T) {
ctx := context.Background()
cat, err := Open(t.TempDir() + "/catalog.db")
@@ -944,6 +1265,19 @@ func mustTagByLabel(t *testing.T, ctx context.Context, cat *Catalog, label strin
return Tag{}
}
func videoUpdatedAtByID(t *testing.T, ctx context.Context, cat *Catalog, ids ...string) map[string]int64 {
t.Helper()
out := make(map[string]int64, len(ids))
for _, id := range ids {
var updatedAt int64
if err := cat.db.QueryRowContext(ctx, `SELECT updated_at FROM videos WHERE id = ?`, id).Scan(&updatedAt); err != nil {
t.Fatalf("read updated_at for %s: %v", id, err)
}
out[id] = updatedAt
}
return out
}
// 删除 collection 标签的最后一个引用视频后,标签应当自动从 tags 表里消失。
// user/system 标签不受影响:用户/系统标签的语义由人维护,孤儿状态保留。
func TestDeleteVideoPrunesOrphanCollectionTag(t *testing.T) {
+1 -1
View File
@@ -202,7 +202,7 @@ type Nightly struct {
// 这里保留 yaml 中的静态定义,用于启动时预置盘。生产建议只在 DB 里维护。
type Drive struct {
ID string `yaml:"id"`
Kind string `yaml:"kind"` // quark / p115 / pikpak / wopan / onedrive / googledrive / localstorage
Kind string `yaml:"kind"` // quark / p115 / p123 / pikpak / wopan / onedrive / googledrive / localstorage
Name string `yaml:"name"`
RootID string `yaml:"root_id"`
Params map[string]string `yaml:"params,omitempty"`
+1 -1
View File
@@ -10,7 +10,7 @@ import (
// Drive 是多家网盘统一抽象。上层不区分盘,只区分 Kind。
type Drive interface {
// Kind 返回驱动代号:"quark" / "p115" / "pikpak" / "wopan" / "onedrive" / "googledrive" / "localstorage"
// Kind 返回驱动代号:"quark" / "p115" / "p123" / "pikpak" / "wopan" / "onedrive" / "googledrive" / "localstorage"
Kind() string
// ID 返回该盘在 catalog 中的唯一标识
+28 -2
View File
@@ -47,7 +47,7 @@ func (d *Driver) Init(context.Context) error {
}
info, err := os.Stat(root)
if err != nil {
return fmt.Errorf("localstorage: stat root: %w", err)
return fmt.Errorf("localstorage: stat root %q: %w%s", root, err, localStoragePathHint(d.rootPath))
}
if !info.IsDir() {
return fmt.Errorf("localstorage: root is not a directory: %s", root)
@@ -174,12 +174,38 @@ func (d *Driver) pathForID(id string) (string, string, error) {
if err != nil {
return "", "", err
}
if p != root && !strings.HasPrefix(p, root+string(os.PathSeparator)) {
if !pathWithinRoot(root, p) {
return "", "", errors.New("localstorage: path escapes root")
}
return p, rel, nil
}
func pathWithinRoot(root, path string) bool {
rel, err := filepath.Rel(root, path)
if err != nil {
return false
}
return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator)))
}
func localStoragePathHint(configured string) string {
cwd, _ := os.Getwd()
parts := []string{}
if strings.TrimSpace(configured) != "" {
parts = append(parts, fmt.Sprintf("configured=%q", strings.TrimSpace(configured)))
}
if cwd != "" {
parts = append(parts, fmt.Sprintf("cwd=%q", cwd))
}
if _, err := os.Stat("/.dockerenv"); err == nil {
parts = append(parts, "docker=host paths must be bind-mounted into the container")
}
if len(parts) == 0 {
return ""
}
return " (" + strings.Join(parts, ", ") + ")"
}
func decodeRel(id string) (string, error) {
id = strings.TrimSpace(id)
if id == "" || id == "/" {
@@ -70,13 +70,34 @@ func TestStreamURLRejectsEscapingID(t *testing.T) {
}
func TestInitRequiresExistingDirectory(t *testing.T) {
drv := New(Config{ID: "local", RootPath: filepath.Join(t.TempDir(), "missing")})
missing := filepath.Join(t.TempDir(), "missing")
drv := New(Config{ID: "local", RootPath: missing})
err := drv.Init(context.Background())
if err == nil || !strings.Contains(err.Error(), "stat root") {
t.Fatalf("error = %v, want stat root failure", err)
}
if !strings.Contains(err.Error(), missing) || !strings.Contains(err.Error(), "configured=") {
t.Fatalf("error = %v, want diagnostic path details", err)
}
}
func TestPathForIDAllowsRootPathSlash(t *testing.T) {
drv := New(Config{ID: "local", RootPath: string(os.PathSeparator)})
childID := encodeRel("tmp")
path, rel, err := drv.pathForID(childID)
if err != nil {
t.Fatalf("pathForID: %v", err)
}
if rel != "tmp" {
t.Fatalf("rel = %q, want tmp", rel)
}
if path != filepath.Join(string(os.PathSeparator), "tmp") {
t.Fatalf("path = %q, want /tmp", path)
}
}
func TestScannerPersistsLocalStorageVideo(t *testing.T) {
+773
View File
@@ -0,0 +1,773 @@
package p123
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"hash/crc32"
"io"
"log"
"math"
"math/rand"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"sync"
"time"
"github.com/go-resty/resty/v2"
"github.com/video-site/backend/internal/drives"
)
const (
Kind = "p123"
defaultMainAPIBase = "https://www.123pan.com/b/api"
defaultLoginAPIBase = "https://login.123pan.com/api"
defaultReferer = "https://www.123pan.com/"
defaultPlatform = "web"
defaultAppVersion = "3"
defaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) video-site-123pan"
endpointSignIn = "/user/sign_in"
endpointUserInfo = "/user/info"
endpointFileList = "/file/list/new"
endpointDownloadInfo = "/file/download_info"
endpointMkdir = "/file/upload_request"
listInterval = 700 * time.Millisecond
listCooldown = 10 * time.Minute
)
type Driver struct {
id string
rootID string
username string
password string
accessToken string
platform string
mainAPIBase string
loginAPIBase string
referer string
userAgent string
client *resty.Client
httpClient *http.Client
onTokenUpdate func(access string)
tokenMu sync.RWMutex
listMu sync.Mutex
lastListAt time.Time
fileMu sync.RWMutex
files map[string]cachedFile
}
type Config struct {
ID string
RootID string
Username string
Password string
AccessToken string
Platform string
MainAPIBaseURL string
LoginAPIBaseURL string
OnTokenUpdate func(access string)
}
func New(c Config) *Driver {
rootID := strings.TrimSpace(c.RootID)
if rootID == "" {
rootID = "0"
}
platform := strings.TrimSpace(c.Platform)
if platform == "" {
platform = defaultPlatform
}
mainAPIBase := strings.TrimRight(strings.TrimSpace(c.MainAPIBaseURL), "/")
if mainAPIBase == "" {
mainAPIBase = defaultMainAPIBase
}
loginAPIBase := strings.TrimRight(strings.TrimSpace(c.LoginAPIBaseURL), "/")
if loginAPIBase == "" {
loginAPIBase = defaultLoginAPIBase
}
return &Driver{
id: c.ID,
rootID: rootID,
username: strings.TrimSpace(c.Username),
password: strings.TrimSpace(c.Password),
accessToken: normalizeAccessToken(c.AccessToken),
platform: platform,
mainAPIBase: mainAPIBase,
loginAPIBase: loginAPIBase,
referer: defaultReferer,
userAgent: defaultUserAgent,
onTokenUpdate: c.OnTokenUpdate,
client: resty.New().
SetTimeout(30*time.Second).
SetHeader("Accept", "application/json, text/plain, */*"),
httpClient: &http.Client{
Timeout: 30 * time.Second,
CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
},
},
files: make(map[string]cachedFile),
}
}
func (d *Driver) Kind() string { return Kind }
func (d *Driver) ID() string { return d.id }
func (d *Driver) RootID() string { return d.rootID }
func (d *Driver) Init(ctx context.Context) error {
if d.currentToken() == "" {
if err := d.login(ctx); err != nil {
return err
}
}
_, err := d.request(ctx, endpointUserInfo, http.MethodGet, nil, nil)
return err
}
func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error) {
if strings.TrimSpace(dirID) == "" {
dirID = d.rootID
}
d.listMu.Lock()
defer d.listMu.Unlock()
page := 1
total := 0
out := make([]drives.Entry, 0)
for {
var resp fileListResp
query := map[string]string{
"driveId": "0",
"limit": "100",
"next": "0",
"orderBy": "file_id",
"orderDirection": "desc",
"parentFileId": dirID,
"trashed": "false",
"SearchData": "",
"Page": strconv.Itoa(page),
"OnlyLookAbnormalFile": "0",
"event": "homeListFile",
"operateType": "4",
"inDirectSpace": "false",
}
for attempt := 0; ; attempt++ {
if err := d.waitForListSlotLocked(ctx); err != nil {
return nil, err
}
if _, err := d.request(ctx, endpointFileList, http.MethodGet, func(req *resty.Request) {
req.SetQueryParams(query)
}, &resp); err != nil {
wait, ok := drives.RateLimitRetryAfter(err)
if !ok {
return nil, fmt.Errorf("123pan list: %w", err)
}
if wait <= 0 {
wait = listCooldown
}
log.Printf("[p123] list cooling down drive=%s dir=%s page=%d cooldown=%s attempt=%d err=%v",
d.id, dirID, page, wait, attempt+1, err)
if err := sleepContext(ctx, wait); err != nil {
return nil, err
}
continue
}
break
}
for _, f := range resp.Data.InfoList {
d.cacheFile(f, dirID)
out = append(out, fileToEntry(f, dirID))
}
total = resp.Data.Total
page++
if len(resp.Data.InfoList) == 0 || resp.Data.Next == "-1" || (total > 0 && len(out) >= total) {
return out, nil
}
}
}
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
f, parentID, err := d.findFile(ctx, fileID)
if err != nil {
return nil, err
}
e := fileToEntry(f, parentID)
return &e, nil
}
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
f, _, err := d.findFile(ctx, fileID)
if err != nil {
return nil, fmt.Errorf("123pan stream metadata: %w", err)
}
body := map[string]any{
"driveId": 0,
"etag": f.Etag,
"fileId": f.FileID,
"fileName": f.FileName,
"s3keyFlag": f.S3KeyFlag,
"size": f.Size,
"type": f.Type,
}
var resp downloadInfoResp
if _, err := d.request(ctx, endpointDownloadInfo, http.MethodPost, func(req *resty.Request) {
req.SetBody(body)
}, &resp); err != nil {
return nil, fmt.Errorf("123pan download info: %w", err)
}
downloadURL := strings.TrimSpace(resp.URL())
if downloadURL == "" {
return nil, errors.New("123pan download info: empty url")
}
return d.resolveDownloadURL(ctx, downloadURL)
}
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
return "", drives.ErrNotSupported
}
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
parts := splitPath(pathFromRoot)
currentID := d.rootID
for _, name := range parts {
childID, err := d.findChildDir(ctx, currentID, name)
if err != nil {
return "", err
}
if childID == "" {
id, err := d.makeDir(ctx, currentID, name)
if err != nil {
return "", err
}
childID = id
}
currentID = childID
}
return currentID, nil
}
func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, error) {
body := map[string]any{
"driveId": 0,
"etag": "",
"fileName": name,
"parentFileId": parentID,
"size": 0,
"type": 1,
}
var resp mkdirResp
if _, err := d.request(ctx, endpointMkdir, http.MethodPost, func(req *resty.Request) {
req.SetBody(body)
}, &resp); err != nil {
return "", fmt.Errorf("123pan mkdir %s: %w", name, err)
}
if resp.Data.FileID != 0 {
return strconv.FormatInt(resp.Data.FileID, 10), nil
}
// 123 云盘创建目录的返回字段不稳定;创建成功但没回 fileId 时回读父目录确认。
childID, err := d.findChildDir(ctx, parentID, name)
if err != nil {
return "", err
}
if childID == "" {
return "", errors.New("123pan mkdir: empty file id")
}
return childID, nil
}
func (d *Driver) findChildDir(ctx context.Context, parentID, name string) (string, error) {
entries, err := d.List(ctx, parentID)
if err != nil {
return "", err
}
for _, e := range entries {
if e.IsDir && e.Name == name {
return e.ID, nil
}
}
return "", nil
}
func (d *Driver) resolveDownloadURL(ctx context.Context, downloadURL string) (*drives.StreamLink, error) {
original, err := url.Parse(downloadURL)
if err != nil {
return nil, err
}
target := original.String()
if params := original.Query().Get("params"); params != "" {
if decoded, err := base64.StdEncoding.DecodeString(params); err == nil && len(decoded) > 0 {
if u, err := url.Parse(string(decoded)); err == nil {
target = u.String()
}
}
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
if err != nil {
return nil, err
}
req.Header.Set("Referer", defaultReferer)
req.Header.Set("User-Agent", d.userAgent)
res, err := d.httpClient.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
finalURL := ""
if res.StatusCode >= 300 && res.StatusCode < 400 {
finalURL = strings.TrimSpace(res.Header.Get("Location"))
} else if res.StatusCode < 300 {
var redirect redirectResp
if err := json.NewDecoder(res.Body).Decode(&redirect); err == nil {
finalURL = redirect.URL()
}
if finalURL == "" {
finalURL = target
}
} else {
body, _ := io.ReadAll(io.LimitReader(res.Body, 4096))
if isP123RateLimitHTTPResponse(res.StatusCode, res.Header.Get("Retry-After"), string(body)) {
return nil, p123RateLimitErrorFromHTTP("download redirect", res.StatusCode, res.Header.Get("Retry-After"), string(body))
}
return nil, fmt.Errorf("123pan download redirect: status %d", res.StatusCode)
}
if finalURL == "" {
return nil, errors.New("123pan download redirect: empty url")
}
headers := http.Header{}
if original.Scheme != "" && original.Host != "" {
headers.Set("Referer", fmt.Sprintf("%s://%s/", original.Scheme, original.Host))
} else {
headers.Set("Referer", defaultReferer)
}
headers.Set("User-Agent", d.userAgent)
return &drives.StreamLink{
URL: finalURL,
Headers: headers,
Expires: time.Now().Add(10 * time.Minute),
}, nil
}
func (d *Driver) request(ctx context.Context, endpoint, method string, configure func(*resty.Request), out any) ([]byte, error) {
if d.currentToken() == "" {
if err := d.login(ctx); err != nil {
return nil, err
}
}
rawURL := d.mainAPIBase + endpoint
for attempt := 0; attempt < 2; attempt++ {
req := d.client.R().
SetContext(ctx).
SetHeaders(map[string]string{
"origin": "https://www.123pan.com",
"referer": d.referer,
"authorization": "Bearer " + d.currentToken(),
"user-agent": d.userAgent,
"platform": d.platform,
"app-version": defaultAppVersion,
})
if configure != nil {
configure(req)
}
if out != nil {
req.SetResult(out)
}
res, err := req.Execute(method, signAPIURL(rawURL))
if err != nil {
return nil, err
}
body := res.Body()
var env apiEnvelope
decodeErr := json.Unmarshal(body, &env)
if isP123RateLimitResponse(res, env.Code, env.Message) {
return nil, p123RateLimitError(res, env.Code, env.Message)
}
if decodeErr != nil {
if res.IsError() {
return nil, fmt.Errorf("123pan request: status=%d body=%s", res.StatusCode(), strings.TrimSpace(res.String()))
}
return nil, fmt.Errorf("parse 123pan response: %w", decodeErr)
}
if env.Code == 0 {
return body, nil
}
if env.Code == 401 && attempt == 0 {
if err := d.login(ctx); err != nil {
return nil, err
}
continue
}
if env.Message == "" {
env.Message = fmt.Sprintf("code=%d", env.Code)
}
return nil, errors.New(env.Message)
}
return nil, errors.New("123pan request: unauthorized")
}
func isP123RateLimitResponse(res *resty.Response, code int, message string) bool {
if code == http.StatusTooManyRequests || isP123RateLimitMessage(message) {
return true
}
if res == nil {
return false
}
return isP123RateLimitHTTPResponse(res.StatusCode(), res.Header().Get("Retry-After"), res.String())
}
func isP123RateLimitHTTPResponse(status int, retryAfter, body string) bool {
if status == http.StatusTooManyRequests {
return true
}
if retryAfter != "" {
switch status {
case http.StatusTooManyRequests, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
return true
}
}
if isP123RateLimitMessage(body) {
return true
}
return false
}
func isP123RateLimitMessage(message string) bool {
text := strings.ToLower(strings.TrimSpace(message))
if text == "" {
return false
}
return strings.Contains(text, "请求太频繁") ||
strings.Contains(text, "请求过于频繁") ||
strings.Contains(text, "请求频繁") ||
strings.Contains(text, "操作频繁") ||
strings.Contains(text, "频率限制") ||
strings.Contains(text, "请求次数过多") ||
strings.Contains(text, "too many request") ||
strings.Contains(text, "too many requests") ||
strings.Contains(text, "rate limit") ||
strings.Contains(text, "rate-limit") ||
strings.Contains(text, "ratelimit") ||
strings.Contains(text, "throttl") ||
strings.Contains(text, "temporarily blocked") ||
strings.Contains(text, "request has been blocked") ||
strings.Contains(text, "blocked") ||
strings.Contains(text, "访问被阻断")
}
func p123RateLimitError(res *resty.Response, code int, message string) error {
if strings.TrimSpace(message) == "" {
message = "123pan rate limited"
}
if code != 0 {
message = fmt.Sprintf("code=%d %s", code, message)
}
if res != nil && strings.TrimSpace(res.String()) != "" {
message = fmt.Sprintf("%s: status=%d body=%s", message, res.StatusCode(), strings.TrimSpace(res.String()))
}
return &drives.RateLimitError{
Provider: Kind,
RetryAfter: parseRetryAfterHeader(responseRetryAfter(res)),
Err: errors.New(message),
}
}
func p123RateLimitErrorFromHTTP(step string, status int, retryAfter, body string) error {
message := fmt.Sprintf("123pan %s rate limited: status=%d", step, status)
if strings.TrimSpace(body) != "" {
message += " body=" + strings.TrimSpace(body)
}
return &drives.RateLimitError{
Provider: Kind,
RetryAfter: parseRetryAfterHeader(retryAfter),
Err: errors.New(message),
}
}
func responseRetryAfter(res *resty.Response) string {
if res == nil {
return ""
}
return res.Header().Get("Retry-After")
}
func parseRetryAfterHeader(raw string) time.Duration {
raw = strings.TrimSpace(raw)
if raw == "" {
return 0
}
if seconds, err := strconv.Atoi(raw); err == nil && seconds > 0 {
return time.Duration(seconds) * time.Second
}
if when, err := http.ParseTime(raw); err == nil {
if wait := time.Until(when); wait > 0 {
return wait
}
}
return 0
}
func (d *Driver) login(ctx context.Context) error {
if d.username == "" || d.password == "" {
return errors.New("123pan login: username and password are required")
}
body := map[string]any{
"passport": d.username,
"password": d.password,
"remember": true,
}
if strings.Contains(d.username, "@") {
body = map[string]any{
"mail": d.username,
"password": d.password,
"type": 2,
}
}
var resp loginResp
res, err := d.client.R().
SetContext(ctx).
SetHeaders(map[string]string{
"origin": "https://www.123pan.com",
"referer": d.referer,
"user-agent": "Dart/2.19(dart:io)-video-site",
"platform": d.platform,
"app-version": defaultAppVersion,
}).
SetBody(body).
SetResult(&resp).
Post(d.loginAPIBase + endpointSignIn)
if err != nil {
return err
}
if resp.Code != 200 {
if resp.Message == "" {
resp.Message = fmt.Sprintf("status=%d code=%d", res.StatusCode(), resp.Code)
}
return loginError(resp.Message)
}
if strings.TrimSpace(resp.Data.Token) == "" {
return errors.New("123pan login: empty token")
}
d.setToken(resp.Data.Token)
return nil
}
func (d *Driver) currentToken() string {
d.tokenMu.RLock()
defer d.tokenMu.RUnlock()
return d.accessToken
}
func (d *Driver) setToken(token string) {
token = normalizeAccessToken(token)
d.tokenMu.Lock()
d.accessToken = token
d.tokenMu.Unlock()
if d.onTokenUpdate != nil {
d.onTokenUpdate(token)
}
}
func (d *Driver) waitForListSlotLocked(ctx context.Context) error {
if d.lastListAt.IsZero() {
d.lastListAt = time.Now()
return ctx.Err()
}
next := d.lastListAt.Add(listInterval)
now := time.Now()
if now.Before(next) {
timer := time.NewTimer(next.Sub(now))
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
}
}
d.lastListAt = time.Now()
return ctx.Err()
}
func sleepContext(ctx context.Context, d time.Duration) error {
if d <= 0 {
return ctx.Err()
}
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
func (d *Driver) cacheFile(f panFile, parentID string) {
id := strconv.FormatInt(f.FileID, 10)
if id == "0" {
return
}
d.fileMu.Lock()
d.files[id] = cachedFile{file: f, parentID: parentID}
d.fileMu.Unlock()
}
func (d *Driver) cachedFile(fileID string) (panFile, string, bool) {
d.fileMu.RLock()
defer d.fileMu.RUnlock()
c, ok := d.files[fileID]
return c.file, c.parentID, ok
}
func (d *Driver) findFile(ctx context.Context, fileID string) (panFile, string, error) {
fileID = strings.TrimSpace(fileID)
if fileID == "" {
return panFile{}, "", errors.New("empty file id")
}
if f, parentID, ok := d.cachedFile(fileID); ok {
return f, parentID, nil
}
f, parentID, ok, err := d.findFileInDir(ctx, fileID, d.rootID, make(map[string]struct{}))
if err != nil {
return panFile{}, "", err
}
if !ok {
return panFile{}, "", fmt.Errorf("file %s not found", fileID)
}
return f, parentID, nil
}
func (d *Driver) findFileInDir(ctx context.Context, targetID, dirID string, visited map[string]struct{}) (panFile, string, bool, error) {
if _, ok := visited[dirID]; ok {
return panFile{}, "", false, nil
}
visited[dirID] = struct{}{}
entries, err := d.List(ctx, dirID)
if err != nil {
return panFile{}, "", false, err
}
for _, e := range entries {
if e.ID == targetID {
f, parentID, ok := d.cachedFile(e.ID)
if !ok {
return panFile{}, "", false, nil
}
return f, parentID, true, nil
}
}
for _, e := range entries {
if !e.IsDir {
continue
}
if f, parentID, ok, err := d.findFileInDir(ctx, targetID, e.ID, visited); err != nil || ok {
return f, parentID, ok, err
}
}
return panFile{}, "", false, nil
}
func normalizeAccessToken(token string) string {
token = strings.TrimSpace(token)
if len(token) >= len("Bearer ") && strings.EqualFold(token[:len("Bearer ")], "Bearer ") {
token = strings.TrimSpace(token[len("Bearer "):])
}
return token
}
func loginError(message string) error {
message = strings.TrimSpace(message)
if strings.Contains(message, "境外登录风险") ||
(strings.Contains(message, "短信验证码") && strings.Contains(message, "微信")) {
return errors.New("123pan login: 账号密码登录被 123 云盘风控拦截,请在浏览器完成短信/微信验证后复制 access_token,并在后台编辑该 123 云盘时只填写 access_token")
}
if message == "" {
message = "login failed"
}
return errors.New(message)
}
func signPath(apiPath, platform, version string) (string, string) {
table := []byte{'a', 'd', 'e', 'f', 'g', 'h', 'l', 'm', 'y', 'i', 'j', 'n', 'o', 'p', 'k', 'q', 'r', 's', 't', 'u', 'b', 'c', 'v', 'w', 's', 'z'}
random := fmt.Sprintf("%.f", math.Round(1e7*rand.Float64()))
now := time.Now().In(time.FixedZone("CST", 8*3600))
timestamp := fmt.Sprint(now.Unix())
nowStr := []byte(now.Format("200601021504"))
for i := 0; i < len(nowStr); i++ {
nowStr[i] = table[nowStr[i]-48]
}
timeSign := fmt.Sprint(crc32.ChecksumIEEE(nowStr))
data := strings.Join([]string{timestamp, random, apiPath, platform, version, timeSign}, "|")
dataSign := fmt.Sprint(crc32.ChecksumIEEE([]byte(data)))
return timeSign, strings.Join([]string{timestamp, random, dataSign}, "-")
}
func signAPIURL(rawURL string) string {
u, err := url.Parse(rawURL)
if err != nil {
return rawURL
}
query := u.Query()
k, v := signPath(u.Path, defaultPlatform, defaultAppVersion)
query.Add(k, v)
u.RawQuery = query.Encode()
return u.String()
}
func splitPath(p string) []string {
p = strings.Trim(p, "/")
if p == "" {
return nil
}
return strings.Split(p, "/")
}
func fileToEntry(f panFile, parentID string) drives.Entry {
return drives.Entry{
ID: strconv.FormatInt(f.FileID, 10),
Name: f.FileName,
Size: f.Size,
Hash: strings.ToLower(f.Etag),
IsDir: f.Type == 1,
ParentID: parentID,
MimeType: guessMime(f.FileName),
ModTime: f.UpdateAt.Time(),
}
}
func guessMime(name string) string {
ext := strings.ToLower(path.Ext(name))
switch ext {
case ".mp4":
return "video/mp4"
case ".mkv":
return "video/x-matroska"
case ".mov":
return "video/quicktime"
case ".webm":
return "video/webm"
case ".jpg", ".jpeg":
return "image/jpeg"
case ".png":
return "image/png"
}
return "application/octet-stream"
}
var _ drives.Drive = (*Driver)(nil)
+256
View File
@@ -0,0 +1,256 @@
package p123
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/video-site/backend/internal/drives"
)
func TestStreamURLResolvesDownloadInfoRedirect(t *testing.T) {
ctx := context.Background()
var downloadReferer string
var download *httptest.Server
download = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/resolve":
downloadReferer = r.Header.Get("Referer")
http.Redirect(w, r, download.URL+"/cdn/video.mp4", http.StatusFound)
case "/cdn/video.mp4":
t.Fatalf("driver followed redirect unexpectedly")
default:
http.NotFound(w, r)
}
}))
defer download.Close()
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/user/sign_in":
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 200,
"data": map[string]string{"token": "token-1"},
})
case "/b/api/user/info":
if got := r.Header.Get("Authorization"); got != "Bearer token-1" {
t.Fatalf("Authorization = %q, want bearer token", got)
}
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{}})
case "/b/api/file/list/new":
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 0,
"data": map[string]any{
"Next": "-1",
"Total": 1,
"InfoList": []map[string]any{
{
"FileName": "video.mp4",
"Size": 1234,
"UpdateAt": "2026-01-02 03:04:05",
"FileId": 100,
"Type": 0,
"Etag": "ABCDEF",
"S3KeyFlag": "flag-1",
},
},
},
})
case "/b/api/file/download_info":
var body map[string]any
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatalf("decode download_info body: %v", err)
}
if got := body["fileName"]; got != "video.mp4" {
t.Fatalf("fileName = %#v, want cached file metadata", got)
}
if got := body["etag"]; got != "ABCDEF" {
t.Fatalf("etag = %#v, want cached etag", got)
}
entryURL := download.URL + "/entry?params=" + base64.StdEncoding.EncodeToString([]byte(download.URL+"/resolve"))
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 0,
"data": map[string]string{"DownloadUrl": entryURL},
})
default:
http.NotFound(w, r)
}
}))
defer api.Close()
var savedToken string
d := New(Config{
ID: "123-main",
Username: "user@example.com",
Password: "secret",
MainAPIBaseURL: api.URL + "/b/api",
LoginAPIBaseURL: api.URL + "/api",
OnTokenUpdate: func(access string) {
savedToken = access
},
})
if err := d.Init(ctx); err != nil {
t.Fatalf("Init() error = %v", err)
}
if savedToken != "token-1" {
t.Fatalf("saved token = %q, want token-1", savedToken)
}
if _, err := d.List(ctx, d.RootID()); err != nil {
t.Fatalf("List() error = %v", err)
}
link, err := d.StreamURL(ctx, "100")
if err != nil {
t.Fatalf("StreamURL() error = %v", err)
}
if got := link.URL; got != download.URL+"/cdn/video.mp4" {
t.Fatalf("URL = %q, want final CDN URL", got)
}
if got := link.Headers.Get("Referer"); !strings.HasPrefix(got, download.URL) {
t.Fatalf("Referer = %q, want original download host", got)
}
if downloadReferer != defaultReferer {
t.Fatalf("resolve Referer = %q, want %q", downloadReferer, defaultReferer)
}
}
func TestInitUsesAccessTokenWithoutLogin(t *testing.T) {
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/user/sign_in":
t.Fatalf("driver should not password-login when access_token is configured")
case "/b/api/user/info":
if got := r.Header.Get("Authorization"); got != "Bearer token-1" {
t.Fatalf("Authorization = %q, want bearer token", got)
}
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{}})
default:
http.NotFound(w, r)
}
}))
defer api.Close()
d := New(Config{
ID: "123-main",
AccessToken: "Bearer token-1",
MainAPIBaseURL: api.URL + "/b/api",
LoginAPIBaseURL: api.URL + "/api",
})
if err := d.Init(context.Background()); err != nil {
t.Fatalf("Init() error = %v", err)
}
}
func TestLoginRiskErrorSuggestsAccessToken(t *testing.T) {
err := loginError("当前账号存在境外登录风险,请使用短信验证码或者微信进行登录。")
if err == nil || !strings.Contains(err.Error(), "access_token") {
t.Fatalf("loginError() = %v, want access_token guidance", err)
}
}
func TestRequestCode429ReturnsRateLimitError(t *testing.T) {
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Retry-After", "2")
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 429,
"message": "请求太频繁",
})
}))
defer api.Close()
d := New(Config{
ID: "123-main",
AccessToken: "token-1",
MainAPIBaseURL: api.URL,
})
_, err := d.request(context.Background(), endpointFileList, http.MethodGet, nil, nil)
var rateLimit *drives.RateLimitError
if !errors.As(err, &rateLimit) {
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
}
if rateLimit.RetryAfter != 2*time.Second {
t.Fatalf("RetryAfter = %s, want 2s", rateLimit.RetryAfter)
}
}
func TestListCoolsDownAndRetriesRateLimit(t *testing.T) {
var listCalls int
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.URL.Path != "/file/list/new" {
http.NotFound(w, r)
return
}
listCalls++
if listCalls == 1 {
w.Header().Set("Retry-After", "1")
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 429,
"message": "请求太频繁",
})
return
}
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 0,
"data": map[string]any{
"Next": "-1",
"Total": 1,
"InfoList": []map[string]any{
{
"FileName": "video.mp4",
"Size": 1234,
"UpdateAt": "2026-01-02 03:04:05",
"FileId": 100,
"Type": 0,
"Etag": "ABCDEF",
"S3KeyFlag": "flag-1",
},
},
},
})
}))
defer api.Close()
d := New(Config{
ID: "123-main",
AccessToken: "token-1",
MainAPIBaseURL: api.URL,
})
entries, err := d.List(context.Background(), d.RootID())
if err != nil {
t.Fatalf("List() error = %v", err)
}
if listCalls != 2 {
t.Fatalf("list calls = %d, want 2", listCalls)
}
if len(entries) != 1 || entries[0].ID != "100" {
t.Fatalf("entries = %#v, want one file", entries)
}
}
func TestResolveDownloadURL429ReturnsRateLimitError(t *testing.T) {
download := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Retry-After", "3")
http.Error(w, "too many requests", http.StatusTooManyRequests)
}))
defer download.Close()
d := New(Config{ID: "123-main"})
_, err := d.resolveDownloadURL(context.Background(), download.URL)
var rateLimit *drives.RateLimitError
if !errors.As(err, &rateLimit) {
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
}
if rateLimit.RetryAfter != 3*time.Second {
t.Fatalf("RetryAfter = %s, want 3s", rateLimit.RetryAfter)
}
}
+285
View File
@@ -0,0 +1,285 @@
package p123
import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"github.com/go-resty/resty/v2"
"github.com/skip2/go-qrcode"
)
const (
defaultUserAPIBase = "https://user.123pan.cn/api"
defaultQRLoginPage = "https://www.123pan.com/wx-app-login.html"
defaultQRReferer = "https://user.123pan.com/centerlogin"
defaultQROrigin = "https://user.123pan.com"
defaultQRUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36"
endpointQRCodeGenerate = "/user/qr-code/generate"
endpointQRCodeResult = "/user/qr-code/result"
endpointQRCodeWXCode = "/user/qr-code/wx_code"
)
type QRConfig struct {
UserAPIBaseURL string
HTTPClient *http.Client
Now func() time.Time
}
type QRClient struct {
userAPIBase string
client *resty.Client
now func() time.Time
}
type QRCodeSession struct {
LoginUUID string `json:"loginUuid"`
UniID string `json:"uniID"`
QRCodeURL string `json:"qrCodeUrl"`
QRImageDataURL string `json:"qrImageDataUrl"`
ExpiresAt string `json:"expiresAt,omitempty"`
}
type QRCodeStatus struct {
LoginStatus int `json:"loginStatus"`
StatusText string `json:"statusText"`
ScanPlatform int `json:"scanPlatform,omitempty"`
PlatformText string `json:"platformText,omitempty"`
AccessToken string `json:"accessToken,omitempty"`
}
func NewQRClient(c QRConfig) *QRClient {
userAPIBase := strings.TrimRight(strings.TrimSpace(c.UserAPIBaseURL), "/")
if userAPIBase == "" {
userAPIBase = defaultUserAPIBase
}
httpClient := c.HTTPClient
if httpClient == nil {
httpClient = &http.Client{Timeout: 20 * time.Second}
}
now := c.Now
if now == nil {
now = time.Now
}
return &QRClient{
userAPIBase: userAPIBase,
client: resty.NewWithClient(httpClient).
SetTimeout(20*time.Second).
SetHeader("Accept", "application/json, text/plain, */*"),
now: now,
}
}
func (c *QRClient) Generate(ctx context.Context) (QRCodeSession, error) {
loginUUID, err := newLoginUUID()
if err != nil {
return QRCodeSession{}, err
}
var resp qrGenerateResp
res, err := c.request(ctx, loginUUID).
SetResult(&resp).
Get(c.userAPIBase + endpointQRCodeGenerate)
if err != nil {
return QRCodeSession{}, err
}
if resp.Code != 0 {
return QRCodeSession{}, qrAPIError(resp.Message, res.StatusCode(), resp.Code)
}
uniID := strings.TrimSpace(resp.Data.UniID)
if uniID == "" {
return QRCodeSession{}, errors.New("123pan qr: empty uniID")
}
qrURL := buildQRLoginURL(resp.Data.URL, uniID)
png, err := qrcode.Encode(qrURL, qrcode.Medium, 220)
if err != nil {
return QRCodeSession{}, err
}
return QRCodeSession{
LoginUUID: loginUUID,
UniID: uniID,
QRCodeURL: qrURL,
QRImageDataURL: "data:image/png;base64," + base64.StdEncoding.EncodeToString(png),
ExpiresAt: c.now().Add(5 * time.Minute).Format(time.RFC3339),
}, nil
}
func (c *QRClient) Poll(ctx context.Context, loginUUID, uniID string) (QRCodeStatus, error) {
loginUUID = strings.TrimSpace(loginUUID)
uniID = strings.TrimSpace(uniID)
if loginUUID == "" {
return QRCodeStatus{}, errors.New("loginUuid is required")
}
if uniID == "" {
return QRCodeStatus{}, errors.New("uniID is required")
}
var resp qrResultResp
res, err := c.request(ctx, loginUUID).
SetQueryParam("uniID", uniID).
SetResult(&resp).
Get(c.userAPIBase + endpointQRCodeResult)
if err != nil {
return QRCodeStatus{}, err
}
if resp.Code != 0 && resp.Code != 200 {
return QRCodeStatus{}, qrAPIError(resp.Message, res.StatusCode(), resp.Code)
}
if resp.Code == 200 {
resp.Data.LoginStatus = 3
if resp.Data.ScanPlatform == 0 {
resp.Data.ScanPlatform = resp.Data.LoginType
}
}
status := QRCodeStatus{
LoginStatus: resp.Data.LoginStatus,
StatusText: qrLoginStatusText(resp.Data.LoginStatus),
ScanPlatform: resp.Data.ScanPlatform,
PlatformText: qrScanPlatformText(resp.Data.ScanPlatform),
}
if status.LoginStatus != 3 {
return status, nil
}
if token := resp.TokenValue(); token != "" {
status.AccessToken = normalizeAccessToken(token)
return status, nil
}
if resp.Data.ScanPlatform == 4 {
token, err := c.finishWechatLogin(ctx, loginUUID, uniID)
if err != nil {
return QRCodeStatus{}, err
}
status.AccessToken = normalizeAccessToken(token)
return status, nil
}
return QRCodeStatus{}, errors.New("123pan qr: confirmed login returned empty token")
}
func (c *QRClient) finishWechatLogin(ctx context.Context, loginUUID, uniID string) (string, error) {
var wxResp qrWXCodeResp
res, err := c.request(ctx, loginUUID).
SetBody(map[string]string{"uniID": uniID}).
SetResult(&wxResp).
Post(c.userAPIBase + endpointQRCodeWXCode)
if err != nil {
return "", err
}
if wxResp.Code != 0 {
return "", qrAPIError(wxResp.Message, res.StatusCode(), wxResp.Code)
}
wxCode := strings.TrimSpace(wxResp.WXCode())
if wxCode == "" {
return "", errors.New("123pan qr: empty wechat code")
}
var signIn loginResp
res, err = c.request(ctx, loginUUID).
SetBody(map[string]any{
"from": "web",
"wechat_code": wxCode,
"type": 4,
}).
SetResult(&signIn).
Post(c.userAPIBase + endpointSignIn)
if err != nil {
return "", err
}
if signIn.Code != 200 && signIn.Code != 0 {
return "", qrAPIError(signIn.Message, res.StatusCode(), signIn.Code)
}
token := strings.TrimSpace(signIn.Data.Token)
if token == "" {
return "", errors.New("123pan qr: empty token")
}
return token, nil
}
func (c *QRClient) request(ctx context.Context, loginUUID string) *resty.Request {
return c.client.R().
SetContext(ctx).
SetHeaders(map[string]string{
"Content-Type": "application/json;charset=UTF-8",
"platform": defaultPlatform,
"App-Version": defaultAppVersion,
"LoginUuid": loginUUID,
"Referer": defaultQRReferer,
"Origin": defaultQROrigin,
"User-Agent": defaultQRUserAgent,
})
}
func buildQRLoginURL(raw, uniID string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
raw = defaultQRLoginPage
}
u, err := url.Parse(raw)
if err != nil {
return defaultQRLoginPage + "?env=production&uniID=" + url.QueryEscape(uniID) + "&source=123pan&type=login"
}
q := u.Query()
q.Set("env", "production")
q.Set("uniID", uniID)
q.Set("source", "123pan")
q.Set("type", "login")
u.RawQuery = q.Encode()
return u.String()
}
func newLoginUUID() (string, error) {
var b [16]byte
if _, err := rand.Read(b[:]); err != nil {
return "", err
}
b[6] = (b[6] & 0x0f) | 0x40
b[8] = (b[8] & 0x3f) | 0x80
parts := []string{
hex.EncodeToString(b[0:4]),
hex.EncodeToString(b[4:6]),
hex.EncodeToString(b[6:8]),
hex.EncodeToString(b[8:10]),
hex.EncodeToString(b[10:16]),
}
return strings.Join(parts, "-"), nil
}
func qrAPIError(message string, httpStatus, apiCode int) error {
message = strings.TrimSpace(message)
if message == "" {
message = fmt.Sprintf("HTTP %d code=%d", httpStatus, apiCode)
}
return errors.New(message)
}
func qrLoginStatusText(status int) string {
switch status {
case 0:
return "等待扫码"
case 1:
return "已扫码,等待确认"
case 2:
return "已拒绝"
case 3:
return "已确认"
case 4:
return "已过期"
default:
return "未知状态"
}
}
func qrScanPlatformText(platform int) string {
switch platform {
case 4:
return "微信"
case 7:
return "123 云盘 App"
default:
return ""
}
}
+182
View File
@@ -0,0 +1,182 @@
package p123
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestQRCodeGenerateBuildsImage(t *testing.T) {
var seenLoginUUID string
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.URL.Path != "/api/user/qr-code/generate" {
http.NotFound(w, r)
return
}
seenLoginUUID = r.Header.Get("LoginUuid")
if seenLoginUUID == "" {
t.Fatalf("missing LoginUuid header")
}
if r.Header.Get("platform") != defaultPlatform {
t.Fatalf("platform header = %q, want %q", r.Header.Get("platform"), defaultPlatform)
}
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 0,
"message": "ok",
"data": map[string]string{
"uniID": "uni-1",
"url": "https://www.123pan.com/wx-app-login.html",
},
})
}))
t.Cleanup(api.Close)
got, err := NewQRClient(QRConfig{UserAPIBaseURL: api.URL + "/api"}).Generate(context.Background())
if err != nil {
t.Fatalf("Generate() error = %v", err)
}
if got.LoginUUID != seenLoginUUID {
t.Fatalf("loginUuid = %q, want header %q", got.LoginUUID, seenLoginUUID)
}
if got.UniID != "uni-1" {
t.Fatalf("uniID = %q, want uni-1", got.UniID)
}
if !strings.Contains(got.QRCodeURL, "uniID=uni-1") || !strings.Contains(got.QRCodeURL, "type=login") {
t.Fatalf("qrCodeUrl = %q, want login params", got.QRCodeURL)
}
if !strings.HasPrefix(got.QRImageDataURL, "data:image/png;base64,") {
t.Fatalf("qrImageDataUrl missing png data url prefix")
}
if got.ExpiresAt == "" {
t.Fatalf("expiresAt is empty")
}
}
func TestQRCodePollCompletesWechatLogin(t *testing.T) {
var wxCodeRequested bool
var signInRequested bool
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.Header.Get("LoginUuid") != "login-1" {
t.Fatalf("LoginUuid = %q, want login-1", r.Header.Get("LoginUuid"))
}
switch r.URL.Path {
case "/api/user/qr-code/result":
if r.URL.Query().Get("uniID") != "uni-1" {
t.Fatalf("uniID = %q, want uni-1", r.URL.Query().Get("uniID"))
}
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 0,
"data": map[string]any{
"loginStatus": 3,
"scanPlatform": 4,
},
})
case "/api/user/qr-code/wx_code":
wxCodeRequested = true
var body map[string]string
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatalf("decode wx_code body: %v", err)
}
if body["uniID"] != "uni-1" {
t.Fatalf("wx_code uniID = %q, want uni-1", body["uniID"])
}
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 0,
"data": map[string]string{"wxCode": "wx-code-1"},
})
case "/api/user/sign_in":
signInRequested = true
var body map[string]any
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatalf("decode sign_in body: %v", err)
}
if body["wechat_code"] != "wx-code-1" {
t.Fatalf("wechat_code = %#v, want wx-code-1", body["wechat_code"])
}
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 200,
"data": map[string]string{"token": "Bearer token-1"},
})
default:
http.NotFound(w, r)
}
}))
t.Cleanup(api.Close)
got, err := NewQRClient(QRConfig{UserAPIBaseURL: api.URL + "/api"}).Poll(context.Background(), "login-1", "uni-1")
if err != nil {
t.Fatalf("Poll() error = %v", err)
}
if !wxCodeRequested || !signInRequested {
t.Fatalf("wechat completion calls wx=%v signIn=%v, want both", wxCodeRequested, signInRequested)
}
if got.LoginStatus != 3 || got.AccessToken != "token-1" || got.PlatformText != "微信" {
t.Fatalf("status = %#v, want confirmed wechat token", got)
}
}
func TestQRCodePollUsesAppToken(t *testing.T) {
var wxCodeRequested bool
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/user/qr-code/result":
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 0,
"data": map[string]any{
"loginStatus": 3,
"scanPlatform": 7,
"token": "app-token",
},
})
case "/api/user/qr-code/wx_code":
wxCodeRequested = true
http.Error(w, "unexpected wx_code", http.StatusInternalServerError)
default:
http.NotFound(w, r)
}
}))
t.Cleanup(api.Close)
got, err := NewQRClient(QRConfig{UserAPIBaseURL: api.URL + "/api"}).Poll(context.Background(), "login-1", "uni-1")
if err != nil {
t.Fatalf("Poll() error = %v", err)
}
if wxCodeRequested {
t.Fatalf("wx_code should not be called when app token is already returned")
}
if got.AccessToken != "app-token" || got.PlatformText != "123 云盘 App" {
t.Fatalf("status = %#v, want app token", got)
}
}
func TestQRCodePollUsesOfficialAppSuccessCode(t *testing.T) {
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.URL.Path != "/api/user/qr-code/result" {
http.NotFound(w, r)
return
}
_ = json.NewEncoder(w).Encode(map[string]any{
"code": 200,
"data": map[string]any{
"login_type": 7,
"token": "app-token",
},
})
}))
t.Cleanup(api.Close)
got, err := NewQRClient(QRConfig{UserAPIBaseURL: api.URL + "/api"}).Poll(context.Background(), "login-1", "uni-1")
if err != nil {
t.Fatalf("Poll() error = %v", err)
}
if got.LoginStatus != 3 || got.ScanPlatform != 7 || got.AccessToken != "app-token" {
t.Fatalf("status = %#v, want official app success token", got)
}
}
+183
View File
@@ -0,0 +1,183 @@
package p123
import (
"encoding/json"
"strconv"
"strings"
"time"
)
type apiEnvelope struct {
Code int `json:"code"`
Message string `json:"message"`
}
type loginResp struct {
Code int `json:"code"`
Message string `json:"message"`
Data struct {
Token string `json:"token"`
} `json:"data"`
}
type qrGenerateResp struct {
Code int `json:"code"`
Message string `json:"message"`
Data struct {
UniID string `json:"uniID"`
URL string `json:"url"`
} `json:"data"`
}
type qrResultResp struct {
Code int `json:"code"`
Message string `json:"message"`
Data struct {
LoginStatus int `json:"loginStatus"`
ScanPlatform int `json:"scanPlatform"`
LoginType int `json:"login_type"`
Token string `json:"token"`
AccessToken string `json:"accessToken"`
} `json:"data"`
}
func (r qrResultResp) TokenValue() string {
if strings.TrimSpace(r.Data.Token) != "" {
return r.Data.Token
}
return r.Data.AccessToken
}
type qrWXCodeResp struct {
Code int `json:"code"`
Message string `json:"message"`
Data struct {
WXCodeLower string `json:"wxCode"`
WXCodeTitle string `json:"WxCode"`
Code string `json:"code"`
} `json:"data"`
}
func (r qrWXCodeResp) WXCode() string {
if r.Data.WXCodeLower != "" {
return r.Data.WXCodeLower
}
if r.Data.WXCodeTitle != "" {
return r.Data.WXCodeTitle
}
return r.Data.Code
}
type fileListResp struct {
Data struct {
Next string `json:"Next"`
Total int `json:"Total"`
InfoList []panFile `json:"InfoList"`
} `json:"data"`
}
type panFile struct {
FileName string `json:"FileName"`
Size int64 `json:"Size"`
UpdateAt flexibleTime `json:"UpdateAt"`
FileID int64 `json:"FileId"`
Type int `json:"Type"`
Etag string `json:"Etag"`
S3KeyFlag string `json:"S3KeyFlag"`
}
type cachedFile struct {
file panFile
parentID string
}
type downloadInfoResp struct {
Data struct {
DownloadURL string `json:"DownloadUrl"`
DownloadURLLower string `json:"downloadUrl"`
} `json:"data"`
}
func (r downloadInfoResp) URL() string {
if r.Data.DownloadURL != "" {
return r.Data.DownloadURL
}
return r.Data.DownloadURLLower
}
type redirectResp struct {
Data struct {
RedirectURL string `json:"redirect_url"`
RedirectURLCamel string `json:"redirectUrl"`
RedirectURLTitle string `json:"RedirectUrl"`
} `json:"data"`
}
func (r redirectResp) URL() string {
if r.Data.RedirectURL != "" {
return r.Data.RedirectURL
}
if r.Data.RedirectURLCamel != "" {
return r.Data.RedirectURLCamel
}
return r.Data.RedirectURLTitle
}
type mkdirResp struct {
Data struct {
FileID int64 `json:"FileId"`
} `json:"data"`
}
type flexibleTime struct {
t time.Time
}
func (t *flexibleTime) UnmarshalJSON(data []byte) error {
if string(data) == "null" || string(data) == `""` {
return nil
}
var s string
if err := json.Unmarshal(data, &s); err == nil {
t.t = parseTimeString(s)
return nil
}
var n int64
if err := json.Unmarshal(data, &n); err == nil {
if n > 1_000_000_000_000 {
t.t = time.UnixMilli(n)
} else {
t.t = time.Unix(n, 0)
}
return nil
}
return nil
}
func (t flexibleTime) Time() time.Time {
return t.t
}
func parseTimeString(s string) time.Time {
s = strings.TrimSpace(s)
if s == "" {
return time.Time{}
}
for _, layout := range []string{
time.RFC3339Nano,
time.RFC3339,
"2006-01-02 15:04:05",
"2006-01-02T15:04:05",
} {
if parsed, err := time.ParseInLocation(layout, s, time.FixedZone("UTC+8", 8*3600)); err == nil {
return parsed
}
}
if n, err := strconv.ParseInt(s, 10, 64); err == nil {
if n > 1_000_000_000_000 {
return time.UnixMilli(n)
}
return time.Unix(n, 0)
}
return time.Time{}
}
+154 -20
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"io"
"log"
"net"
"net/http"
"net/url"
"os"
@@ -20,6 +21,7 @@ import (
"time"
"github.com/video-site/backend/internal/catalog"
"golang.org/x/net/proxy"
)
// 默认 author/tag 标签,便于在前端筛选 spider91 来源的视频。
@@ -79,29 +81,123 @@ func NewCrawler(cfg CrawlerConfig) *Crawler {
cfg.DownloadTimeout = 30 * time.Minute
}
if cfg.HTTPClient == nil {
// 选 proxy 函数:显式 ProxyURL > 环境变量 > 直连
proxyFn := http.ProxyFromEnvironment
if strings.TrimSpace(cfg.ProxyURL) != "" {
if u, err := url.Parse(cfg.ProxyURL); err == nil {
proxyFn = http.ProxyURL(u)
} else {
log.Printf("[spider91] invalid proxy URL %q, falling back to env: %v", cfg.ProxyURL, err)
}
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
ResponseHeaderTimeout: 60 * time.Second,
MaxIdleConns: 10,
IdleConnTimeout: 90 * time.Second,
}
if err := configureExplicitProxy(transport, cfg.ProxyURL); err != nil {
log.Printf("[spider91] invalid configured proxy URL, falling back to env: %v", err)
}
cfg.HTTPClient = &http.Client{
// 不限制总下载时长,靠 ctx 控制;只挡 dial / handshake / header
Timeout: 0,
Transport: &http.Transport{
Proxy: proxyFn,
ResponseHeaderTimeout: 60 * time.Second,
MaxIdleConns: 10,
IdleConnTimeout: 90 * time.Second,
},
Timeout: 0,
Transport: transport,
}
}
return &Crawler{cfg: cfg}
}
func configureExplicitProxy(transport *http.Transport, raw string) error {
proxyURL := strings.TrimSpace(raw)
if proxyURL == "" {
return nil
}
u, err := url.Parse(proxyURL)
if err != nil || u.Scheme == "" || u.Host == "" {
return fmt.Errorf("invalid proxy URL")
}
switch strings.ToLower(u.Scheme) {
case "http", "https":
transport.Proxy = http.ProxyURL(u)
transport.DialContext = nil
return nil
case "socks5", "socks5h":
dialContext, err := socksProxyDialContext(u)
if err != nil {
return err
}
transport.Proxy = nil
transport.DialContext = dialContext
return nil
default:
return fmt.Errorf("unsupported proxy scheme %q", u.Scheme)
}
}
func socksProxyDialContext(proxyURL *url.URL) (func(context.Context, string, string) (net.Conn, error), error) {
var auth *proxy.Auth
if proxyURL.User != nil {
username := proxyURL.User.Username()
password, _ := proxyURL.User.Password()
auth = &proxy.Auth{User: username, Password: password}
}
dialer, err := proxy.SOCKS5("tcp", proxyURL.Host, auth, &net.Dialer{Timeout: 60 * time.Second})
if err != nil {
return nil, err
}
remoteDNS := strings.EqualFold(proxyURL.Scheme, "socks5h")
return func(ctx context.Context, network, addr string) (net.Conn, error) {
target := addr
if !remoteDNS {
resolved, err := resolveSocksTarget(ctx, addr)
if err != nil {
return nil, err
}
target = resolved
}
if ctxDialer, ok := dialer.(proxy.ContextDialer); ok {
return ctxDialer.DialContext(ctx, network, target)
}
type result struct {
conn net.Conn
err error
}
ch := make(chan result, 1)
go func() {
conn, err := dialer.Dial(network, target)
ch <- result{conn: conn, err: err}
}()
select {
case <-ctx.Done():
return nil, ctx.Err()
case res := <-ch:
return res.conn, res.err
}
}, nil
}
func resolveSocksTarget(ctx context.Context, addr string) (string, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil || net.ParseIP(host) != nil {
return addr, nil
}
ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
if err != nil {
return "", err
}
ip := selectSocksTargetIP(ips)
if ip == nil {
return "", fmt.Errorf("resolve %s: no address", host)
}
return net.JoinHostPort(ip.String(), port), nil
}
func selectSocksTargetIP(ips []net.IPAddr) net.IP {
for _, addr := range ips {
if ip4 := addr.IP.To4(); ip4 != nil {
return ip4
}
}
for _, addr := range ips {
if addr.IP != nil {
return addr.IP
}
}
return nil
}
// CrawlResult 汇总一次 RunOnce 的结果。
type CrawlResult struct {
// TargetNew 是本次 RunOnce 的目标新增数(来自 drive.Credentials.target_new)。
@@ -324,6 +420,16 @@ func (c *Crawler) startSpiderTargetNew(ctx context.Context, targetNew int, seenP
if c.cfg.WorkDir != "" {
cmd.Dir = c.cfg.WorkDir
}
if proxyURL := strings.TrimSpace(c.cfg.ProxyURL); proxyURL != "" {
cmd.Env = append(os.Environ(),
"HTTP_PROXY="+proxyURL,
"HTTPS_PROXY="+proxyURL,
"http_proxy="+proxyURL,
"https_proxy="+proxyURL,
"NO_PROXY=",
"no_proxy=",
)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, nil, fmt.Errorf("stdout pipe: %w", err)
@@ -427,6 +533,17 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
}
}
title := strings.TrimSpace(item.Title)
if title == "" {
title = sourceID
}
tags := []string{DefaultTag}
if matched, err := c.cfg.Catalog.MatchTags(ctx, title+" "+DefaultAuthor); err == nil {
tags = mergeCatalogTags(tags, matched)
} else {
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s match tags: %v", c.cfg.Driver.ID(), viewkey, sourceID, err)
}
// 入库
now := time.Now()
v := &catalog.Video{
@@ -434,9 +551,9 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
DriveID: c.cfg.Driver.ID(),
FileID: videoFile,
FileName: videoFile,
Title: strings.TrimSpace(item.Title),
Title: title,
Author: DefaultAuthor,
Tags: []string{DefaultTag},
Tags: tags,
Ext: strings.TrimPrefix(videoExt, "."),
Quality: "HD",
Size: videoSize,
@@ -445,9 +562,6 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
CreatedAt: now,
UpdatedAt: now,
}
if v.Title == "" {
v.Title = sourceID
}
if thumbReady {
// 设了 ThumbnailURL 后 thumb worker 会跳过这条视频,
// 不再尝试用 ffmpeg 抽帧(封面已经是网站原图)。
@@ -888,6 +1002,26 @@ func copyFileAtomic(src, dst string) error {
return os.Rename(tmp, dst)
}
func mergeCatalogTags(lists ...[]string) []string {
out := []string{}
seen := map[string]bool{}
for _, list := range lists {
for _, tag := range list {
tag = strings.TrimSpace(tag)
if tag == "" {
continue
}
key := strings.ToLower(tag)
if seen[key] {
continue
}
seen[key] = true
out = append(out, tag)
}
}
return out
}
// BuildVideoID 给定 driveID + 91 源视频 ID,按统一规则生成 catalog 中 videos.id。
// 与 scanner 用法一致:<kind>-<driveID>-<fileID>。
func BuildVideoID(driveID, sourceID string) string {
@@ -3,6 +3,8 @@ package spider91
import (
"context"
"encoding/json"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
@@ -53,7 +55,7 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
// 同时仍写 --output 文件作归档。
videoEntries := []map[string]string{
{
"title": "Video One",
"title": "Video One 口交",
"thumb_url": srv.URL + "/thumb/not-120001.jpg",
"video_url": srv.URL + "/videos/120001.mp4",
"viewkey": "vk-001",
@@ -94,6 +96,9 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
}); err != nil {
t.Fatalf("upsert drive: %v", err)
}
if _, err := cat.CreateTagAndClassify(context.Background(), "Video One", nil, "user"); err != nil {
t.Fatalf("create user tag: %v", err)
}
var newVideos []*catalog.Video
c := NewCrawler(CrawlerConfig{
@@ -188,6 +193,17 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
if !hasDefaultTag {
t.Fatalf("video %s tags = %v, want contain %q", videoID, v.Tags, DefaultTag)
}
if sourceID == "120001" {
if !containsString(v.Tags, "口交") {
t.Fatalf("video %s tags = %v, want contain built-in tag 口交", videoID, v.Tags)
}
if !containsString(v.Tags, "Video One") {
t.Fatalf("video %s tags = %v, want contain user tag Video One", videoID, v.Tags)
}
}
if sourceID == "120002" && (containsString(v.Tags, "口交") || containsString(v.Tags, "Video One")) {
t.Fatalf("video %s tags = %v, should not inherit tags from other spider91 videos", videoID, v.Tags)
}
}
// 7. 第二次 RunOnce:源视频 ID 已存在 → 全部 skipped,无新文件下载
@@ -233,6 +249,108 @@ func TestCrawlerRunOnceMissingScript(t *testing.T) {
}
}
func TestCrawlerPassesProxyToSpiderProcess(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("shell-based fake script only on unix")
}
tmp := t.TempDir()
scriptPath := filepath.Join(tmp, "print_proxy_env.sh")
script := `#!/bin/sh
printf 'HTTP_PROXY=%s\n' "$HTTP_PROXY"
printf 'HTTPS_PROXY=%s\n' "$HTTPS_PROXY"
printf 'http_proxy=%s\n' "$http_proxy"
printf 'https_proxy=%s\n' "$https_proxy"
printf 'NO_PROXY=%s\n' "$NO_PROXY"
printf 'no_proxy=%s\n' "$no_proxy"
`
if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
t.Fatalf("write script: %v", err)
}
proxyURL := "socks5h://proxy.local:1080"
drv := New(Config{ID: "proxy-drive", RootDir: filepath.Join(tmp, "proxy-drive")})
c := NewCrawler(CrawlerConfig{
Driver: drv,
PythonPath: "sh",
ScriptPath: scriptPath,
ProxyURL: proxyURL,
})
cmd, stdout, err := c.startSpiderTargetNew(
context.Background(),
1,
filepath.Join(tmp, "seen.txt"),
filepath.Join(tmp, "out.json"),
)
if err != nil {
t.Fatalf("startSpiderTargetNew: %v", err)
}
raw, err := io.ReadAll(stdout)
if err != nil {
t.Fatalf("read stdout: %v", err)
}
if err := cmd.Wait(); err != nil {
t.Fatalf("wait: %v", err)
}
want := strings.Join([]string{
"HTTP_PROXY=" + proxyURL,
"HTTPS_PROXY=" + proxyURL,
"http_proxy=" + proxyURL,
"https_proxy=" + proxyURL,
"NO_PROXY=",
"no_proxy=",
}, "\n") + "\n"
if string(raw) != want {
t.Fatalf("proxy env = %q, want %q", string(raw), want)
}
}
func TestConfigureExplicitProxySupportsSocksSchemes(t *testing.T) {
for _, raw := range []string{
"socks5://127.0.0.1:1080",
"socks5h://proxy-user:proxy-pass@127.0.0.1:1080",
} {
t.Run(raw, func(t *testing.T) {
transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
if err := configureExplicitProxy(transport, raw); err != nil {
t.Fatalf("configureExplicitProxy: %v", err)
}
if transport.Proxy != nil {
t.Fatalf("Transport.Proxy should be nil for SOCKS proxy")
}
if transport.DialContext == nil {
t.Fatalf("Transport.DialContext should be set for SOCKS proxy")
}
})
}
transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
if err := configureExplicitProxy(transport, "http://127.0.0.1:7890"); err != nil {
t.Fatalf("configureExplicitProxy http: %v", err)
}
if transport.Proxy == nil {
t.Fatalf("Transport.Proxy should be set for HTTP proxy")
}
if transport.DialContext != nil {
t.Fatalf("Transport.DialContext should not be set for HTTP proxy")
}
if err := configureExplicitProxy(&http.Transport{}, "ftp://127.0.0.1:21"); err == nil {
t.Fatalf("expected unsupported proxy scheme error")
}
}
func TestSelectSocksTargetIPPrefersIPv4(t *testing.T) {
got := selectSocksTargetIP([]net.IPAddr{
{IP: net.ParseIP("2606:4700:20::681a:229")},
{IP: net.ParseIP("104.26.3.41")},
})
if got == nil || got.String() != "104.26.3.41" {
t.Fatalf("selectSocksTargetIP = %v, want IPv4 104.26.3.41", got)
}
}
// TestCrawlerThumbDownloadFailureMarksStatusFailed 验证:网站封面下载失败时
// crawler 把 thumbnail_status 显式标 'failed',避免后续封面补队列一直重复
// 捞到这条 spider91 视频。
@@ -658,3 +776,12 @@ func buildFakeSpiderScript(entries []map[string]string) string {
sb.WriteString("fi\n")
return sb.String()
}
func containsString(values []string, want string) bool {
for _, value := range values {
if value == want {
return true
}
}
return false
}
+57 -5
View File
@@ -237,13 +237,33 @@ func appendUniqueStart(starts []float64, start, eachSec float64) []float64 {
}
// thumbnailOffsets 选封面抽帧的时间点(秒)。独立于 teaser。
func thumbnailOffsets() []float64 {
return []float64{5, 1, 0}
// 默认取视频中间帧;时长未知时退回早期帧。
func thumbnailOffsets(duration float64) []float64 {
if duration <= 0 {
return []float64{5, 1, 0}
}
mid := duration / 2
out := []float64{mid}
for _, fallback := range []float64{5, 1, 0} {
if !containsOffset(out, fallback) {
out = append(out, fallback)
}
}
return out
}
func containsOffset(offsets []float64, target float64) bool {
for _, offset := range offsets {
if math.Abs(offset-target) < 0.01 {
return true
}
}
return false
}
// --- 封面 ---
// GenerateThumbnail 抽一张 jpg 封面。默认从第 5 秒抽帧,失败时回退到更早时间点。
// GenerateThumbnail 抽一张 jpg 封面。默认从视频中间抽帧,失败时回退到更早时间点。
func (g *Generator) GenerateThumbnail(ctx context.Context, link *drives.StreamLink, videoID string, duration float64) (string, error) {
dir := filepath.Join(g.cfg.LocalDir, "thumbs")
if err := os.MkdirAll(dir, 0o755); err != nil {
@@ -252,7 +272,7 @@ func (g *Generator) GenerateThumbnail(ctx context.Context, link *drives.StreamLi
dst := filepath.Join(dir, videoID+".jpg")
var lastErr error
offsets := thumbnailOffsets()
offsets := thumbnailOffsets(duration)
for i, offset := range offsets {
if i > 0 {
_ = os.Remove(dst)
@@ -1507,6 +1527,29 @@ func driveErrorShouldCooldown(d drives.Drive, err error) bool {
strings.Contains(text, "moov atom not found") ||
strings.Contains(text, "partial file") ||
strings.Contains(text, "service unavailable")
case "p123":
// 123 云盘直链解析 / ffmpeg 读取阶段可能返回 429、5xx,或 WAF 类
// blocked / 访问阻断文本。命中时冷却,避免封面和预览视频生成连续打接口。
text := strings.ToLower(err.Error())
return strings.Contains(text, "请求太频繁") ||
strings.Contains(text, "请求过于频繁") ||
strings.Contains(text, "请求频繁") ||
strings.Contains(text, "操作频繁") ||
strings.Contains(text, "频率限制") ||
strings.Contains(text, "请求次数过多") ||
strings.Contains(text, "429") ||
strings.Contains(text, "http 500") ||
strings.Contains(text, "http 502") ||
strings.Contains(text, "http 503") ||
strings.Contains(text, "http 504") ||
strings.Contains(text, "server returned 403") ||
strings.Contains(text, "403 forbidden") ||
strings.Contains(text, "too many request") ||
strings.Contains(text, "too many requests") ||
strings.Contains(text, "rate limit") ||
strings.Contains(text, "blocked") ||
strings.Contains(text, "访问被阻断") ||
strings.Contains(text, "service unavailable")
}
return false
}
@@ -1553,6 +1596,11 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) bool {
return false
}
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "pending"})
if isSpider91OriginVideo(v) {
log.Printf("[thumb] skip %s: spider91-origin video must use crawled thumbnail", v.Title)
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
return false
}
link, err := w.streamLink(ctx, v)
if err != nil {
if w.pauseForRecoverableError(ctx, v, err, "streamURL") {
@@ -1618,7 +1666,7 @@ func (w *ThumbWorker) probeDuration(ctx context.Context, v *catalog.Video, link
}
func (w *ThumbWorker) generateThumbnailFromLink(ctx context.Context, v *catalog.Video, link *drives.StreamLink) error {
if _, err := w.Gen.GenerateThumbnail(ctx, link, v.ID, 0); err != nil {
if _, err := w.Gen.GenerateThumbnail(ctx, link, v.ID, float64(v.DurationSeconds)); err != nil {
return err
}
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
@@ -1629,6 +1677,10 @@ func (w *ThumbWorker) generateThumbnailFromLink(ctx context.Context, v *catalog.
return nil
}
func isSpider91OriginVideo(v *catalog.Video) bool {
return v != nil && strings.HasPrefix(v.ID, "spider91-")
}
func localPreviewLink(v *catalog.Video) (*drives.StreamLink, bool) {
if v.PreviewLocal == "" {
return nil, false
+22 -9
View File
@@ -168,16 +168,29 @@ func TestMediumAndLongVideosStillRequirePlannedTeaserSegments(t *testing.T) {
}
}
func TestThumbnailOffsetsUseFiveSecondsWithEarlyFallbacks(t *testing.T) {
got := thumbnailOffsets()
want := []float64{5, 1, 0}
if len(got) != len(want) {
t.Fatalf("offsets = %#v, want %#v", got, want)
func TestThumbnailOffsetsPreferMiddleFrame(t *testing.T) {
tests := []struct {
name string
duration float64
want []float64
}{
{name: "unknown duration", duration: 0, want: []float64{5, 1, 0}},
{name: "long video", duration: 2804.9, want: []float64{1402.45, 5, 1, 0}},
{name: "short video", duration: 8.9, want: []float64{4.45, 5, 1, 0}},
{name: "middle equals fallback", duration: 10, want: []float64{5, 1, 0}},
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("offset[%d] = %.2f, want %.2f", i, got[i], want[i])
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := thumbnailOffsets(tt.duration)
if len(got) != len(tt.want) {
t.Fatalf("offsets = %#v, want %#v", got, tt.want)
}
for i := range tt.want {
if math.Abs(got[i]-tt.want[i]) > 0.001 {
t.Fatalf("offset[%d] = %.2f, want %.2f", i, got[i], tt.want[i])
}
}
})
}
}
+47 -2
View File
@@ -39,8 +39,8 @@ func TestThumbWorkerUpdatesThumbnailAndDurationWithoutChangingPreviewStatus(t *t
if gen.thumbnailVideoID != video.ID {
t.Fatalf("thumbnail video id = %q, want %q", gen.thumbnailVideoID, video.ID)
}
if gen.thumbnailDuration != 0 {
t.Fatalf("thumbnail duration = %.1f, want fixed-offset thumbnail generation", gen.thumbnailDuration)
if gen.thumbnailDuration != 42 {
t.Fatalf("thumbnail duration = %.1f, want probed duration", gen.thumbnailDuration)
}
if gen.probeCalls != 1 {
t.Fatalf("probe calls = %d, want 1 for thumbnail generation", gen.probeCalls)
@@ -89,6 +89,35 @@ func TestThumbWorkerBackfillsDurationWhenThumbnailAlreadyExists(t *testing.T) {
}
}
func TestThumbWorkerDoesNotGenerateThumbnailForSpider91OriginVideo(t *testing.T) {
ctx := context.Background()
cat, video := seedPreviewTestVideo(t, "spider91-91-spider-1200001")
gen := &fakeThumbGenerator{probeDuration: 42}
drv := &previewFakeDrive{kind: "pikpak"}
worker := NewThumbWorker(gen, cat, drv)
worker.process(ctx, video)
got, err := cat.GetVideo(ctx, video.ID)
if err != nil {
t.Fatalf("get video: %v", err)
}
if got.ThumbnailURL != "" {
t.Fatalf("thumbnail = %q, want empty when crawled spider91 thumbnail is missing", got.ThumbnailURL)
}
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
if err != nil {
t.Fatalf("list failed thumbnails: %v", err)
}
if len(failed) != 1 || failed[0].ID != video.ID {
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
}
if gen.probeCalls != 0 || gen.generateCalls != 0 {
t.Fatalf("generator calls probe=%d generate=%d, want no ffmpeg work for spider91-origin thumbnail", gen.probeCalls, gen.generateCalls)
}
}
func TestThumbWorkerSkipsDurationBackfillWhenExistingThumbnailCannotBeProbed(t *testing.T) {
ctx := context.Background()
cat, video := seedPreviewTestVideo(t, "thumb-worker-existing-thumbnail-probe-fails")
@@ -587,6 +616,22 @@ func TestPreviewWorkerP115TransientErrorKeepsVideoPending(t *testing.T) {
}
}
func TestP123TransientErrorsShouldCooldown(t *testing.T) {
drv := &previewFakeDrive{kind: "p123"}
for _, err := range []error{
errors.New("Server returned 403 Forbidden"),
errors.New("请求太频繁"),
errors.New("http 503 service unavailable"),
} {
if !driveErrorShouldCooldown(drv, err) {
t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err)
}
}
if driveErrorShouldCooldown(drv, errors.New("invalid credential")) {
t.Fatal("invalid credential should not trigger p123 cooldown")
}
}
func assertCooldownAround(t *testing.T, until time.Time, before time.Time, want time.Duration) {
t.Helper()
if until.IsZero() {
+3 -1
View File
@@ -147,13 +147,15 @@ func (p *Proxy) ServeStream(w http.ResponseWriter, r *http.Request, driveID, fil
// CDN 不校验请求头,直连可获得最佳带宽并避免占用 backend 出站
// - onedriveMicrosoft Graph 返回的 @microsoft.graph.downloadUrl 是短期
// 免鉴权下载 URL,不需要后端继续代传视频字节
// - p123123 云盘 download_info 返回的下载页会再跳 CDNdriver 已在后端
// 先解出最终 Location,浏览器可直接 302 到该短期地址
//
// 其余网盘(如沃盘 / 夸克等)仍走反代,因为它们的下载
// 链接通常需要随请求带上后端持有的 Cookie / Authorization / Range
// 的特殊处理,浏览器拿不到这些上下文。
func shouldRedirect(d drives.Drive) bool {
switch d.Kind() {
case "p115", "pikpak", "onedrive":
case "p115", "pikpak", "onedrive", "p123":
return true
}
return false
+25
View File
@@ -176,6 +176,31 @@ func TestServeStreamRedirectsOneDrive(t *testing.T) {
}
}
func TestServeStreamRedirectsP123(t *testing.T) {
reg := NewRegistry()
drv := &proxyFakeSimpleDrive{
kind: "p123",
url: "https://cdn.123pan.example/video.mp4",
}
reg.Set("p123", drv)
p := New(reg)
req := httptest.NewRequest(http.MethodGet, "/p/stream/p123/file-1", nil)
rr := httptest.NewRecorder()
p.ServeStream(rr, req, "p123", "file-1")
if rr.Code != http.StatusFound {
t.Fatalf("status = %d, want %d", rr.Code, http.StatusFound)
}
if got := rr.Header().Get("Location"); got != "https://cdn.123pan.example/video.mp4" {
t.Fatalf("Location = %q", got)
}
if drv.calls != 1 {
t.Fatalf("link calls = %d, want 1", drv.calls)
}
}
func TestServeStreamServesLocalFilePath(t *testing.T) {
path := filepath.Join(t.TempDir(), "video.mp4")
if err := os.WriteFile(path, []byte("0123456789"), 0o644); err != nil {
+1 -14
View File
@@ -181,14 +181,10 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
if existing.Category == "" && dirName != "" {
patch.Category = dirName
}
if existing.ThumbnailURL == "" && e.ThumbnailURL != "" {
patch.ThumbnailURL = e.ThumbnailURL
}
if patch.Category != "" || patch.ThumbnailURL != "" || patch.ContentHash != "" || patch.FileName != "" {
if patch.Category != "" || patch.ContentHash != "" || patch.FileName != "" {
_ = s.Catalog.UpdateVideoMeta(ctx, id, patch)
}
if dup := s.findDuplicate(ctx, e.Hash, e.Name, e.Size, id); dup != nil {
s.backfillDuplicateThumbnail(ctx, dup, e.ThumbnailURL)
continue
}
if !sameTags(existing.Tags, tags) {
@@ -198,7 +194,6 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
}
if dup := s.findDuplicate(ctx, e.Hash, e.Name, e.Size, id); dup != nil {
s.backfillDuplicateThumbnail(ctx, dup, e.ThumbnailURL)
continue
}
@@ -216,7 +211,6 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
Ext: strings.TrimPrefix(ext, "."),
Quality: "HD",
Size: e.Size,
ThumbnailURL: e.ThumbnailURL,
PreviewStatus: "pending",
Category: dirName,
PublishedAt: now,
@@ -268,13 +262,6 @@ func (s *Scanner) findDuplicateByFileSignature(ctx context.Context, fileName str
return dup
}
func (s *Scanner) backfillDuplicateThumbnail(ctx context.Context, canonical *catalog.Video, thumbnailURL string) {
if canonical.ThumbnailURL != "" || thumbnailURL == "" {
return
}
_ = s.Catalog.UpdateVideoMeta(ctx, canonical.ID, catalog.VideoMetaPatch{ThumbnailURL: thumbnailURL})
}
func sameTags(a, b []string) bool {
if len(a) != len(b) {
return false
+6 -6
View File
@@ -14,7 +14,7 @@ import (
"github.com/video-site/backend/internal/drives"
)
func TestRunPersistsRemoteThumbnailFromDriveEntry(t *testing.T) {
func TestRunIgnoresRemoteThumbnailFromDriveEntry(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
@@ -50,8 +50,8 @@ func TestRunPersistsRemoteThumbnailFromDriveEntry(t *testing.T) {
if err != nil {
t.Fatalf("get video: %v", err)
}
if got.ThumbnailURL != "https://thumbnail.example/clip.jpg" {
t.Fatalf("thumbnail = %q, want remote thumbnail", got.ThumbnailURL)
if got.ThumbnailURL != "" {
t.Fatalf("thumbnail = %q, want empty so local thumbnail worker regenerates it", got.ThumbnailURL)
}
}
@@ -90,7 +90,7 @@ func TestRunIgnoresZeroSizeVideoFiles(t *testing.T) {
}
}
func TestRunBackfillsRemoteThumbnailForExistingVideo(t *testing.T) {
func TestRunDoesNotBackfillRemoteThumbnailForExistingVideo(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
@@ -140,8 +140,8 @@ func TestRunBackfillsRemoteThumbnailForExistingVideo(t *testing.T) {
if err != nil {
t.Fatalf("get video: %v", err)
}
if got.ThumbnailURL != "https://thumbnail.example/backfilled.jpg" {
t.Fatalf("thumbnail = %q, want backfilled remote thumbnail", got.ThumbnailURL)
if got.ThumbnailURL != "" {
t.Fatalf("thumbnail = %q, want empty so local thumbnail worker regenerates it", got.ThumbnailURL)
}
}
+84 -2
View File
@@ -4,7 +4,8 @@
// - 改写 catalog 行:drive_id / file_id / content_hash 改成目标盘的;
// 视频自身的 id 不变(仍是 spider91-<driveID>-<viewkey>),video_tags、
// 收藏、点赞、views 等关联数据全部保留
// - 删除本地 mp4spider91/<id>/videos/<viewkey>.<ext>)和 thumbspider91/<id>/thumbs/<viewkey>.jpg
// - 删除本地 mp4spider91/<id>/videos/<viewkey>.<ext>)和 thumb
// spider91/<id>/thumbs/<viewkey>.jpg);公共 /p/thumb/<videoID> 副本会保留
//
// 之后回放时,videoSource() 自动落到 /p/stream/<target>/<file_id>
// proxy 层走对应盘的直链 / 302 直连。
@@ -175,6 +176,7 @@ type Config struct {
// 4002 / 9)后整体进入冷却的时长。冷却期间 runOnce 直接返回,不再发起任何
// PikPak API 请求,避免被进一步风控。0 时默认 5 分钟;< 0 关闭冷却(仅用于测试)。
CaptchaCooldown time.Duration
CommonThumbDir string
OnMigrated func(videoID string)
}
@@ -571,18 +573,98 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
if err := m.cfg.Catalog.MigrateVideoToDrive(ctx, v.ID, targetDriveID, res.FileID, res.Hash); err != nil {
return false, fmt.Errorf("catalog migrate: %w", err)
}
m.preserveCrawledThumbnail(ctx, src, v)
// 同步 catalog 里的 file_name,让下次目标盘扫盘时 (file_name, size) 也能匹配上
if err := m.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{FileName: uploadName}); err != nil {
log.Printf("[spider91migrate] %s update file_name after migrate: %v", v.ID, err)
}
// 删除本地 mp4 和 thumbthumb 在 previews/thumbs/ 还有副本,不影响展示)
// 删除本地 mp4 和 thumb公共 /p/thumb 副本已在 preserveCrawledThumbnail 中保留)。
CleanupSpider91Local(src, v.FileID)
log.Printf("[spider91migrate] %s migrated to drive=%s(kind=%s) file=%s name=%q", v.ID, targetDriveID, pp.Kind(), res.FileID, uploadName)
return true, nil
}
func (m *Migrator) preserveCrawledThumbnail(ctx context.Context, src *spider91.Driver, v *catalog.Video) {
if m == nil || m.cfg.Catalog == nil || src == nil || v == nil || v.ID == "" || v.FileID == "" {
return
}
commonDir := strings.TrimSpace(m.cfg.CommonThumbDir)
if commonDir == "" {
return
}
thumbPath, ok := findSpider91ThumbPath(src, v.FileID)
if !ok {
if v.ThumbnailURL == "" {
log.Printf("[spider91migrate] %s crawled thumbnail missing before migration cleanup", v.ID)
}
return
}
if err := os.MkdirAll(commonDir, 0o755); err != nil {
log.Printf("[spider91migrate] %s mkdir common thumbs: %v", v.ID, err)
return
}
dst := filepath.Join(commonDir, v.ID+".jpg")
if _, err := os.Stat(dst); err != nil {
if !os.IsNotExist(err) {
log.Printf("[spider91migrate] %s stat common thumb: %v", v.ID, err)
return
}
if err := copyFileAtomic(thumbPath, dst); err != nil {
log.Printf("[spider91migrate] %s preserve crawled thumbnail: %v", v.ID, err)
return
}
}
if err := m.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
ThumbnailURL: "/p/thumb/" + v.ID,
}); err != nil {
log.Printf("[spider91migrate] %s update crawled thumbnail url: %v", v.ID, err)
return
}
v.ThumbnailURL = "/p/thumb/" + v.ID
}
func findSpider91ThumbPath(src *spider91.Driver, fileID string) (string, bool) {
thumbBase := stripExt(fileID)
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
thumbPath, err := src.ThumbPath(thumbBase + ext)
if err != nil {
continue
}
info, statErr := os.Stat(thumbPath)
if statErr == nil && info.Mode().IsRegular() && info.Size() > 0 {
return thumbPath, true
}
}
return "", false
}
func copyFileAtomic(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
tmp := dst + ".part"
out, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
if err != nil {
return err
}
_, copyErr := io.Copy(out, in)
closeErr := out.Close()
if copyErr != nil {
_ = os.Remove(tmp)
return copyErr
}
if closeErr != nil {
_ = os.Remove(tmp)
return closeErr
}
return os.Rename(tmp, dst)
}
// CleanupSpider91Local 删除已迁移视频的本地 mp4 和 thumb。
//
// thumb 删除是 best-effort —— 找不到就算了(spider91 thumb 文件名带后缀,
@@ -339,12 +339,14 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
now := time.Now()
id := writeSpider91Video(t, cat, src, "vk001", ".mp4", []byte("video bytes here"), now)
commonThumbDir := t.TempDir()
m := New(Config{
Catalog: cat,
Registry: reg,
GetTargetDriveID: func() string { return pp.ID() },
KeepLatestN: -1, // 关闭"保留最新 N 个",让 1 条也能立即上传
CommonThumbDir: commonThumbDir,
})
m.runOnce(context.Background())
@@ -382,8 +384,15 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
if got.ContentHash == "" {
t.Fatalf("content_hash should be set after migration")
}
if got.ThumbnailURL != "/p/thumb/"+id {
t.Fatalf("thumbnail_url = %q, want preserved crawled thumbnail URL", got.ThumbnailURL)
}
commonThumbPath := filepath.Join(commonThumbDir, id+".jpg")
if data, err := os.ReadFile(commonThumbPath); err != nil || string(data) != "thumb" {
t.Fatalf("common thumb = %q, %v; want copied crawled thumb", string(data), err)
}
// 3) 本地视频和 thumb 都被删了
// 3) 本地视频和 thumb 都被删了;公共 /p/thumb 副本保留。
videoPath, _ := src.VideoPath("vk001.mp4")
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
t.Fatalf("local mp4 still exists or stat error %v", err)
+168
View File
@@ -0,0 +1,168 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package socks
import (
"context"
"errors"
"io"
"net"
"strconv"
"time"
)
var (
noDeadline = time.Time{}
aLongTimeAgo = time.Unix(1, 0)
)
func (d *Dialer) connect(ctx context.Context, c net.Conn, address string) (_ net.Addr, ctxErr error) {
host, port, err := splitHostPort(address)
if err != nil {
return nil, err
}
if deadline, ok := ctx.Deadline(); ok && !deadline.IsZero() {
c.SetDeadline(deadline)
defer c.SetDeadline(noDeadline)
}
if ctx != context.Background() {
errCh := make(chan error, 1)
done := make(chan struct{})
defer func() {
close(done)
if ctxErr == nil {
ctxErr = <-errCh
}
}()
go func() {
select {
case <-ctx.Done():
c.SetDeadline(aLongTimeAgo)
errCh <- ctx.Err()
case <-done:
errCh <- nil
}
}()
}
b := make([]byte, 0, 6+len(host)) // the size here is just an estimate
b = append(b, Version5)
if len(d.AuthMethods) == 0 || d.Authenticate == nil {
b = append(b, 1, byte(AuthMethodNotRequired))
} else {
ams := d.AuthMethods
if len(ams) > 255 {
return nil, errors.New("too many authentication methods")
}
b = append(b, byte(len(ams)))
for _, am := range ams {
b = append(b, byte(am))
}
}
if _, ctxErr = c.Write(b); ctxErr != nil {
return
}
if _, ctxErr = io.ReadFull(c, b[:2]); ctxErr != nil {
return
}
if b[0] != Version5 {
return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0])))
}
am := AuthMethod(b[1])
if am == AuthMethodNoAcceptableMethods {
return nil, errors.New("no acceptable authentication methods")
}
if d.Authenticate != nil {
if ctxErr = d.Authenticate(ctx, c, am); ctxErr != nil {
return
}
}
b = b[:0]
b = append(b, Version5, byte(d.cmd), 0)
if ip := net.ParseIP(host); ip != nil {
if ip4 := ip.To4(); ip4 != nil {
b = append(b, AddrTypeIPv4)
b = append(b, ip4...)
} else if ip6 := ip.To16(); ip6 != nil {
b = append(b, AddrTypeIPv6)
b = append(b, ip6...)
} else {
return nil, errors.New("unknown address type")
}
} else {
if len(host) > 255 {
return nil, errors.New("FQDN too long")
}
b = append(b, AddrTypeFQDN)
b = append(b, byte(len(host)))
b = append(b, host...)
}
b = append(b, byte(port>>8), byte(port))
if _, ctxErr = c.Write(b); ctxErr != nil {
return
}
if _, ctxErr = io.ReadFull(c, b[:4]); ctxErr != nil {
return
}
if b[0] != Version5 {
return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0])))
}
if cmdErr := Reply(b[1]); cmdErr != StatusSucceeded {
return nil, errors.New("unknown error " + cmdErr.String())
}
if b[2] != 0 {
return nil, errors.New("non-zero reserved field")
}
l := 2
var a Addr
switch b[3] {
case AddrTypeIPv4:
l += net.IPv4len
a.IP = make(net.IP, net.IPv4len)
case AddrTypeIPv6:
l += net.IPv6len
a.IP = make(net.IP, net.IPv6len)
case AddrTypeFQDN:
if _, err := io.ReadFull(c, b[:1]); err != nil {
return nil, err
}
l += int(b[0])
default:
return nil, errors.New("unknown address type " + strconv.Itoa(int(b[3])))
}
if cap(b) < l {
b = make([]byte, l)
} else {
b = b[:l]
}
if _, ctxErr = io.ReadFull(c, b); ctxErr != nil {
return
}
if a.IP != nil {
copy(a.IP, b)
} else {
a.Name = string(b[:len(b)-2])
}
a.Port = int(b[len(b)-2])<<8 | int(b[len(b)-1])
return &a, nil
}
func splitHostPort(address string) (string, int, error) {
host, port, err := net.SplitHostPort(address)
if err != nil {
return "", 0, err
}
portnum, err := strconv.Atoi(port)
if err != nil {
return "", 0, err
}
if 1 > portnum || portnum > 0xffff {
return "", 0, errors.New("port number out of range " + port)
}
return host, portnum, nil
}
+317
View File
@@ -0,0 +1,317 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package socks provides a SOCKS version 5 client implementation.
//
// SOCKS protocol version 5 is defined in RFC 1928.
// Username/Password authentication for SOCKS version 5 is defined in
// RFC 1929.
package socks
import (
"context"
"errors"
"io"
"net"
"strconv"
)
// A Command represents a SOCKS command.
type Command int
func (cmd Command) String() string {
switch cmd {
case CmdConnect:
return "socks connect"
case cmdBind:
return "socks bind"
default:
return "socks " + strconv.Itoa(int(cmd))
}
}
// An AuthMethod represents a SOCKS authentication method.
type AuthMethod int
// A Reply represents a SOCKS command reply code.
type Reply int
func (code Reply) String() string {
switch code {
case StatusSucceeded:
return "succeeded"
case 0x01:
return "general SOCKS server failure"
case 0x02:
return "connection not allowed by ruleset"
case 0x03:
return "network unreachable"
case 0x04:
return "host unreachable"
case 0x05:
return "connection refused"
case 0x06:
return "TTL expired"
case 0x07:
return "command not supported"
case 0x08:
return "address type not supported"
default:
return "unknown code: " + strconv.Itoa(int(code))
}
}
// Wire protocol constants.
const (
Version5 = 0x05
AddrTypeIPv4 = 0x01
AddrTypeFQDN = 0x03
AddrTypeIPv6 = 0x04
CmdConnect Command = 0x01 // establishes an active-open forward proxy connection
cmdBind Command = 0x02 // establishes a passive-open forward proxy connection
AuthMethodNotRequired AuthMethod = 0x00 // no authentication required
AuthMethodUsernamePassword AuthMethod = 0x02 // use username/password
AuthMethodNoAcceptableMethods AuthMethod = 0xff // no acceptable authentication methods
StatusSucceeded Reply = 0x00
)
// An Addr represents a SOCKS-specific address.
// Either Name or IP is used exclusively.
type Addr struct {
Name string // fully-qualified domain name
IP net.IP
Port int
}
func (a *Addr) Network() string { return "socks" }
func (a *Addr) String() string {
if a == nil {
return "<nil>"
}
port := strconv.Itoa(a.Port)
if a.IP == nil {
return net.JoinHostPort(a.Name, port)
}
return net.JoinHostPort(a.IP.String(), port)
}
// A Conn represents a forward proxy connection.
type Conn struct {
net.Conn
boundAddr net.Addr
}
// BoundAddr returns the address assigned by the proxy server for
// connecting to the command target address from the proxy server.
func (c *Conn) BoundAddr() net.Addr {
if c == nil {
return nil
}
return c.boundAddr
}
// A Dialer holds SOCKS-specific options.
type Dialer struct {
cmd Command // either CmdConnect or cmdBind
proxyNetwork string // network between a proxy server and a client
proxyAddress string // proxy server address
// ProxyDial specifies the optional dial function for
// establishing the transport connection.
ProxyDial func(context.Context, string, string) (net.Conn, error)
// AuthMethods specifies the list of request authentication
// methods.
// If empty, SOCKS client requests only AuthMethodNotRequired.
AuthMethods []AuthMethod
// Authenticate specifies the optional authentication
// function. It must be non-nil when AuthMethods is not empty.
// It must return an error when the authentication is failed.
Authenticate func(context.Context, io.ReadWriter, AuthMethod) error
}
// DialContext connects to the provided address on the provided
// network.
//
// The returned error value may be a net.OpError. When the Op field of
// net.OpError contains "socks", the Source field contains a proxy
// server address and the Addr field contains a command target
// address.
//
// See func Dial of the net package of standard library for a
// description of the network and address parameters.
func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
if err := d.validateTarget(network, address); err != nil {
proxy, dst, _ := d.pathAddrs(address)
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
}
if ctx == nil {
proxy, dst, _ := d.pathAddrs(address)
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")}
}
var err error
var c net.Conn
if d.ProxyDial != nil {
c, err = d.ProxyDial(ctx, d.proxyNetwork, d.proxyAddress)
} else {
var dd net.Dialer
c, err = dd.DialContext(ctx, d.proxyNetwork, d.proxyAddress)
}
if err != nil {
proxy, dst, _ := d.pathAddrs(address)
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
}
a, err := d.connect(ctx, c, address)
if err != nil {
c.Close()
proxy, dst, _ := d.pathAddrs(address)
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
}
return &Conn{Conn: c, boundAddr: a}, nil
}
// DialWithConn initiates a connection from SOCKS server to the target
// network and address using the connection c that is already
// connected to the SOCKS server.
//
// It returns the connection's local address assigned by the SOCKS
// server.
func (d *Dialer) DialWithConn(ctx context.Context, c net.Conn, network, address string) (net.Addr, error) {
if err := d.validateTarget(network, address); err != nil {
proxy, dst, _ := d.pathAddrs(address)
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
}
if ctx == nil {
proxy, dst, _ := d.pathAddrs(address)
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")}
}
a, err := d.connect(ctx, c, address)
if err != nil {
proxy, dst, _ := d.pathAddrs(address)
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
}
return a, nil
}
// Dial connects to the provided address on the provided network.
//
// Unlike DialContext, it returns a raw transport connection instead
// of a forward proxy connection.
//
// Deprecated: Use DialContext or DialWithConn instead.
func (d *Dialer) Dial(network, address string) (net.Conn, error) {
if err := d.validateTarget(network, address); err != nil {
proxy, dst, _ := d.pathAddrs(address)
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
}
var err error
var c net.Conn
if d.ProxyDial != nil {
c, err = d.ProxyDial(context.Background(), d.proxyNetwork, d.proxyAddress)
} else {
c, err = net.Dial(d.proxyNetwork, d.proxyAddress)
}
if err != nil {
proxy, dst, _ := d.pathAddrs(address)
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
}
if _, err := d.DialWithConn(context.Background(), c, network, address); err != nil {
c.Close()
return nil, err
}
return c, nil
}
func (d *Dialer) validateTarget(network, address string) error {
switch network {
case "tcp", "tcp6", "tcp4":
default:
return errors.New("network not implemented")
}
switch d.cmd {
case CmdConnect, cmdBind:
default:
return errors.New("command not implemented")
}
return nil
}
func (d *Dialer) pathAddrs(address string) (proxy, dst net.Addr, err error) {
for i, s := range []string{d.proxyAddress, address} {
host, port, err := splitHostPort(s)
if err != nil {
return nil, nil, err
}
a := &Addr{Port: port}
a.IP = net.ParseIP(host)
if a.IP == nil {
a.Name = host
}
if i == 0 {
proxy = a
} else {
dst = a
}
}
return
}
// NewDialer returns a new Dialer that dials through the provided
// proxy server's network and address.
func NewDialer(network, address string) *Dialer {
return &Dialer{proxyNetwork: network, proxyAddress: address, cmd: CmdConnect}
}
const (
authUsernamePasswordVersion = 0x01
authStatusSucceeded = 0x00
)
// UsernamePassword are the credentials for the username/password
// authentication method.
type UsernamePassword struct {
Username string
Password string
}
// Authenticate authenticates a pair of username and password with the
// proxy server.
func (up *UsernamePassword) Authenticate(ctx context.Context, rw io.ReadWriter, auth AuthMethod) error {
switch auth {
case AuthMethodNotRequired:
return nil
case AuthMethodUsernamePassword:
if len(up.Username) == 0 || len(up.Username) > 255 || len(up.Password) > 255 {
return errors.New("invalid username/password")
}
b := []byte{authUsernamePasswordVersion}
b = append(b, byte(len(up.Username)))
b = append(b, up.Username...)
b = append(b, byte(len(up.Password)))
b = append(b, up.Password...)
// TODO(mikio): handle IO deadlines and cancelation if
// necessary
if _, err := rw.Write(b); err != nil {
return err
}
if _, err := io.ReadFull(rw, b[:2]); err != nil {
return err
}
if b[0] != authUsernamePasswordVersion {
return errors.New("invalid username/password version")
}
if b[1] != authStatusSucceeded {
return errors.New("username/password authentication failed")
}
return nil
}
return errors.New("unsupported authentication method " + strconv.Itoa(int(auth)))
}
+54
View File
@@ -0,0 +1,54 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package proxy
import (
"context"
"net"
)
// A ContextDialer dials using a context.
type ContextDialer interface {
DialContext(ctx context.Context, network, address string) (net.Conn, error)
}
// Dial works like DialContext on net.Dialer but using a dialer returned by FromEnvironment.
//
// The passed ctx is only used for returning the Conn, not the lifetime of the Conn.
//
// Custom dialers (registered via RegisterDialerType) that do not implement ContextDialer
// can leak a goroutine for as long as it takes the underlying Dialer implementation to timeout.
//
// A Conn returned from a successful Dial after the context has been cancelled will be immediately closed.
func Dial(ctx context.Context, network, address string) (net.Conn, error) {
d := FromEnvironment()
if xd, ok := d.(ContextDialer); ok {
return xd.DialContext(ctx, network, address)
}
return dialContext(ctx, d, network, address)
}
// WARNING: this can leak a goroutine for as long as the underlying Dialer implementation takes to timeout
// A Conn returned from a successful Dial after the context has been cancelled will be immediately closed.
func dialContext(ctx context.Context, d Dialer, network, address string) (net.Conn, error) {
var (
conn net.Conn
done = make(chan struct{}, 1)
err error
)
go func() {
conn, err = d.Dial(network, address)
close(done)
if conn != nil && ctx.Err() != nil {
conn.Close()
}
}()
select {
case <-ctx.Done():
err = ctx.Err()
case <-done:
}
return conn, err
}
+31
View File
@@ -0,0 +1,31 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package proxy
import (
"context"
"net"
)
type direct struct{}
// Direct implements Dialer by making network connections directly using net.Dial or net.DialContext.
var Direct = direct{}
var (
_ Dialer = Direct
_ ContextDialer = Direct
)
// Dial directly invokes net.Dial with the supplied parameters.
func (direct) Dial(network, addr string) (net.Conn, error) {
return net.Dial(network, addr)
}
// DialContext instantiates a net.Dialer and invokes its DialContext receiver with the supplied parameters.
func (direct) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, network, addr)
}
+151
View File
@@ -0,0 +1,151 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package proxy
import (
"context"
"net"
"strings"
)
// A PerHost directs connections to a default Dialer unless the host name
// requested matches one of a number of exceptions.
type PerHost struct {
def, bypass Dialer
bypassNetworks []*net.IPNet
bypassIPs []net.IP
bypassZones []string
bypassHosts []string
}
// NewPerHost returns a PerHost Dialer that directs connections to either
// defaultDialer or bypass, depending on whether the connection matches one of
// the configured rules.
func NewPerHost(defaultDialer, bypass Dialer) *PerHost {
return &PerHost{
def: defaultDialer,
bypass: bypass,
}
}
// Dial connects to the address addr on the given network through either
// defaultDialer or bypass.
func (p *PerHost) Dial(network, addr string) (c net.Conn, err error) {
host, _, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
return p.dialerForRequest(host).Dial(network, addr)
}
// DialContext connects to the address addr on the given network through either
// defaultDialer or bypass.
func (p *PerHost) DialContext(ctx context.Context, network, addr string) (c net.Conn, err error) {
host, _, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
d := p.dialerForRequest(host)
if x, ok := d.(ContextDialer); ok {
return x.DialContext(ctx, network, addr)
}
return dialContext(ctx, d, network, addr)
}
func (p *PerHost) dialerForRequest(host string) Dialer {
if ip := net.ParseIP(host); ip != nil {
for _, net := range p.bypassNetworks {
if net.Contains(ip) {
return p.bypass
}
}
for _, bypassIP := range p.bypassIPs {
if bypassIP.Equal(ip) {
return p.bypass
}
}
return p.def
}
for _, zone := range p.bypassZones {
if strings.HasSuffix(host, zone) {
return p.bypass
}
if host == zone[1:] {
// For a zone ".example.com", we match "example.com"
// too.
return p.bypass
}
}
for _, bypassHost := range p.bypassHosts {
if bypassHost == host {
return p.bypass
}
}
return p.def
}
// AddFromString parses a string that contains comma-separated values
// specifying hosts that should use the bypass proxy. Each value is either an
// IP address, a CIDR range, a zone (*.example.com) or a host name
// (localhost). A best effort is made to parse the string and errors are
// ignored.
func (p *PerHost) AddFromString(s string) {
hosts := strings.Split(s, ",")
for _, host := range hosts {
host = strings.TrimSpace(host)
if len(host) == 0 {
continue
}
if strings.Contains(host, "/") {
// We assume that it's a CIDR address like 127.0.0.0/8
if _, net, err := net.ParseCIDR(host); err == nil {
p.AddNetwork(net)
}
continue
}
if ip := net.ParseIP(host); ip != nil {
p.AddIP(ip)
continue
}
if strings.HasPrefix(host, "*.") {
p.AddZone(host[1:])
continue
}
p.AddHost(host)
}
}
// AddIP specifies an IP address that will use the bypass proxy. Note that
// this will only take effect if a literal IP address is dialed. A connection
// to a named host will never match an IP.
func (p *PerHost) AddIP(ip net.IP) {
p.bypassIPs = append(p.bypassIPs, ip)
}
// AddNetwork specifies an IP range that will use the bypass proxy. Note that
// this will only take effect if a literal IP address is dialed. A connection
// to a named host will never match.
func (p *PerHost) AddNetwork(net *net.IPNet) {
p.bypassNetworks = append(p.bypassNetworks, net)
}
// AddZone specifies a DNS suffix that will use the bypass proxy. A zone of
// "example.com" matches "example.com" and all of its subdomains.
func (p *PerHost) AddZone(zone string) {
zone = strings.TrimSuffix(zone, ".")
if !strings.HasPrefix(zone, ".") {
zone = "." + zone
}
p.bypassZones = append(p.bypassZones, zone)
}
// AddHost specifies a host name that will use the bypass proxy.
func (p *PerHost) AddHost(host string) {
host = strings.TrimSuffix(host, ".")
p.bypassHosts = append(p.bypassHosts, host)
}
+149
View File
@@ -0,0 +1,149 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package proxy provides support for a variety of protocols to proxy network
// data.
package proxy // import "golang.org/x/net/proxy"
import (
"errors"
"net"
"net/url"
"os"
"sync"
)
// A Dialer is a means to establish a connection.
// Custom dialers should also implement ContextDialer.
type Dialer interface {
// Dial connects to the given address via the proxy.
Dial(network, addr string) (c net.Conn, err error)
}
// Auth contains authentication parameters that specific Dialers may require.
type Auth struct {
User, Password string
}
// FromEnvironment returns the dialer specified by the proxy-related
// variables in the environment and makes underlying connections
// directly.
func FromEnvironment() Dialer {
return FromEnvironmentUsing(Direct)
}
// FromEnvironmentUsing returns the dialer specify by the proxy-related
// variables in the environment and makes underlying connections
// using the provided forwarding Dialer (for instance, a *net.Dialer
// with desired configuration).
func FromEnvironmentUsing(forward Dialer) Dialer {
allProxy := allProxyEnv.Get()
if len(allProxy) == 0 {
return forward
}
proxyURL, err := url.Parse(allProxy)
if err != nil {
return forward
}
proxy, err := FromURL(proxyURL, forward)
if err != nil {
return forward
}
noProxy := noProxyEnv.Get()
if len(noProxy) == 0 {
return proxy
}
perHost := NewPerHost(proxy, forward)
perHost.AddFromString(noProxy)
return perHost
}
// proxySchemes is a map from URL schemes to a function that creates a Dialer
// from a URL with such a scheme.
var proxySchemes map[string]func(*url.URL, Dialer) (Dialer, error)
// RegisterDialerType takes a URL scheme and a function to generate Dialers from
// a URL with that scheme and a forwarding Dialer. Registered schemes are used
// by FromURL.
func RegisterDialerType(scheme string, f func(*url.URL, Dialer) (Dialer, error)) {
if proxySchemes == nil {
proxySchemes = make(map[string]func(*url.URL, Dialer) (Dialer, error))
}
proxySchemes[scheme] = f
}
// FromURL returns a Dialer given a URL specification and an underlying
// Dialer for it to make network requests.
func FromURL(u *url.URL, forward Dialer) (Dialer, error) {
var auth *Auth
if u.User != nil {
auth = new(Auth)
auth.User = u.User.Username()
if p, ok := u.User.Password(); ok {
auth.Password = p
}
}
switch u.Scheme {
case "socks5", "socks5h":
addr := u.Hostname()
port := u.Port()
if port == "" {
port = "1080"
}
return SOCKS5("tcp", net.JoinHostPort(addr, port), auth, forward)
}
// If the scheme doesn't match any of the built-in schemes, see if it
// was registered by another package.
if proxySchemes != nil {
if f, ok := proxySchemes[u.Scheme]; ok {
return f(u, forward)
}
}
return nil, errors.New("proxy: unknown scheme: " + u.Scheme)
}
var (
allProxyEnv = &envOnce{
names: []string{"ALL_PROXY", "all_proxy"},
}
noProxyEnv = &envOnce{
names: []string{"NO_PROXY", "no_proxy"},
}
)
// envOnce looks up an environment variable (optionally by multiple
// names) once. It mitigates expensive lookups on some platforms
// (e.g. Windows).
// (Borrowed from net/http/transport.go)
type envOnce struct {
names []string
once sync.Once
val string
}
func (e *envOnce) Get() string {
e.once.Do(e.init)
return e.val
}
func (e *envOnce) init() {
for _, n := range e.names {
e.val = os.Getenv(n)
if e.val != "" {
return
}
}
}
// reset is used by tests
func (e *envOnce) reset() {
e.once = sync.Once{}
e.val = ""
}
+42
View File
@@ -0,0 +1,42 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package proxy
import (
"context"
"net"
"golang.org/x/net/internal/socks"
)
// SOCKS5 returns a Dialer that makes SOCKSv5 connections to the given
// address with an optional username and password.
// See RFC 1928 and RFC 1929.
func SOCKS5(network, address string, auth *Auth, forward Dialer) (Dialer, error) {
d := socks.NewDialer(network, address)
if forward != nil {
if f, ok := forward.(ContextDialer); ok {
d.ProxyDial = func(ctx context.Context, network string, address string) (net.Conn, error) {
return f.DialContext(ctx, network, address)
}
} else {
d.ProxyDial = func(ctx context.Context, network string, address string) (net.Conn, error) {
return dialContext(ctx, forward, network, address)
}
}
}
if auth != nil {
up := socks.UsernamePassword{
Username: auth.User,
Password: auth.Password,
}
d.AuthMethods = []socks.AuthMethod{
socks.AuthMethodNotRequired,
socks.AuthMethodUsernamePassword,
}
d.Authenticate = up.Authenticate
}
return d, nil
}
+2
View File
@@ -67,6 +67,8 @@ github.com/skip2/go-qrcode/reedsolomon
golang.org/x/crypto/curve25519
# golang.org/x/net v0.27.0
## explicit; go 1.18
golang.org/x/net/internal/socks
golang.org/x/net/proxy
golang.org/x/net/publicsuffix
# golang.org/x/sys v0.30.0
## explicit; go 1.18
+26 -5
View File
@@ -122,6 +122,27 @@ asset_name() {
printf '%s-linux-%s.tar.gz' "$APP_NAME" "$ARCH"
}
verify_runtime_deps() {
local cmd
for cmd in curl tar ffmpeg ffprobe openssl python3; do
command -v "$cmd" >/dev/null 2>&1 || die "missing command: $cmd"
done
python3 - <<'PY' || die "missing Python modules for 91Spider: requests, bs4, lxml, socks"
import importlib.util
import sys
missing = [
name
for name in ("requests", "bs4", "lxml", "socks")
if importlib.util.find_spec(name) is None
]
if missing:
print("missing Python modules: " + ", ".join(missing), file=sys.stderr)
sys.exit(1)
PY
}
install_deps() {
if [[ "$INSTALL_DEPS" != "1" ]]; then
return
@@ -130,13 +151,12 @@ install_deps() {
export DEBIAN_FRONTEND=noninteractive
log "installing runtime dependencies"
apt-get update
apt-get install -y ca-certificates curl tar ffmpeg openssl iproute2 python3 python3-requests python3-bs4 python3-lxml
apt-get install -y ca-certificates curl tar ffmpeg openssl iproute2 python3 python3-requests python3-bs4 python3-lxml python3-socks
verify_runtime_deps
return
fi
for cmd in curl tar ffmpeg ffprobe openssl; do
command -v "$cmd" >/dev/null 2>&1 || die "missing command: $cmd"
done
verify_runtime_deps
}
check_system() {
@@ -689,13 +709,14 @@ update_app() {
exec_latest_manager_update
fi
install_deps
if should_skip_update; then
log "already up to date; skipped app update"
return 0
fi
check_disk_space
install_deps
local backup
backup="$(mktemp -d)"
+1
View File
@@ -7,6 +7,7 @@
"": {
"name": "video-site",
"version": "0.1.0",
"license": "MIT",
"dependencies": {
"lucide-react": "0.453.0",
"react": "18.3.1",
+1
View File
@@ -1,6 +1,7 @@
{
"name": "video-site",
"private": true,
"license": "MIT",
"version": "0.1.0",
"type": "module",
"scripts": {
+342 -42
View File
@@ -1,5 +1,6 @@
import { useCallback, useEffect, useMemo, useState } from "react";
import {
AlertTriangle,
ArrowLeft,
ChevronDown,
ChevronRight,
@@ -10,6 +11,7 @@ import {
Plus,
Power,
PowerOff,
QrCode,
RefreshCw,
RotateCcw,
Trash2,
@@ -23,6 +25,7 @@ import { makeUniqueDriveId } from "./driveId";
const kindLabel: Record<string, string> = {
quark: "夸克网盘",
p115: "115 网盘",
p123: "123 云盘",
pikpak: "PikPak",
wopan: "联通沃盘",
onedrive: "OneDrive",
@@ -90,12 +93,14 @@ export function DrivesPage() {
useState<api.NightlyJobStatus>(idleNightlyStatus);
const [loading, setLoading] = useState(true);
const [modalOpen, setModalOpen] = useState(false);
const [deleteTarget, setDeleteTarget] = useState<api.AdminDrive | null>(null);
const [form, setForm] = useState<FormState>(emptyForm);
const [saving, setSaving] = useState(false);
const [deletingId, setDeletingId] = useState("");
const [regenFailedId, setRegenFailedId] = useState("");
// 与 regenFailedId 并列:失败封面重新入队按钮的 disable 状态。两套独立按钮 →
// 两个 state 互不阻塞,避免操作 teaser 时锁住封面那条按钮(反之亦然)。
// 失败重试按钮各自维护 pending 状态,避免操作 teaser / 封面 / 指纹时互相锁住。
const [regenFailedThumbId, setRegenFailedThumbId] = useState("");
const [regenFailedFingerprintId, setRegenFailedFingerprintId] = useState("");
// togglingTeaserId 在请求未返回前禁用按钮,避免连点导致两次切换互相覆盖。
const [togglingTeaserId, setTogglingTeaserId] = useState("");
const [scanningAll, setScanningAll] = useState(false);
@@ -169,7 +174,7 @@ export function DrivesPage() {
kind: d.kind,
name: d.name,
rootId: d.rootId,
creds: {},
creds: d.kind === "spider91" ? { proxy: d.spider91Proxy ?? "" } : {},
spider91UploadDriveId: settings?.spider91UploadDriveId ?? "",
});
setModalOpen(true);
@@ -185,7 +190,9 @@ export function DrivesPage() {
const driveID = existing
? form.id
: makeUniqueDriveId(form.kind, name, list);
const rootId = form.rootId.trim() || defaultRootId(form.kind);
const rootId = usesRootDirectoryID(form.kind)
? form.rootId.trim() || defaultRootId(form.kind)
: defaultRootId(form.kind);
// 若编辑且没有提供凭证,提示一下但仍允许保存(不改凭证)
setSaving(true);
try {
@@ -233,14 +240,22 @@ export function DrivesPage() {
}
}
async function handleDelete(d: api.AdminDrive) {
if (!window.confirm(`确定删除 ${d.name || d.id}\n这会移除盘配置,但不会删除其中的视频元数据。`)) return;
async function confirmDeleteDrive() {
if (!deleteTarget) return;
const d = deleteTarget;
setDeletingId(d.id);
try {
await api.deleteDrive(d.id);
show("已删除", "success");
const resp = await api.deleteDrive(d.id, { deleteVideos: true });
show(`已删除,并清理 ${resp.deletedVideos ?? 0} 个视频`, "success");
setDeleteTarget(null);
if (selectedDriveId === d.id) {
setSelectedDriveId(null);
}
refresh();
} catch (e) {
show(e instanceof Error ? e.message : "删除失败", "error");
} finally {
setDeletingId("");
}
}
@@ -310,6 +325,19 @@ export function DrivesPage() {
}
}
async function handleRegenFailedFingerprints(d: api.AdminDrive) {
setRegenFailedFingerprintId(d.id);
try {
await api.regenFailedFingerprints(d.id);
show("已触发失败指纹重新生成", "success");
refresh();
} catch (e) {
show(e instanceof Error ? e.message : "触发失败", "error");
} finally {
setRegenFailedFingerprintId("");
}
}
async function handleToggleTeaser(d: api.AdminDrive) {
const next = !d.teaserEnabled;
setTogglingTeaserId(d.id);
@@ -351,6 +379,19 @@ export function DrivesPage() {
return selectedDriveId ? list.find((d) => d.id === selectedDriveId) : null;
}, [selectedDriveId, list]);
const deleteModal = (
<DeleteDriveModal
drive={deleteTarget}
deleting={deletingId === deleteTarget?.id}
onCancel={() => {
if (!deletingId) {
setDeleteTarget(null);
}
}}
onConfirm={confirmDeleteDrive}
/>
);
if (selectedDriveId && selectedDrive) {
const d = selectedDrive;
const driveStorage = storage?.drives[d.id];
@@ -395,7 +436,7 @@ export function DrivesPage() {
<span className="admin-detail-label"> ID</span>
<span className="admin-detail-value admin-mono-cell">{d.id}</span>
</div>
{d.kind !== "spider91" && (
{usesRootDirectoryID(d.kind) && (
<>
<div className="admin-detail-row">
<span className="admin-detail-label"> ID</span>
@@ -436,10 +477,7 @@ export function DrivesPage() {
<button className="admin-btn" onClick={() => openEdit(d)}>
{d.kind === "spider91" ? "编辑配置" : "编辑配置凭证"}
</button>
<button className="admin-btn is-danger" onClick={() => {
handleDelete(d);
setSelectedDriveId(null);
}} style={{ marginLeft: "auto" }}>
<button className="admin-btn is-danger" onClick={() => setDeleteTarget(d)} style={{ marginLeft: "auto" }}>
<Trash2 size={13} />
</button>
</div>
@@ -479,14 +517,14 @@ export function DrivesPage() {
style={{ padding: "4px 10px", fontSize: "11px" }}
>
{d.teaserEnabled ? <Power size={11} /> : <PowerOff size={11} />}
<span>{d.teaserEnabled ? "Teaser: 开" : "Teaser: 关"}</span>
<span>{d.teaserEnabled ? "预览视频生成:开" : "预览视频生成:关"}</span>
</button>
</div>
</header>
<div className="admin-detail-grid">
<div className="admin-detail-row">
<span className="admin-detail-label"></span>
<span className="admin-detail-label"></span>
<div className="admin-detail-value">
<GenerationStatusLine label="封面" status={d.thumbnailGenerationStatus} />
</div>
@@ -503,13 +541,13 @@ export function DrivesPage() {
</div>
</div>
<div className="admin-detail-row">
<span className="admin-detail-label">Teaser </span>
<span className="admin-detail-label"></span>
<div className="admin-detail-value">
<GenerationStatusLine label="预览" status={d.previewGenerationStatus} />
</div>
</div>
<div className="admin-detail-row">
<span className="admin-detail-label">Teaser </span>
<span className="admin-detail-label"></span>
<div className="admin-detail-value">
<GenerationCounts
ready={d.teaserReadyCount}
@@ -519,13 +557,13 @@ export function DrivesPage() {
</div>
</div>
<div className="admin-detail-row">
<span className="admin-detail-label"></span>
<span className="admin-detail-label"></span>
<div className="admin-detail-value">
<GenerationStatusLine label="指纹" status={d.fingerprintGenerationStatus} />
</div>
</div>
<div className="admin-detail-row">
<span className="admin-detail-label"></span>
<span className="admin-detail-label"></span>
<div className="admin-detail-value">
<GenerationCounts
ready={d.fingerprintReadyCount}
@@ -537,14 +575,6 @@ export function DrivesPage() {
</div>
<div className="admin-detail-actions">
<button
className="admin-btn"
disabled={(d.teaserFailedCount ?? 0) <= 0 || regenFailedId === d.id}
onClick={() => handleRegenFailed(d)}
>
<RotateCcw size={13} />
<span> Teaser</span>
</button>
<button
className="admin-btn"
disabled={(d.thumbnailFailedCount ?? 0) <= 0 || regenFailedThumbId === d.id}
@@ -553,6 +583,25 @@ export function DrivesPage() {
<RotateCcw size={13} />
<span></span>
</button>
<button
className="admin-btn"
disabled={(d.teaserFailedCount ?? 0) <= 0 || regenFailedId === d.id}
onClick={() => handleRegenFailed(d)}
>
<RotateCcw size={13} />
<span></span>
</button>
<button
className="admin-btn"
disabled={
(d.fingerprintFailedCount ?? 0) <= 0 ||
regenFailedFingerprintId === d.id
}
onClick={() => handleRegenFailedFingerprints(d)}
>
<RotateCcw size={13} />
<span></span>
</button>
</div>
</div>
@@ -570,7 +619,7 @@ export function DrivesPage() {
<span className="admin-detail-value">{formatBytes(driveStorage?.thumbnailBytes ?? 0)}</span>
</div>
<div className="admin-detail-row">
<span className="admin-detail-label">Teaser </span>
<span className="admin-detail-label"></span>
<span className="admin-detail-value">{formatBytes(driveStorage?.teaserBytes ?? 0)}</span>
</div>
<div className="admin-detail-row">
@@ -610,6 +659,7 @@ export function DrivesPage() {
uploadTargets={uploadTargets}
/>
</Modal>
{deleteModal}
</section>
);
}
@@ -727,6 +777,7 @@ export function DrivesPage() {
uploadTargets={uploadTargets}
/>
</Modal>
{deleteModal}
</section>
);
}
@@ -739,7 +790,7 @@ function StorageSummary({ storage }: { storage: api.AdminDriveStorage }) {
<strong>{formatBytes(storage.thumbnailBytes)}</strong>
</div>
<div className="admin-storage-summary__metric">
<span>Teaser </span>
<span></span>
<strong>{formatBytes(storage.teaserBytes)}</strong>
</div>
<div className="admin-storage-summary__metric">
@@ -894,6 +945,75 @@ function StatusTag({
return <span className="admin-status">{status || "未连接"}</span>;
}
function DeleteDriveModal({
drive,
deleting,
onCancel,
onConfirm,
}: {
drive: api.AdminDrive | null;
deleting: boolean;
onCancel: () => void;
onConfirm: () => void;
}) {
const name = drive?.name || drive?.id || "";
const isSpider91 = drive?.kind === "spider91";
const isLocalStorage = drive?.kind === "localstorage";
const title = isSpider91 ? "删除 91Spider" : "删除存储";
const primaryText = deleting ? "删除中..." : "确认删除并清理";
return (
<Modal
open={!!drive}
title={title}
onClose={onCancel}
footer={
<>
<button className="admin-btn" onClick={onCancel} disabled={deleting}>
</button>
<button className="admin-btn is-danger" onClick={onConfirm} disabled={deleting}>
<Trash2 size={13} />
{primaryText}
</button>
</>
}
>
<div className="admin-delete-confirm">
<div className="admin-delete-confirm__icon">
<AlertTriangle size={20} />
</div>
<div className="admin-delete-confirm__content">
<p className="admin-delete-confirm__title">
{isSpider91
? `确定删除「${name}」吗?`
: `确定删除「${name}」并清理该存储的视频数据吗?`}
</p>
<p className="admin-delete-confirm__text">
</p>
<ul className="admin-delete-confirm__list">
<li></li>
<li></li>
<li></li>
{isSpider91 && (
<li> 91Spider 91 </li>
)}
{isLocalStorage && (
<li></li>
)}
</ul>
{!isSpider91 && !isLocalStorage && (
<p className="admin-delete-confirm__text">
</p>
)}
</div>
</div>
</Modal>
);
}
function DriveForm({
form,
onChange,
@@ -941,6 +1061,7 @@ function DriveForm({
disabled={isEdit}
>
<option value="p115">115 </option>
<option value="p123">123 </option>
<option value="pikpak">PikPak</option>
<option value="onedrive">OneDrive</option>
<option value="googledrive">Google Drive</option>
@@ -950,17 +1071,19 @@ function DriveForm({
<option value="wopan"></option>
</select>
</div>
<div className="admin-form__row">
<label> ID</label>
<input
value={form.rootId}
onChange={(e) => set("rootId", e.target.value)}
placeholder={rootIdPlaceholder(form.kind)}
/>
<div className="admin-form__help">
使ID获取方式请参考OpenList文档
{usesRootDirectoryID(form.kind) && (
<div className="admin-form__row">
<label> ID</label>
<input
value={form.rootId}
onChange={(e) => set("rootId", e.target.value)}
placeholder={rootIdPlaceholder(form.kind)}
/>
<div className="admin-form__help">
使ID获取方式请参考OpenList文档
</div>
</div>
</div>
)}
{(help || fields.length > 0) && (
<>
@@ -972,6 +1095,12 @@ function DriveForm({
</div>
)}
{form.kind === "p123" && (
<P123QRCodeLogin
onToken={(token) => setCred("access_token", token)}
/>
)}
{fields.map((f) => (
<div key={f.key} className="admin-form__row">
<label>{f.label}{f.required && " *"}</label>
@@ -1008,6 +1137,144 @@ function DriveForm({
);
}
function P123QRCodeLogin({ onToken }: { onToken: (token: string) => void }) {
const { show } = useToast();
const [session, setSession] = useState<api.P123QRSession | null>(null);
const [status, setStatus] = useState<api.P123QRStatus | null>(null);
const [starting, setStarting] = useState(false);
const [pollingError, setPollingError] = useState("");
const [completed, setCompleted] = useState(false);
async function start() {
setStarting(true);
setPollingError("");
setCompleted(false);
setStatus(null);
try {
const next = await api.startP123QRLogin();
setSession(next);
} catch (e) {
setSession(null);
show(e instanceof Error ? e.message : "生成二维码失败", "error");
} finally {
setStarting(false);
}
}
useEffect(() => {
if (!session || completed) return;
const activeSession = session;
let stopped = false;
let inFlight = false;
let timer: number | undefined;
async function poll() {
if (stopped || inFlight) return;
inFlight = true;
try {
const next = await api.getP123QRStatus(activeSession.uniID, activeSession.loginUuid);
if (stopped) return;
setStatus(next);
setPollingError("");
if (next.accessToken) {
stopped = true;
if (timer) window.clearInterval(timer);
setCompleted(true);
onToken(next.accessToken);
show("扫码成功,已填入 access_token,保存后生效", "success");
return;
}
if (next.loginStatus === 2 || next.loginStatus === 4) {
stopped = true;
if (timer) window.clearInterval(timer);
}
} catch (e) {
if (stopped) return;
setPollingError(e instanceof Error ? e.message : "查询扫码状态失败");
} finally {
inFlight = false;
}
}
poll();
timer = window.setInterval(poll, 1800);
return () => {
stopped = true;
if (timer) window.clearInterval(timer);
};
}, [session, completed, onToken, show]);
const statusText = completed
? "已获取 token"
: pollingError || status?.statusText || (session ? "等待扫码" : "未生成二维码");
const statusClass = p123QRStatusClass(status, completed, pollingError);
const platform = status?.platformText ? ` · ${status.platformText}` : "";
return (
<div className="admin-form__row">
<label></label>
<div className="admin-p123-qr">
<div className="admin-p123-qr__actions">
<button
type="button"
className="admin-btn"
onClick={start}
disabled={starting}
>
<QrCode size={14} />
{starting ? "生成中..." : session ? "重新生成二维码" : "生成二维码"}
</button>
<span className={`admin-status ${statusClass}`}>
{statusText}
{platform}
</span>
</div>
{session && (
<div className="admin-p123-qr__body">
<img
className="admin-p123-qr__image"
src={session.qrImageDataUrl}
alt="123 云盘扫码登录二维码"
/>
<div className="admin-p123-qr__meta">
<div className="admin-form__help">
使 123 App access_token
</div>
{session.expiresAt && (
<div className="admin-form__help">
{new Date(session.expiresAt).toLocaleTimeString("zh-CN", {
hour: "2-digit",
minute: "2-digit",
second: "2-digit",
})}
</div>
)}
{(status?.loginStatus === 2 || status?.loginStatus === 4) && (
<div className="admin-form__help">
{status.loginStatus === 2 ? "已被拒绝" : "已过期"}
</div>
)}
</div>
</div>
)}
</div>
</div>
);
}
function p123QRStatusClass(
status: api.P123QRStatus | null,
completed: boolean,
error: string
): string {
if (completed || status?.loginStatus === 3) return "is-ok";
if (error || status?.loginStatus === 2 || status?.loginStatus === 4) {
return "is-error";
}
return "is-pending";
}
/**
* Spider91UploadTargetField spider91 drive "上传目标"
*
@@ -1052,6 +1319,8 @@ function credentialHelp(kind: Kind, isEdit: boolean): string {
return `在 pan.quark.cn 登录后,F12 → Network → 任意请求 → Request Headers 里复制整段 Cookie 粘贴到下方。${note}`;
case "p115":
return `登录 115.com 后复制 Cookie,形如 "UID=...; CID=...; SEID=...; KID=..."。${note}`;
case "p123":
return `推荐使用扫码登录自动获取 access_token;账号密码登录被 123 云盘风控拦截时,也可以只填写 access_token。播放走 302 跳转到 123 云盘返回的短期 CDN 地址。${note}`;
case "pikpak":
return `填写 PikPak 账号和密码即可。平台、设备 ID、验证码 token 和 refresh token 会由服务端自动处理并保存。${note}`;
case "wopan":
@@ -1063,7 +1332,7 @@ function credentialHelp(kind: Kind, isEdit: boolean): string {
case "localstorage":
return `把服务器上的一个已有目录作为视频来源扫描。填写绝对路径,例如 /mnt/videos;系统会读取该目录及子目录中的视频,并生成封面、Teaser 和指纹。${note}`;
case "spider91":
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;它不是外部网盘,不需要填写 Cookie 或目录 ID。后续流水线会把较早的视频上传到你选择的 115 / PikPak / OneDrive 目标盘。";
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;可按服务器网络情况单独配置代理。后续流水线会把较早的视频上传到你选择的 115 / PikPak / OneDrive 目标盘。";
default:
return "";
}
@@ -1098,6 +1367,26 @@ function credentialFields(kind: Kind): Array<{
required: true,
},
];
case "p123":
return [
{
key: "username",
label: "用户名 / 邮箱(可选)",
placeholder: "user@example.com",
},
{
key: "password",
label: "密码(可选)",
placeholder: "123 云盘密码",
},
{
key: "access_token",
label: "access_token(推荐用于风控场景)",
placeholder: "Bearer eyJ... 或直接粘贴 token",
multiline: true,
help: "扫码成功后会自动填入该字段;如果 token 过期,重新扫码后保存即可。",
},
];
case "pikpak":
return [
{
@@ -1164,7 +1453,14 @@ function credentialFields(kind: Kind): Array<{
},
];
case "spider91":
return [];
return [
{
key: "proxy",
label: "代理地址(可选)",
placeholder: "http://127.0.0.1:7890",
help: "仅用于 91Spider 的列表/详情请求和视频、封面下载;留空则使用服务器环境变量 HTTP_PROXY / HTTPS_PROXY 或直连。支持 http://、https://、socks5:// 或 socks5h://。",
},
];
}
}
@@ -1177,6 +1473,10 @@ function defaultRootId(kind: Kind): string {
return "0";
}
function usesRootDirectoryID(kind: Kind): boolean {
return kind !== "localstorage" && kind !== "spider91";
}
function rootIdPlaceholder(kind: Kind): string {
const rootId = defaultRootId(kind);
return rootId ? `默认:${rootId}` : "留空表示根目录";
@@ -1310,7 +1610,7 @@ function SelectedDirsChips({
className="admin-text-faint"
style={{ fontSize: "13px", padding: "6px 0" }}
>
{drive.kind === "p115" ? "115 网盘" : drive.kind}{" "}
{kindLabel[drive.kind] ?? drive.kind}{" "}
</div>
);
+45 -4
View File
@@ -77,7 +77,7 @@ export function checkUpdate() {
export type AdminDrive = {
id: string;
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
kind: "quark" | "p115" | "p123" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
name: string;
rootId: string;
status: string;
@@ -93,6 +93,8 @@ export type AdminDrive = {
skipDirIds: string[];
// spider91 上次成功爬取时间(unix 秒);其它 kind 留空。
lastCrawlAt?: number;
// spider91 专用代理地址;仅后台管理接口返回,用于编辑表单回显。
spider91Proxy?: string;
thumbnailGenerationStatus?: DriveGenerationStatus;
previewGenerationStatus?: DriveGenerationStatus;
fingerprintGenerationStatus?: DriveGenerationStatus;
@@ -137,7 +139,7 @@ export function getDriveStorage() {
export type UpsertDriveInput = {
id: string;
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
kind: "quark" | "p115" | "p123" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
name: string;
rootId: string;
credentials: Record<string, string>;
@@ -156,9 +158,14 @@ export function upsertDrive(body: UpsertDriveInput) {
});
}
export function deleteDrive(id: string) {
return request<{ ok: boolean }>(`/drives/${encodeURIComponent(id)}`, {
export type DeleteDriveInput = {
deleteVideos: true;
};
export function deleteDrive(id: string, body: DeleteDriveInput) {
return request<{ ok: boolean; deletedVideos: number }>(`/drives/${encodeURIComponent(id)}`, {
method: "DELETE",
body: JSON.stringify(body),
});
}
@@ -169,6 +176,33 @@ export function rescan(id: string) {
);
}
export type P123QRSession = {
loginUuid: string;
uniID: string;
qrCodeUrl: string;
qrImageDataUrl: string;
expiresAt?: string;
};
export type P123QRStatus = {
loginStatus: number;
statusText: string;
scanPlatform?: number;
platformText?: string;
accessToken?: string;
};
export function startP123QRLogin() {
return request<P123QRSession>("/drives/p123/qr", { method: "POST" });
}
export function getP123QRStatus(uniID: string, loginUuid: string) {
const qs = new URLSearchParams({ loginUuid });
return request<P123QRStatus>(
`/drives/p123/qr/${encodeURIComponent(uniID)}?${qs.toString()}`
);
}
/**
* teaser toggle
*
@@ -243,6 +277,13 @@ export function regenFailedThumbnails(id: string) {
);
}
export function regenFailedFingerprints(id: string) {
return request<{ ok: boolean }>(
`/drives/${encodeURIComponent(id)}/fingerprints/failed/regenerate`,
{ method: "POST" }
);
}
// ---------- Videos ----------
export type AdminVideo = {
+6 -3
View File
@@ -10,11 +10,14 @@ export function TagCloud() {
useEffect(() => {
let active = true;
fetchTags().then((list) => {
if (active) setTags(list);
});
const timer = window.setTimeout(() => {
fetchTags().then((list) => {
if (active) setTags(list);
});
}, 500);
return () => {
active = false;
window.clearTimeout(timer);
};
}, []);
+4 -2
View File
@@ -12,6 +12,7 @@ import { PreviewVideo } from "./PreviewVideo";
type Props = {
video: VideoItem;
priority?: boolean;
};
const HOVER_DELAY_MS = 300;
@@ -24,7 +25,7 @@ function useActivePreviewId(): string | null {
);
}
export function VideoCard({ video }: Props) {
export function VideoCard({ video, priority = false }: Props) {
const [previewState, setPreviewState] = useState<PreviewState>("idle");
const [shouldRenderPreview, setShouldRenderPreview] = useState(false);
const [progress, setProgress] = useState(0); // 0~1
@@ -204,7 +205,7 @@ export function VideoCard({ video }: Props) {
className="thumb-image"
src={thumbnailSrc}
alt={video.title}
loading="lazy"
loading={priority ? "eager" : "lazy"}
onError={handleThumbnailError}
/>
@@ -291,6 +292,7 @@ function sourceKindFromLabel(label: string): string {
const value = label.toLowerCase();
if (value.includes("夸克") || value.includes("quark")) return "quark";
if (value.includes("115") || value.includes("p115")) return "p115";
if (value.includes("123") || value.includes("p123")) return "p123";
if (value.includes("pikpak")) return "pikpak";
if (value.includes("沃盘") || value.includes("wopan") || value.includes("联通")) return "wopan";
if (value.includes("onedrive") || value.includes("one drive")) return "onedrive";
+4 -2
View File
@@ -6,6 +6,7 @@ type Props = {
loading?: boolean;
compact?: boolean;
emptyText?: string;
priorityCount?: number;
skeletonCount?: number;
};
@@ -14,6 +15,7 @@ export function VideoGrid({
loading,
compact,
emptyText = "暂时没有视频",
priorityCount = 0,
skeletonCount = 8,
}: Props) {
if (loading) {
@@ -32,8 +34,8 @@ export function VideoGrid({
return (
<div className={`video-grid ${compact ? "is-compact" : ""}`}>
{(videos ?? []).map((v) => (
<VideoCard key={v.id} video={v} />
{(videos ?? []).map((v, index) => (
<VideoCard key={v.id} video={v} priority={index < priorityCount} />
))}
</div>
);
+1
View File
@@ -68,6 +68,7 @@ function sourceKindFromLabel(label: string): string {
const value = label.toLowerCase();
if (value.includes("夸克") || value.includes("quark")) return "quark";
if (value.includes("115") || value.includes("p115")) return "p115";
if (value.includes("123") || value.includes("p123")) return "p123";
if (value.includes("pikpak")) return "pikpak";
if (value.includes("沃盘") || value.includes("wopan") || value.includes("联通"))
return "wopan";
+30 -4
View File
@@ -1,14 +1,19 @@
import type { VideoDetail, VideoItem } from "@/types";
// 真实后端接口调用。未配置网盘时,各接口返回空数据。
export function fetchHomeVideos(): Promise<VideoItem[]> {
return apiGet<VideoItem[]>("/api/home").catch(() => []);
export function fetchHomeVideos(excludeIds?: string[]): Promise<VideoItem[]> {
const qs = new URLSearchParams();
for (const id of excludeIds ?? []) {
if (id.trim()) qs.append("exclude", id.trim());
}
const suffix = qs.toString() ? `?${qs.toString()}` : "";
return apiGet<VideoItem[]>(`/api/home${suffix}`).catch(() => []);
}
export function fetchListing(
page: number,
pageSize: number,
params?: { q?: string; tag?: string; cat?: string; sort?: string }
params?: { q?: string; tag?: string; cat?: string; sort?: string; includeTotal?: boolean }
): Promise<{ items: VideoItem[]; total: number }> {
const qs = new URLSearchParams({
page: String(page),
@@ -18,6 +23,7 @@ export function fetchListing(
if (params?.tag) qs.set("tag", params.tag);
if (params?.cat) qs.set("cat", params.cat);
if (params?.sort) qs.set("sort", params.sort);
if (params?.includeTotal === false) qs.set("count", "false");
return apiGet<{ items: VideoItem[]; total: number }>(
`/api/list?${qs.toString()}`
).catch(() => ({ items: [], total: 0 }));
@@ -73,8 +79,28 @@ export function uploadVideo(input: UploadVideoInput): Promise<VideoItem> {
export type TagItem = { id: string; label: string; count?: number };
const TAG_CACHE_TTL_MS = 30_000;
let cachedTags: TagItem[] | null = null;
let cachedTagsAt = 0;
let pendingTags: Promise<TagItem[]> | null = null;
export function fetchTags(): Promise<TagItem[]> {
return apiGet<TagItem[]>("/api/tags").catch(() => []);
const now = Date.now();
if (cachedTags && now - cachedTagsAt < TAG_CACHE_TTL_MS) {
return Promise.resolve(cachedTags);
}
if (pendingTags) return pendingTags;
pendingTags = apiGet<TagItem[]>("/api/tags")
.then((tags) => {
cachedTags = tags;
cachedTagsAt = Date.now();
return tags;
})
.catch(() => cachedTags ?? [])
.finally(() => {
pendingTags = null;
});
return pendingTags;
}
/** 短视频模式单条记录。比 VideoItem 多 videoSrc / poster。 */
+69 -18
View File
@@ -10,6 +10,8 @@ import type { VideoItem } from "@/types";
const DESKTOP_COUNT = 12;
const MOBILE_COUNT = 8;
const HOME_RECENT_KEY = "home.random.recentVideoIds";
const HOME_RECENT_LIMIT = 72;
function useIsMobile() {
const [mobile, setMobile] = useState(window.innerWidth <= 640);
@@ -26,31 +28,75 @@ function useIsMobile() {
let cachedRanking: VideoItem[] | null = null;
let cachedLatest: VideoItem[] | null = null;
function loadRecentHomeVideoIds(): string[] {
try {
const raw = window.localStorage.getItem(HOME_RECENT_KEY);
const parsed = raw ? JSON.parse(raw) : [];
return Array.isArray(parsed)
? parsed.filter((id): id is string => typeof id === "string" && id.length > 0)
: [];
} catch {
return [];
}
}
function rememberHomeVideos(items: VideoItem[]) {
const merged = [...items.map((item) => item.id), ...loadRecentHomeVideoIds()];
const seen = new Set<string>();
const recent: string[] = [];
for (const id of merged) {
if (!id || seen.has(id)) continue;
seen.add(id);
recent.push(id);
if (recent.length >= HOME_RECENT_LIMIT) break;
}
try {
window.localStorage.setItem(HOME_RECENT_KEY, JSON.stringify(recent));
} catch {
// localStorage 不可用时只影响连续刷新去重,不影响首页展示。
}
}
export default function HomePage() {
const [rankingVideos, setRankingVideos] = useState<VideoItem[]>(cachedRanking ?? []);
const [latestVideos, setLatestVideos] = useState<VideoItem[]>(cachedLatest ?? []);
const [loading, setLoading] = useState(cachedRanking === null);
const [rankingLoading, setRankingLoading] = useState(cachedRanking === null);
const [latestLoading, setLatestLoading] = useState(cachedLatest === null);
const isMobile = useIsMobile();
useEffect(() => {
document.title = "首页 · 91";
// 有缓存说明是 SPA 内导航返回,不重新请求
if (cachedRanking !== null) return;
let active = true;
setLoading(true);
Promise.all([
fetchHomeVideos(),
fetchListing(1, DESKTOP_COUNT, { sort: "latest" }),
]).then(([rankingItems, latestResult]) => {
if (!active) return;
cachedRanking = rankingItems;
cachedLatest = latestResult.items;
setRankingVideos(rankingItems);
setLatestVideos(latestResult.items);
setLoading(false);
});
if (cachedRanking === null) {
setRankingLoading(true);
const excludeIds = loadRecentHomeVideoIds();
fetchHomeVideos(excludeIds)
.then((rankingItems) => {
if (!active) return;
rememberHomeVideos(rankingItems);
cachedRanking = rankingItems;
setRankingVideos(rankingItems);
})
.finally(() => {
if (active) setRankingLoading(false);
});
}
if (cachedLatest === null) {
setLatestLoading(true);
fetchListing(1, DESKTOP_COUNT, { sort: "latest", includeTotal: false })
.then((latestResult) => {
if (!active) return;
cachedLatest = latestResult.items;
setLatestVideos(latestResult.items);
})
.finally(() => {
if (active) setLatestLoading(false);
});
}
return () => { active = false; };
}, []);
@@ -68,12 +114,17 @@ export default function HomePage() {
<div className="container page-section">
<SectionHeader title="随机推荐" extra={`随机展示 ${ranking.length} 个作品`} />
<VideoGrid videos={ranking} loading={loading} skeletonCount={displayCount} />
<VideoGrid
videos={ranking}
loading={rankingLoading}
priorityCount={Math.min(4, displayCount)}
skeletonCount={displayCount}
/>
</div>
<div className="container page-section">
<SectionHeader title="最新视频" extra={latest.length > 0 ? `${latest.length}` : undefined} />
<VideoGrid videos={latest} loading={loading} skeletonCount={displayCount} />
<VideoGrid videos={latest} loading={latestLoading} skeletonCount={displayCount} />
</div>
</AppShell>
);
+16 -6
View File
@@ -33,6 +33,7 @@ export default function ListingPage() {
);
const initialState = useMemo(() => readListingState(listKey), [listKey]);
const activeListKeyRef = useRef(listKey);
const hasLoadedListingRef = useRef(false);
const pendingScrollYRef = useRef<number | null>(
initialState ? initialState.scrollY : null
);
@@ -40,9 +41,11 @@ export default function ListingPage() {
const [sort, setSort] = useState<SortKey>(initialState?.sort ?? "latest");
const [view, setView] = useState<ViewMode>(initialState?.view ?? "grid");
const [page, setPage] = useState(initialState?.page ?? 1);
const [loading, setLoading] = useState(true);
const [initialLoading, setInitialLoading] = useState(true);
const [refreshing, setRefreshing] = useState(false);
const [items, setItems] = useState<VideoItem[]>([]);
const [total, setTotal] = useState(0);
const isFetching = initialLoading || refreshing;
useEffect(() => {
if (activeListKeyRef.current === listKey) return;
@@ -64,12 +67,19 @@ export default function ListingPage() {
: "视频列表 · 91";
let active = true;
setLoading(true);
const isInitialLoad = !hasLoadedListingRef.current;
if (isInitialLoad) {
setInitialLoading(true);
} else {
setRefreshing(true);
}
fetchListing(page, tag ? PAGE_SIZE_TAG : PAGE_SIZE_DEFAULT, { q: keyword, tag, cat, sort }).then((r) => {
if (!active) return;
setItems(r.items ?? []);
setTotal(r.total ?? 0);
setLoading(false);
hasLoadedListingRef.current = true;
setInitialLoading(false);
setRefreshing(false);
});
return () => {
active = false;
@@ -109,7 +119,7 @@ export default function ListingPage() {
}, [listKey, sort, view, page]);
useEffect(() => {
if (loading) return;
if (isFetching) return;
const scrollY = pendingScrollYRef.current;
if (scrollY === null) return;
pendingScrollYRef.current = null;
@@ -118,7 +128,7 @@ export default function ListingPage() {
window.scrollTo({ top: scrollY, behavior: "auto" });
});
});
}, [loading, items.length, listKey]);
}, [isFetching, items.length, listKey]);
const title = keyword
? `搜索结果:${keyword}`
@@ -153,7 +163,7 @@ export default function ListingPage() {
/>
<VideoGrid
videos={items}
loading={loading}
loading={initialLoading}
compact={view === "compact"}
skeletonCount={12}
emptyText="没有找到匹配的视频"
+1
View File
@@ -1353,6 +1353,7 @@ function formatCount(n: number) {
function getDriveShortName(source: string): string {
const s = source.toLowerCase();
if (s.includes("115")) return "115";
if (s.includes("123")) return "123";
if (s.includes("pikpak")) return "PikP";
if (s.includes("quark") || s.includes("夸克")) return "Quak";
if (s.includes("onedrive")) return "OneDrive";
+92 -6
View File
@@ -324,6 +324,43 @@
line-height: var(--line-relaxed);
}
.admin-p123-qr {
display: grid;
gap: var(--space-3);
padding: var(--space-3);
border: 1px solid var(--border-subtle);
border-radius: var(--radius-sm);
background: var(--bg-elevated);
}
.admin-p123-qr__actions {
display: flex;
flex-wrap: wrap;
gap: var(--space-2);
align-items: center;
}
.admin-p123-qr__body {
display: grid;
grid-template-columns: 160px minmax(0, 1fr);
gap: var(--space-3);
align-items: center;
}
.admin-p123-qr__image {
width: 160px;
height: 160px;
padding: 8px;
border-radius: var(--radius-sm);
background: #fff;
}
.admin-p123-qr__meta {
display: grid;
gap: 6px;
min-width: 0;
}
/* =========================================================
* Buttons
* ========================================================= */
@@ -704,6 +741,50 @@
background: var(--bg-elevated);
}
.admin-delete-confirm {
display: grid;
grid-template-columns: 42px minmax(0, 1fr);
gap: var(--space-3);
align-items: start;
}
.admin-delete-confirm__icon {
width: 42px;
height: 42px;
border-radius: var(--radius-sm);
display: grid;
place-items: center;
color: var(--danger);
background: var(--danger-soft);
border: 1px solid rgba(241, 85, 108, 0.34);
}
.admin-delete-confirm__content {
min-width: 0;
}
.admin-delete-confirm__title {
margin: 0 0 8px;
color: var(--text-strong);
font-size: var(--font-md);
font-weight: var(--weight-semibold);
}
.admin-delete-confirm__text {
margin: 0 0 10px;
color: var(--text-muted);
font-size: var(--font-sm);
line-height: 1.6;
}
.admin-delete-confirm__list {
margin: 0 0 10px;
padding-left: 18px;
color: var(--text-default);
font-size: var(--font-sm);
line-height: 1.7;
}
/* =========================================================
* Toast
* ========================================================= */
@@ -1106,6 +1187,14 @@
width: 100%;
}
.admin-p123-qr__body {
grid-template-columns: 1fr;
}
.admin-p123-qr__image {
justify-self: center;
}
/* Table card-grid conversion for videos & tags tables */
.admin-table:not(.admin-drives-table) {
display: block;
@@ -1418,13 +1507,11 @@
text-align: left;
color: inherit;
transition: border-color var(--transition-fast),
transform var(--transition-fast),
box-shadow var(--transition-fast);
}
.theme-card:hover:not(:disabled) {
border-color: var(--border-accent);
transform: translateY(-2px);
box-shadow: var(--shadow-md);
}
@@ -1781,12 +1868,11 @@
flex-direction: column;
gap: var(--space-4);
cursor: pointer;
transition: transform var(--transition-fast), border-color var(--transition-fast), box-shadow var(--transition-fast);
transition: border-color var(--transition-fast), box-shadow var(--transition-fast);
position: relative;
}
.admin-drive-card:hover {
transform: translateY(-2px);
border-color: var(--border-accent);
box-shadow: var(--shadow-md);
}
@@ -1822,6 +1908,7 @@
.admin-drive-card__brand-icon[data-kind="quark"] { background: var(--drive-quark); }
.admin-drive-card__brand-icon[data-kind="p115"] { background: var(--drive-p115); }
.admin-drive-card__brand-icon[data-kind="p123"] { background: var(--drive-p123); }
.admin-drive-card__brand-icon[data-kind="pikpak"] { background: var(--drive-pikpak); }
.admin-drive-card__brand-icon[data-kind="wopan"] { background: var(--drive-wopan); }
.admin-drive-card__brand-icon[data-kind="onedrive"] { background: var(--drive-onedrive); }
@@ -2178,12 +2265,11 @@
border-radius: var(--radius-sm);
padding: var(--space-4);
gap: var(--space-3);
transition: all var(--transition-fast);
transition: border-color var(--transition-fast), box-shadow var(--transition-fast);
box-shadow: var(--shadow-sm);
}
.admin-tag-card:hover {
transform: translateY(-2px);
border-color: var(--border-accent);
box-shadow: var(--shadow-md);
}
+2
View File
@@ -19,6 +19,8 @@ body,
html {
/* 让锚点跳转有平滑动画,但尊重系统 reduce-motion */
scroll-behavior: smooth;
scrollbar-gutter: stable;
overflow-y: scroll;
}
@media (prefers-reduced-motion: reduce) {
-3
View File
@@ -90,7 +90,6 @@
.promo-card:hover {
border-color: var(--border-accent);
transform: translateY(-3px);
box-shadow: var(--shadow-md);
color: var(--text-strong);
}
@@ -149,7 +148,6 @@
}
.back-to-top:hover {
transform: translateY(-3px);
filter: brightness(1.05);
}
@@ -372,7 +370,6 @@
}
.upload-submit:hover:not(:disabled) {
transform: translateY(-1px);
filter: brightness(1.05);
box-shadow: 0 10px 22px var(--accent-glow), var(--shadow-inset);
}
+1 -3
View File
@@ -16,13 +16,12 @@
border: 1px solid var(--border-default);
border-radius: var(--radius-xl);
box-shadow: 0 8px 30px rgba(0, 0, 0, 0.4), var(--shadow-inset);
transition: border-color var(--transition-fast), box-shadow var(--transition-fast), transform var(--transition-fast);
transition: border-color var(--transition-fast), box-shadow var(--transition-fast);
}
.search-panel__form:focus-within {
border-color: var(--border-accent);
box-shadow: 0 0 0 1px var(--border-accent), 0 8px 24px var(--accent-glow);
transform: translateY(-1px);
}
.search-panel__input-wrapper {
@@ -83,7 +82,6 @@
.search-panel__submit:hover {
filter: brightness(1.05);
transform: translateY(-1px);
box-shadow: 0 8px 20px var(--accent-glow), var(--shadow-inset);
}
+2
View File
@@ -131,6 +131,7 @@
/* ----- 网盘品牌色 ----- */
--drive-quark: #5b8def;
--drive-p115: #f56b76;
--drive-p123: #22b8c8;
--drive-pikpak: #8a6dff;
--drive-wopan: #ff8a3c;
--drive-onedrive: #4cabea;
@@ -211,6 +212,7 @@
/* ----- 网盘品牌色(粉白基底下重新调谐,整体柔化、降饱和) ----- */
--drive-quark: #4f7be0;
--drive-p115: #e0556a;
--drive-p123: #1596a8;
--drive-pikpak: #8466e6;
--drive-wopan: #e57a36;
--drive-onedrive: #2f95cf;
+16 -4
View File
@@ -78,7 +78,7 @@
border-radius: var(--radius-md);
padding: 8px;
color: var(--text-default);
transition: transform var(--transition-normal), border-color var(--transition-normal),
transition: border-color var(--transition-normal),
box-shadow var(--transition-normal), background var(--transition-normal);
}
@@ -86,7 +86,6 @@
.video-card:focus-within {
background: var(--bg-elevated);
border-color: var(--border-default);
transform: translateY(-4px);
box-shadow: var(--shadow-lg);
}
@@ -248,11 +247,11 @@
backdrop-filter: blur(8px);
border: 1px solid var(--drive-border, rgba(255, 255, 255, 0.09));
box-shadow: 0 4px 12px var(--drive-shadow, rgba(0, 0, 0, 0.3)), inset 0 1px 0 rgba(255, 255, 255, 0.05);
transition: transform var(--transition-fast), border-color var(--transition-fast), box-shadow var(--transition-fast);
transition: border-color var(--transition-fast), box-shadow var(--transition-fast);
}
.source-badge:hover {
transform: translateY(-1px) scale(1.04);
transform: scale(1.04);
border-color: var(--drive-color);
box-shadow: 0 0 12px var(--drive-shadow-strong, var(--accent-glow));
}
@@ -282,6 +281,14 @@
--drive-shadow: rgba(245, 107, 118, 0.15);
--drive-shadow-strong: rgba(245, 107, 118, 0.4);
}
.source-badge[data-kind="p123"] {
--drive-color: var(--drive-p123);
--drive-bg: rgba(34, 184, 200, 0.12);
--drive-text: #98edf5;
--drive-border: rgba(34, 184, 200, 0.35);
--drive-shadow: rgba(34, 184, 200, 0.15);
--drive-shadow-strong: rgba(34, 184, 200, 0.4);
}
.source-badge[data-kind="pikpak"] {
--drive-color: var(--drive-pikpak);
--drive-bg: rgba(138, 109, 255, 0.12);
@@ -446,6 +453,7 @@
.video-card__source[data-kind="quark"] { --source-color: var(--drive-quark); }
.video-card__source[data-kind="p115"] { --source-color: var(--drive-p115); }
.video-card__source[data-kind="p123"] { --source-color: var(--drive-p123); }
.video-card__source[data-kind="pikpak"] { --source-color: var(--drive-pikpak); }
.video-card__source[data-kind="wopan"] { --source-color: var(--drive-wopan); }
.video-card__source[data-kind="onedrive"] { --source-color: var(--drive-onedrive); }
@@ -455,7 +463,11 @@
* Empty / Loading / Skeleton
* ========================================================= */
.video-grid-empty {
min-height: 220px;
padding: var(--space-9) var(--space-5);
display: flex;
align-items: center;
justify-content: center;
text-align: center;
color: var(--text-muted);
background: var(--bg-surface);
+6 -1
View File
@@ -333,6 +333,12 @@
color: var(--drive-p115);
}
.vd-meta__chip[data-tone="p123"] {
background: rgba(34, 184, 200, 0.14);
border-color: rgba(34, 184, 200, 0.3);
color: var(--drive-p123);
}
.vd-meta__chip[data-tone="pikpak"] {
background: rgba(138, 109, 255, 0.14);
border-color: rgba(138, 109, 255, 0.3);
@@ -883,7 +889,6 @@
.vd-rail__link:hover {
background: rgba(255, 255, 255, 0.04);
transform: translateY(-1px);
}
.vd-rail__link:focus-visible {
+13 -2
View File
@@ -8,6 +8,9 @@ const drivesPageSource = readFileSync(
);
test("spider91 drive form does not expose advanced crawler credentials", () => {
assert.match(drivesPageSource, /key: "proxy"/);
assert.match(drivesPageSource, /label: "代理地址(可选)"/);
assert.match(drivesPageSource, /支持 http:\/\/、https:\/\/、socks5:\/\/ 或 socks5h:\/\//);
assert.doesNotMatch(drivesPageSource, /target_new/);
assert.doesNotMatch(drivesPageSource, /crawl_hour/);
assert.doesNotMatch(drivesPageSource, /python_path/);
@@ -24,8 +27,14 @@ test("spider91 upload target uses explicit local-save option instead of auto tar
assert.doesNotMatch(drivesPageSource, /自动模式/);
});
test("drive form shows a root directory id field for all drive kinds", () => {
test("drive form hides root directory id for localstorage and spider91", () => {
assert.match(drivesPageSource, /<label>根目录 ID<\/label>/);
assert.match(
drivesPageSource,
/function usesRootDirectoryID\(kind: Kind\): boolean \{\s*return kind !== "localstorage" && kind !== "spider91";\s*\}/
);
assert.match(drivesPageSource, /\{usesRootDirectoryID\(form\.kind\) && \(/);
assert.match(drivesPageSource, /\{usesRootDirectoryID\(d\.kind\) && \(/);
assert.match(drivesPageSource, /placeholder=\{rootIdPlaceholder\(form\.kind\)\}/);
assert.doesNotMatch(drivesPageSource, /扫描起点目录 ID/);
assert.doesNotMatch(drivesPageSource, /set\("scanRootId"/);
@@ -94,6 +103,7 @@ test("localstorage drive form asks for a server directory path", () => {
assert.match(fields, /key: "path"/);
assert.match(fields, /label: "本地目录路径"/);
assert.match(drivesPageSource, /if \(kind === "localstorage"\) return "\/"/);
assert.match(drivesPageSource, /kind !== "localstorage" && kind !== "spider91"/);
});
test("drive type selector keeps primary source order", () => {
@@ -101,10 +111,11 @@ test("drive type selector keeps primary source order", () => {
drivesPageSource.matchAll(/<option value="([^"]+)">([^<]+)<\/option>/g),
(match) => ({ value: match[1], label: match[2] })
);
const driveOptions = options.slice(0, 8);
const driveOptions = options.slice(0, 9);
assert.deepEqual(driveOptions, [
{ value: "p115", label: "115 网盘" },
{ value: "p123", label: "123 云盘" },
{ value: "pikpak", label: "PikPak" },
{ value: "onedrive", label: "OneDrive" },
{ value: "googledrive", label: "Google Drive" },