mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-16 01:05:42 +08:00
Compare commits
15 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5cd5e8a126 | |||
| 233b55c0f8 | |||
| 11c8585ea9 | |||
| c3ada441c1 | |||
| 2250d3ca4c | |||
| e465eac568 | |||
| 667a0c4e1c | |||
| 33ff07314a | |||
| 858abd7532 | |||
| 91352ac681 | |||
| d70cea7320 | |||
| 6654a1b730 | |||
| bcc887e088 | |||
| 2c0cfe1d15 | |||
| a276e6b32d |
@@ -9,7 +9,7 @@
|
||||
- 视频直链 (MP4)
|
||||
|
||||
依赖安装:
|
||||
pip install requests beautifulsoup4 lxml
|
||||
pip install requests beautifulsoup4 lxml PySocks
|
||||
|
||||
使用方法:
|
||||
# 全量爬取(默认行为,从 page=1 一直爬到末尾,写到 OUTPUT_FILE)
|
||||
@@ -68,6 +68,7 @@ import time
|
||||
import random
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import html
|
||||
from urllib.parse import urljoin, unquote, urlparse
|
||||
@@ -80,6 +81,28 @@ except ImportError:
|
||||
print("请运行: pip install beautifulsoup4 lxml")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def prefer_ipv4_for_plain_socks5_proxy():
|
||||
"""PySocks may pick IPv6 first for socks5://; some SOCKS5 servers only accept IPv4."""
|
||||
proxy_envs = (
|
||||
os.environ.get("HTTPS_PROXY", ""),
|
||||
os.environ.get("HTTP_PROXY", ""),
|
||||
os.environ.get("https_proxy", ""),
|
||||
os.environ.get("http_proxy", ""),
|
||||
)
|
||||
uses_plain_socks5 = any(v.strip().lower().startswith("socks5://") for v in proxy_envs)
|
||||
if not uses_plain_socks5 or getattr(socket, "_spider91_ipv4_first", False):
|
||||
return
|
||||
|
||||
original_getaddrinfo = socket.getaddrinfo
|
||||
|
||||
def getaddrinfo_ipv4_first(*args, **kwargs):
|
||||
infos = original_getaddrinfo(*args, **kwargs)
|
||||
return sorted(infos, key=lambda info: 0 if info[0] == socket.AF_INET else 1)
|
||||
|
||||
socket.getaddrinfo = getaddrinfo_ipv4_first
|
||||
socket._spider91_ipv4_first = True
|
||||
|
||||
# ===================== 配置区域 =====================
|
||||
BASE_URL = "https://www.91porn.com/v.php"
|
||||
LIST_PARAMS = {
|
||||
@@ -706,7 +729,7 @@ def print_help():
|
||||
- 视频直链 (MP4)
|
||||
|
||||
依赖安装:
|
||||
pip install requests beautifulsoup4 lxml
|
||||
pip install requests beautifulsoup4 lxml PySocks
|
||||
|
||||
使用方法:
|
||||
python spider_91porn.py
|
||||
@@ -757,13 +780,15 @@ def main():
|
||||
"日志改走 stderr。配合 backend 边读边下载使用。")
|
||||
|
||||
args, _ = parser.parse_known_args()
|
||||
cli_out = sys.stderr if args.stream_output else sys.stdout
|
||||
prefer_ipv4_for_plain_socks5_proxy()
|
||||
|
||||
print("""
|
||||
================================================
|
||||
91porn 视频爬虫启动中...
|
||||
================================================
|
||||
按 Ctrl+C 可随时中断并保存进度
|
||||
""")
|
||||
""", file=cli_out)
|
||||
|
||||
# 加载已知 ID(来自 backend 的 catalog 已入库列表;兼容旧参数名)
|
||||
seen_viewkeys = []
|
||||
@@ -775,9 +800,9 @@ def main():
|
||||
if line:
|
||||
seen_viewkeys.append(line)
|
||||
except FileNotFoundError:
|
||||
print(f"警告: --seen-viewkeys-file 不存在: {args.seen_viewkeys_file}")
|
||||
print(f"警告: --seen-viewkeys-file 不存在: {args.seen_viewkeys_file}", file=cli_out)
|
||||
except Exception as e:
|
||||
print(f"警告: 读取 --seen-viewkeys-file 失败: {e}")
|
||||
print(f"警告: 读取 --seen-viewkeys-file 失败: {e}", file=cli_out)
|
||||
|
||||
# 决定运行模式
|
||||
if args.target_new is not None:
|
||||
|
||||
@@ -36,10 +36,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
python3-bs4 \
|
||||
python3-lxml \
|
||||
python3-requests \
|
||||
python3-socks \
|
||||
tar \
|
||||
tzdata \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN python3 -c "import requests, bs4, lxml, socks"
|
||||
|
||||
WORKDIR /opt/video-site-91
|
||||
|
||||
COPY --from=backend /out/server ./server
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 nianzhibai
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -12,7 +12,8 @@
|
||||
<a href="#快速开始">快速开始</a> ·
|
||||
<a href="#功能特性">功能特性</a> ·
|
||||
<a href="#预览图">预览图</a> ·
|
||||
<a href="#数据存放位置">数据目录</a>
|
||||
<a href="#数据存放位置">数据目录</a> ·
|
||||
<a href="#许可证">许可证</a>
|
||||
</p>
|
||||
|
||||
---
|
||||
@@ -20,7 +21,7 @@
|
||||
## 功能特性
|
||||
|
||||
- **多后端支持** — 兼容 115 云盘、PikPak 云盘、OneDrive、Google Drive 和本地存储
|
||||
- **低带宽播放** — 支持 302 的云盘可直连播放;Google Drive 等需鉴权直链的来源走后端代理
|
||||
- **低带宽播放** — 115 云盘、PikPak 云盘、OneDrive 都支持302模式,在线播放视频时,不占用服务器带宽,播放体验不受服务器带宽影响;Google Drive 不支持302模式,走服务器中转,观看体验会受服务器带宽影响
|
||||
- **封面 & 预览片段** — 自动为每个视频生成封面图和预览片段,首页快速选片
|
||||
- **91 爬虫** — 内置爬虫,支持抓取 91 本月最热视频
|
||||
- **双主题** — 黑黄经典主题 / 粉白清新主题,随时切换
|
||||
@@ -196,6 +197,12 @@ docker compose up -d # 更新并重启
|
||||
|
||||
---
|
||||
|
||||
## 许可证
|
||||
|
||||
本项目基于 [MIT License](LICENSE) 开源。
|
||||
|
||||
---
|
||||
|
||||
## 致谢
|
||||
|
||||
- [OpenList](https://github.com/OpenListTeam/OpenList) — 优秀的开源项目
|
||||
|
||||
+225
-5
@@ -30,6 +30,7 @@ import (
|
||||
"github.com/video-site/backend/internal/drives/localupload"
|
||||
"github.com/video-site/backend/internal/drives/onedrive"
|
||||
"github.com/video-site/backend/internal/drives/p115"
|
||||
"github.com/video-site/backend/internal/drives/p123"
|
||||
"github.com/video-site/backend/internal/drives/pikpak"
|
||||
"github.com/video-site/backend/internal/drives/quark"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
@@ -81,6 +82,7 @@ func main() {
|
||||
Catalog: cat,
|
||||
Registry: app.registry,
|
||||
GetTargetDriveID: func() string { return app.Spider91UploadDriveID() },
|
||||
CommonThumbDir: app.commonThumbsDir(),
|
||||
})
|
||||
|
||||
// 初始化本地内置盘;外部云盘放到 HTTP 服务启动后异步挂载,避免上游
|
||||
@@ -90,6 +92,11 @@ func main() {
|
||||
|
||||
app.loadTheme(ctx)
|
||||
app.loadSpider91UploadDriveID(ctx)
|
||||
if removed, err := app.cleanupOrphanDriveVideos(ctx); err != nil {
|
||||
log.Printf("[cleanup] orphan drive videos: %v", err)
|
||||
} else if removed > 0 {
|
||||
log.Printf("[cleanup] removed %d orphan drive videos", removed)
|
||||
}
|
||||
if err := app.attachLocalUpload(ctx); err != nil {
|
||||
log.Printf("[local-upload] attach failed: %v", err)
|
||||
}
|
||||
@@ -156,6 +163,9 @@ func main() {
|
||||
}
|
||||
return app.attachDrive(ctx, d)
|
||||
},
|
||||
OnDriveDeleteCleanup: func(cleanupCtx context.Context, driveID string) (int, error) {
|
||||
return app.cleanupDriveVideosForDelete(cleanupCtx, driveID)
|
||||
},
|
||||
OnDriveRemoved: func(driveID string) {
|
||||
app.detachDrive(driveID)
|
||||
},
|
||||
@@ -182,6 +192,9 @@ func main() {
|
||||
OnRegenFailedThumbnails: func(driveID string) {
|
||||
go app.regenFailedThumbnails(ctx, driveID)
|
||||
},
|
||||
OnRegenFailedFingerprints: func(driveID string) {
|
||||
go app.regenFailedFingerprints(ctx, driveID)
|
||||
},
|
||||
GetDriveGenerationStatuses: func() map[string]api.DriveGenerationStatuses {
|
||||
return app.driveGenerationStatuses()
|
||||
},
|
||||
@@ -608,6 +621,22 @@ func (a *App) attachDriveUnlocked(ctx context.Context, d *catalog.Drive) error {
|
||||
Cookie: d.Credentials["cookie"],
|
||||
RootID: d.RootID,
|
||||
})
|
||||
case p123.Kind:
|
||||
drv = p123.New(p123.Config{
|
||||
ID: d.ID,
|
||||
Username: d.Credentials["username"],
|
||||
Password: d.Credentials["password"],
|
||||
AccessToken: d.Credentials["access_token"],
|
||||
Platform: d.Credentials["platform"],
|
||||
RootID: d.RootID,
|
||||
OnTokenUpdate: func(access string) {
|
||||
if d.Credentials == nil {
|
||||
d.Credentials = make(map[string]string)
|
||||
}
|
||||
d.Credentials["access_token"] = access
|
||||
_ = a.cat.UpsertDrive(ctx, d)
|
||||
},
|
||||
})
|
||||
case "pikpak":
|
||||
drv = pikpak.New(pikpak.Config{
|
||||
ID: d.ID,
|
||||
@@ -774,7 +803,7 @@ func fingerprintConfigForDrive(drv drives.Drive) fingerprint.Config {
|
||||
return cfg
|
||||
}
|
||||
switch strings.ToLower(drv.Kind()) {
|
||||
case "p115", "onedrive":
|
||||
case "p115", "p123", "onedrive":
|
||||
cfg.RateLimitCooldown = 10 * time.Minute
|
||||
case "pikpak":
|
||||
cfg.RateLimitCooldown = 5 * time.Minute
|
||||
@@ -854,13 +883,14 @@ func (a *App) attachSpider91Crawler(d *catalog.Drive, drv *spider91.Driver) {
|
||||
a.spider91Crawlers[driveID] = c
|
||||
a.mu.Unlock()
|
||||
|
||||
// 确保 "91porn" 系统标签存在,并把已入库的 spider91 视频按 author 字段
|
||||
// 匹配补打这个标签(CreateTagAndClassify 内部对所有视频走一遍 classify)。
|
||||
// 重复调用是幂等的:tags 用 INSERT OR IGNORE,video_tags 也是 INSERT OR IGNORE。
|
||||
// 确保 "91porn" 系统标签存在,并按 spider91 来源前缀给历史视频补打。
|
||||
// 不能只靠文本匹配:老版本入库的视频可能没有 author/tags 字段,但 id 前缀
|
||||
// "spider91-<driveID>-" 会一直保留,即使后续迁移到 PikPak/115 也不变。
|
||||
bgCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
go func() {
|
||||
defer cancel()
|
||||
if _, err := a.cat.CreateTagAndClassify(bgCtx, spider91.DefaultTag, nil, "system"); err != nil {
|
||||
prefix := "spider91-" + driveID + "-"
|
||||
if _, err := a.cat.EnsureTagForVideoIDPrefix(bgCtx, prefix, spider91.DefaultTag, nil, "system"); err != nil {
|
||||
log.Printf("[spider91] ensure %q tag: %v", spider91.DefaultTag, err)
|
||||
}
|
||||
}()
|
||||
@@ -1231,6 +1261,160 @@ func (a *App) cleanupMissingDriveVideos(ctx context.Context, driveID string, liv
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
func (a *App) cleanupDriveVideosForDelete(ctx context.Context, driveID string) (int, error) {
|
||||
if a == nil || a.cat == nil {
|
||||
return 0, nil
|
||||
}
|
||||
d, err := a.cat.GetDrive(ctx, driveID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Stop generation/crawl workers before deleting assets so they do not keep
|
||||
// writing files for a drive that is being removed.
|
||||
a.detachDrive(driveID)
|
||||
|
||||
items, err := a.videosForDriveDelete(ctx, d)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
localDir := ""
|
||||
if a.cfg != nil {
|
||||
localDir = a.cfg.Storage.LocalPreviewDir
|
||||
}
|
||||
for _, v := range items {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := removeLocalVideoAssets(localDir, v); err != nil {
|
||||
return 0, fmt.Errorf("remove local assets for %s: %w", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
if strings.EqualFold(d.Kind, spider91.Kind) {
|
||||
if err := a.removeSpider91DriveDir(driveID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
removed := 0
|
||||
for _, v := range items {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return removed, err
|
||||
}
|
||||
if err := a.cat.DeleteVideo(ctx, v.ID); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
continue
|
||||
}
|
||||
return removed, fmt.Errorf("delete catalog video %s: %w", v.ID, err)
|
||||
}
|
||||
removed++
|
||||
}
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
func (a *App) cleanupOrphanDriveVideos(ctx context.Context) (int, error) {
|
||||
if a == nil || a.cat == nil {
|
||||
return 0, nil
|
||||
}
|
||||
items, err := a.cat.ListVideosWithMissingDrive(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
localDir := ""
|
||||
if a.cfg != nil {
|
||||
localDir = a.cfg.Storage.LocalPreviewDir
|
||||
}
|
||||
spider91Dirs := map[string]struct{}{}
|
||||
for _, v := range items {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := removeLocalVideoAssets(localDir, v); err != nil {
|
||||
return 0, fmt.Errorf("remove local assets for orphan %s: %w", v.ID, err)
|
||||
}
|
||||
if strings.HasPrefix(v.ID, "spider91-"+v.DriveID+"-") {
|
||||
spider91Dirs[v.DriveID] = struct{}{}
|
||||
}
|
||||
}
|
||||
for driveID := range spider91Dirs {
|
||||
if err := a.removeSpider91DriveDir(driveID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
removed := 0
|
||||
for _, v := range items {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return removed, err
|
||||
}
|
||||
if err := a.cat.DeleteVideo(ctx, v.ID); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
continue
|
||||
}
|
||||
return removed, fmt.Errorf("delete orphan catalog video %s: %w", v.ID, err)
|
||||
}
|
||||
removed++
|
||||
}
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
func (a *App) videosForDriveDelete(ctx context.Context, d *catalog.Drive) ([]*catalog.Video, error) {
|
||||
if d == nil {
|
||||
return nil, nil
|
||||
}
|
||||
items, err := a.cat.ListVideosByDrive(ctx, d.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
byID := make(map[string]*catalog.Video, len(items))
|
||||
for _, v := range items {
|
||||
byID[v.ID] = v
|
||||
}
|
||||
|
||||
if strings.EqualFold(d.Kind, spider91.Kind) {
|
||||
prefix := "spider91-" + d.ID + "-"
|
||||
originItems, err := a.cat.ListVideosByIDPrefix(ctx, prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, v := range originItems {
|
||||
byID[v.ID] = v
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]*catalog.Video, 0, len(byID))
|
||||
for _, v := range byID {
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (a *App) removeSpider91DriveDir(driveID string) error {
|
||||
if strings.TrimSpace(driveID) == "" {
|
||||
return errors.New("remove spider91 drive dir: empty drive id")
|
||||
}
|
||||
root := a.spider91RootDir()
|
||||
dir := a.spider91DriveDir(driveID)
|
||||
clean, ok := localPathWithin(root, dir)
|
||||
if !ok {
|
||||
return fmt.Errorf("remove spider91 drive dir: unsafe path %s", dir)
|
||||
}
|
||||
rootClean, ok := localPathWithin(root, root)
|
||||
if !ok || clean == rootClean {
|
||||
return fmt.Errorf("remove spider91 drive dir: refusing to remove root %s", root)
|
||||
}
|
||||
if err := os.RemoveAll(clean); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("remove spider91 drive dir %s: %w", clean, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeLocalVideoAssets(localDir string, v *catalog.Video) error {
|
||||
if localDir == "" || v == nil || v.ID == "" {
|
||||
return nil
|
||||
@@ -1551,6 +1735,42 @@ func (a *App) regenFailedThumbnails(ctx context.Context, driveID string) {
|
||||
log.Printf("[thumb] enqueued failed thumbnails for regen drive=%s queued=%d", driveID, queued)
|
||||
}
|
||||
|
||||
func (a *App) regenFailedFingerprints(ctx context.Context, driveID string) {
|
||||
items, err := a.cat.ListVideosByFingerprintStatus(ctx, driveID, "failed", 0)
|
||||
if err != nil {
|
||||
log.Printf("[fingerprint] list failed videos for regen drive=%s: %v", driveID, err)
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
fingerprintWorker := a.fingerprintWorkers[driveID]
|
||||
a.mu.Unlock()
|
||||
if fingerprintWorker == nil {
|
||||
log.Printf("[fingerprint] regen failed drive=%s skipped: fingerprint worker not found", driveID)
|
||||
return
|
||||
}
|
||||
log.Printf("[fingerprint] enqueue failed videos for regen drive=%s count=%d", driveID, len(items))
|
||||
queued := 0
|
||||
for _, v := range items {
|
||||
if err := ctx.Err(); err != nil {
|
||||
log.Printf("[fingerprint] enqueue failed canceled drive=%s queued=%d: %v", driveID, queued, err)
|
||||
return
|
||||
}
|
||||
if err := a.cat.UpdateVideoFingerprint(ctx, v.ID, "", "pending", ""); err != nil {
|
||||
log.Printf("[fingerprint] reset failed video %s drive=%s: %v", v.ID, driveID, err)
|
||||
continue
|
||||
}
|
||||
v.SampledSHA256 = ""
|
||||
v.FingerprintStatus = "pending"
|
||||
v.FingerprintError = ""
|
||||
if !fingerprintWorker.EnqueueBlocking(ctx, v) {
|
||||
log.Printf("[fingerprint] enqueue failed canceled drive=%s queued=%d", driveID, queued)
|
||||
return
|
||||
}
|
||||
queued++
|
||||
}
|
||||
log.Printf("[fingerprint] enqueued failed videos for regen drive=%s queued=%d", driveID, queued)
|
||||
}
|
||||
|
||||
// listScanTargetIDs 返回 nightly Phase 1 应扫描的所有 drive ID
|
||||
// (非 spider91、非 localupload)。它直接读 catalog,而不是 registry,这样
|
||||
// 进程刚启动、云盘还在后台挂载时,nightly 也不会漏掉配置过的 drive。
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/config"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
"github.com/video-site/backend/internal/fingerprint"
|
||||
"github.com/video-site/backend/internal/preview"
|
||||
"github.com/video-site/backend/internal/proxy"
|
||||
@@ -671,6 +672,309 @@ func TestCleanupMissingPikPakVideosRemovesDatabaseRowsAndLocalAssets(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupDriveVideosForDeleteRemovesRowsAndGeneratedAssetsOnly(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
localDir := filepath.Join(root, "previews")
|
||||
originalDir := filepath.Join(root, "local-videos")
|
||||
originalVideo := filepath.Join(originalDir, "clip.mp4")
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, path := range []string{originalVideo} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("original"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "local-main",
|
||||
Kind: "localstorage",
|
||||
Name: "Local",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{"path": originalDir},
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
previewPath := filepath.Join(localDir, "localstorage-local-main-file.mp4")
|
||||
thumbPath := filepath.Join(localDir, "thumbs", "localstorage-local-main-file.jpg")
|
||||
for _, path := range []string{previewPath, thumbPath} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("generated"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "localstorage-local-main-file",
|
||||
DriveID: "local-main",
|
||||
FileID: "encoded-local-file",
|
||||
Title: "Local File",
|
||||
PreviewLocal: previewPath,
|
||||
PreviewStatus: "ready",
|
||||
ThumbnailURL: "/p/thumb/localstorage-local-main-file",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed local video: %v", err)
|
||||
}
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: "pikpak-other",
|
||||
DriveID: "PikPak",
|
||||
FileID: "other",
|
||||
Title: "Other",
|
||||
PreviewStatus: "ready",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed other video: %v", err)
|
||||
}
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
registry: proxy.NewRegistry(),
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
fingerprintWorkers: make(map[string]*fingerprint.Worker),
|
||||
spider91Crawlers: make(map[string]*spider91.Crawler),
|
||||
}
|
||||
removed, err := app.cleanupDriveVideosForDelete(ctx, "local-main")
|
||||
if err != nil {
|
||||
t.Fatalf("cleanup drive videos: %v", err)
|
||||
}
|
||||
if removed != 1 {
|
||||
t.Fatalf("removed = %d, want 1", removed)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "localstorage-local-main-file"); err != sql.ErrNoRows {
|
||||
t.Fatalf("deleted video lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "pikpak-other"); err != nil {
|
||||
t.Fatalf("other drive video missing: %v", err)
|
||||
}
|
||||
for _, path := range []string{previewPath, thumbPath} {
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("generated asset %s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(originalVideo); err != nil {
|
||||
t.Fatalf("original local video should remain, stat err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupDriveVideosForDeleteSpider91RemovesCrawledDirAndOriginRecords(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
localDir := filepath.Join(root, "previews")
|
||||
driveID := "spider-main"
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: driveID,
|
||||
Kind: "spider91",
|
||||
Name: "91 Spider",
|
||||
RootID: "/",
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed spider91 drive: %v", err)
|
||||
}
|
||||
|
||||
spiderDriveDir := filepath.Join(root, "spider91", driveID)
|
||||
sourceVideo := filepath.Join(spiderDriveDir, "videos", "source.mp4")
|
||||
sourceThumb := filepath.Join(spiderDriveDir, "thumbs", "source.jpg")
|
||||
localPreview := filepath.Join(localDir, "spider91-spider-main-source.mp4")
|
||||
localThumb := filepath.Join(localDir, "thumbs", "spider91-spider-main-source.jpg")
|
||||
migratedPreview := filepath.Join(localDir, "spider91-spider-main-migrated.mp4")
|
||||
migratedThumb := filepath.Join(localDir, "thumbs", "spider91-spider-main-migrated.jpg")
|
||||
for _, path := range []string{sourceVideo, sourceThumb, localPreview, localThumb, migratedPreview, migratedThumb} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("asset"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*catalog.Video{
|
||||
{
|
||||
ID: "spider91-spider-main-source",
|
||||
DriveID: driveID,
|
||||
FileID: "source.mp4",
|
||||
Title: "Source",
|
||||
PreviewLocal: localPreview,
|
||||
PreviewStatus: "ready",
|
||||
ThumbnailURL: "/p/thumb/spider91-spider-main-source",
|
||||
},
|
||||
{
|
||||
ID: "spider91-spider-main-migrated",
|
||||
DriveID: "PikPak",
|
||||
FileID: "pikpak-file-id",
|
||||
Title: "Migrated",
|
||||
PreviewLocal: migratedPreview,
|
||||
PreviewStatus: "ready",
|
||||
ThumbnailURL: "/p/thumb/spider91-spider-main-migrated",
|
||||
},
|
||||
{
|
||||
ID: "pikpak-PikPak-other",
|
||||
DriveID: "PikPak",
|
||||
FileID: "other",
|
||||
Title: "Other",
|
||||
PreviewStatus: "ready",
|
||||
},
|
||||
} {
|
||||
v.PublishedAt = now
|
||||
v.CreatedAt = now
|
||||
v.UpdatedAt = now
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed video %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
registry: proxy.NewRegistry(),
|
||||
workers: make(map[string]*preview.Worker),
|
||||
thumbWorkers: make(map[string]*preview.ThumbWorker),
|
||||
fingerprintWorkers: make(map[string]*fingerprint.Worker),
|
||||
spider91Crawlers: make(map[string]*spider91.Crawler),
|
||||
}
|
||||
removed, err := app.cleanupDriveVideosForDelete(ctx, driveID)
|
||||
if err != nil {
|
||||
t.Fatalf("cleanup spider91 videos: %v", err)
|
||||
}
|
||||
if removed != 2 {
|
||||
t.Fatalf("removed = %d, want 2", removed)
|
||||
}
|
||||
for _, id := range []string{"spider91-spider-main-source", "spider91-spider-main-migrated"} {
|
||||
if _, err := cat.GetVideo(ctx, id); err != sql.ErrNoRows {
|
||||
t.Fatalf("%s lookup error = %v, want sql.ErrNoRows", id, err)
|
||||
}
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "pikpak-PikPak-other"); err != nil {
|
||||
t.Fatalf("unrelated pikpak video missing: %v", err)
|
||||
}
|
||||
for _, path := range []string{spiderDriveDir, localPreview, localThumb, migratedPreview, migratedThumb} {
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("%s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupOrphanDriveVideosRemovesRowsAndGeneratedAssets(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
localDir := filepath.Join(root, "previews")
|
||||
cat, err := catalog.Open(filepath.Join(t.TempDir(), "catalog.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "active-drive",
|
||||
Kind: "pikpak",
|
||||
Name: "Active",
|
||||
RootID: "root",
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed active drive: %v", err)
|
||||
}
|
||||
|
||||
previewPath := filepath.Join(localDir, "p123-123-orphan.mp4")
|
||||
thumbPath := filepath.Join(localDir, "thumbs", "p123-123-orphan.jpg")
|
||||
for _, path := range []string{previewPath, thumbPath} {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("generated"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*catalog.Video{
|
||||
{
|
||||
ID: "p123-123-orphan",
|
||||
DriveID: "123",
|
||||
FileID: "orphan-file",
|
||||
Title: "Orphan",
|
||||
PreviewLocal: previewPath,
|
||||
PreviewStatus: "ready",
|
||||
ThumbnailURL: "/p/thumb/p123-123-orphan",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "pikpak-active",
|
||||
DriveID: "active-drive",
|
||||
FileID: "active-file",
|
||||
Title: "Active",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed video %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
app := &App{
|
||||
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
|
||||
cat: cat,
|
||||
}
|
||||
removed, err := app.cleanupOrphanDriveVideos(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("cleanup orphan videos: %v", err)
|
||||
}
|
||||
if removed != 1 {
|
||||
t.Fatalf("removed = %d, want 1", removed)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "p123-123-orphan"); err != sql.ErrNoRows {
|
||||
t.Fatalf("orphan video lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
if _, err := cat.GetVideo(ctx, "pikpak-active"); err != nil {
|
||||
t.Fatalf("active video missing: %v", err)
|
||||
}
|
||||
for _, path := range []string{previewPath, thumbPath} {
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("orphan asset %s still exists, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupDuplicateVideoAssetsRemovesOnlyDuplicateLocalAssets(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
localDir := t.TempDir()
|
||||
|
||||
@@ -59,7 +59,7 @@ preview:
|
||||
width: 480
|
||||
|
||||
# 盘列表。上线后请通过管理后台添加,本文件可留空。
|
||||
# kind 支持 quark / p115 / pikpak / wopan / onedrive / googledrive / localstorage。
|
||||
# kind 支持 quark / p115 / p123 / pikpak / wopan / onedrive / googledrive / localstorage。
|
||||
# OneDrive 示例:
|
||||
# - id: "my-onedrive"
|
||||
# kind: "onedrive"
|
||||
|
||||
+4
-4
@@ -7,15 +7,18 @@ toolchain go1.23.4
|
||||
require (
|
||||
github.com/OpenListTeam/wopan-sdk-go v0.2.0
|
||||
github.com/SheltonZhu/115driver v1.3.2
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
|
||||
github.com/go-chi/chi/v5 v5.1.0
|
||||
github.com/go-resty/resty/v2 v2.14.0
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
|
||||
golang.org/x/net v0.27.0
|
||||
golang.org/x/sys v0.30.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
modernc.org/sqlite v1.33.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aead/ecdh v0.2.0 // indirect
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect
|
||||
github.com/andreburgaud/crypt2go v1.1.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
@@ -26,10 +29,7 @@ require (
|
||||
github.com/pierrec/lz4/v4 v4.1.17 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
|
||||
golang.org/x/crypto v0.25.0 // indirect
|
||||
golang.org/x/net v0.27.0 // indirect
|
||||
golang.org/x/sys v0.30.0 // indirect
|
||||
golang.org/x/time v0.8.0 // indirect
|
||||
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
|
||||
modernc.org/libc v1.55.3 // indirect
|
||||
|
||||
@@ -6,7 +6,9 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -16,6 +18,7 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/auth"
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives/p123"
|
||||
)
|
||||
|
||||
type AdminServer struct {
|
||||
@@ -40,12 +43,14 @@ type AdminServer struct {
|
||||
LocalPreviewDir string
|
||||
// Hooks:外层注入实际执行者
|
||||
OnDriveSaved func(driveID string) error
|
||||
OnDriveDeleteCleanup func(ctx context.Context, driveID string) (int, error)
|
||||
OnDriveRemoved func(driveID string)
|
||||
OnScanRequested func(driveID string)
|
||||
OnRegenPreview func(videoID string)
|
||||
OnRegenAllPreviews func()
|
||||
OnRegenFailedPreviews func(driveID string)
|
||||
OnRegenFailedThumbnails func(driveID string)
|
||||
OnRegenFailedFingerprints func(driveID string)
|
||||
GetDriveGenerationStatuses func() map[string]DriveGenerationStatuses
|
||||
// OnTeaserEnabledChanged 在 per-drive teaser 开关被切换后调用。
|
||||
// enabled=true 时上层应该重新把 pending teaser 入队(类似旧的全局开关从关到开);
|
||||
@@ -68,6 +73,9 @@ type AdminServer struct {
|
||||
// 用于"设置跳过目录"弹窗按需展开浏览网盘目录树;只返回目录条目,文件忽略。
|
||||
// 调用方应当处理 error 并以 5xx 返回前端。
|
||||
ListDriveDirChildren func(ctx context.Context, driveID, parentID string) ([]DriveDirEntry, error)
|
||||
// 123 云盘扫码登录接口测试注入;生产留空走官方 user.123pan.cn。
|
||||
P123UserAPIBaseURL string
|
||||
P123HTTPClient *http.Client
|
||||
}
|
||||
|
||||
// DriveDirEntry 是 dirtree 接口的一条返回项:网盘上的一个目录节点。
|
||||
@@ -114,6 +122,8 @@ func (a *AdminServer) Register(r chi.Router) {
|
||||
r.Get("/drives", a.handleListDrives)
|
||||
r.Get("/drives/storage", a.handleDriveStorage)
|
||||
r.Post("/drives", a.handleUpsertDrive)
|
||||
r.Post("/drives/p123/qr", a.handleP123QRStart)
|
||||
r.Get("/drives/p123/qr/{uniID}", a.handleP123QRStatus)
|
||||
r.Delete("/drives/{id}", a.handleDeleteDrive)
|
||||
r.Post("/drives/{id}/rescan", a.handleRescan)
|
||||
r.Post("/drives/{id}/teaser-enabled", a.handleSetDriveTeaserEnabled)
|
||||
@@ -121,6 +131,7 @@ func (a *AdminServer) Register(r chi.Router) {
|
||||
r.Get("/drives/{id}/dirtree", a.handleListDriveDirTree)
|
||||
r.Post("/drives/{id}/previews/failed/regenerate", a.handleRegenFailedPreviews)
|
||||
r.Post("/drives/{id}/thumbnails/failed/regenerate", a.handleRegenFailedThumbnails)
|
||||
r.Post("/drives/{id}/fingerprints/failed/regenerate", a.handleRegenFailedFingerprints)
|
||||
|
||||
// 视频
|
||||
r.Get("/videos", a.handleAdminListVideos)
|
||||
@@ -393,6 +404,7 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
SkipDirIDs []string `json:"skipDirIds"`
|
||||
// LastCrawlAt 是 spider91 上次成功爬取的 unix 秒(来自 credentials.last_crawl_at)。
|
||||
// 其它 kind 留 0;前端用它显示"上次抓取: N 小时前"。
|
||||
Spider91Proxy string `json:"spider91Proxy,omitempty"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
|
||||
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
|
||||
PreviewGenerationStatus GenerationStatus `json:"previewGenerationStatus"`
|
||||
@@ -451,6 +463,7 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
|
||||
HasCredential: hasCred,
|
||||
TeaserEnabled: d.TeaserEnabled,
|
||||
SkipDirIDs: append([]string{}, d.SkipDirIDs...),
|
||||
Spider91Proxy: spider91ProxyForDrive(d),
|
||||
LastCrawlAt: lastCrawlAt,
|
||||
ThumbnailGenerationStatus: generation.Thumbnail,
|
||||
PreviewGenerationStatus: generation.Preview,
|
||||
@@ -503,7 +516,14 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
|
||||
if existingDrive, err := a.Catalog.GetDrive(r.Context(), body.ID); err == nil {
|
||||
existing = existingDrive
|
||||
}
|
||||
if len(body.Credentials) == 0 && existing != nil && len(existing.Credentials) > 0 {
|
||||
if body.Kind == "spider91" {
|
||||
credentials, err := mergeSpider91Credentials(existing, body.Credentials)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
body.Credentials = credentials
|
||||
} else if len(body.Credentials) == 0 && existing != nil && len(existing.Credentials) > 0 {
|
||||
body.Credentials = existing.Credentials
|
||||
}
|
||||
|
||||
@@ -552,8 +572,82 @@ func (a *AdminServer) handleUpsertDrive(w http.ResponseWriter, r *http.Request)
|
||||
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
func spider91ProxyForDrive(d *catalog.Drive) string {
|
||||
if d == nil || d.Kind != "spider91" || d.Credentials == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(d.Credentials["proxy"])
|
||||
}
|
||||
|
||||
func mergeSpider91Credentials(existing *catalog.Drive, incoming map[string]string) (map[string]string, error) {
|
||||
merged := map[string]string{}
|
||||
if existing != nil {
|
||||
for k, v := range existing.Credentials {
|
||||
merged[k] = v
|
||||
}
|
||||
}
|
||||
for k, v := range incoming {
|
||||
if strings.TrimSpace(k) == "" {
|
||||
continue
|
||||
}
|
||||
if k == "proxy" {
|
||||
proxy, err := normalizeSpider91ProxyURL(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if proxy == "" {
|
||||
delete(merged, "proxy")
|
||||
} else {
|
||||
merged["proxy"] = proxy
|
||||
}
|
||||
continue
|
||||
}
|
||||
merged[k] = v
|
||||
}
|
||||
return merged, nil
|
||||
}
|
||||
|
||||
func normalizeSpider91ProxyURL(raw string) (string, error) {
|
||||
proxy := strings.TrimSpace(raw)
|
||||
if proxy == "" {
|
||||
return "", nil
|
||||
}
|
||||
u, err := url.Parse(proxy)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return "", fmt.Errorf("91Spider 代理地址格式无效,请填写类似 http://127.0.0.1:7890 的地址")
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https", "socks5", "socks5h":
|
||||
return proxy, nil
|
||||
default:
|
||||
return "", fmt.Errorf("91Spider 代理地址仅支持 http://、https://、socks5:// 或 socks5h://")
|
||||
}
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleDeleteDrive(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
var body deleteDriveReq
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
|
||||
writeErr(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
if !body.DeleteVideos {
|
||||
http.Error(w, "deleteVideos=true is required when deleting a drive", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
deletedVideos := 0
|
||||
if a.OnDriveDeleteCleanup == nil {
|
||||
http.Error(w, "drive video cleanup is not available", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
removed, err := a.OnDriveDeleteCleanup(r.Context(), id)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
deletedVideos = removed
|
||||
|
||||
if err := a.Catalog.DeleteDrive(r.Context(), id); err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
@@ -561,7 +655,11 @@ func (a *AdminServer) handleDeleteDrive(w http.ResponseWriter, r *http.Request)
|
||||
if a.OnDriveRemoved != nil {
|
||||
a.OnDriveRemoved(id)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
|
||||
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "deletedVideos": deletedVideos})
|
||||
}
|
||||
|
||||
type deleteDriveReq struct {
|
||||
DeleteVideos bool `json:"deleteVideos"`
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleRescan(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -572,6 +670,39 @@ func (a *AdminServer) handleRescan(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
func (a *AdminServer) p123QRClient() *p123.QRClient {
|
||||
return p123.NewQRClient(p123.QRConfig{
|
||||
UserAPIBaseURL: a.P123UserAPIBaseURL,
|
||||
HTTPClient: a.P123HTTPClient,
|
||||
})
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleP123QRStart(w http.ResponseWriter, r *http.Request) {
|
||||
session, err := a.p123QRClient().Generate(r.Context())
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusBadGateway, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, http.StatusOK, session)
|
||||
}
|
||||
|
||||
func (a *AdminServer) handleP123QRStatus(w http.ResponseWriter, r *http.Request) {
|
||||
uniID := chi.URLParam(r, "uniID")
|
||||
loginUUID := r.URL.Query().Get("loginUuid")
|
||||
if strings.TrimSpace(uniID) == "" || strings.TrimSpace(loginUUID) == "" {
|
||||
http.Error(w, "uniID and loginUuid are required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
status, err := a.p123QRClient().Poll(r.Context(), loginUUID, uniID)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusBadGateway, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, http.StatusOK, status)
|
||||
}
|
||||
|
||||
// handleRunNightlyJob 触发一次完整的凌晨流水线(不论当前时间,不论今日是否已跑)。
|
||||
// 立即返回 202;进度通过 backend 日志和下次 GET /admin/api/drives 的状态变化观察。
|
||||
// 流水线已在跑或已排队时,Runner 会拒绝重复触发。
|
||||
@@ -907,6 +1038,16 @@ func (a *AdminServer) handleRegenFailedThumbnails(w http.ResponseWriter, r *http
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
// handleRegenFailedFingerprints triggers regeneration for all failed sampled
|
||||
// fingerprints on a drive. It mirrors the failed teaser/thumbnail retry endpoints.
|
||||
func (a *AdminServer) handleRegenFailedFingerprints(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
if a.OnRegenFailedFingerprints != nil {
|
||||
a.OnRegenFailedFingerprints(id)
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
// ---------- Settings ----------
|
||||
|
||||
// settingsDTO 是 GET/PUT /admin/api/settings 的入参/出参。
|
||||
|
||||
@@ -3,6 +3,7 @@ package api
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -439,6 +440,300 @@ func TestHandleUpsertDriveReplacesExistingCredentialsWhenProvided(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertSpider91ProxyPreservesRuntimeCredentials(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "spider91-main",
|
||||
Kind: "spider91",
|
||||
Name: "91 Spider",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"last_crawl_at": "1800000000",
|
||||
"proxy": "http://old-proxy.local:7890",
|
||||
"script_path": "/opt/video-site-91/91VideoSpider/spider_91porn.py",
|
||||
},
|
||||
Status: "ok",
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"kind": "spider91",
|
||||
"name": "91 Spider",
|
||||
"rootId": "/",
|
||||
"credentials": {"proxy": " socks5h://proxy-user:proxy-pass@127.0.0.1:7891 "}
|
||||
}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
got, err := cat.GetDrive(ctx, "spider91-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get drive: %v", err)
|
||||
}
|
||||
if got.Credentials["proxy"] != "socks5h://proxy-user:proxy-pass@127.0.0.1:7891" {
|
||||
t.Fatalf("proxy = %q, want trimmed new proxy", got.Credentials["proxy"])
|
||||
}
|
||||
if got.Credentials["last_crawl_at"] != "1800000000" {
|
||||
t.Fatalf("last_crawl_at = %q, want preserved", got.Credentials["last_crawl_at"])
|
||||
}
|
||||
if got.Credentials["script_path"] == "" {
|
||||
t.Fatalf("script_path should be preserved")
|
||||
}
|
||||
|
||||
req = httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"kind": "spider91",
|
||||
"name": "91 Spider",
|
||||
"rootId": "/",
|
||||
"credentials": {"proxy": " "}
|
||||
}`))
|
||||
rr = httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("clear status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
got, err = cat.GetDrive(ctx, "spider91-main")
|
||||
if err != nil {
|
||||
t.Fatalf("get cleared drive: %v", err)
|
||||
}
|
||||
if _, ok := got.Credentials["proxy"]; ok {
|
||||
t.Fatalf("proxy should be removed after empty save, got %q", got.Credentials["proxy"])
|
||||
}
|
||||
if got.Credentials["last_crawl_at"] != "1800000000" {
|
||||
t.Fatalf("last_crawl_at after clear = %q, want preserved", got.Credentials["last_crawl_at"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpsertSpider91RejectsUnsupportedProxyScheme(t *testing.T) {
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives", strings.NewReader(`{
|
||||
"id": "spider91-main",
|
||||
"kind": "spider91",
|
||||
"name": "91 Spider",
|
||||
"rootId": "/",
|
||||
"credentials": {"proxy": "ftp://127.0.0.1:21"}
|
||||
}`))
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleUpsertDrive(rr, req)
|
||||
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), "socks5:// 或 socks5h://") {
|
||||
t.Fatalf("body = %q, want supported schemes message", rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleDeleteDriveRunsRequestedCleanupBeforeDeletingDrive(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "drive-one",
|
||||
Kind: "pikpak",
|
||||
Name: "Drive One",
|
||||
RootID: "root",
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
cleanupCalled := ""
|
||||
removedCalled := ""
|
||||
req := httptest.NewRequest(http.MethodDelete, "/admin/api/drives/drive-one", strings.NewReader(`{"deleteVideos":true}`))
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "drive-one")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
Catalog: cat,
|
||||
OnDriveDeleteCleanup: func(cleanupCtx context.Context, driveID string) (int, error) {
|
||||
cleanupCalled = driveID
|
||||
if _, err := cat.GetDrive(cleanupCtx, driveID); err != nil {
|
||||
t.Fatalf("drive should still exist during cleanup: %v", err)
|
||||
}
|
||||
return 3, nil
|
||||
},
|
||||
OnDriveRemoved: func(driveID string) {
|
||||
removedCalled = driveID
|
||||
},
|
||||
}).handleDeleteDrive(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if cleanupCalled != "drive-one" {
|
||||
t.Fatalf("cleanup called with %q, want drive-one", cleanupCalled)
|
||||
}
|
||||
if removedCalled != "drive-one" {
|
||||
t.Fatalf("removed hook called with %q, want drive-one", removedCalled)
|
||||
}
|
||||
if _, err := cat.GetDrive(ctx, "drive-one"); err != sql.ErrNoRows {
|
||||
t.Fatalf("drive lookup error = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
var got struct {
|
||||
OK bool `json:"ok"`
|
||||
DeletedVideos int `json:"deletedVideos"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if !got.OK || got.DeletedVideos != 3 {
|
||||
t.Fatalf("response = %#v, want ok with deletedVideos=3", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleDeleteDriveRequiresCleanupConfirmation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
if err := cat.UpsertDrive(ctx, &catalog.Drive{
|
||||
ID: "drive-one",
|
||||
Kind: "pikpak",
|
||||
Name: "Drive One",
|
||||
RootID: "root",
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodDelete, "/admin/api/drives/drive-one", strings.NewReader(`{"deleteVideos":false}`))
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "drive-one")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
Catalog: cat,
|
||||
OnDriveDeleteCleanup: func(context.Context, string) (int, error) {
|
||||
t.Fatal("cleanup hook should not be called without confirmation")
|
||||
return 0, nil
|
||||
},
|
||||
}).handleDeleteDrive(rr, req)
|
||||
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400; body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if _, err := cat.GetDrive(ctx, "drive-one"); err != nil {
|
||||
t.Fatalf("drive should remain after rejected delete: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListDrivesIncludesSpider91Proxy(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, d := range []*catalog.Drive{
|
||||
{
|
||||
ID: "spider91-main",
|
||||
Kind: "spider91",
|
||||
Name: "91 Spider",
|
||||
RootID: "/",
|
||||
Credentials: map[string]string{
|
||||
"last_crawl_at": "1800000000",
|
||||
"proxy": " http://127.0.0.1:7890 ",
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
{
|
||||
ID: "onedrive-main",
|
||||
Kind: "onedrive",
|
||||
Name: "OneDrive",
|
||||
RootID: "root",
|
||||
Credentials: map[string]string{
|
||||
"proxy": "http://should-not-leak.local:7890",
|
||||
},
|
||||
Status: "ok",
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, d); err != nil {
|
||||
t.Fatalf("seed drive %s: %v", d.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/drives", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminServer{Catalog: cat}).handleListDrives(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
var got []struct {
|
||||
ID string `json:"id"`
|
||||
Spider91Proxy string `json:"spider91Proxy"`
|
||||
LastCrawlAt int64 `json:"lastCrawlAt"`
|
||||
}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
byID := map[string]struct {
|
||||
Spider91Proxy string
|
||||
LastCrawlAt int64
|
||||
}{}
|
||||
for _, d := range got {
|
||||
byID[d.ID] = struct {
|
||||
Spider91Proxy string
|
||||
LastCrawlAt int64
|
||||
}{Spider91Proxy: d.Spider91Proxy, LastCrawlAt: d.LastCrawlAt}
|
||||
}
|
||||
if byID["spider91-main"].Spider91Proxy != "http://127.0.0.1:7890" {
|
||||
t.Fatalf("spider91 proxy = %q, want trimmed proxy", byID["spider91-main"].Spider91Proxy)
|
||||
}
|
||||
if byID["spider91-main"].LastCrawlAt != 1800000000 {
|
||||
t.Fatalf("lastCrawlAt = %d, want 1800000000", byID["spider91-main"].LastCrawlAt)
|
||||
}
|
||||
if byID["onedrive-main"].Spider91Proxy != "" {
|
||||
t.Fatalf("onedrive spider91Proxy = %q, want empty", byID["onedrive-main"].Spider91Proxy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -598,6 +893,28 @@ func TestHandleListDrivesIncludesTeaserCounts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRegenFailedFingerprintsInvokesHook(t *testing.T) {
|
||||
called := ""
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/api/drives/drive-one/fingerprints/failed/regenerate", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "drive-one")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
(&AdminServer{
|
||||
OnRegenFailedFingerprints: func(driveID string) {
|
||||
called = driveID
|
||||
},
|
||||
}).handleRegenFailedFingerprints(rr, req)
|
||||
|
||||
if rr.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if called != "drive-one" {
|
||||
t.Fatalf("called drive = %q, want drive-one", called)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleDriveStorageReportsLocalMediaUsage(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
|
||||
+93
-35
@@ -15,6 +15,7 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
@@ -53,6 +54,10 @@ type Server struct {
|
||||
UploadDir string
|
||||
OnVideoUploaded func(*catalog.Video)
|
||||
|
||||
tagCacheMu sync.Mutex
|
||||
tagCacheUntil time.Time
|
||||
tagCache []TagDTO
|
||||
|
||||
// GetTheme 返回当前生效的主题("dark" | "pink")。前台 /api/settings/theme 用,
|
||||
// 不需要登录。无注入时返回 "dark"。
|
||||
GetTheme func() string
|
||||
@@ -86,6 +91,12 @@ type VideoDTO struct {
|
||||
Category string `json:"category,omitempty"`
|
||||
}
|
||||
|
||||
type TagDTO struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
type VideoDetailDTO struct {
|
||||
VideoDTO
|
||||
VideoSrc string `json:"videoSrc"`
|
||||
@@ -156,41 +167,60 @@ func (s *Server) handleGetTheme(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) {
|
||||
// 首页优先展示封面已经生成好的视频,避免新盘扫盘时大量黑封面占满首页。
|
||||
// 候选仍按发布时间覆盖最近 200 个,随后随机洗牌;封面不足时再用普通可见视频补齐。
|
||||
const candidatePool = 200
|
||||
readyItems, _, err := s.Catalog.ListVideos(r.Context(), catalog.ListParams{
|
||||
Sort: "latest", Page: 1, PageSize: candidatePool, ThumbnailReadyOnly: true,
|
||||
})
|
||||
// 首页优先从全量已有封面的视频里随机抽取,避免只在最近一小段候选中反复出现。
|
||||
excludeIDs := parseVideoIDQuery(r, "exclude", 120)
|
||||
items, err := s.Catalog.RandomVideosWithReadyThumbnailsExcluding(r.Context(), excludeIDs, homePageSize)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
rand.Shuffle(len(readyItems), func(i, j int) {
|
||||
readyItems[i], readyItems[j] = readyItems[j], readyItems[i]
|
||||
})
|
||||
|
||||
items := appendUniqueVideos(nil, readyItems, homePageSize)
|
||||
if len(items) > homePageSize {
|
||||
items = items[:homePageSize]
|
||||
}
|
||||
if len(items) < homePageSize {
|
||||
fallback, _, err := s.Catalog.ListVideos(r.Context(), catalog.ListParams{
|
||||
Sort: "latest", Page: 1, PageSize: candidatePool,
|
||||
})
|
||||
fallbackExclude := append([]string{}, excludeIDs...)
|
||||
for _, item := range items {
|
||||
if item != nil {
|
||||
fallbackExclude = append(fallbackExclude, item.ID)
|
||||
}
|
||||
}
|
||||
fallback, err := s.Catalog.RandomVideosExcluding(r.Context(), fallbackExclude, homePageSize-len(items))
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
rand.Shuffle(len(fallback), func(i, j int) {
|
||||
fallback[i], fallback[j] = fallback[j], fallback[i]
|
||||
})
|
||||
items = appendUniqueVideos(items, fallback, homePageSize)
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, http.StatusOK, mapVideos(items))
|
||||
}
|
||||
|
||||
func parseVideoIDQuery(r *http.Request, key string, limit int) []string {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
values := r.URL.Query()[key]
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
out := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
for _, id := range strings.Split(value, ",") {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
out = append(out, id)
|
||||
if limit > 0 && len(out) >= limit {
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func appendUniqueVideos(dst []*catalog.Video, candidates []*catalog.Video, limit int) []*catalog.Video {
|
||||
if len(dst) >= limit {
|
||||
return dst[:limit]
|
||||
@@ -226,12 +256,13 @@ func (s *Server) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
sort := q.Get("sort")
|
||||
params := catalog.ListParams{
|
||||
Keyword: q.Get("q"),
|
||||
Tag: q.Get("tag"),
|
||||
Category: q.Get("cat"),
|
||||
Sort: sort,
|
||||
Page: page,
|
||||
PageSize: size,
|
||||
Keyword: q.Get("q"),
|
||||
Tag: q.Get("tag"),
|
||||
Category: q.Get("cat"),
|
||||
Sort: sort,
|
||||
Page: page,
|
||||
PageSize: size,
|
||||
SkipTotal: strings.EqualFold(q.Get("count"), "false"),
|
||||
}
|
||||
if sort == "" || sort == "latest" {
|
||||
params.PreferReadyThumbnails = true
|
||||
@@ -260,6 +291,15 @@ func (s *Server) handleVideoDetail(w http.ResponseWriter, r *http.Request) {
|
||||
writeErr(w, http.StatusNotFound, sql.ErrNoRows)
|
||||
return
|
||||
}
|
||||
if v.DriveID != localUploadDriveID {
|
||||
if _, err := s.Catalog.GetDrive(r.Context(), v.DriveID); err != nil {
|
||||
drives, listErr := s.Catalog.ListDrives(r.Context())
|
||||
if listErr != nil || len(drives) > 0 {
|
||||
writeErr(w, http.StatusNotFound, sql.ErrNoRows)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
related := s.pickRelatedVideos(r.Context(), v, 6)
|
||||
dto := mapVideo(v)
|
||||
if d, err := s.Catalog.GetDrive(r.Context(), v.DriveID); err == nil {
|
||||
@@ -423,20 +463,32 @@ func appendRandomRelated(picked []*catalog.Video, pool []*catalog.Video, targetL
|
||||
}
|
||||
|
||||
func (s *Server) handleTags(w http.ResponseWriter, r *http.Request) {
|
||||
now := time.Now()
|
||||
s.tagCacheMu.Lock()
|
||||
if s.tagCache != nil && now.Before(s.tagCacheUntil) {
|
||||
out := append([]TagDTO(nil), s.tagCache...)
|
||||
s.tagCacheMu.Unlock()
|
||||
w.Header().Set("Cache-Control", "private, max-age=15")
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
return
|
||||
}
|
||||
s.tagCacheMu.Unlock()
|
||||
|
||||
stats, err := s.Catalog.ListTags(r.Context())
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
type tag struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
out := make([]tag, 0, len(stats))
|
||||
out := make([]TagDTO, 0, len(stats))
|
||||
for _, stat := range stats {
|
||||
out = append(out, tag{ID: stat.Label, Label: stat.Label, Count: stat.Count})
|
||||
out = append(out, TagDTO{ID: stat.Label, Label: stat.Label, Count: stat.Count})
|
||||
}
|
||||
s.tagCacheMu.Lock()
|
||||
s.tagCache = append([]TagDTO(nil), out...)
|
||||
s.tagCacheUntil = now.Add(30 * time.Second)
|
||||
s.tagCacheMu.Unlock()
|
||||
|
||||
w.Header().Set("Cache-Control", "private, max-age=15")
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
@@ -867,10 +919,14 @@ func previewURL(v *catalog.Video) string {
|
||||
}
|
||||
|
||||
func thumbnailURL(v *catalog.Video) string {
|
||||
base := "/p/thumb/" + v.ID
|
||||
if v.ThumbnailURL != "" {
|
||||
return v.ThumbnailURL
|
||||
base = v.ThumbnailURL
|
||||
}
|
||||
return "/p/thumb/" + v.ID
|
||||
if !strings.HasPrefix(base, "/p/thumb/") || v.UpdatedAt.IsZero() {
|
||||
return base
|
||||
}
|
||||
return base + "?v=" + strconv.FormatInt(v.UpdatedAt.UnixMilli(), 10)
|
||||
}
|
||||
|
||||
func (s *Server) videoSource(v *catalog.Video) string {
|
||||
@@ -900,6 +956,8 @@ func driveKindLabel(kind string) string {
|
||||
return "夸克网盘"
|
||||
case "p115":
|
||||
return "115 网盘"
|
||||
case "p123":
|
||||
return "123 云盘"
|
||||
case "pikpak":
|
||||
return "PikPak"
|
||||
case "wopan":
|
||||
|
||||
@@ -99,6 +99,27 @@ func TestPreviewURLFallsBackWithoutUpdatedAt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailURLVersionsLocalGeneratedThumbnails(t *testing.T) {
|
||||
got := thumbnailURL(&catalog.Video{
|
||||
ID: "video-1",
|
||||
ThumbnailURL: "/p/thumb/video-1",
|
||||
UpdatedAt: time.UnixMilli(1778863000123),
|
||||
})
|
||||
if got != "/p/thumb/video-1?v=1778863000123" {
|
||||
t.Fatalf("thumbnail URL = %q, want versioned local URL", got)
|
||||
}
|
||||
|
||||
remote := "https://thumb.example/video-1.jpg"
|
||||
got = thumbnailURL(&catalog.Video{
|
||||
ID: "video-1",
|
||||
ThumbnailURL: remote,
|
||||
UpdatedAt: time.UnixMilli(1778863000123),
|
||||
})
|
||||
if got != remote {
|
||||
t.Fatalf("remote thumbnail URL = %q, want unchanged %q", got, remote)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleHomePrioritizesVideosWithReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -166,6 +187,59 @@ func TestHandleHomePrioritizesVideosWithReadyThumbnails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleHomeExcludesRecentlyShownVideos(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < homePageSize+4; i++ {
|
||||
id := "ready-video-" + strconv.Itoa(i)
|
||||
if err := cat.UpsertVideo(ctx, &catalog.Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: id,
|
||||
ThumbnailURL: "https://thumb.example/" + id + ".jpg",
|
||||
PublishedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
CreatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
UpdatedAt: now.Add(time.Duration(i) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready video %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/home?exclude=ready-video-0&exclude=ready-video-1", nil)
|
||||
(&Server{Catalog: cat}).handleHome(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var got []VideoDTO
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(got) != homePageSize {
|
||||
t.Fatalf("home items = %d, want %d", len(got), homePageSize)
|
||||
}
|
||||
for _, item := range got {
|
||||
if item.ID == "ready-video-0" || item.ID == "ready-video-1" {
|
||||
t.Fatalf("home returned excluded video %q; items=%#v", item.ID, got)
|
||||
}
|
||||
if !strings.HasPrefix(item.ID, "ready-video-") {
|
||||
t.Fatalf("home returned %q without a ready thumbnail; items=%#v", item.ID, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListLatestPrefersReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
@@ -237,6 +311,26 @@ func TestHandleListLatestPrefersReadyThumbnails(t *testing.T) {
|
||||
t.Fatalf("thumbnail for %q = %q, want ready thumbnail URL", item.ID, item.Thumbnail)
|
||||
}
|
||||
}
|
||||
|
||||
rr = httptest.NewRecorder()
|
||||
req = httptest.NewRequest(http.MethodGet, "/api/list?page=1&size=12&sort=latest&count=false", nil)
|
||||
(&Server{Catalog: cat}).handleList(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("count=false status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
got = struct {
|
||||
Items []VideoDTO `json:"items"`
|
||||
Total int `json:"total"`
|
||||
}{}
|
||||
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode count=false response: %v", err)
|
||||
}
|
||||
if got.Total != 0 {
|
||||
t.Fatalf("count=false total = %d, want 0", got.Total)
|
||||
}
|
||||
if len(got.Items) != 12 {
|
||||
t.Fatalf("count=false items = %d, want 12", len(got.Items))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUploadVideoSavesFileVideoTagsAndQueuesPreview(t *testing.T) {
|
||||
|
||||
@@ -622,6 +622,56 @@ func (c *Catalog) ListVideosByDrive(ctx context.Context, driveID string) ([]*Vid
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) ListVideosByIDPrefix(ctx context.Context, prefix string) ([]*Video, error) {
|
||||
prefix = strings.TrimSpace(prefix)
|
||||
if prefix == "" {
|
||||
return nil, fmt.Errorf("catalog: list videos by id prefix: empty prefix")
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE SUBSTR(id, 1, LENGTH(?)) = ?
|
||||
ORDER BY created_at ASC, id ASC`,
|
||||
prefix, prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) ListVideosWithMissingDrive(ctx context.Context) ([]*Video, error) {
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE drive_id != 'local-upload'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM drives
|
||||
WHERE drives.id = videos.drive_id
|
||||
)
|
||||
ORDER BY drive_id ASC, id ASC`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListVideoFileIDsByDrive 只返回某 drive 下所有视频的 file_id 集合,
|
||||
// 比 ListVideosByDrive 轻量。
|
||||
func (c *Catalog) ListVideoFileIDsByDrive(ctx context.Context, driveID string) ([]string, error) {
|
||||
@@ -765,6 +815,37 @@ func (c *Catalog) ListVideosNeedingFingerprint(ctx context.Context, driveID stri
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListVideosByFingerprintStatus lists visible videos on a drive by fingerprint status.
|
||||
// It is used by the admin "retry failed fingerprints" action to reset failed rows
|
||||
// back to pending and enqueue them again.
|
||||
func (c *Catalog) ListVideosByFingerprintStatus(ctx context.Context, driveID, status string, limit int) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10000
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
`SELECT `+allVideoCols+` FROM videos
|
||||
WHERE drive_id = ?
|
||||
AND COALESCE(sampled_sha256, '') = ''
|
||||
AND COALESCE(fingerprint_status, 'pending') = ?
|
||||
AND COALESCE(hidden, 0) = 0
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT ?`,
|
||||
driveID, status, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*Video
|
||||
for rows.Next() {
|
||||
v, err := scanVideo(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) UpdateVideoFingerprint(ctx context.Context, id, sampledSHA256, status, errText string) error {
|
||||
sampledSHA256 = normalizeContentHash(sampledSHA256)
|
||||
if status == "" {
|
||||
@@ -798,6 +879,7 @@ type ListParams struct {
|
||||
Sort string // latest | hot | week | long
|
||||
ThumbnailReadyOnly bool
|
||||
PreferReadyThumbnails bool
|
||||
SkipTotal bool
|
||||
Page int
|
||||
PageSize int
|
||||
}
|
||||
@@ -822,12 +904,7 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
|
||||
args = append(args, p.DriveID)
|
||||
}
|
||||
if p.Tag != "" {
|
||||
where = append(where, `EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
JOIN tags t ON t.id = vt.tag_id
|
||||
WHERE vt.video_id = videos.id AND t.label = ? COLLATE NOCASE
|
||||
)`)
|
||||
where = append(where, videoMatchesTagLabelSQL("videos"))
|
||||
args = append(args, p.Tag)
|
||||
}
|
||||
if p.Category != "" && p.Category != "all" {
|
||||
@@ -838,6 +915,7 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
|
||||
where = append(where, "COALESCE(thumbnail_url, '') != ''")
|
||||
}
|
||||
where = append(where, "COALESCE(hidden, 0) = 0")
|
||||
where = append(where, activeDriveWhereSQL)
|
||||
where = append(where, uniqueVideoWhereSQL)
|
||||
|
||||
whereSQL := ""
|
||||
@@ -859,10 +937,11 @@ func (c *Catalog) ListVideos(ctx context.Context, p ListParams) ([]*Video, int,
|
||||
orderBy = " ORDER BY " + readyOrderPrefix + "duration_seconds DESC"
|
||||
}
|
||||
|
||||
// count
|
||||
var total int
|
||||
if err := c.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM videos"+whereSQL, args...).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
if !p.SkipTotal {
|
||||
if err := c.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM videos"+whereSQL, args...).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
}
|
||||
|
||||
// list
|
||||
@@ -893,6 +972,7 @@ func (c *Catalog) CountVisibleVideos(ctx context.Context) (int, error) {
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM videos
|
||||
WHERE COALESCE(hidden, 0) = 0
|
||||
AND `+activeDriveWhereSQL+`
|
||||
AND `+uniqueVideoWhereSQL,
|
||||
).Scan(&total)
|
||||
if err != nil {
|
||||
@@ -906,6 +986,14 @@ func (c *Catalog) CountVisibleVideos(ctx context.Context) (int, error) {
|
||||
// 如果剩余可选数量 < limit,就返回所有可选项;调用方负责判断是否需要开新一轮。
|
||||
// limit <= 0 时返回 nil, nil。
|
||||
func (c *Catalog) RandomVideosExcluding(ctx context.Context, excludeIDs []string, limit int) ([]*Video, error) {
|
||||
return c.randomVideosExcluding(ctx, excludeIDs, limit, false)
|
||||
}
|
||||
|
||||
func (c *Catalog) RandomVideosWithReadyThumbnailsExcluding(ctx context.Context, excludeIDs []string, limit int) ([]*Video, error) {
|
||||
return c.randomVideosExcluding(ctx, excludeIDs, limit, true)
|
||||
}
|
||||
|
||||
func (c *Catalog) randomVideosExcluding(ctx context.Context, excludeIDs []string, limit int, thumbnailReadyOnly bool) ([]*Video, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -913,7 +1001,11 @@ func (c *Catalog) RandomVideosExcluding(ctx context.Context, excludeIDs []string
|
||||
cleaned := cleanVideoIDs(excludeIDs)
|
||||
args := make([]any, 0, len(cleaned)+1)
|
||||
whereSQL := `WHERE COALESCE(hidden, 0) = 0
|
||||
AND ` + activeDriveWhereSQL + `
|
||||
AND ` + uniqueVideoWhereSQL
|
||||
if thumbnailReadyOnly {
|
||||
whereSQL += " AND COALESCE(thumbnail_url, '') != ''"
|
||||
}
|
||||
if len(cleaned) > 0 {
|
||||
placeholders := strings.Repeat("?,", len(cleaned))
|
||||
placeholders = placeholders[:len(placeholders)-1]
|
||||
@@ -993,6 +1085,7 @@ func (c *Catalog) LeastPopulatedVisibleUniqueTag(ctx context.Context, labels []s
|
||||
`SELECT COUNT(*)
|
||||
FROM videos
|
||||
WHERE COALESCE(hidden, 0) = 0
|
||||
AND `+activeDriveWhereSQL+`
|
||||
AND `+uniqueVideoWhereSQL+`
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
@@ -1029,6 +1122,7 @@ func (c *Catalog) RandomVideosByTagExcluding(ctx context.Context, tag string, ex
|
||||
args := make([]any, 0, len(cleaned)+2)
|
||||
args = append(args, tag)
|
||||
whereSQL := `WHERE COALESCE(hidden, 0) = 0
|
||||
AND ` + activeDriveWhereSQL + `
|
||||
AND ` + uniqueVideoWhereSQL + `
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
@@ -1472,10 +1566,7 @@ func normalizeDriveRootID(kind, rootID string) string {
|
||||
}
|
||||
return rootID
|
||||
case "localstorage", "spider91":
|
||||
if rootID == "" {
|
||||
return "/"
|
||||
}
|
||||
return rootID
|
||||
return "/"
|
||||
default:
|
||||
if rootID == "" {
|
||||
return "0"
|
||||
@@ -1679,6 +1770,17 @@ COALESCE(category, ''), COALESCE(hidden, 0), COALESCE(badges, '[]'), COALESCE(de
|
||||
published_at, created_at, updated_at
|
||||
`
|
||||
|
||||
const activeDriveWhereSQL = `(videos.drive_id = 'local-upload'
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM drives
|
||||
WHERE drives.id = videos.drive_id
|
||||
)
|
||||
OR NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM drives
|
||||
))`
|
||||
|
||||
const uniqueVideoWhereSQL = `((COALESCE(videos.content_hash, '') = ''
|
||||
OR NOT EXISTS (
|
||||
SELECT 1
|
||||
|
||||
@@ -61,6 +61,7 @@ func TestUpsertDriveDefaultsRootIDByKind(t *testing.T) {
|
||||
{id: "onedrive", kind: "onedrive", want: "root"},
|
||||
{id: "googledrive", kind: "googledrive", want: "root"},
|
||||
{id: "localstorage", kind: "localstorage", want: "/"},
|
||||
{id: "spider91", kind: "spider91", want: "/"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
@@ -82,3 +83,44 @@ func TestUpsertDriveDefaultsRootIDByKind(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertDriveIgnoresRootIDForLocalStorageAndSpider91(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for _, tc := range []struct {
|
||||
id string
|
||||
kind string
|
||||
}{
|
||||
{id: "localstorage", kind: "localstorage"},
|
||||
{id: "spider91", kind: "spider91"},
|
||||
} {
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: tc.id,
|
||||
Kind: tc.kind,
|
||||
Name: tc.kind,
|
||||
RootID: "manual-root",
|
||||
ScanRootID: "manual-scan-root",
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert %s: %v", tc.kind, err)
|
||||
}
|
||||
got, err := cat.GetDrive(ctx, tc.id)
|
||||
if err != nil {
|
||||
t.Fatalf("get %s: %v", tc.kind, err)
|
||||
}
|
||||
if got.RootID != "/" {
|
||||
t.Fatalf("%s rootId = %q, want /", tc.kind, got.RootID)
|
||||
}
|
||||
if got.ScanRootID != "/" {
|
||||
t.Fatalf("%s scanRootId = %q, want /", tc.kind, got.ScanRootID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestListVideosHidesMissingDriveVideosWhenDrivesExist(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: "active-drive",
|
||||
Kind: "pikpak",
|
||||
Name: "Active",
|
||||
RootID: "root",
|
||||
TeaserEnabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed drive: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{
|
||||
ID: "visible-video",
|
||||
DriveID: "active-drive",
|
||||
FileID: "visible-file",
|
||||
Title: "Visible",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "orphan-video",
|
||||
DriveID: "deleted-drive",
|
||||
FileID: "orphan-file",
|
||||
Title: "Orphan",
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed video %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{Page: 1, PageSize: 10, Sort: "latest"})
|
||||
if err != nil {
|
||||
t.Fatalf("list videos: %v", err)
|
||||
}
|
||||
if total != 1 || len(items) != 1 || items[0].ID != "visible-video" {
|
||||
t.Fatalf("items total=%d items=%v, want only visible-video", total, items)
|
||||
}
|
||||
}
|
||||
@@ -73,7 +73,7 @@ CREATE TABLE IF NOT EXISTS deleted_tags (
|
||||
-- 网盘账户
|
||||
CREATE TABLE IF NOT EXISTS drives (
|
||||
id TEXT PRIMARY KEY,
|
||||
kind TEXT NOT NULL, -- quark / p115 / pikpak / wopan / onedrive / googledrive / localstorage / spider91
|
||||
kind TEXT NOT NULL, -- quark / p115 / p123 / pikpak / wopan / onedrive / googledrive / localstorage / spider91
|
||||
name TEXT NOT NULL,
|
||||
root_id TEXT NOT NULL DEFAULT '0',
|
||||
scan_root_id TEXT, -- deprecated: 扫描起点固定等于 root_id
|
||||
|
||||
@@ -110,6 +110,62 @@ func TestRandomVideosExcluding(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosWithReadyThumbnailsExcluding(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cat.Close() })
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < 4; i++ {
|
||||
id := "ready-" + string(rune('a'+i))
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: "f-" + id,
|
||||
Title: id,
|
||||
ThumbnailURL: "/p/thumb/" + id,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
id := "pending-" + string(rune('a'+i))
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: "f-" + id,
|
||||
Title: id,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := cat.RandomVideosWithReadyThumbnailsExcluding(ctx, []string{"ready-a"}, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("random ready excluding: %v", err)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("ready random count = %d, want 3", len(got))
|
||||
}
|
||||
for _, v := range got {
|
||||
if v.ID == "ready-a" {
|
||||
t.Fatal("excluded ready video was returned")
|
||||
}
|
||||
if v.ThumbnailURL == "" {
|
||||
t.Fatalf("pending video %q was returned", v.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomVideosForPreferredVideoChoosesLeastPopulatedTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
|
||||
@@ -100,15 +100,27 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_content_hash ON videos(content_hash)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_content_hash_created ON videos(content_hash, created_at, id)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_sampled_sha256 ON videos(size_bytes, sampled_sha256)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_sampled_sha256_created ON videos(size_bytes, sampled_sha256, created_at, id)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_hidden ON videos(hidden)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_visible_pub ON videos(COALESCE(hidden, 0), published_at DESC)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_file_name_size ON videos(file_name, size_bytes)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_videos_file_name_size_created ON videos(file_name, size_bytes, created_at, id)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.seedSystemTags(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -127,6 +139,12 @@ func (c *Catalog) migrate(ctx context.Context) error {
|
||||
if err := c.clearVolatileOneDriveThumbnails(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.clearRemoteP123ThumbnailsOnce(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.clearRemoteNonSpider91Thumbnails(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.hideZeroSizeVideosFromKnownDrives(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -257,6 +275,85 @@ UPDATE videos
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) clearRemoteP123ThumbnailsOnce(ctx context.Context) error {
|
||||
// 123 云盘列表返回的缩略图尺寸和稳定性都不适合作为站内封面;清空历史写入的
|
||||
// 远程 URL,让封面 worker 统一从视频直链抽帧生成本地 /p/thumb/<id>。
|
||||
const markerKey = "videos.p123.remote_thumbnails_cleared"
|
||||
marker, err := c.GetSetting(ctx, markerKey, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read %s marker: %w", markerKey, err)
|
||||
}
|
||||
if strings.TrimSpace(marker) == "1" {
|
||||
return nil
|
||||
}
|
||||
|
||||
var p123Drives int
|
||||
if err := c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM drives WHERE kind = 'p123'`).Scan(&p123Drives); err != nil {
|
||||
return fmt.Errorf("count p123 drives: %w", err)
|
||||
}
|
||||
if p123Drives == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
res, err := c.db.ExecContext(ctx, `
|
||||
UPDATE videos
|
||||
SET thumbnail_url = '',
|
||||
thumbnail_status = 'pending',
|
||||
thumbnail_failures = 0,
|
||||
updated_at = ?
|
||||
WHERE EXISTS (
|
||||
SELECT 1
|
||||
FROM drives
|
||||
WHERE drives.id = videos.drive_id
|
||||
AND drives.kind = 'p123'
|
||||
)
|
||||
AND (
|
||||
lower(COALESCE(thumbnail_url, '')) LIKE 'http://%'
|
||||
OR lower(COALESCE(thumbnail_url, '')) LIKE 'https://%'
|
||||
)
|
||||
`, time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected > 0 {
|
||||
log.Printf("[catalog] cleared %d remote 123pan thumbnail(s) for local regeneration", affected)
|
||||
}
|
||||
if err := c.SetSetting(ctx, markerKey, "1"); err != nil {
|
||||
return fmt.Errorf("write %s marker: %w", markerKey, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Catalog) clearRemoteNonSpider91Thumbnails(ctx context.Context) error {
|
||||
// 非 91Spider 视频不再使用网盘侧返回的远程缩略图。清空历史 http/https
|
||||
// thumbnail_url 后,封面 worker 会重新从视频中间帧生成本地 /p/thumb/<id>。
|
||||
// 91Spider 的封面是爬虫下载后保存到本地 /p/thumb/<id>,不受这条规则影响。
|
||||
res, err := c.db.ExecContext(ctx, `
|
||||
UPDATE videos
|
||||
SET thumbnail_url = '',
|
||||
thumbnail_status = 'pending',
|
||||
thumbnail_failures = 0,
|
||||
updated_at = ?
|
||||
WHERE (
|
||||
lower(COALESCE(thumbnail_url, '')) LIKE 'http://%'
|
||||
OR lower(COALESCE(thumbnail_url, '')) LIKE 'https://%'
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM drives
|
||||
WHERE drives.id = videos.drive_id
|
||||
AND drives.kind = 'spider91'
|
||||
)
|
||||
`, time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected > 0 {
|
||||
log.Printf("[catalog] cleared %d remote non-91Spider thumbnail(s) for local regeneration", affected)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Catalog) hideZeroSizeVideosFromKnownDrives(ctx context.Context) error {
|
||||
_, err := c.db.ExecContext(ctx, `
|
||||
UPDATE videos
|
||||
@@ -302,7 +399,15 @@ func (c *Catalog) classifySystemTags(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (c *Catalog) backfillVideoTags(ctx context.Context) error {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT id, COALESCE(tags, '[]') FROM videos`)
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT id, COALESCE(tags, '[]')
|
||||
FROM videos
|
||||
WHERE COALESCE(tags, '') NOT IN ('', '[]', 'null')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
WHERE vt.video_id = videos.id
|
||||
)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -319,11 +424,14 @@ func (c *Catalog) backfillVideoTags(ctx context.Context) error {
|
||||
if len(labels) == 0 {
|
||||
continue
|
||||
}
|
||||
if err := c.addVideoTags(ctx, videoID, labels, "legacy", true); err != nil {
|
||||
added, err := c.addVideoTags(ctx, videoID, labels, "legacy", true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return err
|
||||
if added {
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -392,6 +500,57 @@ func (c *Catalog) CreateTagAndClassify(ctx context.Context, label string, aliase
|
||||
return c.classifyTag(ctx, tag)
|
||||
}
|
||||
|
||||
func (c *Catalog) EnsureTagForVideoIDPrefix(ctx context.Context, prefix, label string, aliases []string, source string) (int, error) {
|
||||
prefix = strings.TrimSpace(prefix)
|
||||
if prefix == "" {
|
||||
return 0, errors.New("video id prefix is required")
|
||||
}
|
||||
tag, err := c.ensureTag(ctx, label, aliases, source)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT v.id
|
||||
FROM videos v
|
||||
WHERE v.id LIKE ? || '%'
|
||||
AND COALESCE(v.tags_manual, 0) = 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
WHERE vt.video_id = v.id
|
||||
AND vt.tag_id = ?
|
||||
)
|
||||
ORDER BY v.id ASC`, prefix, tag.ID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var videoIDs []string
|
||||
for rows.Next() {
|
||||
var videoID string
|
||||
if err := rows.Scan(&videoID); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
videoIDs = append(videoIDs, videoID)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, videoID := range videoIDs {
|
||||
if err := c.insertVideoTag(ctx, videoID, tag.ID, "auto"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return len(videoIDs), nil
|
||||
}
|
||||
|
||||
func (c *Catalog) DeleteTag(ctx context.Context, tagID int64) (int, error) {
|
||||
tx, err := c.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
@@ -453,10 +612,66 @@ func (c *Catalog) DeleteTag(ctx context.Context, tagID int64) (int, error) {
|
||||
|
||||
func (c *Catalog) ListTags(ctx context.Context) ([]Tag, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT t.id, t.label, t.aliases, t.source, COUNT(v.id) AS cnt
|
||||
WITH tagged_tags AS (
|
||||
SELECT vt.tag_id,
|
||||
tagged.id,
|
||||
COALESCE(tagged.content_hash, '') AS content_hash,
|
||||
COALESCE(tagged.sampled_sha256, '') AS sampled_sha256,
|
||||
tagged.size_bytes,
|
||||
COALESCE(tagged.file_name, '') AS file_name
|
||||
FROM video_tags vt
|
||||
JOIN videos tagged ON tagged.id = vt.video_id
|
||||
WHERE COALESCE(tagged.hidden, 0) = 0
|
||||
),
|
||||
tag_candidates AS (
|
||||
SELECT tag_id, id AS video_id
|
||||
FROM tagged_tags
|
||||
UNION ALL
|
||||
SELECT tag_id,
|
||||
(SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_tags.content_hash != ''
|
||||
AND canonical.content_hash = tagged_tags.content_hash
|
||||
AND COALESCE(canonical.content_hash, '') != ''
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_tags
|
||||
WHERE content_hash != ''
|
||||
UNION ALL
|
||||
SELECT tag_id,
|
||||
(SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_tags.sampled_sha256 != ''
|
||||
AND tagged_tags.size_bytes > 0
|
||||
AND canonical.sampled_sha256 = tagged_tags.sampled_sha256
|
||||
AND canonical.size_bytes = tagged_tags.size_bytes
|
||||
AND COALESCE(canonical.sampled_sha256, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_tags
|
||||
WHERE sampled_sha256 != '' AND size_bytes > 0
|
||||
UNION ALL
|
||||
SELECT tag_id,
|
||||
(SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_tags.file_name != ''
|
||||
AND tagged_tags.size_bytes > 0
|
||||
AND canonical.file_name = tagged_tags.file_name
|
||||
AND canonical.size_bytes = tagged_tags.size_bytes
|
||||
AND COALESCE(canonical.file_name, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_tags
|
||||
WHERE file_name != '' AND size_bytes > 0
|
||||
)
|
||||
SELECT t.id, t.label, t.aliases, t.source, COUNT(DISTINCT videos.id) AS cnt
|
||||
FROM tags t
|
||||
LEFT JOIN video_tags vt ON vt.tag_id = t.id
|
||||
LEFT JOIN videos v ON v.id = vt.video_id AND COALESCE(v.hidden, 0) = 0
|
||||
LEFT JOIN tag_candidates tc ON tc.tag_id = t.id AND tc.video_id IS NOT NULL
|
||||
LEFT JOIN videos ON videos.id = tc.video_id
|
||||
AND COALESCE(videos.hidden, 0) = 0
|
||||
AND `+uniqueVideoWhereSQL+`
|
||||
GROUP BY t.id, t.label, t.aliases, t.source
|
||||
ORDER BY cnt DESC, t.label ASC`)
|
||||
if err != nil {
|
||||
@@ -474,6 +689,66 @@ ORDER BY cnt DESC, t.label ASC`)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func videoMatchesTagLabelSQL(videoAlias string) string {
|
||||
return fmt.Sprintf(`%s.id IN (
|
||||
WITH tagged_videos AS (
|
||||
SELECT tagged.id,
|
||||
COALESCE(tagged.content_hash, '') AS content_hash,
|
||||
COALESCE(tagged.sampled_sha256, '') AS sampled_sha256,
|
||||
tagged.size_bytes,
|
||||
COALESCE(tagged.file_name, '') AS file_name
|
||||
FROM video_tags vt
|
||||
JOIN tags tag_filter ON tag_filter.id = vt.tag_id
|
||||
JOIN videos tagged ON tagged.id = vt.video_id
|
||||
WHERE tag_filter.label = ? COLLATE NOCASE
|
||||
AND COALESCE(tagged.hidden, 0) = 0
|
||||
),
|
||||
tag_candidates AS (
|
||||
SELECT id AS video_id
|
||||
FROM tagged_videos
|
||||
UNION ALL
|
||||
SELECT (SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_videos.content_hash != ''
|
||||
AND canonical.content_hash = tagged_videos.content_hash
|
||||
AND COALESCE(canonical.content_hash, '') != ''
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_videos
|
||||
WHERE content_hash != ''
|
||||
UNION ALL
|
||||
SELECT (SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_videos.sampled_sha256 != ''
|
||||
AND tagged_videos.size_bytes > 0
|
||||
AND canonical.sampled_sha256 = tagged_videos.sampled_sha256
|
||||
AND canonical.size_bytes = tagged_videos.size_bytes
|
||||
AND COALESCE(canonical.sampled_sha256, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_videos
|
||||
WHERE sampled_sha256 != '' AND size_bytes > 0
|
||||
UNION ALL
|
||||
SELECT (SELECT canonical.id
|
||||
FROM videos canonical
|
||||
WHERE tagged_videos.file_name != ''
|
||||
AND tagged_videos.size_bytes > 0
|
||||
AND canonical.file_name = tagged_videos.file_name
|
||||
AND canonical.size_bytes = tagged_videos.size_bytes
|
||||
AND COALESCE(canonical.file_name, '') != ''
|
||||
AND canonical.size_bytes > 0
|
||||
ORDER BY canonical.created_at ASC, canonical.id ASC
|
||||
LIMIT 1) AS video_id
|
||||
FROM tagged_videos
|
||||
WHERE file_name != '' AND size_bytes > 0
|
||||
)
|
||||
SELECT video_id
|
||||
FROM tag_candidates
|
||||
WHERE video_id IS NOT NULL
|
||||
)`, videoAlias)
|
||||
}
|
||||
|
||||
func (c *Catalog) SetManualVideoTags(ctx context.Context, videoID string, labels []string) error {
|
||||
if _, err := c.GetVideo(ctx, videoID); err != nil {
|
||||
return err
|
||||
@@ -592,6 +867,10 @@ func (c *Catalog) getTagByLabel(ctx context.Context, label string) (Tag, error)
|
||||
}
|
||||
|
||||
func (c *Catalog) classifyTag(ctx context.Context, tag Tag) (int, error) {
|
||||
existingIDs, err := c.videoIDSetForTagID(ctx, tag.ID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT id, title, COALESCE(author, ''), COALESCE(category, ''), COALESCE(tags_manual, 0)
|
||||
FROM videos`)
|
||||
@@ -623,13 +902,14 @@ FROM videos`)
|
||||
continue
|
||||
}
|
||||
}
|
||||
added, err := c.addVideoTag(ctx, videoID, tag.Label, "auto", false)
|
||||
if err != nil {
|
||||
if existingIDs[videoID] {
|
||||
continue
|
||||
}
|
||||
if err := c.insertVideoTag(ctx, videoID, tag.ID, "auto"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if added {
|
||||
classified++
|
||||
}
|
||||
existingIDs[videoID] = true
|
||||
classified++
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -689,17 +969,22 @@ func (c *Catalog) replaceVideoTags(ctx context.Context, videoID string, labels [
|
||||
return c.syncVideoTagsJSON(ctx, videoID, manual)
|
||||
}
|
||||
|
||||
func (c *Catalog) addVideoTags(ctx context.Context, videoID string, labels []string, source string, createMissing bool) error {
|
||||
func (c *Catalog) addVideoTags(ctx context.Context, videoID string, labels []string, source string, createMissing bool) (bool, error) {
|
||||
labels = uniqueStrings(cleanLabels(labels))
|
||||
if source != "manual" {
|
||||
labels = c.filterDeletedTagLabels(ctx, labels)
|
||||
}
|
||||
changed := false
|
||||
for _, label := range labels {
|
||||
if _, err := c.addVideoTag(ctx, videoID, label, source, createMissing); err != nil {
|
||||
return err
|
||||
added, err := c.addVideoTag(ctx, videoID, label, source, createMissing)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if added {
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) addVideoTag(ctx context.Context, videoID, label, source string, createMissing bool) (bool, error) {
|
||||
@@ -729,12 +1014,33 @@ func (c *Catalog) addVideoTag(ctx context.Context, videoID, label, source string
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
func (c *Catalog) insertVideoTag(ctx context.Context, videoID string, tagID int64, source string) error {
|
||||
_, err := c.db.ExecContext(ctx,
|
||||
`INSERT OR IGNORE INTO video_tags (video_id, tag_id, source, created_at) VALUES (?, ?, ?, ?)`,
|
||||
videoID, tagID, source, time.Now().UnixMilli())
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Catalog) addCollectionTagToVideos(ctx context.Context, category string) error {
|
||||
return c.addTagToVideosByCategory(ctx, category, category, "auto")
|
||||
}
|
||||
|
||||
func (c *Catalog) addTagToVideosByCategory(ctx context.Context, category, label, source string) error {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT id FROM videos WHERE category = ? AND COALESCE(tags_manual, 0) = 0`, category)
|
||||
tag, err := c.getTagByLabel(ctx, label)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT v.id
|
||||
FROM videos v
|
||||
WHERE v.category = ?
|
||||
AND COALESCE(v.tags_manual, 0) = 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM video_tags vt
|
||||
WHERE vt.video_id = v.id
|
||||
AND vt.tag_id = ?
|
||||
)`, category, tag.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -753,7 +1059,7 @@ func (c *Catalog) addTagToVideosByCategory(ctx context.Context, category, label,
|
||||
return err
|
||||
}
|
||||
for _, videoID := range videoIDs {
|
||||
if _, err := c.addVideoTag(ctx, videoID, label, source, false); err != nil {
|
||||
if err := c.insertVideoTag(ctx, videoID, tag.ID, source); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.syncVideoTagsJSON(ctx, videoID, false); err != nil {
|
||||
@@ -837,6 +1143,23 @@ func (c *Catalog) videoIDsForTagID(ctx context.Context, tagID int64) ([]string,
|
||||
return videoIDs, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) videoIDSetForTagID(ctx context.Context, tagID int64) (map[string]bool, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `SELECT video_id FROM video_tags WHERE tag_id = ?`, tagID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var videoID string
|
||||
if err := rows.Scan(&videoID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[videoID] = true
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Catalog) validateTagsExist(ctx context.Context, labels []string) error {
|
||||
for _, label := range labels {
|
||||
if _, err := c.getTagByLabel(ctx, label); err != nil {
|
||||
|
||||
@@ -326,6 +326,75 @@ func TestCreateTagAndClassifyRestoresDeletedTag(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureTagForVideoIDPrefixBackfillsSourceTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, seed := range []struct {
|
||||
id string
|
||||
manual bool
|
||||
}{
|
||||
{id: "spider91-91-spider-1200001"},
|
||||
{id: "spider91-91-spider-1200002", manual: true},
|
||||
{id: "spider91-other-1200003"},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: seed.id,
|
||||
DriveID: "91-spider",
|
||||
FileID: seed.id + ".mp4",
|
||||
Title: "legacy title without source text",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", seed.id, err)
|
||||
}
|
||||
if seed.manual {
|
||||
if err := cat.SetManualVideoTags(ctx, seed.id, nil); err != nil {
|
||||
t.Fatalf("mark %s manual: %v", seed.id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
added, err := cat.EnsureTagForVideoIDPrefix(ctx, "spider91-91-spider-", "91porn", nil, "system")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure prefix tag: %v", err)
|
||||
}
|
||||
if added != 1 {
|
||||
t.Fatalf("added = %d, want 1", added)
|
||||
}
|
||||
got, err := cat.GetVideo(ctx, "spider91-91-spider-1200001")
|
||||
if err != nil {
|
||||
t.Fatalf("get tagged video: %v", err)
|
||||
}
|
||||
if !sameStrings(got.Tags, []string{"91porn"}) {
|
||||
t.Fatalf("tagged video tags = %#v, want 91porn", got.Tags)
|
||||
}
|
||||
manual, err := cat.GetVideo(ctx, "spider91-91-spider-1200002")
|
||||
if err != nil {
|
||||
t.Fatalf("get manual video: %v", err)
|
||||
}
|
||||
if len(manual.Tags) != 0 {
|
||||
t.Fatalf("manual video tags = %#v, want unchanged", manual.Tags)
|
||||
}
|
||||
other, err := cat.GetVideo(ctx, "spider91-other-1200003")
|
||||
if err != nil {
|
||||
t.Fatalf("get other prefix video: %v", err)
|
||||
}
|
||||
if len(other.Tags) != 0 {
|
||||
t.Fatalf("other prefix video tags = %#v, want unchanged", other.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteTagRejectsSystemTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
@@ -398,6 +467,84 @@ VALUES
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateDoesNotRewriteAlreadySyncedVideoTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, id := range []string{"video-1", "video-2", "video-3"} {
|
||||
if err := cat.UpsertVideo(ctx, &Video{
|
||||
ID: id,
|
||||
DriveID: "drive",
|
||||
FileID: id,
|
||||
Title: "巨乳后入合集",
|
||||
Category: "Better Call Saul S03",
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
if err := cat.migrate(ctx); err != nil {
|
||||
t.Fatalf("first migrate: %v", err)
|
||||
}
|
||||
|
||||
before := videoUpdatedAtByID(t, ctx, cat, "video-1", "video-2", "video-3")
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
if err := cat.migrate(ctx); err != nil {
|
||||
t.Fatalf("second migrate: %v", err)
|
||||
}
|
||||
after := videoUpdatedAtByID(t, ctx, cat, "video-1", "video-2", "video-3")
|
||||
for id, want := range before {
|
||||
if got := after[id]; got != want {
|
||||
t.Fatalf("%s updated_at changed on no-op migrate: got %d, want %d", id, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateBackfillsLegacyTagsWithoutRelations(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
if _, err := cat.db.ExecContext(ctx, `
|
||||
INSERT INTO videos (id, drive_id, file_id, title, tags, tags_manual, published_at, created_at, updated_at)
|
||||
VALUES ('legacy-video', 'drive', 'file-legacy', 'legacy title', '["legacy-tag"]', 0, ?, ?, ?)`,
|
||||
now, now, now); err != nil {
|
||||
t.Fatalf("seed legacy video: %v", err)
|
||||
}
|
||||
if err := cat.migrate(ctx); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
|
||||
tag := mustTagByLabel(t, ctx, cat, "legacy-tag")
|
||||
var count int
|
||||
if err := cat.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM video_tags WHERE video_id = 'legacy-video' AND tag_id = ?`, tag.ID).Scan(&count); err != nil {
|
||||
t.Fatalf("count video tag: %v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("legacy video tag relation count = %d, want 1", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenMigratesLegacyVideosWithoutFileName(t *testing.T) {
|
||||
path := t.TempDir() + "/catalog.db"
|
||||
db, err := sql.Open("sqlite", path)
|
||||
@@ -657,7 +804,7 @@ func TestMigrateCollapsesAVCodeTagsIntoAV(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateClearsVolatileOneDriveThumbnailURLs(t *testing.T) {
|
||||
func TestMigrateClearsRemoteNonSpiderThumbnailURLs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -680,6 +827,36 @@ func TestMigrateClearsVolatileOneDriveThumbnailURLs(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatalf("seed onedrive: %v", err)
|
||||
}
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: "p123-main",
|
||||
Kind: "p123",
|
||||
Name: "123Pan",
|
||||
RootID: "root",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed p123: %v", err)
|
||||
}
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: "pikpak-main",
|
||||
Kind: "pikpak",
|
||||
Name: "PikPak",
|
||||
RootID: "root",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed pikpak: %v", err)
|
||||
}
|
||||
if err := cat.UpsertDrive(ctx, &Drive{
|
||||
ID: "spider91-main",
|
||||
Kind: "spider91",
|
||||
Name: "91Spider",
|
||||
RootID: "root",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed spider91: %v", err)
|
||||
}
|
||||
|
||||
videos := []*Video{
|
||||
{
|
||||
@@ -703,6 +880,27 @@ func TestMigrateClearsVolatileOneDriveThumbnailURLs(t *testing.T) {
|
||||
Title: "PikPak",
|
||||
ThumbnailURL: "https://sg-thumbnail-drive.mypikpak.net/v0/screenshot-thumbnails/demo",
|
||||
},
|
||||
{
|
||||
ID: "p123-remote-thumb-video",
|
||||
DriveID: "p123-main",
|
||||
FileID: "file-4",
|
||||
Title: "123Pan remote thumb",
|
||||
ThumbnailURL: "https://download.123pan.com/thumb/file_70_70?w=70&h=70",
|
||||
},
|
||||
{
|
||||
ID: "p123-local-thumb-video",
|
||||
DriveID: "p123-main",
|
||||
FileID: "file-5",
|
||||
Title: "123Pan local thumb",
|
||||
ThumbnailURL: "/p/thumb/p123-local-thumb-video",
|
||||
},
|
||||
{
|
||||
ID: "spider91-local-thumb-video",
|
||||
DriveID: "spider91-main",
|
||||
FileID: "file-6",
|
||||
Title: "91Spider local thumb",
|
||||
ThumbnailURL: "/p/thumb/spider91-local-thumb-video",
|
||||
},
|
||||
}
|
||||
for _, v := range videos {
|
||||
v.PublishedAt = now
|
||||
@@ -737,8 +935,39 @@ func TestMigrateClearsVolatileOneDriveThumbnailURLs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("get pikpak video: %v", err)
|
||||
}
|
||||
if pikpak.ThumbnailURL == "" {
|
||||
t.Fatal("pikpak thumbnail was cleared")
|
||||
if pikpak.ThumbnailURL != "" {
|
||||
t.Fatalf("pikpak thumbnail = %q, want cleared", pikpak.ThumbnailURL)
|
||||
}
|
||||
|
||||
p123Remote, err := cat.GetVideo(ctx, "p123-remote-thumb-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get p123 remote thumb video: %v", err)
|
||||
}
|
||||
if p123Remote.ThumbnailURL != "" {
|
||||
t.Fatalf("p123 remote thumbnail = %q, want cleared", p123Remote.ThumbnailURL)
|
||||
}
|
||||
var p123Status string
|
||||
if err := cat.db.QueryRowContext(ctx, `SELECT thumbnail_status FROM videos WHERE id = ?`, "p123-remote-thumb-video").Scan(&p123Status); err != nil {
|
||||
t.Fatalf("read p123 thumbnail status: %v", err)
|
||||
}
|
||||
if p123Status != "pending" {
|
||||
t.Fatalf("p123 remote thumbnail_status = %q, want pending", p123Status)
|
||||
}
|
||||
|
||||
p123Local, err := cat.GetVideo(ctx, "p123-local-thumb-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get p123 local thumb video: %v", err)
|
||||
}
|
||||
if p123Local.ThumbnailURL != "/p/thumb/p123-local-thumb-video" {
|
||||
t.Fatalf("p123 local thumbnail = %q, want preserved", p123Local.ThumbnailURL)
|
||||
}
|
||||
|
||||
spider91Local, err := cat.GetVideo(ctx, "spider91-local-thumb-video")
|
||||
if err != nil {
|
||||
t.Fatalf("get spider91 local thumb video: %v", err)
|
||||
}
|
||||
if spider91Local.ThumbnailURL != "/p/thumb/spider91-local-thumb-video" {
|
||||
t.Fatalf("spider91 local thumbnail = %q, want preserved", spider91Local.ThumbnailURL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -859,6 +1088,98 @@ func TestListVideosHidesDuplicateContentHashes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTagFilterMatchesCanonicalDuplicateVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
t.Fatalf("open catalog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cat.Close(); err != nil {
|
||||
t.Fatalf("close catalog: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
for _, v := range []*Video{
|
||||
{
|
||||
ID: "pikpak-canonical",
|
||||
DriveID: "pikpak",
|
||||
FileID: "canonical.mp4",
|
||||
Title: "Canonical",
|
||||
Size: 1024,
|
||||
PublishedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
},
|
||||
{
|
||||
ID: "spider91-dup-1",
|
||||
DriveID: "91-spider",
|
||||
FileID: "dup-1.mp4",
|
||||
Title: "Spider duplicate 1",
|
||||
Tags: []string{"91porn"},
|
||||
Size: 1024,
|
||||
PublishedAt: now.Add(time.Second),
|
||||
CreatedAt: now.Add(time.Second),
|
||||
UpdatedAt: now.Add(time.Second),
|
||||
},
|
||||
{
|
||||
ID: "spider91-dup-2",
|
||||
DriveID: "91-spider",
|
||||
FileID: "dup-2.mp4",
|
||||
Title: "Spider duplicate 2",
|
||||
Tags: []string{"91porn"},
|
||||
Size: 1024,
|
||||
PublishedAt: now.Add(2 * time.Second),
|
||||
CreatedAt: now.Add(2 * time.Second),
|
||||
UpdatedAt: now.Add(2 * time.Second),
|
||||
},
|
||||
{
|
||||
ID: "spider91-visible",
|
||||
DriveID: "91-spider",
|
||||
FileID: "visible.mp4",
|
||||
Title: "Spider visible",
|
||||
Tags: []string{"91porn"},
|
||||
Size: 2048,
|
||||
PublishedAt: now.Add(3 * time.Second),
|
||||
CreatedAt: now.Add(3 * time.Second),
|
||||
UpdatedAt: now.Add(3 * time.Second),
|
||||
},
|
||||
} {
|
||||
if err := cat.UpsertVideo(ctx, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", v.ID, err)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"pikpak-canonical", "spider91-dup-1", "spider91-dup-2"} {
|
||||
if err := cat.UpdateVideoFingerprint(ctx, id, "same-sampled-sha256", "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
if err := cat.UpdateVideoFingerprint(ctx, "spider91-visible", "unique-sampled-sha256", "ready", ""); err != nil {
|
||||
t.Fatalf("fingerprint visible: %v", err)
|
||||
}
|
||||
|
||||
items, total, err := cat.ListVideos(ctx, ListParams{Tag: "91porn", Page: 1, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list videos by tag: %v", err)
|
||||
}
|
||||
if total != 2 || len(items) != 2 {
|
||||
t.Fatalf("tagged videos total=%d len=%d, want 2", total, len(items))
|
||||
}
|
||||
gotIDs := map[string]bool{}
|
||||
for _, item := range items {
|
||||
gotIDs[item.ID] = true
|
||||
}
|
||||
for _, want := range []string{"pikpak-canonical", "spider91-visible"} {
|
||||
if !gotIDs[want] {
|
||||
t.Fatalf("tagged video ids = %#v, want %s", gotIDs, want)
|
||||
}
|
||||
}
|
||||
if got := mustTagByLabel(t, ctx, cat, "91porn").Count; got != 2 {
|
||||
t.Fatalf("91porn count = %d, want 2 visible canonical videos", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListVideosCanFilterReadyThumbnails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := Open(t.TempDir() + "/catalog.db")
|
||||
@@ -944,6 +1265,19 @@ func mustTagByLabel(t *testing.T, ctx context.Context, cat *Catalog, label strin
|
||||
return Tag{}
|
||||
}
|
||||
|
||||
func videoUpdatedAtByID(t *testing.T, ctx context.Context, cat *Catalog, ids ...string) map[string]int64 {
|
||||
t.Helper()
|
||||
out := make(map[string]int64, len(ids))
|
||||
for _, id := range ids {
|
||||
var updatedAt int64
|
||||
if err := cat.db.QueryRowContext(ctx, `SELECT updated_at FROM videos WHERE id = ?`, id).Scan(&updatedAt); err != nil {
|
||||
t.Fatalf("read updated_at for %s: %v", id, err)
|
||||
}
|
||||
out[id] = updatedAt
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// 删除 collection 标签的最后一个引用视频后,标签应当自动从 tags 表里消失。
|
||||
// user/system 标签不受影响:用户/系统标签的语义由人维护,孤儿状态保留。
|
||||
func TestDeleteVideoPrunesOrphanCollectionTag(t *testing.T) {
|
||||
|
||||
@@ -202,7 +202,7 @@ type Nightly struct {
|
||||
// 这里保留 yaml 中的静态定义,用于启动时预置盘。生产建议只在 DB 里维护。
|
||||
type Drive struct {
|
||||
ID string `yaml:"id"`
|
||||
Kind string `yaml:"kind"` // quark / p115 / pikpak / wopan / onedrive / googledrive / localstorage
|
||||
Kind string `yaml:"kind"` // quark / p115 / p123 / pikpak / wopan / onedrive / googledrive / localstorage
|
||||
Name string `yaml:"name"`
|
||||
RootID string `yaml:"root_id"`
|
||||
Params map[string]string `yaml:"params,omitempty"`
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
// Drive 是多家网盘统一抽象。上层不区分盘,只区分 Kind。
|
||||
type Drive interface {
|
||||
// Kind 返回驱动代号:"quark" / "p115" / "pikpak" / "wopan" / "onedrive" / "googledrive" / "localstorage"
|
||||
// Kind 返回驱动代号:"quark" / "p115" / "p123" / "pikpak" / "wopan" / "onedrive" / "googledrive" / "localstorage"
|
||||
Kind() string
|
||||
|
||||
// ID 返回该盘在 catalog 中的唯一标识
|
||||
|
||||
@@ -47,7 +47,7 @@ func (d *Driver) Init(context.Context) error {
|
||||
}
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("localstorage: stat root: %w", err)
|
||||
return fmt.Errorf("localstorage: stat root %q: %w%s", root, err, localStoragePathHint(d.rootPath))
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("localstorage: root is not a directory: %s", root)
|
||||
@@ -174,12 +174,38 @@ func (d *Driver) pathForID(id string) (string, string, error) {
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if p != root && !strings.HasPrefix(p, root+string(os.PathSeparator)) {
|
||||
if !pathWithinRoot(root, p) {
|
||||
return "", "", errors.New("localstorage: path escapes root")
|
||||
}
|
||||
return p, rel, nil
|
||||
}
|
||||
|
||||
func pathWithinRoot(root, path string) bool {
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator)))
|
||||
}
|
||||
|
||||
func localStoragePathHint(configured string) string {
|
||||
cwd, _ := os.Getwd()
|
||||
parts := []string{}
|
||||
if strings.TrimSpace(configured) != "" {
|
||||
parts = append(parts, fmt.Sprintf("configured=%q", strings.TrimSpace(configured)))
|
||||
}
|
||||
if cwd != "" {
|
||||
parts = append(parts, fmt.Sprintf("cwd=%q", cwd))
|
||||
}
|
||||
if _, err := os.Stat("/.dockerenv"); err == nil {
|
||||
parts = append(parts, "docker=host paths must be bind-mounted into the container")
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
return " (" + strings.Join(parts, ", ") + ")"
|
||||
}
|
||||
|
||||
func decodeRel(id string) (string, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" || id == "/" {
|
||||
|
||||
@@ -70,13 +70,34 @@ func TestStreamURLRejectsEscapingID(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestInitRequiresExistingDirectory(t *testing.T) {
|
||||
drv := New(Config{ID: "local", RootPath: filepath.Join(t.TempDir(), "missing")})
|
||||
missing := filepath.Join(t.TempDir(), "missing")
|
||||
drv := New(Config{ID: "local", RootPath: missing})
|
||||
|
||||
err := drv.Init(context.Background())
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "stat root") {
|
||||
t.Fatalf("error = %v, want stat root failure", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), missing) || !strings.Contains(err.Error(), "configured=") {
|
||||
t.Fatalf("error = %v, want diagnostic path details", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathForIDAllowsRootPathSlash(t *testing.T) {
|
||||
drv := New(Config{ID: "local", RootPath: string(os.PathSeparator)})
|
||||
childID := encodeRel("tmp")
|
||||
|
||||
path, rel, err := drv.pathForID(childID)
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("pathForID: %v", err)
|
||||
}
|
||||
if rel != "tmp" {
|
||||
t.Fatalf("rel = %q, want tmp", rel)
|
||||
}
|
||||
if path != filepath.Join(string(os.PathSeparator), "tmp") {
|
||||
t.Fatalf("path = %q, want /tmp", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerPersistsLocalStorageVideo(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,773 @@
|
||||
package p123
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-resty/resty/v2"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
const (
|
||||
Kind = "p123"
|
||||
|
||||
defaultMainAPIBase = "https://www.123pan.com/b/api"
|
||||
defaultLoginAPIBase = "https://login.123pan.com/api"
|
||||
defaultReferer = "https://www.123pan.com/"
|
||||
defaultPlatform = "web"
|
||||
defaultAppVersion = "3"
|
||||
defaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) video-site-123pan"
|
||||
|
||||
endpointSignIn = "/user/sign_in"
|
||||
endpointUserInfo = "/user/info"
|
||||
endpointFileList = "/file/list/new"
|
||||
endpointDownloadInfo = "/file/download_info"
|
||||
endpointMkdir = "/file/upload_request"
|
||||
|
||||
listInterval = 700 * time.Millisecond
|
||||
listCooldown = 10 * time.Minute
|
||||
)
|
||||
|
||||
type Driver struct {
|
||||
id string
|
||||
rootID string
|
||||
username string
|
||||
password string
|
||||
accessToken string
|
||||
platform string
|
||||
mainAPIBase string
|
||||
loginAPIBase string
|
||||
referer string
|
||||
userAgent string
|
||||
|
||||
client *resty.Client
|
||||
httpClient *http.Client
|
||||
|
||||
onTokenUpdate func(access string)
|
||||
|
||||
tokenMu sync.RWMutex
|
||||
|
||||
listMu sync.Mutex
|
||||
lastListAt time.Time
|
||||
|
||||
fileMu sync.RWMutex
|
||||
files map[string]cachedFile
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ID string
|
||||
RootID string
|
||||
Username string
|
||||
Password string
|
||||
AccessToken string
|
||||
Platform string
|
||||
|
||||
MainAPIBaseURL string
|
||||
LoginAPIBaseURL string
|
||||
|
||||
OnTokenUpdate func(access string)
|
||||
}
|
||||
|
||||
func New(c Config) *Driver {
|
||||
rootID := strings.TrimSpace(c.RootID)
|
||||
if rootID == "" {
|
||||
rootID = "0"
|
||||
}
|
||||
platform := strings.TrimSpace(c.Platform)
|
||||
if platform == "" {
|
||||
platform = defaultPlatform
|
||||
}
|
||||
mainAPIBase := strings.TrimRight(strings.TrimSpace(c.MainAPIBaseURL), "/")
|
||||
if mainAPIBase == "" {
|
||||
mainAPIBase = defaultMainAPIBase
|
||||
}
|
||||
loginAPIBase := strings.TrimRight(strings.TrimSpace(c.LoginAPIBaseURL), "/")
|
||||
if loginAPIBase == "" {
|
||||
loginAPIBase = defaultLoginAPIBase
|
||||
}
|
||||
return &Driver{
|
||||
id: c.ID,
|
||||
rootID: rootID,
|
||||
username: strings.TrimSpace(c.Username),
|
||||
password: strings.TrimSpace(c.Password),
|
||||
accessToken: normalizeAccessToken(c.AccessToken),
|
||||
platform: platform,
|
||||
mainAPIBase: mainAPIBase,
|
||||
loginAPIBase: loginAPIBase,
|
||||
referer: defaultReferer,
|
||||
userAgent: defaultUserAgent,
|
||||
onTokenUpdate: c.OnTokenUpdate,
|
||||
client: resty.New().
|
||||
SetTimeout(30*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
httpClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
},
|
||||
files: make(map[string]cachedFile),
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Kind() string { return Kind }
|
||||
func (d *Driver) ID() string { return d.id }
|
||||
func (d *Driver) RootID() string { return d.rootID }
|
||||
|
||||
func (d *Driver) Init(ctx context.Context) error {
|
||||
if d.currentToken() == "" {
|
||||
if err := d.login(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err := d.request(ctx, endpointUserInfo, http.MethodGet, nil, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *Driver) List(ctx context.Context, dirID string) ([]drives.Entry, error) {
|
||||
if strings.TrimSpace(dirID) == "" {
|
||||
dirID = d.rootID
|
||||
}
|
||||
d.listMu.Lock()
|
||||
defer d.listMu.Unlock()
|
||||
|
||||
page := 1
|
||||
total := 0
|
||||
out := make([]drives.Entry, 0)
|
||||
for {
|
||||
var resp fileListResp
|
||||
query := map[string]string{
|
||||
"driveId": "0",
|
||||
"limit": "100",
|
||||
"next": "0",
|
||||
"orderBy": "file_id",
|
||||
"orderDirection": "desc",
|
||||
"parentFileId": dirID,
|
||||
"trashed": "false",
|
||||
"SearchData": "",
|
||||
"Page": strconv.Itoa(page),
|
||||
"OnlyLookAbnormalFile": "0",
|
||||
"event": "homeListFile",
|
||||
"operateType": "4",
|
||||
"inDirectSpace": "false",
|
||||
}
|
||||
for attempt := 0; ; attempt++ {
|
||||
if err := d.waitForListSlotLocked(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := d.request(ctx, endpointFileList, http.MethodGet, func(req *resty.Request) {
|
||||
req.SetQueryParams(query)
|
||||
}, &resp); err != nil {
|
||||
wait, ok := drives.RateLimitRetryAfter(err)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("123pan list: %w", err)
|
||||
}
|
||||
if wait <= 0 {
|
||||
wait = listCooldown
|
||||
}
|
||||
log.Printf("[p123] list cooling down drive=%s dir=%s page=%d cooldown=%s attempt=%d err=%v",
|
||||
d.id, dirID, page, wait, attempt+1, err)
|
||||
if err := sleepContext(ctx, wait); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
for _, f := range resp.Data.InfoList {
|
||||
d.cacheFile(f, dirID)
|
||||
out = append(out, fileToEntry(f, dirID))
|
||||
}
|
||||
total = resp.Data.Total
|
||||
page++
|
||||
if len(resp.Data.InfoList) == 0 || resp.Data.Next == "-1" || (total > 0 && len(out) >= total) {
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Stat(ctx context.Context, fileID string) (*drives.Entry, error) {
|
||||
f, parentID, err := d.findFile(ctx, fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e := fileToEntry(f, parentID)
|
||||
return &e, nil
|
||||
}
|
||||
|
||||
func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLink, error) {
|
||||
f, _, err := d.findFile(ctx, fileID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("123pan stream metadata: %w", err)
|
||||
}
|
||||
body := map[string]any{
|
||||
"driveId": 0,
|
||||
"etag": f.Etag,
|
||||
"fileId": f.FileID,
|
||||
"fileName": f.FileName,
|
||||
"s3keyFlag": f.S3KeyFlag,
|
||||
"size": f.Size,
|
||||
"type": f.Type,
|
||||
}
|
||||
var resp downloadInfoResp
|
||||
if _, err := d.request(ctx, endpointDownloadInfo, http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(body)
|
||||
}, &resp); err != nil {
|
||||
return nil, fmt.Errorf("123pan download info: %w", err)
|
||||
}
|
||||
downloadURL := strings.TrimSpace(resp.URL())
|
||||
if downloadURL == "" {
|
||||
return nil, errors.New("123pan download info: empty url")
|
||||
}
|
||||
return d.resolveDownloadURL(ctx, downloadURL)
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
parts := splitPath(pathFromRoot)
|
||||
currentID := d.rootID
|
||||
for _, name := range parts {
|
||||
childID, err := d.findChildDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if childID == "" {
|
||||
id, err := d.makeDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
childID = id
|
||||
}
|
||||
currentID = childID
|
||||
}
|
||||
return currentID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
body := map[string]any{
|
||||
"driveId": 0,
|
||||
"etag": "",
|
||||
"fileName": name,
|
||||
"parentFileId": parentID,
|
||||
"size": 0,
|
||||
"type": 1,
|
||||
}
|
||||
var resp mkdirResp
|
||||
if _, err := d.request(ctx, endpointMkdir, http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(body)
|
||||
}, &resp); err != nil {
|
||||
return "", fmt.Errorf("123pan mkdir %s: %w", name, err)
|
||||
}
|
||||
if resp.Data.FileID != 0 {
|
||||
return strconv.FormatInt(resp.Data.FileID, 10), nil
|
||||
}
|
||||
// 123 云盘创建目录的返回字段不稳定;创建成功但没回 fileId 时回读父目录确认。
|
||||
childID, err := d.findChildDir(ctx, parentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if childID == "" {
|
||||
return "", errors.New("123pan mkdir: empty file id")
|
||||
}
|
||||
return childID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) findChildDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
entries, err := d.List(ctx, parentID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir && e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (d *Driver) resolveDownloadURL(ctx context.Context, downloadURL string) (*drives.StreamLink, error) {
|
||||
original, err := url.Parse(downloadURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target := original.String()
|
||||
if params := original.Query().Get("params"); params != "" {
|
||||
if decoded, err := base64.StdEncoding.DecodeString(params); err == nil && len(decoded) > 0 {
|
||||
if u, err := url.Parse(string(decoded)); err == nil {
|
||||
target = u.String()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Referer", defaultReferer)
|
||||
req.Header.Set("User-Agent", d.userAgent)
|
||||
res, err := d.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
finalURL := ""
|
||||
if res.StatusCode >= 300 && res.StatusCode < 400 {
|
||||
finalURL = strings.TrimSpace(res.Header.Get("Location"))
|
||||
} else if res.StatusCode < 300 {
|
||||
var redirect redirectResp
|
||||
if err := json.NewDecoder(res.Body).Decode(&redirect); err == nil {
|
||||
finalURL = redirect.URL()
|
||||
}
|
||||
if finalURL == "" {
|
||||
finalURL = target
|
||||
}
|
||||
} else {
|
||||
body, _ := io.ReadAll(io.LimitReader(res.Body, 4096))
|
||||
if isP123RateLimitHTTPResponse(res.StatusCode, res.Header.Get("Retry-After"), string(body)) {
|
||||
return nil, p123RateLimitErrorFromHTTP("download redirect", res.StatusCode, res.Header.Get("Retry-After"), string(body))
|
||||
}
|
||||
return nil, fmt.Errorf("123pan download redirect: status %d", res.StatusCode)
|
||||
}
|
||||
if finalURL == "" {
|
||||
return nil, errors.New("123pan download redirect: empty url")
|
||||
}
|
||||
|
||||
headers := http.Header{}
|
||||
if original.Scheme != "" && original.Host != "" {
|
||||
headers.Set("Referer", fmt.Sprintf("%s://%s/", original.Scheme, original.Host))
|
||||
} else {
|
||||
headers.Set("Referer", defaultReferer)
|
||||
}
|
||||
headers.Set("User-Agent", d.userAgent)
|
||||
return &drives.StreamLink{
|
||||
URL: finalURL,
|
||||
Headers: headers,
|
||||
Expires: time.Now().Add(10 * time.Minute),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) request(ctx context.Context, endpoint, method string, configure func(*resty.Request), out any) ([]byte, error) {
|
||||
if d.currentToken() == "" {
|
||||
if err := d.login(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
rawURL := d.mainAPIBase + endpoint
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
req := d.client.R().
|
||||
SetContext(ctx).
|
||||
SetHeaders(map[string]string{
|
||||
"origin": "https://www.123pan.com",
|
||||
"referer": d.referer,
|
||||
"authorization": "Bearer " + d.currentToken(),
|
||||
"user-agent": d.userAgent,
|
||||
"platform": d.platform,
|
||||
"app-version": defaultAppVersion,
|
||||
})
|
||||
if configure != nil {
|
||||
configure(req)
|
||||
}
|
||||
if out != nil {
|
||||
req.SetResult(out)
|
||||
}
|
||||
res, err := req.Execute(method, signAPIURL(rawURL))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body := res.Body()
|
||||
var env apiEnvelope
|
||||
decodeErr := json.Unmarshal(body, &env)
|
||||
if isP123RateLimitResponse(res, env.Code, env.Message) {
|
||||
return nil, p123RateLimitError(res, env.Code, env.Message)
|
||||
}
|
||||
if decodeErr != nil {
|
||||
if res.IsError() {
|
||||
return nil, fmt.Errorf("123pan request: status=%d body=%s", res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return nil, fmt.Errorf("parse 123pan response: %w", decodeErr)
|
||||
}
|
||||
if env.Code == 0 {
|
||||
return body, nil
|
||||
}
|
||||
if env.Code == 401 && attempt == 0 {
|
||||
if err := d.login(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if env.Message == "" {
|
||||
env.Message = fmt.Sprintf("code=%d", env.Code)
|
||||
}
|
||||
return nil, errors.New(env.Message)
|
||||
}
|
||||
return nil, errors.New("123pan request: unauthorized")
|
||||
}
|
||||
|
||||
func isP123RateLimitResponse(res *resty.Response, code int, message string) bool {
|
||||
if code == http.StatusTooManyRequests || isP123RateLimitMessage(message) {
|
||||
return true
|
||||
}
|
||||
if res == nil {
|
||||
return false
|
||||
}
|
||||
return isP123RateLimitHTTPResponse(res.StatusCode(), res.Header().Get("Retry-After"), res.String())
|
||||
}
|
||||
|
||||
func isP123RateLimitHTTPResponse(status int, retryAfter, body string) bool {
|
||||
if status == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
if retryAfter != "" {
|
||||
switch status {
|
||||
case http.StatusTooManyRequests, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
|
||||
return true
|
||||
}
|
||||
}
|
||||
if isP123RateLimitMessage(body) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isP123RateLimitMessage(message string) bool {
|
||||
text := strings.ToLower(strings.TrimSpace(message))
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(text, "请求太频繁") ||
|
||||
strings.Contains(text, "请求过于频繁") ||
|
||||
strings.Contains(text, "请求频繁") ||
|
||||
strings.Contains(text, "操作频繁") ||
|
||||
strings.Contains(text, "频率限制") ||
|
||||
strings.Contains(text, "请求次数过多") ||
|
||||
strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "rate-limit") ||
|
||||
strings.Contains(text, "ratelimit") ||
|
||||
strings.Contains(text, "throttl") ||
|
||||
strings.Contains(text, "temporarily blocked") ||
|
||||
strings.Contains(text, "request has been blocked") ||
|
||||
strings.Contains(text, "blocked") ||
|
||||
strings.Contains(text, "访问被阻断")
|
||||
}
|
||||
|
||||
func p123RateLimitError(res *resty.Response, code int, message string) error {
|
||||
if strings.TrimSpace(message) == "" {
|
||||
message = "123pan rate limited"
|
||||
}
|
||||
if code != 0 {
|
||||
message = fmt.Sprintf("code=%d %s", code, message)
|
||||
}
|
||||
if res != nil && strings.TrimSpace(res.String()) != "" {
|
||||
message = fmt.Sprintf("%s: status=%d body=%s", message, res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: parseRetryAfterHeader(responseRetryAfter(res)),
|
||||
Err: errors.New(message),
|
||||
}
|
||||
}
|
||||
|
||||
func p123RateLimitErrorFromHTTP(step string, status int, retryAfter, body string) error {
|
||||
message := fmt.Sprintf("123pan %s rate limited: status=%d", step, status)
|
||||
if strings.TrimSpace(body) != "" {
|
||||
message += " body=" + strings.TrimSpace(body)
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: Kind,
|
||||
RetryAfter: parseRetryAfterHeader(retryAfter),
|
||||
Err: errors.New(message),
|
||||
}
|
||||
}
|
||||
|
||||
func responseRetryAfter(res *resty.Response) string {
|
||||
if res == nil {
|
||||
return ""
|
||||
}
|
||||
return res.Header().Get("Retry-After")
|
||||
}
|
||||
|
||||
func parseRetryAfterHeader(raw string) time.Duration {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return 0
|
||||
}
|
||||
if seconds, err := strconv.Atoi(raw); err == nil && seconds > 0 {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
if when, err := http.ParseTime(raw); err == nil {
|
||||
if wait := time.Until(when); wait > 0 {
|
||||
return wait
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (d *Driver) login(ctx context.Context) error {
|
||||
if d.username == "" || d.password == "" {
|
||||
return errors.New("123pan login: username and password are required")
|
||||
}
|
||||
body := map[string]any{
|
||||
"passport": d.username,
|
||||
"password": d.password,
|
||||
"remember": true,
|
||||
}
|
||||
if strings.Contains(d.username, "@") {
|
||||
body = map[string]any{
|
||||
"mail": d.username,
|
||||
"password": d.password,
|
||||
"type": 2,
|
||||
}
|
||||
}
|
||||
var resp loginResp
|
||||
res, err := d.client.R().
|
||||
SetContext(ctx).
|
||||
SetHeaders(map[string]string{
|
||||
"origin": "https://www.123pan.com",
|
||||
"referer": d.referer,
|
||||
"user-agent": "Dart/2.19(dart:io)-video-site",
|
||||
"platform": d.platform,
|
||||
"app-version": defaultAppVersion,
|
||||
}).
|
||||
SetBody(body).
|
||||
SetResult(&resp).
|
||||
Post(d.loginAPIBase + endpointSignIn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.Code != 200 {
|
||||
if resp.Message == "" {
|
||||
resp.Message = fmt.Sprintf("status=%d code=%d", res.StatusCode(), resp.Code)
|
||||
}
|
||||
return loginError(resp.Message)
|
||||
}
|
||||
if strings.TrimSpace(resp.Data.Token) == "" {
|
||||
return errors.New("123pan login: empty token")
|
||||
}
|
||||
d.setToken(resp.Data.Token)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) currentToken() string {
|
||||
d.tokenMu.RLock()
|
||||
defer d.tokenMu.RUnlock()
|
||||
return d.accessToken
|
||||
}
|
||||
|
||||
func (d *Driver) setToken(token string) {
|
||||
token = normalizeAccessToken(token)
|
||||
d.tokenMu.Lock()
|
||||
d.accessToken = token
|
||||
d.tokenMu.Unlock()
|
||||
if d.onTokenUpdate != nil {
|
||||
d.onTokenUpdate(token)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) waitForListSlotLocked(ctx context.Context) error {
|
||||
if d.lastListAt.IsZero() {
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
next := d.lastListAt.Add(listInterval)
|
||||
now := time.Now()
|
||||
if now.Before(next) {
|
||||
timer := time.NewTimer(next.Sub(now))
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
d.lastListAt = time.Now()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, d time.Duration) error {
|
||||
if d <= 0 {
|
||||
return ctx.Err()
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) cacheFile(f panFile, parentID string) {
|
||||
id := strconv.FormatInt(f.FileID, 10)
|
||||
if id == "0" {
|
||||
return
|
||||
}
|
||||
d.fileMu.Lock()
|
||||
d.files[id] = cachedFile{file: f, parentID: parentID}
|
||||
d.fileMu.Unlock()
|
||||
}
|
||||
|
||||
func (d *Driver) cachedFile(fileID string) (panFile, string, bool) {
|
||||
d.fileMu.RLock()
|
||||
defer d.fileMu.RUnlock()
|
||||
c, ok := d.files[fileID]
|
||||
return c.file, c.parentID, ok
|
||||
}
|
||||
|
||||
func (d *Driver) findFile(ctx context.Context, fileID string) (panFile, string, error) {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return panFile{}, "", errors.New("empty file id")
|
||||
}
|
||||
if f, parentID, ok := d.cachedFile(fileID); ok {
|
||||
return f, parentID, nil
|
||||
}
|
||||
f, parentID, ok, err := d.findFileInDir(ctx, fileID, d.rootID, make(map[string]struct{}))
|
||||
if err != nil {
|
||||
return panFile{}, "", err
|
||||
}
|
||||
if !ok {
|
||||
return panFile{}, "", fmt.Errorf("file %s not found", fileID)
|
||||
}
|
||||
return f, parentID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) findFileInDir(ctx context.Context, targetID, dirID string, visited map[string]struct{}) (panFile, string, bool, error) {
|
||||
if _, ok := visited[dirID]; ok {
|
||||
return panFile{}, "", false, nil
|
||||
}
|
||||
visited[dirID] = struct{}{}
|
||||
entries, err := d.List(ctx, dirID)
|
||||
if err != nil {
|
||||
return panFile{}, "", false, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.ID == targetID {
|
||||
f, parentID, ok := d.cachedFile(e.ID)
|
||||
if !ok {
|
||||
return panFile{}, "", false, nil
|
||||
}
|
||||
return f, parentID, true, nil
|
||||
}
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir {
|
||||
continue
|
||||
}
|
||||
if f, parentID, ok, err := d.findFileInDir(ctx, targetID, e.ID, visited); err != nil || ok {
|
||||
return f, parentID, ok, err
|
||||
}
|
||||
}
|
||||
return panFile{}, "", false, nil
|
||||
}
|
||||
|
||||
func normalizeAccessToken(token string) string {
|
||||
token = strings.TrimSpace(token)
|
||||
if len(token) >= len("Bearer ") && strings.EqualFold(token[:len("Bearer ")], "Bearer ") {
|
||||
token = strings.TrimSpace(token[len("Bearer "):])
|
||||
}
|
||||
return token
|
||||
}
|
||||
|
||||
func loginError(message string) error {
|
||||
message = strings.TrimSpace(message)
|
||||
if strings.Contains(message, "境外登录风险") ||
|
||||
(strings.Contains(message, "短信验证码") && strings.Contains(message, "微信")) {
|
||||
return errors.New("123pan login: 账号密码登录被 123 云盘风控拦截,请在浏览器完成短信/微信验证后复制 access_token,并在后台编辑该 123 云盘时只填写 access_token")
|
||||
}
|
||||
if message == "" {
|
||||
message = "login failed"
|
||||
}
|
||||
return errors.New(message)
|
||||
}
|
||||
|
||||
func signPath(apiPath, platform, version string) (string, string) {
|
||||
table := []byte{'a', 'd', 'e', 'f', 'g', 'h', 'l', 'm', 'y', 'i', 'j', 'n', 'o', 'p', 'k', 'q', 'r', 's', 't', 'u', 'b', 'c', 'v', 'w', 's', 'z'}
|
||||
random := fmt.Sprintf("%.f", math.Round(1e7*rand.Float64()))
|
||||
now := time.Now().In(time.FixedZone("CST", 8*3600))
|
||||
timestamp := fmt.Sprint(now.Unix())
|
||||
nowStr := []byte(now.Format("200601021504"))
|
||||
for i := 0; i < len(nowStr); i++ {
|
||||
nowStr[i] = table[nowStr[i]-48]
|
||||
}
|
||||
timeSign := fmt.Sprint(crc32.ChecksumIEEE(nowStr))
|
||||
data := strings.Join([]string{timestamp, random, apiPath, platform, version, timeSign}, "|")
|
||||
dataSign := fmt.Sprint(crc32.ChecksumIEEE([]byte(data)))
|
||||
return timeSign, strings.Join([]string{timestamp, random, dataSign}, "-")
|
||||
}
|
||||
|
||||
func signAPIURL(rawURL string) string {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return rawURL
|
||||
}
|
||||
query := u.Query()
|
||||
k, v := signPath(u.Path, defaultPlatform, defaultAppVersion)
|
||||
query.Add(k, v)
|
||||
u.RawQuery = query.Encode()
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func splitPath(p string) []string {
|
||||
p = strings.Trim(p, "/")
|
||||
if p == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(p, "/")
|
||||
}
|
||||
|
||||
func fileToEntry(f panFile, parentID string) drives.Entry {
|
||||
return drives.Entry{
|
||||
ID: strconv.FormatInt(f.FileID, 10),
|
||||
Name: f.FileName,
|
||||
Size: f.Size,
|
||||
Hash: strings.ToLower(f.Etag),
|
||||
IsDir: f.Type == 1,
|
||||
ParentID: parentID,
|
||||
MimeType: guessMime(f.FileName),
|
||||
ModTime: f.UpdateAt.Time(),
|
||||
}
|
||||
}
|
||||
|
||||
func guessMime(name string) string {
|
||||
ext := strings.ToLower(path.Ext(name))
|
||||
switch ext {
|
||||
case ".mp4":
|
||||
return "video/mp4"
|
||||
case ".mkv":
|
||||
return "video/x-matroska"
|
||||
case ".mov":
|
||||
return "video/quicktime"
|
||||
case ".webm":
|
||||
return "video/webm"
|
||||
case ".jpg", ".jpeg":
|
||||
return "image/jpeg"
|
||||
case ".png":
|
||||
return "image/png"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
var _ drives.Drive = (*Driver)(nil)
|
||||
@@ -0,0 +1,256 @@
|
||||
package p123
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestStreamURLResolvesDownloadInfoRedirect(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var downloadReferer string
|
||||
var download *httptest.Server
|
||||
download = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/resolve":
|
||||
downloadReferer = r.Header.Get("Referer")
|
||||
http.Redirect(w, r, download.URL+"/cdn/video.mp4", http.StatusFound)
|
||||
case "/cdn/video.mp4":
|
||||
t.Fatalf("driver followed redirect unexpectedly")
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer download.Close()
|
||||
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/api/user/sign_in":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 200,
|
||||
"data": map[string]string{"token": "token-1"},
|
||||
})
|
||||
case "/b/api/user/info":
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer token-1" {
|
||||
t.Fatalf("Authorization = %q, want bearer token", got)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{}})
|
||||
case "/b/api/file/list/new":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"Next": "-1",
|
||||
"Total": 1,
|
||||
"InfoList": []map[string]any{
|
||||
{
|
||||
"FileName": "video.mp4",
|
||||
"Size": 1234,
|
||||
"UpdateAt": "2026-01-02 03:04:05",
|
||||
"FileId": 100,
|
||||
"Type": 0,
|
||||
"Etag": "ABCDEF",
|
||||
"S3KeyFlag": "flag-1",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
case "/b/api/file/download_info":
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode download_info body: %v", err)
|
||||
}
|
||||
if got := body["fileName"]; got != "video.mp4" {
|
||||
t.Fatalf("fileName = %#v, want cached file metadata", got)
|
||||
}
|
||||
if got := body["etag"]; got != "ABCDEF" {
|
||||
t.Fatalf("etag = %#v, want cached etag", got)
|
||||
}
|
||||
entryURL := download.URL + "/entry?params=" + base64.StdEncoding.EncodeToString([]byte(download.URL+"/resolve"))
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]string{"DownloadUrl": entryURL},
|
||||
})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
var savedToken string
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
Username: "user@example.com",
|
||||
Password: "secret",
|
||||
MainAPIBaseURL: api.URL + "/b/api",
|
||||
LoginAPIBaseURL: api.URL + "/api",
|
||||
OnTokenUpdate: func(access string) {
|
||||
savedToken = access
|
||||
},
|
||||
})
|
||||
if err := d.Init(ctx); err != nil {
|
||||
t.Fatalf("Init() error = %v", err)
|
||||
}
|
||||
if savedToken != "token-1" {
|
||||
t.Fatalf("saved token = %q, want token-1", savedToken)
|
||||
}
|
||||
if _, err := d.List(ctx, d.RootID()); err != nil {
|
||||
t.Fatalf("List() error = %v", err)
|
||||
}
|
||||
|
||||
link, err := d.StreamURL(ctx, "100")
|
||||
if err != nil {
|
||||
t.Fatalf("StreamURL() error = %v", err)
|
||||
}
|
||||
if got := link.URL; got != download.URL+"/cdn/video.mp4" {
|
||||
t.Fatalf("URL = %q, want final CDN URL", got)
|
||||
}
|
||||
if got := link.Headers.Get("Referer"); !strings.HasPrefix(got, download.URL) {
|
||||
t.Fatalf("Referer = %q, want original download host", got)
|
||||
}
|
||||
if downloadReferer != defaultReferer {
|
||||
t.Fatalf("resolve Referer = %q, want %q", downloadReferer, defaultReferer)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitUsesAccessTokenWithoutLogin(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/api/user/sign_in":
|
||||
t.Fatalf("driver should not password-login when access_token is configured")
|
||||
case "/b/api/user/info":
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer token-1" {
|
||||
t.Fatalf("Authorization = %q, want bearer token", got)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{}})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "Bearer token-1",
|
||||
MainAPIBaseURL: api.URL + "/b/api",
|
||||
LoginAPIBaseURL: api.URL + "/api",
|
||||
})
|
||||
if err := d.Init(context.Background()); err != nil {
|
||||
t.Fatalf("Init() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginRiskErrorSuggestsAccessToken(t *testing.T) {
|
||||
err := loginError("当前账号存在境外登录风险,请使用短信验证码或者微信进行登录。")
|
||||
if err == nil || !strings.Contains(err.Error(), "access_token") {
|
||||
t.Fatalf("loginError() = %v, want access_token guidance", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestCode429ReturnsRateLimitError(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Retry-After", "2")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 429,
|
||||
"message": "请求太频繁",
|
||||
})
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "token-1",
|
||||
MainAPIBaseURL: api.URL,
|
||||
})
|
||||
_, err := d.request(context.Background(), endpointFileList, http.MethodGet, nil, nil)
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != 2*time.Second {
|
||||
t.Fatalf("RetryAfter = %s, want 2s", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListCoolsDownAndRetriesRateLimit(t *testing.T) {
|
||||
var listCalls int
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/file/list/new" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
listCalls++
|
||||
if listCalls == 1 {
|
||||
w.Header().Set("Retry-After", "1")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 429,
|
||||
"message": "请求太频繁",
|
||||
})
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"Next": "-1",
|
||||
"Total": 1,
|
||||
"InfoList": []map[string]any{
|
||||
{
|
||||
"FileName": "video.mp4",
|
||||
"Size": 1234,
|
||||
"UpdateAt": "2026-01-02 03:04:05",
|
||||
"FileId": 100,
|
||||
"Type": 0,
|
||||
"Etag": "ABCDEF",
|
||||
"S3KeyFlag": "flag-1",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "123-main",
|
||||
AccessToken: "token-1",
|
||||
MainAPIBaseURL: api.URL,
|
||||
})
|
||||
entries, err := d.List(context.Background(), d.RootID())
|
||||
if err != nil {
|
||||
t.Fatalf("List() error = %v", err)
|
||||
}
|
||||
if listCalls != 2 {
|
||||
t.Fatalf("list calls = %d, want 2", listCalls)
|
||||
}
|
||||
if len(entries) != 1 || entries[0].ID != "100" {
|
||||
t.Fatalf("entries = %#v, want one file", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDownloadURL429ReturnsRateLimitError(t *testing.T) {
|
||||
download := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Retry-After", "3")
|
||||
http.Error(w, "too many requests", http.StatusTooManyRequests)
|
||||
}))
|
||||
defer download.Close()
|
||||
|
||||
d := New(Config{ID: "123-main"})
|
||||
_, err := d.resolveDownloadURL(context.Background(), download.URL)
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != 3*time.Second {
|
||||
t.Fatalf("RetryAfter = %s, want 3s", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
package p123
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-resty/resty/v2"
|
||||
"github.com/skip2/go-qrcode"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultUserAPIBase = "https://user.123pan.cn/api"
|
||||
defaultQRLoginPage = "https://www.123pan.com/wx-app-login.html"
|
||||
defaultQRReferer = "https://user.123pan.com/centerlogin"
|
||||
defaultQROrigin = "https://user.123pan.com"
|
||||
defaultQRUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36"
|
||||
|
||||
endpointQRCodeGenerate = "/user/qr-code/generate"
|
||||
endpointQRCodeResult = "/user/qr-code/result"
|
||||
endpointQRCodeWXCode = "/user/qr-code/wx_code"
|
||||
)
|
||||
|
||||
type QRConfig struct {
|
||||
UserAPIBaseURL string
|
||||
HTTPClient *http.Client
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type QRClient struct {
|
||||
userAPIBase string
|
||||
client *resty.Client
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type QRCodeSession struct {
|
||||
LoginUUID string `json:"loginUuid"`
|
||||
UniID string `json:"uniID"`
|
||||
QRCodeURL string `json:"qrCodeUrl"`
|
||||
QRImageDataURL string `json:"qrImageDataUrl"`
|
||||
ExpiresAt string `json:"expiresAt,omitempty"`
|
||||
}
|
||||
|
||||
type QRCodeStatus struct {
|
||||
LoginStatus int `json:"loginStatus"`
|
||||
StatusText string `json:"statusText"`
|
||||
ScanPlatform int `json:"scanPlatform,omitempty"`
|
||||
PlatformText string `json:"platformText,omitempty"`
|
||||
AccessToken string `json:"accessToken,omitempty"`
|
||||
}
|
||||
|
||||
func NewQRClient(c QRConfig) *QRClient {
|
||||
userAPIBase := strings.TrimRight(strings.TrimSpace(c.UserAPIBaseURL), "/")
|
||||
if userAPIBase == "" {
|
||||
userAPIBase = defaultUserAPIBase
|
||||
}
|
||||
httpClient := c.HTTPClient
|
||||
if httpClient == nil {
|
||||
httpClient = &http.Client{Timeout: 20 * time.Second}
|
||||
}
|
||||
now := c.Now
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
return &QRClient{
|
||||
userAPIBase: userAPIBase,
|
||||
client: resty.NewWithClient(httpClient).
|
||||
SetTimeout(20*time.Second).
|
||||
SetHeader("Accept", "application/json, text/plain, */*"),
|
||||
now: now,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *QRClient) Generate(ctx context.Context) (QRCodeSession, error) {
|
||||
loginUUID, err := newLoginUUID()
|
||||
if err != nil {
|
||||
return QRCodeSession{}, err
|
||||
}
|
||||
var resp qrGenerateResp
|
||||
res, err := c.request(ctx, loginUUID).
|
||||
SetResult(&resp).
|
||||
Get(c.userAPIBase + endpointQRCodeGenerate)
|
||||
if err != nil {
|
||||
return QRCodeSession{}, err
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return QRCodeSession{}, qrAPIError(resp.Message, res.StatusCode(), resp.Code)
|
||||
}
|
||||
uniID := strings.TrimSpace(resp.Data.UniID)
|
||||
if uniID == "" {
|
||||
return QRCodeSession{}, errors.New("123pan qr: empty uniID")
|
||||
}
|
||||
qrURL := buildQRLoginURL(resp.Data.URL, uniID)
|
||||
png, err := qrcode.Encode(qrURL, qrcode.Medium, 220)
|
||||
if err != nil {
|
||||
return QRCodeSession{}, err
|
||||
}
|
||||
return QRCodeSession{
|
||||
LoginUUID: loginUUID,
|
||||
UniID: uniID,
|
||||
QRCodeURL: qrURL,
|
||||
QRImageDataURL: "data:image/png;base64," + base64.StdEncoding.EncodeToString(png),
|
||||
ExpiresAt: c.now().Add(5 * time.Minute).Format(time.RFC3339),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *QRClient) Poll(ctx context.Context, loginUUID, uniID string) (QRCodeStatus, error) {
|
||||
loginUUID = strings.TrimSpace(loginUUID)
|
||||
uniID = strings.TrimSpace(uniID)
|
||||
if loginUUID == "" {
|
||||
return QRCodeStatus{}, errors.New("loginUuid is required")
|
||||
}
|
||||
if uniID == "" {
|
||||
return QRCodeStatus{}, errors.New("uniID is required")
|
||||
}
|
||||
var resp qrResultResp
|
||||
res, err := c.request(ctx, loginUUID).
|
||||
SetQueryParam("uniID", uniID).
|
||||
SetResult(&resp).
|
||||
Get(c.userAPIBase + endpointQRCodeResult)
|
||||
if err != nil {
|
||||
return QRCodeStatus{}, err
|
||||
}
|
||||
if resp.Code != 0 && resp.Code != 200 {
|
||||
return QRCodeStatus{}, qrAPIError(resp.Message, res.StatusCode(), resp.Code)
|
||||
}
|
||||
if resp.Code == 200 {
|
||||
resp.Data.LoginStatus = 3
|
||||
if resp.Data.ScanPlatform == 0 {
|
||||
resp.Data.ScanPlatform = resp.Data.LoginType
|
||||
}
|
||||
}
|
||||
status := QRCodeStatus{
|
||||
LoginStatus: resp.Data.LoginStatus,
|
||||
StatusText: qrLoginStatusText(resp.Data.LoginStatus),
|
||||
ScanPlatform: resp.Data.ScanPlatform,
|
||||
PlatformText: qrScanPlatformText(resp.Data.ScanPlatform),
|
||||
}
|
||||
if status.LoginStatus != 3 {
|
||||
return status, nil
|
||||
}
|
||||
if token := resp.TokenValue(); token != "" {
|
||||
status.AccessToken = normalizeAccessToken(token)
|
||||
return status, nil
|
||||
}
|
||||
if resp.Data.ScanPlatform == 4 {
|
||||
token, err := c.finishWechatLogin(ctx, loginUUID, uniID)
|
||||
if err != nil {
|
||||
return QRCodeStatus{}, err
|
||||
}
|
||||
status.AccessToken = normalizeAccessToken(token)
|
||||
return status, nil
|
||||
}
|
||||
return QRCodeStatus{}, errors.New("123pan qr: confirmed login returned empty token")
|
||||
}
|
||||
|
||||
func (c *QRClient) finishWechatLogin(ctx context.Context, loginUUID, uniID string) (string, error) {
|
||||
var wxResp qrWXCodeResp
|
||||
res, err := c.request(ctx, loginUUID).
|
||||
SetBody(map[string]string{"uniID": uniID}).
|
||||
SetResult(&wxResp).
|
||||
Post(c.userAPIBase + endpointQRCodeWXCode)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if wxResp.Code != 0 {
|
||||
return "", qrAPIError(wxResp.Message, res.StatusCode(), wxResp.Code)
|
||||
}
|
||||
wxCode := strings.TrimSpace(wxResp.WXCode())
|
||||
if wxCode == "" {
|
||||
return "", errors.New("123pan qr: empty wechat code")
|
||||
}
|
||||
var signIn loginResp
|
||||
res, err = c.request(ctx, loginUUID).
|
||||
SetBody(map[string]any{
|
||||
"from": "web",
|
||||
"wechat_code": wxCode,
|
||||
"type": 4,
|
||||
}).
|
||||
SetResult(&signIn).
|
||||
Post(c.userAPIBase + endpointSignIn)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if signIn.Code != 200 && signIn.Code != 0 {
|
||||
return "", qrAPIError(signIn.Message, res.StatusCode(), signIn.Code)
|
||||
}
|
||||
token := strings.TrimSpace(signIn.Data.Token)
|
||||
if token == "" {
|
||||
return "", errors.New("123pan qr: empty token")
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func (c *QRClient) request(ctx context.Context, loginUUID string) *resty.Request {
|
||||
return c.client.R().
|
||||
SetContext(ctx).
|
||||
SetHeaders(map[string]string{
|
||||
"Content-Type": "application/json;charset=UTF-8",
|
||||
"platform": defaultPlatform,
|
||||
"App-Version": defaultAppVersion,
|
||||
"LoginUuid": loginUUID,
|
||||
"Referer": defaultQRReferer,
|
||||
"Origin": defaultQROrigin,
|
||||
"User-Agent": defaultQRUserAgent,
|
||||
})
|
||||
}
|
||||
|
||||
func buildQRLoginURL(raw, uniID string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
raw = defaultQRLoginPage
|
||||
}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return defaultQRLoginPage + "?env=production&uniID=" + url.QueryEscape(uniID) + "&source=123pan&type=login"
|
||||
}
|
||||
q := u.Query()
|
||||
q.Set("env", "production")
|
||||
q.Set("uniID", uniID)
|
||||
q.Set("source", "123pan")
|
||||
q.Set("type", "login")
|
||||
u.RawQuery = q.Encode()
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func newLoginUUID() (string, error) {
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
b[6] = (b[6] & 0x0f) | 0x40
|
||||
b[8] = (b[8] & 0x3f) | 0x80
|
||||
parts := []string{
|
||||
hex.EncodeToString(b[0:4]),
|
||||
hex.EncodeToString(b[4:6]),
|
||||
hex.EncodeToString(b[6:8]),
|
||||
hex.EncodeToString(b[8:10]),
|
||||
hex.EncodeToString(b[10:16]),
|
||||
}
|
||||
return strings.Join(parts, "-"), nil
|
||||
}
|
||||
|
||||
func qrAPIError(message string, httpStatus, apiCode int) error {
|
||||
message = strings.TrimSpace(message)
|
||||
if message == "" {
|
||||
message = fmt.Sprintf("HTTP %d code=%d", httpStatus, apiCode)
|
||||
}
|
||||
return errors.New(message)
|
||||
}
|
||||
|
||||
func qrLoginStatusText(status int) string {
|
||||
switch status {
|
||||
case 0:
|
||||
return "等待扫码"
|
||||
case 1:
|
||||
return "已扫码,等待确认"
|
||||
case 2:
|
||||
return "已拒绝"
|
||||
case 3:
|
||||
return "已确认"
|
||||
case 4:
|
||||
return "已过期"
|
||||
default:
|
||||
return "未知状态"
|
||||
}
|
||||
}
|
||||
|
||||
func qrScanPlatformText(platform int) string {
|
||||
switch platform {
|
||||
case 4:
|
||||
return "微信"
|
||||
case 7:
|
||||
return "123 云盘 App"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
package p123
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestQRCodeGenerateBuildsImage(t *testing.T) {
|
||||
var seenLoginUUID string
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/api/user/qr-code/generate" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
seenLoginUUID = r.Header.Get("LoginUuid")
|
||||
if seenLoginUUID == "" {
|
||||
t.Fatalf("missing LoginUuid header")
|
||||
}
|
||||
if r.Header.Get("platform") != defaultPlatform {
|
||||
t.Fatalf("platform header = %q, want %q", r.Header.Get("platform"), defaultPlatform)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"message": "ok",
|
||||
"data": map[string]string{
|
||||
"uniID": "uni-1",
|
||||
"url": "https://www.123pan.com/wx-app-login.html",
|
||||
},
|
||||
})
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{UserAPIBaseURL: api.URL + "/api"}).Generate(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
if got.LoginUUID != seenLoginUUID {
|
||||
t.Fatalf("loginUuid = %q, want header %q", got.LoginUUID, seenLoginUUID)
|
||||
}
|
||||
if got.UniID != "uni-1" {
|
||||
t.Fatalf("uniID = %q, want uni-1", got.UniID)
|
||||
}
|
||||
if !strings.Contains(got.QRCodeURL, "uniID=uni-1") || !strings.Contains(got.QRCodeURL, "type=login") {
|
||||
t.Fatalf("qrCodeUrl = %q, want login params", got.QRCodeURL)
|
||||
}
|
||||
if !strings.HasPrefix(got.QRImageDataURL, "data:image/png;base64,") {
|
||||
t.Fatalf("qrImageDataUrl missing png data url prefix")
|
||||
}
|
||||
if got.ExpiresAt == "" {
|
||||
t.Fatalf("expiresAt is empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQRCodePollCompletesWechatLogin(t *testing.T) {
|
||||
var wxCodeRequested bool
|
||||
var signInRequested bool
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.Header.Get("LoginUuid") != "login-1" {
|
||||
t.Fatalf("LoginUuid = %q, want login-1", r.Header.Get("LoginUuid"))
|
||||
}
|
||||
switch r.URL.Path {
|
||||
case "/api/user/qr-code/result":
|
||||
if r.URL.Query().Get("uniID") != "uni-1" {
|
||||
t.Fatalf("uniID = %q, want uni-1", r.URL.Query().Get("uniID"))
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"loginStatus": 3,
|
||||
"scanPlatform": 4,
|
||||
},
|
||||
})
|
||||
case "/api/user/qr-code/wx_code":
|
||||
wxCodeRequested = true
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode wx_code body: %v", err)
|
||||
}
|
||||
if body["uniID"] != "uni-1" {
|
||||
t.Fatalf("wx_code uniID = %q, want uni-1", body["uniID"])
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]string{"wxCode": "wx-code-1"},
|
||||
})
|
||||
case "/api/user/sign_in":
|
||||
signInRequested = true
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode sign_in body: %v", err)
|
||||
}
|
||||
if body["wechat_code"] != "wx-code-1" {
|
||||
t.Fatalf("wechat_code = %#v, want wx-code-1", body["wechat_code"])
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 200,
|
||||
"data": map[string]string{"token": "Bearer token-1"},
|
||||
})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{UserAPIBaseURL: api.URL + "/api"}).Poll(context.Background(), "login-1", "uni-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Poll() error = %v", err)
|
||||
}
|
||||
if !wxCodeRequested || !signInRequested {
|
||||
t.Fatalf("wechat completion calls wx=%v signIn=%v, want both", wxCodeRequested, signInRequested)
|
||||
}
|
||||
if got.LoginStatus != 3 || got.AccessToken != "token-1" || got.PlatformText != "微信" {
|
||||
t.Fatalf("status = %#v, want confirmed wechat token", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQRCodePollUsesAppToken(t *testing.T) {
|
||||
var wxCodeRequested bool
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/api/user/qr-code/result":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 0,
|
||||
"data": map[string]any{
|
||||
"loginStatus": 3,
|
||||
"scanPlatform": 7,
|
||||
"token": "app-token",
|
||||
},
|
||||
})
|
||||
case "/api/user/qr-code/wx_code":
|
||||
wxCodeRequested = true
|
||||
http.Error(w, "unexpected wx_code", http.StatusInternalServerError)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{UserAPIBaseURL: api.URL + "/api"}).Poll(context.Background(), "login-1", "uni-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Poll() error = %v", err)
|
||||
}
|
||||
if wxCodeRequested {
|
||||
t.Fatalf("wx_code should not be called when app token is already returned")
|
||||
}
|
||||
if got.AccessToken != "app-token" || got.PlatformText != "123 云盘 App" {
|
||||
t.Fatalf("status = %#v, want app token", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQRCodePollUsesOfficialAppSuccessCode(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path != "/api/user/qr-code/result" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"code": 200,
|
||||
"data": map[string]any{
|
||||
"login_type": 7,
|
||||
"token": "app-token",
|
||||
},
|
||||
})
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
|
||||
got, err := NewQRClient(QRConfig{UserAPIBaseURL: api.URL + "/api"}).Poll(context.Background(), "login-1", "uni-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Poll() error = %v", err)
|
||||
}
|
||||
if got.LoginStatus != 3 || got.ScanPlatform != 7 || got.AccessToken != "app-token" {
|
||||
t.Fatalf("status = %#v, want official app success token", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package p123
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type apiEnvelope struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
type loginResp struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
Token string `json:"token"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type qrGenerateResp struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
UniID string `json:"uniID"`
|
||||
URL string `json:"url"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type qrResultResp struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
LoginStatus int `json:"loginStatus"`
|
||||
ScanPlatform int `json:"scanPlatform"`
|
||||
LoginType int `json:"login_type"`
|
||||
Token string `json:"token"`
|
||||
AccessToken string `json:"accessToken"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func (r qrResultResp) TokenValue() string {
|
||||
if strings.TrimSpace(r.Data.Token) != "" {
|
||||
return r.Data.Token
|
||||
}
|
||||
return r.Data.AccessToken
|
||||
}
|
||||
|
||||
type qrWXCodeResp struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
WXCodeLower string `json:"wxCode"`
|
||||
WXCodeTitle string `json:"WxCode"`
|
||||
Code string `json:"code"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func (r qrWXCodeResp) WXCode() string {
|
||||
if r.Data.WXCodeLower != "" {
|
||||
return r.Data.WXCodeLower
|
||||
}
|
||||
if r.Data.WXCodeTitle != "" {
|
||||
return r.Data.WXCodeTitle
|
||||
}
|
||||
return r.Data.Code
|
||||
}
|
||||
|
||||
type fileListResp struct {
|
||||
Data struct {
|
||||
Next string `json:"Next"`
|
||||
Total int `json:"Total"`
|
||||
InfoList []panFile `json:"InfoList"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type panFile struct {
|
||||
FileName string `json:"FileName"`
|
||||
Size int64 `json:"Size"`
|
||||
UpdateAt flexibleTime `json:"UpdateAt"`
|
||||
FileID int64 `json:"FileId"`
|
||||
Type int `json:"Type"`
|
||||
Etag string `json:"Etag"`
|
||||
S3KeyFlag string `json:"S3KeyFlag"`
|
||||
}
|
||||
|
||||
type cachedFile struct {
|
||||
file panFile
|
||||
parentID string
|
||||
}
|
||||
|
||||
type downloadInfoResp struct {
|
||||
Data struct {
|
||||
DownloadURL string `json:"DownloadUrl"`
|
||||
DownloadURLLower string `json:"downloadUrl"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func (r downloadInfoResp) URL() string {
|
||||
if r.Data.DownloadURL != "" {
|
||||
return r.Data.DownloadURL
|
||||
}
|
||||
return r.Data.DownloadURLLower
|
||||
}
|
||||
|
||||
type redirectResp struct {
|
||||
Data struct {
|
||||
RedirectURL string `json:"redirect_url"`
|
||||
RedirectURLCamel string `json:"redirectUrl"`
|
||||
RedirectURLTitle string `json:"RedirectUrl"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func (r redirectResp) URL() string {
|
||||
if r.Data.RedirectURL != "" {
|
||||
return r.Data.RedirectURL
|
||||
}
|
||||
if r.Data.RedirectURLCamel != "" {
|
||||
return r.Data.RedirectURLCamel
|
||||
}
|
||||
return r.Data.RedirectURLTitle
|
||||
}
|
||||
|
||||
type mkdirResp struct {
|
||||
Data struct {
|
||||
FileID int64 `json:"FileId"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type flexibleTime struct {
|
||||
t time.Time
|
||||
}
|
||||
|
||||
func (t *flexibleTime) UnmarshalJSON(data []byte) error {
|
||||
if string(data) == "null" || string(data) == `""` {
|
||||
return nil
|
||||
}
|
||||
var s string
|
||||
if err := json.Unmarshal(data, &s); err == nil {
|
||||
t.t = parseTimeString(s)
|
||||
return nil
|
||||
}
|
||||
var n int64
|
||||
if err := json.Unmarshal(data, &n); err == nil {
|
||||
if n > 1_000_000_000_000 {
|
||||
t.t = time.UnixMilli(n)
|
||||
} else {
|
||||
t.t = time.Unix(n, 0)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t flexibleTime) Time() time.Time {
|
||||
return t.t
|
||||
}
|
||||
|
||||
func parseTimeString(s string) time.Time {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
for _, layout := range []string{
|
||||
time.RFC3339Nano,
|
||||
time.RFC3339,
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02T15:04:05",
|
||||
} {
|
||||
if parsed, err := time.ParseInLocation(layout, s, time.FixedZone("UTC+8", 8*3600)); err == nil {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
if n, err := strconv.ParseInt(s, 10, 64); err == nil {
|
||||
if n > 1_000_000_000_000 {
|
||||
return time.UnixMilli(n)
|
||||
}
|
||||
return time.Unix(n, 0)
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -20,6 +21,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"golang.org/x/net/proxy"
|
||||
)
|
||||
|
||||
// 默认 author/tag 标签,便于在前端筛选 spider91 来源的视频。
|
||||
@@ -79,29 +81,123 @@ func NewCrawler(cfg CrawlerConfig) *Crawler {
|
||||
cfg.DownloadTimeout = 30 * time.Minute
|
||||
}
|
||||
if cfg.HTTPClient == nil {
|
||||
// 选 proxy 函数:显式 ProxyURL > 环境变量 > 直连
|
||||
proxyFn := http.ProxyFromEnvironment
|
||||
if strings.TrimSpace(cfg.ProxyURL) != "" {
|
||||
if u, err := url.Parse(cfg.ProxyURL); err == nil {
|
||||
proxyFn = http.ProxyURL(u)
|
||||
} else {
|
||||
log.Printf("[spider91] invalid proxy URL %q, falling back to env: %v", cfg.ProxyURL, err)
|
||||
}
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
ResponseHeaderTimeout: 60 * time.Second,
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
}
|
||||
if err := configureExplicitProxy(transport, cfg.ProxyURL); err != nil {
|
||||
log.Printf("[spider91] invalid configured proxy URL, falling back to env: %v", err)
|
||||
}
|
||||
cfg.HTTPClient = &http.Client{
|
||||
// 不限制总下载时长,靠 ctx 控制;只挡 dial / handshake / header
|
||||
Timeout: 0,
|
||||
Transport: &http.Transport{
|
||||
Proxy: proxyFn,
|
||||
ResponseHeaderTimeout: 60 * time.Second,
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
},
|
||||
Timeout: 0,
|
||||
Transport: transport,
|
||||
}
|
||||
}
|
||||
return &Crawler{cfg: cfg}
|
||||
}
|
||||
|
||||
func configureExplicitProxy(transport *http.Transport, raw string) error {
|
||||
proxyURL := strings.TrimSpace(raw)
|
||||
if proxyURL == "" {
|
||||
return nil
|
||||
}
|
||||
u, err := url.Parse(proxyURL)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return fmt.Errorf("invalid proxy URL")
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https":
|
||||
transport.Proxy = http.ProxyURL(u)
|
||||
transport.DialContext = nil
|
||||
return nil
|
||||
case "socks5", "socks5h":
|
||||
dialContext, err := socksProxyDialContext(u)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
transport.Proxy = nil
|
||||
transport.DialContext = dialContext
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("unsupported proxy scheme %q", u.Scheme)
|
||||
}
|
||||
}
|
||||
|
||||
func socksProxyDialContext(proxyURL *url.URL) (func(context.Context, string, string) (net.Conn, error), error) {
|
||||
var auth *proxy.Auth
|
||||
if proxyURL.User != nil {
|
||||
username := proxyURL.User.Username()
|
||||
password, _ := proxyURL.User.Password()
|
||||
auth = &proxy.Auth{User: username, Password: password}
|
||||
}
|
||||
dialer, err := proxy.SOCKS5("tcp", proxyURL.Host, auth, &net.Dialer{Timeout: 60 * time.Second})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remoteDNS := strings.EqualFold(proxyURL.Scheme, "socks5h")
|
||||
return func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
target := addr
|
||||
if !remoteDNS {
|
||||
resolved, err := resolveSocksTarget(ctx, addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target = resolved
|
||||
}
|
||||
if ctxDialer, ok := dialer.(proxy.ContextDialer); ok {
|
||||
return ctxDialer.DialContext(ctx, network, target)
|
||||
}
|
||||
type result struct {
|
||||
conn net.Conn
|
||||
err error
|
||||
}
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
conn, err := dialer.Dial(network, target)
|
||||
ch <- result{conn: conn, err: err}
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case res := <-ch:
|
||||
return res.conn, res.err
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveSocksTarget(ctx context.Context, addr string) (string, error) {
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil || net.ParseIP(host) != nil {
|
||||
return addr, nil
|
||||
}
|
||||
ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
ip := selectSocksTargetIP(ips)
|
||||
if ip == nil {
|
||||
return "", fmt.Errorf("resolve %s: no address", host)
|
||||
}
|
||||
return net.JoinHostPort(ip.String(), port), nil
|
||||
}
|
||||
|
||||
func selectSocksTargetIP(ips []net.IPAddr) net.IP {
|
||||
for _, addr := range ips {
|
||||
if ip4 := addr.IP.To4(); ip4 != nil {
|
||||
return ip4
|
||||
}
|
||||
}
|
||||
for _, addr := range ips {
|
||||
if addr.IP != nil {
|
||||
return addr.IP
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CrawlResult 汇总一次 RunOnce 的结果。
|
||||
type CrawlResult struct {
|
||||
// TargetNew 是本次 RunOnce 的目标新增数(来自 drive.Credentials.target_new)。
|
||||
@@ -324,6 +420,16 @@ func (c *Crawler) startSpiderTargetNew(ctx context.Context, targetNew int, seenP
|
||||
if c.cfg.WorkDir != "" {
|
||||
cmd.Dir = c.cfg.WorkDir
|
||||
}
|
||||
if proxyURL := strings.TrimSpace(c.cfg.ProxyURL); proxyURL != "" {
|
||||
cmd.Env = append(os.Environ(),
|
||||
"HTTP_PROXY="+proxyURL,
|
||||
"HTTPS_PROXY="+proxyURL,
|
||||
"http_proxy="+proxyURL,
|
||||
"https_proxy="+proxyURL,
|
||||
"NO_PROXY=",
|
||||
"no_proxy=",
|
||||
)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("stdout pipe: %w", err)
|
||||
@@ -427,6 +533,17 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
}
|
||||
}
|
||||
|
||||
title := strings.TrimSpace(item.Title)
|
||||
if title == "" {
|
||||
title = sourceID
|
||||
}
|
||||
tags := []string{DefaultTag}
|
||||
if matched, err := c.cfg.Catalog.MatchTags(ctx, title+" "+DefaultAuthor); err == nil {
|
||||
tags = mergeCatalogTags(tags, matched)
|
||||
} else {
|
||||
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s match tags: %v", c.cfg.Driver.ID(), viewkey, sourceID, err)
|
||||
}
|
||||
|
||||
// 入库
|
||||
now := time.Now()
|
||||
v := &catalog.Video{
|
||||
@@ -434,9 +551,9 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
DriveID: c.cfg.Driver.ID(),
|
||||
FileID: videoFile,
|
||||
FileName: videoFile,
|
||||
Title: strings.TrimSpace(item.Title),
|
||||
Title: title,
|
||||
Author: DefaultAuthor,
|
||||
Tags: []string{DefaultTag},
|
||||
Tags: tags,
|
||||
Ext: strings.TrimPrefix(videoExt, "."),
|
||||
Quality: "HD",
|
||||
Size: videoSize,
|
||||
@@ -445,9 +562,6 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if v.Title == "" {
|
||||
v.Title = sourceID
|
||||
}
|
||||
if thumbReady {
|
||||
// 设了 ThumbnailURL 后 thumb worker 会跳过这条视频,
|
||||
// 不再尝试用 ffmpeg 抽帧(封面已经是网站原图)。
|
||||
@@ -888,6 +1002,26 @@ func copyFileAtomic(src, dst string) error {
|
||||
return os.Rename(tmp, dst)
|
||||
}
|
||||
|
||||
func mergeCatalogTags(lists ...[]string) []string {
|
||||
out := []string{}
|
||||
seen := map[string]bool{}
|
||||
for _, list := range lists {
|
||||
for _, tag := range list {
|
||||
tag = strings.TrimSpace(tag)
|
||||
if tag == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(tag)
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, tag)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BuildVideoID 给定 driveID + 91 源视频 ID,按统一规则生成 catalog 中 videos.id。
|
||||
// 与 scanner 用法一致:<kind>-<driveID>-<fileID>。
|
||||
func BuildVideoID(driveID, sourceID string) string {
|
||||
|
||||
@@ -3,6 +3,8 @@ package spider91
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
@@ -53,7 +55,7 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
|
||||
// 同时仍写 --output 文件作归档。
|
||||
videoEntries := []map[string]string{
|
||||
{
|
||||
"title": "Video One",
|
||||
"title": "Video One 口交",
|
||||
"thumb_url": srv.URL + "/thumb/not-120001.jpg",
|
||||
"video_url": srv.URL + "/videos/120001.mp4",
|
||||
"viewkey": "vk-001",
|
||||
@@ -94,6 +96,9 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert drive: %v", err)
|
||||
}
|
||||
if _, err := cat.CreateTagAndClassify(context.Background(), "Video One", nil, "user"); err != nil {
|
||||
t.Fatalf("create user tag: %v", err)
|
||||
}
|
||||
|
||||
var newVideos []*catalog.Video
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
@@ -188,6 +193,17 @@ func TestCrawlerRunOnceFullFlow(t *testing.T) {
|
||||
if !hasDefaultTag {
|
||||
t.Fatalf("video %s tags = %v, want contain %q", videoID, v.Tags, DefaultTag)
|
||||
}
|
||||
if sourceID == "120001" {
|
||||
if !containsString(v.Tags, "口交") {
|
||||
t.Fatalf("video %s tags = %v, want contain built-in tag 口交", videoID, v.Tags)
|
||||
}
|
||||
if !containsString(v.Tags, "Video One") {
|
||||
t.Fatalf("video %s tags = %v, want contain user tag Video One", videoID, v.Tags)
|
||||
}
|
||||
}
|
||||
if sourceID == "120002" && (containsString(v.Tags, "口交") || containsString(v.Tags, "Video One")) {
|
||||
t.Fatalf("video %s tags = %v, should not inherit tags from other spider91 videos", videoID, v.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
// 7. 第二次 RunOnce:源视频 ID 已存在 → 全部 skipped,无新文件下载
|
||||
@@ -233,6 +249,108 @@ func TestCrawlerRunOnceMissingScript(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlerPassesProxyToSpiderProcess(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell-based fake script only on unix")
|
||||
}
|
||||
|
||||
tmp := t.TempDir()
|
||||
scriptPath := filepath.Join(tmp, "print_proxy_env.sh")
|
||||
script := `#!/bin/sh
|
||||
printf 'HTTP_PROXY=%s\n' "$HTTP_PROXY"
|
||||
printf 'HTTPS_PROXY=%s\n' "$HTTPS_PROXY"
|
||||
printf 'http_proxy=%s\n' "$http_proxy"
|
||||
printf 'https_proxy=%s\n' "$https_proxy"
|
||||
printf 'NO_PROXY=%s\n' "$NO_PROXY"
|
||||
printf 'no_proxy=%s\n' "$no_proxy"
|
||||
`
|
||||
if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
|
||||
t.Fatalf("write script: %v", err)
|
||||
}
|
||||
|
||||
proxyURL := "socks5h://proxy.local:1080"
|
||||
drv := New(Config{ID: "proxy-drive", RootDir: filepath.Join(tmp, "proxy-drive")})
|
||||
c := NewCrawler(CrawlerConfig{
|
||||
Driver: drv,
|
||||
PythonPath: "sh",
|
||||
ScriptPath: scriptPath,
|
||||
ProxyURL: proxyURL,
|
||||
})
|
||||
cmd, stdout, err := c.startSpiderTargetNew(
|
||||
context.Background(),
|
||||
1,
|
||||
filepath.Join(tmp, "seen.txt"),
|
||||
filepath.Join(tmp, "out.json"),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("startSpiderTargetNew: %v", err)
|
||||
}
|
||||
raw, err := io.ReadAll(stdout)
|
||||
if err != nil {
|
||||
t.Fatalf("read stdout: %v", err)
|
||||
}
|
||||
if err := cmd.Wait(); err != nil {
|
||||
t.Fatalf("wait: %v", err)
|
||||
}
|
||||
|
||||
want := strings.Join([]string{
|
||||
"HTTP_PROXY=" + proxyURL,
|
||||
"HTTPS_PROXY=" + proxyURL,
|
||||
"http_proxy=" + proxyURL,
|
||||
"https_proxy=" + proxyURL,
|
||||
"NO_PROXY=",
|
||||
"no_proxy=",
|
||||
}, "\n") + "\n"
|
||||
if string(raw) != want {
|
||||
t.Fatalf("proxy env = %q, want %q", string(raw), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigureExplicitProxySupportsSocksSchemes(t *testing.T) {
|
||||
for _, raw := range []string{
|
||||
"socks5://127.0.0.1:1080",
|
||||
"socks5h://proxy-user:proxy-pass@127.0.0.1:1080",
|
||||
} {
|
||||
t.Run(raw, func(t *testing.T) {
|
||||
transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
|
||||
if err := configureExplicitProxy(transport, raw); err != nil {
|
||||
t.Fatalf("configureExplicitProxy: %v", err)
|
||||
}
|
||||
if transport.Proxy != nil {
|
||||
t.Fatalf("Transport.Proxy should be nil for SOCKS proxy")
|
||||
}
|
||||
if transport.DialContext == nil {
|
||||
t.Fatalf("Transport.DialContext should be set for SOCKS proxy")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
|
||||
if err := configureExplicitProxy(transport, "http://127.0.0.1:7890"); err != nil {
|
||||
t.Fatalf("configureExplicitProxy http: %v", err)
|
||||
}
|
||||
if transport.Proxy == nil {
|
||||
t.Fatalf("Transport.Proxy should be set for HTTP proxy")
|
||||
}
|
||||
if transport.DialContext != nil {
|
||||
t.Fatalf("Transport.DialContext should not be set for HTTP proxy")
|
||||
}
|
||||
|
||||
if err := configureExplicitProxy(&http.Transport{}, "ftp://127.0.0.1:21"); err == nil {
|
||||
t.Fatalf("expected unsupported proxy scheme error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSocksTargetIPPrefersIPv4(t *testing.T) {
|
||||
got := selectSocksTargetIP([]net.IPAddr{
|
||||
{IP: net.ParseIP("2606:4700:20::681a:229")},
|
||||
{IP: net.ParseIP("104.26.3.41")},
|
||||
})
|
||||
if got == nil || got.String() != "104.26.3.41" {
|
||||
t.Fatalf("selectSocksTargetIP = %v, want IPv4 104.26.3.41", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCrawlerThumbDownloadFailureMarksStatusFailed 验证:网站封面下载失败时
|
||||
// crawler 把 thumbnail_status 显式标 'failed',避免后续封面补队列一直重复
|
||||
// 捞到这条 spider91 视频。
|
||||
@@ -658,3 +776,12 @@ func buildFakeSpiderScript(entries []map[string]string) string {
|
||||
sb.WriteString("fi\n")
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func containsString(values []string, want string) bool {
|
||||
for _, value := range values {
|
||||
if value == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -237,13 +237,33 @@ func appendUniqueStart(starts []float64, start, eachSec float64) []float64 {
|
||||
}
|
||||
|
||||
// thumbnailOffsets 选封面抽帧的时间点(秒)。独立于 teaser。
|
||||
func thumbnailOffsets() []float64 {
|
||||
return []float64{5, 1, 0}
|
||||
// 默认取视频中间帧;时长未知时退回早期帧。
|
||||
func thumbnailOffsets(duration float64) []float64 {
|
||||
if duration <= 0 {
|
||||
return []float64{5, 1, 0}
|
||||
}
|
||||
mid := duration / 2
|
||||
out := []float64{mid}
|
||||
for _, fallback := range []float64{5, 1, 0} {
|
||||
if !containsOffset(out, fallback) {
|
||||
out = append(out, fallback)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func containsOffset(offsets []float64, target float64) bool {
|
||||
for _, offset := range offsets {
|
||||
if math.Abs(offset-target) < 0.01 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// --- 封面 ---
|
||||
|
||||
// GenerateThumbnail 抽一张 jpg 封面。默认从第 5 秒抽帧,失败时回退到更早时间点。
|
||||
// GenerateThumbnail 抽一张 jpg 封面。默认从视频中间抽帧,失败时回退到更早时间点。
|
||||
func (g *Generator) GenerateThumbnail(ctx context.Context, link *drives.StreamLink, videoID string, duration float64) (string, error) {
|
||||
dir := filepath.Join(g.cfg.LocalDir, "thumbs")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
@@ -252,7 +272,7 @@ func (g *Generator) GenerateThumbnail(ctx context.Context, link *drives.StreamLi
|
||||
dst := filepath.Join(dir, videoID+".jpg")
|
||||
|
||||
var lastErr error
|
||||
offsets := thumbnailOffsets()
|
||||
offsets := thumbnailOffsets(duration)
|
||||
for i, offset := range offsets {
|
||||
if i > 0 {
|
||||
_ = os.Remove(dst)
|
||||
@@ -1507,6 +1527,29 @@ func driveErrorShouldCooldown(d drives.Drive, err error) bool {
|
||||
strings.Contains(text, "moov atom not found") ||
|
||||
strings.Contains(text, "partial file") ||
|
||||
strings.Contains(text, "service unavailable")
|
||||
case "p123":
|
||||
// 123 云盘直链解析 / ffmpeg 读取阶段可能返回 429、5xx,或 WAF 类
|
||||
// blocked / 访问阻断文本。命中时冷却,避免封面和预览视频生成连续打接口。
|
||||
text := strings.ToLower(err.Error())
|
||||
return strings.Contains(text, "请求太频繁") ||
|
||||
strings.Contains(text, "请求过于频繁") ||
|
||||
strings.Contains(text, "请求频繁") ||
|
||||
strings.Contains(text, "操作频繁") ||
|
||||
strings.Contains(text, "频率限制") ||
|
||||
strings.Contains(text, "请求次数过多") ||
|
||||
strings.Contains(text, "429") ||
|
||||
strings.Contains(text, "http 500") ||
|
||||
strings.Contains(text, "http 502") ||
|
||||
strings.Contains(text, "http 503") ||
|
||||
strings.Contains(text, "http 504") ||
|
||||
strings.Contains(text, "server returned 403") ||
|
||||
strings.Contains(text, "403 forbidden") ||
|
||||
strings.Contains(text, "too many request") ||
|
||||
strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "blocked") ||
|
||||
strings.Contains(text, "访问被阻断") ||
|
||||
strings.Contains(text, "service unavailable")
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1553,6 +1596,11 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) bool {
|
||||
return false
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "pending"})
|
||||
if isSpider91OriginVideo(v) {
|
||||
log.Printf("[thumb] skip %s: spider91-origin video must use crawled thumbnail", v.Title)
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{ThumbnailStatus: "failed"})
|
||||
return false
|
||||
}
|
||||
link, err := w.streamLink(ctx, v)
|
||||
if err != nil {
|
||||
if w.pauseForRecoverableError(ctx, v, err, "streamURL") {
|
||||
@@ -1618,7 +1666,7 @@ func (w *ThumbWorker) probeDuration(ctx context.Context, v *catalog.Video, link
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) generateThumbnailFromLink(ctx context.Context, v *catalog.Video, link *drives.StreamLink) error {
|
||||
if _, err := w.Gen.GenerateThumbnail(ctx, link, v.ID, 0); err != nil {
|
||||
if _, err := w.Gen.GenerateThumbnail(ctx, link, v.ID, float64(v.DurationSeconds)); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
@@ -1629,6 +1677,10 @@ func (w *ThumbWorker) generateThumbnailFromLink(ctx context.Context, v *catalog.
|
||||
return nil
|
||||
}
|
||||
|
||||
func isSpider91OriginVideo(v *catalog.Video) bool {
|
||||
return v != nil && strings.HasPrefix(v.ID, "spider91-")
|
||||
}
|
||||
|
||||
func localPreviewLink(v *catalog.Video) (*drives.StreamLink, bool) {
|
||||
if v.PreviewLocal == "" {
|
||||
return nil, false
|
||||
|
||||
@@ -168,16 +168,29 @@ func TestMediumAndLongVideosStillRequirePlannedTeaserSegments(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailOffsetsUseFiveSecondsWithEarlyFallbacks(t *testing.T) {
|
||||
got := thumbnailOffsets()
|
||||
want := []float64{5, 1, 0}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("offsets = %#v, want %#v", got, want)
|
||||
func TestThumbnailOffsetsPreferMiddleFrame(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
duration float64
|
||||
want []float64
|
||||
}{
|
||||
{name: "unknown duration", duration: 0, want: []float64{5, 1, 0}},
|
||||
{name: "long video", duration: 2804.9, want: []float64{1402.45, 5, 1, 0}},
|
||||
{name: "short video", duration: 8.9, want: []float64{4.45, 5, 1, 0}},
|
||||
{name: "middle equals fallback", duration: 10, want: []float64{5, 1, 0}},
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("offset[%d] = %.2f, want %.2f", i, got[i], want[i])
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := thumbnailOffsets(tt.duration)
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("offsets = %#v, want %#v", got, tt.want)
|
||||
}
|
||||
for i := range tt.want {
|
||||
if math.Abs(got[i]-tt.want[i]) > 0.001 {
|
||||
t.Fatalf("offset[%d] = %.2f, want %.2f", i, got[i], tt.want[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -39,8 +39,8 @@ func TestThumbWorkerUpdatesThumbnailAndDurationWithoutChangingPreviewStatus(t *t
|
||||
if gen.thumbnailVideoID != video.ID {
|
||||
t.Fatalf("thumbnail video id = %q, want %q", gen.thumbnailVideoID, video.ID)
|
||||
}
|
||||
if gen.thumbnailDuration != 0 {
|
||||
t.Fatalf("thumbnail duration = %.1f, want fixed-offset thumbnail generation", gen.thumbnailDuration)
|
||||
if gen.thumbnailDuration != 42 {
|
||||
t.Fatalf("thumbnail duration = %.1f, want probed duration", gen.thumbnailDuration)
|
||||
}
|
||||
if gen.probeCalls != 1 {
|
||||
t.Fatalf("probe calls = %d, want 1 for thumbnail generation", gen.probeCalls)
|
||||
@@ -89,6 +89,35 @@ func TestThumbWorkerBackfillsDurationWhenThumbnailAlreadyExists(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerDoesNotGenerateThumbnailForSpider91OriginVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "spider91-91-spider-1200001")
|
||||
|
||||
gen := &fakeThumbGenerator{probeDuration: 42}
|
||||
drv := &previewFakeDrive{kind: "pikpak"}
|
||||
worker := NewThumbWorker(gen, cat, drv)
|
||||
|
||||
worker.process(ctx, video)
|
||||
|
||||
got, err := cat.GetVideo(ctx, video.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want empty when crawled spider91 thumbnail is missing", got.ThumbnailURL)
|
||||
}
|
||||
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list failed thumbnails: %v", err)
|
||||
}
|
||||
if len(failed) != 1 || failed[0].ID != video.ID {
|
||||
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
|
||||
}
|
||||
if gen.probeCalls != 0 || gen.generateCalls != 0 {
|
||||
t.Fatalf("generator calls probe=%d generate=%d, want no ffmpeg work for spider91-origin thumbnail", gen.probeCalls, gen.generateCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbWorkerSkipsDurationBackfillWhenExistingThumbnailCannotBeProbed(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, video := seedPreviewTestVideo(t, "thumb-worker-existing-thumbnail-probe-fails")
|
||||
@@ -587,6 +616,22 @@ func TestPreviewWorkerP115TransientErrorKeepsVideoPending(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestP123TransientErrorsShouldCooldown(t *testing.T) {
|
||||
drv := &previewFakeDrive{kind: "p123"}
|
||||
for _, err := range []error{
|
||||
errors.New("Server returned 403 Forbidden"),
|
||||
errors.New("请求太频繁"),
|
||||
errors.New("http 503 service unavailable"),
|
||||
} {
|
||||
if !driveErrorShouldCooldown(drv, err) {
|
||||
t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err)
|
||||
}
|
||||
}
|
||||
if driveErrorShouldCooldown(drv, errors.New("invalid credential")) {
|
||||
t.Fatal("invalid credential should not trigger p123 cooldown")
|
||||
}
|
||||
}
|
||||
|
||||
func assertCooldownAround(t *testing.T, until time.Time, before time.Time, want time.Duration) {
|
||||
t.Helper()
|
||||
if until.IsZero() {
|
||||
|
||||
@@ -147,13 +147,15 @@ func (p *Proxy) ServeStream(w http.ResponseWriter, r *http.Request, driveID, fil
|
||||
// CDN 不校验请求头,直连可获得最佳带宽并避免占用 backend 出站
|
||||
// - onedrive:Microsoft Graph 返回的 @microsoft.graph.downloadUrl 是短期
|
||||
// 免鉴权下载 URL,不需要后端继续代传视频字节
|
||||
// - p123:123 云盘 download_info 返回的下载页会再跳 CDN;driver 已在后端
|
||||
// 先解出最终 Location,浏览器可直接 302 到该短期地址
|
||||
//
|
||||
// 其余网盘(如沃盘 / 夸克等)仍走反代,因为它们的下载
|
||||
// 链接通常需要随请求带上后端持有的 Cookie / Authorization / Range
|
||||
// 的特殊处理,浏览器拿不到这些上下文。
|
||||
func shouldRedirect(d drives.Drive) bool {
|
||||
switch d.Kind() {
|
||||
case "p115", "pikpak", "onedrive":
|
||||
case "p115", "pikpak", "onedrive", "p123":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
|
||||
@@ -176,6 +176,31 @@ func TestServeStreamRedirectsOneDrive(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamRedirectsP123(t *testing.T) {
|
||||
reg := NewRegistry()
|
||||
drv := &proxyFakeSimpleDrive{
|
||||
kind: "p123",
|
||||
url: "https://cdn.123pan.example/video.mp4",
|
||||
}
|
||||
reg.Set("p123", drv)
|
||||
|
||||
p := New(reg)
|
||||
req := httptest.NewRequest(http.MethodGet, "/p/stream/p123/file-1", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
p.ServeStream(rr, req, "p123", "file-1")
|
||||
|
||||
if rr.Code != http.StatusFound {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusFound)
|
||||
}
|
||||
if got := rr.Header().Get("Location"); got != "https://cdn.123pan.example/video.mp4" {
|
||||
t.Fatalf("Location = %q", got)
|
||||
}
|
||||
if drv.calls != 1 {
|
||||
t.Fatalf("link calls = %d, want 1", drv.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeStreamServesLocalFilePath(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "video.mp4")
|
||||
if err := os.WriteFile(path, []byte("0123456789"), 0o644); err != nil {
|
||||
|
||||
@@ -181,14 +181,10 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
if existing.Category == "" && dirName != "" {
|
||||
patch.Category = dirName
|
||||
}
|
||||
if existing.ThumbnailURL == "" && e.ThumbnailURL != "" {
|
||||
patch.ThumbnailURL = e.ThumbnailURL
|
||||
}
|
||||
if patch.Category != "" || patch.ThumbnailURL != "" || patch.ContentHash != "" || patch.FileName != "" {
|
||||
if patch.Category != "" || patch.ContentHash != "" || patch.FileName != "" {
|
||||
_ = s.Catalog.UpdateVideoMeta(ctx, id, patch)
|
||||
}
|
||||
if dup := s.findDuplicate(ctx, e.Hash, e.Name, e.Size, id); dup != nil {
|
||||
s.backfillDuplicateThumbnail(ctx, dup, e.ThumbnailURL)
|
||||
continue
|
||||
}
|
||||
if !sameTags(existing.Tags, tags) {
|
||||
@@ -198,7 +194,6 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
}
|
||||
|
||||
if dup := s.findDuplicate(ctx, e.Hash, e.Name, e.Size, id); dup != nil {
|
||||
s.backfillDuplicateThumbnail(ctx, dup, e.ThumbnailURL)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -216,7 +211,6 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, stats *Stats,
|
||||
Ext: strings.TrimPrefix(ext, "."),
|
||||
Quality: "HD",
|
||||
Size: e.Size,
|
||||
ThumbnailURL: e.ThumbnailURL,
|
||||
PreviewStatus: "pending",
|
||||
Category: dirName,
|
||||
PublishedAt: now,
|
||||
@@ -268,13 +262,6 @@ func (s *Scanner) findDuplicateByFileSignature(ctx context.Context, fileName str
|
||||
return dup
|
||||
}
|
||||
|
||||
func (s *Scanner) backfillDuplicateThumbnail(ctx context.Context, canonical *catalog.Video, thumbnailURL string) {
|
||||
if canonical.ThumbnailURL != "" || thumbnailURL == "" {
|
||||
return
|
||||
}
|
||||
_ = s.Catalog.UpdateVideoMeta(ctx, canonical.ID, catalog.VideoMetaPatch{ThumbnailURL: thumbnailURL})
|
||||
}
|
||||
|
||||
func sameTags(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestRunPersistsRemoteThumbnailFromDriveEntry(t *testing.T) {
|
||||
func TestRunIgnoresRemoteThumbnailFromDriveEntry(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -50,8 +50,8 @@ func TestRunPersistsRemoteThumbnailFromDriveEntry(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "https://thumbnail.example/clip.jpg" {
|
||||
t.Fatalf("thumbnail = %q, want remote thumbnail", got.ThumbnailURL)
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want empty so local thumbnail worker regenerates it", got.ThumbnailURL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,7 +90,7 @@ func TestRunIgnoresZeroSizeVideoFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBackfillsRemoteThumbnailForExistingVideo(t *testing.T) {
|
||||
func TestRunDoesNotBackfillRemoteThumbnailForExistingVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
||||
if err != nil {
|
||||
@@ -140,8 +140,8 @@ func TestRunBackfillsRemoteThumbnailForExistingVideo(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.ThumbnailURL != "https://thumbnail.example/backfilled.jpg" {
|
||||
t.Fatalf("thumbnail = %q, want backfilled remote thumbnail", got.ThumbnailURL)
|
||||
if got.ThumbnailURL != "" {
|
||||
t.Fatalf("thumbnail = %q, want empty so local thumbnail worker regenerates it", got.ThumbnailURL)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
// - 改写 catalog 行:drive_id / file_id / content_hash 改成目标盘的;
|
||||
// 视频自身的 id 不变(仍是 spider91-<driveID>-<viewkey>),video_tags、
|
||||
// 收藏、点赞、views 等关联数据全部保留
|
||||
// - 删除本地 mp4(spider91/<id>/videos/<viewkey>.<ext>)和 thumb(spider91/<id>/thumbs/<viewkey>.jpg)
|
||||
// - 删除本地 mp4(spider91/<id>/videos/<viewkey>.<ext>)和源 thumb
|
||||
// (spider91/<id>/thumbs/<viewkey>.jpg);公共 /p/thumb/<videoID> 副本会保留
|
||||
//
|
||||
// 之后回放时,videoSource() 自动落到 /p/stream/<target>/<file_id>,
|
||||
// proxy 层走对应盘的直链 / 302 直连。
|
||||
@@ -175,6 +176,7 @@ type Config struct {
|
||||
// 4002 / 9)后整体进入冷却的时长。冷却期间 runOnce 直接返回,不再发起任何
|
||||
// PikPak API 请求,避免被进一步风控。0 时默认 5 分钟;< 0 关闭冷却(仅用于测试)。
|
||||
CaptchaCooldown time.Duration
|
||||
CommonThumbDir string
|
||||
OnMigrated func(videoID string)
|
||||
}
|
||||
|
||||
@@ -571,18 +573,98 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
|
||||
if err := m.cfg.Catalog.MigrateVideoToDrive(ctx, v.ID, targetDriveID, res.FileID, res.Hash); err != nil {
|
||||
return false, fmt.Errorf("catalog migrate: %w", err)
|
||||
}
|
||||
m.preserveCrawledThumbnail(ctx, src, v)
|
||||
// 同步 catalog 里的 file_name,让下次目标盘扫盘时 (file_name, size) 也能匹配上
|
||||
if err := m.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{FileName: uploadName}); err != nil {
|
||||
log.Printf("[spider91migrate] %s update file_name after migrate: %v", v.ID, err)
|
||||
}
|
||||
|
||||
// 删除本地 mp4 和 thumb(thumb 在 previews/thumbs/ 还有副本,不影响展示)
|
||||
// 删除本地 mp4 和源 thumb(公共 /p/thumb 副本已在 preserveCrawledThumbnail 中保留)。
|
||||
CleanupSpider91Local(src, v.FileID)
|
||||
|
||||
log.Printf("[spider91migrate] %s migrated to drive=%s(kind=%s) file=%s name=%q", v.ID, targetDriveID, pp.Kind(), res.FileID, uploadName)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *Migrator) preserveCrawledThumbnail(ctx context.Context, src *spider91.Driver, v *catalog.Video) {
|
||||
if m == nil || m.cfg.Catalog == nil || src == nil || v == nil || v.ID == "" || v.FileID == "" {
|
||||
return
|
||||
}
|
||||
commonDir := strings.TrimSpace(m.cfg.CommonThumbDir)
|
||||
if commonDir == "" {
|
||||
return
|
||||
}
|
||||
thumbPath, ok := findSpider91ThumbPath(src, v.FileID)
|
||||
if !ok {
|
||||
if v.ThumbnailURL == "" {
|
||||
log.Printf("[spider91migrate] %s crawled thumbnail missing before migration cleanup", v.ID)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(commonDir, 0o755); err != nil {
|
||||
log.Printf("[spider91migrate] %s mkdir common thumbs: %v", v.ID, err)
|
||||
return
|
||||
}
|
||||
dst := filepath.Join(commonDir, v.ID+".jpg")
|
||||
if _, err := os.Stat(dst); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
log.Printf("[spider91migrate] %s stat common thumb: %v", v.ID, err)
|
||||
return
|
||||
}
|
||||
if err := copyFileAtomic(thumbPath, dst); err != nil {
|
||||
log.Printf("[spider91migrate] %s preserve crawled thumbnail: %v", v.ID, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := m.cfg.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
ThumbnailURL: "/p/thumb/" + v.ID,
|
||||
}); err != nil {
|
||||
log.Printf("[spider91migrate] %s update crawled thumbnail url: %v", v.ID, err)
|
||||
return
|
||||
}
|
||||
v.ThumbnailURL = "/p/thumb/" + v.ID
|
||||
}
|
||||
|
||||
func findSpider91ThumbPath(src *spider91.Driver, fileID string) (string, bool) {
|
||||
thumbBase := stripExt(fileID)
|
||||
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
|
||||
thumbPath, err := src.ThumbPath(thumbBase + ext)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
info, statErr := os.Stat(thumbPath)
|
||||
if statErr == nil && info.Mode().IsRegular() && info.Size() > 0 {
|
||||
return thumbPath, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func copyFileAtomic(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
|
||||
tmp := dst + ".part"
|
||||
out, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, copyErr := io.Copy(out, in)
|
||||
closeErr := out.Close()
|
||||
if copyErr != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return copyErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return closeErr
|
||||
}
|
||||
return os.Rename(tmp, dst)
|
||||
}
|
||||
|
||||
// CleanupSpider91Local 删除已迁移视频的本地 mp4 和 thumb。
|
||||
//
|
||||
// thumb 删除是 best-effort —— 找不到就算了(spider91 thumb 文件名带后缀,
|
||||
|
||||
@@ -339,12 +339,14 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
|
||||
|
||||
now := time.Now()
|
||||
id := writeSpider91Video(t, cat, src, "vk001", ".mp4", []byte("video bytes here"), now)
|
||||
commonThumbDir := t.TempDir()
|
||||
|
||||
m := New(Config{
|
||||
Catalog: cat,
|
||||
Registry: reg,
|
||||
GetTargetDriveID: func() string { return pp.ID() },
|
||||
KeepLatestN: -1, // 关闭"保留最新 N 个",让 1 条也能立即上传
|
||||
CommonThumbDir: commonThumbDir,
|
||||
})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
@@ -382,8 +384,15 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
|
||||
if got.ContentHash == "" {
|
||||
t.Fatalf("content_hash should be set after migration")
|
||||
}
|
||||
if got.ThumbnailURL != "/p/thumb/"+id {
|
||||
t.Fatalf("thumbnail_url = %q, want preserved crawled thumbnail URL", got.ThumbnailURL)
|
||||
}
|
||||
commonThumbPath := filepath.Join(commonThumbDir, id+".jpg")
|
||||
if data, err := os.ReadFile(commonThumbPath); err != nil || string(data) != "thumb" {
|
||||
t.Fatalf("common thumb = %q, %v; want copied crawled thumb", string(data), err)
|
||||
}
|
||||
|
||||
// 3) 本地视频和 thumb 都被删了
|
||||
// 3) 本地视频和源 thumb 都被删了;公共 /p/thumb 副本保留。
|
||||
videoPath, _ := src.VideoPath("vk001.mp4")
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local mp4 still exists or stat error %v", err)
|
||||
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
// Copyright 2018 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package socks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
noDeadline = time.Time{}
|
||||
aLongTimeAgo = time.Unix(1, 0)
|
||||
)
|
||||
|
||||
func (d *Dialer) connect(ctx context.Context, c net.Conn, address string) (_ net.Addr, ctxErr error) {
|
||||
host, port, err := splitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if deadline, ok := ctx.Deadline(); ok && !deadline.IsZero() {
|
||||
c.SetDeadline(deadline)
|
||||
defer c.SetDeadline(noDeadline)
|
||||
}
|
||||
if ctx != context.Background() {
|
||||
errCh := make(chan error, 1)
|
||||
done := make(chan struct{})
|
||||
defer func() {
|
||||
close(done)
|
||||
if ctxErr == nil {
|
||||
ctxErr = <-errCh
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
c.SetDeadline(aLongTimeAgo)
|
||||
errCh <- ctx.Err()
|
||||
case <-done:
|
||||
errCh <- nil
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
b := make([]byte, 0, 6+len(host)) // the size here is just an estimate
|
||||
b = append(b, Version5)
|
||||
if len(d.AuthMethods) == 0 || d.Authenticate == nil {
|
||||
b = append(b, 1, byte(AuthMethodNotRequired))
|
||||
} else {
|
||||
ams := d.AuthMethods
|
||||
if len(ams) > 255 {
|
||||
return nil, errors.New("too many authentication methods")
|
||||
}
|
||||
b = append(b, byte(len(ams)))
|
||||
for _, am := range ams {
|
||||
b = append(b, byte(am))
|
||||
}
|
||||
}
|
||||
if _, ctxErr = c.Write(b); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if _, ctxErr = io.ReadFull(c, b[:2]); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
if b[0] != Version5 {
|
||||
return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0])))
|
||||
}
|
||||
am := AuthMethod(b[1])
|
||||
if am == AuthMethodNoAcceptableMethods {
|
||||
return nil, errors.New("no acceptable authentication methods")
|
||||
}
|
||||
if d.Authenticate != nil {
|
||||
if ctxErr = d.Authenticate(ctx, c, am); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
b = b[:0]
|
||||
b = append(b, Version5, byte(d.cmd), 0)
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
b = append(b, AddrTypeIPv4)
|
||||
b = append(b, ip4...)
|
||||
} else if ip6 := ip.To16(); ip6 != nil {
|
||||
b = append(b, AddrTypeIPv6)
|
||||
b = append(b, ip6...)
|
||||
} else {
|
||||
return nil, errors.New("unknown address type")
|
||||
}
|
||||
} else {
|
||||
if len(host) > 255 {
|
||||
return nil, errors.New("FQDN too long")
|
||||
}
|
||||
b = append(b, AddrTypeFQDN)
|
||||
b = append(b, byte(len(host)))
|
||||
b = append(b, host...)
|
||||
}
|
||||
b = append(b, byte(port>>8), byte(port))
|
||||
if _, ctxErr = c.Write(b); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if _, ctxErr = io.ReadFull(c, b[:4]); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
if b[0] != Version5 {
|
||||
return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0])))
|
||||
}
|
||||
if cmdErr := Reply(b[1]); cmdErr != StatusSucceeded {
|
||||
return nil, errors.New("unknown error " + cmdErr.String())
|
||||
}
|
||||
if b[2] != 0 {
|
||||
return nil, errors.New("non-zero reserved field")
|
||||
}
|
||||
l := 2
|
||||
var a Addr
|
||||
switch b[3] {
|
||||
case AddrTypeIPv4:
|
||||
l += net.IPv4len
|
||||
a.IP = make(net.IP, net.IPv4len)
|
||||
case AddrTypeIPv6:
|
||||
l += net.IPv6len
|
||||
a.IP = make(net.IP, net.IPv6len)
|
||||
case AddrTypeFQDN:
|
||||
if _, err := io.ReadFull(c, b[:1]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l += int(b[0])
|
||||
default:
|
||||
return nil, errors.New("unknown address type " + strconv.Itoa(int(b[3])))
|
||||
}
|
||||
if cap(b) < l {
|
||||
b = make([]byte, l)
|
||||
} else {
|
||||
b = b[:l]
|
||||
}
|
||||
if _, ctxErr = io.ReadFull(c, b); ctxErr != nil {
|
||||
return
|
||||
}
|
||||
if a.IP != nil {
|
||||
copy(a.IP, b)
|
||||
} else {
|
||||
a.Name = string(b[:len(b)-2])
|
||||
}
|
||||
a.Port = int(b[len(b)-2])<<8 | int(b[len(b)-1])
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
func splitHostPort(address string) (string, int, error) {
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
portnum, err := strconv.Atoi(port)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if 1 > portnum || portnum > 0xffff {
|
||||
return "", 0, errors.New("port number out of range " + port)
|
||||
}
|
||||
return host, portnum, nil
|
||||
}
|
||||
+317
@@ -0,0 +1,317 @@
|
||||
// Copyright 2018 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package socks provides a SOCKS version 5 client implementation.
|
||||
//
|
||||
// SOCKS protocol version 5 is defined in RFC 1928.
|
||||
// Username/Password authentication for SOCKS version 5 is defined in
|
||||
// RFC 1929.
|
||||
package socks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// A Command represents a SOCKS command.
|
||||
type Command int
|
||||
|
||||
func (cmd Command) String() string {
|
||||
switch cmd {
|
||||
case CmdConnect:
|
||||
return "socks connect"
|
||||
case cmdBind:
|
||||
return "socks bind"
|
||||
default:
|
||||
return "socks " + strconv.Itoa(int(cmd))
|
||||
}
|
||||
}
|
||||
|
||||
// An AuthMethod represents a SOCKS authentication method.
|
||||
type AuthMethod int
|
||||
|
||||
// A Reply represents a SOCKS command reply code.
|
||||
type Reply int
|
||||
|
||||
func (code Reply) String() string {
|
||||
switch code {
|
||||
case StatusSucceeded:
|
||||
return "succeeded"
|
||||
case 0x01:
|
||||
return "general SOCKS server failure"
|
||||
case 0x02:
|
||||
return "connection not allowed by ruleset"
|
||||
case 0x03:
|
||||
return "network unreachable"
|
||||
case 0x04:
|
||||
return "host unreachable"
|
||||
case 0x05:
|
||||
return "connection refused"
|
||||
case 0x06:
|
||||
return "TTL expired"
|
||||
case 0x07:
|
||||
return "command not supported"
|
||||
case 0x08:
|
||||
return "address type not supported"
|
||||
default:
|
||||
return "unknown code: " + strconv.Itoa(int(code))
|
||||
}
|
||||
}
|
||||
|
||||
// Wire protocol constants.
|
||||
const (
|
||||
Version5 = 0x05
|
||||
|
||||
AddrTypeIPv4 = 0x01
|
||||
AddrTypeFQDN = 0x03
|
||||
AddrTypeIPv6 = 0x04
|
||||
|
||||
CmdConnect Command = 0x01 // establishes an active-open forward proxy connection
|
||||
cmdBind Command = 0x02 // establishes a passive-open forward proxy connection
|
||||
|
||||
AuthMethodNotRequired AuthMethod = 0x00 // no authentication required
|
||||
AuthMethodUsernamePassword AuthMethod = 0x02 // use username/password
|
||||
AuthMethodNoAcceptableMethods AuthMethod = 0xff // no acceptable authentication methods
|
||||
|
||||
StatusSucceeded Reply = 0x00
|
||||
)
|
||||
|
||||
// An Addr represents a SOCKS-specific address.
|
||||
// Either Name or IP is used exclusively.
|
||||
type Addr struct {
|
||||
Name string // fully-qualified domain name
|
||||
IP net.IP
|
||||
Port int
|
||||
}
|
||||
|
||||
func (a *Addr) Network() string { return "socks" }
|
||||
|
||||
func (a *Addr) String() string {
|
||||
if a == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
port := strconv.Itoa(a.Port)
|
||||
if a.IP == nil {
|
||||
return net.JoinHostPort(a.Name, port)
|
||||
}
|
||||
return net.JoinHostPort(a.IP.String(), port)
|
||||
}
|
||||
|
||||
// A Conn represents a forward proxy connection.
|
||||
type Conn struct {
|
||||
net.Conn
|
||||
|
||||
boundAddr net.Addr
|
||||
}
|
||||
|
||||
// BoundAddr returns the address assigned by the proxy server for
|
||||
// connecting to the command target address from the proxy server.
|
||||
func (c *Conn) BoundAddr() net.Addr {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
return c.boundAddr
|
||||
}
|
||||
|
||||
// A Dialer holds SOCKS-specific options.
|
||||
type Dialer struct {
|
||||
cmd Command // either CmdConnect or cmdBind
|
||||
proxyNetwork string // network between a proxy server and a client
|
||||
proxyAddress string // proxy server address
|
||||
|
||||
// ProxyDial specifies the optional dial function for
|
||||
// establishing the transport connection.
|
||||
ProxyDial func(context.Context, string, string) (net.Conn, error)
|
||||
|
||||
// AuthMethods specifies the list of request authentication
|
||||
// methods.
|
||||
// If empty, SOCKS client requests only AuthMethodNotRequired.
|
||||
AuthMethods []AuthMethod
|
||||
|
||||
// Authenticate specifies the optional authentication
|
||||
// function. It must be non-nil when AuthMethods is not empty.
|
||||
// It must return an error when the authentication is failed.
|
||||
Authenticate func(context.Context, io.ReadWriter, AuthMethod) error
|
||||
}
|
||||
|
||||
// DialContext connects to the provided address on the provided
|
||||
// network.
|
||||
//
|
||||
// The returned error value may be a net.OpError. When the Op field of
|
||||
// net.OpError contains "socks", the Source field contains a proxy
|
||||
// server address and the Addr field contains a command target
|
||||
// address.
|
||||
//
|
||||
// See func Dial of the net package of standard library for a
|
||||
// description of the network and address parameters.
|
||||
func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
if err := d.validateTarget(network, address); err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
if ctx == nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")}
|
||||
}
|
||||
var err error
|
||||
var c net.Conn
|
||||
if d.ProxyDial != nil {
|
||||
c, err = d.ProxyDial(ctx, d.proxyNetwork, d.proxyAddress)
|
||||
} else {
|
||||
var dd net.Dialer
|
||||
c, err = dd.DialContext(ctx, d.proxyNetwork, d.proxyAddress)
|
||||
}
|
||||
if err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
a, err := d.connect(ctx, c, address)
|
||||
if err != nil {
|
||||
c.Close()
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
return &Conn{Conn: c, boundAddr: a}, nil
|
||||
}
|
||||
|
||||
// DialWithConn initiates a connection from SOCKS server to the target
|
||||
// network and address using the connection c that is already
|
||||
// connected to the SOCKS server.
|
||||
//
|
||||
// It returns the connection's local address assigned by the SOCKS
|
||||
// server.
|
||||
func (d *Dialer) DialWithConn(ctx context.Context, c net.Conn, network, address string) (net.Addr, error) {
|
||||
if err := d.validateTarget(network, address); err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
if ctx == nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")}
|
||||
}
|
||||
a, err := d.connect(ctx, c, address)
|
||||
if err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// Dial connects to the provided address on the provided network.
|
||||
//
|
||||
// Unlike DialContext, it returns a raw transport connection instead
|
||||
// of a forward proxy connection.
|
||||
//
|
||||
// Deprecated: Use DialContext or DialWithConn instead.
|
||||
func (d *Dialer) Dial(network, address string) (net.Conn, error) {
|
||||
if err := d.validateTarget(network, address); err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
var err error
|
||||
var c net.Conn
|
||||
if d.ProxyDial != nil {
|
||||
c, err = d.ProxyDial(context.Background(), d.proxyNetwork, d.proxyAddress)
|
||||
} else {
|
||||
c, err = net.Dial(d.proxyNetwork, d.proxyAddress)
|
||||
}
|
||||
if err != nil {
|
||||
proxy, dst, _ := d.pathAddrs(address)
|
||||
return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
|
||||
}
|
||||
if _, err := d.DialWithConn(context.Background(), c, network, address); err != nil {
|
||||
c.Close()
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (d *Dialer) validateTarget(network, address string) error {
|
||||
switch network {
|
||||
case "tcp", "tcp6", "tcp4":
|
||||
default:
|
||||
return errors.New("network not implemented")
|
||||
}
|
||||
switch d.cmd {
|
||||
case CmdConnect, cmdBind:
|
||||
default:
|
||||
return errors.New("command not implemented")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Dialer) pathAddrs(address string) (proxy, dst net.Addr, err error) {
|
||||
for i, s := range []string{d.proxyAddress, address} {
|
||||
host, port, err := splitHostPort(s)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
a := &Addr{Port: port}
|
||||
a.IP = net.ParseIP(host)
|
||||
if a.IP == nil {
|
||||
a.Name = host
|
||||
}
|
||||
if i == 0 {
|
||||
proxy = a
|
||||
} else {
|
||||
dst = a
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// NewDialer returns a new Dialer that dials through the provided
|
||||
// proxy server's network and address.
|
||||
func NewDialer(network, address string) *Dialer {
|
||||
return &Dialer{proxyNetwork: network, proxyAddress: address, cmd: CmdConnect}
|
||||
}
|
||||
|
||||
const (
|
||||
authUsernamePasswordVersion = 0x01
|
||||
authStatusSucceeded = 0x00
|
||||
)
|
||||
|
||||
// UsernamePassword are the credentials for the username/password
|
||||
// authentication method.
|
||||
type UsernamePassword struct {
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
// Authenticate authenticates a pair of username and password with the
|
||||
// proxy server.
|
||||
func (up *UsernamePassword) Authenticate(ctx context.Context, rw io.ReadWriter, auth AuthMethod) error {
|
||||
switch auth {
|
||||
case AuthMethodNotRequired:
|
||||
return nil
|
||||
case AuthMethodUsernamePassword:
|
||||
if len(up.Username) == 0 || len(up.Username) > 255 || len(up.Password) > 255 {
|
||||
return errors.New("invalid username/password")
|
||||
}
|
||||
b := []byte{authUsernamePasswordVersion}
|
||||
b = append(b, byte(len(up.Username)))
|
||||
b = append(b, up.Username...)
|
||||
b = append(b, byte(len(up.Password)))
|
||||
b = append(b, up.Password...)
|
||||
// TODO(mikio): handle IO deadlines and cancelation if
|
||||
// necessary
|
||||
if _, err := rw.Write(b); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.ReadFull(rw, b[:2]); err != nil {
|
||||
return err
|
||||
}
|
||||
if b[0] != authUsernamePasswordVersion {
|
||||
return errors.New("invalid username/password version")
|
||||
}
|
||||
if b[1] != authStatusSucceeded {
|
||||
return errors.New("username/password authentication failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errors.New("unsupported authentication method " + strconv.Itoa(int(auth)))
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
// Copyright 2019 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
)
|
||||
|
||||
// A ContextDialer dials using a context.
|
||||
type ContextDialer interface {
|
||||
DialContext(ctx context.Context, network, address string) (net.Conn, error)
|
||||
}
|
||||
|
||||
// Dial works like DialContext on net.Dialer but using a dialer returned by FromEnvironment.
|
||||
//
|
||||
// The passed ctx is only used for returning the Conn, not the lifetime of the Conn.
|
||||
//
|
||||
// Custom dialers (registered via RegisterDialerType) that do not implement ContextDialer
|
||||
// can leak a goroutine for as long as it takes the underlying Dialer implementation to timeout.
|
||||
//
|
||||
// A Conn returned from a successful Dial after the context has been cancelled will be immediately closed.
|
||||
func Dial(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
d := FromEnvironment()
|
||||
if xd, ok := d.(ContextDialer); ok {
|
||||
return xd.DialContext(ctx, network, address)
|
||||
}
|
||||
return dialContext(ctx, d, network, address)
|
||||
}
|
||||
|
||||
// WARNING: this can leak a goroutine for as long as the underlying Dialer implementation takes to timeout
|
||||
// A Conn returned from a successful Dial after the context has been cancelled will be immediately closed.
|
||||
func dialContext(ctx context.Context, d Dialer, network, address string) (net.Conn, error) {
|
||||
var (
|
||||
conn net.Conn
|
||||
done = make(chan struct{}, 1)
|
||||
err error
|
||||
)
|
||||
go func() {
|
||||
conn, err = d.Dial(network, address)
|
||||
close(done)
|
||||
if conn != nil && ctx.Err() != nil {
|
||||
conn.Close()
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
case <-done:
|
||||
}
|
||||
return conn, err
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
)
|
||||
|
||||
type direct struct{}
|
||||
|
||||
// Direct implements Dialer by making network connections directly using net.Dial or net.DialContext.
|
||||
var Direct = direct{}
|
||||
|
||||
var (
|
||||
_ Dialer = Direct
|
||||
_ ContextDialer = Direct
|
||||
)
|
||||
|
||||
// Dial directly invokes net.Dial with the supplied parameters.
|
||||
func (direct) Dial(network, addr string) (net.Conn, error) {
|
||||
return net.Dial(network, addr)
|
||||
}
|
||||
|
||||
// DialContext instantiates a net.Dialer and invokes its DialContext receiver with the supplied parameters.
|
||||
func (direct) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, network, addr)
|
||||
}
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A PerHost directs connections to a default Dialer unless the host name
|
||||
// requested matches one of a number of exceptions.
|
||||
type PerHost struct {
|
||||
def, bypass Dialer
|
||||
|
||||
bypassNetworks []*net.IPNet
|
||||
bypassIPs []net.IP
|
||||
bypassZones []string
|
||||
bypassHosts []string
|
||||
}
|
||||
|
||||
// NewPerHost returns a PerHost Dialer that directs connections to either
|
||||
// defaultDialer or bypass, depending on whether the connection matches one of
|
||||
// the configured rules.
|
||||
func NewPerHost(defaultDialer, bypass Dialer) *PerHost {
|
||||
return &PerHost{
|
||||
def: defaultDialer,
|
||||
bypass: bypass,
|
||||
}
|
||||
}
|
||||
|
||||
// Dial connects to the address addr on the given network through either
|
||||
// defaultDialer or bypass.
|
||||
func (p *PerHost) Dial(network, addr string) (c net.Conn, err error) {
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return p.dialerForRequest(host).Dial(network, addr)
|
||||
}
|
||||
|
||||
// DialContext connects to the address addr on the given network through either
|
||||
// defaultDialer or bypass.
|
||||
func (p *PerHost) DialContext(ctx context.Context, network, addr string) (c net.Conn, err error) {
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := p.dialerForRequest(host)
|
||||
if x, ok := d.(ContextDialer); ok {
|
||||
return x.DialContext(ctx, network, addr)
|
||||
}
|
||||
return dialContext(ctx, d, network, addr)
|
||||
}
|
||||
|
||||
func (p *PerHost) dialerForRequest(host string) Dialer {
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
for _, net := range p.bypassNetworks {
|
||||
if net.Contains(ip) {
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
for _, bypassIP := range p.bypassIPs {
|
||||
if bypassIP.Equal(ip) {
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
return p.def
|
||||
}
|
||||
|
||||
for _, zone := range p.bypassZones {
|
||||
if strings.HasSuffix(host, zone) {
|
||||
return p.bypass
|
||||
}
|
||||
if host == zone[1:] {
|
||||
// For a zone ".example.com", we match "example.com"
|
||||
// too.
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
for _, bypassHost := range p.bypassHosts {
|
||||
if bypassHost == host {
|
||||
return p.bypass
|
||||
}
|
||||
}
|
||||
return p.def
|
||||
}
|
||||
|
||||
// AddFromString parses a string that contains comma-separated values
|
||||
// specifying hosts that should use the bypass proxy. Each value is either an
|
||||
// IP address, a CIDR range, a zone (*.example.com) or a host name
|
||||
// (localhost). A best effort is made to parse the string and errors are
|
||||
// ignored.
|
||||
func (p *PerHost) AddFromString(s string) {
|
||||
hosts := strings.Split(s, ",")
|
||||
for _, host := range hosts {
|
||||
host = strings.TrimSpace(host)
|
||||
if len(host) == 0 {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(host, "/") {
|
||||
// We assume that it's a CIDR address like 127.0.0.0/8
|
||||
if _, net, err := net.ParseCIDR(host); err == nil {
|
||||
p.AddNetwork(net)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
p.AddIP(ip)
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(host, "*.") {
|
||||
p.AddZone(host[1:])
|
||||
continue
|
||||
}
|
||||
p.AddHost(host)
|
||||
}
|
||||
}
|
||||
|
||||
// AddIP specifies an IP address that will use the bypass proxy. Note that
|
||||
// this will only take effect if a literal IP address is dialed. A connection
|
||||
// to a named host will never match an IP.
|
||||
func (p *PerHost) AddIP(ip net.IP) {
|
||||
p.bypassIPs = append(p.bypassIPs, ip)
|
||||
}
|
||||
|
||||
// AddNetwork specifies an IP range that will use the bypass proxy. Note that
|
||||
// this will only take effect if a literal IP address is dialed. A connection
|
||||
// to a named host will never match.
|
||||
func (p *PerHost) AddNetwork(net *net.IPNet) {
|
||||
p.bypassNetworks = append(p.bypassNetworks, net)
|
||||
}
|
||||
|
||||
// AddZone specifies a DNS suffix that will use the bypass proxy. A zone of
|
||||
// "example.com" matches "example.com" and all of its subdomains.
|
||||
func (p *PerHost) AddZone(zone string) {
|
||||
zone = strings.TrimSuffix(zone, ".")
|
||||
if !strings.HasPrefix(zone, ".") {
|
||||
zone = "." + zone
|
||||
}
|
||||
p.bypassZones = append(p.bypassZones, zone)
|
||||
}
|
||||
|
||||
// AddHost specifies a host name that will use the bypass proxy.
|
||||
func (p *PerHost) AddHost(host string) {
|
||||
host = strings.TrimSuffix(host, ".")
|
||||
p.bypassHosts = append(p.bypassHosts, host)
|
||||
}
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package proxy provides support for a variety of protocols to proxy network
|
||||
// data.
|
||||
package proxy // import "golang.org/x/net/proxy"
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// A Dialer is a means to establish a connection.
|
||||
// Custom dialers should also implement ContextDialer.
|
||||
type Dialer interface {
|
||||
// Dial connects to the given address via the proxy.
|
||||
Dial(network, addr string) (c net.Conn, err error)
|
||||
}
|
||||
|
||||
// Auth contains authentication parameters that specific Dialers may require.
|
||||
type Auth struct {
|
||||
User, Password string
|
||||
}
|
||||
|
||||
// FromEnvironment returns the dialer specified by the proxy-related
|
||||
// variables in the environment and makes underlying connections
|
||||
// directly.
|
||||
func FromEnvironment() Dialer {
|
||||
return FromEnvironmentUsing(Direct)
|
||||
}
|
||||
|
||||
// FromEnvironmentUsing returns the dialer specify by the proxy-related
|
||||
// variables in the environment and makes underlying connections
|
||||
// using the provided forwarding Dialer (for instance, a *net.Dialer
|
||||
// with desired configuration).
|
||||
func FromEnvironmentUsing(forward Dialer) Dialer {
|
||||
allProxy := allProxyEnv.Get()
|
||||
if len(allProxy) == 0 {
|
||||
return forward
|
||||
}
|
||||
|
||||
proxyURL, err := url.Parse(allProxy)
|
||||
if err != nil {
|
||||
return forward
|
||||
}
|
||||
proxy, err := FromURL(proxyURL, forward)
|
||||
if err != nil {
|
||||
return forward
|
||||
}
|
||||
|
||||
noProxy := noProxyEnv.Get()
|
||||
if len(noProxy) == 0 {
|
||||
return proxy
|
||||
}
|
||||
|
||||
perHost := NewPerHost(proxy, forward)
|
||||
perHost.AddFromString(noProxy)
|
||||
return perHost
|
||||
}
|
||||
|
||||
// proxySchemes is a map from URL schemes to a function that creates a Dialer
|
||||
// from a URL with such a scheme.
|
||||
var proxySchemes map[string]func(*url.URL, Dialer) (Dialer, error)
|
||||
|
||||
// RegisterDialerType takes a URL scheme and a function to generate Dialers from
|
||||
// a URL with that scheme and a forwarding Dialer. Registered schemes are used
|
||||
// by FromURL.
|
||||
func RegisterDialerType(scheme string, f func(*url.URL, Dialer) (Dialer, error)) {
|
||||
if proxySchemes == nil {
|
||||
proxySchemes = make(map[string]func(*url.URL, Dialer) (Dialer, error))
|
||||
}
|
||||
proxySchemes[scheme] = f
|
||||
}
|
||||
|
||||
// FromURL returns a Dialer given a URL specification and an underlying
|
||||
// Dialer for it to make network requests.
|
||||
func FromURL(u *url.URL, forward Dialer) (Dialer, error) {
|
||||
var auth *Auth
|
||||
if u.User != nil {
|
||||
auth = new(Auth)
|
||||
auth.User = u.User.Username()
|
||||
if p, ok := u.User.Password(); ok {
|
||||
auth.Password = p
|
||||
}
|
||||
}
|
||||
|
||||
switch u.Scheme {
|
||||
case "socks5", "socks5h":
|
||||
addr := u.Hostname()
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
port = "1080"
|
||||
}
|
||||
return SOCKS5("tcp", net.JoinHostPort(addr, port), auth, forward)
|
||||
}
|
||||
|
||||
// If the scheme doesn't match any of the built-in schemes, see if it
|
||||
// was registered by another package.
|
||||
if proxySchemes != nil {
|
||||
if f, ok := proxySchemes[u.Scheme]; ok {
|
||||
return f(u, forward)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.New("proxy: unknown scheme: " + u.Scheme)
|
||||
}
|
||||
|
||||
var (
|
||||
allProxyEnv = &envOnce{
|
||||
names: []string{"ALL_PROXY", "all_proxy"},
|
||||
}
|
||||
noProxyEnv = &envOnce{
|
||||
names: []string{"NO_PROXY", "no_proxy"},
|
||||
}
|
||||
)
|
||||
|
||||
// envOnce looks up an environment variable (optionally by multiple
|
||||
// names) once. It mitigates expensive lookups on some platforms
|
||||
// (e.g. Windows).
|
||||
// (Borrowed from net/http/transport.go)
|
||||
type envOnce struct {
|
||||
names []string
|
||||
once sync.Once
|
||||
val string
|
||||
}
|
||||
|
||||
func (e *envOnce) Get() string {
|
||||
e.once.Do(e.init)
|
||||
return e.val
|
||||
}
|
||||
|
||||
func (e *envOnce) init() {
|
||||
for _, n := range e.names {
|
||||
e.val = os.Getenv(n)
|
||||
if e.val != "" {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reset is used by tests
|
||||
func (e *envOnce) reset() {
|
||||
e.once = sync.Once{}
|
||||
e.val = ""
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
|
||||
"golang.org/x/net/internal/socks"
|
||||
)
|
||||
|
||||
// SOCKS5 returns a Dialer that makes SOCKSv5 connections to the given
|
||||
// address with an optional username and password.
|
||||
// See RFC 1928 and RFC 1929.
|
||||
func SOCKS5(network, address string, auth *Auth, forward Dialer) (Dialer, error) {
|
||||
d := socks.NewDialer(network, address)
|
||||
if forward != nil {
|
||||
if f, ok := forward.(ContextDialer); ok {
|
||||
d.ProxyDial = func(ctx context.Context, network string, address string) (net.Conn, error) {
|
||||
return f.DialContext(ctx, network, address)
|
||||
}
|
||||
} else {
|
||||
d.ProxyDial = func(ctx context.Context, network string, address string) (net.Conn, error) {
|
||||
return dialContext(ctx, forward, network, address)
|
||||
}
|
||||
}
|
||||
}
|
||||
if auth != nil {
|
||||
up := socks.UsernamePassword{
|
||||
Username: auth.User,
|
||||
Password: auth.Password,
|
||||
}
|
||||
d.AuthMethods = []socks.AuthMethod{
|
||||
socks.AuthMethodNotRequired,
|
||||
socks.AuthMethodUsernamePassword,
|
||||
}
|
||||
d.Authenticate = up.Authenticate
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
Vendored
+2
@@ -67,6 +67,8 @@ github.com/skip2/go-qrcode/reedsolomon
|
||||
golang.org/x/crypto/curve25519
|
||||
# golang.org/x/net v0.27.0
|
||||
## explicit; go 1.18
|
||||
golang.org/x/net/internal/socks
|
||||
golang.org/x/net/proxy
|
||||
golang.org/x/net/publicsuffix
|
||||
# golang.org/x/sys v0.30.0
|
||||
## explicit; go 1.18
|
||||
|
||||
+26
-5
@@ -122,6 +122,27 @@ asset_name() {
|
||||
printf '%s-linux-%s.tar.gz' "$APP_NAME" "$ARCH"
|
||||
}
|
||||
|
||||
verify_runtime_deps() {
|
||||
local cmd
|
||||
for cmd in curl tar ffmpeg ffprobe openssl python3; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || die "missing command: $cmd"
|
||||
done
|
||||
|
||||
python3 - <<'PY' || die "missing Python modules for 91Spider: requests, bs4, lxml, socks"
|
||||
import importlib.util
|
||||
import sys
|
||||
|
||||
missing = [
|
||||
name
|
||||
for name in ("requests", "bs4", "lxml", "socks")
|
||||
if importlib.util.find_spec(name) is None
|
||||
]
|
||||
if missing:
|
||||
print("missing Python modules: " + ", ".join(missing), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
install_deps() {
|
||||
if [[ "$INSTALL_DEPS" != "1" ]]; then
|
||||
return
|
||||
@@ -130,13 +151,12 @@ install_deps() {
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
log "installing runtime dependencies"
|
||||
apt-get update
|
||||
apt-get install -y ca-certificates curl tar ffmpeg openssl iproute2 python3 python3-requests python3-bs4 python3-lxml
|
||||
apt-get install -y ca-certificates curl tar ffmpeg openssl iproute2 python3 python3-requests python3-bs4 python3-lxml python3-socks
|
||||
verify_runtime_deps
|
||||
return
|
||||
fi
|
||||
|
||||
for cmd in curl tar ffmpeg ffprobe openssl; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || die "missing command: $cmd"
|
||||
done
|
||||
verify_runtime_deps
|
||||
}
|
||||
|
||||
check_system() {
|
||||
@@ -689,13 +709,14 @@ update_app() {
|
||||
exec_latest_manager_update
|
||||
fi
|
||||
|
||||
install_deps
|
||||
|
||||
if should_skip_update; then
|
||||
log "already up to date; skipped app update"
|
||||
return 0
|
||||
fi
|
||||
|
||||
check_disk_space
|
||||
install_deps
|
||||
|
||||
local backup
|
||||
backup="$(mktemp -d)"
|
||||
|
||||
Generated
+1
@@ -7,6 +7,7 @@
|
||||
"": {
|
||||
"name": "video-site",
|
||||
"version": "0.1.0",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"lucide-react": "0.453.0",
|
||||
"react": "18.3.1",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
{
|
||||
"name": "video-site",
|
||||
"private": true,
|
||||
"license": "MIT",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
+342
-42
@@ -1,5 +1,6 @@
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import {
|
||||
AlertTriangle,
|
||||
ArrowLeft,
|
||||
ChevronDown,
|
||||
ChevronRight,
|
||||
@@ -10,6 +11,7 @@ import {
|
||||
Plus,
|
||||
Power,
|
||||
PowerOff,
|
||||
QrCode,
|
||||
RefreshCw,
|
||||
RotateCcw,
|
||||
Trash2,
|
||||
@@ -23,6 +25,7 @@ import { makeUniqueDriveId } from "./driveId";
|
||||
const kindLabel: Record<string, string> = {
|
||||
quark: "夸克网盘",
|
||||
p115: "115 网盘",
|
||||
p123: "123 云盘",
|
||||
pikpak: "PikPak",
|
||||
wopan: "联通沃盘",
|
||||
onedrive: "OneDrive",
|
||||
@@ -90,12 +93,14 @@ export function DrivesPage() {
|
||||
useState<api.NightlyJobStatus>(idleNightlyStatus);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [modalOpen, setModalOpen] = useState(false);
|
||||
const [deleteTarget, setDeleteTarget] = useState<api.AdminDrive | null>(null);
|
||||
const [form, setForm] = useState<FormState>(emptyForm);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [deletingId, setDeletingId] = useState("");
|
||||
const [regenFailedId, setRegenFailedId] = useState("");
|
||||
// 与 regenFailedId 并列:失败封面重新入队按钮的 disable 状态。两套独立按钮 →
|
||||
// 两个 state 互不阻塞,避免操作 teaser 时锁住封面那条按钮(反之亦然)。
|
||||
// 失败重试按钮各自维护 pending 状态,避免操作 teaser / 封面 / 指纹时互相锁住。
|
||||
const [regenFailedThumbId, setRegenFailedThumbId] = useState("");
|
||||
const [regenFailedFingerprintId, setRegenFailedFingerprintId] = useState("");
|
||||
// togglingTeaserId 在请求未返回前禁用按钮,避免连点导致两次切换互相覆盖。
|
||||
const [togglingTeaserId, setTogglingTeaserId] = useState("");
|
||||
const [scanningAll, setScanningAll] = useState(false);
|
||||
@@ -169,7 +174,7 @@ export function DrivesPage() {
|
||||
kind: d.kind,
|
||||
name: d.name,
|
||||
rootId: d.rootId,
|
||||
creds: {},
|
||||
creds: d.kind === "spider91" ? { proxy: d.spider91Proxy ?? "" } : {},
|
||||
spider91UploadDriveId: settings?.spider91UploadDriveId ?? "",
|
||||
});
|
||||
setModalOpen(true);
|
||||
@@ -185,7 +190,9 @@ export function DrivesPage() {
|
||||
const driveID = existing
|
||||
? form.id
|
||||
: makeUniqueDriveId(form.kind, name, list);
|
||||
const rootId = form.rootId.trim() || defaultRootId(form.kind);
|
||||
const rootId = usesRootDirectoryID(form.kind)
|
||||
? form.rootId.trim() || defaultRootId(form.kind)
|
||||
: defaultRootId(form.kind);
|
||||
// 若编辑且没有提供凭证,提示一下但仍允许保存(不改凭证)
|
||||
setSaving(true);
|
||||
try {
|
||||
@@ -233,14 +240,22 @@ export function DrivesPage() {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleDelete(d: api.AdminDrive) {
|
||||
if (!window.confirm(`确定删除 ${d.name || d.id}?\n这会移除盘配置,但不会删除其中的视频元数据。`)) return;
|
||||
async function confirmDeleteDrive() {
|
||||
if (!deleteTarget) return;
|
||||
const d = deleteTarget;
|
||||
setDeletingId(d.id);
|
||||
try {
|
||||
await api.deleteDrive(d.id);
|
||||
show("已删除", "success");
|
||||
const resp = await api.deleteDrive(d.id, { deleteVideos: true });
|
||||
show(`已删除,并清理 ${resp.deletedVideos ?? 0} 个视频`, "success");
|
||||
setDeleteTarget(null);
|
||||
if (selectedDriveId === d.id) {
|
||||
setSelectedDriveId(null);
|
||||
}
|
||||
refresh();
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "删除失败", "error");
|
||||
} finally {
|
||||
setDeletingId("");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -310,6 +325,19 @@ export function DrivesPage() {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleRegenFailedFingerprints(d: api.AdminDrive) {
|
||||
setRegenFailedFingerprintId(d.id);
|
||||
try {
|
||||
await api.regenFailedFingerprints(d.id);
|
||||
show("已触发失败指纹重新生成", "success");
|
||||
refresh();
|
||||
} catch (e) {
|
||||
show(e instanceof Error ? e.message : "触发失败", "error");
|
||||
} finally {
|
||||
setRegenFailedFingerprintId("");
|
||||
}
|
||||
}
|
||||
|
||||
async function handleToggleTeaser(d: api.AdminDrive) {
|
||||
const next = !d.teaserEnabled;
|
||||
setTogglingTeaserId(d.id);
|
||||
@@ -351,6 +379,19 @@ export function DrivesPage() {
|
||||
return selectedDriveId ? list.find((d) => d.id === selectedDriveId) : null;
|
||||
}, [selectedDriveId, list]);
|
||||
|
||||
const deleteModal = (
|
||||
<DeleteDriveModal
|
||||
drive={deleteTarget}
|
||||
deleting={deletingId === deleteTarget?.id}
|
||||
onCancel={() => {
|
||||
if (!deletingId) {
|
||||
setDeleteTarget(null);
|
||||
}
|
||||
}}
|
||||
onConfirm={confirmDeleteDrive}
|
||||
/>
|
||||
);
|
||||
|
||||
if (selectedDriveId && selectedDrive) {
|
||||
const d = selectedDrive;
|
||||
const driveStorage = storage?.drives[d.id];
|
||||
@@ -395,7 +436,7 @@ export function DrivesPage() {
|
||||
<span className="admin-detail-label">网盘 ID</span>
|
||||
<span className="admin-detail-value admin-mono-cell">{d.id}</span>
|
||||
</div>
|
||||
{d.kind !== "spider91" && (
|
||||
{usesRootDirectoryID(d.kind) && (
|
||||
<>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">根目录 ID</span>
|
||||
@@ -436,10 +477,7 @@ export function DrivesPage() {
|
||||
<button className="admin-btn" onClick={() => openEdit(d)}>
|
||||
{d.kind === "spider91" ? "编辑配置" : "编辑配置凭证"}
|
||||
</button>
|
||||
<button className="admin-btn is-danger" onClick={() => {
|
||||
handleDelete(d);
|
||||
setSelectedDriveId(null);
|
||||
}} style={{ marginLeft: "auto" }}>
|
||||
<button className="admin-btn is-danger" onClick={() => setDeleteTarget(d)} style={{ marginLeft: "auto" }}>
|
||||
<Trash2 size={13} /> 删除网盘
|
||||
</button>
|
||||
</div>
|
||||
@@ -479,14 +517,14 @@ export function DrivesPage() {
|
||||
style={{ padding: "4px 10px", fontSize: "11px" }}
|
||||
>
|
||||
{d.teaserEnabled ? <Power size={11} /> : <PowerOff size={11} />}
|
||||
<span>{d.teaserEnabled ? "Teaser: 开" : "Teaser: 关"}</span>
|
||||
<span>{d.teaserEnabled ? "预览视频生成:开" : "预览视频生成:关"}</span>
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div className="admin-detail-grid">
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">封面状态</span>
|
||||
<span className="admin-detail-label">封面生成状态</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationStatusLine label="封面" status={d.thumbnailGenerationStatus} />
|
||||
</div>
|
||||
@@ -503,13 +541,13 @@ export function DrivesPage() {
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">Teaser 状态</span>
|
||||
<span className="admin-detail-label">预览视频生成状态</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationStatusLine label="预览" status={d.previewGenerationStatus} />
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">Teaser 数量</span>
|
||||
<span className="admin-detail-label">预览视频数量</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationCounts
|
||||
ready={d.teaserReadyCount}
|
||||
@@ -519,13 +557,13 @@ export function DrivesPage() {
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">指纹状态</span>
|
||||
<span className="admin-detail-label">视频指纹生成状态</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationStatusLine label="指纹" status={d.fingerprintGenerationStatus} />
|
||||
</div>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">指纹数量</span>
|
||||
<span className="admin-detail-label">视频指纹数量</span>
|
||||
<div className="admin-detail-value">
|
||||
<GenerationCounts
|
||||
ready={d.fingerprintReadyCount}
|
||||
@@ -537,14 +575,6 @@ export function DrivesPage() {
|
||||
</div>
|
||||
|
||||
<div className="admin-detail-actions">
|
||||
<button
|
||||
className="admin-btn"
|
||||
disabled={(d.teaserFailedCount ?? 0) <= 0 || regenFailedId === d.id}
|
||||
onClick={() => handleRegenFailed(d)}
|
||||
>
|
||||
<RotateCcw size={13} />
|
||||
<span>重试失败 Teaser</span>
|
||||
</button>
|
||||
<button
|
||||
className="admin-btn"
|
||||
disabled={(d.thumbnailFailedCount ?? 0) <= 0 || regenFailedThumbId === d.id}
|
||||
@@ -553,6 +583,25 @@ export function DrivesPage() {
|
||||
<RotateCcw size={13} />
|
||||
<span>重试失败封面</span>
|
||||
</button>
|
||||
<button
|
||||
className="admin-btn"
|
||||
disabled={(d.teaserFailedCount ?? 0) <= 0 || regenFailedId === d.id}
|
||||
onClick={() => handleRegenFailed(d)}
|
||||
>
|
||||
<RotateCcw size={13} />
|
||||
<span>重试失败预览视频</span>
|
||||
</button>
|
||||
<button
|
||||
className="admin-btn"
|
||||
disabled={
|
||||
(d.fingerprintFailedCount ?? 0) <= 0 ||
|
||||
regenFailedFingerprintId === d.id
|
||||
}
|
||||
onClick={() => handleRegenFailedFingerprints(d)}
|
||||
>
|
||||
<RotateCcw size={13} />
|
||||
<span>重试失败指纹</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -570,7 +619,7 @@ export function DrivesPage() {
|
||||
<span className="admin-detail-value">{formatBytes(driveStorage?.thumbnailBytes ?? 0)}</span>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
<span className="admin-detail-label">Teaser 占用</span>
|
||||
<span className="admin-detail-label">预览视频占用</span>
|
||||
<span className="admin-detail-value">{formatBytes(driveStorage?.teaserBytes ?? 0)}</span>
|
||||
</div>
|
||||
<div className="admin-detail-row">
|
||||
@@ -610,6 +659,7 @@ export function DrivesPage() {
|
||||
uploadTargets={uploadTargets}
|
||||
/>
|
||||
</Modal>
|
||||
{deleteModal}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -727,6 +777,7 @@ export function DrivesPage() {
|
||||
uploadTargets={uploadTargets}
|
||||
/>
|
||||
</Modal>
|
||||
{deleteModal}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -739,7 +790,7 @@ function StorageSummary({ storage }: { storage: api.AdminDriveStorage }) {
|
||||
<strong>{formatBytes(storage.thumbnailBytes)}</strong>
|
||||
</div>
|
||||
<div className="admin-storage-summary__metric">
|
||||
<span>Teaser 占用</span>
|
||||
<span>预览视频占用</span>
|
||||
<strong>{formatBytes(storage.teaserBytes)}</strong>
|
||||
</div>
|
||||
<div className="admin-storage-summary__metric">
|
||||
@@ -894,6 +945,75 @@ function StatusTag({
|
||||
return <span className="admin-status">{status || "未连接"}</span>;
|
||||
}
|
||||
|
||||
function DeleteDriveModal({
|
||||
drive,
|
||||
deleting,
|
||||
onCancel,
|
||||
onConfirm,
|
||||
}: {
|
||||
drive: api.AdminDrive | null;
|
||||
deleting: boolean;
|
||||
onCancel: () => void;
|
||||
onConfirm: () => void;
|
||||
}) {
|
||||
const name = drive?.name || drive?.id || "";
|
||||
const isSpider91 = drive?.kind === "spider91";
|
||||
const isLocalStorage = drive?.kind === "localstorage";
|
||||
const title = isSpider91 ? "删除 91Spider" : "删除存储";
|
||||
const primaryText = deleting ? "删除中..." : "确认删除并清理";
|
||||
|
||||
return (
|
||||
<Modal
|
||||
open={!!drive}
|
||||
title={title}
|
||||
onClose={onCancel}
|
||||
footer={
|
||||
<>
|
||||
<button className="admin-btn" onClick={onCancel} disabled={deleting}>
|
||||
取消
|
||||
</button>
|
||||
<button className="admin-btn is-danger" onClick={onConfirm} disabled={deleting}>
|
||||
<Trash2 size={13} />
|
||||
{primaryText}
|
||||
</button>
|
||||
</>
|
||||
}
|
||||
>
|
||||
<div className="admin-delete-confirm">
|
||||
<div className="admin-delete-confirm__icon">
|
||||
<AlertTriangle size={20} />
|
||||
</div>
|
||||
<div className="admin-delete-confirm__content">
|
||||
<p className="admin-delete-confirm__title">
|
||||
{isSpider91
|
||||
? `确定删除「${name}」吗?`
|
||||
: `确定删除「${name}」并清理该存储的视频数据吗?`}
|
||||
</p>
|
||||
<p className="admin-delete-confirm__text">
|
||||
取消或关闭此弹窗不会删除存储配置,也不会清理任何文件。
|
||||
</p>
|
||||
<ul className="admin-delete-confirm__list">
|
||||
<li>删除该存储配置</li>
|
||||
<li>删除数据库中的相关视频记录,网站首页、列表、标签页和详情页不再展示这些视频</li>
|
||||
<li>删除本机保存的封面图和预览视频</li>
|
||||
{isSpider91 && (
|
||||
<li>删除通过 91Spider 爬取到的本机 91 视频文件</li>
|
||||
)}
|
||||
{isLocalStorage && (
|
||||
<li>不会删除用户配置的本地目录中的原始视频文件</li>
|
||||
)}
|
||||
</ul>
|
||||
{!isSpider91 && !isLocalStorage && (
|
||||
<p className="admin-delete-confirm__text">
|
||||
此操作只清理本项目生成和记录的数据,不会删除云盘上的原始视频文件。
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
|
||||
function DriveForm({
|
||||
form,
|
||||
onChange,
|
||||
@@ -941,6 +1061,7 @@ function DriveForm({
|
||||
disabled={isEdit}
|
||||
>
|
||||
<option value="p115">115 网盘</option>
|
||||
<option value="p123">123 云盘</option>
|
||||
<option value="pikpak">PikPak</option>
|
||||
<option value="onedrive">OneDrive</option>
|
||||
<option value="googledrive">Google Drive</option>
|
||||
@@ -950,17 +1071,19 @@ function DriveForm({
|
||||
<option value="wopan">联通沃盘</option>
|
||||
</select>
|
||||
</div>
|
||||
<div className="admin-form__row">
|
||||
<label>根目录 ID</label>
|
||||
<input
|
||||
value={form.rootId}
|
||||
onChange={(e) => set("rootId", e.target.value)}
|
||||
placeholder={rootIdPlaceholder(form.kind)}
|
||||
/>
|
||||
<div className="admin-form__help">
|
||||
留空时使用该网盘类型的默认根目录,具体目录ID获取方式请参考OpenList文档
|
||||
{usesRootDirectoryID(form.kind) && (
|
||||
<div className="admin-form__row">
|
||||
<label>根目录 ID</label>
|
||||
<input
|
||||
value={form.rootId}
|
||||
onChange={(e) => set("rootId", e.target.value)}
|
||||
placeholder={rootIdPlaceholder(form.kind)}
|
||||
/>
|
||||
<div className="admin-form__help">
|
||||
留空时使用该网盘类型的默认根目录,具体目录ID获取方式请参考OpenList文档
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{(help || fields.length > 0) && (
|
||||
<>
|
||||
@@ -972,6 +1095,12 @@ function DriveForm({
|
||||
</div>
|
||||
)}
|
||||
|
||||
{form.kind === "p123" && (
|
||||
<P123QRCodeLogin
|
||||
onToken={(token) => setCred("access_token", token)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{fields.map((f) => (
|
||||
<div key={f.key} className="admin-form__row">
|
||||
<label>{f.label}{f.required && " *"}</label>
|
||||
@@ -1008,6 +1137,144 @@ function DriveForm({
|
||||
);
|
||||
}
|
||||
|
||||
function P123QRCodeLogin({ onToken }: { onToken: (token: string) => void }) {
|
||||
const { show } = useToast();
|
||||
const [session, setSession] = useState<api.P123QRSession | null>(null);
|
||||
const [status, setStatus] = useState<api.P123QRStatus | null>(null);
|
||||
const [starting, setStarting] = useState(false);
|
||||
const [pollingError, setPollingError] = useState("");
|
||||
const [completed, setCompleted] = useState(false);
|
||||
|
||||
async function start() {
|
||||
setStarting(true);
|
||||
setPollingError("");
|
||||
setCompleted(false);
|
||||
setStatus(null);
|
||||
try {
|
||||
const next = await api.startP123QRLogin();
|
||||
setSession(next);
|
||||
} catch (e) {
|
||||
setSession(null);
|
||||
show(e instanceof Error ? e.message : "生成二维码失败", "error");
|
||||
} finally {
|
||||
setStarting(false);
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (!session || completed) return;
|
||||
const activeSession = session;
|
||||
let stopped = false;
|
||||
let inFlight = false;
|
||||
let timer: number | undefined;
|
||||
|
||||
async function poll() {
|
||||
if (stopped || inFlight) return;
|
||||
inFlight = true;
|
||||
try {
|
||||
const next = await api.getP123QRStatus(activeSession.uniID, activeSession.loginUuid);
|
||||
if (stopped) return;
|
||||
setStatus(next);
|
||||
setPollingError("");
|
||||
if (next.accessToken) {
|
||||
stopped = true;
|
||||
if (timer) window.clearInterval(timer);
|
||||
setCompleted(true);
|
||||
onToken(next.accessToken);
|
||||
show("扫码成功,已填入 access_token,保存后生效", "success");
|
||||
return;
|
||||
}
|
||||
if (next.loginStatus === 2 || next.loginStatus === 4) {
|
||||
stopped = true;
|
||||
if (timer) window.clearInterval(timer);
|
||||
}
|
||||
} catch (e) {
|
||||
if (stopped) return;
|
||||
setPollingError(e instanceof Error ? e.message : "查询扫码状态失败");
|
||||
} finally {
|
||||
inFlight = false;
|
||||
}
|
||||
}
|
||||
|
||||
poll();
|
||||
timer = window.setInterval(poll, 1800);
|
||||
return () => {
|
||||
stopped = true;
|
||||
if (timer) window.clearInterval(timer);
|
||||
};
|
||||
}, [session, completed, onToken, show]);
|
||||
|
||||
const statusText = completed
|
||||
? "已获取 token"
|
||||
: pollingError || status?.statusText || (session ? "等待扫码" : "未生成二维码");
|
||||
const statusClass = p123QRStatusClass(status, completed, pollingError);
|
||||
const platform = status?.platformText ? ` · ${status.platformText}` : "";
|
||||
|
||||
return (
|
||||
<div className="admin-form__row">
|
||||
<label>扫码登录</label>
|
||||
<div className="admin-p123-qr">
|
||||
<div className="admin-p123-qr__actions">
|
||||
<button
|
||||
type="button"
|
||||
className="admin-btn"
|
||||
onClick={start}
|
||||
disabled={starting}
|
||||
>
|
||||
<QrCode size={14} />
|
||||
{starting ? "生成中..." : session ? "重新生成二维码" : "生成二维码"}
|
||||
</button>
|
||||
<span className={`admin-status ${statusClass}`}>
|
||||
{statusText}
|
||||
{platform}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{session && (
|
||||
<div className="admin-p123-qr__body">
|
||||
<img
|
||||
className="admin-p123-qr__image"
|
||||
src={session.qrImageDataUrl}
|
||||
alt="123 云盘扫码登录二维码"
|
||||
/>
|
||||
<div className="admin-p123-qr__meta">
|
||||
<div className="admin-form__help">
|
||||
使用微信或 123 云盘 App 扫码并确认登录;确认后系统会自动填入 access_token。
|
||||
</div>
|
||||
{session.expiresAt && (
|
||||
<div className="admin-form__help">
|
||||
过期时间:{new Date(session.expiresAt).toLocaleTimeString("zh-CN", {
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
{(status?.loginStatus === 2 || status?.loginStatus === 4) && (
|
||||
<div className="admin-form__help">
|
||||
当前二维码{status.loginStatus === 2 ? "已被拒绝" : "已过期"},请重新生成。
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function p123QRStatusClass(
|
||||
status: api.P123QRStatus | null,
|
||||
completed: boolean,
|
||||
error: string
|
||||
): string {
|
||||
if (completed || status?.loginStatus === 3) return "is-ok";
|
||||
if (error || status?.loginStatus === 2 || status?.loginStatus === 4) {
|
||||
return "is-error";
|
||||
}
|
||||
return "is-pending";
|
||||
}
|
||||
|
||||
/**
|
||||
* Spider91UploadTargetField 是 spider91 drive 表单专属的"上传目标"下拉。
|
||||
*
|
||||
@@ -1052,6 +1319,8 @@ function credentialHelp(kind: Kind, isEdit: boolean): string {
|
||||
return `在 pan.quark.cn 登录后,F12 → Network → 任意请求 → Request Headers 里复制整段 Cookie 粘贴到下方。${note}`;
|
||||
case "p115":
|
||||
return `登录 115.com 后复制 Cookie,形如 "UID=...; CID=...; SEID=...; KID=..."。${note}`;
|
||||
case "p123":
|
||||
return `推荐使用扫码登录自动获取 access_token;账号密码登录被 123 云盘风控拦截时,也可以只填写 access_token。播放走 302 跳转到 123 云盘返回的短期 CDN 地址。${note}`;
|
||||
case "pikpak":
|
||||
return `填写 PikPak 账号和密码即可。平台、设备 ID、验证码 token 和 refresh token 会由服务端自动处理并保存。${note}`;
|
||||
case "wopan":
|
||||
@@ -1063,7 +1332,7 @@ function credentialHelp(kind: Kind, isEdit: boolean): string {
|
||||
case "localstorage":
|
||||
return `把服务器上的一个已有目录作为视频来源扫描。填写绝对路径,例如 /mnt/videos;系统会读取该目录及子目录中的视频,并生成封面、Teaser 和指纹。${note}`;
|
||||
case "spider91":
|
||||
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;它不是外部网盘,不需要填写 Cookie 或目录 ID。后续流水线会把较早的视频上传到你选择的 115 / PikPak / OneDrive 目标盘。";
|
||||
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;可按服务器网络情况单独配置代理。后续流水线会把较早的视频上传到你选择的 115 / PikPak / OneDrive 目标盘。";
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
@@ -1098,6 +1367,26 @@ function credentialFields(kind: Kind): Array<{
|
||||
required: true,
|
||||
},
|
||||
];
|
||||
case "p123":
|
||||
return [
|
||||
{
|
||||
key: "username",
|
||||
label: "用户名 / 邮箱(可选)",
|
||||
placeholder: "user@example.com",
|
||||
},
|
||||
{
|
||||
key: "password",
|
||||
label: "密码(可选)",
|
||||
placeholder: "123 云盘密码",
|
||||
},
|
||||
{
|
||||
key: "access_token",
|
||||
label: "access_token(推荐用于风控场景)",
|
||||
placeholder: "Bearer eyJ... 或直接粘贴 token",
|
||||
multiline: true,
|
||||
help: "扫码成功后会自动填入该字段;如果 token 过期,重新扫码后保存即可。",
|
||||
},
|
||||
];
|
||||
case "pikpak":
|
||||
return [
|
||||
{
|
||||
@@ -1164,7 +1453,14 @@ function credentialFields(kind: Kind): Array<{
|
||||
},
|
||||
];
|
||||
case "spider91":
|
||||
return [];
|
||||
return [
|
||||
{
|
||||
key: "proxy",
|
||||
label: "代理地址(可选)",
|
||||
placeholder: "http://127.0.0.1:7890",
|
||||
help: "仅用于 91Spider 的列表/详情请求和视频、封面下载;留空则使用服务器环境变量 HTTP_PROXY / HTTPS_PROXY 或直连。支持 http://、https://、socks5:// 或 socks5h://。",
|
||||
},
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1177,6 +1473,10 @@ function defaultRootId(kind: Kind): string {
|
||||
return "0";
|
||||
}
|
||||
|
||||
function usesRootDirectoryID(kind: Kind): boolean {
|
||||
return kind !== "localstorage" && kind !== "spider91";
|
||||
}
|
||||
|
||||
function rootIdPlaceholder(kind: Kind): string {
|
||||
const rootId = defaultRootId(kind);
|
||||
return rootId ? `默认:${rootId}` : "留空表示根目录";
|
||||
@@ -1310,7 +1610,7 @@ function SelectedDirsChips({
|
||||
className="admin-text-faint"
|
||||
style={{ fontSize: "13px", padding: "6px 0" }}
|
||||
>
|
||||
当前未勾选任何跳过目录({drive.kind === "p115" ? "115 网盘" : drive.kind}{" "}
|
||||
当前未勾选任何跳过目录({kindLabel[drive.kind] ?? drive.kind}{" "}
|
||||
将完整扫描)。
|
||||
</div>
|
||||
);
|
||||
|
||||
+45
-4
@@ -77,7 +77,7 @@ export function checkUpdate() {
|
||||
|
||||
export type AdminDrive = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
kind: "quark" | "p115" | "p123" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
name: string;
|
||||
rootId: string;
|
||||
status: string;
|
||||
@@ -93,6 +93,8 @@ export type AdminDrive = {
|
||||
skipDirIds: string[];
|
||||
// spider91 上次成功爬取时间(unix 秒);其它 kind 留空。
|
||||
lastCrawlAt?: number;
|
||||
// spider91 专用代理地址;仅后台管理接口返回,用于编辑表单回显。
|
||||
spider91Proxy?: string;
|
||||
thumbnailGenerationStatus?: DriveGenerationStatus;
|
||||
previewGenerationStatus?: DriveGenerationStatus;
|
||||
fingerprintGenerationStatus?: DriveGenerationStatus;
|
||||
@@ -137,7 +139,7 @@ export function getDriveStorage() {
|
||||
|
||||
export type UpsertDriveInput = {
|
||||
id: string;
|
||||
kind: "quark" | "p115" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
kind: "quark" | "p115" | "p123" | "pikpak" | "wopan" | "onedrive" | "googledrive" | "localstorage" | "spider91";
|
||||
name: string;
|
||||
rootId: string;
|
||||
credentials: Record<string, string>;
|
||||
@@ -156,9 +158,14 @@ export function upsertDrive(body: UpsertDriveInput) {
|
||||
});
|
||||
}
|
||||
|
||||
export function deleteDrive(id: string) {
|
||||
return request<{ ok: boolean }>(`/drives/${encodeURIComponent(id)}`, {
|
||||
export type DeleteDriveInput = {
|
||||
deleteVideos: true;
|
||||
};
|
||||
|
||||
export function deleteDrive(id: string, body: DeleteDriveInput) {
|
||||
return request<{ ok: boolean; deletedVideos: number }>(`/drives/${encodeURIComponent(id)}`, {
|
||||
method: "DELETE",
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -169,6 +176,33 @@ export function rescan(id: string) {
|
||||
);
|
||||
}
|
||||
|
||||
export type P123QRSession = {
|
||||
loginUuid: string;
|
||||
uniID: string;
|
||||
qrCodeUrl: string;
|
||||
qrImageDataUrl: string;
|
||||
expiresAt?: string;
|
||||
};
|
||||
|
||||
export type P123QRStatus = {
|
||||
loginStatus: number;
|
||||
statusText: string;
|
||||
scanPlatform?: number;
|
||||
platformText?: string;
|
||||
accessToken?: string;
|
||||
};
|
||||
|
||||
export function startP123QRLogin() {
|
||||
return request<P123QRSession>("/drives/p123/qr", { method: "POST" });
|
||||
}
|
||||
|
||||
export function getP123QRStatus(uniID: string, loginUuid: string) {
|
||||
const qs = new URLSearchParams({ loginUuid });
|
||||
return request<P123QRStatus>(
|
||||
`/drives/p123/qr/${encodeURIComponent(uniID)}?${qs.toString()}`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 切换某个云盘的 teaser 生成开关。点击网盘列表里行内的 toggle 按钮时调用。
|
||||
*
|
||||
@@ -243,6 +277,13 @@ export function regenFailedThumbnails(id: string) {
|
||||
);
|
||||
}
|
||||
|
||||
export function regenFailedFingerprints(id: string) {
|
||||
return request<{ ok: boolean }>(
|
||||
`/drives/${encodeURIComponent(id)}/fingerprints/failed/regenerate`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- Videos ----------
|
||||
|
||||
export type AdminVideo = {
|
||||
|
||||
@@ -10,11 +10,14 @@ export function TagCloud() {
|
||||
|
||||
useEffect(() => {
|
||||
let active = true;
|
||||
fetchTags().then((list) => {
|
||||
if (active) setTags(list);
|
||||
});
|
||||
const timer = window.setTimeout(() => {
|
||||
fetchTags().then((list) => {
|
||||
if (active) setTags(list);
|
||||
});
|
||||
}, 500);
|
||||
return () => {
|
||||
active = false;
|
||||
window.clearTimeout(timer);
|
||||
};
|
||||
}, []);
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ import { PreviewVideo } from "./PreviewVideo";
|
||||
|
||||
type Props = {
|
||||
video: VideoItem;
|
||||
priority?: boolean;
|
||||
};
|
||||
|
||||
const HOVER_DELAY_MS = 300;
|
||||
@@ -24,7 +25,7 @@ function useActivePreviewId(): string | null {
|
||||
);
|
||||
}
|
||||
|
||||
export function VideoCard({ video }: Props) {
|
||||
export function VideoCard({ video, priority = false }: Props) {
|
||||
const [previewState, setPreviewState] = useState<PreviewState>("idle");
|
||||
const [shouldRenderPreview, setShouldRenderPreview] = useState(false);
|
||||
const [progress, setProgress] = useState(0); // 0~1
|
||||
@@ -204,7 +205,7 @@ export function VideoCard({ video }: Props) {
|
||||
className="thumb-image"
|
||||
src={thumbnailSrc}
|
||||
alt={video.title}
|
||||
loading="lazy"
|
||||
loading={priority ? "eager" : "lazy"}
|
||||
onError={handleThumbnailError}
|
||||
/>
|
||||
|
||||
@@ -291,6 +292,7 @@ function sourceKindFromLabel(label: string): string {
|
||||
const value = label.toLowerCase();
|
||||
if (value.includes("夸克") || value.includes("quark")) return "quark";
|
||||
if (value.includes("115") || value.includes("p115")) return "p115";
|
||||
if (value.includes("123") || value.includes("p123")) return "p123";
|
||||
if (value.includes("pikpak")) return "pikpak";
|
||||
if (value.includes("沃盘") || value.includes("wopan") || value.includes("联通")) return "wopan";
|
||||
if (value.includes("onedrive") || value.includes("one drive")) return "onedrive";
|
||||
|
||||
@@ -6,6 +6,7 @@ type Props = {
|
||||
loading?: boolean;
|
||||
compact?: boolean;
|
||||
emptyText?: string;
|
||||
priorityCount?: number;
|
||||
skeletonCount?: number;
|
||||
};
|
||||
|
||||
@@ -14,6 +15,7 @@ export function VideoGrid({
|
||||
loading,
|
||||
compact,
|
||||
emptyText = "暂时没有视频",
|
||||
priorityCount = 0,
|
||||
skeletonCount = 8,
|
||||
}: Props) {
|
||||
if (loading) {
|
||||
@@ -32,8 +34,8 @@ export function VideoGrid({
|
||||
|
||||
return (
|
||||
<div className={`video-grid ${compact ? "is-compact" : ""}`}>
|
||||
{(videos ?? []).map((v) => (
|
||||
<VideoCard key={v.id} video={v} />
|
||||
{(videos ?? []).map((v, index) => (
|
||||
<VideoCard key={v.id} video={v} priority={index < priorityCount} />
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -68,6 +68,7 @@ function sourceKindFromLabel(label: string): string {
|
||||
const value = label.toLowerCase();
|
||||
if (value.includes("夸克") || value.includes("quark")) return "quark";
|
||||
if (value.includes("115") || value.includes("p115")) return "p115";
|
||||
if (value.includes("123") || value.includes("p123")) return "p123";
|
||||
if (value.includes("pikpak")) return "pikpak";
|
||||
if (value.includes("沃盘") || value.includes("wopan") || value.includes("联通"))
|
||||
return "wopan";
|
||||
|
||||
+30
-4
@@ -1,14 +1,19 @@
|
||||
import type { VideoDetail, VideoItem } from "@/types";
|
||||
|
||||
// 真实后端接口调用。未配置网盘时,各接口返回空数据。
|
||||
export function fetchHomeVideos(): Promise<VideoItem[]> {
|
||||
return apiGet<VideoItem[]>("/api/home").catch(() => []);
|
||||
export function fetchHomeVideos(excludeIds?: string[]): Promise<VideoItem[]> {
|
||||
const qs = new URLSearchParams();
|
||||
for (const id of excludeIds ?? []) {
|
||||
if (id.trim()) qs.append("exclude", id.trim());
|
||||
}
|
||||
const suffix = qs.toString() ? `?${qs.toString()}` : "";
|
||||
return apiGet<VideoItem[]>(`/api/home${suffix}`).catch(() => []);
|
||||
}
|
||||
|
||||
export function fetchListing(
|
||||
page: number,
|
||||
pageSize: number,
|
||||
params?: { q?: string; tag?: string; cat?: string; sort?: string }
|
||||
params?: { q?: string; tag?: string; cat?: string; sort?: string; includeTotal?: boolean }
|
||||
): Promise<{ items: VideoItem[]; total: number }> {
|
||||
const qs = new URLSearchParams({
|
||||
page: String(page),
|
||||
@@ -18,6 +23,7 @@ export function fetchListing(
|
||||
if (params?.tag) qs.set("tag", params.tag);
|
||||
if (params?.cat) qs.set("cat", params.cat);
|
||||
if (params?.sort) qs.set("sort", params.sort);
|
||||
if (params?.includeTotal === false) qs.set("count", "false");
|
||||
return apiGet<{ items: VideoItem[]; total: number }>(
|
||||
`/api/list?${qs.toString()}`
|
||||
).catch(() => ({ items: [], total: 0 }));
|
||||
@@ -73,8 +79,28 @@ export function uploadVideo(input: UploadVideoInput): Promise<VideoItem> {
|
||||
|
||||
export type TagItem = { id: string; label: string; count?: number };
|
||||
|
||||
const TAG_CACHE_TTL_MS = 30_000;
|
||||
let cachedTags: TagItem[] | null = null;
|
||||
let cachedTagsAt = 0;
|
||||
let pendingTags: Promise<TagItem[]> | null = null;
|
||||
|
||||
export function fetchTags(): Promise<TagItem[]> {
|
||||
return apiGet<TagItem[]>("/api/tags").catch(() => []);
|
||||
const now = Date.now();
|
||||
if (cachedTags && now - cachedTagsAt < TAG_CACHE_TTL_MS) {
|
||||
return Promise.resolve(cachedTags);
|
||||
}
|
||||
if (pendingTags) return pendingTags;
|
||||
pendingTags = apiGet<TagItem[]>("/api/tags")
|
||||
.then((tags) => {
|
||||
cachedTags = tags;
|
||||
cachedTagsAt = Date.now();
|
||||
return tags;
|
||||
})
|
||||
.catch(() => cachedTags ?? [])
|
||||
.finally(() => {
|
||||
pendingTags = null;
|
||||
});
|
||||
return pendingTags;
|
||||
}
|
||||
|
||||
/** 短视频模式单条记录。比 VideoItem 多 videoSrc / poster。 */
|
||||
|
||||
+69
-18
@@ -10,6 +10,8 @@ import type { VideoItem } from "@/types";
|
||||
|
||||
const DESKTOP_COUNT = 12;
|
||||
const MOBILE_COUNT = 8;
|
||||
const HOME_RECENT_KEY = "home.random.recentVideoIds";
|
||||
const HOME_RECENT_LIMIT = 72;
|
||||
|
||||
function useIsMobile() {
|
||||
const [mobile, setMobile] = useState(window.innerWidth <= 640);
|
||||
@@ -26,31 +28,75 @@ function useIsMobile() {
|
||||
let cachedRanking: VideoItem[] | null = null;
|
||||
let cachedLatest: VideoItem[] | null = null;
|
||||
|
||||
function loadRecentHomeVideoIds(): string[] {
|
||||
try {
|
||||
const raw = window.localStorage.getItem(HOME_RECENT_KEY);
|
||||
const parsed = raw ? JSON.parse(raw) : [];
|
||||
return Array.isArray(parsed)
|
||||
? parsed.filter((id): id is string => typeof id === "string" && id.length > 0)
|
||||
: [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function rememberHomeVideos(items: VideoItem[]) {
|
||||
const merged = [...items.map((item) => item.id), ...loadRecentHomeVideoIds()];
|
||||
const seen = new Set<string>();
|
||||
const recent: string[] = [];
|
||||
for (const id of merged) {
|
||||
if (!id || seen.has(id)) continue;
|
||||
seen.add(id);
|
||||
recent.push(id);
|
||||
if (recent.length >= HOME_RECENT_LIMIT) break;
|
||||
}
|
||||
try {
|
||||
window.localStorage.setItem(HOME_RECENT_KEY, JSON.stringify(recent));
|
||||
} catch {
|
||||
// localStorage 不可用时只影响连续刷新去重,不影响首页展示。
|
||||
}
|
||||
}
|
||||
|
||||
export default function HomePage() {
|
||||
const [rankingVideos, setRankingVideos] = useState<VideoItem[]>(cachedRanking ?? []);
|
||||
const [latestVideos, setLatestVideos] = useState<VideoItem[]>(cachedLatest ?? []);
|
||||
const [loading, setLoading] = useState(cachedRanking === null);
|
||||
const [rankingLoading, setRankingLoading] = useState(cachedRanking === null);
|
||||
const [latestLoading, setLatestLoading] = useState(cachedLatest === null);
|
||||
const isMobile = useIsMobile();
|
||||
|
||||
useEffect(() => {
|
||||
document.title = "首页 · 91";
|
||||
|
||||
// 有缓存说明是 SPA 内导航返回,不重新请求
|
||||
if (cachedRanking !== null) return;
|
||||
|
||||
let active = true;
|
||||
setLoading(true);
|
||||
Promise.all([
|
||||
fetchHomeVideos(),
|
||||
fetchListing(1, DESKTOP_COUNT, { sort: "latest" }),
|
||||
]).then(([rankingItems, latestResult]) => {
|
||||
if (!active) return;
|
||||
cachedRanking = rankingItems;
|
||||
cachedLatest = latestResult.items;
|
||||
setRankingVideos(rankingItems);
|
||||
setLatestVideos(latestResult.items);
|
||||
setLoading(false);
|
||||
});
|
||||
|
||||
if (cachedRanking === null) {
|
||||
setRankingLoading(true);
|
||||
const excludeIds = loadRecentHomeVideoIds();
|
||||
fetchHomeVideos(excludeIds)
|
||||
.then((rankingItems) => {
|
||||
if (!active) return;
|
||||
rememberHomeVideos(rankingItems);
|
||||
cachedRanking = rankingItems;
|
||||
setRankingVideos(rankingItems);
|
||||
})
|
||||
.finally(() => {
|
||||
if (active) setRankingLoading(false);
|
||||
});
|
||||
}
|
||||
|
||||
if (cachedLatest === null) {
|
||||
setLatestLoading(true);
|
||||
fetchListing(1, DESKTOP_COUNT, { sort: "latest", includeTotal: false })
|
||||
.then((latestResult) => {
|
||||
if (!active) return;
|
||||
cachedLatest = latestResult.items;
|
||||
setLatestVideos(latestResult.items);
|
||||
})
|
||||
.finally(() => {
|
||||
if (active) setLatestLoading(false);
|
||||
});
|
||||
}
|
||||
|
||||
return () => { active = false; };
|
||||
}, []);
|
||||
|
||||
@@ -68,12 +114,17 @@ export default function HomePage() {
|
||||
|
||||
<div className="container page-section">
|
||||
<SectionHeader title="随机推荐" extra={`随机展示 ${ranking.length} 个作品`} />
|
||||
<VideoGrid videos={ranking} loading={loading} skeletonCount={displayCount} />
|
||||
<VideoGrid
|
||||
videos={ranking}
|
||||
loading={rankingLoading}
|
||||
priorityCount={Math.min(4, displayCount)}
|
||||
skeletonCount={displayCount}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="container page-section">
|
||||
<SectionHeader title="最新视频" extra={latest.length > 0 ? `共 ${latest.length} 个` : undefined} />
|
||||
<VideoGrid videos={latest} loading={loading} skeletonCount={displayCount} />
|
||||
<VideoGrid videos={latest} loading={latestLoading} skeletonCount={displayCount} />
|
||||
</div>
|
||||
</AppShell>
|
||||
);
|
||||
|
||||
@@ -33,6 +33,7 @@ export default function ListingPage() {
|
||||
);
|
||||
const initialState = useMemo(() => readListingState(listKey), [listKey]);
|
||||
const activeListKeyRef = useRef(listKey);
|
||||
const hasLoadedListingRef = useRef(false);
|
||||
const pendingScrollYRef = useRef<number | null>(
|
||||
initialState ? initialState.scrollY : null
|
||||
);
|
||||
@@ -40,9 +41,11 @@ export default function ListingPage() {
|
||||
const [sort, setSort] = useState<SortKey>(initialState?.sort ?? "latest");
|
||||
const [view, setView] = useState<ViewMode>(initialState?.view ?? "grid");
|
||||
const [page, setPage] = useState(initialState?.page ?? 1);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [initialLoading, setInitialLoading] = useState(true);
|
||||
const [refreshing, setRefreshing] = useState(false);
|
||||
const [items, setItems] = useState<VideoItem[]>([]);
|
||||
const [total, setTotal] = useState(0);
|
||||
const isFetching = initialLoading || refreshing;
|
||||
|
||||
useEffect(() => {
|
||||
if (activeListKeyRef.current === listKey) return;
|
||||
@@ -64,12 +67,19 @@ export default function ListingPage() {
|
||||
: "视频列表 · 91";
|
||||
|
||||
let active = true;
|
||||
setLoading(true);
|
||||
const isInitialLoad = !hasLoadedListingRef.current;
|
||||
if (isInitialLoad) {
|
||||
setInitialLoading(true);
|
||||
} else {
|
||||
setRefreshing(true);
|
||||
}
|
||||
fetchListing(page, tag ? PAGE_SIZE_TAG : PAGE_SIZE_DEFAULT, { q: keyword, tag, cat, sort }).then((r) => {
|
||||
if (!active) return;
|
||||
setItems(r.items ?? []);
|
||||
setTotal(r.total ?? 0);
|
||||
setLoading(false);
|
||||
hasLoadedListingRef.current = true;
|
||||
setInitialLoading(false);
|
||||
setRefreshing(false);
|
||||
});
|
||||
return () => {
|
||||
active = false;
|
||||
@@ -109,7 +119,7 @@ export default function ListingPage() {
|
||||
}, [listKey, sort, view, page]);
|
||||
|
||||
useEffect(() => {
|
||||
if (loading) return;
|
||||
if (isFetching) return;
|
||||
const scrollY = pendingScrollYRef.current;
|
||||
if (scrollY === null) return;
|
||||
pendingScrollYRef.current = null;
|
||||
@@ -118,7 +128,7 @@ export default function ListingPage() {
|
||||
window.scrollTo({ top: scrollY, behavior: "auto" });
|
||||
});
|
||||
});
|
||||
}, [loading, items.length, listKey]);
|
||||
}, [isFetching, items.length, listKey]);
|
||||
|
||||
const title = keyword
|
||||
? `搜索结果:${keyword}`
|
||||
@@ -153,7 +163,7 @@ export default function ListingPage() {
|
||||
/>
|
||||
<VideoGrid
|
||||
videos={items}
|
||||
loading={loading}
|
||||
loading={initialLoading}
|
||||
compact={view === "compact"}
|
||||
skeletonCount={12}
|
||||
emptyText="没有找到匹配的视频"
|
||||
|
||||
@@ -1353,6 +1353,7 @@ function formatCount(n: number) {
|
||||
function getDriveShortName(source: string): string {
|
||||
const s = source.toLowerCase();
|
||||
if (s.includes("115")) return "115";
|
||||
if (s.includes("123")) return "123";
|
||||
if (s.includes("pikpak")) return "PikP";
|
||||
if (s.includes("quark") || s.includes("夸克")) return "Quak";
|
||||
if (s.includes("onedrive")) return "OneDrive";
|
||||
|
||||
+92
-6
@@ -324,6 +324,43 @@
|
||||
line-height: var(--line-relaxed);
|
||||
}
|
||||
|
||||
.admin-p123-qr {
|
||||
display: grid;
|
||||
gap: var(--space-3);
|
||||
padding: var(--space-3);
|
||||
border: 1px solid var(--border-subtle);
|
||||
border-radius: var(--radius-sm);
|
||||
background: var(--bg-elevated);
|
||||
}
|
||||
|
||||
.admin-p123-qr__actions {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: var(--space-2);
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.admin-p123-qr__body {
|
||||
display: grid;
|
||||
grid-template-columns: 160px minmax(0, 1fr);
|
||||
gap: var(--space-3);
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.admin-p123-qr__image {
|
||||
width: 160px;
|
||||
height: 160px;
|
||||
padding: 8px;
|
||||
border-radius: var(--radius-sm);
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
.admin-p123-qr__meta {
|
||||
display: grid;
|
||||
gap: 6px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
/* =========================================================
|
||||
* Buttons
|
||||
* ========================================================= */
|
||||
@@ -704,6 +741,50 @@
|
||||
background: var(--bg-elevated);
|
||||
}
|
||||
|
||||
.admin-delete-confirm {
|
||||
display: grid;
|
||||
grid-template-columns: 42px minmax(0, 1fr);
|
||||
gap: var(--space-3);
|
||||
align-items: start;
|
||||
}
|
||||
|
||||
.admin-delete-confirm__icon {
|
||||
width: 42px;
|
||||
height: 42px;
|
||||
border-radius: var(--radius-sm);
|
||||
display: grid;
|
||||
place-items: center;
|
||||
color: var(--danger);
|
||||
background: var(--danger-soft);
|
||||
border: 1px solid rgba(241, 85, 108, 0.34);
|
||||
}
|
||||
|
||||
.admin-delete-confirm__content {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.admin-delete-confirm__title {
|
||||
margin: 0 0 8px;
|
||||
color: var(--text-strong);
|
||||
font-size: var(--font-md);
|
||||
font-weight: var(--weight-semibold);
|
||||
}
|
||||
|
||||
.admin-delete-confirm__text {
|
||||
margin: 0 0 10px;
|
||||
color: var(--text-muted);
|
||||
font-size: var(--font-sm);
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.admin-delete-confirm__list {
|
||||
margin: 0 0 10px;
|
||||
padding-left: 18px;
|
||||
color: var(--text-default);
|
||||
font-size: var(--font-sm);
|
||||
line-height: 1.7;
|
||||
}
|
||||
|
||||
/* =========================================================
|
||||
* Toast
|
||||
* ========================================================= */
|
||||
@@ -1106,6 +1187,14 @@
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.admin-p123-qr__body {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.admin-p123-qr__image {
|
||||
justify-self: center;
|
||||
}
|
||||
|
||||
/* Table card-grid conversion for videos & tags tables */
|
||||
.admin-table:not(.admin-drives-table) {
|
||||
display: block;
|
||||
@@ -1418,13 +1507,11 @@
|
||||
text-align: left;
|
||||
color: inherit;
|
||||
transition: border-color var(--transition-fast),
|
||||
transform var(--transition-fast),
|
||||
box-shadow var(--transition-fast);
|
||||
}
|
||||
|
||||
.theme-card:hover:not(:disabled) {
|
||||
border-color: var(--border-accent);
|
||||
transform: translateY(-2px);
|
||||
box-shadow: var(--shadow-md);
|
||||
}
|
||||
|
||||
@@ -1781,12 +1868,11 @@
|
||||
flex-direction: column;
|
||||
gap: var(--space-4);
|
||||
cursor: pointer;
|
||||
transition: transform var(--transition-fast), border-color var(--transition-fast), box-shadow var(--transition-fast);
|
||||
transition: border-color var(--transition-fast), box-shadow var(--transition-fast);
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.admin-drive-card:hover {
|
||||
transform: translateY(-2px);
|
||||
border-color: var(--border-accent);
|
||||
box-shadow: var(--shadow-md);
|
||||
}
|
||||
@@ -1822,6 +1908,7 @@
|
||||
|
||||
.admin-drive-card__brand-icon[data-kind="quark"] { background: var(--drive-quark); }
|
||||
.admin-drive-card__brand-icon[data-kind="p115"] { background: var(--drive-p115); }
|
||||
.admin-drive-card__brand-icon[data-kind="p123"] { background: var(--drive-p123); }
|
||||
.admin-drive-card__brand-icon[data-kind="pikpak"] { background: var(--drive-pikpak); }
|
||||
.admin-drive-card__brand-icon[data-kind="wopan"] { background: var(--drive-wopan); }
|
||||
.admin-drive-card__brand-icon[data-kind="onedrive"] { background: var(--drive-onedrive); }
|
||||
@@ -2178,12 +2265,11 @@
|
||||
border-radius: var(--radius-sm);
|
||||
padding: var(--space-4);
|
||||
gap: var(--space-3);
|
||||
transition: all var(--transition-fast);
|
||||
transition: border-color var(--transition-fast), box-shadow var(--transition-fast);
|
||||
box-shadow: var(--shadow-sm);
|
||||
}
|
||||
|
||||
.admin-tag-card:hover {
|
||||
transform: translateY(-2px);
|
||||
border-color: var(--border-accent);
|
||||
box-shadow: var(--shadow-md);
|
||||
}
|
||||
|
||||
@@ -19,6 +19,8 @@ body,
|
||||
html {
|
||||
/* 让锚点跳转有平滑动画,但尊重系统 reduce-motion */
|
||||
scroll-behavior: smooth;
|
||||
scrollbar-gutter: stable;
|
||||
overflow-y: scroll;
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
|
||||
@@ -90,7 +90,6 @@
|
||||
|
||||
.promo-card:hover {
|
||||
border-color: var(--border-accent);
|
||||
transform: translateY(-3px);
|
||||
box-shadow: var(--shadow-md);
|
||||
color: var(--text-strong);
|
||||
}
|
||||
@@ -149,7 +148,6 @@
|
||||
}
|
||||
|
||||
.back-to-top:hover {
|
||||
transform: translateY(-3px);
|
||||
filter: brightness(1.05);
|
||||
}
|
||||
|
||||
@@ -372,7 +370,6 @@
|
||||
}
|
||||
|
||||
.upload-submit:hover:not(:disabled) {
|
||||
transform: translateY(-1px);
|
||||
filter: brightness(1.05);
|
||||
box-shadow: 0 10px 22px var(--accent-glow), var(--shadow-inset);
|
||||
}
|
||||
|
||||
@@ -16,13 +16,12 @@
|
||||
border: 1px solid var(--border-default);
|
||||
border-radius: var(--radius-xl);
|
||||
box-shadow: 0 8px 30px rgba(0, 0, 0, 0.4), var(--shadow-inset);
|
||||
transition: border-color var(--transition-fast), box-shadow var(--transition-fast), transform var(--transition-fast);
|
||||
transition: border-color var(--transition-fast), box-shadow var(--transition-fast);
|
||||
}
|
||||
|
||||
.search-panel__form:focus-within {
|
||||
border-color: var(--border-accent);
|
||||
box-shadow: 0 0 0 1px var(--border-accent), 0 8px 24px var(--accent-glow);
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.search-panel__input-wrapper {
|
||||
@@ -83,7 +82,6 @@
|
||||
|
||||
.search-panel__submit:hover {
|
||||
filter: brightness(1.05);
|
||||
transform: translateY(-1px);
|
||||
box-shadow: 0 8px 20px var(--accent-glow), var(--shadow-inset);
|
||||
}
|
||||
|
||||
|
||||
@@ -131,6 +131,7 @@
|
||||
/* ----- 网盘品牌色 ----- */
|
||||
--drive-quark: #5b8def;
|
||||
--drive-p115: #f56b76;
|
||||
--drive-p123: #22b8c8;
|
||||
--drive-pikpak: #8a6dff;
|
||||
--drive-wopan: #ff8a3c;
|
||||
--drive-onedrive: #4cabea;
|
||||
@@ -211,6 +212,7 @@
|
||||
/* ----- 网盘品牌色(粉白基底下重新调谐,整体柔化、降饱和) ----- */
|
||||
--drive-quark: #4f7be0;
|
||||
--drive-p115: #e0556a;
|
||||
--drive-p123: #1596a8;
|
||||
--drive-pikpak: #8466e6;
|
||||
--drive-wopan: #e57a36;
|
||||
--drive-onedrive: #2f95cf;
|
||||
|
||||
@@ -78,7 +78,7 @@
|
||||
border-radius: var(--radius-md);
|
||||
padding: 8px;
|
||||
color: var(--text-default);
|
||||
transition: transform var(--transition-normal), border-color var(--transition-normal),
|
||||
transition: border-color var(--transition-normal),
|
||||
box-shadow var(--transition-normal), background var(--transition-normal);
|
||||
}
|
||||
|
||||
@@ -86,7 +86,6 @@
|
||||
.video-card:focus-within {
|
||||
background: var(--bg-elevated);
|
||||
border-color: var(--border-default);
|
||||
transform: translateY(-4px);
|
||||
box-shadow: var(--shadow-lg);
|
||||
}
|
||||
|
||||
@@ -248,11 +247,11 @@
|
||||
backdrop-filter: blur(8px);
|
||||
border: 1px solid var(--drive-border, rgba(255, 255, 255, 0.09));
|
||||
box-shadow: 0 4px 12px var(--drive-shadow, rgba(0, 0, 0, 0.3)), inset 0 1px 0 rgba(255, 255, 255, 0.05);
|
||||
transition: transform var(--transition-fast), border-color var(--transition-fast), box-shadow var(--transition-fast);
|
||||
transition: border-color var(--transition-fast), box-shadow var(--transition-fast);
|
||||
}
|
||||
|
||||
.source-badge:hover {
|
||||
transform: translateY(-1px) scale(1.04);
|
||||
transform: scale(1.04);
|
||||
border-color: var(--drive-color);
|
||||
box-shadow: 0 0 12px var(--drive-shadow-strong, var(--accent-glow));
|
||||
}
|
||||
@@ -282,6 +281,14 @@
|
||||
--drive-shadow: rgba(245, 107, 118, 0.15);
|
||||
--drive-shadow-strong: rgba(245, 107, 118, 0.4);
|
||||
}
|
||||
.source-badge[data-kind="p123"] {
|
||||
--drive-color: var(--drive-p123);
|
||||
--drive-bg: rgba(34, 184, 200, 0.12);
|
||||
--drive-text: #98edf5;
|
||||
--drive-border: rgba(34, 184, 200, 0.35);
|
||||
--drive-shadow: rgba(34, 184, 200, 0.15);
|
||||
--drive-shadow-strong: rgba(34, 184, 200, 0.4);
|
||||
}
|
||||
.source-badge[data-kind="pikpak"] {
|
||||
--drive-color: var(--drive-pikpak);
|
||||
--drive-bg: rgba(138, 109, 255, 0.12);
|
||||
@@ -446,6 +453,7 @@
|
||||
|
||||
.video-card__source[data-kind="quark"] { --source-color: var(--drive-quark); }
|
||||
.video-card__source[data-kind="p115"] { --source-color: var(--drive-p115); }
|
||||
.video-card__source[data-kind="p123"] { --source-color: var(--drive-p123); }
|
||||
.video-card__source[data-kind="pikpak"] { --source-color: var(--drive-pikpak); }
|
||||
.video-card__source[data-kind="wopan"] { --source-color: var(--drive-wopan); }
|
||||
.video-card__source[data-kind="onedrive"] { --source-color: var(--drive-onedrive); }
|
||||
@@ -455,7 +463,11 @@
|
||||
* Empty / Loading / Skeleton
|
||||
* ========================================================= */
|
||||
.video-grid-empty {
|
||||
min-height: 220px;
|
||||
padding: var(--space-9) var(--space-5);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
text-align: center;
|
||||
color: var(--text-muted);
|
||||
background: var(--bg-surface);
|
||||
|
||||
@@ -333,6 +333,12 @@
|
||||
color: var(--drive-p115);
|
||||
}
|
||||
|
||||
.vd-meta__chip[data-tone="p123"] {
|
||||
background: rgba(34, 184, 200, 0.14);
|
||||
border-color: rgba(34, 184, 200, 0.3);
|
||||
color: var(--drive-p123);
|
||||
}
|
||||
|
||||
.vd-meta__chip[data-tone="pikpak"] {
|
||||
background: rgba(138, 109, 255, 0.14);
|
||||
border-color: rgba(138, 109, 255, 0.3);
|
||||
@@ -883,7 +889,6 @@
|
||||
|
||||
.vd-rail__link:hover {
|
||||
background: rgba(255, 255, 255, 0.04);
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.vd-rail__link:focus-visible {
|
||||
|
||||
@@ -8,6 +8,9 @@ const drivesPageSource = readFileSync(
|
||||
);
|
||||
|
||||
test("spider91 drive form does not expose advanced crawler credentials", () => {
|
||||
assert.match(drivesPageSource, /key: "proxy"/);
|
||||
assert.match(drivesPageSource, /label: "代理地址(可选)"/);
|
||||
assert.match(drivesPageSource, /支持 http:\/\/、https:\/\/、socks5:\/\/ 或 socks5h:\/\//);
|
||||
assert.doesNotMatch(drivesPageSource, /target_new/);
|
||||
assert.doesNotMatch(drivesPageSource, /crawl_hour/);
|
||||
assert.doesNotMatch(drivesPageSource, /python_path/);
|
||||
@@ -24,8 +27,14 @@ test("spider91 upload target uses explicit local-save option instead of auto tar
|
||||
assert.doesNotMatch(drivesPageSource, /自动模式/);
|
||||
});
|
||||
|
||||
test("drive form shows a root directory id field for all drive kinds", () => {
|
||||
test("drive form hides root directory id for localstorage and spider91", () => {
|
||||
assert.match(drivesPageSource, /<label>根目录 ID<\/label>/);
|
||||
assert.match(
|
||||
drivesPageSource,
|
||||
/function usesRootDirectoryID\(kind: Kind\): boolean \{\s*return kind !== "localstorage" && kind !== "spider91";\s*\}/
|
||||
);
|
||||
assert.match(drivesPageSource, /\{usesRootDirectoryID\(form\.kind\) && \(/);
|
||||
assert.match(drivesPageSource, /\{usesRootDirectoryID\(d\.kind\) && \(/);
|
||||
assert.match(drivesPageSource, /placeholder=\{rootIdPlaceholder\(form\.kind\)\}/);
|
||||
assert.doesNotMatch(drivesPageSource, /扫描起点目录 ID/);
|
||||
assert.doesNotMatch(drivesPageSource, /set\("scanRootId"/);
|
||||
@@ -94,6 +103,7 @@ test("localstorage drive form asks for a server directory path", () => {
|
||||
assert.match(fields, /key: "path"/);
|
||||
assert.match(fields, /label: "本地目录路径"/);
|
||||
assert.match(drivesPageSource, /if \(kind === "localstorage"\) return "\/"/);
|
||||
assert.match(drivesPageSource, /kind !== "localstorage" && kind !== "spider91"/);
|
||||
});
|
||||
|
||||
test("drive type selector keeps primary source order", () => {
|
||||
@@ -101,10 +111,11 @@ test("drive type selector keeps primary source order", () => {
|
||||
drivesPageSource.matchAll(/<option value="([^"]+)">([^<]+)<\/option>/g),
|
||||
(match) => ({ value: match[1], label: match[2] })
|
||||
);
|
||||
const driveOptions = options.slice(0, 8);
|
||||
const driveOptions = options.slice(0, 9);
|
||||
|
||||
assert.deepEqual(driveOptions, [
|
||||
{ value: "p115", label: "115 网盘" },
|
||||
{ value: "p123", label: "123 云盘" },
|
||||
{ value: "pikpak", label: "PikPak" },
|
||||
{ value: "onedrive", label: "OneDrive" },
|
||||
{ value: "googledrive", label: "Google Drive" },
|
||||
|
||||
Reference in New Issue
Block a user