admin/91
1
0
Fork 0
mirror of https://github.com/nianzhibai/91.git synced 2026-06-15 08:45:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add crawler preview generation toggle

Browse Source 
Expose per-crawler teaser settings on crawler cards and persist them through the admin API.\n\nWhen preview generation is disabled, crawler imports still create thumbnails and fingerprints while marking previews disabled and allowing migration without waiting for teaser files.\n\nPreserve the latest teaser setting after crawler runs so stale crawl state cannot overwrite a user toggle.
This commit is contained in:
nianzhibai
2026-06-14 17:52:29 +08:00
parent aa856db1f6
commit bb83277d62
15 changed files with 432 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/cmd/server/main.go
+21 -12
View File
@@ -1216,6 +1216,7 @@ func (a *App) attachScriptCrawler(d *catalog.Drive, drv *scriptcrawler.Driver) {
CommonThumbDir: a.commonThumbsDir(),
ProxyURL: proxyURL,
ConfigJSON: configJSON,
DisablePreview: !d.TeaserEnabled,
OnProgress: func(progress scriptcrawler.CrawlProgress) {
scanned := progress.Checked
if scanned < progress.TotalEntries {
@@ -3143,18 +3144,7 @@ func (a *App) runScriptCrawlerCrawlWithTaskContext(ctx context.Context, driveID
driveID, res.TargetNew, res.CandidateBudget, res.TotalEntries, res.NewVideos, res.Skipped, res.Failed, res.SeenSnapshot)
}
if d.Credentials == nil {
d.Credentials = make(map[string]string)
}
d.Credentials["last_crawl_at"] = strconv.FormatInt(time.Now().Unix(), 10)
if runErr != nil {
d.Status = "error"
d.LastError = runErr.Error()
} else {
d.Status = "ok"
d.LastError = ""
}
if err := a.cat.UpsertDrive(ctx, d); err != nil {
if err := a.updateScriptCrawlerRunState(ctx, driveID, runErr); err != nil {
log.Printf("[scriptcrawler] drive=%s update last_crawl_at: %v", driveID, err)
}
if err := ctx.Err(); err != nil {
@@ -3172,6 +3162,25 @@ func (a *App) runScriptCrawlerCrawlWithTaskContext(ctx context.Context, driveID
return runErr == nil
}
func (a *App) updateScriptCrawlerRunState(ctx context.Context, driveID string, runErr error) error {
d, err := a.cat.GetDrive(ctx, driveID)
if err != nil {
return err
}
if d.Credentials == nil {
d.Credentials = make(map[string]string)
}
d.Credentials["last_crawl_at"] = strconv.FormatInt(time.Now().Unix(), 10)
if runErr != nil {
d.Status = "error"
d.LastError = runErr.Error()
} else {
d.Status = "ok"
d.LastError = ""
}
return a.cat.UpsertDrive(ctx, d)
}
func (a *App) runSpider91MigrationAfterManualCrawl(ctx context.Context, driveID string) {
a.runCrawlerMigrationAfterManualCrawl(ctx, driveID)
}
backend/cmd/server/main_test.go
+47
View File
@@ -227,6 +227,53 @@ func TestRegisterPreviewWorkersBackfillsHistoricalFingerprints(t *testing.T) {
t.Fatalf("fingerprint status=%q sampled=%q, want ready with hash", got.FingerprintStatus, got.SampledSHA256)
}
func TestUpdateScriptCrawlerRunStatePreservesCurrentTeaserSwitch(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
if err := cat.UpsertDrive(ctx, &catalog.Drive{
ID: "crawler-id",
Kind: scriptcrawler.Kind,
Name: "Crawler",
RootID: "/",
Credentials: map[string]string{
"script_path": "/tmp/crawler.py",
"target_new": "10",
},
TeaserEnabled: false,
}); err != nil {
t.Fatalf("seed crawler drive: %v", err)
}
if err := cat.SetDriveTeaserEnabled(ctx, "crawler-id", true); err != nil {
t.Fatalf("toggle teaser: %v", err)
}
app := &App{cat: cat}
if err := app.updateScriptCrawlerRunState(ctx, "crawler-id", nil); err != nil {
t.Fatalf("update run state: %v", err)
}
got, err := cat.GetDrive(ctx, "crawler-id")
if err != nil {
t.Fatalf("get crawler drive: %v", err)
}
if !got.TeaserEnabled {
t.Fatal("teaserEnabled = false after run state update, want preserved true")
}
if got.Status != "ok" || got.LastError != "" {
t.Fatalf("status=%q lastError=%q, want ok with no error", got.Status, got.LastError)
}
if got.Credentials["last_crawl_at"] == "" || got.Credentials["target_new"] != "10" {
t.Fatalf("credentials after run state update = %#v", got.Credentials)
}
}
func TestStopDriveTasksCancelsQueuedTasksAndReplacesWorkers(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
backend/internal/api/admin.go
+17 -6
View File
@@ -469,7 +469,8 @@ func (a *AdminServer) handleListDrives(w http.ResponseWriter, r *http.Request) {
Status string `json:"status"`
LastError string `json:"lastError,omitempty"`
HasCredential bool `json:"hasCredential"`
// TeaserEnabled 控制是否给本盘生成预览视频/封面。前端用它在网盘列表/编辑表单展示开关状态。
// TeaserEnabled 控制是否给本盘生成预览视频；封面生成不受影响。
// 前端用它在网盘列表/编辑表单展示开关状态。
TeaserEnabled bool `json:"teaserEnabled"`
// SkipDirIDs 是用户在 admin 配置的"扫描跳过目录"集合（drive 侧目录 fileID）。
// 前端用它在"设置跳过目录"弹窗里回显已选项；JSON 字段名 camelCase 与
@@ -591,7 +592,7 @@ type upsertDriveReq struct {
// Deprecated: 扫描起点已固定为 rootId；保留字段只为兼容旧客户端请求体。
ScanRootID string `json:"scanRootId"`
Credentials map[string]string `json:"credentials"`
// TeaserEnabled 是 per-drive 预览视频/封面生成开关。
// TeaserEnabled 是 per-drive 预览视频生成开关；封面生成不受影响。
// 用 *bool 区分 "未传" / "传了 false"：未传时表示客户端不打算改这个字段，
// 沿用 catalog 现有值；新建时未传一律默认开启（true）。
TeaserEnabled *bool `json:"teaserEnabled,omitempty"`
@@ -690,6 +691,7 @@ type crawlerDTO struct {
Proxy string `json:"proxy,omitempty"`
TargetNew string `json:"targetNew,omitempty"`
UploadDriveID string `json:"uploadDriveId,omitempty"`
TeaserEnabled bool `json:"teaserEnabled"`
LastCrawlAt int64 `json:"lastCrawlAt,omitempty"`
ScanGenerationStatus GenerationStatus `json:"scanGenerationStatus"`
ThumbnailGenerationStatus GenerationStatus `json:"thumbnailGenerationStatus"`
@@ -717,6 +719,7 @@ type upsertCrawlerReq struct {
Proxy string `json:"proxy"`
TargetNew string `json:"targetNew"`
UploadDriveID string `json:"uploadDriveId"`
TeaserEnabled *bool `json:"teaserEnabled,omitempty"`
}
func (a *AdminServer) handleListCrawlers(w http.ResponseWriter, r *http.Request) {
@@ -778,6 +781,7 @@ func (a *AdminServer) crawlerDTOForDrive(d *catalog.Drive, assets catalog.Crawle
Proxy: strings.TrimSpace(d.Credentials["proxy"]),
TargetNew: strings.TrimSpace(d.Credentials["target_new"]),
UploadDriveID: strings.TrimSpace(d.Credentials["upload_drive_id"]),
TeaserEnabled: d.TeaserEnabled,
LastCrawlAt: lastCrawlAt,
ScanGenerationStatus: generation.Scan,
ThumbnailGenerationStatus: generation.Thumbnail,
@@ -864,6 +868,13 @@ func (a *AdminServer) handleUpsertCrawler(w http.ResponseWriter, r *http.Request
return
}
name := meta.Name
teaserEnabled := true
if existing != nil {
teaserEnabled = existing.TeaserEnabled
}
if body.TeaserEnabled != nil {
teaserEnabled = *body.TeaserEnabled
}
if id == "" {
generatedID, err := a.generateCrawlerID(r.Context(), name)
if err != nil {
@@ -879,15 +890,15 @@ func (a *AdminServer) handleUpsertCrawler(w http.ResponseWriter, r *http.Request
RootID: "/",
Credentials: merged,
Status: "disconnected",
TeaserEnabled: true,
}
if existing != nil {
d.TeaserEnabled = existing.TeaserEnabled
TeaserEnabled: teaserEnabled,
}
if err := a.Catalog.UpsertDrive(r.Context(), d); err != nil {
writeErr(w, http.StatusInternalServerError, err)
return
}
if existing != nil && existing.TeaserEnabled != teaserEnabled && a.OnTeaserEnabledChanged != nil {
a.OnTeaserEnabledChanged(id, teaserEnabled)
}
if a.OnDriveSaved != nil {
if err := a.OnDriveSaved(id); err != nil {
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "id": id, "warning": err.Error()})
backend/internal/api/admin_test.go
+62 -19
View File
@@ -944,7 +944,8 @@ func TestHandleListCrawlersOnlyIncludesCrawlerPageScripts(t *testing.T) {
"script_path": scriptPath,
"upload_drive_id": "p115-target",
},
Status: "ok",
Status: "ok",
TeaserEnabled: false,
},
{
ID: "p115-target",
@@ -1027,6 +1028,7 @@ func TestHandleListCrawlersOnlyIncludesCrawlerPageScripts(t *testing.T) {
Kind string `json:"kind"`
Proxy string `json:"proxy"`
UploadDriveID string `json:"uploadDriveId"`
TeaserEnabled bool `json:"teaserEnabled"`
LastCrawlAt int64 `json:"lastCrawlAt"`
TotalCrawled int `json:"totalCrawledCount"`
LocalVideos int `json:"localVideoCount"`
@@ -1038,11 +1040,12 @@ func TestHandleListCrawlersOnlyIncludesCrawlerPageScripts(t *testing.T) {
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
t.Fatalf("decode: %v", err)
}
byID := map[string]struct {
type crawlerListRow struct {
Name string
Kind string
Proxy string
UploadDriveID string
TeaserEnabled bool
LastCrawlAt int64
TotalCrawled int
LocalVideos int
@@ -1050,25 +1053,15 @@ func TestHandleListCrawlersOnlyIncludesCrawlerPageScripts(t *testing.T) {
ThumbnailReady int
TeaserReady int
FingerprintReady int
}{}
}
byID := map[string]crawlerListRow{}
for _, d := range got {
byID[d.ID] = struct {
Name string
Kind string
Proxy string
UploadDriveID string
LastCrawlAt int64
TotalCrawled int
LocalVideos int
MigratedVideo int
ThumbnailReady int
TeaserReady int
FingerprintReady int
}{
byID[d.ID] = crawlerListRow{
Name: d.Name,
Kind: d.Kind,
Proxy: d.Proxy,
UploadDriveID: d.UploadDriveID,
TeaserEnabled: d.TeaserEnabled,
LastCrawlAt: d.LastCrawlAt,
TotalCrawled: d.TotalCrawled,
LocalVideos: d.LocalVideos,
@@ -1096,6 +1089,9 @@ func TestHandleListCrawlersOnlyIncludesCrawlerPageScripts(t *testing.T) {
if byID["crawler-spider91"].UploadDriveID != "p115-target" {
t.Fatalf("uploadDriveId = %q, want p115-target", byID["crawler-spider91"].UploadDriveID)
}
if byID["crawler-spider91"].TeaserEnabled {
t.Fatal("teaserEnabled = true, want false from crawler drive")
}
if byID["crawler-spider91"].LastCrawlAt != 1800000000 {
t.Fatalf("lastCrawlAt = %d, want 1800000000", byID["crawler-spider91"].LastCrawlAt)
}
@@ -1171,7 +1167,8 @@ func TestHandleUpsertCrawlerRequiresScriptPath(t *testing.T) {
"id": "spider91-main",
"builtin": "spider91",
"scriptPath": "`+scriptPath+`",
"targetNew": "15"
"targetNew": "15",
"teaserEnabled": false
}`))
rr = httptest.NewRecorder()
srv.handleUpsertCrawler(rr, req)
@@ -1195,6 +1192,9 @@ func TestHandleUpsertCrawlerRequiresScriptPath(t *testing.T) {
if got.Credentials["script_path"] != scriptPath {
t.Fatalf("script_path = %q, want %q", got.Credentials["script_path"], scriptPath)
}
if got.TeaserEnabled {
t.Fatal("teaserEnabled = true, want false from request")
}
}
func TestHandleUpsertCrawlerGeneratesIDFromScriptName(t *testing.T) {
@@ -1277,12 +1277,21 @@ func TestHandleUpsertCrawlerPersistsAndValidatesUploadDrive(t *testing.T) {
t.Fatalf("seed drive %s: %v", d.ID, err)
}
}
srv := &AdminServer{Catalog: cat}
var teaserCallbackID string
var teaserCallbackEnabled bool
srv := &AdminServer{
Catalog: cat,
OnTeaserEnabledChanged: func(id string, enabled bool) {
teaserCallbackID = id
teaserCallbackEnabled = enabled
},
}
req := httptest.NewRequest(http.MethodPost, "/admin/api/crawlers", strings.NewReader(`{
"id": "crawler-upload",
"scriptPath": "`+scriptPath+`",
"uploadDriveId": "p115-target"
"uploadDriveId": "p115-target",
"teaserEnabled": false
}`))
rr := httptest.NewRecorder()
srv.handleUpsertCrawler(rr, req)
@@ -1296,6 +1305,12 @@ func TestHandleUpsertCrawlerPersistsAndValidatesUploadDrive(t *testing.T) {
if got.Credentials["upload_drive_id"] != "p115-target" {
t.Fatalf("upload_drive_id = %q, want p115-target", got.Credentials["upload_drive_id"])
}
if got.TeaserEnabled {
t.Fatal("teaserEnabled = true, want false")
}
if teaserCallbackID != "" {
t.Fatalf("teaser callback on create = %q, want none", teaserCallbackID)
}
req = httptest.NewRequest(http.MethodPost, "/admin/api/crawlers", strings.NewReader(`{
"id": "crawler-upload",
@@ -1314,6 +1329,34 @@ func TestHandleUpsertCrawlerPersistsAndValidatesUploadDrive(t *testing.T) {
if got.Credentials["upload_drive_id"] != "wopan-target" {
t.Fatalf("upload_drive_id = %q, want wopan-target", got.Credentials["upload_drive_id"])
}
if got.TeaserEnabled {
t.Fatal("teaserEnabled after edit without field = true, want preserved false")
}
if teaserCallbackID != "" {
t.Fatalf("teaser callback after preserved edit = %q, want none", teaserCallbackID)
}
req = httptest.NewRequest(http.MethodPost, "/admin/api/crawlers", strings.NewReader(`{
"id": "crawler-upload",
"scriptPath": "`+scriptPath+`",
"uploadDriveId": "wopan-target",
"teaserEnabled": true
}`))
rr = httptest.NewRecorder()
srv.handleUpsertCrawler(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("enable teaser status = %d, body = %s", rr.Code, rr.Body.String())
}
got, err = cat.GetDrive(ctx, "crawler-upload")
if err != nil {
t.Fatalf("get crawler after teaser enable: %v", err)
}
if !got.TeaserEnabled {
t.Fatal("teaserEnabled after explicit enable = false, want true")
}
if teaserCallbackID != "crawler-upload" || !teaserCallbackEnabled {
t.Fatalf("teaser callback = %q/%v, want crawler-upload/true", teaserCallbackID, teaserCallbackEnabled)
}
req = httptest.NewRequest(http.MethodPost, "/admin/api/crawlers", strings.NewReader(`{
"id": "crawler-upload",
backend/internal/catalog/catalog.go
+2 -2
View File
@@ -1937,7 +1937,7 @@ type Drive struct {
Credentials map[string]string `json:"credentials,omitempty"`
Status string `json:"status"`
LastError string `json:"lastError,omitempty"`
// TeaserEnabled 控制是否给本盘生成预览视频/封面。
// TeaserEnabled 控制是否给本盘生成预览视频；封面生成不受影响。
// 替代早期的全局 preview.enabled 开关；新建 drive 时 UpsertDrive 默认置 true。
TeaserEnabled bool `json:"teaserEnabled"`
// SkipDirIDs 是用户在管理后台为该盘选定的"扫描跳过目录"集合（网盘侧的目录 fileID）。
@@ -2070,7 +2070,7 @@ func (c *Catalog) DeleteDrive(ctx context.Context, id string) error {
return err
}
// SetDriveTeaserEnabled 切换某盘的预览视频/封面生成开关。
// SetDriveTeaserEnabled 切换某盘的预览视频生成开关。
//
// 与 UpsertDrive 的区别：只动 teaser_enabled + updated_at 一列，不要求调用方
// 重传 kind / name / credentials 等容易踩坑的字段。
backend/internal/catalog/schema.sql
+2 -2
View File
@@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS videos (
thumbnail_failures INTEGER DEFAULT 0, -- consecutive transient thumbnail generation failures
preview_file_id TEXT, -- deprecated: 旧版回写网盘后的预览视频 file id
preview_local TEXT, -- 本地预览视频路径（兜底）
preview_status TEXT DEFAULT 'pending', -- pending / ready / failed
preview_status TEXT DEFAULT 'pending', -- pending / ready / failed / disabled
transcode_status TEXT DEFAULT '', -- '' / pending / ready / skipped / failed（浏览器兼容性转码）
transcode_error TEXT DEFAULT '',
transcoded_file_id TEXT DEFAULT '', -- 转码产物在同一 drive 上的 fileID，播放源优先用它
@@ -121,7 +121,7 @@ CREATE TABLE IF NOT EXISTS drives (
credentials TEXT, -- JSON: cookie / refresh_token 等
status TEXT DEFAULT 'disconnected', -- disconnected / ok / error
last_error TEXT,
-- 是否给该盘生成预览视频/封面：1 开 / 0 关。
-- 是否给该盘生成预览视频：1 开 / 0 关。封面生成不受影响。
-- 替代了早期的全局 preview.enabled 设置（保留旧 setting 行不再读）。
teaser_enabled INTEGER NOT NULL DEFAULT 1,
-- 扫描时要跳过的目录 ID 集合（JSON array of string）。命中其中任意一个的目录及其
backend/internal/drives/scriptcrawler/crawler.go
+18 -1
View File
@@ -50,6 +50,7 @@ type CrawlerConfig struct {
CommonThumbDir string
ProxyURL string
ConfigJSON string
DisablePreview bool
HTTPClient *http.Client
DownloadTimeout time.Duration
OnProgress func(CrawlProgress)
@@ -562,6 +563,10 @@ func (c *Crawler) processItem(ctx context.Context, item Item) (bool, error) {
if quality == "" {
quality = "HD"
}
previewStatus := "pending"
if c.previewDisabled(ctx) {
previewStatus = "disabled"
}
v := &catalog.Video{
ID: videoID,
DriveID: c.cfg.Driver.ID(),
@@ -576,7 +581,7 @@ func (c *Crawler) processItem(ctx context.Context, item Item) (bool, error) {
Quality: quality,
Category: strings.TrimSpace(item.Category),
Description: strings.TrimSpace(item.Description),
PreviewStatus: "pending",
PreviewStatus: previewStatus,
PublishedAt: publishedAt,
CreatedAt: now,
UpdatedAt: now,
@@ -632,6 +637,18 @@ func (c *Crawler) processItem(ctx context.Context, item Item) (bool, error) {
return true, nil
}
func (c *Crawler) previewDisabled(ctx context.Context) bool {
if c == nil {
return false
}
if c.cfg.Catalog != nil && c.cfg.Driver != nil {
if d, err := c.cfg.Catalog.GetDrive(ctx, c.cfg.Driver.ID()); err == nil && d != nil {
return !d.TeaserEnabled
}
}
return c.cfg.DisablePreview
}
func (c *Crawler) materializeMedia(ctx context.Context, ref MediaRef, dst, referer string, required bool) (int64, error) {
if local := strings.TrimSpace(ref.LocalFile); local != "" {
return c.copyLocalOutput(local, dst)
backend/internal/drives/scriptcrawler/crawler_test.go
+122
View File
@@ -114,6 +114,128 @@ func TestCrawlerRunOnceImportsLocalFileAndSkipsExisting(t *testing.T) {
}
}
func TestCrawlerRunOnceMarksPreviewDisabledWhenConfigured(t *testing.T) {
ctx := context.Background()
tmp := t.TempDir()
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
if err := drv.Init(ctx); err != nil {
t.Fatalf("driver init: %v", err)
}
dummyScript := filepath.Join(tmp, "helper-script")
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
t.Fatalf("write dummy script: %v", err)
}
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
t.Fatalf("write helper wrapper: %v", err)
}
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
c := NewCrawler(CrawlerConfig{
Driver: drv,
Catalog: cat,
PythonPath: wrapper,
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
ScriptPath: dummyScript,
DisablePreview: true,
})
res, err := c.RunOnce(ctx, 1)
if err != nil {
t.Fatalf("run once: %v", err)
}
if res.NewVideos != 1 || res.Failed != 0 {
t.Fatalf("result = new:%d failed:%d, want 1/0", res.NewVideos, res.Failed)
}
v, err := cat.GetVideo(ctx, BuildVideoID("demo", "abc-123"))
if err != nil {
t.Fatalf("get video: %v", err)
}
if v.PreviewStatus != "disabled" {
t.Fatalf("preview status = %q, want disabled", v.PreviewStatus)
}
if v.FingerprintStatus != "ready" || v.SampledSHA256 == "" {
t.Fatalf("fingerprint status=%q sampled=%q, want ready and sampled hash", v.FingerprintStatus, v.SampledSHA256)
}
pending, err := cat.ListVideosByPreviewStatus(ctx, "demo", "pending", 0)
if err != nil {
t.Fatalf("list pending previews: %v", err)
}
if len(pending) != 0 {
t.Fatalf("pending previews = %d, want 0", len(pending))
}
}
func TestCrawlerRunOnceUsesCurrentDrivePreviewSwitch(t *testing.T) {
ctx := context.Background()
tmp := t.TempDir()
cat, err := catalog.Open(filepath.Join(tmp, "catalog.db"))
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
drv := New(Config{ID: "demo", RootDir: filepath.Join(tmp, "crawler")})
if err := drv.Init(ctx); err != nil {
t.Fatalf("driver init: %v", err)
}
if err := cat.UpsertDrive(ctx, &catalog.Drive{
ID: drv.ID(),
Kind: Kind,
Name: "Demo",
RootID: "/",
Credentials: map[string]string{"script_path": "/tmp/crawler.py"},
TeaserEnabled: true,
}); err != nil {
t.Fatalf("seed drive: %v", err)
}
dummyScript := filepath.Join(tmp, "helper-script")
if err := os.WriteFile(dummyScript, []byte("helper"), 0o755); err != nil {
t.Fatalf("write dummy script: %v", err)
}
wrapper := filepath.Join(tmp, "helper-wrapper.sh")
wrapperScript := fmt.Sprintf("#!/bin/sh\nexec %q -test.run=TestScriptCrawlerHelperProcess \"$@\"\n", os.Args[0])
if err := os.WriteFile(wrapper, []byte(wrapperScript), 0o755); err != nil {
t.Fatalf("write helper wrapper: %v", err)
}
t.Setenv("GO_WANT_SCRIPTCRAWLER_HELPER", "1")
c := NewCrawler(CrawlerConfig{
Driver: drv,
Catalog: cat,
PythonPath: wrapper,
FFprobePath: writeScriptCrawlerFFprobeStub(t, tmp, true),
ScriptPath: dummyScript,
DisablePreview: true,
})
res, err := c.RunOnce(ctx, 1)
if err != nil {
t.Fatalf("run once: %v", err)
}
if res.NewVideos != 1 || res.Failed != 0 {
t.Fatalf("result = new:%d failed:%d, want 1/0", res.NewVideos, res.Failed)
}
v, err := cat.GetVideo(ctx, BuildVideoID("demo", "abc-123"))
if err != nil {
t.Fatalf("get video: %v", err)
}
if v.PreviewStatus != "pending" {
t.Fatalf("preview status = %q, want pending from current drive switch", v.PreviewStatus)
}
}
func TestCrawlerRunOnceUsesSourceKindNamespace(t *testing.T) {
ctx := context.Background()
tmp := t.TempDir()
backend/internal/spider91migrate/migrator.go
+24 -19
View File
@@ -101,15 +101,16 @@ const (
)
type migrationPlan struct {
source Spider91LocalSource
row *catalog.Drive
sourceKinds []string
targetDriveID string
target uploadTarget
uploadDir string
keepLatestN int
requireAssetsReady bool
legacyBackfill bool
source Spider91LocalSource
row *catalog.Drive
sourceKinds []string
targetDriveID string
target uploadTarget
uploadDir string
keepLatestN int
requireAssetsReady bool
requirePreviewReady bool
legacyBackfill bool
}
// pikpakAdapter / p115Adapter / p123Adapter / onedriveAdapter / googledriveAdapter / wopanAdapter / guangyapanAdapter 把具体 driver 包装成 uploadTarget。
@@ -597,14 +598,15 @@ func (m *Migrator) migrationPlans(ctx context.Context) []migrationPlan {
continue
}
out = append(out, migrationPlan{
source: src,
row: row,
sourceKinds: crawlerSourceKindsForRow(row),
targetDriveID: resolvedID,
target: target,
uploadDir: scriptCrawlerUploadDir(row.ID),
keepLatestN: 0,
requireAssetsReady: true,
source: src,
row: row,
sourceKinds: crawlerSourceKindsForRow(row),
targetDriveID: resolvedID,
target: target,
uploadDir: scriptCrawlerUploadDir(row.ID),
keepLatestN: 0,
requireAssetsReady: true,
requirePreviewReady: row.TeaserEnabled,
})
case spider91.Kind:
if m.cfg.GetTargetDriveID == nil {
@@ -838,7 +840,7 @@ func (m *Migrator) migrateDrive(ctx context.Context, plan migrationPlan) (int, e
}
if plan.requireAssetsReady {
ready, err := m.crawlerVideoAssetsReady(ctx, v)
ready, err := m.crawlerVideoAssetsReady(ctx, v, plan.requirePreviewReady)
if err != nil {
log.Printf("[spider91migrate] %s check generated assets: %v", v.ID, err)
continue
@@ -914,7 +916,7 @@ func (m *Migrator) findVideoForLocalFile(ctx context.Context, plan migrationPlan
return nil
}
func (m *Migrator) crawlerVideoAssetsReady(ctx context.Context, v *catalog.Video) (bool, error) {
func (m *Migrator) crawlerVideoAssetsReady(ctx context.Context, v *catalog.Video, requirePreview bool) (bool, error) {
if v == nil {
return false, nil
}
@@ -922,6 +924,9 @@ func (m *Migrator) crawlerVideoAssetsReady(ctx context.Context, v *catalog.Video
if !fingerprintReady {
return false, nil
}
if !requirePreview {
return true, nil
}
if strings.EqualFold(strings.TrimSpace(v.PreviewStatus), "ready") {
return true, nil
}
backend/internal/spider91migrate/migrator_test.go
+49
View File
@@ -365,11 +365,19 @@ func seedScriptCrawlerDrive(t *testing.T, cat *catalog.Catalog, d *scriptcrawler
"script_path": "/tmp/crawler.py",
"upload_drive_id": uploadDriveID,
},
TeaserEnabled: true,
}); err != nil {
t.Fatalf("seed scriptcrawler drive: %v", err)
}
}
func setScriptCrawlerTeaserEnabled(t *testing.T, cat *catalog.Catalog, driveID string, enabled bool) {
t.Helper()
if err := cat.SetDriveTeaserEnabled(context.Background(), driveID, enabled); err != nil {
t.Fatalf("set scriptcrawler teaser enabled: %v", err)
}
}
func writeScriptCrawlerVideo(t *testing.T, cat *catalog.Catalog, d *scriptcrawler.Driver, sourceID, ext string, content []byte, readyAssets bool) string {
t.Helper()
fileID := sourceID + ext
@@ -587,6 +595,47 @@ func TestRunOnceSkipsScriptCrawlerVideoUntilPreviewAndFingerprintReady(t *testin
}
}
func TestRunOnceMigratesScriptCrawlerVideoWithoutPreviewWhenTeaserDisabled(t *testing.T) {
cat := setupCatalog(t)
src := setupScriptCrawler(t, "crawler-no-preview")
pp := newFakePikPak("pikpak-target", "pikpak-root-id")
seedScriptCrawlerDrive(t, cat, src, pp.ID())
setScriptCrawlerTeaserEnabled(t, cat, src.ID(), false)
reg := newFakeRegistry()
reg.Add(src)
reg.Add(pp)
id := writeScriptCrawlerVideo(t, cat, src, "fingerprint-ready", ".mp4", []byte("script video bytes"), false)
if err := cat.UpdateVideoFingerprint(context.Background(), id, "sampled-fingerprint-ready", "ready", ""); err != nil {
t.Fatalf("mark fingerprint ready: %v", err)
}
if err := cat.UpdatePreview(context.Background(), id, "", "disabled"); err != nil {
t.Fatalf("mark preview disabled: %v", err)
}
m := New(Config{Catalog: cat, Registry: reg})
m.runOnce(context.Background())
if pp.uploadCalls != 1 {
t.Fatalf("upload calls = %d, want 1 when preview generation is disabled", pp.uploadCalls)
}
got, err := cat.GetVideo(context.Background(), id)
if err != nil {
t.Fatalf("get migrated video: %v", err)
}
if got.DriveID != pp.ID() {
t.Fatalf("drive_id = %q, want %q", got.DriveID, pp.ID())
}
if got.PreviewStatus != "disabled" || got.FingerprintStatus != "ready" || got.SampledSHA256 == "" {
t.Fatalf("asset status after migration = preview %q fingerprint %q sampled %q, want disabled/ready/non-empty", got.PreviewStatus, got.FingerprintStatus, got.SampledSHA256)
}
videoPath, _ := src.VideoPath("fingerprint-ready.mp4")
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
t.Fatalf("local scriptcrawler video still exists or stat error %v", err)
}
}
func TestRunOnceBindsScriptCrawlerDuplicateToExistingTargetWithoutUpload(t *testing.T) {
cat := setupCatalog(t)
src := setupScriptCrawler(t, "crawler-duplicate")
src/admin/CrawlersPage.tsx
+37
View File
@@ -18,6 +18,8 @@ import {
Link as LinkIcon,
Pencil,
Plus,
Power,
PowerOff,
RefreshCw,
TestTube,
Trash2,
@@ -56,6 +58,7 @@ export function CrawlersPage() {
const [expandedId, setExpandedId] = useState("");
const [runningId, setRunningId] = useState("");
const [stoppingId, setStoppingId] = useState("");
const [togglingTeaserId, setTogglingTeaserId] = useState("");
// undefined = 编辑器关闭；null = 新建；其余 = 编辑已有爬虫
const [editorTarget, setEditorTarget] = useState<api.AdminCrawler | null | undefined>(undefined);
const [deleteTarget, setDeleteTarget] = useState<api.AdminCrawler | null>(null);
@@ -136,6 +139,23 @@ export function CrawlersPage() {
}
}
async function toggleTeaser(crawler: api.AdminCrawler) {
const next = !crawler.teaserEnabled;
setTogglingTeaserId(crawler.id);
setList((prev) => prev.map((item) => (item.id === crawler.id ? { ...item, teaserEnabled: next } : item)));
try {
const resp = await api.setDriveTeaserEnabled(crawler.id, next);
setList((prev) => prev.map((item) => (item.id === crawler.id ? { ...item, teaserEnabled: resp.teaserEnabled } : item)));
show(resp.teaserEnabled ? `已开启「${crawler.name}」预览视频生成` : `已关闭「${crawler.name}」预览视频生成`, "success");
await refresh(true);
} catch (e) {
setList((prev) => prev.map((item) => (item.id === crawler.id ? { ...item, teaserEnabled: crawler.teaserEnabled } : item)));
show(e instanceof Error ? e.message : "切换预览视频失败", "error");
} finally {
setTogglingTeaserId("");
}
}
async function confirmDelete() {
if (!deleteTarget) return;
setDeleting(true);
@@ -214,9 +234,11 @@ export function CrawlersPage() {
expanded={expandedId === crawler.id}
running={runningId === crawler.id}
stopping={stoppingId === crawler.id}
togglingTeaser={togglingTeaserId === crawler.id}
onToggle={() => setExpandedId(expandedId === crawler.id ? "" : crawler.id)}
onRun={() => run(crawler)}
onStop={() => stop(crawler)}
onToggleTeaser={() => toggleTeaser(crawler)}
onEdit={() => setEditorTarget(crawler)}
onDelete={() => setDeleteTarget(crawler)}
/>
@@ -290,9 +312,11 @@ function CrawlerRow({
expanded,
running,
stopping,
togglingTeaser,
onToggle,
onRun,
onStop,
onToggleTeaser,
onEdit,
onDelete,
}: {
@@ -300,9 +324,11 @@ function CrawlerRow({
expanded: boolean;
running: boolean;
stopping: boolean;
togglingTeaser: boolean;
onToggle: () => void;
onRun: () => void;
onStop: () => void;
onToggleTeaser: () => void;
onEdit: () => void;
onDelete: () => void;
}) {
@@ -343,6 +369,17 @@ function CrawlerRow({
<ChevronDown size={16} className="admin-crawler-row__chevron" />
</button>
<div className="admin-crawler-row__actions">
<button
className={`admin-btn admin-crawler-preview-card-toggle ${crawler.teaserEnabled ? "is-on" : ""}`}
type="button"
onClick={onToggleTeaser}
disabled={togglingTeaser}
aria-pressed={crawler.teaserEnabled}
title={crawler.teaserEnabled ? "关闭后，该爬虫新爬取的视频不再生成预览视频" : "开启后，该爬虫新爬取的视频会生成预览视频"}
>
{crawler.teaserEnabled ? <Power size={13} /> : <PowerOff size={13} />}
<span>{crawler.teaserEnabled ? "预览：开" : "预览：关"}</span>
</button>
{busy ? (
<button className="admin-btn is-stop" type="button" onClick={onStop} disabled={stopping}>
<CircleStop size={13} /> {stopping ? "停止中..." : "停止"}
src/admin/VideosPage.tsx
+1
View File
@@ -834,6 +834,7 @@ function VideoTitleCell({ video: v }: { video: api.AdminVideo }) {
function PreviewStatus({ s }: { s: string }) {
if (s === "ready") return <span className="admin-status is-ok">就绪</span>;
if (s === "failed") return <span className="admin-status is-error">失败</span>;
if (s === "disabled") return <span className="admin-status">已关闭</span>;
if (s === "skipped") return <span className="admin-status">跳过</span>;
return <span className="admin-status is-pending">待生成</span>;
}
src/admin/api.ts
+2 -1
View File
@@ -84,7 +84,7 @@ export type AdminDrive = {
status: string;
lastError?: string;
hasCredential: boolean;
/** 当前是否给该盘生成预览视频/封面（per-drive 开关，替代旧的全局 preview.enabled）。 */
/** 当前是否给该盘生成预览视频（per-drive 开关，替代旧的全局 preview.enabled；封面不受影响）。 */
teaserEnabled: boolean;
/**
* 用户在 admin 配置的"扫描跳过目录"集合（drive 侧目录 fileID 列表）。
@@ -212,6 +212,7 @@ export type AdminCrawler = {
proxy?: string;
targetNew?: string;
uploadDriveId?: string;
teaserEnabled: boolean;
lastCrawlAt?: number;
scanGenerationStatus?: DriveGenerationStatus;
thumbnailGenerationStatus?: DriveGenerationStatus;
src/styles/admin.css
+19
View File
@@ -768,6 +768,25 @@
gap: var(--space-2);
}
.admin-crawler-preview-card-toggle {
min-width: 96px;
padding-inline: 10px;
}
.admin-crawler-preview-card-toggle.is-on {
border-color: var(--accent);
background: var(--accent);
color: var(--text-on-accent);
box-shadow: none;
}
.admin-crawler-preview-card-toggle.is-on:hover:not(:disabled) {
border-color: var(--accent-hover);
background: var(--accent-hover);
color: var(--text-on-accent);
box-shadow: none;
}
.admin-crawler-row__delete {
padding-inline: 10px;
}
tests/adminDriveForm.test.ts
+9
View File
@@ -259,6 +259,13 @@ test("crawler management is a separate admin section", () => {
assert.match(crawlerPageSource, /测试通过/);
assert.match(crawlerPageSource, /Spider91UploadTargetField/);
assert.match(crawlerPageSource, /uploadDriveId/);
assert.match(crawlerPageSource, /api\.setDriveTeaserEnabled/);
assert.match(crawlerPageSource, /admin-crawler-preview-card-toggle/);
assert.match(crawlerPageSource, /预览：开/);
assert.match(crawlerPageSource, /预览：关/);
assert.match(crawlerPageSource, /aria-pressed=\{crawler\.teaserEnabled\}/);
assert.doesNotMatch(crawlerPageSource, /teaserEnabled: form\.teaserEnabled/);
assert.doesNotMatch(crawlerPageSource, /aria-pressed=\{form\.teaserEnabled\}/);
assert.match(crawlerPageSource, /UPLOAD_TARGET_KINDS/);
assert.doesNotMatch(crawlerPageSource, /新建脚本/);
assert.doesNotMatch(crawlerPageSource, /爬虫 ID/);
@@ -274,6 +281,8 @@ test("crawler management is a separate admin section", () => {
assert.doesNotMatch(crawlerPageSource, /内置 91/);
assert.match(apiSource, /type AdminCrawler/);
assert.match(apiSource, /uploadDriveId\?: string/);
assert.match(apiSource, /teaserEnabled: boolean/);
assert.doesNotMatch(apiSource, /teaserEnabled\?: boolean/);
assert.match(apiSource, /"\/crawlers"/);
assert.match(apiSource, /"\/crawlers\/import-file"/);
assert.match(apiSource, /"\/crawlers\/import-url"/);