admin/91
1
0
Fork 0
mirror of https://github.com/nianzhibai/91.git synced 2026-06-15 08:45:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

Exclude 115 movies folder and relax short teasers

Browse Source
This commit is contained in:
Codex
2026-05-20 13:46:11 +08:00
parent cda88ac1f5
commit d98d4e0aeb
6 changed files with 189 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2
View File
@@ -16,6 +16,7 @@
- 视频管理支持按网盘筛选、每页 100 条分页、每个网盘的 Teaser 已生成/待生成/失败统计、单条或全量重生 teaser、编辑标题/作者/分类/标签等元数据。
- 标签管理支持创建标签并自动分类已有视频；内置规则会把常见番号污染归并到 `AV` 等系统标签，降低标签列表噪声。
- 115 生成 teaser 时会顺序取链并分段生成，降低 CDN 403 / WAF 风控导致的大量失败概率；遇到疑似风控会进入冷却并保留任务为 `pending`。
- 115 扫描会跳过名为 `影视` 的目录及其全部子目录文件；这些文件不会新增到目录、不会计入扫描统计，已入库的同源文件会在后续扫描中清理。
## 快速开始
@@ -139,6 +140,7 @@ OneDrive 当前采用 OpenList 在线 API 的续期方式，不要求用户提
- Teaser：每段固定 3 秒；30 秒以下最多 3 段，30 秒及以上固定 4 段；长视频在 20% 到 80% 区间均匀取段
- 生成的封面和 teaser 都只保存在本地 `backend/data/previews/`，不会回写到网盘；旧数据中的 `preview_file_id` 会被忽略
- 极短视频会按可容纳的完整 3 秒片段数自动降级
- 30 秒以下短视频会尽量生成多段 teaser，但只要生成到至少 1 个有效片段就会视为成功，避免短视频随机切点无有效视频流时反复失败
- 首次失败的任务标 `preview_status = failed`，不再自动重试；管理后台可手动重新生成
- 封面或 teaser 生成遇到明确频率限制（如 429）时，对应 worker 固定冷却 5 分钟。
- 服务启动或网盘重新挂载时，如果 Teaser 开关已开启，会自动把历史 `pending` 任务重新入队，避免重启后停在“待生成”。
backend/cmd/server/main.go
+37
View File
@@ -605,6 +605,14 @@ func (a *App) runScan(ctx context.Context, driveID string) {
return
}
log.Printf("[scan] drive=%s done scanned=%d added=%d errors=%d", driveID, stats.Scanned, stats.Added, stats.Errors)
if drv.Kind() == "p115" && len(stats.ExcludedFileIDs) > 0 {
removed, err := a.cleanupExcludedDriveVideos(ctx, driveID, stats.ExcludedFileIDs)
if err != nil {
log.Printf("[cleanup] excluded 115 videos drive=%s error: %v", driveID, err)
} else if removed > 0 {
log.Printf("[cleanup] removed %d excluded 115 videos for drive=%s", removed, driveID)
}
}
if drv.Kind() == "pikpak" {
if stats.Errors > 0 {
log.Printf("[cleanup] skip stale PikPak cleanup for drive=%s: scan had %d directory errors", driveID, stats.Errors)
@@ -620,6 +628,35 @@ func (a *App) runScan(ctx context.Context, driveID string) {
a.enqueueDriveGeneration(ctx, driveID, worker, thumbWorker)
}
func (a *App) cleanupExcludedDriveVideos(ctx context.Context, driveID string, excludedFileIDs map[string]struct{}) (int, error) {
if len(excludedFileIDs) == 0 {
return 0, nil
}
items, err := a.cat.ListVideosByDrive(ctx, driveID)
if err != nil {
return 0, err
}
localDir := ""
if a.cfg != nil {
localDir = a.cfg.Storage.LocalPreviewDir
}
removed := 0
for _, v := range items {
if _, ok := excludedFileIDs[v.FileID]; !ok {
continue
}
if err := removeLocalVideoAssets(localDir, v); err != nil {
return removed, fmt.Errorf("remove local assets for %s: %w", v.ID, err)
}
if err := a.cat.DeleteVideo(ctx, v.ID); err != nil {
return removed, fmt.Errorf("delete catalog video %s: %w", v.ID, err)
}
removed++
}
return removed, nil
}
func (a *App) cleanupMissingDriveVideos(ctx context.Context, driveID string, liveFileIDs map[string]struct{}, visitedDirIDs map[string]struct{}, fullDriveScan bool) (int, error) {
items, err := a.cat.ListVideosByDrive(ctx, driveID)
if err != nil {
backend/internal/preview/ffmpeg.go
+12 -7
View File
@@ -517,6 +517,7 @@ func (g *Generator) generateSequential(ctx context.Context, duration float64, li
candidates := teaserCandidateStarts(duration, starts, eachSec)
targetSegments := len(starts)
requiredSegments := requiredTeaserSegments(duration, targetSegments)
var lastErr error
for i, start := range candidates {
if len(segmentPaths) >= targetSegments {
@@ -532,13 +533,7 @@ func (g *Generator) generateSequential(ctx context.Context, duration float64, li
}
segmentPaths = append(segmentPaths, seg)
}
if len(segmentPaths) == 0 {
if lastErr != nil {
return "", lastErr
}
return "", errors.New("no usable teaser segment")
}
if len(segmentPaths) < targetSegments {
if len(segmentPaths) < requiredSegments {
if lastErr != nil {
return "", fmt.Errorf("only generated %d/%d teaser segments: %w", len(segmentPaths), targetSegments, lastErr)
}
@@ -607,6 +602,16 @@ func (g *Generator) generateSequential(ctx context.Context, duration float64, li
return tmpPath, nil
}
func requiredTeaserSegments(duration float64, targetSegments int) int {
if targetSegments <= 0 {
return 0
}
if duration > 0 && duration < 30 {
return 1
}
return targetSegments
}
func (g *Generator) generateSingleSegment(ctx context.Context, index int, start, eachSec float64, linkForInput func(int) (*drives.StreamLink, error)) (string, error) {
ctx, cancel := context.WithTimeout(ctx, teaserSegmentTimeout)
defer cancel()
backend/internal/preview/ffmpeg_test.go
+18
View File
@@ -130,6 +130,24 @@ func TestTeaserSegmentFallbackRequiresPlannedSegmentCount(t *testing.T) {
}
}
func TestShortVideoRequiresOnlyOneUsableTeaserSegment(t *testing.T) {
if got := requiredTeaserSegments(12, 3); got != 1 {
t.Fatalf("required segments = %d, want 1 for short video", got)
}
if got := requiredTeaserSegments(29.999, 3); got != 1 {
t.Fatalf("required segments = %d, want 1 below 30 seconds", got)
}
}
func TestMediumAndLongVideosStillRequirePlannedTeaserSegments(t *testing.T) {
if got := requiredTeaserSegments(30, 4); got != 4 {
t.Fatalf("required segments = %d, want planned count at 30 seconds", got)
}
if got := requiredTeaserSegments(204, 4); got != 4 {
t.Fatalf("required segments = %d, want planned count for longer video", got)
}
}
func TestThumbnailOffsetsUseFiveSecondsWithEarlyFallbacks(t *testing.T) {
got := thumbnailOffsets()
want := []float64{5, 1, 0}
backend/internal/scanner/scanner.go
+47 -7
View File
@@ -39,11 +39,12 @@ func New(cat *catalog.Catalog, drv drives.Drive, exts []string, maxDepth int, on
}
type Stats struct {
Scanned int
Added int
Errors int
SeenFileIDs map[string]struct{}
VisitedDirIDs map[string]struct{}
Scanned int
Added int
Errors int
SeenFileIDs map[string]struct{}
VisitedDirIDs map[string]struct{}
ExcludedFileIDs map[string]struct{}
}
// Run 从 Drive.RootID 开始扫描
@@ -52,8 +53,9 @@ func (s *Scanner) Run(ctx context.Context, startDirID string) (Stats, error) {
startDirID = s.Drive.RootID()
}
stats := Stats{
SeenFileIDs: make(map[string]struct{}),
VisitedDirIDs: make(map[string]struct{}),
SeenFileIDs: make(map[string]struct{}),
VisitedDirIDs: make(map[string]struct{}),
ExcludedFileIDs: make(map[string]struct{}),
}
if err := s.walk(ctx, startDirID, "", 0, &stats); err != nil {
return stats, err
@@ -81,6 +83,13 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, depth int, st
if strings.EqualFold(e.Name, "previews") {
continue
}
if s.shouldExcludeDir(e.Name) {
if err := s.collectExcludedFiles(ctx, e.ID, depth+1, stats); err != nil {
stats.Errors++
log.Printf("[scanner] exclude %s error: %v", e.Name, err)
}
continue
}
if err := s.walk(ctx, e.ID, e.Name, depth+1, stats); err != nil {
stats.Errors++
log.Printf("[scanner] walk %s error: %v", e.Name, err)
@@ -180,6 +189,37 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, depth int, st
return nil
}
func (s *Scanner) shouldExcludeDir(name string) bool {
return s.Drive != nil &&
s.Drive.Kind() == "p115" &&
strings.TrimSpace(name) == "影视"
}
func (s *Scanner) collectExcludedFiles(ctx context.Context, dirID string, depth int, stats *Stats) error {
if depth >= s.MaxDepth {
return nil
}
if err := ctx.Err(); err != nil {
return err
}
entries, err := s.Drive.List(ctx, dirID)
if err != nil {
return fmt.Errorf("list excluded %s: %w", dirID, err)
}
for _, e := range entries {
if e.IsDir {
if err := s.collectExcludedFiles(ctx, e.ID, depth+1, stats); err != nil {
return err
}
continue
}
if e.ID != "" {
stats.ExcludedFileIDs[e.ID] = struct{}{}
}
}
return nil
}
func (s *Scanner) findDuplicate(ctx context.Context, hash, fileName string, size int64, currentID string) *catalog.Video {
if dup := s.findDuplicateByHash(ctx, hash, currentID); dup != nil {
return dup
backend/internal/scanner/scanner_test.go
+73 -2
View File
@@ -479,6 +479,65 @@ func TestRunReportsSeenVideoFileIDsAndVisitedDirectories(t *testing.T) {
}
}
func TestRunExcludesP115MoviesDirectoryFromStatsAndCatalog(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
drv := &scannerTreeFakeDrive{
kind: "p115",
id: "115",
entries: map[string][]drives.Entry{
"root": {
{ID: "movies-dir", Name: "影视", IsDir: true},
{ID: "normal-file", Name: "normal.mp4", Size: 123},
},
"movies-dir": {
{ID: "movie-file", ParentID: "movies-dir", Name: "movie.mp4", Size: 456},
{ID: "nested-dir", Name: "Nested", IsDir: true},
},
"nested-dir": {
{ID: "nested-movie-file", ParentID: "nested-dir", Name: "nested.mp4", Size: 789},
},
},
}
sc := New(cat, drv, []string{".mp4"}, 5, nil)
stats, err := sc.Run(ctx, "")
if err != nil {
t.Fatalf("scan: %v", err)
}
if stats.Scanned != 1 {
t.Fatalf("scanned = %d, want only non-excluded file counted", stats.Scanned)
}
if stats.Added != 1 {
t.Fatalf("added = %d, want only non-excluded file added", stats.Added)
}
if _, ok := stats.ExcludedFileIDs["movie-file"]; !ok {
t.Fatalf("excluded file ids = %#v, want movie-file", stats.ExcludedFileIDs)
}
if _, ok := stats.ExcludedFileIDs["nested-movie-file"]; !ok {
t.Fatalf("excluded file ids = %#v, want nested-movie-file", stats.ExcludedFileIDs)
}
if _, err := cat.GetVideo(ctx, "p115-115-movie-file"); err != sql.ErrNoRows {
t.Fatalf("excluded direct movie get error = %v, want sql.ErrNoRows", err)
}
if _, err := cat.GetVideo(ctx, "p115-115-nested-movie-file"); err != sql.ErrNoRows {
t.Fatalf("excluded nested movie get error = %v, want sql.ErrNoRows", err)
}
if _, err := cat.GetVideo(ctx, "p115-115-normal-file"); err != nil {
t.Fatalf("normal video was not added: %v", err)
}
}
type scannerFakeDrive struct {
entries []drives.Entry
}
@@ -506,11 +565,23 @@ func (d *scannerFakeDrive) EnsureDir(context.Context, string) (string, error) {
func (d *scannerFakeDrive) RootID() string { return "root" }
type scannerTreeFakeDrive struct {
kind string
id string
entries map[string][]drives.Entry
}
func (d *scannerTreeFakeDrive) Kind() string { return "fake" }
func (d *scannerTreeFakeDrive) ID() string { return "drive" }
func (d *scannerTreeFakeDrive) Kind() string {
if d.kind != "" {
return d.kind
}
return "fake"
}
func (d *scannerTreeFakeDrive) ID() string {
if d.id != "" {
return d.id
}
return "drive"
}
func (d *scannerTreeFakeDrive) Init(context.Context) error {
return nil
}