admin/91
1
0
Fork 0
mirror of https://github.com/nianzhibai/91.git synced 2026-06-15 08:45:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: cleanup stale pikpak videos

Browse Source
This commit is contained in:
nianzhibai
2026-05-14 23:27:26 +08:00
parent 164d96c940
commit 59c2826deb
6 changed files with 302 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/cmd/server/main.go
+101
View File
@@ -9,6 +9,7 @@ import (
"os/signal"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
"time"
@@ -507,6 +508,18 @@ func (a *App) runScan(ctx context.Context, driveID string) {
return
}
log.Printf("[scan] drive=%s done scanned=%d added=%d", driveID, stats.Scanned, stats.Added)
if drv.Kind() == "pikpak" {
if stats.Errors > 0 {
log.Printf("[cleanup] skip stale PikPak cleanup for drive=%s: scan had %d directory errors", driveID, stats.Errors)
} else {
removed, err := a.cleanupMissingDriveVideos(ctx, driveID, stats.SeenFileIDs, stats.VisitedDirIDs, startID == drv.RootID())
if err != nil {
log.Printf("[cleanup] stale PikPak cleanup drive=%s error: %v", driveID, err)
} else if removed > 0 {
log.Printf("[cleanup] removed %d stale PikPak videos for drive=%s", removed, driveID)
}
}
}
if thumbWorker != nil {
a.enqueueThumbnails(ctx, driveID, thumbWorker)
}
@@ -515,6 +528,94 @@ func (a *App) runScan(ctx context.Context, driveID string) {
}
}
func (a *App) cleanupMissingDriveVideos(ctx context.Context, driveID string, liveFileIDs map[string]struct{}, visitedDirIDs map[string]struct{}, fullDriveScan bool) (int, error) {
items, err := a.cat.ListVideosByDrive(ctx, driveID)
if err != nil {
return 0, err
}
localDir := ""
if a.cfg != nil {
localDir = a.cfg.Storage.LocalPreviewDir
}
removed := 0
for _, v := range items {
if _, ok := liveFileIDs[v.FileID]; ok {
continue
}
if !fullDriveScan {
if _, ok := visitedDirIDs[v.ParentID]; !ok {
continue
}
}
if err := removeLocalVideoAssets(localDir, v); err != nil {
return removed, fmt.Errorf("remove local assets for %s: %w", v.ID, err)
}
if err := a.cat.DeleteVideo(ctx, v.ID); err != nil {
return removed, fmt.Errorf("delete catalog video %s: %w", v.ID, err)
}
removed++
}
return removed, nil
}
func removeLocalVideoAssets(localDir string, v *catalog.Video) error {
if localDir == "" || v == nil || v.ID == "" {
return nil
}
candidates := []string{
v.PreviewLocal,
filepath.Join(localDir, v.ID+".mp4"),
filepath.Join(localDir, "thumbs", v.ID+".jpg"),
filepath.Join(localDir, "transcodes", v.ID+".mp4"),
filepath.Join(localDir, "transcodes", v.ID+".tmp.mp4"),
}
seen := make(map[string]struct{}, len(candidates))
for _, candidate := range candidates {
clean, ok := localPathWithin(localDir, candidate)
if !ok {
continue
}
if _, ok := seen[clean]; ok {
continue
}
seen[clean] = struct{}{}
info, err := os.Stat(clean)
if err != nil {
if os.IsNotExist(err) {
continue
}
return err
}
if !info.Mode().IsRegular() {
continue
}
if err := os.Remove(clean); err != nil && !os.IsNotExist(err) {
return err
}
}
return nil
}
func localPathWithin(root, path string) (string, bool) {
if strings.TrimSpace(root) == "" || strings.TrimSpace(path) == "" {
return "", false
}
rootAbs, err := filepath.Abs(root)
if err != nil {
return "", false
}
pathAbs, err := filepath.Abs(path)
if err != nil {
return "", false
}
rel, err := filepath.Rel(rootAbs, pathAbs)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
return "", false
}
return pathAbs, true
}
func (a *App) enqueueUploadedVideo(ctx context.Context, v *catalog.Video) {
if v == nil {
return
backend/cmd/server/main_test.go
+95
View File
@@ -2,11 +2,15 @@ package main
import (
"context"
"database/sql"
"io"
"os"
"path/filepath"
"testing"
"time"
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/config"
"github.com/video-site/backend/internal/drives"
"github.com/video-site/backend/internal/preview"
)
@@ -265,6 +269,97 @@ func TestShouldScanDriveSkipsLocalUpload(t *testing.T) {
}
}
func TestCleanupMissingPikPakVideosRemovesDatabaseRowsAndLocalAssets(t *testing.T) {
ctx := context.Background()
localDir := t.TempDir()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
obsoletePreview := filepath.Join(localDir, "obsolete.mp4")
obsoleteThumb := filepath.Join(localDir, "thumbs", "pikpak-PikPak-obsolete.jpg")
obsoleteTranscode := filepath.Join(localDir, "transcodes", "pikpak-PikPak-obsolete.mp4")
obsoleteTranscodeTmp := filepath.Join(localDir, "transcodes", "pikpak-PikPak-obsolete.tmp.mp4")
keptPreview := filepath.Join(localDir, "kept.mp4")
for _, path := range []string{obsoletePreview, obsoleteThumb, obsoleteTranscode, obsoleteTranscodeTmp, keptPreview} {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", path, err)
}
if err := os.WriteFile(path, []byte("asset"), 0o644); err != nil {
t.Fatalf("write %s: %v", path, err)
}
}
now := time.Now()
for _, v := range []*catalog.Video{
{
ID: "pikpak-PikPak-obsolete",
DriveID: "PikPak",
FileID: "obsolete",
Title: "Obsolete",
PreviewStatus: "ready",
PreviewLocal: obsoletePreview,
},
{
ID: "pikpak-PikPak-kept",
DriveID: "PikPak",
FileID: "kept",
Title: "Kept",
PreviewStatus: "ready",
PreviewLocal: keptPreview,
},
{
ID: "onedrive-OneDrive-obsolete",
DriveID: "OneDrive",
FileID: "obsolete",
Title: "Other Drive",
PreviewStatus: "ready",
},
} {
v.PublishedAt = now
v.CreatedAt = now
v.UpdatedAt = now
if err := cat.UpsertVideo(ctx, v); err != nil {
t.Fatalf("seed %s: %v", v.ID, err)
}
}
app := &App{
cfg: &config.Config{Storage: config.Storage{LocalPreviewDir: localDir}},
cat: cat,
}
removed, err := app.cleanupMissingDriveVideos(ctx, "PikPak", map[string]struct{}{"kept": {}}, nil, true)
if err != nil {
t.Fatalf("cleanup missing videos: %v", err)
}
if removed != 1 {
t.Fatalf("removed = %d, want 1", removed)
}
if _, err := cat.GetVideo(ctx, "pikpak-PikPak-obsolete"); err != sql.ErrNoRows {
t.Fatalf("obsolete video lookup error = %v, want sql.ErrNoRows", err)
}
if _, err := cat.GetVideo(ctx, "pikpak-PikPak-kept"); err != nil {
t.Fatalf("kept video missing after cleanup: %v", err)
}
if _, err := cat.GetVideo(ctx, "onedrive-OneDrive-obsolete"); err != nil {
t.Fatalf("other drive video missing after cleanup: %v", err)
}
for _, path := range []string{obsoletePreview, obsoleteThumb, obsoleteTranscode, obsoleteTranscodeTmp} {
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("obsolete asset %s still exists, stat err=%v", path, err)
}
}
if _, err := os.Stat(keptPreview); err != nil {
t.Fatalf("kept preview missing: %v", err)
}
}
type serverFakeTeaserGenerator struct{}
func (g *serverFakeTeaserGenerator) Probe(context.Context, *drives.StreamLink) (float64, error) {
backend/internal/catalog/catalog.go
+38
View File
@@ -354,6 +354,44 @@ func (c *Catalog) GetVideo(ctx context.Context, id string) (*Video, error) {
return scanVideo(row)
}
func (c *Catalog) ListVideosByDrive(ctx context.Context, driveID string) ([]*Video, error) {
rows, err := c.db.QueryContext(ctx,
`SELECT `+allVideoCols+` FROM videos WHERE drive_id = ? ORDER BY created_at ASC, id ASC`,
driveID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []*Video
for rows.Next() {
v, err := scanVideo(rows)
if err != nil {
return nil, err
}
out = append(out, v)
}
return out, rows.Err()
}
func (c *Catalog) DeleteVideo(ctx context.Context, id string) error {
tx, err := c.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if _, err := tx.ExecContext(ctx, `DELETE FROM video_tags WHERE video_id = ?`, id); err != nil {
return err
}
res, err := tx.ExecContext(ctx, `DELETE FROM videos WHERE id = ?`, id)
if err != nil {
return err
}
if rows, err := res.RowsAffected(); err == nil && rows == 0 {
return sql.ErrNoRows
}
return tx.Commit()
}
func (c *Catalog) FindVideoByContentHash(ctx context.Context, hash string) (*Video, error) {
hash = normalizeContentHash(hash)
if hash == "" {
backend/internal/scanner/scanner.go
+12 -3
View File
@@ -39,8 +39,11 @@ func New(cat *catalog.Catalog, drv drives.Drive, exts []string, maxDepth int, on
}
type Stats struct {
Scanned int
Added int
Scanned int
Added int
Errors int
SeenFileIDs map[string]struct{}
VisitedDirIDs map[string]struct{}
}
// Run 从 Drive.RootID 开始扫描
@@ -48,7 +51,10 @@ func (s *Scanner) Run(ctx context.Context, startDirID string) (Stats, error) {
if startDirID == "" {
startDirID = s.Drive.RootID()
}
stats := Stats{}
stats := Stats{
SeenFileIDs: make(map[string]struct{}),
VisitedDirIDs: make(map[string]struct{}),
}
if err := s.walk(ctx, startDirID, "", 0, &stats); err != nil {
return stats, err
}
@@ -62,6 +68,7 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, depth int, st
if err := ctx.Err(); err != nil {
return err
}
stats.VisitedDirIDs[dirID] = struct{}{}
entries, err := s.Drive.List(ctx, dirID)
if err != nil {
@@ -75,6 +82,7 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, depth int, st
continue
}
if err := s.walk(ctx, e.ID, e.Name, depth+1, stats); err != nil {
stats.Errors++
log.Printf("[scanner] walk %s error: %v", e.Name, err)
}
continue
@@ -88,6 +96,7 @@ func (s *Scanner) walk(ctx context.Context, dirID, dirName string, depth int, st
if e.Size <= 0 {
continue
}
stats.SeenFileIDs[e.ID] = struct{}{}
id := s.Drive.Kind() + "-" + s.Drive.ID() + "-" + e.ID
parsed := Parse(e.Name)
backend/internal/scanner/scanner_test.go
+55
View File
@@ -424,6 +424,61 @@ func TestRunSkipsDuplicateFileNamesWithSameSizeWhenHashesMissing(t *testing.T) {
}
}
func TestRunReportsSeenVideoFileIDsAndVisitedDirectories(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
if err != nil {
t.Fatalf("open catalog: %v", err)
}
t.Cleanup(func() {
if err := cat.Close(); err != nil {
t.Fatalf("close catalog: %v", err)
}
})
drv := &scannerTreeFakeDrive{
entries: map[string][]drives.Entry{
"root": {
{ID: "dir-1", Name: "Folder", IsDir: true},
{ID: "root-file", Name: "root.mp4", Size: 123},
{ID: "note", Name: "note.txt", Size: 123},
},
"dir-1": {
{ID: "nested-file", ParentID: "dir-1", Name: "nested.mp4", Size: 456},
{ID: "empty-video", ParentID: "dir-1", Name: "empty.mp4", Size: 0},
},
},
}
sc := New(cat, drv, []string{".mp4"}, 5, nil)
stats, err := sc.Run(ctx, "")
if err != nil {
t.Fatalf("scan: %v", err)
}
if _, ok := stats.SeenFileIDs["root-file"]; !ok {
t.Fatalf("seen file ids = %#v, want root-file", stats.SeenFileIDs)
}
if _, ok := stats.SeenFileIDs["nested-file"]; !ok {
t.Fatalf("seen file ids = %#v, want live non-empty videos", stats.SeenFileIDs)
}
if _, ok := stats.SeenFileIDs["note"]; ok {
t.Fatalf("seen file ids = %#v, want non-video entries excluded", stats.SeenFileIDs)
}
if _, ok := stats.SeenFileIDs["empty-video"]; ok {
t.Fatalf("seen file ids = %#v, want zero-size entries excluded", stats.SeenFileIDs)
}
if _, ok := stats.VisitedDirIDs["root"]; !ok {
t.Fatalf("visited dir ids = %#v, want root", stats.VisitedDirIDs)
}
if _, ok := stats.VisitedDirIDs["dir-1"]; !ok {
t.Fatalf("visited dir ids = %#v, want nested dir", stats.VisitedDirIDs)
}
if stats.Errors != 0 {
t.Fatalf("errors = %d, want 0", stats.Errors)
}
}
type scannerFakeDrive struct {
entries []drives.Entry
}
tsconfig.tsbuildinfo
+1 -1
View File
@@ -1 +1 @@
{"root":["./src/App.tsx","./src/main.tsx","./src/types.ts","./src/admin/AdminLayout.tsx","./src/admin/AuthContext.tsx","./src/admin/DrivesPage.tsx","./src/admin/LoginPage.tsx","./src/admin/Modal.tsx","./src/admin/PreviewToggle.tsx","./src/admin/RequireAuth.tsx","./src/admin/TagsPage.tsx","./src/admin/ToastContext.tsx","./src/admin/VideosPage.tsx","./src/admin/api.ts","./src/components/AppShell.tsx","./src/components/BackToTop.tsx","./src/components/CommentPanel.tsx","./src/components/Footer.tsx","./src/components/MainNav.tsx","./src/components/Pagination.tsx","./src/components/PreviewVideo.tsx","./src/components/PromoStrip.tsx","./src/components/RecommendedRail.tsx","./src/components/SearchPanel.tsx","./src/components/SectionHeader.tsx","./src/components/SortToolbar.tsx","./src/components/SubNav.tsx","./src/components/TagCloud.tsx","./src/components/TopBar.tsx","./src/components/VideoActions.tsx","./src/components/VideoCard.tsx","./src/components/VideoGrid.tsx","./src/components/VideoInfoPanel.tsx","./src/components/VideoPlayer.tsx","./src/data/categories.ts","./src/data/tags.ts","./src/data/videos.ts","./src/lib/format.ts","./src/lib/previewController.ts","./src/lib/previewIntent.ts","./src/lib/useInViewport.ts","./src/pages/HomePage.tsx","./src/pages/ListingPage.tsx","./src/pages/UploadPage.tsx","./src/pages/VideoDetailPage.tsx"],"version":"5.6.3"}
{"root":["./src/App.tsx","./src/main.tsx","./src/types.ts","./src/admin/AdminLayout.tsx","./src/admin/AuthContext.tsx","./src/admin/DrivesPage.tsx","./src/admin/LoginPage.tsx","./src/admin/Modal.tsx","./src/admin/PreviewToggle.tsx","./src/admin/RequireAuth.tsx","./src/admin/TagsPage.tsx","./src/admin/ToastContext.tsx","./src/admin/VideosPage.tsx","./src/admin/api.ts","./src/admin/storageFormat.ts","./src/components/AppShell.tsx","./src/components/BackToTop.tsx","./src/components/CommentPanel.tsx","./src/components/Footer.tsx","./src/components/MainNav.tsx","./src/components/Pagination.tsx","./src/components/PreviewVideo.tsx","./src/components/PromoStrip.tsx","./src/components/RecommendedRail.tsx","./src/components/SearchPanel.tsx","./src/components/SectionHeader.tsx","./src/components/SortToolbar.tsx","./src/components/SubNav.tsx","./src/components/TagCloud.tsx","./src/components/TopBar.tsx","./src/components/VideoActions.tsx","./src/components/VideoCard.tsx","./src/components/VideoGrid.tsx","./src/components/VideoInfoPanel.tsx","./src/components/VideoPlayer.tsx","./src/data/categories.ts","./src/data/tags.ts","./src/data/videos.ts","./src/lib/format.ts","./src/lib/previewController.ts","./src/lib/previewIntent.ts","./src/lib/uploadTitle.ts","./src/lib/useInViewport.ts","./src/pages/HomePage.tsx","./src/pages/ListingPage.tsx","./src/pages/UploadPage.tsx","./src/pages/VideoDetailPage.tsx"],"version":"5.6.3"}