fix: hash long local media asset filenames

This commit is contained in:
nianzhibai
2026-06-03 20:35:53 +08:00
parent 53327c9b8e
commit 8f0d52aec4
10 changed files with 228 additions and 31 deletions
+30 -8
View File
@@ -36,6 +36,7 @@ import (
"github.com/video-site/backend/internal/drives/spider91"
"github.com/video-site/backend/internal/drives/wopan"
"github.com/video-site/backend/internal/fingerprint"
"github.com/video-site/backend/internal/mediaasset"
"github.com/video-site/backend/internal/nightly"
"github.com/video-site/backend/internal/preview"
"github.com/video-site/backend/internal/proxy"
@@ -1421,9 +1422,9 @@ func removeLocalVideoAssets(localDir string, v *catalog.Video) error {
}
candidates := []string{
v.PreviewLocal,
filepath.Join(localDir, v.ID+".mp4"),
filepath.Join(localDir, "thumbs", v.ID+".jpg"),
}
candidates = append(candidates, mediaasset.PreviewPathCandidates(localDir, v.ID)...)
candidates = append(candidates, mediaasset.ThumbnailPathCandidates(localDir, v.ID)...)
seen := make(map[string]struct{}, len(candidates))
for _, candidate := range candidates {
clean, ok := localPathWithin(localDir, candidate)
@@ -1540,14 +1541,35 @@ func cleanupDuplicateThumbnailAsset(localDir, videoID, thumbnailURL string) (cle
if thumbnailURL != "/p/thumb/"+videoID {
return false, false, false, nil
}
clean, ok := localPathWithin(localDir, filepath.Join(localDir, "thumbs", videoID+".jpg"))
if !ok {
candidates := mediaasset.ThumbnailPathCandidates(localDir, videoID)
seen := make(map[string]struct{}, len(candidates))
anyChecked := false
allMissing := true
for _, candidate := range candidates {
clean, ok := localPathWithin(localDir, candidate)
if !ok {
continue
}
if _, ok := seen[clean]; ok {
continue
}
seen[clean] = struct{}{}
anyChecked = true
removedOne, missingOne, removeErr := removeRegularFileIfExists(clean)
if removeErr != nil {
return false, false, false, removeErr
}
if removedOne {
removed = true
}
if !missingOne {
allMissing = false
}
}
if !anyChecked {
return false, false, false, nil
}
removed, missing, err = removeRegularFileIfExists(clean)
if err != nil {
return false, false, false, err
}
missing = allMissing && !removed
return true, removed, missing, nil
}
+13 -7
View File
@@ -25,6 +25,7 @@ import (
"github.com/video-site/backend/internal/drives/localstorage"
"github.com/video-site/backend/internal/drives/localupload"
"github.com/video-site/backend/internal/drives/spider91"
"github.com/video-site/backend/internal/mediaasset"
"github.com/video-site/backend/internal/proxy"
)
@@ -860,14 +861,19 @@ func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) {
func (s *Server) handleThumb(w http.ResponseWriter, r *http.Request) {
videoID := chi.URLParam(r, "videoID")
// 直接读本地 thumbs 目录中 <videoID>.jpg
path := filepath.Join(s.LocalDir, "thumbs", videoID+".jpg")
clean := filepath.Clean(path)
if !strings.HasPrefix(clean, filepath.Clean(s.LocalDir)) {
http.Error(w, "invalid path", http.StatusForbidden)
return
var clean string
for _, path := range mediaasset.ThumbnailPathCandidates(s.LocalDir, videoID) {
candidate := filepath.Clean(path)
if !strings.HasPrefix(candidate, filepath.Clean(s.LocalDir)) {
http.Error(w, "invalid path", http.StatusForbidden)
return
}
if _, err := os.Stat(candidate); err == nil {
clean = candidate
break
}
}
if _, err := os.Stat(clean); err != nil {
if clean == "" {
w.Header().Set("Cache-Control", "no-store")
http.NotFound(w, r)
return
+29
View File
@@ -17,6 +17,7 @@ import (
"github.com/go-chi/chi/v5"
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/mediaasset"
"github.com/video-site/backend/internal/proxy"
)
@@ -552,6 +553,34 @@ func TestHandlePreviewIgnoresRemotePreviewFileIDAndServesLocalFile(t *testing.T)
}
}
func TestHandleThumbServesHashedPathForLongVideoID(t *testing.T) {
localDir := t.TempDir()
longID := "localstorage-" + strings.Repeat("x", 240)
thumbPath := mediaasset.ThumbnailPath(localDir, longID)
if err := os.MkdirAll(filepath.Dir(thumbPath), 0o755); err != nil {
t.Fatalf("mkdir thumb dir: %v", err)
}
if err := os.WriteFile(thumbPath, []byte("thumb-bytes"), 0o644); err != nil {
t.Fatalf("write thumb: %v", err)
}
server := &Server{
LocalDir: localDir,
Proxy: proxy.New(proxy.NewRegistry()),
}
req := requestWithRouteParam(http.MethodGet, "/p/thumb/"+longID, "videoID", longID, strings.NewReader(``))
rr := httptest.NewRecorder()
server.handleThumb(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
if rr.Body.String() != "thumb-bytes" {
t.Fatalf("body = %q, want thumb bytes", rr.Body.String())
}
}
func TestHandleTagsReturnsUnifiedTagPool(t *testing.T) {
ctx := context.Background()
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
+2 -1
View File
@@ -21,6 +21,7 @@ import (
"time"
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/mediaasset"
"golang.org/x/net/proxy"
)
@@ -525,7 +526,7 @@ func (c *Crawler) processOne(ctx context.Context, videoID string, item spiderVid
log.Printf("[spider91] drive=%s mkdir common thumbs: %v", c.cfg.Driver.ID(), err)
thumbReady = false
} else {
dst := filepath.Join(c.cfg.CommonThumbDir, videoID+".jpg")
dst := mediaasset.ThumbnailPathInDir(c.cfg.CommonThumbDir, videoID)
if err := copyFileAtomic(thumbPath, dst); err != nil {
log.Printf("[spider91] drive=%s viewkey=%s source_id=%s copy thumb to common dir: %v", c.cfg.Driver.ID(), viewkey, sourceID, err)
thumbReady = false
+69
View File
@@ -0,0 +1,69 @@
package mediaasset
import (
"crypto/sha256"
"encoding/hex"
"path/filepath"
"strings"
)
const maxPlainStemBytes = 180
const maxLegacyFilenameBytes = 255
func PreviewPath(localDir, videoID string) string {
return filepath.Join(localDir, PreviewFilename(videoID))
}
func ThumbnailPath(localDir, videoID string) string {
return ThumbnailPathInDir(filepath.Join(localDir, "thumbs"), videoID)
}
func ThumbnailPathInDir(thumbDir, videoID string) string {
return filepath.Join(thumbDir, ThumbnailFilename(videoID))
}
func PreviewPathCandidates(localDir, videoID string) []string {
return pathCandidates(localDir, videoID, ".mp4", "")
}
func ThumbnailPathCandidates(localDir, videoID string) []string {
return pathCandidates(localDir, videoID, ".jpg", "thumbs")
}
func PreviewFilename(videoID string) string {
return safeFilename(videoID, ".mp4")
}
func ThumbnailFilename(videoID string) string {
return safeFilename(videoID, ".jpg")
}
func pathCandidates(localDir, videoID, ext, subdir string) []string {
safe := safeFilename(videoID, ext)
legacy := videoID + ext
base := localDir
if subdir != "" {
base = filepath.Join(base, subdir)
}
out := []string{filepath.Join(base, safe)}
if legacy != safe && isPlainSafeStem(videoID) && len([]byte(legacy)) <= maxLegacyFilenameBytes {
out = append(out, filepath.Join(base, legacy))
}
return out
}
func safeFilename(videoID, ext string) string {
if isPlainSafeStem(videoID) && len([]byte(videoID))+len(ext) <= maxPlainStemBytes {
return videoID + ext
}
sum := sha256.Sum256([]byte(videoID))
return "v-" + hex.EncodeToString(sum[:]) + ext
}
func isPlainSafeStem(value string) bool {
value = strings.TrimSpace(value)
if value == "" || value == "." || value == ".." {
return false
}
return !strings.ContainsAny(value, `/\`+"\x00")
}
+56
View File
@@ -0,0 +1,56 @@
package mediaasset
import (
"path/filepath"
"strings"
"testing"
)
func TestFilenamesKeepShortSafeIDs(t *testing.T) {
if got := ThumbnailFilename("video-1"); got != "video-1.jpg" {
t.Fatalf("thumbnail filename = %q, want video-1.jpg", got)
}
if got := PreviewFilename("video-1"); got != "video-1.mp4" {
t.Fatalf("preview filename = %q, want video-1.mp4", got)
}
}
func TestFilenamesHashLongOrUnsafeIDs(t *testing.T) {
longID := "localstorage-" + strings.Repeat("x", 240)
got := ThumbnailFilename(longID)
if !strings.HasPrefix(got, "v-") || !strings.HasSuffix(got, ".jpg") {
t.Fatalf("thumbnail filename = %q, want hashed jpg", got)
}
if len([]byte(got)) >= len([]byte(longID+".jpg")) {
t.Fatalf("thumbnail filename = %q should be shorter than original id", got)
}
unsafe := ThumbnailFilename("dir/video")
if unsafe == "dir/video.jpg" || strings.ContainsAny(unsafe, `/\`) {
t.Fatalf("unsafe thumbnail filename = %q, want hashed single filename", unsafe)
}
}
func TestThumbnailPathCandidatesIncludeLegacyForHashedIDs(t *testing.T) {
localDir := t.TempDir()
mediumID := "localstorage-" + strings.Repeat("x", 190)
got := ThumbnailPathCandidates(localDir, mediumID)
if len(got) != 2 {
t.Fatalf("candidates = %#v, want hashed and legacy paths", got)
}
if got[0] != ThumbnailPath(localDir, mediumID) {
t.Fatalf("first candidate = %q, want safe path %q", got[0], ThumbnailPath(localDir, mediumID))
}
if filepath.Base(got[1]) != mediumID+".jpg" {
t.Fatalf("legacy candidate = %q, want original id jpg", got[1])
}
}
func TestThumbnailPathCandidatesSkipOverlongLegacy(t *testing.T) {
localDir := t.TempDir()
longID := "localstorage-" + strings.Repeat("x", 240)
got := ThumbnailPathCandidates(localDir, longID)
if len(got) != 1 {
t.Fatalf("candidates = %#v, want only hashed path for overlong id", got)
}
}
+6 -2
View File
@@ -21,6 +21,7 @@ import (
"github.com/video-site/backend/internal/catalog"
"github.com/video-site/backend/internal/drives"
"github.com/video-site/backend/internal/mediaasset"
)
type Config struct {
@@ -269,7 +270,7 @@ func (g *Generator) GenerateThumbnail(ctx context.Context, link *drives.StreamLi
if err := os.MkdirAll(dir, 0o755); err != nil {
return "", err
}
dst := filepath.Join(dir, videoID+".jpg")
dst := mediaasset.ThumbnailPath(g.cfg.LocalDir, videoID)
var lastErr error
offsets := thumbnailOffsets(duration)
@@ -966,7 +967,10 @@ func ffmpegOutputLooksRateLimited(output []byte) bool {
// MoveToLocal 把临时文件改名到稳定位置,返回最终路径
func (g *Generator) MoveToLocal(tmpPath, videoID string) (string, error) {
dst := filepath.Join(g.cfg.LocalDir, videoID+".mp4")
if err := os.MkdirAll(g.cfg.LocalDir, 0o755); err != nil {
return "", err
}
dst := mediaasset.PreviewPath(g.cfg.LocalDir, videoID)
if err := os.Rename(tmpPath, dst); err != nil {
// 跨盘 rename 可能失败,fallback 到 copy
if cerr := copyFile(tmpPath, dst); cerr != nil {
+2 -1
View File
@@ -33,6 +33,7 @@ import (
"github.com/video-site/backend/internal/drives/p115"
"github.com/video-site/backend/internal/drives/pikpak"
"github.com/video-site/backend/internal/drives/spider91"
"github.com/video-site/backend/internal/mediaasset"
)
// uploadTarget 是 migrator 调用目标 drive 的最小接口。任何一种"接收 spider91 上传"的
@@ -605,7 +606,7 @@ func (m *Migrator) preserveCrawledThumbnail(ctx context.Context, src *spider91.D
log.Printf("[spider91migrate] %s mkdir common thumbs: %v", v.ID, err)
return
}
dst := filepath.Join(commonDir, v.ID+".jpg")
dst := mediaasset.ThumbnailPathInDir(commonDir, v.ID)
if _, err := os.Stat(dst); err != nil {
if !os.IsNotExist(err) {
log.Printf("[spider91migrate] %s stat common thumb: %v", v.ID, err)
+11 -8
View File
@@ -5,6 +5,8 @@ import (
"os"
"path/filepath"
"strings"
"github.com/video-site/backend/internal/mediaasset"
)
type VideoAssetRef struct {
@@ -71,14 +73,15 @@ func Compute(
continue
}
driveUsage := out.Drives[ref.DriveID]
thumbPath := filepath.Join(localDir, "thumbs", ref.ID+".jpg")
if size, exists, err := regularFileSize(thumbPath); err != nil {
return Usage{}, err
} else if exists {
key := ref.DriveID + "\x00thumb\x00" + thumbPath
if !seen[key] {
driveUsage.ThumbnailBytes += size
seen[key] = true
for _, thumbPath := range mediaasset.ThumbnailPathCandidates(localDir, ref.ID) {
if size, exists, err := regularFileSize(thumbPath); err != nil {
return Usage{}, err
} else if exists {
key := ref.DriveID + "\x00thumb\x00" + thumbPath
if !seen[key] {
driveUsage.ThumbnailBytes += size
seen[key] = true
}
}
}
@@ -3,7 +3,10 @@ package storageusage
import (
"os"
"path/filepath"
"strings"
"testing"
"github.com/video-site/backend/internal/mediaasset"
)
func TestComputeCountsLocalThumbnailsAndTeasersByDrive(t *testing.T) {
@@ -13,6 +16,8 @@ func TestComputeCountsLocalThumbnailsAndTeasersByDrive(t *testing.T) {
}
writeSizedFile(t, filepath.Join(localDir, "thumbs", "video-a.jpg"), 3)
writeSizedFile(t, filepath.Join(localDir, "thumbs", "video-b.jpg"), 5)
longID := "localstorage-" + strings.Repeat("x", 240)
writeSizedFile(t, mediaasset.ThumbnailPath(localDir, longID), 13)
teaserA := filepath.Join(localDir, "video-a.mp4")
teaserB := filepath.Join(localDir, "video-b.mp4")
writeSizedFile(t, teaserA, 7)
@@ -24,6 +29,7 @@ func TestComputeCountsLocalThumbnailsAndTeasersByDrive(t *testing.T) {
{ID: "video-a", DriveID: "drive-a", PreviewLocal: teaserA},
{ID: "video-a-copy", DriveID: "drive-a", PreviewLocal: teaserA},
{ID: "video-b", DriveID: "drive-b", PreviewLocal: teaserB},
{ID: longID, DriveID: "drive-b"},
{ID: "outside", DriveID: "drive-b", PreviewLocal: outside},
{ID: "unknown-drive-video", DriveID: "missing", PreviewLocal: teaserB},
}, []string{"drive-a", "drive-b"}, func(string) (DiskStats, error) {
@@ -41,11 +47,11 @@ func TestComputeCountsLocalThumbnailsAndTeasersByDrive(t *testing.T) {
t.Fatalf("drive-a usage = %#v, want thumbnails=3 teaser=7 total=10", driveA)
}
driveB := got.Drives["drive-b"]
if driveB.ThumbnailBytes != 5 || driveB.TeaserBytes != 11 || driveB.TotalBytes != 16 {
t.Fatalf("drive-b usage = %#v, want thumbnails=5 teaser=11 total=16", driveB)
if driveB.ThumbnailBytes != 18 || driveB.TeaserBytes != 11 || driveB.TotalBytes != 29 {
t.Fatalf("drive-b usage = %#v, want thumbnails=18 teaser=11 total=29", driveB)
}
if got.ThumbnailBytes != 8 || got.TeaserBytes != 18 || got.TotalBytes != 26 {
t.Fatalf("totals = %#v, want thumbnails=8 teaser=18 total=26", got)
if got.ThumbnailBytes != 21 || got.TeaserBytes != 18 || got.TotalBytes != 39 {
t.Fatalf("totals = %#v, want thumbnails=21 teaser=18 total=39", got)
}
}