mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-15 00:44:30 +08:00
7e5e67697e
Implement a new GuangYaPan cloud drive integration across the backend, admin UI, playback proxy, and Spider91 migration flow. Backend changes:\n- Add a GuangYaPan drive driver with token refresh, QR/device login support, directory listing, stream link resolution, directory creation, rename/delete operations, OSS multipart upload, and upload task polling.\n- Register GuangYaPan as a supported storage kind in configuration, catalog normalization, admin APIs, public drive labels, and 302 playback redirects.\n- Allow Spider91 crawler uploads to target GuangYaPan through a dedicated migration adapter.\n- Add scan, thumbnail, preview, and fingerprint cooldown handling for GuangYaPan based on explicit HTTP status codes, Retry-After values, and structured provider codes instead of natural-language message matching.\n- Tighten existing provider cooldown detectors so OneDrive, Google Drive, 115, PikPak, 123pan, Wopan, and media workers avoid treating arbitrary response text as a rate-limit signal.\n- Keep large videos eligible for preview generation unless the user disables preview generation. Admin and tooling changes:\n- Add GuangYaPan as a selectable drive type with QR login UI and token/root-path credential fields.\n- Add crawler upload target support for GuangYaPan in the admin UI.\n- Add drive branding, labels, metadata display, and docs/config examples for GuangYaPan.\n- Include a standalone GuangYaPan QR login helper script for manual credential acquisition. Tests:\n- Add GuangYaPan driver, QR login, proxy, admin API, crawler upload target, fingerprint, cooldown, and form coverage.\n- Update rate-limit tests to assert that message-only throttling text no longer starts cooldowns.\n- Cover explicit HTTP status parsing through shared drive helper tests.
900 lines
27 KiB
Go
900 lines
27 KiB
Go
package preview
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/video-site/backend/internal/catalog"
|
|
"github.com/video-site/backend/internal/drives"
|
|
)
|
|
|
|
func TestThumbWorkerUpdatesThumbnailAndDurationWithoutChangingPreviewStatus(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "thumb-worker-video")
|
|
|
|
gen := &fakeThumbGenerator{probeDuration: 42}
|
|
drv := &previewFakeDrive{}
|
|
worker := NewThumbWorker(gen, cat, drv)
|
|
|
|
worker.process(ctx, video)
|
|
|
|
got, err := cat.GetVideo(ctx, video.ID)
|
|
if err != nil {
|
|
t.Fatalf("get video: %v", err)
|
|
}
|
|
if got.ThumbnailURL != "/p/thumb/"+video.ID {
|
|
t.Fatalf("thumbnail = %q, want generated thumb URL", got.ThumbnailURL)
|
|
}
|
|
if got.PreviewStatus != "pending" {
|
|
t.Fatalf("preview status = %q, want pending", got.PreviewStatus)
|
|
}
|
|
if got.DurationSeconds != 42 {
|
|
t.Fatalf("duration = %d, want probed duration", got.DurationSeconds)
|
|
}
|
|
if gen.thumbnailVideoID != video.ID {
|
|
t.Fatalf("thumbnail video id = %q, want %q", gen.thumbnailVideoID, video.ID)
|
|
}
|
|
if gen.thumbnailDuration != 42 {
|
|
t.Fatalf("thumbnail duration = %.1f, want probed duration", gen.thumbnailDuration)
|
|
}
|
|
if gen.probeCalls != 1 {
|
|
t.Fatalf("probe calls = %d, want 1 for thumbnail generation", gen.probeCalls)
|
|
}
|
|
if drv.streamFileID != video.FileID {
|
|
t.Fatalf("stream file id = %q, want %q", drv.streamFileID, video.FileID)
|
|
}
|
|
}
|
|
|
|
func TestThumbWorkerBackfillsDurationWhenThumbnailAlreadyExists(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "thumb-worker-existing-thumbnail")
|
|
video.ThumbnailURL = "/p/thumb/" + video.ID
|
|
if err := cat.UpsertVideo(ctx, video); err != nil {
|
|
t.Fatalf("update video: %v", err)
|
|
}
|
|
|
|
gen := &fakeThumbGenerator{probeDuration: 19}
|
|
drv := &previewFakeDrive{}
|
|
worker := NewThumbWorker(gen, cat, drv)
|
|
|
|
worker.process(ctx, video)
|
|
|
|
got, err := cat.GetVideo(ctx, video.ID)
|
|
if err != nil {
|
|
t.Fatalf("get video: %v", err)
|
|
}
|
|
if got.DurationSeconds != 19 {
|
|
t.Fatalf("duration = %d, want probed duration", got.DurationSeconds)
|
|
}
|
|
if got.ThumbnailURL != "/p/thumb/"+video.ID {
|
|
t.Fatalf("thumbnail = %q, want unchanged existing thumbnail", got.ThumbnailURL)
|
|
}
|
|
ready, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "ready", 0)
|
|
if err != nil {
|
|
t.Fatalf("list ready thumbnails: %v", err)
|
|
}
|
|
if len(ready) != 1 || ready[0].ID != video.ID {
|
|
t.Fatalf("ready thumbnails = %#v, want only %s", ready, video.ID)
|
|
}
|
|
if gen.probeCalls != 1 {
|
|
t.Fatalf("probe calls = %d, want 1", gen.probeCalls)
|
|
}
|
|
if gen.thumbnailVideoID != "" {
|
|
t.Fatalf("thumbnail generation video id = %q, want no regeneration", gen.thumbnailVideoID)
|
|
}
|
|
}
|
|
|
|
func TestThumbWorkerDoesNotGenerateThumbnailForSpider91OriginVideo(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "spider91-91-spider-1200001")
|
|
|
|
gen := &fakeThumbGenerator{probeDuration: 42}
|
|
drv := &previewFakeDrive{kind: "pikpak"}
|
|
worker := NewThumbWorker(gen, cat, drv)
|
|
|
|
worker.process(ctx, video)
|
|
|
|
got, err := cat.GetVideo(ctx, video.ID)
|
|
if err != nil {
|
|
t.Fatalf("get video: %v", err)
|
|
}
|
|
if got.ThumbnailURL != "" {
|
|
t.Fatalf("thumbnail = %q, want empty when crawled spider91 thumbnail is missing", got.ThumbnailURL)
|
|
}
|
|
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
|
|
if err != nil {
|
|
t.Fatalf("list failed thumbnails: %v", err)
|
|
}
|
|
if len(failed) != 1 || failed[0].ID != video.ID {
|
|
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
|
|
}
|
|
if gen.probeCalls != 0 || gen.generateCalls != 0 {
|
|
t.Fatalf("generator calls probe=%d generate=%d, want no ffmpeg work for spider91-origin thumbnail", gen.probeCalls, gen.generateCalls)
|
|
}
|
|
}
|
|
|
|
func TestThumbWorkerSkipsDurationBackfillWhenExistingThumbnailCannotBeProbed(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "thumb-worker-existing-thumbnail-probe-fails")
|
|
video.ThumbnailURL = "/p/thumb/" + video.ID
|
|
if err := cat.UpsertVideo(ctx, video); err != nil {
|
|
t.Fatalf("update video: %v", err)
|
|
}
|
|
|
|
gen := &fakeThumbGenerator{probeErr: errors.New("invalid media")}
|
|
drv := &previewFakeDrive{}
|
|
worker := NewThumbWorker(gen, cat, drv)
|
|
|
|
worker.process(ctx, video)
|
|
|
|
got, err := cat.GetVideo(ctx, video.ID)
|
|
if err != nil {
|
|
t.Fatalf("get video: %v", err)
|
|
}
|
|
if got.ThumbnailURL != "/p/thumb/"+video.ID {
|
|
t.Fatalf("thumbnail = %q, want unchanged existing thumbnail", got.ThumbnailURL)
|
|
}
|
|
if got.DurationSeconds != 0 {
|
|
t.Fatalf("duration = %d, want still unknown", got.DurationSeconds)
|
|
}
|
|
skipped, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "skipped", 0)
|
|
if err != nil {
|
|
t.Fatalf("list skipped thumbnails: %v", err)
|
|
}
|
|
if len(skipped) != 1 || skipped[0].ID != video.ID {
|
|
t.Fatalf("skipped thumbnails = %#v, want only %s", skipped, video.ID)
|
|
}
|
|
missing, err := cat.CountVideosNeedingThumbnail(ctx, video.DriveID)
|
|
if err != nil {
|
|
t.Fatalf("count videos needing thumbnail: %v", err)
|
|
}
|
|
if missing != 0 {
|
|
t.Fatalf("missing thumbnails = %d, want 0 after duration backfill is skipped", missing)
|
|
}
|
|
}
|
|
|
|
func TestThumbWorkerFallsBackToLocalPreviewWhenDriveStreamFails(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "thumb-worker-local-preview")
|
|
localPreview := filepath.Join(t.TempDir(), "preview.mp4")
|
|
if err := os.WriteFile(localPreview, []byte("preview"), 0o644); err != nil {
|
|
t.Fatalf("write local preview: %v", err)
|
|
}
|
|
video.PreviewLocal = localPreview
|
|
if err := cat.UpsertVideo(ctx, video); err != nil {
|
|
t.Fatalf("update video: %v", err)
|
|
}
|
|
|
|
gen := &fakeThumbGenerator{}
|
|
drv := &previewFakeDrive{streamErr: errors.New("remote unavailable")}
|
|
worker := NewThumbWorker(gen, cat, drv)
|
|
|
|
worker.process(ctx, video)
|
|
|
|
got, err := cat.GetVideo(ctx, video.ID)
|
|
if err != nil {
|
|
t.Fatalf("get video: %v", err)
|
|
}
|
|
if got.ThumbnailURL != "/p/thumb/"+video.ID {
|
|
t.Fatalf("thumbnail = %q, want generated thumb URL", got.ThumbnailURL)
|
|
}
|
|
if gen.thumbnailURL != localPreview {
|
|
t.Fatalf("thumbnail source = %q, want local preview %q", gen.thumbnailURL, localPreview)
|
|
}
|
|
}
|
|
|
|
func TestPreviewWorkerGeneratesTeaserWithoutReplacingExistingThumbnail(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "preview-worker-video")
|
|
video.ThumbnailURL = "https://thumbnail.example/original.jpg"
|
|
if err := cat.UpsertVideo(ctx, video); err != nil {
|
|
t.Fatalf("update video: %v", err)
|
|
}
|
|
|
|
gen := &fakeTeaserGenerator{}
|
|
drv := &previewFakeDrive{}
|
|
worker := NewWorker(gen, cat, drv)
|
|
|
|
worker.process(ctx, video)
|
|
|
|
got, err := cat.GetVideo(ctx, video.ID)
|
|
if err != nil {
|
|
t.Fatalf("get video: %v", err)
|
|
}
|
|
if got.ThumbnailURL != "https://thumbnail.example/original.jpg" {
|
|
t.Fatalf("thumbnail = %q, want existing thumbnail unchanged", got.ThumbnailURL)
|
|
}
|
|
if got.PreviewStatus != "ready" {
|
|
t.Fatalf("preview status = %q, want ready", got.PreviewStatus)
|
|
}
|
|
if got.PreviewLocal != "/tmp/"+video.ID+".mp4" {
|
|
t.Fatalf("preview local = %q, want moved teaser path", got.PreviewLocal)
|
|
}
|
|
}
|
|
|
|
func TestPreviewWorkerDeduplicatesQueuedVideos(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "preview-dedupe-video")
|
|
|
|
gen := &fakeTeaserGenerator{}
|
|
drv := &previewFakeDrive{}
|
|
worker := NewWorker(gen, cat, drv)
|
|
|
|
if !worker.EnqueueBlocking(ctx, video) {
|
|
t.Fatal("first enqueue returned false, want true")
|
|
}
|
|
if !worker.EnqueueBlocking(ctx, video) {
|
|
t.Fatal("duplicate enqueue returned false, want idempotent success")
|
|
}
|
|
if got := worker.Status().QueueLength; got != 1 {
|
|
t.Fatalf("queue length = %d, want 1 unique video", got)
|
|
}
|
|
|
|
queued := <-worker.ch
|
|
if !worker.Enqueue(video) {
|
|
t.Fatal("enqueue while the same video is reserved returned false, want idempotent success")
|
|
}
|
|
select {
|
|
case <-worker.ch:
|
|
t.Fatal("duplicate enqueue added another queued video")
|
|
default:
|
|
}
|
|
|
|
worker.processQueued(ctx, queued)
|
|
if !worker.Enqueue(video) {
|
|
t.Fatal("enqueue after processing returned false, want true")
|
|
}
|
|
}
|
|
|
|
func TestThumbWorkerDeduplicatesQueuedVideos(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "thumb-dedupe-video")
|
|
|
|
gen := &fakeThumbGenerator{}
|
|
drv := &previewFakeDrive{}
|
|
worker := NewThumbWorker(gen, cat, drv)
|
|
|
|
if !worker.Enqueue(video) {
|
|
t.Fatal("first enqueue returned false, want true")
|
|
}
|
|
if !worker.Enqueue(video) {
|
|
t.Fatal("duplicate enqueue returned false, want idempotent success")
|
|
}
|
|
if got := worker.Status().QueueLength; got != 1 {
|
|
t.Fatalf("queue length = %d, want 1 unique video", got)
|
|
}
|
|
|
|
queued := <-worker.ch
|
|
if !worker.Enqueue(video) {
|
|
t.Fatal("enqueue while the same thumbnail is reserved returned false, want idempotent success")
|
|
}
|
|
select {
|
|
case <-worker.ch:
|
|
t.Fatal("duplicate enqueue added another queued thumbnail")
|
|
default:
|
|
}
|
|
|
|
worker.processQueued(ctx, queued)
|
|
if !worker.Enqueue(video) {
|
|
t.Fatal("enqueue after release returned false, want true")
|
|
}
|
|
}
|
|
|
|
func TestPreviewWorkerRemovesPreviousLocalTeaserAfterNewTeaserIsReady(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "preview-cleanup-video")
|
|
oldPath := filepath.Join(t.TempDir(), "old-teaser.mp4")
|
|
if err := os.WriteFile(oldPath, []byte("old teaser"), 0o644); err != nil {
|
|
t.Fatalf("write old teaser: %v", err)
|
|
}
|
|
video.PreviewLocal = oldPath
|
|
video.PreviewStatus = "ready"
|
|
if err := cat.UpsertVideo(ctx, video); err != nil {
|
|
t.Fatalf("update video: %v", err)
|
|
}
|
|
|
|
gen := &fakeTeaserGenerator{
|
|
localPath: filepath.Join(t.TempDir(), "new-teaser.mp4"),
|
|
}
|
|
drv := &previewFakeDrive{}
|
|
worker := NewWorker(gen, cat, drv)
|
|
|
|
worker.process(ctx, video)
|
|
|
|
if _, err := os.Stat(oldPath); !os.IsNotExist(err) {
|
|
t.Fatalf("old teaser still exists or stat failed with unexpected error: %v", err)
|
|
}
|
|
got, err := cat.GetVideo(ctx, video.ID)
|
|
if err != nil {
|
|
t.Fatalf("get video: %v", err)
|
|
}
|
|
if got.PreviewLocal != gen.localPath {
|
|
t.Fatalf("preview local = %q, want %q", got.PreviewLocal, gen.localPath)
|
|
}
|
|
}
|
|
|
|
func TestPreviewWorkerNeverCallsDriveUploadOrEnsureDir(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "preview-local-only-video")
|
|
localPath := filepath.Join(t.TempDir(), "local-only-teaser.mp4")
|
|
gen := &fakeTeaserGenerator{localPath: localPath}
|
|
drv := &previewFakeDrive{}
|
|
worker := NewWorker(gen, cat, drv)
|
|
|
|
worker.process(ctx, video)
|
|
|
|
got, err := cat.GetVideo(ctx, video.ID)
|
|
if err != nil {
|
|
t.Fatalf("get video: %v", err)
|
|
}
|
|
if got.PreviewStatus != "ready" {
|
|
t.Fatalf("preview status = %q, want ready", got.PreviewStatus)
|
|
}
|
|
if got.PreviewLocal != localPath {
|
|
t.Fatalf("preview local = %q, want %q", got.PreviewLocal, localPath)
|
|
}
|
|
if got.PreviewFileID != "" {
|
|
t.Fatalf("preview file id = %q, want empty for local-only teaser", got.PreviewFileID)
|
|
}
|
|
if drv.ensureDirCalls != 0 {
|
|
t.Fatalf("ensure dir calls = %d, want 0 (teaser/cover must not write back to drive)", drv.ensureDirCalls)
|
|
}
|
|
if drv.uploadCalls != 0 {
|
|
t.Fatalf("upload calls = %d, want 0 (teaser/cover must not write back to drive)", drv.uploadCalls)
|
|
}
|
|
}
|
|
|
|
func TestPreviewWorkerGeneratesTeaserForLargeVideo(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "preview-large-video")
|
|
video.Size = 6 * 1024 * 1024 * 1024
|
|
if err := cat.UpsertVideo(ctx, video); err != nil {
|
|
t.Fatalf("update video: %v", err)
|
|
}
|
|
|
|
gen := &fakeTeaserGenerator{}
|
|
drv := &previewFakeDrive{}
|
|
worker := NewWorker(gen, cat, drv)
|
|
|
|
worker.process(ctx, video)
|
|
|
|
got, err := cat.GetVideo(ctx, video.ID)
|
|
if err != nil {
|
|
t.Fatalf("get video: %v", err)
|
|
}
|
|
if got.PreviewStatus != "ready" {
|
|
t.Fatalf("preview status = %q, want ready", got.PreviewStatus)
|
|
}
|
|
if drv.streamCalls != 1 {
|
|
t.Fatalf("stream calls = %d, want 1", drv.streamCalls)
|
|
}
|
|
if gen.generateCalls != 1 {
|
|
t.Fatalf("generate calls = %d, want 1", gen.generateCalls)
|
|
}
|
|
}
|
|
|
|
func TestPreviewWorkerRateLimitLeavesCurrentPendingAndSkipsNextVideo(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, first := seedPreviewTestVideo(t, "preview-rate-limit-1")
|
|
second := *first
|
|
second.ID = "preview-rate-limit-2"
|
|
second.FileID = "file-id-2"
|
|
if err := cat.UpsertVideo(ctx, &second); err != nil {
|
|
t.Fatalf("seed second video: %v", err)
|
|
}
|
|
|
|
gen := &fakeTeaserGenerator{
|
|
generateErr: &drives.RateLimitError{
|
|
Provider: "onedrive",
|
|
RetryAfter: 2 * time.Hour,
|
|
Err: errors.New("429 Too Many Requests"),
|
|
},
|
|
}
|
|
drv := &previewFakeDrive{}
|
|
worker := NewWorker(gen, cat, drv)
|
|
|
|
before := time.Now()
|
|
worker.process(ctx, first)
|
|
gotFirst, err := cat.GetVideo(ctx, first.ID)
|
|
if err != nil {
|
|
t.Fatalf("get first video: %v", err)
|
|
}
|
|
if gotFirst.PreviewStatus != "pending" {
|
|
t.Fatalf("first preview status = %q, want pending after rate limit", gotFirst.PreviewStatus)
|
|
}
|
|
if gen.generateCalls != 1 {
|
|
t.Fatalf("generate calls = %d, want 1", gen.generateCalls)
|
|
}
|
|
assertCooldownAround(t, worker.Status().CooldownUntil, before, 2*time.Hour)
|
|
|
|
gen.generateErr = nil
|
|
worker.process(ctx, &second)
|
|
gotSecond, err := cat.GetVideo(ctx, second.ID)
|
|
if err != nil {
|
|
t.Fatalf("get second video: %v", err)
|
|
}
|
|
if gotSecond.PreviewStatus != "pending" {
|
|
t.Fatalf("second preview status = %q, want pending while drive is cooling down", gotSecond.PreviewStatus)
|
|
}
|
|
if gen.generateCalls != 1 {
|
|
t.Fatalf("generate calls = %d, want second video skipped during cooldown", gen.generateCalls)
|
|
}
|
|
}
|
|
|
|
func TestThumbWorkerRateLimitHonorsRetryAfter(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "thumb-rate-limit")
|
|
|
|
gen := &fakeThumbGenerator{
|
|
generateErr: &drives.RateLimitError{
|
|
Provider: "media source",
|
|
RetryAfter: 2 * time.Hour,
|
|
Err: errors.New("429 Too Many Requests"),
|
|
},
|
|
}
|
|
drv := &previewFakeDrive{}
|
|
worker := NewThumbWorker(gen, cat, drv)
|
|
|
|
before := time.Now()
|
|
worker.process(ctx, video)
|
|
|
|
got, err := cat.GetVideo(ctx, video.ID)
|
|
if err != nil {
|
|
t.Fatalf("get video: %v", err)
|
|
}
|
|
if got.ThumbnailURL != "" {
|
|
t.Fatalf("thumbnail = %q, want unchanged after rate limit", got.ThumbnailURL)
|
|
}
|
|
assertCooldownAround(t, worker.Status().CooldownUntil, before, 2*time.Hour)
|
|
}
|
|
|
|
func TestThumbWorkerP115MessageOnlyErrorFailsWithoutCooldown(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "thumb-p115-message-only")
|
|
|
|
gen := &fakeThumbGenerator{
|
|
generateErr: errors.New("ffmpeg thumb: exit status 183, stderr: partial file Cannot determine format of input 0:0 after EOF"),
|
|
}
|
|
drv := &previewFakeDrive{kind: "p115"}
|
|
worker := NewThumbWorker(gen, cat, drv)
|
|
|
|
worker.process(ctx, video)
|
|
|
|
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
|
|
if err != nil {
|
|
t.Fatalf("list failed thumbnails: %v", err)
|
|
}
|
|
if len(failed) != 1 || failed[0].ID != video.ID {
|
|
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
|
|
}
|
|
if !worker.Status().CooldownUntil.IsZero() {
|
|
t.Fatalf("cooldown until = %s, want no cooldown for message-only media error", worker.Status().CooldownUntil)
|
|
}
|
|
if gen.generateCalls != 1 {
|
|
t.Fatalf("generate calls = %d, want 1", gen.generateCalls)
|
|
}
|
|
}
|
|
|
|
func TestThumbWorkerDoesNotRequeueP115MessageOnlyError(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "thumb-p115-no-requeue")
|
|
|
|
gen := &fakeThumbGenerator{
|
|
generateErr: errors.New("ffmpeg thumb: partial file Cannot determine format of input 0:0 after EOF"),
|
|
}
|
|
drv := &previewFakeDrive{kind: "p115"}
|
|
worker := NewThumbWorker(gen, cat, drv)
|
|
|
|
worker.processQueued(ctx, video)
|
|
|
|
select {
|
|
case queued := <-worker.ch:
|
|
t.Fatalf("unexpected requeued video id = %q", queued.ID)
|
|
default:
|
|
}
|
|
|
|
got, err := cat.GetVideo(ctx, video.ID)
|
|
if err != nil {
|
|
t.Fatalf("get video: %v", err)
|
|
}
|
|
if got.ThumbnailURL != "" {
|
|
t.Fatalf("thumbnail = %q, want empty after message-only failure", got.ThumbnailURL)
|
|
}
|
|
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
|
|
if err != nil {
|
|
t.Fatalf("list failed thumbnails: %v", err)
|
|
}
|
|
if len(failed) != 1 || failed[0].ID != video.ID {
|
|
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
|
|
}
|
|
}
|
|
|
|
func TestThumbWorkerPikPakMoovAtomErrorFailsWithoutCooldown(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "thumb-pikpak-missing-moov")
|
|
|
|
mediaErr := errors.New("ffprobe: exit status 1, stderr: moov atom not found Invalid data found when processing input")
|
|
gen := &fakeThumbGenerator{
|
|
probeErr: mediaErr,
|
|
generateErr: mediaErr,
|
|
}
|
|
drv := &previewFakeDrive{kind: "pikpak"}
|
|
worker := NewThumbWorker(gen, cat, drv)
|
|
|
|
worker.process(ctx, video)
|
|
|
|
failed, err := cat.ListVideosByThumbnailStatus(ctx, video.DriveID, "failed", 0)
|
|
if err != nil {
|
|
t.Fatalf("list failed thumbnails: %v", err)
|
|
}
|
|
if len(failed) != 1 || failed[0].ID != video.ID {
|
|
t.Fatalf("failed thumbnails = %#v, want only %s", failed, video.ID)
|
|
}
|
|
if !worker.Status().CooldownUntil.IsZero() {
|
|
t.Fatalf("cooldown until = %s, want no cooldown for invalid PikPak MP4", worker.Status().CooldownUntil)
|
|
}
|
|
if gen.generateCalls != 1 {
|
|
t.Fatalf("generate calls = %d, want 1", gen.generateCalls)
|
|
}
|
|
}
|
|
|
|
func TestPreviewWorkerP115TransientErrorKeepsVideoPending(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "preview-p115-transient")
|
|
|
|
gen := &fakeTeaserGenerator{
|
|
generateErr: errors.New("ffmpeg: exit status 1, stderr: Server returned 403 Forbidden"),
|
|
}
|
|
drv := &previewFakeDrive{kind: "p115"}
|
|
worker := NewWorker(gen, cat, drv)
|
|
|
|
worker.process(ctx, video)
|
|
|
|
got, err := cat.GetVideo(ctx, video.ID)
|
|
if err != nil {
|
|
t.Fatalf("get video: %v", err)
|
|
}
|
|
if got.PreviewStatus != "pending" {
|
|
t.Fatalf("preview status = %q, want pending for transient 115 media error", got.PreviewStatus)
|
|
}
|
|
if gen.generateCalls != 1 {
|
|
t.Fatalf("generate calls = %d, want 1", gen.generateCalls)
|
|
}
|
|
}
|
|
|
|
func TestP123TransientErrorsShouldCooldown(t *testing.T) {
|
|
drv := &previewFakeDrive{kind: "p123"}
|
|
for _, err := range []error{
|
|
errors.New("Server returned 403 Forbidden"),
|
|
errors.New("http 503 service unavailable"),
|
|
} {
|
|
if !driveErrorShouldCooldown(drv, err) {
|
|
t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err)
|
|
}
|
|
}
|
|
if driveErrorShouldCooldown(drv, errors.New("请求太频繁")) {
|
|
t.Fatal("message-only throttling text should not trigger p123 cooldown")
|
|
}
|
|
if driveErrorShouldCooldown(drv, errors.New("invalid credential")) {
|
|
t.Fatal("invalid credential should not trigger p123 cooldown")
|
|
}
|
|
}
|
|
|
|
func TestWopanTransientErrorsShouldCooldown(t *testing.T) {
|
|
drv := &previewFakeDrive{kind: "wopan"}
|
|
for _, err := range []error{
|
|
errors.New("ffmpeg: Server returned 403 Forbidden"),
|
|
errors.New("wopan download url: request failed with status: 429 Too Many Requests"),
|
|
errors.New("http 503 service unavailable"),
|
|
} {
|
|
if !driveErrorShouldCooldown(drv, err) {
|
|
t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err)
|
|
}
|
|
}
|
|
if driveErrorShouldCooldown(drv, errors.New("操作频繁,请稍后重试")) {
|
|
t.Fatal("message-only throttling text should not trigger wopan cooldown")
|
|
}
|
|
if driveErrorShouldCooldown(drv, errors.New("invalid access token")) {
|
|
t.Fatal("invalid access token should not trigger wopan cooldown")
|
|
}
|
|
}
|
|
|
|
func TestGuangYaPanTransientErrorsShouldCooldown(t *testing.T) {
|
|
drv := &previewFakeDrive{kind: "guangyapan"}
|
|
for _, err := range []error{
|
|
errors.New("ffmpeg: Server returned 403 Forbidden"),
|
|
errors.New("guangyapan api rate limited: status=429 msg=操作频繁,请稍后重试"),
|
|
errors.New("http 503 service unavailable"),
|
|
} {
|
|
if !driveErrorShouldCooldown(drv, err) {
|
|
t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err)
|
|
}
|
|
}
|
|
if driveErrorShouldCooldown(drv, errors.New("操作频繁,请稍后重试")) {
|
|
t.Fatal("message-only throttling text should not trigger guangyapan cooldown")
|
|
}
|
|
if driveErrorShouldCooldown(drv, errors.New("invalid access token")) {
|
|
t.Fatal("invalid access token should not trigger guangyapan cooldown")
|
|
}
|
|
}
|
|
|
|
func TestGoogleDriveMediaErrorsShouldCooldown(t *testing.T) {
|
|
drv := &previewFakeDrive{kind: "googledrive"}
|
|
for _, err := range []error{
|
|
errors.New("ffmpeg: Server returned 403 Forbidden"),
|
|
errors.New("http 503 service unavailable"),
|
|
} {
|
|
if !driveErrorShouldCooldown(drv, err) {
|
|
t.Fatalf("driveErrorShouldCooldown(%v) = false, want true", err)
|
|
}
|
|
}
|
|
for _, err := range []error{
|
|
errors.New("google drive api error: usageLimits userRateLimitExceeded"),
|
|
errors.New("downloadQuotaExceeded: The download quota for this file has been exceeded"),
|
|
errors.New("sharingRateLimitExceeded"),
|
|
} {
|
|
if driveErrorShouldCooldown(drv, err) {
|
|
t.Fatalf("message-only google drive error %v should not trigger cooldown", err)
|
|
}
|
|
}
|
|
if driveErrorShouldCooldown(drv, errors.New("invalid credentials")) {
|
|
t.Fatal("invalid credentials should not trigger googledrive cooldown")
|
|
}
|
|
}
|
|
|
|
func assertCooldownAround(t *testing.T, until time.Time, before time.Time, want time.Duration) {
|
|
t.Helper()
|
|
if until.IsZero() {
|
|
t.Fatal("cooldown is zero, want active cooldown")
|
|
}
|
|
min := before.Add(want - time.Second)
|
|
max := time.Now().Add(want + time.Second)
|
|
if until.Before(min) || until.After(max) {
|
|
t.Fatalf("cooldown until = %s, want around %s from now", until.Format(time.RFC3339Nano), want)
|
|
}
|
|
}
|
|
|
|
func TestPreviewWorkerRefreshesP115LinksPerTeaserInput(t *testing.T) {
|
|
ctx := context.Background()
|
|
cat, video := seedPreviewTestVideo(t, "preview-p115-refresh")
|
|
video.DurationSeconds = 81
|
|
if err := cat.UpsertVideo(ctx, video); err != nil {
|
|
t.Fatalf("update video: %v", err)
|
|
}
|
|
|
|
gen := &fakeTeaserGenerator{}
|
|
drv := &previewFakeDrive{kind: "p115"}
|
|
worker := NewWorker(gen, cat, drv)
|
|
|
|
worker.process(ctx, video)
|
|
|
|
if gen.refreshCalls != 3 {
|
|
t.Fatalf("refresh calls = %d, want 3 extra links for a four-input p115 teaser", gen.refreshCalls)
|
|
}
|
|
if drv.streamCalls != 4 {
|
|
t.Fatalf("stream calls = %d, want initial link plus 3 refreshed links", drv.streamCalls)
|
|
}
|
|
}
|
|
|
|
func seedPreviewTestVideo(t *testing.T, id string) (*catalog.Catalog, *catalog.Video) {
|
|
t.Helper()
|
|
ctx := context.Background()
|
|
cat, err := catalog.Open(t.TempDir() + "/catalog.db")
|
|
if err != nil {
|
|
t.Fatalf("open catalog: %v", err)
|
|
}
|
|
t.Cleanup(func() {
|
|
if err := cat.Close(); err != nil {
|
|
t.Fatalf("close catalog: %v", err)
|
|
}
|
|
})
|
|
|
|
video := &catalog.Video{
|
|
ID: id,
|
|
DriveID: "drive-id",
|
|
FileID: "file-id",
|
|
Title: "Clip",
|
|
PreviewStatus: "pending",
|
|
PublishedAt: time.Now(),
|
|
CreatedAt: time.Now(),
|
|
UpdatedAt: time.Now(),
|
|
}
|
|
if err := cat.UpsertVideo(ctx, video); err != nil {
|
|
t.Fatalf("seed video: %v", err)
|
|
}
|
|
return cat, video
|
|
}
|
|
|
|
type fakeThumbGenerator struct {
|
|
thumbnailVideoID string
|
|
thumbnailDuration float64
|
|
thumbnailURL string
|
|
probeCalls int
|
|
generateCalls int
|
|
probeDuration float64
|
|
probeErr error
|
|
generateErr error
|
|
}
|
|
|
|
func (g *fakeThumbGenerator) Probe(context.Context, *drives.StreamLink) (float64, error) {
|
|
g.probeCalls++
|
|
if g.probeErr != nil {
|
|
return 0, g.probeErr
|
|
}
|
|
return g.probeDuration, nil
|
|
}
|
|
|
|
func (g *fakeThumbGenerator) GenerateThumbnail(_ context.Context, link *drives.StreamLink, videoID string, duration float64) (string, error) {
|
|
g.generateCalls++
|
|
g.thumbnailVideoID = videoID
|
|
g.thumbnailDuration = duration
|
|
if link != nil {
|
|
g.thumbnailURL = link.URL
|
|
}
|
|
if g.generateErr != nil {
|
|
return "", g.generateErr
|
|
}
|
|
return "/tmp/" + videoID + ".jpg", nil
|
|
}
|
|
|
|
type fakeTeaserGenerator struct {
|
|
localPath string
|
|
generateErr error
|
|
generateCalls int
|
|
refreshCalls int
|
|
}
|
|
|
|
func (g *fakeTeaserGenerator) Probe(context.Context, *drives.StreamLink) (float64, error) {
|
|
return 0, nil
|
|
}
|
|
|
|
func (g *fakeTeaserGenerator) Generate(context.Context, *drives.StreamLink, float64) (string, error) {
|
|
g.generateCalls++
|
|
if g.generateErr != nil {
|
|
return "", g.generateErr
|
|
}
|
|
return "/tmp/source-teaser.mp4", nil
|
|
}
|
|
|
|
func (g *fakeTeaserGenerator) GenerateWithLinkProvider(ctx context.Context, first *drives.StreamLink, duration float64, refresh func(context.Context) (*drives.StreamLink, error)) (string, error) {
|
|
for i := 0; i < 3; i++ {
|
|
if _, err := refresh(ctx); err != nil {
|
|
return "", err
|
|
}
|
|
g.refreshCalls++
|
|
}
|
|
return g.Generate(ctx, first, duration)
|
|
}
|
|
|
|
func (g *fakeTeaserGenerator) MoveToLocal(_ string, videoID string) (string, error) {
|
|
if g.localPath != "" {
|
|
return g.localPath, nil
|
|
}
|
|
return "/tmp/" + videoID + ".mp4", nil
|
|
}
|
|
|
|
type previewFakeDrive struct {
|
|
kind string
|
|
streamFileID string
|
|
streamCalls int
|
|
streamErr error
|
|
ensureDirCalls int
|
|
uploadCalls int
|
|
}
|
|
|
|
func (d *previewFakeDrive) Kind() string {
|
|
if d.kind != "" {
|
|
return d.kind
|
|
}
|
|
return "fake"
|
|
}
|
|
func (d *previewFakeDrive) ID() string { return "drive-id" }
|
|
func (d *previewFakeDrive) Init(context.Context) error {
|
|
return nil
|
|
}
|
|
func (d *previewFakeDrive) List(context.Context, string) ([]drives.Entry, error) {
|
|
return nil, nil
|
|
}
|
|
func (d *previewFakeDrive) Stat(context.Context, string) (*drives.Entry, error) {
|
|
return nil, drives.ErrNotSupported
|
|
}
|
|
func (d *previewFakeDrive) StreamURL(_ context.Context, fileID string) (*drives.StreamLink, error) {
|
|
d.streamCalls++
|
|
d.streamFileID = fileID
|
|
if d.streamErr != nil {
|
|
return nil, d.streamErr
|
|
}
|
|
return &drives.StreamLink{URL: "https://video.example/clip.mp4"}, nil
|
|
}
|
|
func (d *previewFakeDrive) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
|
d.uploadCalls++
|
|
return "", drives.ErrNotSupported
|
|
}
|
|
func (d *previewFakeDrive) EnsureDir(context.Context, string) (string, error) {
|
|
d.ensureDirCalls++
|
|
return "", drives.ErrNotSupported
|
|
}
|
|
func (d *previewFakeDrive) RootID() string { return "root" }
|
|
|
|
func TestWorkerWaitIdleReturnsImmediatelyWhenQueueEmpty(t *testing.T) {
|
|
worker := NewWorker(&fakeTeaserGenerator{}, nil, &previewFakeDrive{})
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
|
|
start := time.Now()
|
|
if err := worker.WaitIdle(ctx); err != nil {
|
|
t.Fatalf("WaitIdle on empty queue: %v", err)
|
|
}
|
|
if took := time.Since(start); took > 50*time.Millisecond {
|
|
t.Fatalf("WaitIdle on empty queue took %s, want immediate return", took)
|
|
}
|
|
}
|
|
|
|
func TestWorkerWaitIdleBlocksUntilQueueDrains(t *testing.T) {
|
|
worker := NewWorker(&fakeTeaserGenerator{}, nil, &previewFakeDrive{})
|
|
v := &catalog.Video{ID: "wait-idle-vid"}
|
|
if !worker.queue.reserve(v) {
|
|
t.Fatalf("reserve should succeed on fresh queue")
|
|
}
|
|
|
|
go func() {
|
|
time.Sleep(120 * time.Millisecond)
|
|
worker.queue.release(v)
|
|
}()
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
|
|
start := time.Now()
|
|
if err := worker.WaitIdle(ctx); err != nil {
|
|
t.Fatalf("WaitIdle: %v", err)
|
|
}
|
|
took := time.Since(start)
|
|
if took < 100*time.Millisecond {
|
|
t.Fatalf("WaitIdle returned in %s, expected to wait until release", took)
|
|
}
|
|
if took > time.Second {
|
|
t.Fatalf("WaitIdle took %s, expected to return shortly after release", took)
|
|
}
|
|
}
|
|
|
|
func TestWorkerWaitIdleHonoursContextCancel(t *testing.T) {
|
|
worker := NewWorker(&fakeTeaserGenerator{}, nil, &previewFakeDrive{})
|
|
v := &catalog.Video{ID: "ctx-cancel"}
|
|
if !worker.queue.reserve(v) {
|
|
t.Fatalf("reserve should succeed")
|
|
}
|
|
t.Cleanup(func() { worker.queue.release(v) })
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
|
defer cancel()
|
|
|
|
if err := worker.WaitIdle(ctx); err == nil {
|
|
t.Fatalf("WaitIdle expected ctx.Err, got nil")
|
|
}
|
|
}
|
|
|
|
func TestThumbWorkerWaitIdleBlocksUntilQueueDrains(t *testing.T) {
|
|
worker := NewThumbWorker(&fakeThumbGenerator{}, nil, &previewFakeDrive{})
|
|
v := &catalog.Video{ID: "thumb-wait-idle"}
|
|
if !worker.queue.reserve(v) {
|
|
t.Fatalf("reserve should succeed")
|
|
}
|
|
|
|
go func() {
|
|
time.Sleep(80 * time.Millisecond)
|
|
worker.queue.release(v)
|
|
}()
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
if err := worker.WaitIdle(ctx); err != nil {
|
|
t.Fatalf("ThumbWorker.WaitIdle: %v", err)
|
|
}
|
|
}
|