mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-15 00:44:30 +08:00
Handle OneDrive rate limits and use preview server
This commit is contained in:
@@ -31,18 +31,21 @@ Linux / WSL 环境推荐用仓库根目录的脚本同时启动前后端:
|
||||
|
||||
```bash
|
||||
npm install
|
||||
./start.sh # 前端 9191,后端 9192
|
||||
./start.sh # 前端 9191,后端 9192;默认使用生产预览模式,无热更新
|
||||
./start.sh --status # 查看运行状态
|
||||
./start.sh --restart # 重启
|
||||
./start.sh --stop # 停止
|
||||
```
|
||||
|
||||
如果需要开发热更新,可临时使用 `FRONTEND_MODE=dev ./start.sh --restart`。
|
||||
|
||||
也可以分两个终端手动启动:
|
||||
|
||||
```bash
|
||||
# 前端
|
||||
npm install
|
||||
npm run dev # 监听 http://127.0.0.1:9191
|
||||
npm run build
|
||||
npm run preview # 监听 http://127.0.0.1:9191,无热更新
|
||||
|
||||
# 后端(另开终端)
|
||||
cd backend
|
||||
@@ -55,7 +58,7 @@ go run ./cmd/server # 默认监听 127.0.0.1:9192,依赖已 vendor 入库
|
||||
- `backend/data/video-site.db`(SQLite)
|
||||
- `backend/data/previews/`(teaser 和封面本地目录)
|
||||
|
||||
Vite dev server 已配置把 `/api`、`/p`、`/admin/api` 反代到 `127.0.0.1:9192`。浏览器访问 `http://127.0.0.1:9191/` 进入前台,`/admin` 进入管理后台(默认 `admin` / `admin123`,请在 `backend/config.yaml` 里改)。如果本地已经存在旧的 `backend/config.yaml`,请确认 `server.listen` 与 Vite 代理端口一致。
|
||||
Vite dev / preview server 都已配置把 `/api`、`/p`、`/admin/api` 反代到 `127.0.0.1:9192`。浏览器访问 `http://127.0.0.1:9191/` 进入前台,`/admin` 进入管理后台(默认 `admin` / `admin123`,请在 `backend/config.yaml` 里改)。如果本地已经存在旧的 `backend/config.yaml`,请确认 `server.listen` 与 Vite 代理端口一致。
|
||||
|
||||
## 目录
|
||||
|
||||
@@ -118,7 +121,7 @@ OneDrive 当前采用 OpenList 在线 API 的续期方式,不要求用户提
|
||||
- 极短视频会按可容纳的完整 3 秒片段数自动降级
|
||||
- 首次失败的任务标 `preview_status = failed`,不再自动重试;管理后台可手动重新生成
|
||||
- 服务启动或网盘重新挂载时,如果 Teaser 开关已开启,会自动把历史 `pending` 任务重新入队,避免重启后停在“待生成”。
|
||||
- OneDrive 直链生成 teaser 时可能触发 Microsoft 429 限流;这种失败会被记录为 `failed`,可稍后在管理后台重生。
|
||||
- OneDrive 直链生成 teaser 时可能触发 Microsoft 429 限流;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。
|
||||
- 详见 plan 15.12 节
|
||||
|
||||
## 常用管理能力
|
||||
|
||||
+6
-3
@@ -47,9 +47,11 @@ Git Bash / WSL 环境推荐从仓库根目录启动完整开发环境:
|
||||
|
||||
```bash
|
||||
npm install
|
||||
./start.sh
|
||||
./start.sh # 默认前端 production preview,无热更新
|
||||
```
|
||||
|
||||
需要前端开发热更新时再用 `FRONTEND_MODE=dev ./start.sh --restart`。
|
||||
|
||||
PowerShell 下可以分两个终端手动启动,后端命令如下:
|
||||
|
||||
```powershell
|
||||
@@ -70,7 +72,8 @@ go run ./cmd/server
|
||||
`vite.config.ts` 已经把 `/api`、`/p`、`/admin/api` 代理到 `127.0.0.1:9192`。
|
||||
|
||||
```
|
||||
npm run dev 前端 9191
|
||||
npm run build 构建前端静态资源
|
||||
npm run preview 前端 9191,无热更新
|
||||
go run ./cmd/server 后端 9192
|
||||
```
|
||||
|
||||
@@ -138,7 +141,7 @@ ffmpeg -ss <起点> -headers "UA/Cookie/Referer" -i <直链> \
|
||||
|
||||
当前策略是每段固定 3 秒;30 秒以下最多 3 段,30 秒及以上固定 4 段;长视频在 20% 到 80% 区间均匀取段。优先把 teaser 上传回网盘的 `previews/` 目录;失败时保留本地 `data/previews/<videoID>.mp4` 作为兜底。
|
||||
|
||||
服务启动或网盘重新挂载时,如果 Teaser 开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 直链生成 teaser 时可能触发 Microsoft 429 限流;这类任务会标记为 `failed`,可稍后在视频管理页重生。
|
||||
服务启动或网盘重新挂载时,如果 Teaser 开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 直链生成 teaser 时可能触发 Microsoft 429 限流;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。
|
||||
|
||||
前端卡片的 `previewSrc` 统一指向 `/p/preview/<videoID>`,后端自动选择网盘代理或本地文件。
|
||||
|
||||
|
||||
@@ -64,3 +64,38 @@ type StreamLink struct {
|
||||
|
||||
// ErrNotSupported 代表某家盘不支持某操作
|
||||
var ErrNotSupported = errors.New("operation not supported by this drive")
|
||||
|
||||
// RateLimitError 表示上游服务正在限流。RetryAfter 为 0 时由调用方选择默认冷却时间。
|
||||
type RateLimitError struct {
|
||||
Provider string
|
||||
RetryAfter time.Duration
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *RateLimitError) Error() string {
|
||||
if e == nil {
|
||||
return "rate limited"
|
||||
}
|
||||
if e.Err != nil {
|
||||
return e.Err.Error()
|
||||
}
|
||||
if e.Provider != "" {
|
||||
return e.Provider + " rate limited"
|
||||
}
|
||||
return "rate limited"
|
||||
}
|
||||
|
||||
func (e *RateLimitError) Unwrap() error {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return e.Err
|
||||
}
|
||||
|
||||
func RateLimitRetryAfter(err error) (time.Duration, bool) {
|
||||
var rateLimit *RateLimitError
|
||||
if errors.As(err, &rateLimit) {
|
||||
return rateLimit.RetryAfter, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -265,6 +266,9 @@ func (d *Driver) requestOnce(ctx context.Context, rawURL, method string, configu
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isRateLimitResponse(res, graphErr.Error.Code) {
|
||||
return onedriveRateLimitError(res, graphErr.Error.Message)
|
||||
}
|
||||
if graphErr.Error.Code != "" {
|
||||
if graphErr.Error.Code == "InvalidAuthenticationToken" && retry {
|
||||
if err := d.refresh(ctx); err != nil {
|
||||
@@ -298,6 +302,9 @@ func (d *Driver) refresh(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("onedrive refresh token: %w", err)
|
||||
}
|
||||
if res.StatusCode() == http.StatusTooManyRequests {
|
||||
return onedriveRateLimitError(res, "token refresh throttled")
|
||||
}
|
||||
if out.Text != "" {
|
||||
return fmt.Errorf("onedrive refresh token: %s", out.Text)
|
||||
}
|
||||
@@ -321,6 +328,47 @@ func (d *Driver) refresh(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func isRateLimitResponse(res *resty.Response, code string) bool {
|
||||
if code == "TooManyRequests" || code == "activityLimitReached" {
|
||||
return true
|
||||
}
|
||||
return res != nil && res.StatusCode() == http.StatusTooManyRequests
|
||||
}
|
||||
|
||||
func onedriveRateLimitError(res *resty.Response, message string) error {
|
||||
if strings.TrimSpace(message) == "" {
|
||||
message = "onedrive rate limited"
|
||||
}
|
||||
if res != nil && strings.TrimSpace(res.String()) != "" {
|
||||
message = fmt.Sprintf("%s: status=%d body=%s", message, res.StatusCode(), strings.TrimSpace(res.String()))
|
||||
}
|
||||
return &drives.RateLimitError{
|
||||
Provider: "onedrive",
|
||||
RetryAfter: parseRetryAfter(res),
|
||||
Err: errors.New(message),
|
||||
}
|
||||
}
|
||||
|
||||
func parseRetryAfter(res *resty.Response) time.Duration {
|
||||
if res == nil {
|
||||
return 0
|
||||
}
|
||||
raw := strings.TrimSpace(res.Header().Get("Retry-After"))
|
||||
if raw == "" {
|
||||
return 0
|
||||
}
|
||||
if seconds, err := strconv.Atoi(raw); err == nil && seconds > 0 {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
if when, err := http.ParseTime(raw); err == nil {
|
||||
d := time.Until(when)
|
||||
if d > 0 {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (d *Driver) driveBaseURL() string {
|
||||
if d.isSharePoint {
|
||||
return fmt.Sprintf("%s/v1.0/sites/%s/drive", d.apiBaseURL, url.PathEscape(d.siteID))
|
||||
|
||||
@@ -3,11 +3,13 @@ package onedrive
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
@@ -174,6 +176,42 @@ func TestGraphItemWithoutFolderFacetIsNotDirectory(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraph429ReturnsRateLimitErrorWithRetryAfter(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Retry-After", "120")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{
|
||||
"error": map[string]any{
|
||||
"code": "TooManyRequests",
|
||||
"message": "throttled",
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("write json: %v", err)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
|
||||
_, err := d.List(context.Background(), "root")
|
||||
if err == nil {
|
||||
t.Fatal("list succeeded, want rate limit error")
|
||||
}
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != 2*time.Minute {
|
||||
t.Fatalf("retry after = %v, want 2m", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatAndStreamURLUseDriveItemMetadata(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
|
||||
@@ -2,6 +2,7 @@ package preview
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -11,6 +12,7 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
@@ -228,7 +230,7 @@ func (g *Generator) GenerateThumbnail(ctx context.Context, link *drives.StreamLi
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
os.Remove(dst)
|
||||
return "", fmt.Errorf("ffmpeg thumb: %w, stderr: %s", err, string(out))
|
||||
return "", ffmpegCommandError("ffmpeg thumb", err, out)
|
||||
}
|
||||
if info, statErr := os.Stat(dst); statErr != nil || info.Size() == 0 {
|
||||
os.Remove(dst)
|
||||
@@ -256,9 +258,9 @@ func (g *Generator) Probe(ctx context.Context, link *drives.StreamLink) (float64
|
||||
args = append(args, link.URL)
|
||||
|
||||
cmd := exec.CommandContext(ctx2, g.cfg.FFprobePath, args...)
|
||||
out, err := cmd.Output()
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("ffprobe: %w", err)
|
||||
return 0, ffmpegCommandError("ffprobe", err, out)
|
||||
}
|
||||
raw := strings.TrimSpace(string(out))
|
||||
if raw == "" || raw == "N/A" {
|
||||
@@ -362,7 +364,7 @@ func (g *Generator) Generate(ctx context.Context, link *drives.StreamLink, durat
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
os.Remove(tmpPath)
|
||||
return "", fmt.Errorf("ffmpeg: %w, stderr: %s", err, string(out))
|
||||
return "", ffmpegCommandError("ffmpeg", err, out)
|
||||
}
|
||||
|
||||
if info, statErr := os.Stat(tmpPath); statErr != nil || info.Size() == 0 {
|
||||
@@ -372,6 +374,48 @@ func (g *Generator) Generate(ctx context.Context, link *drives.StreamLink, durat
|
||||
return tmpPath, nil
|
||||
}
|
||||
|
||||
func ffmpegCommandError(tool string, err error, output []byte) error {
|
||||
msg := fmt.Sprintf("%s: %v, stderr: %s", tool, err, redactURLs(string(output)))
|
||||
wrapped := errors.New(msg)
|
||||
if ffmpegOutputLooksRateLimited(output) {
|
||||
return &drives.RateLimitError{
|
||||
Provider: "media source",
|
||||
Err: wrapped,
|
||||
}
|
||||
}
|
||||
return wrapped
|
||||
}
|
||||
|
||||
func redactURLs(text string) string {
|
||||
fields := strings.Fields(text)
|
||||
for i, field := range fields {
|
||||
if strings.HasPrefix(field, "http://") || strings.HasPrefix(field, "https://") {
|
||||
suffix := ""
|
||||
for len(field) > 0 {
|
||||
last := field[len(field)-1]
|
||||
if last != '.' && last != ',' && last != ';' && last != ')' {
|
||||
break
|
||||
}
|
||||
suffix = string(last) + suffix
|
||||
field = field[:len(field)-1]
|
||||
}
|
||||
fields[i] = "https://<redacted>" + suffix
|
||||
}
|
||||
}
|
||||
return strings.Join(fields, " ")
|
||||
}
|
||||
|
||||
func ffmpegOutputLooksRateLimited(output []byte) bool {
|
||||
text := strings.ToLower(string(output))
|
||||
if !strings.Contains(text, "429") {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(text, "too many requests") ||
|
||||
strings.Contains(text, "rate limit") ||
|
||||
strings.Contains(text, "rate-limit") ||
|
||||
strings.Contains(text, "server returned 429")
|
||||
}
|
||||
|
||||
// --- 本地落盘 ---
|
||||
|
||||
// MoveToLocal 把临时文件改名到稳定位置,返回最终路径
|
||||
@@ -410,6 +454,9 @@ type Worker struct {
|
||||
Drive drives.Drive
|
||||
RemoteDir string
|
||||
ch chan *catalog.Video
|
||||
|
||||
RateLimitCooldown time.Duration
|
||||
rateLimit rateLimitState
|
||||
}
|
||||
|
||||
func NewWorker(gen TeaserGenerator, cat *catalog.Catalog, drv drives.Drive, remoteDir string) *Worker {
|
||||
@@ -451,6 +498,46 @@ type ThumbWorker struct {
|
||||
Catalog *catalog.Catalog
|
||||
Drive drives.Drive
|
||||
ch chan *catalog.Video
|
||||
|
||||
RateLimitCooldown time.Duration
|
||||
rateLimit rateLimitState
|
||||
}
|
||||
|
||||
const defaultRateLimitCooldown = 30 * time.Minute
|
||||
|
||||
type rateLimitState struct {
|
||||
mu sync.Mutex
|
||||
until time.Time
|
||||
lastSkipLog time.Time
|
||||
}
|
||||
|
||||
func (s *rateLimitState) active(now time.Time) (time.Time, bool, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.until.IsZero() || !now.Before(s.until) {
|
||||
return time.Time{}, false, false
|
||||
}
|
||||
shouldLog := s.lastSkipLog.IsZero() || now.Sub(s.lastSkipLog) >= 5*time.Minute
|
||||
if shouldLog {
|
||||
s.lastSkipLog = now
|
||||
}
|
||||
return s.until, true, shouldLog
|
||||
}
|
||||
|
||||
func (s *rateLimitState) pause(now time.Time, d time.Duration) time.Time {
|
||||
if d <= 0 {
|
||||
d = defaultRateLimitCooldown
|
||||
}
|
||||
until := now.Add(d)
|
||||
s.mu.Lock()
|
||||
if until.After(s.until) {
|
||||
s.until = until
|
||||
} else {
|
||||
until = s.until
|
||||
}
|
||||
s.lastSkipLog = time.Time{}
|
||||
s.mu.Unlock()
|
||||
return until
|
||||
}
|
||||
|
||||
func NewThumbWorker(gen ThumbnailGenerator, cat *catalog.Catalog, drv drives.Drive) *ThumbWorker {
|
||||
@@ -520,9 +607,63 @@ func (w *ThumbWorker) Run(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) skipIfRateLimited(v *catalog.Video) bool {
|
||||
until, ok, shouldLog := w.rateLimit.active(time.Now())
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if shouldLog {
|
||||
log.Printf("[preview] drive=%s rate-limited until=%s; skip queued videos and keep them pending", w.Drive.ID(), until.Format(time.RFC3339))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (w *Worker) pauseForRateLimit(err error, step, title string) bool {
|
||||
retryAfter, ok := drives.RateLimitRetryAfter(err)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if retryAfter <= 0 {
|
||||
retryAfter = w.RateLimitCooldown
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), retryAfter)
|
||||
log.Printf("[preview] drive=%s rate-limited until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err)
|
||||
return true
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) skipIfRateLimited(v *catalog.Video) bool {
|
||||
until, ok, shouldLog := w.rateLimit.active(time.Now())
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if shouldLog {
|
||||
log.Printf("[thumb] drive=%s rate-limited until=%s; skip queued thumbnails and keep them pending", w.Drive.ID(), until.Format(time.RFC3339))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) pauseForRateLimit(err error, step, title string) bool {
|
||||
retryAfter, ok := drives.RateLimitRetryAfter(err)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if retryAfter <= 0 {
|
||||
retryAfter = w.RateLimitCooldown
|
||||
}
|
||||
until := w.rateLimit.pause(time.Now(), retryAfter)
|
||||
log.Printf("[thumb] drive=%s rate-limited until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err)
|
||||
return true
|
||||
}
|
||||
|
||||
func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
|
||||
if w.skipIfRateLimited(v) {
|
||||
return
|
||||
}
|
||||
link, err := w.Drive.StreamURL(ctx, v.FileID)
|
||||
if err != nil {
|
||||
if w.pauseForRateLimit(err, "streamURL", v.Title) {
|
||||
return
|
||||
}
|
||||
log.Printf("[thumb] streamURL %s: %v", v.Title, err)
|
||||
return
|
||||
}
|
||||
@@ -534,10 +675,15 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
DurationSeconds: int(dur),
|
||||
})
|
||||
} else if err != nil && w.pauseForRateLimit(err, "probe", v.Title) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := w.Gen.GenerateThumbnail(ctx, link, v.ID, duration); err != nil {
|
||||
if w.pauseForRateLimit(err, "generate", v.Title) {
|
||||
return
|
||||
}
|
||||
log.Printf("[thumb] generate %s: %v", v.Title, err)
|
||||
return
|
||||
}
|
||||
@@ -548,8 +694,14 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
|
||||
}
|
||||
|
||||
func (w *Worker) process(ctx context.Context, v *catalog.Video) {
|
||||
if w.skipIfRateLimited(v) {
|
||||
return
|
||||
}
|
||||
link, err := w.Drive.StreamURL(ctx, v.FileID)
|
||||
if err != nil {
|
||||
if w.pauseForRateLimit(err, "streamURL", v.Title) {
|
||||
return
|
||||
}
|
||||
log.Printf("[preview] streamURL %s: %v", v.Title, err)
|
||||
w.Catalog.UpdatePreview(ctx, v.ID, "", "", "failed")
|
||||
return
|
||||
@@ -563,12 +715,17 @@ func (w *Worker) process(ctx context.Context, v *catalog.Video) {
|
||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||
DurationSeconds: int(dur),
|
||||
})
|
||||
} else if err != nil && w.pauseForRateLimit(err, "probe", v.Title) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 2) teaser
|
||||
tmp, err := w.Gen.Generate(ctx, link, duration)
|
||||
if err != nil {
|
||||
if w.pauseForRateLimit(err, "generate", v.Title) {
|
||||
return
|
||||
}
|
||||
log.Printf("[preview] generate %s: %v", v.Title, err)
|
||||
w.Catalog.UpdatePreview(ctx, v.ID, "", "", "failed")
|
||||
return
|
||||
@@ -585,7 +742,11 @@ func (w *Worker) process(ctx context.Context, v *catalog.Video) {
|
||||
if fid, uerr := w.uploadToDrive(ctx, v.ID, local); uerr == nil {
|
||||
previewFileID = fid
|
||||
} else {
|
||||
log.Printf("[preview] upload %s: %v (local only)", v.Title, uerr)
|
||||
if w.pauseForRateLimit(uerr, "upload", v.Title) {
|
||||
log.Printf("[preview] upload %s: %v (local only; drive cooling down)", v.Title, uerr)
|
||||
} else {
|
||||
log.Printf("[preview] upload %s: %v (local only)", v.Title, uerr)
|
||||
}
|
||||
}
|
||||
}
|
||||
removePreviousLocalTeaser(v.PreviewLocal, local)
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
package preview
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestNewDefaultsToThreeSecondTeaserSegments(t *testing.T) {
|
||||
@@ -85,3 +89,25 @@ func TestShortVideoPreviewPlanReturnsNoSegmentsWhenOneSegmentCannotFit(t *testin
|
||||
t.Fatalf("eachSec = %.2f, want 3", plan.eachSec)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpeg429OutputBecomesRateLimitError(t *testing.T) {
|
||||
err := ffmpegCommandError("ffmpeg", errors.New("exit status 8"), []byte("Server returned 429 Too Many Requests"))
|
||||
var rateLimit *drives.RateLimitError
|
||||
if !errors.As(err, &rateLimit) {
|
||||
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||
}
|
||||
if rateLimit.RetryAfter != 0 {
|
||||
t.Fatalf("retry after = %v, want none from ffmpeg stderr", rateLimit.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegCommandErrorRedactsSignedURLs(t *testing.T) {
|
||||
err := ffmpegCommandError("ffmpeg", errors.New("exit status 8"), []byte("Error opening input file https://download.example/file.mp4?tempauth=secret."))
|
||||
got := err.Error()
|
||||
if strings.Contains(got, "tempauth=secret") {
|
||||
t.Fatalf("error leaked signed URL: %s", got)
|
||||
}
|
||||
if !strings.Contains(got, "https://<redacted>.") {
|
||||
t.Fatalf("error = %q, want redacted URL with punctuation preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package preview
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -108,6 +109,52 @@ func TestPreviewWorkerRemovesPreviousLocalTeaserAfterNewTeaserIsReady(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewWorkerRateLimitLeavesCurrentPendingAndSkipsNextVideo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cat, first := seedPreviewTestVideo(t, "preview-rate-limit-1")
|
||||
second := *first
|
||||
second.ID = "preview-rate-limit-2"
|
||||
second.FileID = "file-id-2"
|
||||
if err := cat.UpsertVideo(ctx, &second); err != nil {
|
||||
t.Fatalf("seed second video: %v", err)
|
||||
}
|
||||
|
||||
gen := &fakeTeaserGenerator{
|
||||
generateErr: &drives.RateLimitError{
|
||||
Provider: "onedrive",
|
||||
RetryAfter: 2 * time.Hour,
|
||||
Err: errors.New("429 Too Many Requests"),
|
||||
},
|
||||
}
|
||||
drv := &previewFakeDrive{}
|
||||
worker := NewWorker(gen, cat, drv, "")
|
||||
|
||||
worker.process(ctx, first)
|
||||
gotFirst, err := cat.GetVideo(ctx, first.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get first video: %v", err)
|
||||
}
|
||||
if gotFirst.PreviewStatus != "pending" {
|
||||
t.Fatalf("first preview status = %q, want pending after rate limit", gotFirst.PreviewStatus)
|
||||
}
|
||||
if gen.generateCalls != 1 {
|
||||
t.Fatalf("generate calls = %d, want 1", gen.generateCalls)
|
||||
}
|
||||
|
||||
gen.generateErr = nil
|
||||
worker.process(ctx, &second)
|
||||
gotSecond, err := cat.GetVideo(ctx, second.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get second video: %v", err)
|
||||
}
|
||||
if gotSecond.PreviewStatus != "pending" {
|
||||
t.Fatalf("second preview status = %q, want pending while drive is cooling down", gotSecond.PreviewStatus)
|
||||
}
|
||||
if gen.generateCalls != 1 {
|
||||
t.Fatalf("generate calls = %d, want second video skipped during cooldown", gen.generateCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func seedPreviewTestVideo(t *testing.T, id string) (*catalog.Catalog, *catalog.Video) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
@@ -153,7 +200,9 @@ func (g *fakeThumbGenerator) GenerateThumbnail(_ context.Context, _ *drives.Stre
|
||||
}
|
||||
|
||||
type fakeTeaserGenerator struct {
|
||||
localPath string
|
||||
localPath string
|
||||
generateErr error
|
||||
generateCalls int
|
||||
}
|
||||
|
||||
func (g *fakeTeaserGenerator) Probe(context.Context, *drives.StreamLink) (float64, error) {
|
||||
@@ -161,6 +210,10 @@ func (g *fakeTeaserGenerator) Probe(context.Context, *drives.StreamLink) (float6
|
||||
}
|
||||
|
||||
func (g *fakeTeaserGenerator) Generate(context.Context, *drives.StreamLink, float64) (string, error) {
|
||||
g.generateCalls++
|
||||
if g.generateErr != nil {
|
||||
return "", g.generateErr
|
||||
}
|
||||
return "/tmp/source-teaser.mp4", nil
|
||||
}
|
||||
|
||||
|
||||
@@ -24,12 +24,6 @@
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 380px) {
|
||||
.video-grid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
|
||||
.video-grid.is-compact {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
@@ -332,6 +326,10 @@
|
||||
}
|
||||
|
||||
@media (max-width: 640px) {
|
||||
.video-grid-loading {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.video-grid {
|
||||
gap: 9px;
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
FRONTEND_HOST="${FRONTEND_HOST:-0.0.0.0}"
|
||||
FRONTEND_PORT="${FRONTEND_PORT:-9191}"
|
||||
FRONTEND_MODE="${FRONTEND_MODE:-preview}"
|
||||
BACKEND_PORT="${BACKEND_PORT:-9192}"
|
||||
LOG_DIR="${LOG_DIR:-/tmp/video-site-91}"
|
||||
|
||||
@@ -18,6 +19,7 @@ Usage: ./start.sh [--restart|--stop|--status]
|
||||
Environment overrides:
|
||||
FRONTEND_HOST=$FRONTEND_HOST
|
||||
FRONTEND_PORT=$FRONTEND_PORT
|
||||
FRONTEND_MODE=$FRONTEND_MODE # preview (default, no HMR) or dev
|
||||
BACKEND_PORT=$BACKEND_PORT
|
||||
LOG_DIR=$LOG_DIR
|
||||
|
||||
@@ -116,11 +118,24 @@ start_frontend() {
|
||||
|
||||
need_cmd npm
|
||||
mkdir -p "$LOG_DIR"
|
||||
echo "starting frontend on $FRONTEND_HOST:$FRONTEND_PORT"
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
setsid nohup npm run dev -- --host "$FRONTEND_HOST" --port "$FRONTEND_PORT" >>"$FRONTEND_LOG" 2>&1 </dev/null &
|
||||
)
|
||||
if [[ "$FRONTEND_MODE" == "dev" ]]; then
|
||||
echo "starting frontend dev server on $FRONTEND_HOST:$FRONTEND_PORT"
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
setsid nohup npm run dev -- --host "$FRONTEND_HOST" --port "$FRONTEND_PORT" >>"$FRONTEND_LOG" 2>&1 </dev/null &
|
||||
)
|
||||
else
|
||||
echo "building frontend for preview mode"
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
npm run build >>"$FRONTEND_LOG" 2>&1
|
||||
)
|
||||
echo "starting frontend preview server on $FRONTEND_HOST:$FRONTEND_PORT"
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
setsid nohup npm run preview -- --host "$FRONTEND_HOST" --port "$FRONTEND_PORT" >>"$FRONTEND_LOG" 2>&1 </dev/null &
|
||||
)
|
||||
fi
|
||||
wait_for_port "frontend" "$FRONTEND_PORT"
|
||||
}
|
||||
|
||||
|
||||
+12
-5
@@ -2,6 +2,12 @@ import { defineConfig } from "vite";
|
||||
import react from "@vitejs/plugin-react";
|
||||
import path from "node:path";
|
||||
|
||||
const backendProxy = {
|
||||
"/api": "http://127.0.0.1:9192",
|
||||
"/p": "http://127.0.0.1:9192",
|
||||
"/admin/api": "http://127.0.0.1:9192",
|
||||
};
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
resolve: {
|
||||
@@ -12,10 +18,11 @@ export default defineConfig({
|
||||
server: {
|
||||
host: "0.0.0.0",
|
||||
port: 9191,
|
||||
proxy: {
|
||||
"/api": "http://127.0.0.1:9192",
|
||||
"/p": "http://127.0.0.1:9192",
|
||||
"/admin/api": "http://127.0.0.1:9192",
|
||||
},
|
||||
proxy: backendProxy,
|
||||
},
|
||||
preview: {
|
||||
host: "0.0.0.0",
|
||||
port: 9191,
|
||||
proxy: backendProxy,
|
||||
},
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user