mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-15 08:45:41 +08:00
Handle OneDrive rate limits and use preview server
This commit is contained in:
@@ -31,18 +31,21 @@ Linux / WSL 环境推荐用仓库根目录的脚本同时启动前后端:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
npm install
|
npm install
|
||||||
./start.sh # 前端 9191,后端 9192
|
./start.sh # 前端 9191,后端 9192;默认使用生产预览模式,无热更新
|
||||||
./start.sh --status # 查看运行状态
|
./start.sh --status # 查看运行状态
|
||||||
./start.sh --restart # 重启
|
./start.sh --restart # 重启
|
||||||
./start.sh --stop # 停止
|
./start.sh --stop # 停止
|
||||||
```
|
```
|
||||||
|
|
||||||
|
如果需要开发热更新,可临时使用 `FRONTEND_MODE=dev ./start.sh --restart`。
|
||||||
|
|
||||||
也可以分两个终端手动启动:
|
也可以分两个终端手动启动:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 前端
|
# 前端
|
||||||
npm install
|
npm install
|
||||||
npm run dev # 监听 http://127.0.0.1:9191
|
npm run build
|
||||||
|
npm run preview # 监听 http://127.0.0.1:9191,无热更新
|
||||||
|
|
||||||
# 后端(另开终端)
|
# 后端(另开终端)
|
||||||
cd backend
|
cd backend
|
||||||
@@ -55,7 +58,7 @@ go run ./cmd/server # 默认监听 127.0.0.1:9192,依赖已 vendor 入库
|
|||||||
- `backend/data/video-site.db`(SQLite)
|
- `backend/data/video-site.db`(SQLite)
|
||||||
- `backend/data/previews/`(teaser 和封面本地目录)
|
- `backend/data/previews/`(teaser 和封面本地目录)
|
||||||
|
|
||||||
Vite dev server 已配置把 `/api`、`/p`、`/admin/api` 反代到 `127.0.0.1:9192`。浏览器访问 `http://127.0.0.1:9191/` 进入前台,`/admin` 进入管理后台(默认 `admin` / `admin123`,请在 `backend/config.yaml` 里改)。如果本地已经存在旧的 `backend/config.yaml`,请确认 `server.listen` 与 Vite 代理端口一致。
|
Vite dev / preview server 都已配置把 `/api`、`/p`、`/admin/api` 反代到 `127.0.0.1:9192`。浏览器访问 `http://127.0.0.1:9191/` 进入前台,`/admin` 进入管理后台(默认 `admin` / `admin123`,请在 `backend/config.yaml` 里改)。如果本地已经存在旧的 `backend/config.yaml`,请确认 `server.listen` 与 Vite 代理端口一致。
|
||||||
|
|
||||||
## 目录
|
## 目录
|
||||||
|
|
||||||
@@ -118,7 +121,7 @@ OneDrive 当前采用 OpenList 在线 API 的续期方式,不要求用户提
|
|||||||
- 极短视频会按可容纳的完整 3 秒片段数自动降级
|
- 极短视频会按可容纳的完整 3 秒片段数自动降级
|
||||||
- 首次失败的任务标 `preview_status = failed`,不再自动重试;管理后台可手动重新生成
|
- 首次失败的任务标 `preview_status = failed`,不再自动重试;管理后台可手动重新生成
|
||||||
- 服务启动或网盘重新挂载时,如果 Teaser 开关已开启,会自动把历史 `pending` 任务重新入队,避免重启后停在“待生成”。
|
- 服务启动或网盘重新挂载时,如果 Teaser 开关已开启,会自动把历史 `pending` 任务重新入队,避免重启后停在“待生成”。
|
||||||
- OneDrive 直链生成 teaser 时可能触发 Microsoft 429 限流;这种失败会被记录为 `failed`,可稍后在管理后台重生。
|
- OneDrive 直链生成 teaser 时可能触发 Microsoft 429 限流;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。
|
||||||
- 详见 plan 15.12 节
|
- 详见 plan 15.12 节
|
||||||
|
|
||||||
## 常用管理能力
|
## 常用管理能力
|
||||||
|
|||||||
+6
-3
@@ -47,9 +47,11 @@ Git Bash / WSL 环境推荐从仓库根目录启动完整开发环境:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
npm install
|
npm install
|
||||||
./start.sh
|
./start.sh # 默认前端 production preview,无热更新
|
||||||
```
|
```
|
||||||
|
|
||||||
|
需要前端开发热更新时再用 `FRONTEND_MODE=dev ./start.sh --restart`。
|
||||||
|
|
||||||
PowerShell 下可以分两个终端手动启动,后端命令如下:
|
PowerShell 下可以分两个终端手动启动,后端命令如下:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
@@ -70,7 +72,8 @@ go run ./cmd/server
|
|||||||
`vite.config.ts` 已经把 `/api`、`/p`、`/admin/api` 代理到 `127.0.0.1:9192`。
|
`vite.config.ts` 已经把 `/api`、`/p`、`/admin/api` 代理到 `127.0.0.1:9192`。
|
||||||
|
|
||||||
```
|
```
|
||||||
npm run dev 前端 9191
|
npm run build 构建前端静态资源
|
||||||
|
npm run preview 前端 9191,无热更新
|
||||||
go run ./cmd/server 后端 9192
|
go run ./cmd/server 后端 9192
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -138,7 +141,7 @@ ffmpeg -ss <起点> -headers "UA/Cookie/Referer" -i <直链> \
|
|||||||
|
|
||||||
当前策略是每段固定 3 秒;30 秒以下最多 3 段,30 秒及以上固定 4 段;长视频在 20% 到 80% 区间均匀取段。优先把 teaser 上传回网盘的 `previews/` 目录;失败时保留本地 `data/previews/<videoID>.mp4` 作为兜底。
|
当前策略是每段固定 3 秒;30 秒以下最多 3 段,30 秒及以上固定 4 段;长视频在 20% 到 80% 区间均匀取段。优先把 teaser 上传回网盘的 `previews/` 目录;失败时保留本地 `data/previews/<videoID>.mp4` 作为兜底。
|
||||||
|
|
||||||
服务启动或网盘重新挂载时,如果 Teaser 开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 直链生成 teaser 时可能触发 Microsoft 429 限流;这类任务会标记为 `failed`,可稍后在视频管理页重生。
|
服务启动或网盘重新挂载时,如果 Teaser 开关已开启,后端会把历史 `pending` 任务重新入队,避免重启后长期停在“待生成”。OneDrive 直链生成 teaser 时可能触发 Microsoft 429 限流;后端会识别这类错误并让当前网盘进入冷却期,保留任务为 `pending`,避免连续请求触发更严重限流。
|
||||||
|
|
||||||
前端卡片的 `previewSrc` 统一指向 `/p/preview/<videoID>`,后端自动选择网盘代理或本地文件。
|
前端卡片的 `previewSrc` 统一指向 `/p/preview/<videoID>`,后端自动选择网盘代理或本地文件。
|
||||||
|
|
||||||
|
|||||||
@@ -64,3 +64,38 @@ type StreamLink struct {
|
|||||||
|
|
||||||
// ErrNotSupported 代表某家盘不支持某操作
|
// ErrNotSupported 代表某家盘不支持某操作
|
||||||
var ErrNotSupported = errors.New("operation not supported by this drive")
|
var ErrNotSupported = errors.New("operation not supported by this drive")
|
||||||
|
|
||||||
|
// RateLimitError 表示上游服务正在限流。RetryAfter 为 0 时由调用方选择默认冷却时间。
|
||||||
|
type RateLimitError struct {
|
||||||
|
Provider string
|
||||||
|
RetryAfter time.Duration
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *RateLimitError) Error() string {
|
||||||
|
if e == nil {
|
||||||
|
return "rate limited"
|
||||||
|
}
|
||||||
|
if e.Err != nil {
|
||||||
|
return e.Err.Error()
|
||||||
|
}
|
||||||
|
if e.Provider != "" {
|
||||||
|
return e.Provider + " rate limited"
|
||||||
|
}
|
||||||
|
return "rate limited"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *RateLimitError) Unwrap() error {
|
||||||
|
if e == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return e.Err
|
||||||
|
}
|
||||||
|
|
||||||
|
func RateLimitRetryAfter(err error) (time.Duration, bool) {
|
||||||
|
var rateLimit *RateLimitError
|
||||||
|
if errors.As(err, &rateLimit) {
|
||||||
|
return rateLimit.RetryAfter, true
|
||||||
|
}
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"path"
|
"path"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -265,6 +266,9 @@ func (d *Driver) requestOnce(ctx context.Context, rawURL, method string, configu
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if isRateLimitResponse(res, graphErr.Error.Code) {
|
||||||
|
return onedriveRateLimitError(res, graphErr.Error.Message)
|
||||||
|
}
|
||||||
if graphErr.Error.Code != "" {
|
if graphErr.Error.Code != "" {
|
||||||
if graphErr.Error.Code == "InvalidAuthenticationToken" && retry {
|
if graphErr.Error.Code == "InvalidAuthenticationToken" && retry {
|
||||||
if err := d.refresh(ctx); err != nil {
|
if err := d.refresh(ctx); err != nil {
|
||||||
@@ -298,6 +302,9 @@ func (d *Driver) refresh(ctx context.Context) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("onedrive refresh token: %w", err)
|
return fmt.Errorf("onedrive refresh token: %w", err)
|
||||||
}
|
}
|
||||||
|
if res.StatusCode() == http.StatusTooManyRequests {
|
||||||
|
return onedriveRateLimitError(res, "token refresh throttled")
|
||||||
|
}
|
||||||
if out.Text != "" {
|
if out.Text != "" {
|
||||||
return fmt.Errorf("onedrive refresh token: %s", out.Text)
|
return fmt.Errorf("onedrive refresh token: %s", out.Text)
|
||||||
}
|
}
|
||||||
@@ -321,6 +328,47 @@ func (d *Driver) refresh(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isRateLimitResponse(res *resty.Response, code string) bool {
|
||||||
|
if code == "TooManyRequests" || code == "activityLimitReached" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return res != nil && res.StatusCode() == http.StatusTooManyRequests
|
||||||
|
}
|
||||||
|
|
||||||
|
func onedriveRateLimitError(res *resty.Response, message string) error {
|
||||||
|
if strings.TrimSpace(message) == "" {
|
||||||
|
message = "onedrive rate limited"
|
||||||
|
}
|
||||||
|
if res != nil && strings.TrimSpace(res.String()) != "" {
|
||||||
|
message = fmt.Sprintf("%s: status=%d body=%s", message, res.StatusCode(), strings.TrimSpace(res.String()))
|
||||||
|
}
|
||||||
|
return &drives.RateLimitError{
|
||||||
|
Provider: "onedrive",
|
||||||
|
RetryAfter: parseRetryAfter(res),
|
||||||
|
Err: errors.New(message),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseRetryAfter(res *resty.Response) time.Duration {
|
||||||
|
if res == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
raw := strings.TrimSpace(res.Header().Get("Retry-After"))
|
||||||
|
if raw == "" {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if seconds, err := strconv.Atoi(raw); err == nil && seconds > 0 {
|
||||||
|
return time.Duration(seconds) * time.Second
|
||||||
|
}
|
||||||
|
if when, err := http.ParseTime(raw); err == nil {
|
||||||
|
d := time.Until(when)
|
||||||
|
if d > 0 {
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
func (d *Driver) driveBaseURL() string {
|
func (d *Driver) driveBaseURL() string {
|
||||||
if d.isSharePoint {
|
if d.isSharePoint {
|
||||||
return fmt.Sprintf("%s/v1.0/sites/%s/drive", d.apiBaseURL, url.PathEscape(d.siteID))
|
return fmt.Sprintf("%s/v1.0/sites/%s/drive", d.apiBaseURL, url.PathEscape(d.siteID))
|
||||||
|
|||||||
@@ -3,11 +3,13 @@ package onedrive
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/video-site/backend/internal/drives"
|
"github.com/video-site/backend/internal/drives"
|
||||||
)
|
)
|
||||||
@@ -174,6 +176,42 @@ func TestGraphItemWithoutFolderFacetIsNotDirectory(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGraph429ReturnsRateLimitErrorWithRetryAfter(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.Header().Set("Retry-After", "120")
|
||||||
|
w.WriteHeader(http.StatusTooManyRequests)
|
||||||
|
if err := json.NewEncoder(w).Encode(map[string]any{
|
||||||
|
"error": map[string]any{
|
||||||
|
"code": "TooManyRequests",
|
||||||
|
"message": "throttled",
|
||||||
|
},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("write json: %v", err)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
d := New(Config{
|
||||||
|
ID: "od-main",
|
||||||
|
AccessToken: "access-token",
|
||||||
|
RefreshToken: "refresh-token",
|
||||||
|
APIBaseURL: srv.URL,
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := d.List(context.Background(), "root")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("list succeeded, want rate limit error")
|
||||||
|
}
|
||||||
|
var rateLimit *drives.RateLimitError
|
||||||
|
if !errors.As(err, &rateLimit) {
|
||||||
|
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||||
|
}
|
||||||
|
if rateLimit.RetryAfter != 2*time.Minute {
|
||||||
|
t.Fatalf("retry after = %v, want 2m", rateLimit.RetryAfter)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestStatAndStreamURLUseDriveItemMetadata(t *testing.T) {
|
func TestStatAndStreamURLUseDriveItemMetadata(t *testing.T) {
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package preview
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
@@ -11,6 +12,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/video-site/backend/internal/catalog"
|
"github.com/video-site/backend/internal/catalog"
|
||||||
@@ -228,7 +230,7 @@ func (g *Generator) GenerateThumbnail(ctx context.Context, link *drives.StreamLi
|
|||||||
out, err := cmd.CombinedOutput()
|
out, err := cmd.CombinedOutput()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
os.Remove(dst)
|
os.Remove(dst)
|
||||||
return "", fmt.Errorf("ffmpeg thumb: %w, stderr: %s", err, string(out))
|
return "", ffmpegCommandError("ffmpeg thumb", err, out)
|
||||||
}
|
}
|
||||||
if info, statErr := os.Stat(dst); statErr != nil || info.Size() == 0 {
|
if info, statErr := os.Stat(dst); statErr != nil || info.Size() == 0 {
|
||||||
os.Remove(dst)
|
os.Remove(dst)
|
||||||
@@ -256,9 +258,9 @@ func (g *Generator) Probe(ctx context.Context, link *drives.StreamLink) (float64
|
|||||||
args = append(args, link.URL)
|
args = append(args, link.URL)
|
||||||
|
|
||||||
cmd := exec.CommandContext(ctx2, g.cfg.FFprobePath, args...)
|
cmd := exec.CommandContext(ctx2, g.cfg.FFprobePath, args...)
|
||||||
out, err := cmd.Output()
|
out, err := cmd.CombinedOutput()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, fmt.Errorf("ffprobe: %w", err)
|
return 0, ffmpegCommandError("ffprobe", err, out)
|
||||||
}
|
}
|
||||||
raw := strings.TrimSpace(string(out))
|
raw := strings.TrimSpace(string(out))
|
||||||
if raw == "" || raw == "N/A" {
|
if raw == "" || raw == "N/A" {
|
||||||
@@ -362,7 +364,7 @@ func (g *Generator) Generate(ctx context.Context, link *drives.StreamLink, durat
|
|||||||
out, err := cmd.CombinedOutput()
|
out, err := cmd.CombinedOutput()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
os.Remove(tmpPath)
|
os.Remove(tmpPath)
|
||||||
return "", fmt.Errorf("ffmpeg: %w, stderr: %s", err, string(out))
|
return "", ffmpegCommandError("ffmpeg", err, out)
|
||||||
}
|
}
|
||||||
|
|
||||||
if info, statErr := os.Stat(tmpPath); statErr != nil || info.Size() == 0 {
|
if info, statErr := os.Stat(tmpPath); statErr != nil || info.Size() == 0 {
|
||||||
@@ -372,6 +374,48 @@ func (g *Generator) Generate(ctx context.Context, link *drives.StreamLink, durat
|
|||||||
return tmpPath, nil
|
return tmpPath, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ffmpegCommandError(tool string, err error, output []byte) error {
|
||||||
|
msg := fmt.Sprintf("%s: %v, stderr: %s", tool, err, redactURLs(string(output)))
|
||||||
|
wrapped := errors.New(msg)
|
||||||
|
if ffmpegOutputLooksRateLimited(output) {
|
||||||
|
return &drives.RateLimitError{
|
||||||
|
Provider: "media source",
|
||||||
|
Err: wrapped,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return wrapped
|
||||||
|
}
|
||||||
|
|
||||||
|
func redactURLs(text string) string {
|
||||||
|
fields := strings.Fields(text)
|
||||||
|
for i, field := range fields {
|
||||||
|
if strings.HasPrefix(field, "http://") || strings.HasPrefix(field, "https://") {
|
||||||
|
suffix := ""
|
||||||
|
for len(field) > 0 {
|
||||||
|
last := field[len(field)-1]
|
||||||
|
if last != '.' && last != ',' && last != ';' && last != ')' {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
suffix = string(last) + suffix
|
||||||
|
field = field[:len(field)-1]
|
||||||
|
}
|
||||||
|
fields[i] = "https://<redacted>" + suffix
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(fields, " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func ffmpegOutputLooksRateLimited(output []byte) bool {
|
||||||
|
text := strings.ToLower(string(output))
|
||||||
|
if !strings.Contains(text, "429") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return strings.Contains(text, "too many requests") ||
|
||||||
|
strings.Contains(text, "rate limit") ||
|
||||||
|
strings.Contains(text, "rate-limit") ||
|
||||||
|
strings.Contains(text, "server returned 429")
|
||||||
|
}
|
||||||
|
|
||||||
// --- 本地落盘 ---
|
// --- 本地落盘 ---
|
||||||
|
|
||||||
// MoveToLocal 把临时文件改名到稳定位置,返回最终路径
|
// MoveToLocal 把临时文件改名到稳定位置,返回最终路径
|
||||||
@@ -410,6 +454,9 @@ type Worker struct {
|
|||||||
Drive drives.Drive
|
Drive drives.Drive
|
||||||
RemoteDir string
|
RemoteDir string
|
||||||
ch chan *catalog.Video
|
ch chan *catalog.Video
|
||||||
|
|
||||||
|
RateLimitCooldown time.Duration
|
||||||
|
rateLimit rateLimitState
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewWorker(gen TeaserGenerator, cat *catalog.Catalog, drv drives.Drive, remoteDir string) *Worker {
|
func NewWorker(gen TeaserGenerator, cat *catalog.Catalog, drv drives.Drive, remoteDir string) *Worker {
|
||||||
@@ -451,6 +498,46 @@ type ThumbWorker struct {
|
|||||||
Catalog *catalog.Catalog
|
Catalog *catalog.Catalog
|
||||||
Drive drives.Drive
|
Drive drives.Drive
|
||||||
ch chan *catalog.Video
|
ch chan *catalog.Video
|
||||||
|
|
||||||
|
RateLimitCooldown time.Duration
|
||||||
|
rateLimit rateLimitState
|
||||||
|
}
|
||||||
|
|
||||||
|
const defaultRateLimitCooldown = 30 * time.Minute
|
||||||
|
|
||||||
|
type rateLimitState struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
until time.Time
|
||||||
|
lastSkipLog time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *rateLimitState) active(now time.Time) (time.Time, bool, bool) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.until.IsZero() || !now.Before(s.until) {
|
||||||
|
return time.Time{}, false, false
|
||||||
|
}
|
||||||
|
shouldLog := s.lastSkipLog.IsZero() || now.Sub(s.lastSkipLog) >= 5*time.Minute
|
||||||
|
if shouldLog {
|
||||||
|
s.lastSkipLog = now
|
||||||
|
}
|
||||||
|
return s.until, true, shouldLog
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *rateLimitState) pause(now time.Time, d time.Duration) time.Time {
|
||||||
|
if d <= 0 {
|
||||||
|
d = defaultRateLimitCooldown
|
||||||
|
}
|
||||||
|
until := now.Add(d)
|
||||||
|
s.mu.Lock()
|
||||||
|
if until.After(s.until) {
|
||||||
|
s.until = until
|
||||||
|
} else {
|
||||||
|
until = s.until
|
||||||
|
}
|
||||||
|
s.lastSkipLog = time.Time{}
|
||||||
|
s.mu.Unlock()
|
||||||
|
return until
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewThumbWorker(gen ThumbnailGenerator, cat *catalog.Catalog, drv drives.Drive) *ThumbWorker {
|
func NewThumbWorker(gen ThumbnailGenerator, cat *catalog.Catalog, drv drives.Drive) *ThumbWorker {
|
||||||
@@ -520,9 +607,63 @@ func (w *ThumbWorker) Run(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (w *Worker) skipIfRateLimited(v *catalog.Video) bool {
|
||||||
|
until, ok, shouldLog := w.rateLimit.active(time.Now())
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if shouldLog {
|
||||||
|
log.Printf("[preview] drive=%s rate-limited until=%s; skip queued videos and keep them pending", w.Drive.ID(), until.Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Worker) pauseForRateLimit(err error, step, title string) bool {
|
||||||
|
retryAfter, ok := drives.RateLimitRetryAfter(err)
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if retryAfter <= 0 {
|
||||||
|
retryAfter = w.RateLimitCooldown
|
||||||
|
}
|
||||||
|
until := w.rateLimit.pause(time.Now(), retryAfter)
|
||||||
|
log.Printf("[preview] drive=%s rate-limited until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *ThumbWorker) skipIfRateLimited(v *catalog.Video) bool {
|
||||||
|
until, ok, shouldLog := w.rateLimit.active(time.Now())
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if shouldLog {
|
||||||
|
log.Printf("[thumb] drive=%s rate-limited until=%s; skip queued thumbnails and keep them pending", w.Drive.ID(), until.Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *ThumbWorker) pauseForRateLimit(err error, step, title string) bool {
|
||||||
|
retryAfter, ok := drives.RateLimitRetryAfter(err)
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if retryAfter <= 0 {
|
||||||
|
retryAfter = w.RateLimitCooldown
|
||||||
|
}
|
||||||
|
until := w.rateLimit.pause(time.Now(), retryAfter)
|
||||||
|
log.Printf("[thumb] drive=%s rate-limited until=%s step=%s video=%s: %v", w.Drive.ID(), until.Format(time.RFC3339), step, title, err)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
|
func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
|
||||||
|
if w.skipIfRateLimited(v) {
|
||||||
|
return
|
||||||
|
}
|
||||||
link, err := w.Drive.StreamURL(ctx, v.FileID)
|
link, err := w.Drive.StreamURL(ctx, v.FileID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if w.pauseForRateLimit(err, "streamURL", v.Title) {
|
||||||
|
return
|
||||||
|
}
|
||||||
log.Printf("[thumb] streamURL %s: %v", v.Title, err)
|
log.Printf("[thumb] streamURL %s: %v", v.Title, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -534,10 +675,15 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
|
|||||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||||
DurationSeconds: int(dur),
|
DurationSeconds: int(dur),
|
||||||
})
|
})
|
||||||
|
} else if err != nil && w.pauseForRateLimit(err, "probe", v.Title) {
|
||||||
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := w.Gen.GenerateThumbnail(ctx, link, v.ID, duration); err != nil {
|
if _, err := w.Gen.GenerateThumbnail(ctx, link, v.ID, duration); err != nil {
|
||||||
|
if w.pauseForRateLimit(err, "generate", v.Title) {
|
||||||
|
return
|
||||||
|
}
|
||||||
log.Printf("[thumb] generate %s: %v", v.Title, err)
|
log.Printf("[thumb] generate %s: %v", v.Title, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -548,8 +694,14 @@ func (w *ThumbWorker) process(ctx context.Context, v *catalog.Video) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (w *Worker) process(ctx context.Context, v *catalog.Video) {
|
func (w *Worker) process(ctx context.Context, v *catalog.Video) {
|
||||||
|
if w.skipIfRateLimited(v) {
|
||||||
|
return
|
||||||
|
}
|
||||||
link, err := w.Drive.StreamURL(ctx, v.FileID)
|
link, err := w.Drive.StreamURL(ctx, v.FileID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if w.pauseForRateLimit(err, "streamURL", v.Title) {
|
||||||
|
return
|
||||||
|
}
|
||||||
log.Printf("[preview] streamURL %s: %v", v.Title, err)
|
log.Printf("[preview] streamURL %s: %v", v.Title, err)
|
||||||
w.Catalog.UpdatePreview(ctx, v.ID, "", "", "failed")
|
w.Catalog.UpdatePreview(ctx, v.ID, "", "", "failed")
|
||||||
return
|
return
|
||||||
@@ -563,12 +715,17 @@ func (w *Worker) process(ctx context.Context, v *catalog.Video) {
|
|||||||
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
_ = w.Catalog.UpdateVideoMeta(ctx, v.ID, catalog.VideoMetaPatch{
|
||||||
DurationSeconds: int(dur),
|
DurationSeconds: int(dur),
|
||||||
})
|
})
|
||||||
|
} else if err != nil && w.pauseForRateLimit(err, "probe", v.Title) {
|
||||||
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2) teaser
|
// 2) teaser
|
||||||
tmp, err := w.Gen.Generate(ctx, link, duration)
|
tmp, err := w.Gen.Generate(ctx, link, duration)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if w.pauseForRateLimit(err, "generate", v.Title) {
|
||||||
|
return
|
||||||
|
}
|
||||||
log.Printf("[preview] generate %s: %v", v.Title, err)
|
log.Printf("[preview] generate %s: %v", v.Title, err)
|
||||||
w.Catalog.UpdatePreview(ctx, v.ID, "", "", "failed")
|
w.Catalog.UpdatePreview(ctx, v.ID, "", "", "failed")
|
||||||
return
|
return
|
||||||
@@ -584,10 +741,14 @@ func (w *Worker) process(ctx context.Context, v *catalog.Video) {
|
|||||||
if w.RemoteDir != "" {
|
if w.RemoteDir != "" {
|
||||||
if fid, uerr := w.uploadToDrive(ctx, v.ID, local); uerr == nil {
|
if fid, uerr := w.uploadToDrive(ctx, v.ID, local); uerr == nil {
|
||||||
previewFileID = fid
|
previewFileID = fid
|
||||||
|
} else {
|
||||||
|
if w.pauseForRateLimit(uerr, "upload", v.Title) {
|
||||||
|
log.Printf("[preview] upload %s: %v (local only; drive cooling down)", v.Title, uerr)
|
||||||
} else {
|
} else {
|
||||||
log.Printf("[preview] upload %s: %v (local only)", v.Title, uerr)
|
log.Printf("[preview] upload %s: %v (local only)", v.Title, uerr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
removePreviousLocalTeaser(v.PreviewLocal, local)
|
removePreviousLocalTeaser(v.PreviewLocal, local)
|
||||||
w.Catalog.UpdatePreview(ctx, v.ID, previewFileID, local, "ready")
|
w.Catalog.UpdatePreview(ctx, v.ID, previewFileID, local, "ready")
|
||||||
log.Printf("[preview] ready %s (duration=%.1fs)", v.Title, duration)
|
log.Printf("[preview] ready %s (duration=%.1fs)", v.Title, duration)
|
||||||
|
|||||||
@@ -1,8 +1,12 @@
|
|||||||
package preview
|
package preview
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"math"
|
"math"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/video-site/backend/internal/drives"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestNewDefaultsToThreeSecondTeaserSegments(t *testing.T) {
|
func TestNewDefaultsToThreeSecondTeaserSegments(t *testing.T) {
|
||||||
@@ -85,3 +89,25 @@ func TestShortVideoPreviewPlanReturnsNoSegmentsWhenOneSegmentCannotFit(t *testin
|
|||||||
t.Fatalf("eachSec = %.2f, want 3", plan.eachSec)
|
t.Fatalf("eachSec = %.2f, want 3", plan.eachSec)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFFmpeg429OutputBecomesRateLimitError(t *testing.T) {
|
||||||
|
err := ffmpegCommandError("ffmpeg", errors.New("exit status 8"), []byte("Server returned 429 Too Many Requests"))
|
||||||
|
var rateLimit *drives.RateLimitError
|
||||||
|
if !errors.As(err, &rateLimit) {
|
||||||
|
t.Fatalf("error = %T %[1]v, want RateLimitError", err)
|
||||||
|
}
|
||||||
|
if rateLimit.RetryAfter != 0 {
|
||||||
|
t.Fatalf("retry after = %v, want none from ffmpeg stderr", rateLimit.RetryAfter)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFmpegCommandErrorRedactsSignedURLs(t *testing.T) {
|
||||||
|
err := ffmpegCommandError("ffmpeg", errors.New("exit status 8"), []byte("Error opening input file https://download.example/file.mp4?tempauth=secret."))
|
||||||
|
got := err.Error()
|
||||||
|
if strings.Contains(got, "tempauth=secret") {
|
||||||
|
t.Fatalf("error leaked signed URL: %s", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got, "https://<redacted>.") {
|
||||||
|
t.Fatalf("error = %q, want redacted URL with punctuation preserved", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package preview
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -108,6 +109,52 @@ func TestPreviewWorkerRemovesPreviousLocalTeaserAfterNewTeaserIsReady(t *testing
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPreviewWorkerRateLimitLeavesCurrentPendingAndSkipsNextVideo(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
cat, first := seedPreviewTestVideo(t, "preview-rate-limit-1")
|
||||||
|
second := *first
|
||||||
|
second.ID = "preview-rate-limit-2"
|
||||||
|
second.FileID = "file-id-2"
|
||||||
|
if err := cat.UpsertVideo(ctx, &second); err != nil {
|
||||||
|
t.Fatalf("seed second video: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
gen := &fakeTeaserGenerator{
|
||||||
|
generateErr: &drives.RateLimitError{
|
||||||
|
Provider: "onedrive",
|
||||||
|
RetryAfter: 2 * time.Hour,
|
||||||
|
Err: errors.New("429 Too Many Requests"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
drv := &previewFakeDrive{}
|
||||||
|
worker := NewWorker(gen, cat, drv, "")
|
||||||
|
|
||||||
|
worker.process(ctx, first)
|
||||||
|
gotFirst, err := cat.GetVideo(ctx, first.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("get first video: %v", err)
|
||||||
|
}
|
||||||
|
if gotFirst.PreviewStatus != "pending" {
|
||||||
|
t.Fatalf("first preview status = %q, want pending after rate limit", gotFirst.PreviewStatus)
|
||||||
|
}
|
||||||
|
if gen.generateCalls != 1 {
|
||||||
|
t.Fatalf("generate calls = %d, want 1", gen.generateCalls)
|
||||||
|
}
|
||||||
|
|
||||||
|
gen.generateErr = nil
|
||||||
|
worker.process(ctx, &second)
|
||||||
|
gotSecond, err := cat.GetVideo(ctx, second.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("get second video: %v", err)
|
||||||
|
}
|
||||||
|
if gotSecond.PreviewStatus != "pending" {
|
||||||
|
t.Fatalf("second preview status = %q, want pending while drive is cooling down", gotSecond.PreviewStatus)
|
||||||
|
}
|
||||||
|
if gen.generateCalls != 1 {
|
||||||
|
t.Fatalf("generate calls = %d, want second video skipped during cooldown", gen.generateCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func seedPreviewTestVideo(t *testing.T, id string) (*catalog.Catalog, *catalog.Video) {
|
func seedPreviewTestVideo(t *testing.T, id string) (*catalog.Catalog, *catalog.Video) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -154,6 +201,8 @@ func (g *fakeThumbGenerator) GenerateThumbnail(_ context.Context, _ *drives.Stre
|
|||||||
|
|
||||||
type fakeTeaserGenerator struct {
|
type fakeTeaserGenerator struct {
|
||||||
localPath string
|
localPath string
|
||||||
|
generateErr error
|
||||||
|
generateCalls int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (g *fakeTeaserGenerator) Probe(context.Context, *drives.StreamLink) (float64, error) {
|
func (g *fakeTeaserGenerator) Probe(context.Context, *drives.StreamLink) (float64, error) {
|
||||||
@@ -161,6 +210,10 @@ func (g *fakeTeaserGenerator) Probe(context.Context, *drives.StreamLink) (float6
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (g *fakeTeaserGenerator) Generate(context.Context, *drives.StreamLink, float64) (string, error) {
|
func (g *fakeTeaserGenerator) Generate(context.Context, *drives.StreamLink, float64) (string, error) {
|
||||||
|
g.generateCalls++
|
||||||
|
if g.generateErr != nil {
|
||||||
|
return "", g.generateErr
|
||||||
|
}
|
||||||
return "/tmp/source-teaser.mp4", nil
|
return "/tmp/source-teaser.mp4", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -24,12 +24,6 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@media (max-width: 380px) {
|
|
||||||
.video-grid {
|
|
||||||
grid-template-columns: 1fr;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
.video-grid.is-compact {
|
.video-grid.is-compact {
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
@@ -332,6 +326,10 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
@media (max-width: 640px) {
|
@media (max-width: 640px) {
|
||||||
|
.video-grid-loading {
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
}
|
||||||
|
|
||||||
.video-grid {
|
.video-grid {
|
||||||
gap: 9px;
|
gap: 9px;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|||||||
|
|
||||||
FRONTEND_HOST="${FRONTEND_HOST:-0.0.0.0}"
|
FRONTEND_HOST="${FRONTEND_HOST:-0.0.0.0}"
|
||||||
FRONTEND_PORT="${FRONTEND_PORT:-9191}"
|
FRONTEND_PORT="${FRONTEND_PORT:-9191}"
|
||||||
|
FRONTEND_MODE="${FRONTEND_MODE:-preview}"
|
||||||
BACKEND_PORT="${BACKEND_PORT:-9192}"
|
BACKEND_PORT="${BACKEND_PORT:-9192}"
|
||||||
LOG_DIR="${LOG_DIR:-/tmp/video-site-91}"
|
LOG_DIR="${LOG_DIR:-/tmp/video-site-91}"
|
||||||
|
|
||||||
@@ -18,6 +19,7 @@ Usage: ./start.sh [--restart|--stop|--status]
|
|||||||
Environment overrides:
|
Environment overrides:
|
||||||
FRONTEND_HOST=$FRONTEND_HOST
|
FRONTEND_HOST=$FRONTEND_HOST
|
||||||
FRONTEND_PORT=$FRONTEND_PORT
|
FRONTEND_PORT=$FRONTEND_PORT
|
||||||
|
FRONTEND_MODE=$FRONTEND_MODE # preview (default, no HMR) or dev
|
||||||
BACKEND_PORT=$BACKEND_PORT
|
BACKEND_PORT=$BACKEND_PORT
|
||||||
LOG_DIR=$LOG_DIR
|
LOG_DIR=$LOG_DIR
|
||||||
|
|
||||||
@@ -116,11 +118,24 @@ start_frontend() {
|
|||||||
|
|
||||||
need_cmd npm
|
need_cmd npm
|
||||||
mkdir -p "$LOG_DIR"
|
mkdir -p "$LOG_DIR"
|
||||||
echo "starting frontend on $FRONTEND_HOST:$FRONTEND_PORT"
|
if [[ "$FRONTEND_MODE" == "dev" ]]; then
|
||||||
|
echo "starting frontend dev server on $FRONTEND_HOST:$FRONTEND_PORT"
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
setsid nohup npm run dev -- --host "$FRONTEND_HOST" --port "$FRONTEND_PORT" >>"$FRONTEND_LOG" 2>&1 </dev/null &
|
setsid nohup npm run dev -- --host "$FRONTEND_HOST" --port "$FRONTEND_PORT" >>"$FRONTEND_LOG" 2>&1 </dev/null &
|
||||||
)
|
)
|
||||||
|
else
|
||||||
|
echo "building frontend for preview mode"
|
||||||
|
(
|
||||||
|
cd "$ROOT_DIR"
|
||||||
|
npm run build >>"$FRONTEND_LOG" 2>&1
|
||||||
|
)
|
||||||
|
echo "starting frontend preview server on $FRONTEND_HOST:$FRONTEND_PORT"
|
||||||
|
(
|
||||||
|
cd "$ROOT_DIR"
|
||||||
|
setsid nohup npm run preview -- --host "$FRONTEND_HOST" --port "$FRONTEND_PORT" >>"$FRONTEND_LOG" 2>&1 </dev/null &
|
||||||
|
)
|
||||||
|
fi
|
||||||
wait_for_port "frontend" "$FRONTEND_PORT"
|
wait_for_port "frontend" "$FRONTEND_PORT"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+11
-4
@@ -2,6 +2,12 @@ import { defineConfig } from "vite";
|
|||||||
import react from "@vitejs/plugin-react";
|
import react from "@vitejs/plugin-react";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
|
|
||||||
|
const backendProxy = {
|
||||||
|
"/api": "http://127.0.0.1:9192",
|
||||||
|
"/p": "http://127.0.0.1:9192",
|
||||||
|
"/admin/api": "http://127.0.0.1:9192",
|
||||||
|
};
|
||||||
|
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
plugins: [react()],
|
plugins: [react()],
|
||||||
resolve: {
|
resolve: {
|
||||||
@@ -12,10 +18,11 @@ export default defineConfig({
|
|||||||
server: {
|
server: {
|
||||||
host: "0.0.0.0",
|
host: "0.0.0.0",
|
||||||
port: 9191,
|
port: 9191,
|
||||||
proxy: {
|
proxy: backendProxy,
|
||||||
"/api": "http://127.0.0.1:9192",
|
|
||||||
"/p": "http://127.0.0.1:9192",
|
|
||||||
"/admin/api": "http://127.0.0.1:9192",
|
|
||||||
},
|
},
|
||||||
|
preview: {
|
||||||
|
host: "0.0.0.0",
|
||||||
|
port: 9191,
|
||||||
|
proxy: backendProxy,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user