mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-15 08:45:41 +08:00
feat: support spider91 uploads to OneDrive
This commit is contained in:
@@ -285,10 +285,10 @@ type App struct {
|
||||
// 全站主题("dark" | "pink"),从 DB 读
|
||||
theme string
|
||||
// 显式指定的 spider91 上传目标 drive ID。
|
||||
// 空字符串表示本地保存不上传,不再自动挑选 pikpak/p115 drive。
|
||||
// 空字符串表示本地保存不上传,不再自动挑选 pikpak/p115/onedrive drive。
|
||||
spider91UploadDriveID string
|
||||
|
||||
// spider91Migrator 周期把 spider91 视频上传到目标 drive(PikPak 或 115)。
|
||||
// spider91Migrator 周期把 spider91 视频上传到目标 drive(PikPak、115 或 OneDrive)。
|
||||
spider91Migrator *spider91migrate.Migrator
|
||||
|
||||
// nightlyRunner 是凌晨流水线调度器:每天 cron_hour 串行跑扫盘 → 91 爬虫 → 迁移。
|
||||
@@ -379,7 +379,7 @@ func (a *App) loadTheme(ctx context.Context) {
|
||||
}
|
||||
|
||||
// Spider91UploadDriveID 返回当前配置的 spider91 上传目标 drive ID。
|
||||
// 空字符串表示本地保存不上传;只有管理员显式选择 pikpak/p115 drive 时才迁移上传。
|
||||
// 空字符串表示本地保存不上传;只有管理员显式选择 pikpak/p115/onedrive drive 时才迁移上传。
|
||||
func (a *App) Spider91UploadDriveID() string {
|
||||
a.mu.Lock()
|
||||
explicit := a.spider91UploadDriveID
|
||||
@@ -396,7 +396,7 @@ func (a *App) Spider91UploadDriveID() string {
|
||||
|
||||
// SetSpider91UploadDriveID 设置 spider91 上传目标 drive ID 并持久化。
|
||||
// 接受空字符串(本地保存不上传)。
|
||||
// 设置一个不存在或 kind 不是 pikpak / p115 的 drive 会返回错误。
|
||||
// 设置一个不存在或 kind 不是 pikpak / p115 / onedrive 的 drive 会返回错误。
|
||||
func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) error {
|
||||
driveID = strings.TrimSpace(driveID)
|
||||
if driveID != "" {
|
||||
@@ -405,7 +405,7 @@ func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) erro
|
||||
return fmt.Errorf("drive %q not found", driveID)
|
||||
}
|
||||
if !isSpider91UploadKind(d.Kind()) {
|
||||
return fmt.Errorf("drive %q kind=%s, only pikpak or p115 can be spider91 upload target", driveID, d.Kind())
|
||||
return fmt.Errorf("drive %q kind=%s, only pikpak, p115 or onedrive can be spider91 upload target", driveID, d.Kind())
|
||||
}
|
||||
}
|
||||
a.mu.Lock()
|
||||
@@ -417,7 +417,7 @@ func (a *App) SetSpider91UploadDriveID(ctx context.Context, driveID string) erro
|
||||
// isSpider91UploadKind 是 spider91 迁移目标盘的 allowlist。
|
||||
// 与 spider91migrate.adaptUploadTarget 的支持范围保持一致。
|
||||
func isSpider91UploadKind(kind string) bool {
|
||||
return kind == "pikpak" || kind == "p115"
|
||||
return kind == "pikpak" || kind == "p115" || kind == "onedrive"
|
||||
}
|
||||
|
||||
// loadSpider91UploadDriveID 从 DB 读上传目标 drive ID 设置;不存在时使用空串。
|
||||
|
||||
@@ -38,6 +38,7 @@ func TestSpider91IntCredFallbacks(t *testing.T) {
|
||||
func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
|
||||
reg := proxy.NewRegistry()
|
||||
reg.Set("p115-one", &spider91UploadTargetFakeDrive{id: "p115-one", kind: "p115"})
|
||||
reg.Set("onedrive-one", &spider91UploadTargetFakeDrive{id: "onedrive-one", kind: "onedrive"})
|
||||
|
||||
app := &App{registry: reg}
|
||||
if got := app.Spider91UploadDriveID(); got != "" {
|
||||
@@ -49,6 +50,11 @@ func TestSpider91UploadDriveIDDoesNotAutoSelectTarget(t *testing.T) {
|
||||
t.Fatalf("explicit upload target = %q, want p115-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "onedrive-one"
|
||||
if got := app.Spider91UploadDriveID(); got != "onedrive-one" {
|
||||
t.Fatalf("explicit onedrive upload target = %q, want onedrive-one", got)
|
||||
}
|
||||
|
||||
app.spider91UploadDriveID = "missing"
|
||||
if got := app.Spider91UploadDriveID(); got != "" {
|
||||
t.Fatalf("missing upload target = %q, want empty", got)
|
||||
|
||||
@@ -3,6 +3,9 @@ package onedrive
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -20,10 +23,17 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
maxSmallUploadSize = 250 * 1024 * 1024
|
||||
defaultRenewAPIURL = "https://api.oplist.org/onedrive/renewapi"
|
||||
onedriveListCooldown = 5 * time.Minute
|
||||
onedriveListInterval = 1 * time.Second
|
||||
maxSmallUploadSize = 250 * 1024 * 1024
|
||||
defaultUploadSessionChunk = 10 * 1024 * 1024
|
||||
uploadSessionRetryAttempts = 3
|
||||
defaultRenewAPIURL = "https://api.oplist.org/onedrive/renewapi"
|
||||
onedriveListCooldown = 5 * time.Minute
|
||||
onedriveListInterval = 1 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
smallUploadThreshold = int64(maxSmallUploadSize)
|
||||
uploadSessionChunk = int64(defaultUploadSessionChunk)
|
||||
)
|
||||
|
||||
type Driver struct {
|
||||
@@ -216,15 +226,49 @@ func (d *Driver) StreamURL(ctx context.Context, fileID string) (*drives.StreamLi
|
||||
}
|
||||
|
||||
func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader, size int64) (string, error) {
|
||||
res, err := d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return res.FileID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
parentID, name, err := d.normalizeUploadArgs(parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
threshold := smallUploadThreshold
|
||||
if threshold <= 0 {
|
||||
threshold = maxSmallUploadSize
|
||||
}
|
||||
if size <= threshold {
|
||||
return d.uploadSmallAndReportHash(ctx, parentID, name, r, size, threshold)
|
||||
}
|
||||
return d.uploadSessionAndReportHash(ctx, parentID, name, r, size)
|
||||
}
|
||||
|
||||
func (d *Driver) normalizeUploadArgs(parentID, name string, r io.Reader, size int64) (string, string, error) {
|
||||
if r == nil {
|
||||
return "", "", errors.New("onedrive upload: body is required")
|
||||
}
|
||||
if size < 0 {
|
||||
return "", "", fmt.Errorf("onedrive upload: invalid size %d", size)
|
||||
}
|
||||
if parentID == "" {
|
||||
parentID = d.rootID
|
||||
}
|
||||
if size > maxSmallUploadSize {
|
||||
return "", fmt.Errorf("onedrive upload: files over %d bytes require upload session", maxSmallUploadSize)
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return "", "", errors.New("onedrive upload: empty file name")
|
||||
}
|
||||
data, err := readSmallUpload(r)
|
||||
return parentID, name, nil
|
||||
}
|
||||
|
||||
func (d *Driver) uploadSmallAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size, limit int64) (UploadResult, error) {
|
||||
data, hash, actualSize, err := readSmallUpload(r, size, limit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return UploadResult{}, err
|
||||
}
|
||||
u := fmt.Sprintf("%s/items/%s:/%s:/content", d.driveBaseURL(), url.PathEscape(parentID), url.PathEscape(name))
|
||||
var item graphItem
|
||||
@@ -233,26 +277,159 @@ func (d *Driver) Upload(ctx context.Context, parentID, name string, r io.Reader,
|
||||
req.SetContentLength(true)
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("onedrive upload: %w", err)
|
||||
return UploadResult{}, fmt.Errorf("onedrive upload: %w", err)
|
||||
}
|
||||
if item.ID == "" {
|
||||
return "", errors.New("onedrive upload: empty item id")
|
||||
return UploadResult{}, errors.New("onedrive upload: empty item id")
|
||||
}
|
||||
return item.ID, nil
|
||||
return UploadResult{FileID: item.ID, Hash: hash, Size: actualSize}, nil
|
||||
}
|
||||
|
||||
func readSmallUpload(r io.Reader) ([]byte, error) {
|
||||
if r == nil {
|
||||
return nil, errors.New("onedrive upload: body is required")
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(r, maxSmallUploadSize+1))
|
||||
func (d *Driver) uploadSessionAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
session, err := d.createUploadSession(ctx, parentID, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("onedrive upload: read body: %w", err)
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if len(data) > maxSmallUploadSize {
|
||||
return nil, fmt.Errorf("onedrive upload: files over %d bytes require upload session", maxSmallUploadSize)
|
||||
if strings.TrimSpace(session.UploadURL) == "" {
|
||||
return UploadResult{}, errors.New("onedrive upload session: empty upload url")
|
||||
}
|
||||
return data, nil
|
||||
|
||||
chunkSize := uploadSessionChunk
|
||||
if chunkSize <= 0 {
|
||||
chunkSize = defaultUploadSessionChunk
|
||||
}
|
||||
buf := make([]byte, int(chunkSize))
|
||||
hasher := sha1.New()
|
||||
var finalItem graphItem
|
||||
var offset int64
|
||||
for offset < size {
|
||||
partSize := minInt64(chunkSize, size-offset)
|
||||
chunk := buf[:int(partSize)]
|
||||
n, err := io.ReadFull(r, chunk)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return UploadResult{}, fmt.Errorf("onedrive upload: size mismatch: declared %d, copied %d", size, offset+int64(n))
|
||||
}
|
||||
return UploadResult{}, fmt.Errorf("onedrive upload: read body: %w", err)
|
||||
}
|
||||
chunk = chunk[:n]
|
||||
_, _ = hasher.Write(chunk)
|
||||
item, err := d.putUploadSessionChunkWithRetry(ctx, session.UploadURL, offset, size, chunk)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
if item != nil {
|
||||
finalItem = *item
|
||||
}
|
||||
offset += int64(n)
|
||||
}
|
||||
if finalItem.ID == "" {
|
||||
return UploadResult{}, errors.New("onedrive upload session: empty item id")
|
||||
}
|
||||
return UploadResult{
|
||||
FileID: finalItem.ID,
|
||||
Hash: hex.EncodeToString(hasher.Sum(nil)),
|
||||
Size: offset,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *Driver) createUploadSession(ctx context.Context, parentID, name string) (uploadSessionResp, error) {
|
||||
u := fmt.Sprintf("%s/items/%s:/%s:/createUploadSession", d.driveBaseURL(), url.PathEscape(parentID), url.PathEscape(name))
|
||||
body := map[string]any{
|
||||
"item": map[string]any{
|
||||
"@microsoft.graph.conflictBehavior": "rename",
|
||||
},
|
||||
}
|
||||
var out uploadSessionResp
|
||||
err := d.request(ctx, u, http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(body)
|
||||
}, &out)
|
||||
if err != nil {
|
||||
return uploadSessionResp{}, fmt.Errorf("onedrive upload session: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunkWithRetry(ctx context.Context, uploadURL string, start, total int64, data []byte) (*graphItem, error) {
|
||||
var last error
|
||||
for attempt := 0; attempt < uploadSessionRetryAttempts; attempt++ {
|
||||
if attempt > 0 {
|
||||
if err := sleepContext(ctx, time.Duration(attempt)*time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
item, retryable, err := d.putUploadSessionChunk(ctx, uploadURL, start, total, data)
|
||||
if err == nil {
|
||||
return item, nil
|
||||
}
|
||||
last = err
|
||||
if !retryable {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if last == nil {
|
||||
last = errors.New("onedrive upload session: retry attempts exhausted")
|
||||
}
|
||||
return nil, last
|
||||
}
|
||||
|
||||
func (d *Driver) putUploadSessionChunk(ctx context.Context, uploadURL string, start, total int64, data []byte) (*graphItem, bool, error) {
|
||||
end := start + int64(len(data)) - 1
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
req.ContentLength = int64(len(data))
|
||||
req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, total))
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK, http.StatusCreated:
|
||||
var item graphItem
|
||||
if err := json.NewDecoder(res.Body).Decode(&item); err != nil {
|
||||
return nil, false, fmt.Errorf("onedrive upload session: decode completed item: %w", err)
|
||||
}
|
||||
return &item, false, nil
|
||||
case http.StatusAccepted:
|
||||
return nil, false, nil
|
||||
default:
|
||||
body, _ := io.ReadAll(io.LimitReader(res.Body, 4096))
|
||||
err := fmt.Errorf("onedrive upload session: status=%d body=%s", res.StatusCode, strings.TrimSpace(string(body)))
|
||||
retryable := res.StatusCode == http.StatusTooManyRequests || (res.StatusCode >= 500 && res.StatusCode <= 504)
|
||||
return nil, retryable, err
|
||||
}
|
||||
}
|
||||
|
||||
func readSmallUpload(r io.Reader, declaredSize, limit int64) ([]byte, string, int64, error) {
|
||||
if r == nil {
|
||||
return nil, "", 0, errors.New("onedrive upload: body is required")
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = maxSmallUploadSize
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(r, limit+1))
|
||||
if err != nil {
|
||||
return nil, "", 0, fmt.Errorf("onedrive upload: read body: %w", err)
|
||||
}
|
||||
if int64(len(data)) > limit {
|
||||
return nil, "", 0, fmt.Errorf("onedrive upload: files over %d bytes require upload session", limit)
|
||||
}
|
||||
if declaredSize >= 0 && int64(len(data)) != declaredSize {
|
||||
return nil, "", 0, fmt.Errorf("onedrive upload: size mismatch: declared %d, copied %d", declaredSize, len(data))
|
||||
}
|
||||
sum := sha1.Sum(data)
|
||||
return data, hex.EncodeToString(sum[:]), int64(len(data)), nil
|
||||
}
|
||||
|
||||
func minInt64(a, b int64) int64 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
@@ -305,6 +482,25 @@ func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, er
|
||||
return item.ID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
fileID = strings.TrimSpace(fileID)
|
||||
if fileID == "" {
|
||||
return errors.New("onedrive rename: empty file id")
|
||||
}
|
||||
newName = strings.TrimSpace(newName)
|
||||
if newName == "" {
|
||||
return errors.New("onedrive rename: empty new name")
|
||||
}
|
||||
var item graphItem
|
||||
err := d.request(ctx, d.itemURL(fileID), http.MethodPatch, func(req *resty.Request) {
|
||||
req.SetBody(map[string]string{"name": newName})
|
||||
}, &item)
|
||||
if err != nil {
|
||||
return fmt.Errorf("onedrive rename: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Driver) request(ctx context.Context, rawURL, method string, configure func(*resty.Request), out any) error {
|
||||
return d.requestOnce(ctx, rawURL, method, configure, out, true)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@ package onedrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
@@ -406,6 +408,36 @@ func TestEnsureDirCreatesMissingFolders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenamePatchesDriveItemName(t *testing.T) {
|
||||
var body map[string]string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPatch || r.URL.EscapedPath() != "/v1.0/me/drive/items/file-id" {
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
|
||||
}
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
t.Fatalf("authorization = %q, want bearer token", got)
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode body: %v", err)
|
||||
}
|
||||
writeJSON(t, w, map[string]any{"id": "file-id", "name": "new name.mp4"})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
if err := d.Rename(context.Background(), "file-id", "new name.mp4"); err != nil {
|
||||
t.Fatalf("rename: %v", err)
|
||||
}
|
||||
if body["name"] != "new name.mp4" {
|
||||
t.Fatalf("rename body = %#v, want new name", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadSmallFileReturnsNewItemID(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
@@ -444,6 +476,86 @@ func TestUploadSmallFileReturnsNewItemID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadLargeFileUsesUploadSessionAndReportsHash(t *testing.T) {
|
||||
oldThreshold := smallUploadThreshold
|
||||
oldChunk := uploadSessionChunk
|
||||
smallUploadThreshold = 8
|
||||
uploadSessionChunk = 4
|
||||
t.Cleanup(func() {
|
||||
smallUploadThreshold = oldThreshold
|
||||
uploadSessionChunk = oldChunk
|
||||
})
|
||||
|
||||
body := "0123456789abc"
|
||||
var ranges []string
|
||||
var chunks []string
|
||||
var createdSession bool
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost && r.URL.EscapedPath() == "/v1.0/me/drive/items/parent-id:/big.mp4:/createUploadSession":
|
||||
createdSession = true
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer access-token" {
|
||||
t.Fatalf("authorization = %q, want bearer token", got)
|
||||
}
|
||||
writeJSON(t, w, map[string]any{"uploadUrl": srv.URL + "/upload-session"})
|
||||
case r.Method == http.MethodPut && r.URL.Path == "/upload-session":
|
||||
ranges = append(ranges, r.Header.Get("Content-Range"))
|
||||
data, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read chunk: %v", err)
|
||||
}
|
||||
chunks = append(chunks, string(data))
|
||||
if len(ranges) < 4 {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
if _, err := w.Write([]byte(`{"nextExpectedRanges":["0-"]}`)); err != nil {
|
||||
t.Fatalf("write accepted: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{"id": "uploaded-big-id"}); err != nil {
|
||||
t.Fatalf("write final item: %v", err)
|
||||
}
|
||||
default:
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
d := New(Config{
|
||||
ID: "od-main",
|
||||
AccessToken: "access-token",
|
||||
RefreshToken: "refresh-token",
|
||||
APIBaseURL: srv.URL,
|
||||
})
|
||||
got, err := d.UploadAndReportHash(context.Background(), "parent-id", "big.mp4", strings.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
if !createdSession {
|
||||
t.Fatal("createUploadSession was not called")
|
||||
}
|
||||
wantRanges := []string{
|
||||
"bytes 0-3/13",
|
||||
"bytes 4-7/13",
|
||||
"bytes 8-11/13",
|
||||
"bytes 12-12/13",
|
||||
}
|
||||
if strings.Join(ranges, "|") != strings.Join(wantRanges, "|") {
|
||||
t.Fatalf("ranges = %#v, want %#v", ranges, wantRanges)
|
||||
}
|
||||
if strings.Join(chunks, "") != body {
|
||||
t.Fatalf("uploaded chunks = %q, want %q", strings.Join(chunks, ""), body)
|
||||
}
|
||||
sum := sha1.Sum([]byte(body))
|
||||
if got.FileID != "uploaded-big-id" || got.Size != int64(len(body)) || got.Hash != hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("upload result = %#v, want file id/hash/size for body", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadRefreshesExpiredTokenAndReplaysBody(t *testing.T) {
|
||||
var uploadAttempts int
|
||||
var tokenRefreshes int
|
||||
|
||||
@@ -82,3 +82,13 @@ type filesResp struct {
|
||||
Value []graphItem `json:"value"`
|
||||
NextLink string `json:"@odata.nextLink"`
|
||||
}
|
||||
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
Hash string
|
||||
Size int64
|
||||
}
|
||||
|
||||
type uploadSessionResp struct {
|
||||
UploadURL string `json:"uploadUrl"`
|
||||
}
|
||||
|
||||
@@ -355,7 +355,60 @@ func (d *Driver) Rename(ctx context.Context, fileID, newName string) error {
|
||||
}
|
||||
|
||||
func (d *Driver) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
currentID := d.rootID
|
||||
for _, name := range splitPath(pathFromRoot) {
|
||||
childID, err := d.findChildDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if childID == "" {
|
||||
childID, err = d.makeDir(ctx, currentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
currentID = childID
|
||||
}
|
||||
return currentID, nil
|
||||
}
|
||||
|
||||
func (d *Driver) findChildDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
entries, err := d.List(ctx, parentID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir && e.Name == name {
|
||||
return e.ID, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (d *Driver) makeDir(ctx context.Context, parentID, name string) (string, error) {
|
||||
var out file
|
||||
err := d.request(ctx, filesURL, http.MethodPost, func(req *resty.Request) {
|
||||
req.SetBody(map[string]any{
|
||||
"kind": "drive#folder",
|
||||
"parent_id": parentID,
|
||||
"name": name,
|
||||
})
|
||||
}, &out)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("pikpak mkdir %s: %w", name, err)
|
||||
}
|
||||
if out.ID == "" {
|
||||
return "", fmt.Errorf("pikpak mkdir %s: empty folder id", name)
|
||||
}
|
||||
return out.ID, nil
|
||||
}
|
||||
|
||||
func splitPath(p string) []string {
|
||||
p = strings.Trim(p, "/")
|
||||
if p == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(p, "/")
|
||||
}
|
||||
|
||||
func (d *Driver) getFiles(ctx context.Context, parentID string) ([]file, error) {
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package pikpak
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
)
|
||||
|
||||
func TestNewDefaults(t *testing.T) {
|
||||
@@ -95,11 +97,85 @@ func TestFolderToEntry(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDirStillUnsupported(t *testing.T) {
|
||||
d := New(Config{ID: "pikpak-main"})
|
||||
func TestEnsureDirReusesExistingFolder(t *testing.T) {
|
||||
var postCalled bool
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
if got := r.URL.Query().Get("parent_id"); got != "root-id" {
|
||||
t.Fatalf("parent_id = %q, want root-id", got)
|
||||
}
|
||||
writePikPakJSON(t, w, map[string]any{
|
||||
"files": []map[string]any{{
|
||||
"id": "existing-folder-id",
|
||||
"kind": "drive#folder",
|
||||
"name": "91 Spider",
|
||||
}},
|
||||
})
|
||||
case http.MethodPost:
|
||||
postCalled = true
|
||||
t.Fatalf("existing folder should not be created again")
|
||||
default:
|
||||
t.Fatalf("unexpected method %s", r.Method)
|
||||
}
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
if _, err := d.EnsureDir(nil, "/previews"); err != drives.ErrNotSupported {
|
||||
t.Fatalf("EnsureDir error = %v, want ErrNotSupported", err)
|
||||
d := newTestDriver(t, srv)
|
||||
got, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure dir: %v", err)
|
||||
}
|
||||
if got != "existing-folder-id" {
|
||||
t.Fatalf("dir id = %q, want existing-folder-id", got)
|
||||
}
|
||||
if postCalled {
|
||||
t.Fatal("POST should not be called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDirCreatesMissingFolder(t *testing.T) {
|
||||
var got uploadRequestBody
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/drive/v1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
writePikPakJSON(t, w, map[string]any{"files": []map[string]any{}})
|
||||
case http.MethodPost:
|
||||
if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode create folder body: %v", err)
|
||||
}
|
||||
writePikPakJSON(t, w, map[string]any{
|
||||
"id": "new-folder-id",
|
||||
"kind": "drive#folder",
|
||||
"name": "91 Spider",
|
||||
})
|
||||
default:
|
||||
t.Fatalf("unexpected method %s", r.Method)
|
||||
}
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
d := newTestDriver(t, srv)
|
||||
id, err := d.EnsureDir(context.Background(), "91 Spider")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure dir: %v", err)
|
||||
}
|
||||
if id != "new-folder-id" {
|
||||
t.Fatalf("dir id = %q, want new-folder-id", id)
|
||||
}
|
||||
if got.Kind != "drive#folder" || got.ParentID != "root-id" || got.Name != "91 Spider" {
|
||||
t.Fatalf("create folder body = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func writePikPakJSON(t *testing.T, w http.ResponseWriter, body any) {
|
||||
t.Helper()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(body); err != nil {
|
||||
t.Fatalf("write json: %v", err)
|
||||
}
|
||||
// Upload 的真实实现见 upload_test.go。
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Package spider91migrate 周期性把 spider91 drive 下载到本地的视频
|
||||
// 上传到一个指定的目标 drive 目录(PikPak 或 115),上传成功后:
|
||||
// 上传到一个指定的目标 drive 目录(PikPak、115 或 OneDrive),上传成功后:
|
||||
//
|
||||
// - 改写 catalog 行:drive_id / file_id / content_hash 改成目标盘的;
|
||||
// 视频自身的 id 不变(仍是 spider91-<driveID>-<viewkey>),video_tags、
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
|
||||
"github.com/video-site/backend/internal/catalog"
|
||||
"github.com/video-site/backend/internal/drives"
|
||||
"github.com/video-site/backend/internal/drives/onedrive"
|
||||
"github.com/video-site/backend/internal/drives/p115"
|
||||
"github.com/video-site/backend/internal/drives/pikpak"
|
||||
"github.com/video-site/backend/internal/drives/spider91"
|
||||
@@ -39,12 +40,14 @@ import (
|
||||
// 这一层抽象把"迁移调用方"和"具体盘的 SDK 协议"解耦:
|
||||
// - PikPak 走 GCID + OSS PutObject(pikpak.UploadResult)
|
||||
// - 115 走 SHA1 + 秒传 / OSS / 分片(p115.UploadResult)
|
||||
// - OneDrive 走 SHA1 + 小文件 PUT / 大文件 upload session
|
||||
//
|
||||
// 两个返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
|
||||
// 各家返回值都被归一成本地的 UploadResult,并在 catalog 改写阶段统一处理。
|
||||
type uploadTarget interface {
|
||||
ID() string
|
||||
Kind() string
|
||||
RootID() string
|
||||
EnsureDir(ctx context.Context, pathFromRoot string) (string, error)
|
||||
UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error)
|
||||
Rename(ctx context.Context, fileID, newName string) error
|
||||
}
|
||||
@@ -52,7 +55,7 @@ type uploadTarget interface {
|
||||
// UploadResult 是 uploadTarget.UploadAndReportHash 的归一返回。
|
||||
//
|
||||
// FileID 目标盘上的新文件 ID;
|
||||
// Hash GCID(PikPak)或 SHA1 HEX 大写(115),写入 catalog.content_hash 用于跨盘去重;
|
||||
// Hash GCID(PikPak)或 SHA1 HEX(115 / OneDrive),写入 catalog.content_hash 用于跨盘去重;
|
||||
// Size 实际上传字节数。
|
||||
type UploadResult struct {
|
||||
FileID string
|
||||
@@ -60,7 +63,9 @@ type UploadResult struct {
|
||||
Size int64
|
||||
}
|
||||
|
||||
// pikpakAdapter / p115Adapter 把具体 driver 包装成 uploadTarget。
|
||||
const spider91UploadDirName = "91 Spider"
|
||||
|
||||
// pikpakAdapter / p115Adapter / onedriveAdapter 把具体 driver 包装成 uploadTarget。
|
||||
//
|
||||
// 之所以不让 driver 直接实现 uploadTarget:
|
||||
//
|
||||
@@ -74,6 +79,9 @@ type pikpakAdapter struct {
|
||||
func (a *pikpakAdapter) ID() string { return a.d.ID() }
|
||||
func (a *pikpakAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *pikpakAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *pikpakAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *pikpakAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
@@ -92,6 +100,9 @@ type p115Adapter struct {
|
||||
func (a *p115Adapter) ID() string { return a.d.ID() }
|
||||
func (a *p115Adapter) Kind() string { return a.d.Kind() }
|
||||
func (a *p115Adapter) RootID() string { return a.d.RootID() }
|
||||
func (a *p115Adapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *p115Adapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportSha1(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
@@ -103,6 +114,27 @@ func (a *p115Adapter) Rename(ctx context.Context, fileID, newName string) error
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
type onedriveAdapter struct {
|
||||
d *onedrive.Driver
|
||||
}
|
||||
|
||||
func (a *onedriveAdapter) ID() string { return a.d.ID() }
|
||||
func (a *onedriveAdapter) Kind() string { return a.d.Kind() }
|
||||
func (a *onedriveAdapter) RootID() string { return a.d.RootID() }
|
||||
func (a *onedriveAdapter) EnsureDir(ctx context.Context, pathFromRoot string) (string, error) {
|
||||
return a.d.EnsureDir(ctx, pathFromRoot)
|
||||
}
|
||||
func (a *onedriveAdapter) UploadAndReportHash(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error) {
|
||||
res, err := a.d.UploadAndReportHash(ctx, parentID, name, r, size)
|
||||
if err != nil {
|
||||
return UploadResult{}, err
|
||||
}
|
||||
return UploadResult{FileID: res.FileID, Hash: res.Hash, Size: res.Size}, nil
|
||||
}
|
||||
func (a *onedriveAdapter) Rename(ctx context.Context, fileID, newName string) error {
|
||||
return a.d.Rename(ctx, fileID, newName)
|
||||
}
|
||||
|
||||
// adaptUploadTarget 把通用 drive 包装成 uploadTarget。
|
||||
// 不支持的盘 kind 返回 error;调用方静默跳过。
|
||||
func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
@@ -111,6 +143,8 @@ func adaptUploadTarget(d drives.Drive) (uploadTarget, error) {
|
||||
return &pikpakAdapter{d: v}, nil
|
||||
case *p115.Driver:
|
||||
return &p115Adapter{d: v}, nil
|
||||
case *onedrive.Driver:
|
||||
return &onedriveAdapter{d: v}, nil
|
||||
case uploadTarget:
|
||||
// 测试或自定义实现可以直接传入;优先使用具体类型分支以拿到适配器。
|
||||
return v, nil
|
||||
@@ -511,15 +545,19 @@ func (m *Migrator) migrateOne(ctx context.Context, v *catalog.Video, src *spider
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// 上传到目标盘的根目录(用户配置的目标 drive 的 rootID)。
|
||||
// 上传到目标盘 rootID 下的固定 "91 Spider" 子目录。若用户把目标盘 rootID
|
||||
// 配成某个自定义目录,这里会在该自定义目录下查找/创建 "91 Spider"。
|
||||
// 上传名走 desiredPikPakName 算出来的方案 B 格式:
|
||||
//
|
||||
// <sanitized title>-<viewkey 后 8 位>.<ext>
|
||||
//
|
||||
// 这样网盘 Web 端列出来的文件名能直接看出是哪个视频,
|
||||
// 又用 viewkey 后 8 位避免同标题撞名。两个目标盘(PikPak / 115)共用同一格式,
|
||||
// 又用 viewkey 后 8 位避免同标题撞名。所有目标盘共用同一格式,
|
||||
// 简化前端 / catalog 的认知。
|
||||
parent := pp.RootID()
|
||||
parent, err := pp.EnsureDir(ctx, spider91UploadDirName)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s ensure %q dir: %w", pp.Kind(), spider91UploadDirName, err)
|
||||
}
|
||||
uploadName := desiredPikPakName(v.Title, extractViewKey(v.ID), v.Ext)
|
||||
res, err := pp.UploadAndReportHash(ctx, parent, uploadName, f, info.Size())
|
||||
if err != nil {
|
||||
@@ -639,7 +677,7 @@ func (m *Migrator) cleanupOldLocalVideos(ctx context.Context, src *spider91.Driv
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// backfillFileNames 扫描目标 drive(PikPak 或 115)下所有 spider91-* 起始 ID 的视频,
|
||||
// backfillFileNames 扫描目标 drive(PikPak、115 或 OneDrive)下所有 spider91-* 起始 ID 的视频,
|
||||
// 对文件名不是 desiredPikPakName(...) 期望格式的,调 target.Rename 修正,
|
||||
// 并把 catalog.file_name 同步到新名字。
|
||||
//
|
||||
|
||||
@@ -53,6 +53,8 @@ type fakePikPak struct {
|
||||
uploadFunc func(ctx context.Context, parentID, name string, r io.Reader, size int64) (UploadResult, error)
|
||||
mu sync.Mutex
|
||||
gotBodies map[string][]byte
|
||||
gotParents map[string]string
|
||||
ensureCalls []string
|
||||
// renameCalls 记录每次 Rename 的 fileID->newName 历史,用于 backfill 测试断言。
|
||||
renameCalls map[string]string
|
||||
}
|
||||
@@ -62,6 +64,7 @@ func newFakePikPak(id, rootID string) *fakePikPak {
|
||||
id: id,
|
||||
rootID: rootID,
|
||||
gotBodies: make(map[string][]byte),
|
||||
gotParents: make(map[string]string),
|
||||
renameCalls: make(map[string]string),
|
||||
}
|
||||
}
|
||||
@@ -80,8 +83,11 @@ func (d *fakePikPak) StreamURL(context.Context, string) (*drives.StreamLink, err
|
||||
func (d *fakePikPak) Upload(context.Context, string, string, io.Reader, int64) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
}
|
||||
func (d *fakePikPak) EnsureDir(context.Context, string) (string, error) {
|
||||
return "", drives.ErrNotSupported
|
||||
func (d *fakePikPak) EnsureDir(_ context.Context, pathFromRoot string) (string, error) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.ensureCalls = append(d.ensureCalls, pathFromRoot)
|
||||
return d.rootID + "/" + pathFromRoot, nil
|
||||
}
|
||||
func (d *fakePikPak) Rename(_ context.Context, fileID, newName string) error {
|
||||
d.mu.Lock()
|
||||
@@ -99,6 +105,7 @@ func (d *fakePikPak) UploadAndReportHash(ctx context.Context, parentID, name str
|
||||
body, _ := io.ReadAll(r)
|
||||
d.mu.Lock()
|
||||
d.gotBodies[name] = body
|
||||
d.gotParents[name] = parentID
|
||||
d.mu.Unlock()
|
||||
return UploadResult{
|
||||
FileID: "remote-" + name,
|
||||
@@ -127,6 +134,19 @@ func (d *fakeP115) Kind() string { return "p115" }
|
||||
var _ drives.Drive = (*fakeP115)(nil)
|
||||
var _ uploadTarget = (*fakeP115)(nil)
|
||||
|
||||
type fakeOneDrive struct {
|
||||
*fakePikPak
|
||||
}
|
||||
|
||||
func newFakeOneDrive(id, rootID string) *fakeOneDrive {
|
||||
return &fakeOneDrive{fakePikPak: newFakePikPak(id, rootID)}
|
||||
}
|
||||
|
||||
func (d *fakeOneDrive) Kind() string { return "onedrive" }
|
||||
|
||||
var _ drives.Drive = (*fakeOneDrive)(nil)
|
||||
var _ uploadTarget = (*fakeOneDrive)(nil)
|
||||
|
||||
// TestBackfillFileNamesRenamesOnlyMismatchedSpider91Videos 验证回填逻辑:
|
||||
//
|
||||
// - 已经是期望格式的不会再调 Rename(幂等)
|
||||
@@ -347,6 +367,12 @@ func TestRunOnceMigratesSpider91VideosAndCleansLocalFiles(t *testing.T) {
|
||||
if _, ok := pp.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("PikPak did not receive expected upload name %q (got names: %v)", wantName, keysOf(pp.gotBodies))
|
||||
}
|
||||
if gotParent := pp.gotParents[wantName]; gotParent != "pikpak-root-id/"+spider91UploadDirName {
|
||||
t.Fatalf("upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(pp.ensureCalls) != 1 || pp.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("ensure calls = %#v, want %q", pp.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
@@ -884,6 +910,12 @@ func TestRunOnceMigratesToP115Target(t *testing.T) {
|
||||
if _, ok := target.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("p115 did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
|
||||
}
|
||||
if gotParent := target.gotParents[wantName]; gotParent != "p115-root-cid/"+spider91UploadDirName {
|
||||
t.Fatalf("p115 upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("p115 ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
@@ -905,7 +937,67 @@ func TestRunOnceMigratesToP115Target(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak 也不是 115 时,
|
||||
func TestRunOnceMigratesToOneDriveTarget(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
src, _ := setupSpider91(t)
|
||||
target := newFakeOneDrive("onedrive-target", "onedrive-root")
|
||||
reg := newFakeRegistry()
|
||||
reg.Add(src)
|
||||
reg.Add(target)
|
||||
|
||||
now := time.Now()
|
||||
id := writeSpider91Video(t, cat, src, "vk-od-001", ".mp4", []byte("video bytes onedrive"), now)
|
||||
|
||||
m := New(Config{
|
||||
Catalog: cat,
|
||||
Registry: reg,
|
||||
GetTargetDriveID: func() string { return target.ID() },
|
||||
KeepLatestN: -1,
|
||||
})
|
||||
m.runOnce(context.Background())
|
||||
|
||||
if target.uploadCalls != 1 {
|
||||
t.Fatalf("onedrive upload calls = %d, want 1", target.uploadCalls)
|
||||
}
|
||||
|
||||
got, err := cat.GetVideo(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("get video: %v", err)
|
||||
}
|
||||
if got.DriveID != target.ID() {
|
||||
t.Fatalf("drive_id = %q, want %q", got.DriveID, target.ID())
|
||||
}
|
||||
wantName := "Sample vk-od-001-001.mp4"
|
||||
if _, ok := target.gotBodies[wantName]; !ok {
|
||||
t.Fatalf("onedrive did not receive expected upload name %q (got names: %v)", wantName, keysOf(target.gotBodies))
|
||||
}
|
||||
if gotParent := target.gotParents[wantName]; gotParent != "onedrive-root/"+spider91UploadDirName {
|
||||
t.Fatalf("onedrive upload parent = %q, want root/91 Spider", gotParent)
|
||||
}
|
||||
if len(target.ensureCalls) != 1 || target.ensureCalls[0] != spider91UploadDirName {
|
||||
t.Fatalf("onedrive ensure calls = %#v, want %q", target.ensureCalls, spider91UploadDirName)
|
||||
}
|
||||
if got.FileID != "remote-"+wantName {
|
||||
t.Fatalf("file_id = %q, want %q", got.FileID, "remote-"+wantName)
|
||||
}
|
||||
if got.FileName != wantName {
|
||||
t.Fatalf("file_name = %q, want %q", got.FileName, wantName)
|
||||
}
|
||||
if got.ContentHash == "" {
|
||||
t.Fatal("content_hash should be set after onedrive migration")
|
||||
}
|
||||
|
||||
videoPath, _ := src.VideoPath("vk-od-001.mp4")
|
||||
if _, err := os.Stat(videoPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local mp4 still exists after onedrive migration or stat error: %v", err)
|
||||
}
|
||||
thumbPath, _ := src.ThumbPath("vk-od-001.jpg")
|
||||
if _, err := os.Stat(thumbPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("local thumb still exists after onedrive migration or stat error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveTargetRejectsUnsupportedKind 验证当目标 drive 既不是 PikPak、115 也不是 OneDrive 时,
|
||||
// resolveTarget 拒绝并返回 error,让 runOnce 静默跳过(不会做破坏性变更)。
|
||||
func TestResolveTargetRejectsUnsupportedKind(t *testing.T) {
|
||||
cat := setupCatalog(t)
|
||||
|
||||
@@ -80,10 +80,10 @@ export function DrivesPage() {
|
||||
const [selectedDriveId, setSelectedDriveId] = useState<string | null>(null);
|
||||
const { show } = useToast();
|
||||
|
||||
// 当前系统中可作为 spider91 上传目标的 drive 列表(pikpak ∪ p115)。
|
||||
// 当前系统中可作为 spider91 上传目标的 drive 列表(pikpak ∪ p115 ∪ onedrive)。
|
||||
// 用户保存 spider91 drive 时从这里挑一个;空表示本地保存不上传。
|
||||
const uploadTargets = useMemo(
|
||||
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115"),
|
||||
() => list.filter((d) => d.kind === "pikpak" || d.kind === "p115" || d.kind === "onedrive"),
|
||||
[list]
|
||||
);
|
||||
|
||||
@@ -979,9 +979,9 @@ function DriveForm({
|
||||
* Spider91UploadTargetField 是 spider91 drive 表单专属的"上传目标"下拉。
|
||||
*
|
||||
* 行为:
|
||||
* - 选项 = "本地保存,不上传" + 系统中所有 pikpak/p115 drive
|
||||
* - 选项 = "本地保存,不上传" + 系统中所有 pikpak/p115/onedrive drive
|
||||
* - value="" 时后端不迁移上传,视频保存在服务器本地
|
||||
* - 没有任何 pikpak/p115 drive 时仍允许选择本地保存
|
||||
* - 没有任何 pikpak/p115/onedrive drive 时仍允许选择本地保存
|
||||
* - 该字段写入的是全局 setting `spider91.upload_drive_id`,不是 drive 自己的
|
||||
* credentials —— 所有 spider91 drive 共享同一个上传目标
|
||||
*/
|
||||
@@ -1006,7 +1006,7 @@ function Spider91UploadTargetField({
|
||||
))}
|
||||
</select>
|
||||
<div className="admin-form__help">
|
||||
选择本地保存时,爬取视频只保存在服务器本地;选择 115 网盘或 PikPak 后,较早的视频会上传到该云盘根目录。该设置全局生效。
|
||||
选择本地保存时,爬取视频只保存在服务器本地;选择 115 网盘、PikPak 或 OneDrive 后,较早的视频会上传到该云盘根目录下的 91 Spider 文件夹。该设置全局生效。
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
@@ -1026,7 +1026,7 @@ function credentialHelp(kind: Kind, isEdit: boolean): string {
|
||||
case "onedrive":
|
||||
return `按 OpenList 默认应用在线挂载,只需要 refresh_token;保存时会自动刷新并保存 token。${note}`;
|
||||
case "spider91":
|
||||
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;它不是外部网盘,不需要填写 Cookie 或目录 ID。后续流水线会把较早的视频上传到你选择的 115 / PikPak 目标盘。";
|
||||
return "91 爬虫会把定时抓取到的视频和封面先保存到本机,并作为一个视频来源接入站点;它不是外部网盘,不需要填写 Cookie 或目录 ID。后续流水线会把较早的视频上传到你选择的 115 / PikPak / OneDrive 目标盘。";
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
|
||||
+2
-2
@@ -340,9 +340,9 @@ export type Theme = "dark" | "pink";
|
||||
export type Settings = {
|
||||
theme: Theme;
|
||||
/**
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak 或 p115 drive)。
|
||||
* spider91 视频迁移到云盘时的目标 drive ID(必须是已挂载的 pikpak、p115 或 onedrive drive)。
|
||||
* - 空字符串:本地保存,不上传到云盘。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115}。
|
||||
* - 非空:显式指定。后端会校验 drive 存在且 kind ∈ {pikpak, p115, onedrive}。
|
||||
*/
|
||||
spider91UploadDriveId: string;
|
||||
};
|
||||
|
||||
@@ -16,6 +16,10 @@ test("spider91 drive form does not expose advanced crawler credentials", () => {
|
||||
|
||||
test("spider91 upload target uses explicit local-save option instead of auto target", () => {
|
||||
assert.match(drivesPageSource, /本地保存,不上传/);
|
||||
assert.match(
|
||||
drivesPageSource,
|
||||
/d\.kind === "pikpak" \|\| d\.kind === "p115" \|\| d\.kind === "onedrive"/
|
||||
);
|
||||
assert.doesNotMatch(drivesPageSource, /自动:唯一/);
|
||||
assert.doesNotMatch(drivesPageSource, /自动模式/);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user