mirror of
https://github.com/nianzhibai/91.git
synced 2026-06-15 00:44:30 +08:00
nianzhibai
c1355385e1
Redesign crawler management around imported Python scripts instead of built-in crawler storage. Crawler scripts now declare CRAWLER_NAME, imports validate metadata, crawler IDs are generated internally, and deleted crawler scripts are detached without deleting already imported videos. Add backend support for file and URL script imports, dry-run testing, metadata parsing, safer job paths, original filename preservation, and crawler listing that ignores detached script records. Remove the legacy built-in Spider91 script path flow and hidden Python/config JSON fields from the crawler API. Rework the admin crawler page into an independent crawler console with script import, dry-run testing, status metrics, spider iconography, and simplified controls. Update docs, examples, installer checks, Docker/release packaging, and tests for the new protocol.
40 lines
873 B
Go
40 lines
873 B
Go
package scriptcrawler
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestExtractMetadataReadsCrawlerName(t *testing.T) {
|
|
meta, err := ExtractMetadata(`
|
|
# comment
|
|
CRAWLER_NAME = "示例爬虫"
|
|
`)
|
|
if err != nil {
|
|
t.Fatalf("extract metadata: %v", err)
|
|
}
|
|
if meta.Name != "示例爬虫" {
|
|
t.Fatalf("name = %q", meta.Name)
|
|
}
|
|
}
|
|
|
|
func TestExtractMetadataRejectsMissingCrawlerName(t *testing.T) {
|
|
_, err := ExtractMetadata(`print("hello")`)
|
|
if err == nil {
|
|
t.Fatal("expected error")
|
|
}
|
|
if !strings.Contains(err.Error(), "CRAWLER_NAME") {
|
|
t.Fatalf("error = %v, want CRAWLER_NAME guidance", err)
|
|
}
|
|
}
|
|
|
|
func TestExtractMetadataRejectsEmptyCrawlerName(t *testing.T) {
|
|
_, err := ExtractMetadata(`CRAWLER_NAME = " "`)
|
|
if err == nil {
|
|
t.Fatal("expected error")
|
|
}
|
|
if !strings.Contains(err.Error(), "不能为空") {
|
|
t.Fatalf("error = %v, want empty-name error", err)
|
|
}
|
|
}
|