feat: add qlrepo sync crontab

2026-03-19 11:06:45 +08:00
parent a03800f196
commit cd818f7b0c
23 changed files with 4126 additions and 529 deletions
@@ -27,7 +27,10 @@ type Config struct {
 	ProxyURL   string
 	AuthToken      string
 	HttpProxy      string
-	WhitelistPaths string // Comma separated paths to preserve (whitelist)
+	WhitelistPaths string // Comma or vertical line separated paths to preserve or filter (whitelist)
+	Blacklist      string // Script filter blacklist keywords, vertical line separated
+	Dependence     string // Script dependence file keywords, vertical line separated
+	Extensions     string // Script file extensions, vertical line separated
 }

 func Run(args []string) {
@@ -43,7 +46,10 @@ func Run(args []string) {
 	fs.StringVar(&cfg.ProxyURL, "proxy-url", "", "Custom proxy url")
 	fs.StringVar(&cfg.AuthToken, "auth-token", "", "Auth token")
 	fs.StringVar(&cfg.HttpProxy, "http-proxy", "", "Http proxy")
-	fs.StringVar(&cfg.WhitelistPaths, "whitelist-paths", "", "Comma separated paths to preserve (whitelist)")
+	fs.StringVar(&cfg.WhitelistPaths, "whitelist-paths", "", "Separated paths to preserve or filter (whitelist)")
+	fs.StringVar(&cfg.Blacklist, "blacklist", "", "Script filter blacklist keywords (| separated)")
+	fs.StringVar(&cfg.Dependence, "dependence", "", "Script dependence keywords (| separated)")
+	fs.StringVar(&cfg.Extensions, "extensions", "", "Script extensions (| separated)")

 	fs.Parse(args)

@@ -59,6 +65,11 @@ func Run(args []string) {
 	} else {
 		syncURL(cfg)
 	}
+
+	// 执行脚本过滤（仅限 git 模式，url 加载通常为单文件，暂不处理过滤）
+	if cfg.SourceType == "git" {
+		filterFiles(cfg)
+	}
 }

 func syncGit(cfg Config) {
@@ -88,7 +99,7 @@ func syncGit(cfg Config) {

 	gitDir := filepath.Join(dest, ".git")
 	if isDir(dest) && !pathExists(gitDir) {
-		repoName := getRepoName(cfg.SourceURL)
+		repoName := utils.GetRepoIdentifier(cfg.SourceURL, cfg.Branch)
 		dest = filepath.Join(dest, repoName)
 		fmt.Printf("目标路径自动追加仓库名: %s\n", dest)
 		gitDir = filepath.Join(dest, ".git")
@@ -220,6 +231,13 @@ func buildProxyURL(url string, proxyType string, proxyURL string) string {
 	if proxyType == "" || proxyType == "none" {
 		return url
 	}
+	
+	// 如果 URL 已经包含明显的代理前缀 (如用户手动填写的 http://ghproxy.com/...)
+	// 则跳过内置代理逻辑
+	if strings.Contains(url, "googo.win") || (proxyType == "custom" && strings.HasPrefix(url, proxyURL)) {
+		return url
+	}
+
 	base := ""
 	if proxyType == "ghproxy" {
 		base = "https://gh-proxy.com/"
@@ -229,7 +247,7 @@ func buildProxyURL(url string, proxyType string, proxyURL string) string {
 		base = strings.TrimSuffix(proxyURL, "/") + "/"
 	}

-	if base != "" && strings.HasPrefix(url, "http") {
+	if base != "" && strings.HasPrefix(url, "http") && !strings.HasPrefix(url, base) {
 		return base + url
 	}
 	return url
@@ -299,11 +317,7 @@ func isRawFileURL(url string) bool {
 	return false
 }

-func getRepoName(url string) string {
-	u := strings.TrimSuffix(url, "/")
-	u = strings.TrimSuffix(u, ".git")
-	return filepath.Base(u)
-}
+

 var ansiRegex = regexp.MustCompile("\x1b\\[[0-9;]*[a-zA-Z]")

@@ -495,3 +509,188 @@ func preserve(baseDir string, paths string) func() {
 		os.RemoveAll(tmpParent)
 	}
 }
+
+// filterFiles performs script filtering based on whitelist, blacklist, dependence and extensions.
+func filterFiles(cfg Config) {
+	// If no filtering is specified, do nothing.
+	if cfg.WhitelistPaths == "" && cfg.Blacklist == "" && cfg.Dependence == "" && cfg.Extensions == "" {
+		return
+	}
+
+	dest := cfg.TargetPath
+	// If the dest appended a repo name in syncGit, we need to find it.
+	// However, BuildRepoCommand already passes the abs path which might already be the specific repo dir.
+	// We'll walk from cfg.TargetPath.
+	
+	gitDir := filepath.Join(dest, ".git")
+	if isDir(dest) && !pathExists(gitDir) {
+		repoName := utils.GetRepoIdentifier(cfg.SourceURL, cfg.Branch)
+		if pathExists(filepath.Join(dest, repoName)) {
+			dest = filepath.Join(dest, repoName)
+		}
+	}
+
+	fmt.Printf("开始执行脚本过滤: %s\n", dest)
+
+	whitelist := splitKeywords(cfg.WhitelistPaths)
+	blacklist := splitKeywords(cfg.Blacklist)
+	dependence := splitKeywords(cfg.Dependence)
+	extensions := splitKeywords(cfg.Extensions)
+
+	// We'll collect files to delete to avoid modifying while walking if possible.
+	// But os.RemoveAll is fine.
+	
+	count := 0
+	filepath.Walk(dest, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+		if info.IsDir() {
+			if info.Name() == ".git" {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+
+		rel, _ := filepath.Rel(dest, path)
+		rel = filepath.ToSlash(rel)
+		filename := info.Name()
+
+		// 1. Check dependence: always keep
+		if matchesAny(rel, filename, dependence) {
+			return nil
+		}
+
+		// 2. Check extensions: delete if not matched and extensions is specified
+		if len(extensions) > 0 {
+			ext := strings.TrimPrefix(filepath.Ext(filename), ".")
+			matchedExt := false
+			for _, e := range extensions {
+				if strings.EqualFold(ext, strings.TrimPrefix(e, ".")) {
+					matchedExt = true
+					break
+				}
+			}
+			if !matchedExt {
+				fmt.Printf("过滤文件 (后缀不符): %s\n", rel)
+				os.Remove(path)
+				count++
+				return nil
+			}
+		}
+
+		// 3. Check blacklist: delete if matched
+		if matchesAny(rel, filename, blacklist) {
+			fmt.Printf("过滤文件 (黑名单): %s\n", rel)
+			os.Remove(path)
+			count++
+			return nil
+		}
+
+		// 4. Check whitelist: delete if NOT matched and whitelist is specified
+		if len(whitelist) > 0 {
+			if !matchesAny(rel, filename, whitelist) {
+				fmt.Printf("过滤文件 (不在白名单): %s\n", rel)
+				os.Remove(path)
+				count++
+				return nil
+			}
+		}
+
+		return nil
+	})
+
+	if count > 0 {
+		fmt.Printf("过滤完成，共删除 %d 个不符合要求的文件\n", count)
+		// Try to clean up empty directories
+		cleanEmptyDirs(dest)
+	}
+}
+
+func splitKeywords(s string) []string {
+	if s == "" {
+		return nil
+	}
+	// Try to split by common separators for compatibility
+	var parts []string
+	if strings.Contains(s, "|") {
+		parts = strings.Split(s, "|")
+	} else if strings.Contains(s, ",") {
+		parts = strings.Split(s, ",")
+	} else {
+		parts = []string{s}
+	}
+	
+	var res []string
+	for _, p := range parts {
+		p = strings.TrimSpace(p)
+		if p != "" {
+			res = append(res, p)
+		}
+	}
+	return res
+}
+
+func matchesAny(rel, filename string, keywords []string) bool {
+	if len(keywords) == 0 {
+		return false
+	}
+	for _, k := range keywords {
+		// 1. 尝试作为正则整体进行匹配，默认不区分大小写 (?i)
+		// 如果关键字不包含正则元字符，则补齐 (?i) 开启忽略大小写
+		pattern := k
+		if !strings.HasPrefix(pattern, "(?i)") {
+			pattern = "(?i)" + pattern
+		}
+
+		reg, err := regexp.Compile(pattern)
+		if err == nil {
+			// 优先匹配文件名（解决 ^jd[^_] 这种锚点在相对路径下失效的问题）
+			if reg.MatchString(filename) || reg.MatchString(rel) {
+				return true
+			}
+		} else {
+			// 回退逻辑：全小写包含判断
+			kLower := strings.ToLower(k)
+			if strings.Contains(strings.ToLower(rel), kLower) || strings.Contains(strings.ToLower(filename), kLower) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func cleanEmptyDirs(root string) {
+	// Post-order traversal to clean up empty dirs
+	filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+		if !info.IsDir() {
+			return nil
+		}
+		if path == root {
+			return nil
+		}
+		if info.Name() == ".git" {
+			return filepath.SkipDir
+		}
+		return nil
+	})
+	
+	// Actually we need to do this recursively or multiple times.
+	// A simpler way:
+	entries, _ := os.ReadDir(root)
+	for _, entry := range entries {
+		if entry.IsDir() {
+			if entry.Name() == ".git" { continue }
+			dirPath := filepath.Join(root, entry.Name())
+			cleanEmptyDirs(dirPath)
+			// Check if now empty
+			if isDirEmpty(dirPath) {
+				os.Remove(dirPath)
+			}
+		}
+	}
+}
+