From 6bce5685c7c6f7212671070736b4cecc37f681e4 Mon Sep 17 00:00:00 2001 From: duorameng <2997944583@qq.com> Date: Wed, 6 May 2026 18:02:35 +0800 Subject: [PATCH] chore: refact reposync --- cmd/reposync/reposync.go | 55 ++- internal/controllers/agent_controller.go | 4 +- internal/controllers/task_controller.go | 96 +++- internal/database/migrate.go | 23 - internal/executor/scheduler.go | 6 +- internal/middleware/auth.go | 22 + internal/models/task.go | 1 + internal/router/api_routes.go | 7 + internal/services/repo/repo_parser.go | 223 +++++++++ internal/services/repo/strategy.go | 27 ++ internal/services/repo/strategy_ql.go | 71 +++ internal/services/repo/strategy_std.go | 31 ++ internal/services/repo/util.go | 236 ++++++++++ internal/services/tasks/executor_service.go | 69 ++- internal/services/tasks/ql_repo_parser.go | 471 -------------------- internal/services/tasks/task_log_service.go | 5 + internal/utils/runtime_env.go | 19 + web/src/api/index.ts | 8 +- web/src/views/tasks/RepoDialog.vue | 189 +++++++- web/src/views/tasks/TaskDialog.vue | 40 +- web/src/views/tasks/Tasks.vue | 23 +- 21 files changed, 1098 insertions(+), 528 deletions(-) create mode 100644 internal/services/repo/repo_parser.go create mode 100644 internal/services/repo/strategy.go create mode 100644 internal/services/repo/strategy_ql.go create mode 100644 internal/services/repo/strategy_std.go create mode 100644 internal/services/repo/util.go delete mode 100644 internal/services/tasks/ql_repo_parser.go diff --git a/cmd/reposync/reposync.go b/cmd/reposync/reposync.go index a551e8b..a2f73e3 100644 --- a/cmd/reposync/reposync.go +++ b/cmd/reposync/reposync.go @@ -2,6 +2,7 @@ package reposync import ( "bytes" + "encoding/json" "flag" "fmt" "io" @@ -13,7 +14,8 @@ import ( "strings" "time" - "github.com/engigu/baihu-panel/internal/services/tasks" + "github.com/engigu/baihu-panel/internal/services" + "github.com/engigu/baihu-panel/internal/services/repo" "github.com/engigu/baihu-panel/internal/utils" ) @@ -35,6 +37,7 @@ type Config struct { TaskID string TaskLanguages string TaskTimeout int + CommentToTask string } func Run(args []string) { @@ -58,8 +61,17 @@ func Run(args []string) { fs.StringVar(&cfg.TaskLanguages, "task-langs", "", "Configured languages (JSON)") fs.StringVar(&cfg.TaskID, "repo-task-id", "", "Original Task ID") fs.IntVar(&cfg.TaskTimeout, "task-timeout", 30, "Task timeout (minutes)") + fs.StringVar(&cfg.CommentToTask, "commenttotask", "false", "Compatible with QL format script comment parsing (true/false)") fs.Parse(args) + + // 处理 $SCRIPTS_DIR$ 代号替换 + if strings.Contains(cfg.TargetPath, "$SCRIPTS_DIR$") { + scriptsDir := os.Getenv("BH_SCRIPTS_DIR") + if scriptsDir != "" { + cfg.TargetPath = filepath.Clean(strings.ReplaceAll(cfg.TargetPath, "$SCRIPTS_DIR$", scriptsDir)) + } + } fmt.Println("========================================") fmt.Println(" 仓库同步任务开始 ") @@ -80,7 +92,10 @@ func Run(args []string) { filterFiles(cfg) if cfg.TaskID != "" { - tasks.ParseRepoScriptsAndAddCron(nil, cfg.TaskID, os.Stdout) + upsertedIDs, deletedIDs := repo.ParseRepoScriptsAndAddCron(cfg.TaskID, os.Stdout, cfg.CommentToTask == "true") + if len(upsertedIDs) > 0 || len(deletedIDs) > 0 { + notifyMainServerToSyncRepoTasks(cfg.TaskID, upsertedIDs, deletedIDs) + } } } fmt.Println("\n========================================") @@ -88,6 +103,38 @@ func Run(args []string) { fmt.Println("========================================") } +func notifyMainServerToSyncRepoTasks(repoID string, upsertedIDs []string, deletedIDs []string) { + appCfg := services.GetConfig() + if appCfg != nil { + url := fmt.Sprintf("http://127.0.0.1:%d/internal/tasks/sync-repo-status", appCfg.Server.Port) + payload := map[string]interface{}{ + "repo_id": repoID, + "upserted_ids": upsertedIDs, + "deleted_ids": deletedIDs, + } + jsonData, _ := json.Marshal(payload) + settings := services.NewSettingsService() + secret := settings.Get("security", "secret") // constant.SectionSecurity = "security", constant.KeySecret = "secret" + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err == nil { + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Internal-Token", secret) + resp, reqErr := http.DefaultClient.Do(req) + if reqErr == nil { + defer resp.Body.Close() + if resp.StatusCode == 200 { + fmt.Println(">> [通知] 已成功将变动任务增量同步至主程序调度器") + } else { + fmt.Printf(">> [通知] 调度器刷新异常，主程序响应状态码: %d\n", resp.StatusCode) + } + } else { + fmt.Printf(">> [通知] 无法连接到主程序进行增量刷新: %v\n", reqErr) + } + } + } +} + func syncGit(cfg Config) { env := os.Environ() @@ -381,9 +428,7 @@ func (c *cleanWriter) Write(p []byte) (n int, err error) { func (c *cleanWriter) Flush() { if len(c.buf) > 0 { s := string(c.buf) - if strings.HasSuffix(s, "\r") { - s = s[:len(s)-1] - } + s = strings.TrimSuffix(s, "\r") s = ansiRegex.ReplaceAllString(s, "") if s != "" { c.out.Write([]byte(s + "\n")) diff --git a/internal/controllers/agent_controller.go b/internal/controllers/agent_controller.go index 21e9471..951afc0 100644 --- a/internal/controllers/agent_controller.go +++ b/internal/controllers/agent_controller.go @@ -543,7 +543,7 @@ func (c *AgentController) handleWSMessage(ac *services.AgentConnection, agent *m } // handleTaskHeartbeat 处理任务心跳 -func (c *AgentController) handleTaskHeartbeat(agent *models.Agent, data json.RawMessage) { +func (c *AgentController) handleTaskHeartbeat(_ *models.Agent, data json.RawMessage) { var req struct { LogID string `json:"log_id"` Duration int64 `json:"duration"` @@ -616,7 +616,7 @@ func (c *AgentController) handleTaskResult(agent *models.Agent, data json.RawMes } // handleTaskLog 处理 Agent 发送的实时日志 -func (c *AgentController) handleTaskLog(agent *models.Agent, data json.RawMessage) { +func (c *AgentController) handleTaskLog(_ *models.Agent, data json.RawMessage) { var logMsg struct { LogID string `json:"log_id"` Content string `json:"content"` diff --git a/internal/controllers/task_controller.go b/internal/controllers/task_controller.go index cba712e..c4ec7f6 100644 --- a/internal/controllers/task_controller.go +++ b/internal/controllers/task_controller.go @@ -11,8 +11,10 @@ import ( "github.com/engigu/baihu-panel/internal/services" "github.com/engigu/baihu-panel/internal/services/tasks" "github.com/engigu/baihu-panel/internal/utils" + "github.com/engigu/baihu-panel/internal/logger" "github.com/gin-gonic/gin" + "os" ) type TaskController struct { @@ -321,11 +323,19 @@ func (tc *TaskController) DeleteTask(c *gin.Context) { return } - // 获取任务信息（用于通知 agent） + // 获取任务信息（用于通知 agent 和物理删除校验） task := tc.taskService.GetTaskByID(id) - var agentID *string - if task != nil { - agentID = task.AgentID + if task == nil { + utils.NotFound(c, "任务不存在") + return + } + + agentID := task.AgentID + deleteFiles := c.Query("delete_files") == "true" + + // 如果需要删除物理文件且是仓库任务 + if deleteFiles && task.Type == constant.TaskTypeRepo { + tc.deleteRepoPhysicalFiles(task) } tc.executorService.RemoveCronTask(id) @@ -344,6 +354,67 @@ func (tc *TaskController) DeleteTask(c *gin.Context) { utils.SuccessMsg(c, "删除成功") } +// deleteRepoPhysicalFiles 删除仓库关联的物理文件 +func (tc *TaskController) deleteRepoPhysicalFiles(task *models.Task) { + if task.Type != constant.TaskTypeRepo { + return + } + + logger.Infof("[Controller] 开始尝试物理删除任务关联文件: %s", task.Name) + var repoCfg models.RepoConfig + if err := json.Unmarshal([]byte(task.Config), &repoCfg); err != nil { + logger.Errorf("[Controller] 解析任务配置失败: %v", err) + return + } + + targetPath := repoCfg.TargetPath + if targetPath == "" { + // 如果 TargetPath 为空，调用系统的计算函数获取默认目录名 + repoId := utils.GetRepoIdentifier(repoCfg.SourceURL, repoCfg.Branch) + if repoId != "" { + targetPath = repoId + logger.Infof("[Controller] TargetPath 为空，使用计算出的标识符: %s", targetPath) + } + } + + if targetPath == "" || targetPath == "$SCRIPTS_DIR$" { + logger.Warnf("[Controller] 任务 %s 无法确定有效的物理删除路径，跳过", task.Name) + return + } + + // 确定绝对路径 + scriptsDir, _ := filepath.Abs(constant.ScriptsWorkDir) + fullPath := targetPath + if strings.HasPrefix(targetPath, "$SCRIPTS_DIR$") { + fullPath = filepath.Join(scriptsDir, strings.TrimPrefix(targetPath, "$SCRIPTS_DIR$")) + } else if !filepath.IsAbs(targetPath) { + fullPath = filepath.Join(scriptsDir, targetPath) + } + + absTargetPath, _ := filepath.Abs(fullPath) + logger.Infof("[Controller] 最终计算的绝对路径: %s, Scripts目录: %s", absTargetPath, scriptsDir) + scriptsDir, _ = filepath.Abs(constant.ScriptsWorkDir) + + // 安全检查：使用 Rel 判断路径关系 + rel, err := filepath.Rel(scriptsDir, absTargetPath) + if err != nil { + logger.Errorf("[Controller] 计算相对路径失败: %v", err) + return + } + + // 必须是在 scripts 目录下（不以 .. 开头）且不能是 scripts 目录本身 (.) + if rel != "." && !strings.HasPrefix(rel, "..") { + err := os.RemoveAll(absTargetPath) + if err != nil { + logger.Errorf("[Controller] 物理删除文件夹失败: %s, 路径: %s, 错误: %v", task.Name, absTargetPath, err) + } else { + logger.Infof("[Controller] 已成功物理删除文件夹: %s, 路径: %s", task.Name, absTargetPath) + } + } else { + logger.Warnf("[Controller] 拒绝物理删除安全目录之外的路径: %s", absTargetPath) + } +} + func (tc *TaskController) BatchDeleteTasks(c *gin.Context) { var req struct { IDs []string `json:"ids" binding:"required"` @@ -475,3 +546,20 @@ func (tc *TaskController) GetTags(c *gin.Context) { } utils.Success(c, tags) } + +// SyncRepoTasks 增量同步仓库任务状态（供本地 reposync 进程调用） +func (tc *TaskController) SyncRepoTasks(c *gin.Context) { + var req struct { + RepoID string `json:"repo_id"` + UpsertedIDs []string `json:"upserted_ids"` + DeletedIDs []string `json:"deleted_ids"` + } + + if err := c.ShouldBindJSON(&req); err != nil { + utils.BadRequest(c, err.Error()) + return + } + + tc.executorService.SyncRepoTasks(req.UpsertedIDs, req.DeletedIDs) + utils.SuccessMsg(c, "增量同步成功") +} diff --git a/internal/database/migrate.go b/internal/database/migrate.go index a03a02e..70414f0 100644 --- a/internal/database/migrate.go +++ b/internal/database/migrate.go @@ -95,29 +95,6 @@ func getModelSignature(models []interface{}) string { return hex.EncodeToString(hash[:]) } -// hasGormTypeText 检查 gorm tag 中是否包含 type:text -func hasGormTypeText(gormTag string) bool { - for _, part := range strings.Split(gormTag, ";") { - if kv := strings.SplitN(strings.TrimSpace(part), ":", 2); len(kv) == 2 { - if strings.TrimSpace(kv[0]) == "type" && strings.EqualFold(strings.TrimSpace(kv[1]), "text") { - return true - } - } - } - return false -} - -// parseGormColumn 从 gorm tag 中提取 column:xxx -func parseGormColumn(gormTag string) string { - for _, part := range strings.Split(gormTag, ";") { - if kv := strings.SplitN(strings.TrimSpace(part), ":", 2); len(kv) == 2 { - if strings.TrimSpace(kv[0]) == "column" { - return strings.TrimSpace(kv[1]) - } - } - } - return "" -} // customMigrations 自定义迁移（处理 AutoMigrate 无法自动完成的变更） func customMigrations() error { diff --git a/internal/executor/scheduler.go b/internal/executor/scheduler.go index 19dd08c..505f841 100644 --- a/internal/executor/scheduler.go +++ b/internal/executor/scheduler.go @@ -372,7 +372,11 @@ func (s *Scheduler) executeTask(req *ExecutionRequest) (*ExecutionResult, error) req.Command = utils.BuildMiseCommand(req.Command, req.Languages) req.UseMise = false } - s.logger.Infof("[Scheduler] 命令: %s", req.Command) + // 确保系统级敏感信息（数据库地址、账号、密码等）始终在脱敏列表中 + allSecrets := append([]string{}, req.Secrets...) + allSecrets = append(allSecrets, utils.GetSystemSecrets()...) + + s.logger.Infof("[Scheduler] 命令: %s", utils.MaskSecrets(req.Command, allSecrets)) if s.config.Verbose { workDir := req.WorkDir diff --git a/internal/middleware/auth.go b/internal/middleware/auth.go index 9c51134..e9839fa 100644 --- a/internal/middleware/auth.go +++ b/internal/middleware/auth.go @@ -263,3 +263,25 @@ func SwaggerAuth() gin.HandlerFunc { c.Abort() } } + +// LocalhostOnly 仅允许本地回环地址访问，并进行简单的内部凭证校验 +func LocalhostOnly() gin.HandlerFunc { + return func(c *gin.Context) { + ip := c.ClientIP() + if ip != "127.0.0.1" && ip != "::1" { + utils.BadRequest(c, "仅允许本地访问") + c.Abort() + return + } + + // 简单的内部通信认证 + token := c.GetHeader("X-Internal-Token") + if token == "" || token != constant.Secret { + utils.Unauthorized(c, "无效的内部调用凭证") + c.Abort() + return + } + + c.Next() + } +} diff --git a/internal/models/task.go b/internal/models/task.go index 0b894ae..26a18ec 100644 --- a/internal/models/task.go +++ b/internal/models/task.go @@ -57,6 +57,7 @@ type RepoConfig struct { Dependence string `json:"dependence"` // 脚本依赖文件关键词，竖线分割 Extensions string `json:"extensions"` // 脚本文件后缀关键词，竖线分割 AutoAddCron bool `json:"auto_add_cron"` // 自动解析脚本注释添加定时任务 + CommentToTask string `json:"commenttotask"` // 兼容 QL 格式任务脚本注释解析 RepoSource string `json:"repo_source"` // 仓库来源，如果是选择了这个 ql 导入的仓库，= ql } diff --git a/internal/router/api_routes.go b/internal/router/api_routes.go index 4bd5014..d531564 100644 --- a/internal/router/api_routes.go +++ b/internal/router/api_routes.go @@ -23,6 +23,13 @@ func initPublicAPIRoutes(api *gin.RouterGroup, c *Controllers) { // 公开的站点设置（无需认证） api.GET("/settings/public", c.Settings.GetPublicSiteSettings) + + // 内部使用的 API（仅限本地调用，无需 Bearer 认证） + internalAPI := api.Group("/internal") + internalAPI.Use(middleware.LocalhostOnly()) + { + internalAPI.POST("/tasks/sync-repo-status", c.Task.SyncRepoTasks) + } } func initAuthorizedAPIRoutes(api *gin.RouterGroup, c *Controllers) { diff --git a/internal/services/repo/repo_parser.go b/internal/services/repo/repo_parser.go new file mode 100644 index 0000000..a46fe5c --- /dev/null +++ b/internal/services/repo/repo_parser.go @@ -0,0 +1,223 @@ +package repo + +import ( + "encoding/json" + "fmt" + "io" + "io/fs" + "path/filepath" + "strings" + + "github.com/engigu/baihu-panel/internal/constant" + "github.com/engigu/baihu-panel/internal/database" + "github.com/engigu/baihu-panel/internal/models" + "github.com/engigu/baihu-panel/internal/utils" +) + +// ParseRepoScriptsAndAddCron 扫描仓库目录中的脚本，解析 cron 和环境注释，并注册任务 +func ParseRepoScriptsAndAddCron(taskID string, logWriter io.Writer, forceCommentToTask bool) ([]string, []string) { + // 帮助函数：如果提供了 logWriter，则将日志输出到该处 + log := func(format string, a ...interface{}) { + msg := fmt.Sprintf(format, a...) + if !strings.HasSuffix(msg, "\n") { + msg += "\n" + } + if logWriter != nil { + logWriter.Write([]byte(msg)) + } + } + + var repoTask models.Task + res := database.DB.Where("id = ?", taskID).Limit(1).Find(&repoTask) + if res.Error != nil || res.RowsAffected == 0 { + return nil, nil + } + + if repoTask.Type != constant.TaskTypeRepo { + return nil, nil + } + + var repoCfg models.RepoConfig + if err := json.Unmarshal([]byte(repoTask.Config), &repoCfg); err != nil { + return nil, nil + } + + // 如果命令行强制开启，则覆盖配置 + if forceCommentToTask { + repoCfg.CommentToTask = "true" + } + + // 1. 确定解析策略 + strategy := GetParserStrategy(repoCfg.RepoSource) + + // 目标路径 + targetPath := repoCfg.TargetPath + if targetPath == "" { + targetPath = repoTask.WorkDir + } else if !filepath.IsAbs(targetPath) { + targetPath = filepath.Join(utils.ResolveAbsScriptsDir(), targetPath) + } + if targetPath == "" { + return nil, nil + } + targetPath = filepath.Clean(targetPath) + + // 获取仓库标识符 + repoId := utils.GetRepoIdentifier(repoCfg.SourceURL, repoCfg.Branch) + + gitDir := filepath.Join(targetPath, ".git") + if !isDir(targetPath) || !pathExists(gitDir) { + repoPath := filepath.Join(targetPath, repoId) + if pathExists(repoPath) { + targetPath = repoPath + } + } + + if !pathExists(targetPath) { + return nil, nil + } + + // 同步过程中使用的标签 + tag := fmt.Sprintf("%s", repoId) + + exts := getValidExtensions(repoCfg.Extensions) + + log("\n----------------------------------------") + log(" 开始扫描脚本并自动注册定时任务 ") + log("----------------------------------------") + + foundSourceIDs := make(map[string]bool) + var upsertedIDs []string + var deletedIDs []string + newTaskCount := 0 + updateTaskCount := 0 + + filepath.WalkDir(targetPath, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + + if strings.Contains(path, ".git") { + return nil + } + + ext := filepath.Ext(path) + if !strategy.SupportExtension(ext, exts) { + return nil + } + + // 2. 元数据提取 + taskName, taskCron := strategy.ExtractMeta(path, ext, repoCfg) + + // 3. 过滤处理 + relRepoPath, _ := filepath.Rel(targetPath, path) + filename := filepath.Base(path) + if !strategy.ShouldProcess(relRepoPath, filename, repoCfg) { + return nil + } + + if taskName != "" && taskCron != "" && repoCfg.AutoAddCron { + // 获取脚本相对于数据目录的路径 + absScriptsDir := utils.ResolveAbsScriptsDir() + // absTargetPath, _ := filepath.Abs(targetPath) + absPath, _ := filepath.Abs(path) + + // 计算 SourceID: 相对于脚本目录的完整路径，并清洗特殊符号 + relPath, _ := filepath.Rel(absScriptsDir, absPath) + sourceID := sanitizeIdentifier(relPath) + + // 替换绝对路径为代号 $SCRIPTS_DIR$ + displayPath := path + displayWorkDir := targetPath + if strings.HasPrefix(absPath, absScriptsDir) { + displayPath = filepath.ToSlash(filepath.Join("$SCRIPTS_DIR$", relPath)) + // 获取目录路径 + relDir, _ := filepath.Rel(absScriptsDir, filepath.Dir(absPath)) + displayWorkDir = filepath.ToSlash(filepath.Join("$SCRIPTS_DIR$", relDir)) + } + + // 找到任务，进行保存 + command := getCommandByExt(ext, displayPath) + taskID, isNew := upsertRepoTask(&repoTask, sourceID, taskName, command, taskCron, displayWorkDir, tag) + + if isNew { + log("[新增] 任务: %s (%s)", taskName, filename) + newTaskCount++ + } else { + log("[更新] 任务: %s (%s)", taskName, filename) + updateTaskCount++ + } + foundSourceIDs[sourceID] = true + upsertedIDs = append(upsertedIDs, taskID) + } + + return nil + }) + + // 清理该仓库下不再存在的旧脚本任务 + deletedTaskCount := 0 + var oldTasks []models.Task + if err := database.DB.Where("repo_task_id = ?", repoTask.ID).Find(&oldTasks).Error; err == nil { + for _, ot := range oldTasks { + if !foundSourceIDs[ot.SourceID] { + log("[移除] 脚本已不存在，删除对应任务: %s", ot.Name) + deletedTaskCount++ + deletedIDs = append(deletedIDs, ot.ID) + database.DB.Unscoped().Where("id = ?", ot.ID).Delete(&models.Task{}) + } + } + } + + log("\n扫描完成: [新增 %d] [更新 %d] [移除 %d]", newTaskCount, updateTaskCount, deletedTaskCount) + log("----------------------------------------") + return upsertedIDs, deletedIDs +} + +// upsertRepoTask 处理来自仓库的任务的创建或更新 +func upsertRepoTask(parentTask *models.Task, sourceID, name, command, cron, workDir, tag string) (string, bool) { + defaultTaskConfig := `{"$task_all_envs":true}` + var existing models.Task + tx := database.DB.Where("source_id = ? AND repo_task_id = ?", sourceID, parentTask.ID).Limit(1).Find(&existing) + + if tx.RowsAffected > 0 { + // 更新操作 + existing.Name = name + existing.Command = models.BigText(command) + existing.Schedule = normalizeCron(cron) + existing.Languages = parentTask.Languages + existing.SourceID = sourceID + existing.RepoTaskID = parentTask.ID + existing.WorkDir = workDir + // 如果原配置为空或者是 {}，则应用默认配置 + if string(existing.Config) == "" || string(existing.Config) == "{}" { + existing.Config = models.BigText(defaultTaskConfig) + } + // 默认开启按条数清理30条 + if existing.CleanConfig == "" { + existing.CleanConfig = `{"type":"count","keep":30}` + } + database.DB.Save(&existing) + return existing.ID, false + } else { + // 创建新任务 + newTask := &models.Task{ + Name: name, + Command: models.BigText(command), + Schedule: normalizeCron(cron), + Type: "task", + TriggerType: constant.TriggerTypeCron, + Tags: tag, + Languages: parentTask.Languages, + Timeout: parentTask.Timeout, + Config: models.BigText(defaultTaskConfig), + Enabled: utils.BoolPtr(true), + WorkDir: workDir, + SourceID: sourceID, + RepoTaskID: parentTask.ID, + CleanConfig: `{"type":"count","keep":30}`, + } + newTask.ID = utils.GenerateID() + database.DB.Create(newTask) + return newTask.ID, true + } +} diff --git a/internal/services/repo/strategy.go b/internal/services/repo/strategy.go new file mode 100644 index 0000000..236ea74 --- /dev/null +++ b/internal/services/repo/strategy.go @@ -0,0 +1,27 @@ +package repo + +import ( + "github.com/engigu/baihu-panel/internal/models" +) + +// RepoParserStrategy 定义不同仓库解析策略的接口 +type RepoParserStrategy interface { + // SupportExtension 判断给定后缀的文件是否应该被处理 + SupportExtension(ext string, exts []string) bool + + // ShouldProcess 应用白名单/黑名单过滤，决定是否处理该文件 + ShouldProcess(relRepoPath, filename string, cfg models.RepoConfig) bool + + // ExtractMeta 从脚本文件中提取任务元数据（名称和 cron 表达式） + ExtractMeta(path string, ext string, cfg models.RepoConfig) (taskName string, taskCron string) +} + +// GetParserStrategy 根据来源类型返回相应的策略实现 +func GetParserStrategy(sourceType string) RepoParserStrategy { + switch sourceType { + case "ql": + return &QinglongStrategy{} + default: + return &StandardStrategy{} + } +} diff --git a/internal/services/repo/strategy_ql.go b/internal/services/repo/strategy_ql.go new file mode 100644 index 0000000..6a4a568 --- /dev/null +++ b/internal/services/repo/strategy_ql.go @@ -0,0 +1,71 @@ +package repo + +import ( + "regexp" + "strings" + "github.com/engigu/baihu-panel/internal/models" +) + +// QinglongStrategy 实现与青龙兼容的解析逻辑 +type QinglongStrategy struct{} + +func (s *QinglongStrategy) SupportExtension(ext string, exts []string) bool { + for _, e := range exts { + if ext == e { + return true + } + } + return false +} + +func (s *QinglongStrategy) ShouldProcess(relRepoPath, filename string, cfg models.RepoConfig) bool { + // 只有在显式设置了白名单时才进行白名单校验 (青龙行为) + if cfg.WhitelistPaths != "" { + if !matchesQLPattern(relRepoPath, filename, cfg.WhitelistPaths) { + return false + } + } + + // 校验黑名单 + if cfg.Blacklist != "" { + if matchesQLPattern(relRepoPath, filename, cfg.Blacklist) { + return false + } + } + return true +} + +func (s *QinglongStrategy) ExtractMeta(path string, ext string, cfg models.RepoConfig) (taskName string, taskCron string) { + return ExtractScriptMeta(path, ext) +} + +// matchesQLPattern 应用关键字过滤逻辑（正则或包含匹配） +func matchesQLPattern(rel, filename string, keywordsStr string) bool { + if keywordsStr == "" { + return false + } + + keywords := splitKeywords(keywordsStr) + for _, k := range keywords { + // 1. 尝试作为正则整体进行匹配，默认不区分大小写 (?i) + pattern := k + if !strings.HasPrefix(pattern, "(?i)") { + pattern = "(?i)" + pattern + } + + reg, err := regexp.Compile(pattern) + if err == nil { + // 优先匹配文件名（解决 ^jd[^_] 这种锚点在相对路径下失效的问题） + if reg.MatchString(filename) || reg.MatchString(rel) { + return true + } + } else { + // 回退逻辑：全小写包含判断 + kLower := strings.ToLower(k) + if strings.Contains(strings.ToLower(rel), kLower) || strings.Contains(strings.ToLower(filename), kLower) { + return true + } + } + } + return false +} diff --git a/internal/services/repo/strategy_std.go b/internal/services/repo/strategy_std.go new file mode 100644 index 0000000..169d15c --- /dev/null +++ b/internal/services/repo/strategy_std.go @@ -0,0 +1,31 @@ +package repo + +import ( + "github.com/engigu/baihu-panel/internal/models" +) + +// StandardStrategy 实现默认的解析逻辑 +type StandardStrategy struct{} + +func (s *StandardStrategy) SupportExtension(ext string, exts []string) bool { + for _, e := range exts { + if ext == e { + return true + } + } + return false +} + +func (s *StandardStrategy) ShouldProcess(relRepoPath, filename string, cfg models.RepoConfig) bool { + // 标准策略未来可能具有不同的过滤规则 + // 目前如果提供了白名单/黑名单，则遵循相同的逻辑，但使用更简单的匹配 + return true +} + +func (s *StandardStrategy) ExtractMeta(path string, ext string, cfg models.RepoConfig) (taskName string, taskCron string) { + // 仅在开启兼容 QL 配置时才解析脚本注释 + if cfg.CommentToTask == "true" { + return ExtractScriptMeta(path, ext) + } + return "", "" +} diff --git a/internal/services/repo/util.go b/internal/services/repo/util.go new file mode 100644 index 0000000..16fb499 --- /dev/null +++ b/internal/services/repo/util.go @@ -0,0 +1,236 @@ +package repo + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "github.com/engigu/baihu-panel/internal/utils" +) + +var ( + // envRegex 匹配脚本中的环境名称设置，如 Env("名称") + envRegex = regexp.MustCompile(`(?i)(?:new[ \t]+)?Env\(['"]?([^'"]+)['"]?\)`) + // cronRegex 匹配脚本中的 cron 表达式设置 + cronRegex = regexp.MustCompile(`(?i)(?:cron[ \t]*[:=][ \t]*['"]?([^'"\r\n]+))|(?:(?:^|[ \t\*\/])(([0-9\*\/\-,L?#]+[ \t]+){4,5}[0-9\*\/\-,L?#]+))`) + // cronFormatRegex 用于校验提取出的字符串是否符合 Cron 表达式格式 (5位或6位) + cronFormatRegex = regexp.MustCompile(`^(([0-9\*\/\-,L?#]+)[ \t]+){4,5}([0-9\*\/\-,L?#]+)$`) +) + +// ExtractScriptMeta 读取文件以提取任务名称和 cron 表达式 +func ExtractScriptMeta(path string, ext string) (taskName string, taskCron string) { + f, err := os.Open(path) + if err != nil { + return "", "" + } + defer f.Close() + + scanner := bufio.NewScanner(f) + var firstCommentLine string + inBlockComment := false + + // 特殊处理：针对当前文件名的 Cron 关联正则表达式 (对标青龙 perl 逻辑) + // 寻找类似 "// 0 0 * * * jd_task.js" 的行 + fileNameEscaped := regexp.QuoteMeta(filepath.Base(path)) + associatedCronRegex := regexp.MustCompile(fmt.Sprintf(`(?i)(?:^|[ \t\*\//])(([0-9\*\/\-,L?#]+[ \t]+){4,5}[0-9\*\/\-,L?#]+)[ \t,"]+.*%s`, fileNameEscaped)) + + for i := 0; i < 15 && scanner.Scan(); i++ { // 限制扫描范围，避免误匹配代码深处的属性 + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + + // 处理块注释开始/结束 + if strings.HasPrefix(line, "/*") { + inBlockComment = true + line = strings.TrimPrefix(line, "/*") + line = strings.TrimPrefix(line, "*") + line = strings.TrimSpace(line) + } + if strings.HasSuffix(line, "*/") { + inBlockComment = false + line = strings.TrimSuffix(line, "*/") + line = strings.TrimSpace(line) + } + + // 1. 尝试提取任务名称 (优先使用 Env) + if taskName == "" { + if envMatch := envRegex.FindStringSubmatch(line); len(envMatch) > 1 { + taskName = strings.TrimSpace(envMatch[1]) + } else if strings.Contains(line, "name:") { + // 兼容 name: "xxx" 格式 + nameRegex := regexp.MustCompile(`(?i)name:[ \t]*['"]([^'"]+)['"]`) + if nameMatch := nameRegex.FindStringSubmatch(line); len(nameMatch) > 1 { + taskName = strings.TrimSpace(nameMatch[1]) + } + } + } + + // 如果还没找到名称，且在注释中，记录第一行非空注释作为备选名称 + if taskName == "" && (inBlockComment || strings.HasPrefix(line, "//") || strings.HasPrefix(line, "*") || strings.HasPrefix(line, "#")) { + cleanLine := line + if strings.HasPrefix(line, "//") { + cleanLine = strings.TrimPrefix(line, "//") + } else if strings.HasPrefix(line, "#") { + cleanLine = strings.TrimPrefix(line, "#") + } else if strings.HasPrefix(line, "*") { + cleanLine = strings.TrimPrefix(line, "*") + } + cleanLine = strings.TrimSpace(cleanLine) + + // 排除掉包含 "Env" 或 "cron" 的行, 且排除掉可能是路径或URL的行 + if cleanLine != "" && !strings.Contains(strings.ToLower(cleanLine), "env") && + !strings.Contains(strings.ToLower(cleanLine), "cron") && + !strings.Contains(cleanLine, "http") && + !strings.Contains(cleanLine, "/") && + firstCommentLine == "" { + // 且排除掉纯 cron 表达式 + if !cronRegex.MatchString(cleanLine) { + firstCommentLine = cleanLine + } + } + } + + // 2. 提取 Cron + if taskCron == "" { + // A. 优先查找关联了当前文件名的 Cron (对标 QL) + if assocMatch := associatedCronRegex.FindStringSubmatch(line); len(assocMatch) > 1 { + tempCron := strings.Trim(strings.TrimSpace(assocMatch[1]), "\"' \t") + if isLikelyCron(tempCron) { + taskCron = tempCron + } + } + + // B. 如果没找到，尝试普通的 cron: "..." 或 cron 表达式 + if taskCron == "" { + if cronMatch := cronRegex.FindStringSubmatch(line); len(cronMatch) > 0 { + for _, m := range cronMatch[1:] { + if m != "" { + tempCron := strings.Trim(strings.TrimSpace(m), "\"' \t") + if isLikelyCron(tempCron) { + taskCron = tempCron + break + } + } + } + } + } + } + + if taskName != "" && taskCron != "" { + break + } + } + + // 如果最后还是没找到 taskName，使用备选名称或文件名 + if taskName == "" { + if firstCommentLine != "" { + taskName = firstCommentLine + } else { + taskName = strings.TrimSuffix(filepath.Base(path), ext) + } + } + + return taskName, taskCron +} + +// isLikelyCron 校验字符串是否符合 Cron 表达式的格式特征 +func isLikelyCron(s string) bool { + return cronFormatRegex.MatchString(s) +} + +// splitKeywords 按竖线或逗号分割字符串 +func splitKeywords(s string) []string { + if s == "" { + return nil + } + var parts []string + if strings.Contains(s, "|") { + parts = strings.Split(s, "|") + } else if strings.Contains(s, ",") { + parts = strings.Split(s, ",") + } else { + parts = []string{s} + } + + var res []string + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + res = append(res, p) + } + } + return res +} + +// getValidExtensions 返回支持的文件扩展名列表，优先考虑自定义扩展名 +func getValidExtensions(customExtensions string) []string { + exts := []string{".js", ".py", ".ts", ".sh"} + if customExtensions != "" { + customExts := splitKeywords(customExtensions) + if len(customExts) > 0 { + exts = nil + for _, e := range customExts { + e = strings.TrimSpace(e) + if e != "" { + if !strings.HasPrefix(e, ".") { + e = "." + e + } + exts = append(exts, e) + } + } + } + } + return exts +} + +// sanitizeIdentifier 将非字母数字字符替换为下划线 +func sanitizeIdentifier(s string) string { + reg := regexp.MustCompile(`[^a-zA-Z0-9]+`) + res := reg.ReplaceAllString(s, "_") + return strings.ToLower(strings.Trim(res, "_")) +} + +// normalizeCron 确保 cron 表达式具有 6 个字段 +func normalizeCron(cron string) string { + fields := strings.Fields(cron) + if len(fields) == 5 { + return "0 " + cron + } + return cron +} + +// pathExists 检查路径是否存在 +func pathExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +// isDir 检查路径是否为目录 +func isDir(path string) bool { + info, err := os.Stat(path) + if err != nil { + return false + } + return info.IsDir() +} + +// getCommandByExt 根据文件扩展名返回默认执行命令 +func getCommandByExt(ext, path string) string { + quotedPath := utils.QuotePath(path) + switch ext { + case ".js", ".ts": + return fmt.Sprintf("node %s", quotedPath) + case ".py": + return fmt.Sprintf("python %s", quotedPath) + case ".sh": + return fmt.Sprintf("bash %s", quotedPath) + case ".php": + return fmt.Sprintf("php %s", quotedPath) + case ".cs": + return fmt.Sprintf("dotnet run %s", quotedPath) + } + return quotedPath +} diff --git a/internal/services/tasks/executor_service.go b/internal/services/tasks/executor_service.go index c599299..e80757f 100644 --- a/internal/services/tasks/executor_service.go +++ b/internal/services/tasks/executor_service.go @@ -128,8 +128,12 @@ func (h *ServerSchedulerHandler) OnTaskExecuting(req *executor.ExecutionRequest) return nil, nil, nil } - // 1. 创建初始日志记录 - taskLog, err := h.es.taskLogService.CreateEmptyLog(task.ID, req.Command) + // 1. 创建初始日志记录（对系统敏感信息进行全面脱敏处理） + masks := append([]string{}, req.Secrets...) + masks = append(masks, utils.GetSystemSecrets()...) + maskedCommand := utils.MaskSecrets(req.Command, masks) + + taskLog, err := h.es.taskLogService.CreateEmptyLog(task.ID, maskedCommand) if err != nil { return nil, nil, fmt.Errorf("创建初始日志失败: %v", err) } @@ -179,8 +183,8 @@ func (h *ServerSchedulerHandler) OnTaskHeartbeat(req *executor.ExecutionRequest, // 每分钟打印一次任务还在运行的日志 if duration >= 60000 && (duration/60000 > (duration-3000)/60000) { - logger.Infof("[Scheduler] 任务运行中... (#%s 已耗时: %v)", - req.TaskID, (time.Duration(duration) * time.Millisecond).Round(time.Second)) + logger.Infof("[Scheduler] 命令: %s (#%s 已耗时: %v)", + utils.MaskSecrets(req.Command, req.Secrets), req.TaskID, (time.Duration(duration) * time.Millisecond).Round(time.Second)) } } @@ -455,6 +459,26 @@ func (es *ExecutorService) ExecuteDispatcher(ctx context.Context, req *executor. if cmd != "" { req.Command = cmd req.WorkDir = workDir + req.UseMise = false // 仓库同步任务不使用 mise，由系统原生执行 + // 强制脱敏并更新数据库日志 + masks := append([]string{}, req.Secrets...) + masks = append(masks, utils.GetSystemSecrets()...) + + // 补充仓库特有的 AuthToken + var repoCfg models.RepoConfig + if err := json.Unmarshal([]byte(task.Config), &repoCfg); err == nil && repoCfg.AuthToken != "" { + masks = append(masks, repoCfg.AuthToken) + } + + maskedCmd := utils.MaskSecrets(req.Command, masks) + + // 更新数据库中的任务日志命令内容 + if req.LogID != "" { + es.taskLogService.UpdateLogCommand(req.LogID, maskedCmd) + } + + // 在控制台打印最终执行的脱敏命令 + logger.Infof("[Executor] 仓库同步最终执行命令: %s", maskedCmd) } } @@ -629,6 +653,27 @@ func (es *ExecutorService) ExecuteTask(taskID string, extraEnvs []string) *execu } } +// SyncRepoTasks 增量同步仓库任务到调度器 +func (es *ExecutorService) SyncRepoTasks(upsertedIDs []string, deletedIDs []string) { + // 处理删除的任务 + for _, id := range deletedIDs { + es.RemoveCronTask(id) + } + + // 处理新增/更新的任务 + if len(upsertedIDs) > 0 { + var tasks []models.Task + database.DB.Where("id IN ?", upsertedIDs).Find(&tasks) + for _, t := range tasks { + if utils.DerefBool(t.Enabled, true) { + es.AddCronTask(&t) + } else { + es.RemoveCronTask(t.ID) + } + } + } +} + // StopTaskExecution stops a running task execution by LogID func (es *ExecutorService) StopTaskExecution(logID string) error { var taskLog models.TaskLog @@ -994,11 +1039,22 @@ func (es *ExecutorService) BuildRepoCommand(task *models.Task) (string, string) exePath = "baihu" // Fallback if executable path can't be found } + // 尽量使用代号 $SCRIPTS_DIR$ 替代绝对路径，增加可读性和可移植性 + scriptsDir, _ := filepath.Abs(constant.ScriptsWorkDir) + displayTargetPath := absTargetPath + if rel, err := filepath.Rel(scriptsDir, absTargetPath); err == nil && !strings.HasPrefix(rel, "..") { + if rel == "." { + displayTargetPath = "$SCRIPTS_DIR$" + } else { + displayTargetPath = "$SCRIPTS_DIR$/" + filepath.ToSlash(rel) + } + } + args := []string{ "reposync", "--source-type", config.SourceType, "--source-url", config.SourceURL, - "--target-path", absTargetPath, + "--target-path", displayTargetPath, } if config.Branch != "" { args = append(args, "--branch", config.Branch) @@ -1027,6 +1083,9 @@ func (es *ExecutorService) BuildRepoCommand(task *models.Task) (string, string) if config.Dependence != "" { args = append(args, "--dependence", config.Dependence) } + if config.CommentToTask == "true" { + args = append(args, "--commenttotask", "true") + } if config.Extensions != "" { args = append(args, "--extensions", config.Extensions) } diff --git a/internal/services/tasks/ql_repo_parser.go b/internal/services/tasks/ql_repo_parser.go deleted file mode 100644 index 859d2a3..0000000 --- a/internal/services/tasks/ql_repo_parser.go +++ /dev/null @@ -1,471 +0,0 @@ -package tasks - -import ( - "bufio" - "encoding/json" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/engigu/baihu-panel/internal/constant" - "github.com/engigu/baihu-panel/internal/database" - "github.com/engigu/baihu-panel/internal/models" - "github.com/engigu/baihu-panel/internal/utils" -) - -// regex patterns for script comment parsing -var ( - envRegex = regexp.MustCompile(`(?i)(?:new[ \t]+)?Env\(['"]?([^'"]+)['"]?\)`) - cronRegex = regexp.MustCompile(`(?i)(?:cron[ \t]*[:=]?[ \t]*['"]?([^'"\r\n]+?)['"]?(?:\s|$))|(?:(?:^|[ \t\*\/])([0-9\*\/\-,L?]+[ \t]+[0-9\*\/\-,L?#]+[ \t]+[0-9\*\/\-,L?#]+[ \t]+[0-9\*\/\-,L?#]+[ \t]+[0-9\*\/\-,L?#]+(?:[ \t]+[0-9\*\/\-,L?#]+)?))`) -) - -// ParseRepoScriptsAndAddCron scans the repo dir for scripts, parses cron and env comments, and registers tasks -func ParseRepoScriptsAndAddCron(es *ExecutorService, taskID string, logWriter io.Writer) { - // help print logs to writer if provided - log := func(format string, a ...interface{}) { - msg := fmt.Sprintf(format, a...) - if !strings.HasSuffix(msg, "\n") { - msg += "\n" - } - if logWriter != nil { - logWriter.Write([]byte(msg)) - } - // logger.Info(msg) - } - - var repoTask models.Task - res := database.DB.Where("id = ?", taskID).Limit(1).Find(&repoTask) - if res.Error != nil || res.RowsAffected == 0 { - return - } - - if repoTask.Type != constant.TaskTypeRepo { - return - } - - var repoCfg models.RepoConfig - if err := json.Unmarshal([]byte(repoTask.Config), &repoCfg); err != nil { - return - } - - if repoCfg.RepoSource != "ql" || !repoCfg.AutoAddCron { - return - } - - // target path - targetPath := repoCfg.TargetPath - if targetPath == "" { - targetPath = repoTask.WorkDir - } else if !filepath.IsAbs(targetPath) { - targetPath = filepath.Join(resolveAbsScriptsDir(), targetPath) - } - if targetPath == "" { - return - } - targetPath = filepath.Clean(targetPath) - - // We might have appended a repo id to targetPath - repoId := utils.GetRepoIdentifier(repoCfg.SourceURL, repoCfg.Branch) - - gitDir := filepath.Join(targetPath, ".git") - if !isDir(targetPath) || !pathExists(gitDir) { - repoPath := filepath.Join(targetPath, repoId) - if pathExists(repoPath) { - targetPath = repoPath - } - } - - if !pathExists(targetPath) { - return - } - - // tag used during sync - tag := fmt.Sprintf("%s", repoId) - - exts := []string{".js", ".py", ".ts", ".sh", ".php"} - if repoCfg.Extensions != "" { - customExts := splitKeywords(repoCfg.Extensions) - if len(customExts) > 0 { - exts = nil - for _, e := range customExts { - e = strings.TrimSpace(e) - if e != "" { - if !strings.HasPrefix(e, ".") { - e = "." + e - } - exts = append(exts, e) - } - } - } - } - - log("\n----------------------------------------") - log(" 开始扫描脚本并自动注册定时任务 ") - log("----------------------------------------") - - foundSourceIDs := make(map[string]bool) - newTaskCount := 0 - updateTaskCount := 0 - - filepath.WalkDir(targetPath, func(path string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { - return nil - } - - if strings.Contains(path, ".git") { - return nil - } - - ext := filepath.Ext(path) - validExt := false - for _, e := range exts { - if ext == e { - validExt = true - break - } - } - if !validExt { - return nil - } - - f, err := os.Open(path) - if err != nil { - return nil - } - defer f.Close() - - var taskName string - var taskCron string - - scanner := bufio.NewScanner(f) - var firstCommentLine string - inBlockComment := false - - // 特殊处理：针对当前文件名的 Cron 关联正则表达式 (对标青龙 perl 逻辑) - // 寻找类似 "// 0 0 * * * jd_task.js" 的行 - fileNameEscaped := regexp.QuoteMeta(filepath.Base(path)) - associatedCronRegex := regexp.MustCompile(fmt.Sprintf(`(?i)(?:^|[ \t\*\//])(([0-9\*\/\-,L?#]+[ \t]+){4,5}[0-9\*\/\-,L?#]+)[ \t,"]+.*%s`, fileNameEscaped)) - - for i := 0; i < 150 && scanner.Scan(); i++ { // QL 扫描范围较大 - line := strings.TrimSpace(scanner.Text()) - if line == "" { - continue - } - - // 处理块注释开始/结束 - if strings.HasPrefix(line, "/*") { - inBlockComment = true - line = strings.TrimPrefix(line, "/*") - line = strings.TrimPrefix(line, "*") - line = strings.TrimSpace(line) - } - if strings.HasSuffix(line, "*/") { - inBlockComment = false - line = strings.TrimSuffix(line, "*/") - line = strings.TrimSpace(line) - } - - // 1. 尝试提取任务名称 (优先使用 Env) - if taskName == "" { - if envMatch := envRegex.FindStringSubmatch(line); len(envMatch) > 1 { - taskName = strings.TrimSpace(envMatch[1]) - } else if strings.Contains(line, "name:") { - // 兼容 name: "xxx" 格式 - nameRegex := regexp.MustCompile(`(?i)name:[ \t]*['"]([^'"]+)['"]`) - if nameMatch := nameRegex.FindStringSubmatch(line); len(nameMatch) > 1 { - taskName = strings.TrimSpace(nameMatch[1]) - } - } - } - - // 如果还没找到名称，且在注释中，记录第一行非空注释作为备选名称 - if taskName == "" && (inBlockComment || strings.HasPrefix(line, "//") || strings.HasPrefix(line, "*") || strings.HasPrefix(line, "#")) { - cleanLine := line - if strings.HasPrefix(line, "//") { - cleanLine = strings.TrimPrefix(line, "//") - } else if strings.HasPrefix(line, "#") { - cleanLine = strings.TrimPrefix(line, "#") - } else if strings.HasPrefix(line, "*") { - cleanLine = strings.TrimPrefix(line, "*") - } - cleanLine = strings.TrimSpace(cleanLine) - - // 排除掉包含 "Env" 或 "cron" 的行, 且排除掉可能是路径或URL的行 - if cleanLine != "" && !strings.Contains(strings.ToLower(cleanLine), "env") && - !strings.Contains(strings.ToLower(cleanLine), "cron") && - !strings.Contains(cleanLine, "http") && - !strings.Contains(cleanLine, "/") && - firstCommentLine == "" { - // 且排除掉纯 cron 表达式 - if !cronRegex.MatchString(cleanLine) { - firstCommentLine = cleanLine - } - } - } - - // 2. 提取 Cron - if taskCron == "" { - // A. 优先查找关联了当前文件名的 Cron (对标 QL) - if assocMatch := associatedCronRegex.FindStringSubmatch(line); len(assocMatch) > 1 { - taskCron = strings.TrimSpace(assocMatch[1]) - } - - // B. 如果没找到，尝试普通的 cron: "..." 或 cron 表达式 - if taskCron == "" { - if cronMatch := cronRegex.FindStringSubmatch(line); len(cronMatch) > 0 { - for _, m := range cronMatch[1:] { - if m != "" { - taskCron = strings.TrimSpace(m) - break - } - } - } - } - } - - if taskName != "" && taskCron != "" { - break - } - } - - // 如果最后还是没找到 taskName，使用备选名称或文件名 - if taskName == "" { - if firstCommentLine != "" { - taskName = firstCommentLine - } else { - taskName = strings.TrimSuffix(filepath.Base(path), ext) - } - } - - // 3. 应用白名单 / 黑名单过滤 (逻辑对标青龙) - // relRepoPath 是相对于仓库根目录的路径，filename 是文件名 - relRepoPath, _ := filepath.Rel(targetPath, path) - filename := filepath.Base(path) - - // 只有在显式设置了白名单时才进行白名单校验 (青龙行为) - if repoCfg.WhitelistPaths != "" { - if !matchesQLPattern(relRepoPath, filename, repoCfg.WhitelistPaths) { - return nil - } - } - - // 校验黑名单 - if repoCfg.Blacklist != "" { - if matchesQLPattern(relRepoPath, filename, repoCfg.Blacklist) { - return nil - } - } - - if taskName != "" && taskCron != "" { - // 获取脚本相对于数据目录的路径 - absScriptsDir := resolveAbsScriptsDir() - absTargetPath, _ := filepath.Abs(targetPath) - absPath, _ := filepath.Abs(path) - - // 计算 SourceID: 相对于脚本目录的完整路径，并清洗特殊符号 - relPath, _ := filepath.Rel(absScriptsDir, absPath) - sourceID := sanitizeIdentifier(relPath) - - // 替换绝对路径为代号 $SCRIPTS_DIR$ - displayPath := path - displayWorkDir := targetPath - if strings.HasPrefix(absPath, absScriptsDir) { - if relCommandPath, err := filepath.Rel(absTargetPath, absPath); err == nil && relCommandPath != "" { - displayPath = filepath.Clean(relCommandPath) - } - // 获取目录路径 - relDir, _ := filepath.Rel(absScriptsDir, absTargetPath) - displayWorkDir = filepath.Join("$SCRIPTS_DIR$", relDir) - } - - // Found task, save it - command := getCommandByExt(ext, displayPath) - - // 注册任务默认开启“全量环境变量注入”，以适配大多数脚本 - defaultTaskConfig := `{"$task_all_envs":true}` - - // See if task exists (优先通过 SourceID 匹配) - var existing models.Task - tx := database.DB.Where("source_id = ? AND repo_task_id = ?", sourceID, repoTask.ID).Limit(1).Find(&existing) - - if tx.RowsAffected > 0 { - // update - existing.Name = taskName - existing.Command = models.BigText(command) - existing.Schedule = normalizeCron(taskCron) - existing.Languages = repoTask.Languages - existing.SourceID = sourceID - existing.RepoTaskID = repoTask.ID - existing.WorkDir = displayWorkDir - // 如果原配置为空或者是 {}，则应用默认配置 - if string(existing.Config) == "" || string(existing.Config) == "{}" { - existing.Config = models.BigText(defaultTaskConfig) - } - // 默认开启按条数清理30条 - if existing.CleanConfig == "" { - existing.CleanConfig = `{"type":"count","keep":30}` - } - database.DB.Save(&existing) - - if utils.DerefBool(existing.Enabled, true) && es != nil { - es.AddCronTask(&existing) - } - log("[更新] 任务: %s (%s)", taskName, filename) - updateTaskCount++ - foundSourceIDs[sourceID] = true - } else { - // create new - newTask := &models.Task{ - Name: taskName, - Command: models.BigText(command), - Schedule: normalizeCron(taskCron), - Type: "task", - TriggerType: constant.TriggerTypeCron, - Tags: tag, - Languages: repoTask.Languages, - Timeout: repoTask.Timeout, - Config: models.BigText(defaultTaskConfig), - Enabled: utils.BoolPtr(true), - WorkDir: displayWorkDir, - SourceID: sourceID, - RepoTaskID: repoTask.ID, - CleanConfig: `{"type":"count","keep":30}`, - } - newTask.ID = utils.GenerateID() - database.DB.Create(newTask) - if es != nil { - es.AddCronTask(newTask) - } - log("[新增] 任务: %s (%s)", taskName, filename) - newTaskCount++ - foundSourceIDs[sourceID] = true - } - } - - return nil - }) - - // 清理该仓库任务下不再存在的旧脚本任务 - deletedTaskCount := 0 - var oldTasks []models.Task - if err := database.DB.Where("repo_task_id = ?", repoTask.ID).Find(&oldTasks).Error; err == nil { - for _, ot := range oldTasks { - if !foundSourceIDs[ot.SourceID] { - log("[移除] 脚本已不存在，删除对应任务: %s", ot.Name) - deletedTaskCount++ - if es != nil { - if es.taskService != nil { - es.taskService.DeleteTask(ot.ID) - } - es.RemoveCronTask(ot.ID) - } else { - // Fallback if es is nil (which shouldn't happen, but just in case) - database.DB.Unscoped().Where("id = ?", ot.ID).Delete(&models.Task{}) - } - } - } - } - - log("\n扫描完成: [新增 %d] [更新 %d] [移除 %d]", newTaskCount, updateTaskCount, deletedTaskCount) - log("----------------------------------------") -} - -func sanitizeIdentifier(s string) string { - // 将所有非字母数字替换为下划线 - reg := regexp.MustCompile(`[^a-zA-Z0-9]+`) - res := reg.ReplaceAllString(s, "_") - return strings.ToLower(strings.Trim(res, "_")) -} - -func normalizeCron(cron string) string { - fields := strings.Fields(cron) - if len(fields) == 5 { - return "0 " + cron - } - return cron -} - -func pathExists(path string) bool { - _, err := os.Stat(path) - return err == nil -} - -func isDir(path string) bool { - info, err := os.Stat(path) - if err != nil { - return false - } - return info.IsDir() -} - -func getCommandByExt(ext, path string) string { - quotedPath := utils.QuotePath(path) - switch ext { - case ".js", ".ts": - return fmt.Sprintf("node %s", quotedPath) - case ".py": - return fmt.Sprintf("python %s", quotedPath) - case ".sh": - return fmt.Sprintf("bash %s", quotedPath) - case ".php": - return fmt.Sprintf("php %s", quotedPath) - } - return quotedPath -} - -func matchesQLPattern(rel, filename string, keywordsStr string) bool { - if keywordsStr == "" { - return false - } - - keywords := splitKeywords(keywordsStr) - for _, k := range keywords { - // 1. 尝试作为正则整体进行匹配，默认不区分大小写 (?i) - pattern := k - if !strings.HasPrefix(pattern, "(?i)") { - pattern = "(?i)" + pattern - } - - reg, err := regexp.Compile(pattern) - if err == nil { - // 优先匹配文件名（解决 ^jd[^_] 这种锚点在相对路径下失效的问题） - if reg.MatchString(filename) || reg.MatchString(rel) { - return true - } - } else { - // 回退逻辑：全小写包含判断 - kLower := strings.ToLower(k) - if strings.Contains(strings.ToLower(rel), kLower) || strings.Contains(strings.ToLower(filename), kLower) { - return true - } - } - } - return false -} - -func splitKeywords(s string) []string { - if s == "" { - return nil - } - var parts []string - if strings.Contains(s, "|") { - parts = strings.Split(s, "|") - } else if strings.Contains(s, ",") { - parts = strings.Split(s, ",") - } else { - parts = []string{s} - } - - var res []string - for _, p := range parts { - p = strings.TrimSpace(p) - if p != "" { - res = append(res, p) - } - } - return res -} diff --git a/internal/services/tasks/task_log_service.go b/internal/services/tasks/task_log_service.go index 765ce96..b39a97f 100644 --- a/internal/services/tasks/task_log_service.go +++ b/internal/services/tasks/task_log_service.go @@ -89,6 +89,11 @@ func (s *TaskLogService) UpdateTaskDuration(logID string, duration int64) error return database.DB.Model(&models.TaskLog{}).Where("id = ?", logID).Update("duration", duration).Error } +// UpdateLogCommand 更新日志中的命令内容（用于动态生成的命令脱敏） +func (s *TaskLogService) UpdateLogCommand(logID string, command string) error { + return database.DB.Model(&models.TaskLog{}).Where("id = ?", logID).Update("command", models.BigText(command)).Error +} + // UpdateTaskStats 更新任务统计 func (s *TaskLogService) UpdateTaskStats(taskID string, status string) { if s.sendStatsService == nil { diff --git a/internal/utils/runtime_env.go b/internal/utils/runtime_env.go index 4d7672e..9b999b3 100644 --- a/internal/utils/runtime_env.go +++ b/internal/utils/runtime_env.go @@ -39,6 +39,25 @@ func BuildRuntimeProcessEnv() []string { return envs } +// GetSystemSecrets 返回当前运行时的所有系统级敏感机密（如数据库账号密码等） +func GetSystemSecrets() []string { + secrets := make([]string, 0, 8) + addIfNotEmpty := func(s string) { + if s != "" { + secrets = append(secrets, s) + } + } + + addIfNotEmpty(constant.RuntimeDBPassword) + addIfNotEmpty(constant.RuntimeDBUser) + addIfNotEmpty(constant.RuntimeDBHost) + addIfNotEmpty(constant.RuntimeDBName) + addIfNotEmpty(constant.RuntimeDBPath) + addIfNotEmpty(constant.RuntimeDBDSN) + + return secrets +} + // BuildShellEnvPrefix 将 KEY=VALUE 环境变量切片转换为 shell 前缀。 func BuildShellEnvPrefix(envs []string) string { parts := make([]string, 0, len(envs)) diff --git a/web/src/api/index.ts b/web/src/api/index.ts index 3faa2a7..d8ec8fa 100644 --- a/web/src/api/index.ts +++ b/web/src/api/index.ts @@ -70,7 +70,12 @@ export const api = { }, create: (data: Partial) => request('/tasks', { method: 'POST', body: JSON.stringify(data) }), update: (id: string, data: Partial) => request(`/tasks/${id}`, { method: 'PUT', body: JSON.stringify(data) }), - delete: (id: string) => request(`/tasks/${id}`, { method: 'DELETE' }), + delete: (id: string, params?: { delete_files?: boolean }) => { + const query = new URLSearchParams() + if (params?.delete_files !== undefined) query.set('delete_files', String(params.delete_files)) + const queryString = query.toString() + return request(`/tasks/${id}${queryString ? '?' + queryString : ''}`, { method: 'DELETE' }) + }, batchDelete: (ids: string[]) => request<{ count: number }>('/tasks/batch-delete', { method: 'POST', body: JSON.stringify({ ids }) }), batchDeleteByQuery: (params?: { name?: string, agent_id?: string, tags?: string, type?: string }) => { const query = new URLSearchParams() @@ -382,6 +387,7 @@ export interface RepoConfig { dependence?: string extensions?: string auto_add_cron?: boolean + commenttotask?: string concurrency?: number repo_source?: string } diff --git a/web/src/views/tasks/RepoDialog.vue b/web/src/views/tasks/RepoDialog.vue index d41c27f..a961830 100644 --- a/web/src/views/tasks/RepoDialog.vue +++ b/web/src/views/tasks/RepoDialog.vue @@ -10,7 +10,7 @@ import { Checkbox } from '@/components/ui/checkbox' import { ScrollArea } from '@/components/ui/scroll-area' import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover' import DirTreeSelect from '@/components/DirTreeSelect.vue' -import { X, Globe, GitBranch, Shield, Zap, Clock, Download, Plus, Search, Check, ChevronsUpDown, Loader2, AlertCircle } from 'lucide-vue-next' +import { X, Globe, GitBranch, Shield, Zap, Clock, Download, Plus, Search, Check, ChevronsUpDown, Loader2, AlertCircle, Terminal } from 'lucide-vue-next' import { api, type Task, type RepoConfig, type Agent, type MiseLanguage } from '@/api' import { toast } from 'vue-sonner' import { cn } from '@/lib/utils' @@ -64,6 +64,7 @@ const repoConfig = ref({ dependence: '', extensions: '', auto_add_cron: false, + commenttotask: 'false', concurrency: 1, repo_source: '', proxy: '' @@ -81,6 +82,13 @@ const autoAddCron = computed({ } }) +const pullQlConfig = computed({ + get: () => repoConfig.value.commenttotask === 'true', + set: (val: boolean) => { + repoConfig.value.commenttotask = val ? 'true' : 'false' + } +}) + // === 语言环境相关 === const installedLangs = ref([]) const loadingLangs = ref(false) @@ -185,6 +193,110 @@ function updateLangName(index: number, name: string) { const showQlImportDialog = ref(false) const qlCommandInput = ref('') +const showBaihuImportDialog = ref(false) +const baihuCommandInput = ref('') + +function importFromBaihu() { + baihuCommandInput.value = '' + showBaihuImportDialog.value = true +} + +function submitBaihuImport() { + const s = baihuCommandInput.value.trim() + if (!s) { + showBaihuImportDialog.value = false + return + } + + // Parse arguments handling quotes + const args: string[] = [] + const regex = /[^\s"']+|"([^"]*)"|'([^']*)'/g + let match + while ((match = regex.exec(s)) !== null) { + args.push(match[1] || match[2] || match[0]) + } + + let i = 0 + // Skip leading 'baihu' or 'reposync' + if (args[i] === 'baihu') i++ + if (args[i] === 'reposync') i++ + + let hasValidField = false + for (; i < args.length; i++) { + const arg = args[i] + if (!arg || !arg.startsWith('--')) continue + + const value = args[i + 1] + if (value === undefined || value.startsWith('--')) continue + + i++ // Skip value in next iteration + hasValidField = true + + switch (arg) { + case '--source-type': repoConfig.value.source_type = value; break + case '--source-url': + repoConfig.value.source_url = value + // Auto-generate name from URL if name is empty + if (!form.value.name) { + try { + const urlPaths = value.split('/') + const name = urlPaths[urlPaths.length - 1]?.replace('.git', '') || '未命名仓库' + form.value.name = '同步 ' + name + } catch { /* ignore */ } + } + break + case '--target-path': + // Strip $SCRIPTS_DIR$/ prefix for UI + if (value.startsWith('$SCRIPTS_DIR$/')) { + repoConfig.value.target_path = value.replace('$SCRIPTS_DIR$/', '') + } else if (value === '$SCRIPTS_DIR$') { + repoConfig.value.target_path = '' + } else { + repoConfig.value.target_path = value + } + break + case '--branch': repoConfig.value.branch = value; break + case '--path': repoConfig.value.sparse_path = value; break + case '--single-file': isSingleFile.value = value === 'true'; break + case '--proxy-url': + repoConfig.value.proxy_url = value + repoConfig.value.proxy = 'custom' + break + case '--auth-token': repoConfig.value.auth_token = value; break + case '--whitelist-paths': repoConfig.value.whitelist_paths = value; break + case '--blacklist': repoConfig.value.blacklist = value; break + case '--dependence': repoConfig.value.dependence = value; break + case '--extensions': repoConfig.value.extensions = value; break + case '--task-timeout': form.value.timeout = parseInt(value) || 30; break + case '--task-langs': + try { + const langs = JSON.parse(value) + if (Array.isArray(langs)) { + selectedLangs.value = langs.map(l => ({ + name: l.name || '', + version: l.version || '', + availableVersions: [] + })) + // Trigger available versions update + selectedLangs.value.forEach(l => updateAvailableVersions(l)) + } + } catch (e) { + console.error('Parse task-langs failed', e) + } + break + } + } + + if (hasValidField) { + repoConfig.value.auto_add_cron = true + repoConfig.value.commenttotask = 'true' + toast.success('命令解析成功，已自动填充表单') + showBaihuImportDialog.value = false + } else { + toast.error('未识别到有效的 reposync 参数') + } +} + function importFromQl() { qlCommandInput.value = '' showQlImportDialog.value = true @@ -234,6 +346,7 @@ function submitQlImport() { if (args[7]) repoConfig.value.extensions = args[7] repoConfig.value.auto_add_cron = true + repoConfig.value.commenttotask = 'true' repoConfig.value.repo_source = 'ql' toast.success('指令解析成功，已开启自动添加任务，请继续完善其他设置') showQlImportDialog.value = false @@ -325,6 +438,7 @@ watch(() => props.open, async (val: boolean) => { dependence: '', extensions: '', auto_add_cron: false, + commenttotask: 'false', concurrency: 1, repo_source: '' } @@ -428,10 +542,16 @@ async function save() { {{ isEdit ? '编辑仓库同步' : '新建仓库同步' }} - +
+ + +
@@ -786,12 +906,12 @@ async function save() {
- 自动添加任务 + 自动添加任务并解析元数据
- +
-

- {{ autoAddCron ? '同步完成后将尝试自动分析脚本并注册定时任务。' : '仅拉取脚本，不自动注册成面板任务。' }} +

+ {{ autoAddCron ? '同步后将自动识别脚本中的 new Env("xxx") 和 cron 信息并注册任务。' : '仅拉取脚本，不自动注册任务。' }}

@@ -879,6 +999,57 @@ async function save() { + + + + + + + + 命令行快速导入 + + + +
+
+

+ 粘贴包含 reposync 及其参数的命令，系统将自动填充表单。 +

+
+ +
+
+ + +
+
+ baihu reposync --source-url 'https://...' --branch 'main' --blacklist '...' +
+
+ +
+