fix: remote agent exec errors

This commit is contained in:
engigu
2026-02-08 10:18:39 +08:00
parent c3c79f25bd
commit b291e3740f
6 changed files with 152 additions and 53 deletions
+3
View File
@@ -439,6 +439,9 @@ func (c *AgentController) WSConnect(ctx *gin.Context) {
// 启动读写协程
go c.wsWritePump(ac)
go c.wsReadPump(ac, agent)
// 主动推送任务列表
go c.wsManager.BroadcastTasks(agent.ID)
}
// wsReadPump 读取消息
+5
View File
@@ -77,6 +77,11 @@ func (m *CronManager) AddTask(task CronTask) error {
timeout := task.GetTimeout()
entryID, err := m.cron.AddFunc(task.GetSchedule(), func() {
defer func() {
if r := recover(); r != nil {
m.logger.Errorf("[CronManager] 任务 #%s 执行过程中发生 Panic: %v", taskID, r)
}
}()
m.logger.Infof("[CronManager] 触发计划任务 #%s (%s)", taskID, name)
req := &ExecutionRequest{
+15 -3
View File
@@ -262,15 +262,27 @@ func (s *Scheduler) worker(id int) {
case <-s.stopCh:
return
case req := <-s.taskQueue:
// 速率限制
<-s.rateLimiter
s.executeTask(req)
func() {
defer func() {
if r := recover(); r != nil {
s.logger.Errorf("[Scheduler] Worker %d panic while processing task %s: %v", id, req.TaskID, r)
}
}()
// 速率限制
<-s.rateLimiter
s.executeTask(req)
}()
}
}
}
// executeTask 执行任务(本地执行)
func (s *Scheduler) executeTask(req *ExecutionRequest) (*ExecutionResult, error) {
defer func() {
if r := recover(); r != nil {
s.logger.Errorf("[Scheduler] 任务 %s 执行过程中发生 Panic: %v", req.TaskID, r)
}
}()
start := time.Now()
s.logger.Infof("[Scheduler] 执行任务 %s (名称: %s, 类型: %s)", req.TaskID, req.Name, req.Type)
+44 -37
View File
@@ -271,48 +271,55 @@ func (m *AgentWSManager) cleanupLoop() {
NewAgentService().ResetAllAgentsToOffline()
for range ticker.C {
m.mu.Lock()
now := time.Now()
func() {
defer func() {
if r := recover(); r != nil {
logger.Errorf("[AgentWS] cleanupLoop panic: %v", r)
}
}()
m.mu.Lock()
now := time.Now()
// 清理超时连接
for agentID, conn := range m.connections {
if now.Sub(conn.LastPing) > 2*time.Minute {
// 减少 IP 连接计数
if conn.IP != "" {
if count, ok := m.ipConnections[conn.IP]; ok && count > 0 {
m.ipConnections[conn.IP] = count - 1
// 清理超时连接
for agentID, conn := range m.connections {
if now.Sub(conn.LastPing) > 2*time.Minute {
// 减少 IP 连接计数
if conn.IP != "" {
if count, ok := m.ipConnections[conn.IP]; ok && count > 0 {
m.ipConnections[conn.IP] = count - 1
}
}
conn.Close()
delete(m.connections, agentID)
// 更新数据库状态
database.DB.Model(&models.Agent{}).Where("id = ?", agentID).Update("status", "offline")
logger.Infof("[AgentWS] Agent #%d 心跳超时,已断开", agentID)
}
}
// 定期清理数据库中的过期状态(处理服务重启或异常终止的情况)
// 有些 Agent 虽然没有连接,但数据库状态可能是 "online"
cutoff := now.Add(-2 * time.Minute)
database.DB.Model(&models.Agent{}).
Where("status = ? AND last_seen < ?", "online", cutoff).
Update("status", "offline")
// 清理过期的限流记录(超过 10 分钟未活动)
// 清理过期的限流记录(超过 10 分钟未活动)
for ip, lastAttempt := range m.ipLastAttempt {
if now.Sub(lastAttempt) > 10*time.Minute {
delete(m.ipLastAttempt, ip)
delete(m.ipFailCount, ip)
// 只清理没有活跃连接的 IP 计数
if m.ipConnections[ip] == 0 {
delete(m.ipConnections, ip)
}
}
conn.Close()
delete(m.connections, agentID)
// 更新数据库状态
database.DB.Model(&models.Agent{}).Where("id = ?", agentID).Update("status", "offline")
logger.Infof("[AgentWS] Agent #%d 心跳超时,已断开", agentID)
}
}
// 定期清理数据库中的过期状态(处理服务重启或异常终止的情况)
// 有些 Agent 虽然没有连接,但数据库状态可能是 "online"
cutoff := now.Add(-2 * time.Minute)
database.DB.Model(&models.Agent{}).
Where("status = ? AND last_seen < ?", "online", cutoff).
Update("status", "offline")
// 清理过期的限流记录(超过 10 分钟未活动)
// 清理过期的限流记录(超过 10 分钟未活动)
for ip, lastAttempt := range m.ipLastAttempt {
if now.Sub(lastAttempt) > 10*time.Minute {
delete(m.ipLastAttempt, ip)
delete(m.ipFailCount, ip)
// 只清理没有活跃连接的 IP 计数
if m.ipConnections[ip] == 0 {
delete(m.ipConnections, ip)
}
}
}
m.mu.Unlock()
m.mu.Unlock()
}()
}
}
@@ -161,6 +161,12 @@ func (h *ServerSchedulerHandler) OnTaskHeartbeat(req *executor.ExecutionRequest,
if req.LogID > 0 {
h.es.taskLogService.UpdateTaskDuration(req.LogID, duration)
}
// 每分钟打印一次任务还在运行的日志
if duration >= 60000 && (duration/60000 > (duration-3000)/60000) {
logger.Infof("[Scheduler] 任务 #%s 仍在运行中... (已耗时: %v)",
req.TaskID, (time.Duration(duration) * time.Millisecond).Round(time.Second))
}
}
func (h *ServerSchedulerHandler) OnTaskStarted(req *executor.ExecutionRequest) {