fix: remote agent exec errors
This commit is contained in:
@@ -439,6 +439,9 @@ func (c *AgentController) WSConnect(ctx *gin.Context) {
|
||||
// 启动读写协程
|
||||
go c.wsWritePump(ac)
|
||||
go c.wsReadPump(ac, agent)
|
||||
|
||||
// 主动推送任务列表
|
||||
go c.wsManager.BroadcastTasks(agent.ID)
|
||||
}
|
||||
|
||||
// wsReadPump 读取消息
|
||||
|
||||
@@ -77,6 +77,11 @@ func (m *CronManager) AddTask(task CronTask) error {
|
||||
timeout := task.GetTimeout()
|
||||
|
||||
entryID, err := m.cron.AddFunc(task.GetSchedule(), func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
m.logger.Errorf("[CronManager] 任务 #%s 执行过程中发生 Panic: %v", taskID, r)
|
||||
}
|
||||
}()
|
||||
m.logger.Infof("[CronManager] 触发计划任务 #%s (%s)", taskID, name)
|
||||
|
||||
req := &ExecutionRequest{
|
||||
|
||||
@@ -262,15 +262,27 @@ func (s *Scheduler) worker(id int) {
|
||||
case <-s.stopCh:
|
||||
return
|
||||
case req := <-s.taskQueue:
|
||||
// 速率限制
|
||||
<-s.rateLimiter
|
||||
s.executeTask(req)
|
||||
func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
s.logger.Errorf("[Scheduler] Worker %d panic while processing task %s: %v", id, req.TaskID, r)
|
||||
}
|
||||
}()
|
||||
// 速率限制
|
||||
<-s.rateLimiter
|
||||
s.executeTask(req)
|
||||
}()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// executeTask 执行任务(本地执行)
|
||||
func (s *Scheduler) executeTask(req *ExecutionRequest) (*ExecutionResult, error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
s.logger.Errorf("[Scheduler] 任务 %s 执行过程中发生 Panic: %v", req.TaskID, r)
|
||||
}
|
||||
}()
|
||||
start := time.Now()
|
||||
|
||||
s.logger.Infof("[Scheduler] 执行任务 %s (名称: %s, 类型: %s)", req.TaskID, req.Name, req.Type)
|
||||
|
||||
@@ -271,48 +271,55 @@ func (m *AgentWSManager) cleanupLoop() {
|
||||
NewAgentService().ResetAllAgentsToOffline()
|
||||
|
||||
for range ticker.C {
|
||||
m.mu.Lock()
|
||||
now := time.Now()
|
||||
func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
logger.Errorf("[AgentWS] cleanupLoop panic: %v", r)
|
||||
}
|
||||
}()
|
||||
m.mu.Lock()
|
||||
now := time.Now()
|
||||
|
||||
// 清理超时连接
|
||||
for agentID, conn := range m.connections {
|
||||
if now.Sub(conn.LastPing) > 2*time.Minute {
|
||||
// 减少 IP 连接计数
|
||||
if conn.IP != "" {
|
||||
if count, ok := m.ipConnections[conn.IP]; ok && count > 0 {
|
||||
m.ipConnections[conn.IP] = count - 1
|
||||
// 清理超时连接
|
||||
for agentID, conn := range m.connections {
|
||||
if now.Sub(conn.LastPing) > 2*time.Minute {
|
||||
// 减少 IP 连接计数
|
||||
if conn.IP != "" {
|
||||
if count, ok := m.ipConnections[conn.IP]; ok && count > 0 {
|
||||
m.ipConnections[conn.IP] = count - 1
|
||||
}
|
||||
}
|
||||
conn.Close()
|
||||
delete(m.connections, agentID)
|
||||
// 更新数据库状态
|
||||
database.DB.Model(&models.Agent{}).Where("id = ?", agentID).Update("status", "offline")
|
||||
logger.Infof("[AgentWS] Agent #%d 心跳超时,已断开", agentID)
|
||||
}
|
||||
}
|
||||
|
||||
// 定期清理数据库中的过期状态(处理服务重启或异常终止的情况)
|
||||
// 有些 Agent 虽然没有连接,但数据库状态可能是 "online"
|
||||
cutoff := now.Add(-2 * time.Minute)
|
||||
database.DB.Model(&models.Agent{}).
|
||||
Where("status = ? AND last_seen < ?", "online", cutoff).
|
||||
Update("status", "offline")
|
||||
|
||||
// 清理过期的限流记录(超过 10 分钟未活动)
|
||||
|
||||
// 清理过期的限流记录(超过 10 分钟未活动)
|
||||
for ip, lastAttempt := range m.ipLastAttempt {
|
||||
if now.Sub(lastAttempt) > 10*time.Minute {
|
||||
delete(m.ipLastAttempt, ip)
|
||||
delete(m.ipFailCount, ip)
|
||||
// 只清理没有活跃连接的 IP 计数
|
||||
if m.ipConnections[ip] == 0 {
|
||||
delete(m.ipConnections, ip)
|
||||
}
|
||||
}
|
||||
conn.Close()
|
||||
delete(m.connections, agentID)
|
||||
// 更新数据库状态
|
||||
database.DB.Model(&models.Agent{}).Where("id = ?", agentID).Update("status", "offline")
|
||||
logger.Infof("[AgentWS] Agent #%d 心跳超时,已断开", agentID)
|
||||
}
|
||||
}
|
||||
|
||||
// 定期清理数据库中的过期状态(处理服务重启或异常终止的情况)
|
||||
// 有些 Agent 虽然没有连接,但数据库状态可能是 "online"
|
||||
cutoff := now.Add(-2 * time.Minute)
|
||||
database.DB.Model(&models.Agent{}).
|
||||
Where("status = ? AND last_seen < ?", "online", cutoff).
|
||||
Update("status", "offline")
|
||||
|
||||
// 清理过期的限流记录(超过 10 分钟未活动)
|
||||
|
||||
// 清理过期的限流记录(超过 10 分钟未活动)
|
||||
for ip, lastAttempt := range m.ipLastAttempt {
|
||||
if now.Sub(lastAttempt) > 10*time.Minute {
|
||||
delete(m.ipLastAttempt, ip)
|
||||
delete(m.ipFailCount, ip)
|
||||
// 只清理没有活跃连接的 IP 计数
|
||||
if m.ipConnections[ip] == 0 {
|
||||
delete(m.ipConnections, ip)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
m.mu.Unlock()
|
||||
m.mu.Unlock()
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -161,6 +161,12 @@ func (h *ServerSchedulerHandler) OnTaskHeartbeat(req *executor.ExecutionRequest,
|
||||
if req.LogID > 0 {
|
||||
h.es.taskLogService.UpdateTaskDuration(req.LogID, duration)
|
||||
}
|
||||
|
||||
// 每分钟打印一次任务还在运行的日志
|
||||
if duration >= 60000 && (duration/60000 > (duration-3000)/60000) {
|
||||
logger.Infof("[Scheduler] 任务 #%s 仍在运行中... (已耗时: %v)",
|
||||
req.TaskID, (time.Duration(duration) * time.Millisecond).Round(time.Second))
|
||||
}
|
||||
}
|
||||
|
||||
func (h *ServerSchedulerHandler) OnTaskStarted(req *executor.ExecutionRequest) {
|
||||
|
||||
Reference in New Issue
Block a user