feat: add scheduler config

This commit is contained in:
engigu
2025-12-22 16:05:07 +08:00
parent 03f4fa7b5c
commit be3527cb61
12 changed files with 387 additions and 52 deletions
+19
View File
@@ -213,6 +213,25 @@ log_retention_days = 30
| 分页大小 | 列表每页显示条数 | 10 |
| Cookie 有效期 | 登录状态保持天数 | 7 |
### 调度设置
以下设置可在管理面板「系统设置 > 调度设置」中配置,用于优化任务执行性能:
| 设置项 | 说明 | 默认值 |
|--------|------|--------|
| Worker 数量 | 并发执行任务的 worker 数量 | 4 |
| 队列大小 | 任务队列缓冲区大小 | 100 |
| 速率间隔 | 任务启动间隔(毫秒),200ms = 每秒最多 5 个任务 | 200 |
**调度机制说明:**
系统采用 Worker Pool + 任务队列的架构来控制任务执行:
1. **Worker Pool**:固定数量的 worker 从队列中取任务执行,避免 cron 触发时无限并发
2. **任务队列**:cron 调度器触发的任务先入队,由 worker 按顺序执行
3. **速率限制**:控制任务启动频率,防止秒级 cron 同时触发多个进程导致 CPU 峰值
4. **热重载**:修改调度设置后立即生效,无需重启服务
---
## 🛠️ 技术栈
+13 -2
View File
@@ -30,8 +30,9 @@ const (
DefaultTaskTimeout = 30
// Settings Section 常量
SectionSite = "site"
SectionSystem = "system"
SectionSite = "site"
SectionSystem = "system"
SectionScheduler = "scheduler"
// Site Settings Key 常量
KeyTitle = "title"
@@ -42,6 +43,11 @@ const (
// System Settings Key 常量
KeyInitialized = "initialized"
// Scheduler Settings Key 常量
KeyWorkerCount = "worker_count"
KeyQueueSize = "queue_size"
KeyRateInterval = "rate_interval"
)
// TablePrefix 表前缀,从配置文件读取
@@ -62,4 +68,9 @@ var DefaultSettings = map[string]map[string]string{
KeyPageSize: "10",
KeyCookieDays: "7",
},
SectionScheduler: {
KeyWorkerCount: "4",
KeyQueueSize: "100",
KeyRateInterval: "200",
},
}
+41 -1
View File
@@ -22,14 +22,16 @@ type SettingsController struct {
settingsService *services.SettingsService
loginLogService *services.LoginLogService
backupService *services.BackupService
executorService *services.ExecutorService
}
func NewSettingsController(userService *services.UserService, loginLogService *services.LoginLogService) *SettingsController {
func NewSettingsController(userService *services.UserService, loginLogService *services.LoginLogService, executorService *services.ExecutorService) *SettingsController {
return &SettingsController{
userService: userService,
settingsService: services.NewSettingsService(),
loginLogService: loginLogService,
backupService: services.NewBackupService(),
executorService: executorService,
}
}
@@ -115,6 +117,44 @@ func (sc *SettingsController) UpdateSiteSettings(c *gin.Context) {
utils.SuccessMsg(c, "保存成功")
}
// GetSchedulerSettings 获取调度设置
func (sc *SettingsController) GetSchedulerSettings(c *gin.Context) {
settings := sc.settingsService.GetSection(constant.SectionScheduler)
utils.Success(c, settings)
}
// UpdateSchedulerSettings 更新调度设置
func (sc *SettingsController) UpdateSchedulerSettings(c *gin.Context) {
var req struct {
WorkerCount string `json:"worker_count"`
QueueSize string `json:"queue_size"`
RateInterval string `json:"rate_interval"`
}
if err := c.ShouldBindJSON(&req); err != nil {
utils.BadRequest(c, "参数错误")
return
}
values := map[string]string{
constant.KeyWorkerCount: req.WorkerCount,
constant.KeyQueueSize: req.QueueSize,
constant.KeyRateInterval: req.RateInterval,
}
if err := sc.settingsService.SetSection(constant.SectionScheduler, values); err != nil {
utils.ServerError(c, "保存失败")
return
}
// 重新加载 executor service
if sc.executorService != nil {
sc.executorService.Reload()
}
utils.SuccessMsg(c, "保存成功")
}
// GetAbout 获取关于信息
func (sc *SettingsController) GetAbout(c *gin.Context) {
var taskCount, logCount, envCount int64
+1 -1
View File
@@ -37,7 +37,7 @@ func RegisterControllers() *Controllers {
Dashboard: controllers.NewDashboardController(cronService, executorService),
Log: controllers.NewLogController(),
Terminal: controllers.NewTerminalController(),
Settings: controllers.NewSettingsController(userService, loginLogService),
Settings: controllers.NewSettingsController(userService, loginLogService, executorService),
Runtime: controllers.NewRuntimeController(),
}
}
+2
View File
@@ -175,6 +175,8 @@ func Setup(c *Controllers) *gin.Engine {
settings.POST("/password", c.Settings.ChangePassword)
settings.GET("/site", c.Settings.GetSiteSettings)
settings.PUT("/site", c.Settings.UpdateSiteSettings)
settings.GET("/scheduler", c.Settings.GetSchedulerSettings)
settings.PUT("/scheduler", c.Settings.UpdateSchedulerSettings)
settings.GET("/about", c.Settings.GetAbout)
settings.GET("/loginlogs", c.Settings.GetLoginLogs)
settings.POST("/backup", c.Settings.CreateBackup)
+2 -2
View File
@@ -110,8 +110,8 @@ func (cs *CronService) runTask(taskID uint) {
now := time.Now()
database.DB.Model(&models.Task{}).Where("id = ?", taskID).Update("last_run", now)
// 执行任务
cs.executorService.ExecuteTask(int(taskID))
// 将任务加入队列执行(通过 worker pool 控制并发)
cs.executorService.EnqueueTask(int(taskID))
// 更新 next_run
cs.updateNextRun(taskID)
+143 -26
View File
@@ -1,19 +1,19 @@
package services
import (
"bytes"
"context"
"encoding/json"
"os"
"os/exec"
"sync"
"time"
"baihu/internal/constant"
"baihu/internal/database"
"baihu/internal/logger"
"baihu/internal/models"
"baihu/internal/utils"
"bytes"
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"sync"
"time"
)
// ExecutionResult represents the result of a task execution
@@ -29,6 +29,11 @@ type ExecutionResult struct {
// ExecutionCallback 任务执行完成后的回调函数类型
type ExecutionCallback func(taskID uint, command string, result *ExecutionResult)
// taskJob 任务队列项
type taskJob struct {
taskID int
}
// ExecutorService handles task execution
type ExecutorService struct {
taskService *TaskService
@@ -36,23 +41,119 @@ type ExecutorService struct {
runningTasks map[int]bool
callbacks []ExecutionCallback
mu sync.RWMutex
resultsMu sync.RWMutex
// 任务队列和 worker pool
taskQueue chan taskJob
workerCount int
rateLimiter <-chan time.Time
stopCh chan struct{}
wg sync.WaitGroup
}
// NewExecutorService creates a new executor service
func NewExecutorService(taskService *TaskService) *ExecutorService {
// 从设置中读取调度配置
settingsService := NewSettingsService()
workerCount := getIntSetting(settingsService, constant.SectionScheduler, constant.KeyWorkerCount, 4)
queueSize := getIntSetting(settingsService, constant.SectionScheduler, constant.KeyQueueSize, 100)
rateInterval := getIntSetting(settingsService, constant.SectionScheduler, constant.KeyRateInterval, 200)
logger.Infof("Executor service config: workers=%d, queue=%d, rate=%dms", workerCount, queueSize, rateInterval)
es := &ExecutorService{
taskService: taskService,
results: make([]ExecutionResult, 0),
results: make([]ExecutionResult, 0, 100),
runningTasks: make(map[int]bool),
callbacks: make([]ExecutionCallback, 0),
taskQueue: make(chan taskJob, queueSize),
workerCount: workerCount,
rateLimiter: time.Tick(time.Duration(rateInterval) * time.Millisecond),
stopCh: make(chan struct{}),
}
// 注册默认回调
es.RegisterCallback(es.saveTaskLogCallback)
es.RegisterCallback(es.updateStatsCallback)
es.RegisterCallback(es.cleanLogsCallback)
// 启动 worker pool
es.startWorkers()
return es
}
// getIntSetting 从设置中获取整数值
func getIntSetting(s *SettingsService, section, key string, defaultVal int) int {
val := s.Get(section, key)
if val == "" {
return defaultVal
}
var result int
if _, err := fmt.Sscanf(val, "%d", &result); err != nil {
return defaultVal
}
return result
}
// startWorkers 启动 worker pool
func (es *ExecutorService) startWorkers() {
for i := 0; i < es.workerCount; i++ {
es.wg.Add(1)
go es.worker(i)
}
}
// worker 从队列中取任务执行
func (es *ExecutorService) worker(id int) {
defer es.wg.Done()
for {
select {
case <-es.stopCh:
return
case job := <-es.taskQueue:
// 速率限制
<-es.rateLimiter
es.executeTaskInternal(job.taskID)
}
}
}
// Stop 停止 executor service
func (es *ExecutorService) Stop() {
close(es.stopCh)
es.wg.Wait()
}
// Reload 重新加载配置并重建 worker pool
func (es *ExecutorService) Reload() {
logger.Info("Reloading executor service...")
// 停止现有 workers
close(es.stopCh)
es.wg.Wait()
logger.Info("Stopped executor service...")
// 从设置中读取新配置
settingsService := NewSettingsService()
workerCount := getIntSetting(settingsService, constant.SectionScheduler, constant.KeyWorkerCount, 4)
queueSize := getIntSetting(settingsService, constant.SectionScheduler, constant.KeyQueueSize, 100)
rateInterval := getIntSetting(settingsService, constant.SectionScheduler, constant.KeyRateInterval, 200)
// 重建 channel 和配置
es.mu.Lock()
es.taskQueue = make(chan taskJob, queueSize)
es.workerCount = workerCount
es.rateLimiter = time.Tick(time.Duration(rateInterval) * time.Millisecond)
es.stopCh = make(chan struct{})
es.mu.Unlock()
// 启动新的 workers
es.startWorkers()
logger.Infof("Executor service reloaded: workers=%d, queue=%d, rate=%dms", workerCount, queueSize, rateInterval)
}
// RegisterCallback 注册执行完成回调
func (es *ExecutorService) RegisterCallback(cb ExecutionCallback) {
es.mu.Lock()
@@ -60,19 +161,21 @@ func (es *ExecutorService) RegisterCallback(cb ExecutionCallback) {
es.mu.Unlock()
}
// executeCallbacks 执行所有回调
func (es *ExecutorService) executeCallbacks(taskID uint, command string, result *ExecutionResult) {
// executeCallbacksAsync 异步执行所有回调
func (es *ExecutorService) executeCallbacksAsync(taskID uint, command string, result *ExecutionResult) {
es.mu.RLock()
callbacks := make([]ExecutionCallback, len(es.callbacks))
copy(callbacks, es.callbacks)
es.mu.RUnlock()
for _, cb := range callbacks {
cb(taskID, command, result)
}
go func() {
for _, cb := range callbacks {
cb(taskID, command, result)
}
}()
}
// saveTaskLogCallback 保存任务日志的回调
// saveTaskLogCallback 保存任务日志的回调(异步执行,压缩在此处进行)
func (es *ExecutorService) saveTaskLogCallback(taskID uint, command string, result *ExecutionResult) {
output := result.Output
if result.Error != "" {
@@ -141,17 +244,13 @@ func (es *ExecutorService) cleanLogsCallback(taskID uint, _ string, _ *Execution
var deleted int64
switch config.Type {
case "day":
// 按天清理:删除 N 天前的日志
cutoff := time.Now().AddDate(0, 0, -config.Keep)
result := database.DB.Where("task_id = ? AND created_at < ?", taskID, cutoff).Delete(&models.TaskLog{})
deleted = result.RowsAffected
case "count":
// 按条数清理:使用子查询删除超出保留数量的旧日志
// 先获取第 N 条的 ID 作为边界
var boundaryLog models.TaskLog
err := database.DB.Where("task_id = ?", taskID).Order("id DESC").Offset(config.Keep - 1).Limit(1).First(&boundaryLog).Error
if err == nil {
// 删除 ID 小于边界的所有日志
result := database.DB.Where("task_id = ? AND id < ?", taskID, boundaryLog.ID).Delete(&models.TaskLog{})
deleted = result.RowsAffected
}
@@ -162,8 +261,25 @@ func (es *ExecutorService) cleanLogsCallback(taskID uint, _ string, _ *Execution
}
}
// ExecuteTask executes a task by ID
// EnqueueTask 将任务加入队列(供 cron 调度器调用)
func (es *ExecutorService) EnqueueTask(taskID int) {
select {
case es.taskQueue <- taskJob{taskID: taskID}:
// 成功入队
default:
// 队列满,直接执行(降级处理)
logger.Warnf("Task queue full, executing task %d directly", taskID)
go es.executeTaskInternal(taskID)
}
}
// ExecuteTask executes a task by ID(同步执行,供 API 调用)
func (es *ExecutorService) ExecuteTask(taskID int) *ExecutionResult {
return es.executeTaskInternal(taskID)
}
// executeTaskInternal 内部执行任务逻辑
func (es *ExecutorService) executeTaskInternal(taskID int) *ExecutionResult {
task := es.taskService.GetTaskByID(taskID)
if task == nil {
return &ExecutionResult{
@@ -197,8 +313,8 @@ func (es *ExecutorService) ExecuteTask(taskID int) *ExecutionResult {
delete(es.runningTasks, taskID)
es.mu.Unlock()
// 执行回调
es.executeCallbacks(uint(taskID), task.Command, result)
// 异步执行回调(日志压缩、统计更新、日志清理)
es.executeCallbacksAsync(uint(taskID), task.Command, result)
return result
}
@@ -255,20 +371,21 @@ func (es *ExecutorService) ExecuteCommandWithEnv(command string, timeout time.Du
result.Success = true
}
es.mu.Lock()
// 使用独立锁保存结果
es.resultsMu.Lock()
es.results = append(es.results, *result)
if len(es.results) > 100 {
es.results = es.results[1:]
}
es.mu.Unlock()
es.resultsMu.Unlock()
return result
}
// GetLastResults returns the last execution results
func (es *ExecutorService) GetLastResults(count int) []ExecutionResult {
es.mu.RLock()
defer es.mu.RUnlock()
es.resultsMu.RLock()
defer es.resultsMu.RUnlock()
start := 0
if len(es.results) > count {
+9
View File
@@ -114,6 +114,9 @@ export const api = {
getPublicSite: () => request<{ title: string; subtitle: string; icon: string }>('/settings/public'),
updateSite: (data: SiteSettings) =>
request('/settings/site', { method: 'PUT', body: JSON.stringify(data) }),
getScheduler: () => request<SchedulerSettings>('/settings/scheduler'),
updateScheduler: (data: SchedulerSettings) =>
request('/settings/scheduler', { method: 'PUT', body: JSON.stringify(data) }),
getAbout: () => request<AboutInfo>('/settings/about'),
getLoginLogs: (params?: { page?: number; page_size?: number; username?: string }) => {
const query = new URLSearchParams()
@@ -307,6 +310,12 @@ export interface SiteSettings {
cookie_days: string
}
export interface SchedulerSettings {
worker_count: string
queue_size: string
rate_interval: string
}
export interface LoginLog {
id: number
+21
View File
@@ -143,3 +143,24 @@
line-height: 1.5;
}
}
/* Number input spinner buttons - dark mode */
input[type="number"]::-webkit-inner-spin-button,
input[type="number"]::-webkit-outer-spin-button {
opacity: 0.6;
}
input[type="number"]:hover::-webkit-inner-spin-button,
input[type="number"]:hover::-webkit-outer-spin-button {
opacity: 1;
}
.dark input[type="number"]::-webkit-inner-spin-button,
.dark input[type="number"]::-webkit-outer-spin-button {
filter: invert(0.7);
}
.dark input[type="number"]:hover::-webkit-inner-spin-button,
.dark input[type="number"]:hover::-webkit-outer-spin-button {
filter: invert(0.85);
}
+14 -20
View File
@@ -21,7 +21,6 @@ const backupLoading = ref(false)
const restoreLoading = ref(false)
const fileInput = ref<HTMLInputElement>()
const showConfirm = ref(false)
const pendingFile = ref<File | null>(null)
async function checkBackupStatus() {
try {
@@ -49,11 +48,20 @@ function downloadBackup() {
setTimeout(checkBackupStatus, 6000)
}
function triggerFileSelect() {
function showRestoreConfirm() {
showConfirm.value = true
}
function confirmRestore() {
showConfirm.value = false
fileInput.value?.click()
}
function handleFileSelect(e: Event) {
function cancelRestore() {
showConfirm.value = false
}
async function handleFileSelect(e: Event) {
const target = e.target as HTMLInputElement
const file = target.files?.[0]
if (!file) return
@@ -64,33 +72,19 @@ function handleFileSelect(e: Event) {
return
}
pendingFile.value = file
showConfirm.value = true
target.value = ''
}
async function confirmRestore() {
if (!pendingFile.value) return
showConfirm.value = false
restoreLoading.value = true
try {
await api.settings.restoreBackup(pendingFile.value)
await api.settings.restoreBackup(file)
toast.success('恢复成功,页面即将刷新')
setTimeout(() => window.location.reload(), 1500)
} catch (e: any) {
toast.error(e.message || '恢复失败')
} finally {
restoreLoading.value = false
pendingFile.value = null
target.value = ''
}
}
function cancelRestore() {
showConfirm.value = false
pendingFile.value = null
}
onMounted(checkBackupStatus)
</script>
@@ -112,7 +106,7 @@ onMounted(checkBackupStatus)
</div>
<div class="border-t pt-4 mt-4">
<div class="flex items-center gap-4">
<Button @click="triggerFileSelect" :disabled="restoreLoading" variant="outline">
<Button @click="showRestoreConfirm" :disabled="restoreLoading" variant="outline">
<Upload class="w-4 h-4 mr-2" />
{{ restoreLoading ? '恢复中...' : '恢复备份' }}
</Button>
@@ -0,0 +1,108 @@
<script setup lang="ts">
import { ref, onMounted } from 'vue'
import { Input } from '@/components/ui/input'
import { Label } from '@/components/ui/label'
import { Button } from '@/components/ui/button'
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle
} from '@/components/ui/alert-dialog'
import { api } from '@/api'
import { toast } from 'vue-sonner'
interface SchedulerSettings {
worker_count: string
queue_size: string
rate_interval: string
}
const form = ref<SchedulerSettings>({
worker_count: '4',
queue_size: '100',
rate_interval: '200'
})
const loading = ref(false)
const showConfirm = ref(false)
async function loadSettings() {
try {
const res = await api.settings.getScheduler()
form.value = res
} catch {}
}
function confirmSave() {
showConfirm.value = true
}
async function saveSettings() {
showConfirm.value = false
loading.value = true
try {
await api.settings.updateScheduler({
worker_count: String(form.value.worker_count),
queue_size: String(form.value.queue_size),
rate_interval: String(form.value.rate_interval)
})
toast.success('保存成功,调度配置已重新加载')
} catch {
toast.error('保存失败')
} finally {
loading.value = false
}
}
onMounted(loadSettings)
</script>
<template>
<div class="space-y-4">
<div class="grid grid-cols-4 items-center gap-4">
<Label class="text-right">Worker 数量</Label>
<div class="col-span-3 flex items-center gap-2">
<Input v-model="form.worker_count" type="number" class="w-24" />
<span class="text-xs text-muted-foreground">并发执行任务的 worker 数量</span>
</div>
</div>
<div class="grid grid-cols-4 items-center gap-4">
<Label class="text-right">队列大小</Label>
<div class="col-span-3 flex items-center gap-2">
<Input v-model="form.queue_size" type="number" class="w-24" />
<span class="text-xs text-muted-foreground">任务队列缓冲区大小</span>
</div>
</div>
<div class="grid grid-cols-4 items-center gap-4">
<Label class="text-right">速率间隔</Label>
<div class="col-span-3 flex items-center gap-2">
<Input v-model="form.rate_interval" type="number" class="w-24" />
<span class="text-xs text-muted-foreground">ms任务启动间隔200ms = 每秒最多5个</span>
</div>
</div>
<div class="flex justify-end pt-2">
<Button @click="confirmSave" :disabled="loading">
{{ loading ? '保存中...' : '保存设置' }}
</Button>
</div>
<AlertDialog :open="showConfirm" @update:open="showConfirm = $event">
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>确认保存</AlertDialogTitle>
<AlertDialogDescription>
保存后调度配置将立即生效正在执行的任务不受影响确定要保存吗
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>取消</AlertDialogCancel>
<AlertDialogAction @click="saveSettings">确认保存</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</div>
</template>
+14
View File
@@ -4,6 +4,7 @@ import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/com
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'
import PasswordSettings from './PasswordSettings.vue'
import SiteSettings from './SiteSettings.vue'
import SchedulerSettings from './SchedulerSettings.vue'
import BackupSettings from './BackupSettings.vue'
import AboutSettings from './AboutSettings.vue'
@@ -21,6 +22,7 @@ const activeTab = ref('password')
<TabsList>
<TabsTrigger value="password">密码修改</TabsTrigger>
<TabsTrigger value="site">站点设置</TabsTrigger>
<TabsTrigger value="scheduler">调度设置</TabsTrigger>
<TabsTrigger value="backup">备份恢复</TabsTrigger>
<TabsTrigger value="about">关于</TabsTrigger>
</TabsList>
@@ -49,6 +51,18 @@ const activeTab = ref('password')
</Card>
</TabsContent>
<TabsContent value="scheduler" class="mt-6">
<Card class="max-w-xl">
<CardHeader>
<CardTitle>调度设置</CardTitle>
<CardDescription>配置任务调度器参数</CardDescription>
</CardHeader>
<CardContent>
<SchedulerSettings />
</CardContent>
</Card>
</TabsContent>
<TabsContent value="backup" class="mt-6">
<Card class="max-w-xl">
<CardHeader>