mirror of
https://github.com/MengMengCode/CLICD.git
synced 2026-08-07 22:24:42 +08:00
@@ -309,6 +309,10 @@ func createContainer(w http.ResponseWriter, r *http.Request) {
|
||||
jsonResponse(w, http.StatusBadRequest, APIResponse{Success: false, Message: err.Error()})
|
||||
return
|
||||
}
|
||||
if err := validateCreateStoragePool(&cfg); err != nil {
|
||||
jsonResponse(w, http.StatusConflict, APIResponse{Success: false, Message: err.Error()})
|
||||
return
|
||||
}
|
||||
if err := validateCreateSSHAuth(cfg); err != nil {
|
||||
jsonResponse(w, http.StatusBadRequest, APIResponse{Success: false, Message: err.Error()})
|
||||
return
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -41,6 +42,9 @@ type ImageInfo struct {
|
||||
|
||||
var imageDownloadsMu sync.Mutex
|
||||
var imageDownloads = map[string]*imageDownloadStatus{}
|
||||
var lxcImageCacheMu sync.Mutex
|
||||
var lxcImageDownloadMu sync.Mutex
|
||||
var lxcImageDownloadActive bool
|
||||
|
||||
type imageDownloadStatus struct {
|
||||
Downloading bool
|
||||
@@ -136,6 +140,22 @@ func isImageDownloadActive(id string) bool {
|
||||
return st != nil && st.Downloading
|
||||
}
|
||||
|
||||
func beginLXCImageDownload() bool {
|
||||
lxcImageDownloadMu.Lock()
|
||||
defer lxcImageDownloadMu.Unlock()
|
||||
if lxcImageDownloadActive {
|
||||
return false
|
||||
}
|
||||
lxcImageDownloadActive = true
|
||||
return true
|
||||
}
|
||||
|
||||
func endLXCImageDownload() {
|
||||
lxcImageDownloadMu.Lock()
|
||||
lxcImageDownloadActive = false
|
||||
lxcImageDownloadMu.Unlock()
|
||||
}
|
||||
|
||||
func lxcImageDownloadTempName(id string) string {
|
||||
return fmt.Sprintf("clicd-img-dl-%s", id)
|
||||
}
|
||||
@@ -307,7 +327,6 @@ func HandleImageDownload(w http.ResponseWriter, r *http.Request) {
|
||||
jsonResponse(w, http.StatusBadRequest, APIResponse{Success: false, Message: "template_id required"})
|
||||
return
|
||||
}
|
||||
|
||||
tmpl := lxc.FindTemplate(req.TemplateID)
|
||||
if tmpl == nil {
|
||||
image := kvm.FindImage(req.TemplateID)
|
||||
@@ -319,6 +338,10 @@ func HandleImageDownload(w http.ResponseWriter, r *http.Request) {
|
||||
jsonResponse(w, http.StatusForbidden, APIResponse{Success: false, Message: "KVM is not available on this host"})
|
||||
return
|
||||
}
|
||||
if _, err := config.SelectStoragePoolForContent(config.StorageContentImages, "", 1024*1024*1024); err != nil {
|
||||
jsonResponse(w, http.StatusConflict, APIResponse{Success: false, Message: err.Error()})
|
||||
return
|
||||
}
|
||||
if ok, _ := kvm.ImageDownloadedInfo(image.ID); ok {
|
||||
ensureImageEnabled(image.ID)
|
||||
clearImageDownload(image.ID)
|
||||
@@ -359,6 +382,29 @@ func HandleImageDownload(w http.ResponseWriter, r *http.Request) {
|
||||
jsonResponse(w, http.StatusAccepted, APIResponse{Success: true, Message: "Download started"})
|
||||
return
|
||||
}
|
||||
if !beginLXCImageDownload() {
|
||||
jsonResponse(w, http.StatusConflict, APIResponse{Success: false, Message: "Another LXC image download is active"})
|
||||
return
|
||||
}
|
||||
lxcDownloadHandedOff := false
|
||||
defer func() {
|
||||
if !lxcDownloadHandedOff {
|
||||
endLXCImageDownload()
|
||||
}
|
||||
}()
|
||||
imagePool, err := config.SelectStoragePoolForContent(
|
||||
config.StorageContentImages,
|
||||
"",
|
||||
dirSizeBytes("/var/cache/lxc/download")+1024*1024*1024,
|
||||
)
|
||||
if err != nil {
|
||||
jsonResponse(w, http.StatusConflict, APIResponse{Success: false, Message: err.Error()})
|
||||
return
|
||||
}
|
||||
if err := ensureLXCImageCachePool(*imagePool); err != nil {
|
||||
jsonResponse(w, http.StatusInternalServerError, APIResponse{Success: false, Message: err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
// Already downloaded? Just enable if needed.
|
||||
if isImageDownloaded(tmpl.Distro, tmpl.Release, tmpl.Arch) {
|
||||
@@ -375,6 +421,7 @@ func HandleImageDownload(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
go func(tmpl lxc.Template) {
|
||||
defer endLXCImageDownload()
|
||||
// Download via lxc-create with a temp container, then destroy it.
|
||||
tmpName := lxcImageDownloadTempName(tmpl.ID)
|
||||
args := []string{"-n", tmpName, "-t", "download", "--",
|
||||
@@ -386,7 +433,7 @@ func HandleImageDownload(w http.ResponseWriter, r *http.Request) {
|
||||
st.Stage = "lxc-create"
|
||||
})
|
||||
cmd := exec.CommandContext(ctx, "lxc-create", args...)
|
||||
output, err := cmd.CombinedOutput()
|
||||
output, err := runLXCImageDownloadCommand(cmd, tmpl.ID)
|
||||
|
||||
// Clean up the temp container unconditionally.
|
||||
cleanupLXCImageDownloadTemp(tmpl.ID)
|
||||
@@ -403,10 +450,154 @@ func HandleImageDownload(w http.ResponseWriter, r *http.Request) {
|
||||
ensureImageEnabled(tmpl.ID)
|
||||
finishImageDownload(tmpl.ID, nil)
|
||||
}(*tmpl)
|
||||
lxcDownloadHandedOff = true
|
||||
|
||||
jsonResponse(w, http.StatusAccepted, APIResponse{Success: true, Message: "Download started"})
|
||||
}
|
||||
|
||||
type lxcImageDownloadCommandResult struct {
|
||||
output []byte
|
||||
err error
|
||||
}
|
||||
|
||||
func runLXCImageDownloadCommand(cmd *exec.Cmd, templateID string) ([]byte, error) {
|
||||
startedAt := time.Now()
|
||||
done := make(chan lxcImageDownloadCommandResult, 1)
|
||||
go func() {
|
||||
output, err := cmd.CombinedOutput()
|
||||
done <- lxcImageDownloadCommandResult{output: output, err: err}
|
||||
}()
|
||||
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
var lastBytes int64
|
||||
for {
|
||||
select {
|
||||
case result := <-done:
|
||||
return result.output, result.err
|
||||
case <-ticker.C:
|
||||
downloadedBytes := newestLXCRootfsDownloadSize(startedAt)
|
||||
if downloadedBytes <= 0 || downloadedBytes == lastBytes {
|
||||
continue
|
||||
}
|
||||
lastBytes = downloadedBytes
|
||||
updateImageDownload(templateID, func(st *imageDownloadStatus) {
|
||||
st.Stage = "downloading"
|
||||
st.DownloadedBytes = downloadedBytes
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newestLXCRootfsDownloadSize(startedAt time.Time) int64 {
|
||||
matches, _ := filepath.Glob("/tmp/tmp.*/rootfs.tar.xz")
|
||||
var newestTime time.Time
|
||||
var newestSize int64
|
||||
for _, match := range matches {
|
||||
info, err := os.Stat(match)
|
||||
if err != nil || info.IsDir() || info.ModTime().Before(startedAt.Add(-5*time.Second)) {
|
||||
continue
|
||||
}
|
||||
if info.ModTime().After(newestTime) {
|
||||
newestTime = info.ModTime()
|
||||
newestSize = info.Size()
|
||||
}
|
||||
}
|
||||
return newestSize
|
||||
}
|
||||
|
||||
func ensureLXCImageCachePool(pool config.StoragePool) error {
|
||||
lxcImageCacheMu.Lock()
|
||||
defer lxcImageCacheMu.Unlock()
|
||||
|
||||
cachePath := "/var/cache/lxc/download"
|
||||
targetPath := filepath.Join(pool.Path, "images", "lxc")
|
||||
targetAbs, err := filepath.Abs(targetPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(targetAbs, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create LXC image storage: %v", err)
|
||||
}
|
||||
|
||||
info, err := os.Lstat(cachePath)
|
||||
if os.IsNotExist(err) {
|
||||
if err := os.MkdirAll(filepath.Dir(cachePath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Symlink(targetAbs, cachePath)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sourcePath := cachePath
|
||||
linked := info.Mode()&os.ModeSymlink != 0
|
||||
if linked {
|
||||
sourcePath, err = filepath.EvalSymlinks(cachePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to resolve LXC image cache: %v", err)
|
||||
}
|
||||
}
|
||||
sourceAbs, err := filepath.Abs(sourcePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if sourceAbs == targetAbs {
|
||||
return nil
|
||||
}
|
||||
if strings.HasPrefix(targetAbs, sourceAbs+string(os.PathSeparator)) || strings.HasPrefix(sourceAbs, targetAbs+string(os.PathSeparator)) {
|
||||
return fmt.Errorf("LXC image cache source and target must not be nested")
|
||||
}
|
||||
if !info.IsDir() && !linked {
|
||||
return fmt.Errorf("LXC image cache is not a directory: %s", cachePath)
|
||||
}
|
||||
|
||||
if output, err := exec.Command("cp", "-a", sourceAbs+string(os.PathSeparator)+".", targetAbs+string(os.PathSeparator)).CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to migrate LXC image cache: %v, output: %s", err, strings.TrimSpace(string(output)))
|
||||
}
|
||||
|
||||
tempLink := fmt.Sprintf("%s.clicd-new-%d", cachePath, time.Now().UnixNano())
|
||||
if err := os.Symlink(targetAbs, tempLink); err != nil {
|
||||
return err
|
||||
}
|
||||
if linked {
|
||||
if err := os.Rename(tempLink, cachePath); err != nil {
|
||||
_ = os.Remove(tempLink)
|
||||
return fmt.Errorf("failed to switch LXC image cache: %v", err)
|
||||
}
|
||||
if isManagedLXCImageCachePath(sourceAbs) {
|
||||
_ = os.RemoveAll(sourceAbs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
backupPath := fmt.Sprintf("%s.clicd-backup-%d", cachePath, time.Now().UnixNano())
|
||||
if err := os.Rename(cachePath, backupPath); err != nil {
|
||||
_ = os.Remove(tempLink)
|
||||
return fmt.Errorf("failed to prepare LXC image cache migration: %v", err)
|
||||
}
|
||||
if err := os.Rename(tempLink, cachePath); err != nil {
|
||||
_ = os.Rename(backupPath, cachePath)
|
||||
_ = os.Remove(tempLink)
|
||||
return fmt.Errorf("failed to activate LXC image storage: %v", err)
|
||||
}
|
||||
if err := os.RemoveAll(backupPath); err != nil {
|
||||
return fmt.Errorf("LXC image cache migrated but old cache cleanup failed: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isManagedLXCImageCachePath(path string) bool {
|
||||
path = filepath.Clean(path)
|
||||
for _, pool := range config.StoragePoolsForContent(config.StorageContentImages) {
|
||||
if path == filepath.Clean(filepath.Join(pool.Path, "images", "lxc")) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return path == filepath.Clean("/var/lib/clicd/images/lxc")
|
||||
}
|
||||
|
||||
// HandleImageCancel cancels an in-progress image download.
|
||||
func HandleImageCancel(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
|
||||
@@ -147,12 +147,12 @@ func trafficByRuntime(id int) map[string]interface{} {
|
||||
return lxcManager.GetTrafficInfo(id)
|
||||
}
|
||||
|
||||
func createSnapshotByRuntime(id int, createdBy string, scheduled bool, rotateLimit int) (config.Snapshot, error) {
|
||||
func createSnapshotByRuntime(id int, createdBy string, scheduled bool, rotateLimit int, storagePoolID ...string) (config.Snapshot, error) {
|
||||
c := config.FindContainer(id)
|
||||
if c != nil && c.IsKVM() {
|
||||
return kvmManager.CreateSnapshot(id, createdBy, scheduled, rotateLimit)
|
||||
return kvmManager.CreateSnapshot(id, createdBy, scheduled, rotateLimit, storagePoolID...)
|
||||
}
|
||||
return lxcManager.CreateSnapshot(id, createdBy, scheduled, rotateLimit)
|
||||
return lxcManager.CreateSnapshot(id, createdBy, scheduled, rotateLimit, storagePoolID...)
|
||||
}
|
||||
|
||||
func deleteSnapshotByRuntime(snapshotID string) error {
|
||||
|
||||
@@ -2,6 +2,7 @@ package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
@@ -48,6 +49,38 @@ func HandleLanguage(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// HandleTaskQueueSettings returns or updates the global task concurrency limit.
|
||||
func HandleTaskQueueSettings(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
jsonResponse(w, http.StatusOK, APIResponse{Success: true, Data: globalQueue.Settings()})
|
||||
case http.MethodPut, http.MethodPost:
|
||||
var req struct {
|
||||
Concurrency int `json:"concurrency"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
jsonResponse(w, http.StatusBadRequest, APIResponse{Success: false, Message: "Invalid request body"})
|
||||
return
|
||||
}
|
||||
if req.Concurrency < 1 || req.Concurrency > config.MaxTaskConcurrency {
|
||||
jsonResponse(w, http.StatusBadRequest, APIResponse{Success: false, Message: "任务并发数必须在 1 到 16 之间"})
|
||||
return
|
||||
}
|
||||
previous := config.AppConfig.TaskConcurrency
|
||||
config.AppConfig.TaskConcurrency = req.Concurrency
|
||||
if err := config.SaveConfig(); err != nil {
|
||||
config.AppConfig.TaskConcurrency = previous
|
||||
jsonResponse(w, http.StatusInternalServerError, APIResponse{Success: false, Message: "保存任务队列设置失败"})
|
||||
return
|
||||
}
|
||||
globalQueue.SetConcurrency(req.Concurrency)
|
||||
auditRequest(r, "settings.task_queue", "task_concurrency", fmt.Sprintf("concurrency=%d", req.Concurrency), true, "")
|
||||
jsonResponse(w, http.StatusOK, APIResponse{Success: true, Message: "任务队列设置已保存", Data: globalQueue.Settings()})
|
||||
default:
|
||||
jsonResponse(w, http.StatusMethodNotAllowed, APIResponse{Success: false, Message: "Method not allowed"})
|
||||
}
|
||||
}
|
||||
|
||||
// RecordLoginLog adds a login attempt to the log (persisted to config)
|
||||
func RecordLoginLog(username, ip, userAgent string, success bool) {
|
||||
config.AddLoginLog(username, ip, userAgent, success)
|
||||
|
||||
@@ -2,6 +2,7 @@ package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -88,6 +89,20 @@ func listContainerSnapshots(w http.ResponseWriter, r *http.Request, containerID
|
||||
|
||||
func createContainerSnapshot(w http.ResponseWriter, r *http.Request, containerID int) {
|
||||
user := requestUser(r)
|
||||
var req struct {
|
||||
StoragePoolID string `json:"storage_pool_id"`
|
||||
}
|
||||
if r.Body != nil {
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil && err != io.EOF {
|
||||
jsonResponse(w, http.StatusBadRequest, APIResponse{Success: false, Message: "Invalid request body"})
|
||||
return
|
||||
}
|
||||
}
|
||||
req.StoragePoolID = strings.TrimSpace(req.StoragePoolID)
|
||||
if _, err := config.SelectStoragePoolForContent(config.StorageContentSnapshots, req.StoragePoolID, 0); err != nil {
|
||||
jsonResponse(w, http.StatusConflict, APIResponse{Success: false, Message: err.Error()})
|
||||
return
|
||||
}
|
||||
if isSubUserRequest(r) {
|
||||
c := config.FindContainer(containerID)
|
||||
limit := config.ContainerSnapshotLimit(c)
|
||||
@@ -96,7 +111,7 @@ func createContainerSnapshot(w http.ResponseWriter, r *http.Request, containerID
|
||||
return
|
||||
}
|
||||
}
|
||||
snapshot, err := createSnapshotByRuntime(containerID, user, false, 0)
|
||||
snapshot, err := createSnapshotByRuntime(containerID, user, false, 0, req.StoragePoolID)
|
||||
if err != nil {
|
||||
jsonResponse(w, http.StatusInternalServerError, APIResponse{Success: false, Message: err.Error()})
|
||||
return
|
||||
@@ -159,6 +174,12 @@ func updateSnapshotSchedule(w http.ResponseWriter, r *http.Request, containerID
|
||||
if req.Time == "" {
|
||||
req.Time = "03:00"
|
||||
}
|
||||
if req.Enabled {
|
||||
if _, err := config.SelectStoragePoolForContent(config.StorageContentSnapshots, "", 0); err != nil {
|
||||
jsonResponse(w, http.StatusConflict, APIResponse{Success: false, Message: err.Error()})
|
||||
return
|
||||
}
|
||||
}
|
||||
user := requestUser(r)
|
||||
c, err := setSnapshotScheduleByRuntime(containerID, req.Enabled, req.IntervalHours, req.Time, user)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,445 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
pathpkg "path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"clicd/internal/config"
|
||||
)
|
||||
|
||||
type storageInfoResponse struct {
|
||||
Pools []storagePoolInfo `json:"pools"`
|
||||
Disks []storageDiskInfo `json:"disks"`
|
||||
ContentTypes []string `json:"content_types"`
|
||||
}
|
||||
|
||||
type storagePoolInfo struct {
|
||||
config.StoragePool
|
||||
Available bool `json:"available"`
|
||||
Exists bool `json:"exists"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
UsedBytes int64 `json:"used_bytes"`
|
||||
FreeBytes int64 `json:"free_bytes"`
|
||||
ClicdUsedBytes int64 `json:"clicd_used_bytes"`
|
||||
ContentUsage []storageContentUsage `json:"content_usage"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type storageContentUsage struct {
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
}
|
||||
|
||||
type storageDiskInfo struct {
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
Type string `json:"type"`
|
||||
FSType string `json:"fstype"`
|
||||
MountPoint string `json:"mount_point"`
|
||||
Model string `json:"model"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
UsedBytes int64 `json:"used_bytes"`
|
||||
FreeBytes int64 `json:"free_bytes"`
|
||||
StoragePoolID string `json:"storage_pool_id,omitempty"`
|
||||
StoragePath string `json:"storage_path,omitempty"`
|
||||
ClicdUsedBytes int64 `json:"clicd_used_bytes"`
|
||||
ContentUsage []storageContentUsage `json:"content_usage"`
|
||||
}
|
||||
|
||||
func HandleStorage(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
jsonResponse(w, http.StatusOK, APIResponse{Success: true, Data: buildStorageInfo()})
|
||||
case http.MethodPut:
|
||||
var req struct {
|
||||
Pools []config.StoragePool `json:"pools"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
jsonResponse(w, http.StatusBadRequest, APIResponse{Success: false, Message: "Invalid request body"})
|
||||
return
|
||||
}
|
||||
pools, err := normalizeStoragePoolsRequest(req.Pools)
|
||||
if err != nil {
|
||||
jsonResponse(w, http.StatusBadRequest, APIResponse{Success: false, Message: err.Error()})
|
||||
return
|
||||
}
|
||||
for _, pool := range pools {
|
||||
if err := os.MkdirAll(pool.Path, 0755); err != nil {
|
||||
jsonResponse(w, http.StatusInternalServerError, APIResponse{Success: false, Message: fmt.Sprintf("Failed to create %s: %v", pool.Path, err)})
|
||||
return
|
||||
}
|
||||
}
|
||||
config.AppConfig.StoragePools = pools
|
||||
if err := config.SaveConfig(); err != nil {
|
||||
jsonResponse(w, http.StatusInternalServerError, APIResponse{Success: false, Message: "Failed to save storage pools"})
|
||||
return
|
||||
}
|
||||
jsonResponse(w, http.StatusOK, APIResponse{Success: true, Data: buildStorageInfo()})
|
||||
default:
|
||||
jsonResponse(w, http.StatusMethodNotAllowed, APIResponse{Success: false, Message: "Method not allowed"})
|
||||
}
|
||||
}
|
||||
|
||||
func buildStorageInfo() storageInfoResponse {
|
||||
disks := detectStorageDisks()
|
||||
pools := make([]storagePoolInfo, 0, len(config.AppConfig.StoragePools))
|
||||
for _, pool := range config.AppConfig.StoragePools {
|
||||
info := storagePoolInfo{StoragePool: pool}
|
||||
if filepath.Clean(pool.MountPoint) == string(os.PathSeparator) {
|
||||
_ = os.MkdirAll(pool.Path, 0755)
|
||||
}
|
||||
if st, err := os.Stat(pool.Path); err == nil && st.IsDir() {
|
||||
info.Exists = true
|
||||
} else if err != nil {
|
||||
info.Error = err.Error()
|
||||
}
|
||||
detectedMountPoint := bestMountPointForPath(pool.Path, disks)
|
||||
if info.MountPoint == "" {
|
||||
info.MountPoint = detectedMountPoint
|
||||
}
|
||||
if detectedMountPoint != "" && filepath.Clean(info.MountPoint) == filepath.Clean(detectedMountPoint) {
|
||||
info.Available = info.Exists
|
||||
info.SizeBytes, info.UsedBytes, info.FreeBytes = dfPath(pool.Path)
|
||||
info.ContentUsage, info.ClicdUsedBytes = contentUsageForPool(pool.Path)
|
||||
} else if info.Error == "" {
|
||||
info.Error = "storage disk is not mounted"
|
||||
}
|
||||
pools = append(pools, info)
|
||||
}
|
||||
for i := range disks {
|
||||
for _, pool := range pools {
|
||||
if pool.MountPoint != disks[i].MountPoint {
|
||||
continue
|
||||
}
|
||||
disks[i].ClicdUsedBytes += pool.ClicdUsedBytes
|
||||
disks[i].ContentUsage = mergeContentUsage(disks[i].ContentUsage, pool.ContentUsage)
|
||||
if disks[i].StoragePoolID == "" {
|
||||
disks[i].StoragePoolID = pool.ID
|
||||
disks[i].StoragePath = pool.Path
|
||||
}
|
||||
}
|
||||
}
|
||||
return storageInfoResponse{
|
||||
Pools: pools,
|
||||
Disks: disks,
|
||||
ContentTypes: []string{
|
||||
config.StorageContentLXC,
|
||||
config.StorageContentKVM,
|
||||
config.StorageContentImages,
|
||||
config.StorageContentSnapshots,
|
||||
config.StorageContentBackups,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeStoragePoolsRequest(items []config.StoragePool) ([]config.StoragePool, error) {
|
||||
if len(items) == 0 {
|
||||
return nil, fmt.Errorf("at least one mounted storage disk configuration must be retained")
|
||||
}
|
||||
result := make([]config.StoragePool, 0, len(items))
|
||||
seen := map[string]bool{}
|
||||
defaultSeen := map[string]bool{}
|
||||
disks := detectStorageDisks()
|
||||
for _, item := range items {
|
||||
item.ID = strings.TrimSpace(item.ID)
|
||||
item.Name = strings.TrimSpace(item.Name)
|
||||
item.Path = filepath.Clean(strings.TrimSpace(item.Path))
|
||||
item.MountPoint = filepath.Clean(strings.TrimSpace(item.MountPoint))
|
||||
if item.MountPoint == "." {
|
||||
item.MountPoint = ""
|
||||
}
|
||||
if item.Name == "" {
|
||||
return nil, fmt.Errorf("storage pool name is required")
|
||||
}
|
||||
if item.ID == "" {
|
||||
item.ID = storageID(item.Name)
|
||||
}
|
||||
if seen[item.ID] {
|
||||
return nil, fmt.Errorf("duplicate storage pool ID: %s", item.ID)
|
||||
}
|
||||
seen[item.ID] = true
|
||||
if !filepath.IsAbs(item.Path) {
|
||||
return nil, fmt.Errorf("%s path must be absolute", item.Name)
|
||||
}
|
||||
detectedMountPoint := bestMountPointForPath(item.Path, disks)
|
||||
if detectedMountPoint == "" {
|
||||
return nil, fmt.Errorf("%s path is not on an available mounted storage disk", item.Name)
|
||||
}
|
||||
if item.MountPoint != "" && filepath.Clean(item.MountPoint) != filepath.Clean(detectedMountPoint) {
|
||||
return nil, fmt.Errorf("%s storage disk mount point has changed; refresh and try again", item.Name)
|
||||
}
|
||||
item.MountPoint = detectedMountPoint
|
||||
item.ContentTypes = normalizeStorageContentTypes(item.ContentTypes)
|
||||
item.DefaultContents = normalizeStorageContentTypes(item.DefaultContents)
|
||||
allowed := map[string]bool{}
|
||||
for _, content := range item.ContentTypes {
|
||||
allowed[content] = true
|
||||
}
|
||||
defaults := make([]string, 0, len(item.DefaultContents))
|
||||
for _, content := range item.DefaultContents {
|
||||
if !allowed[content] {
|
||||
continue
|
||||
}
|
||||
if defaultSeen[content] {
|
||||
return nil, fmt.Errorf("only one default storage disk is allowed for %s", content)
|
||||
}
|
||||
defaultSeen[content] = true
|
||||
defaults = append(defaults, content)
|
||||
}
|
||||
item.DefaultContents = defaults
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func normalizeStorageContentTypes(values []string) []string {
|
||||
valid := map[string]bool{
|
||||
config.StorageContentLXC: true,
|
||||
config.StorageContentKVM: true,
|
||||
config.StorageContentImages: true,
|
||||
config.StorageContentSnapshots: true,
|
||||
config.StorageContentBackups: true,
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
result := []string{}
|
||||
for _, value := range values {
|
||||
next := strings.ToLower(strings.TrimSpace(value))
|
||||
if !valid[next] || seen[next] {
|
||||
continue
|
||||
}
|
||||
seen[next] = true
|
||||
result = append(result, next)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func storageID(name string) string {
|
||||
id := strings.ToLower(strings.TrimSpace(name))
|
||||
id = strings.NewReplacer(" ", "-", "_", "-", ".", "-", "/", "-").Replace(id)
|
||||
id = strings.Trim(id, "-")
|
||||
if id == "" {
|
||||
return "storage"
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func detectStorageDisks() []storageDiskInfo {
|
||||
type lsblkDevice struct {
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
Type string `json:"type"`
|
||||
FSType string `json:"fstype"`
|
||||
MountPoint string `json:"mountpoint"`
|
||||
Model string `json:"model"`
|
||||
Size int64 `json:"size"`
|
||||
ReadOnly bool `json:"ro"`
|
||||
Children []lsblkDevice `json:"children"`
|
||||
}
|
||||
var payload struct {
|
||||
BlockDevices []lsblkDevice `json:"blockdevices"`
|
||||
}
|
||||
out, err := exec.Command("lsblk", "-J", "-b", "-o", "NAME,PATH,SIZE,TYPE,FSTYPE,MOUNTPOINT,MODEL,RO").Output()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(out, &payload); err != nil {
|
||||
return nil
|
||||
}
|
||||
result := []storageDiskInfo{}
|
||||
var walk func(lsblkDevice)
|
||||
walk = func(dev lsblkDevice) {
|
||||
info := storageDiskInfo{
|
||||
Name: dev.Name,
|
||||
Path: dev.Path,
|
||||
Type: dev.Type,
|
||||
FSType: dev.FSType,
|
||||
MountPoint: dev.MountPoint,
|
||||
Model: strings.TrimSpace(dev.Model),
|
||||
SizeBytes: dev.Size,
|
||||
}
|
||||
if isUsableStorageMount(dev.Type, dev.FSType, dev.Path, dev.MountPoint, dev.ReadOnly) && !mountIsReadOnly(dev.MountPoint) {
|
||||
info.SizeBytes, info.UsedBytes, info.FreeBytes = dfPath(dev.MountPoint)
|
||||
result = append(result, info)
|
||||
}
|
||||
for _, child := range dev.Children {
|
||||
walk(child)
|
||||
}
|
||||
}
|
||||
for _, dev := range payload.BlockDevices {
|
||||
walk(dev)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func isUsableStorageMount(deviceType, fsType, devicePath, mountPoint string, readOnly bool) bool {
|
||||
if readOnly || strings.TrimSpace(mountPoint) == "" || !strings.HasPrefix(mountPoint, "/") {
|
||||
return false
|
||||
}
|
||||
|
||||
deviceType = strings.ToLower(strings.TrimSpace(deviceType))
|
||||
devicePath = strings.ToLower(strings.TrimSpace(devicePath))
|
||||
if deviceType == "loop" || deviceType == "rom" || deviceType == "zram" || strings.HasPrefix(devicePath, "/dev/loop") {
|
||||
return false
|
||||
}
|
||||
|
||||
fsType = strings.ToLower(strings.TrimSpace(fsType))
|
||||
unsupportedFileSystems := map[string]bool{
|
||||
"": true,
|
||||
"squashfs": true,
|
||||
"iso9660": true,
|
||||
"udf": true,
|
||||
"swap": true,
|
||||
"tmpfs": true,
|
||||
"devtmpfs": true,
|
||||
"overlay": true,
|
||||
"proc": true,
|
||||
"sysfs": true,
|
||||
"cgroup": true,
|
||||
"cgroup2": true,
|
||||
"efivarfs": true,
|
||||
"securityfs": true,
|
||||
}
|
||||
if unsupportedFileSystems[fsType] {
|
||||
return false
|
||||
}
|
||||
|
||||
mountPoint = pathpkg.Clean(mountPoint)
|
||||
for _, reserved := range []string{"/snap", "/boot"} {
|
||||
if mountPoint == reserved || strings.HasPrefix(mountPoint, reserved+"/") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func mountIsReadOnly(mountPoint string) bool {
|
||||
out, err := exec.Command("findmnt", "-n", "-o", "OPTIONS", "--target", mountPoint).Output()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, option := range strings.Split(strings.TrimSpace(string(out)), ",") {
|
||||
if strings.TrimSpace(option) == "ro" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func contentUsageForPool(poolPath string) ([]storageContentUsage, int64) {
|
||||
mapping := map[string]string{
|
||||
config.StorageContentLXC: "lxc",
|
||||
config.StorageContentKVM: "kvm",
|
||||
config.StorageContentImages: "images",
|
||||
config.StorageContentSnapshots: "snapshots",
|
||||
config.StorageContentBackups: "backups",
|
||||
}
|
||||
result := make([]storageContentUsage, 0, len(mapping))
|
||||
var total int64
|
||||
for _, content := range []string{
|
||||
config.StorageContentLXC,
|
||||
config.StorageContentKVM,
|
||||
config.StorageContentImages,
|
||||
config.StorageContentSnapshots,
|
||||
config.StorageContentBackups,
|
||||
} {
|
||||
size := dirSizeBytes(filepath.Join(poolPath, mapping[content]))
|
||||
result = append(result, storageContentUsage{ContentType: content, SizeBytes: size})
|
||||
total += size
|
||||
}
|
||||
return result, total
|
||||
}
|
||||
|
||||
func mergeContentUsage(current []storageContentUsage, next []storageContentUsage) []storageContentUsage {
|
||||
sizes := map[string]int64{}
|
||||
order := []string{}
|
||||
for _, item := range append(current, next...) {
|
||||
if _, ok := sizes[item.ContentType]; !ok {
|
||||
order = append(order, item.ContentType)
|
||||
}
|
||||
sizes[item.ContentType] += item.SizeBytes
|
||||
}
|
||||
result := make([]storageContentUsage, 0, len(order))
|
||||
for _, content := range order {
|
||||
result = append(result, storageContentUsage{ContentType: content, SizeBytes: sizes[content]})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func dirSizeBytes(path string) int64 {
|
||||
if resolved, err := filepath.EvalSymlinks(path); err == nil {
|
||||
path = resolved
|
||||
}
|
||||
// Count allocated blocks on this filesystem only. LXC rootfs directories can
|
||||
// contain active mounts such as proc/sys; traversing them is slow and reports
|
||||
// enormous virtual sizes that are not actually occupied by CLICD data.
|
||||
out, err := exec.Command("du", "-skx", path).Output()
|
||||
if err == nil {
|
||||
fields := strings.Fields(string(out))
|
||||
if len(fields) > 0 {
|
||||
var sizeKB int64
|
||||
if _, scanErr := fmt.Sscanf(fields[0], "%d", &sizeKB); scanErr == nil && sizeKB <= (1<<63-1)/1024 {
|
||||
return sizeKB * 1024
|
||||
}
|
||||
}
|
||||
}
|
||||
var size int64
|
||||
_ = filepath.WalkDir(path, func(_ string, d os.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if info, statErr := d.Info(); statErr == nil {
|
||||
size += info.Size()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return size
|
||||
}
|
||||
|
||||
func dfPath(path string) (size int64, used int64, free int64) {
|
||||
out, err := exec.Command("df", "-B1", "-P", path).Output()
|
||||
if err != nil {
|
||||
return 0, 0, 0
|
||||
}
|
||||
lines := strings.Split(strings.TrimSpace(string(out)), "\n")
|
||||
if len(lines) < 2 {
|
||||
return 0, 0, 0
|
||||
}
|
||||
fields := strings.Fields(lines[len(lines)-1])
|
||||
if len(fields) < 6 {
|
||||
return 0, 0, 0
|
||||
}
|
||||
fmt.Sscanf(fields[1], "%d", &size)
|
||||
fmt.Sscanf(fields[2], "%d", &used)
|
||||
fmt.Sscanf(fields[3], "%d", &free)
|
||||
return size, used, free
|
||||
}
|
||||
|
||||
func bestMountPointForPath(path string, disks []storageDiskInfo) string {
|
||||
path = strings.ReplaceAll(path, "\\", "/")
|
||||
path = pathpkg.Clean(path)
|
||||
best := ""
|
||||
for _, disk := range disks {
|
||||
mp := pathpkg.Clean(strings.ReplaceAll(disk.MountPoint, "\\", "/"))
|
||||
if disk.MountPoint == "" || mp == "." {
|
||||
continue
|
||||
}
|
||||
matches := path == mp
|
||||
if mp == "/" {
|
||||
matches = pathpkg.IsAbs(path)
|
||||
} else if strings.HasPrefix(path, mp+"/") {
|
||||
matches = true
|
||||
}
|
||||
if matches {
|
||||
if len(mp) > len(best) {
|
||||
best = mp
|
||||
}
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsUsableStorageMount(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
deviceType string
|
||||
fsType string
|
||||
devicePath string
|
||||
mountPoint string
|
||||
readOnly bool
|
||||
wantUsable bool
|
||||
}{
|
||||
{name: "root partition", deviceType: "part", fsType: "ext4", devicePath: "/dev/sda2", mountPoint: "/", wantUsable: true},
|
||||
{name: "mounted data disk", deviceType: "disk", fsType: "xfs", devicePath: "/dev/sdb", mountPoint: "/data", wantUsable: true},
|
||||
{name: "snap loop", deviceType: "loop", fsType: "squashfs", devicePath: "/dev/loop0", mountPoint: "/snap/core20/2105", readOnly: true},
|
||||
{name: "loop without ro flag", deviceType: "loop", fsType: "ext4", devicePath: "/dev/loop7", mountPoint: "/mnt/loop"},
|
||||
{name: "read only disk", deviceType: "part", fsType: "ext4", devicePath: "/dev/sdc1", mountPoint: "/archive", readOnly: true},
|
||||
{name: "optical image", deviceType: "rom", fsType: "iso9660", devicePath: "/dev/sr0", mountPoint: "/media/cdrom"},
|
||||
{name: "efi partition", deviceType: "part", fsType: "vfat", devicePath: "/dev/sda1", mountPoint: "/boot/efi"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := isUsableStorageMount(tt.deviceType, tt.fsType, tt.devicePath, tt.mountPoint, tt.readOnly)
|
||||
if got != tt.wantUsable {
|
||||
t.Fatalf("isUsableStorageMount() = %v, want %v", got, tt.wantUsable)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBestMountPointForPath(t *testing.T) {
|
||||
disks := []storageDiskInfo{
|
||||
{Path: "/dev/sda2", MountPoint: "/"},
|
||||
{Path: "/dev/sdb1", MountPoint: "/mnt/clicd-data"},
|
||||
}
|
||||
tests := []struct {
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{path: "/var/lib/clicd", want: "/"},
|
||||
{path: "/mnt/clicd-data/clicd", want: "/mnt/clicd-data"},
|
||||
{path: "/mnt/clicd-data", want: "/mnt/clicd-data"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := bestMountPointForPath(tt.path, disks); got != tt.want {
|
||||
t.Fatalf("bestMountPointForPath(%q) = %q, want %q", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirSizeBytesUsesAllocatedBlocks(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("allocated-block behavior is provided by the Linux du command")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
file, err := os.Create(filepath.Join(dir, "sparse.img"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := file.Truncate(1 << 30); err != nil {
|
||||
file.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got := dirSizeBytes(dir); got >= 128<<20 {
|
||||
t.Fatalf("dirSizeBytes() = %d, expected allocated size instead of 1 GiB apparent size", got)
|
||||
}
|
||||
}
|
||||
+328
-178
@@ -30,6 +30,8 @@ type Task struct {
|
||||
ContainerName string `json:"container_name"`
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Stage string `json:"stage,omitempty"`
|
||||
StageDetail string `json:"stage_detail,omitempty"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
TemplateID string `json:"template_id,omitempty"`
|
||||
Config lxc.ContainerConfig `json:"config,omitempty"`
|
||||
@@ -37,30 +39,74 @@ type Task struct {
|
||||
User string `json:"user,omitempty"` // who created this task
|
||||
IP string `json:"ip,omitempty"`
|
||||
UserAgent string `json:"user_agent,omitempty"`
|
||||
activeKey string
|
||||
}
|
||||
|
||||
type TaskQueue struct {
|
||||
mu sync.Mutex
|
||||
createQueue []*Task
|
||||
opQueue []*Task
|
||||
tasks map[string]*Task
|
||||
nextID int
|
||||
createCond *sync.Cond
|
||||
opCond *sync.Cond
|
||||
stop chan struct{}
|
||||
mu sync.Mutex
|
||||
createQueue []*Task
|
||||
opQueue []*Task
|
||||
tasks map[string]*Task
|
||||
nextID int
|
||||
createCond *sync.Cond
|
||||
opCond *sync.Cond
|
||||
maxConcurrency int
|
||||
activeTasks int
|
||||
activeTargets map[string]bool
|
||||
stop chan struct{}
|
||||
}
|
||||
|
||||
type TaskQueueSettings struct {
|
||||
Concurrency int `json:"concurrency"`
|
||||
Active int `json:"active"`
|
||||
Pending int `json:"pending"`
|
||||
}
|
||||
|
||||
var globalQueue *TaskQueue
|
||||
|
||||
func init() {
|
||||
globalQueue = &TaskQueue{
|
||||
tasks: make(map[string]*Task),
|
||||
stop: make(chan struct{}),
|
||||
globalQueue = newTaskQueue(config.DefaultTaskConcurrency)
|
||||
go globalQueue.createDispatcher()
|
||||
go globalQueue.opDispatcher()
|
||||
}
|
||||
|
||||
func newTaskQueue(concurrency int) *TaskQueue {
|
||||
q := &TaskQueue{
|
||||
tasks: make(map[string]*Task),
|
||||
maxConcurrency: config.NormalizeTaskConcurrency(concurrency),
|
||||
activeTargets: make(map[string]bool),
|
||||
stop: make(chan struct{}),
|
||||
}
|
||||
globalQueue.createCond = sync.NewCond(&globalQueue.mu)
|
||||
globalQueue.opCond = sync.NewCond(&globalQueue.mu)
|
||||
go globalQueue.createWorker()
|
||||
go globalQueue.opWorker()
|
||||
q.createCond = sync.NewCond(&q.mu)
|
||||
q.opCond = sync.NewCond(&q.mu)
|
||||
return q
|
||||
}
|
||||
|
||||
func ConfigureTaskQueue(concurrency int) {
|
||||
globalQueue.SetConcurrency(concurrency)
|
||||
}
|
||||
|
||||
func (q *TaskQueue) SetConcurrency(concurrency int) {
|
||||
q.mu.Lock()
|
||||
q.maxConcurrency = config.NormalizeTaskConcurrency(concurrency)
|
||||
q.createCond.Broadcast()
|
||||
q.opCond.Broadcast()
|
||||
q.mu.Unlock()
|
||||
}
|
||||
|
||||
func (q *TaskQueue) Settings() TaskQueueSettings {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
return TaskQueueSettings{
|
||||
Concurrency: q.maxConcurrency,
|
||||
Active: q.activeTasks,
|
||||
Pending: len(q.createQueue) + len(q.opQueue),
|
||||
}
|
||||
}
|
||||
|
||||
func (q *TaskQueue) signalDispatchers() {
|
||||
q.createCond.Broadcast()
|
||||
q.opCond.Broadcast()
|
||||
}
|
||||
|
||||
func (q *TaskQueue) enqueueTask(task *Task) {
|
||||
@@ -90,6 +136,8 @@ func (q *TaskQueue) EnqueueWithAudit(containerID int, containerName string, task
|
||||
ContainerID: containerID,
|
||||
ContainerName: containerName,
|
||||
Status: "pending",
|
||||
Stage: "queued",
|
||||
StageDetail: "排队等待",
|
||||
CreatedAt: time.Now().Format("2006-01-02 15:04:05"),
|
||||
TemplateID: templateID,
|
||||
User: user,
|
||||
@@ -172,6 +220,8 @@ func (q *TaskQueue) enqueueBatchCreateList(configs []lxc.ContainerConfig, user s
|
||||
ContainerID: 0,
|
||||
ContainerName: cfgCopy.Name,
|
||||
Status: "pending",
|
||||
Stage: "queued",
|
||||
StageDetail: "排队等待",
|
||||
CreatedAt: time.Now().Format("2006-01-02 15:04:05"),
|
||||
Config: cfgCopy,
|
||||
User: user,
|
||||
@@ -202,6 +252,8 @@ func (q *TaskQueue) enqueueSingleWithAudit(containerID int, containerName string
|
||||
ContainerID: containerID,
|
||||
ContainerName: containerName,
|
||||
Status: "pending",
|
||||
Stage: "queued",
|
||||
StageDetail: "排队等待",
|
||||
CreatedAt: time.Now().Format("2006-01-02 15:04:05"),
|
||||
TemplateID: templateID,
|
||||
User: user,
|
||||
@@ -262,176 +314,251 @@ func (q *TaskQueue) CancelPendingSecurityStops() int {
|
||||
return cancelled
|
||||
}
|
||||
|
||||
// createWorker handles TaskCreate: lxc-create, resource setup, start, and SSH init.
|
||||
// If a restored task already has a same-name container in config, it resumes
|
||||
// initialization instead of creating another ct-{id}.
|
||||
func (q *TaskQueue) createWorker() {
|
||||
// The two dispatchers keep long-running creates from blocking power operations,
|
||||
// while sharing one global concurrency budget.
|
||||
func (q *TaskQueue) createDispatcher() {
|
||||
for {
|
||||
q.mu.Lock()
|
||||
for len(q.createQueue) == 0 {
|
||||
q.createCond.Wait()
|
||||
}
|
||||
task := q.createQueue[0]
|
||||
q.createQueue = q.createQueue[1:]
|
||||
task.Status = "running"
|
||||
q.mu.Unlock()
|
||||
|
||||
createdByTask := false
|
||||
if task.Config.Name == "" {
|
||||
task.Config.Name = task.ContainerName
|
||||
}
|
||||
task.Config.NormalizeResourceAliases()
|
||||
if task.Config.Name == "" {
|
||||
task.Status = "failed"
|
||||
task.Error = "container name is required"
|
||||
config.AddAuditLog(string(task.Type), task.ContainerName, "failed: "+task.Error, "admin")
|
||||
q.mu.Lock()
|
||||
q.persistTasks()
|
||||
q.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
c := config.FindContainerByName(task.Config.Name)
|
||||
if c == nil {
|
||||
// 1) Download image + apply limits (lxc-create)
|
||||
err := createByRuntime(task.Config)
|
||||
if err != nil {
|
||||
task.Status = "failed"
|
||||
task.Error = err.Error()
|
||||
config.AddAuditLog(string(task.Type), task.Config.Name, "失败: "+err.Error(), "admin")
|
||||
q.mu.Lock()
|
||||
q.persistTasks()
|
||||
q.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
createdByTask = true
|
||||
|
||||
// 2) Find created container by name
|
||||
c = config.FindContainerByName(task.Config.Name)
|
||||
if c == nil {
|
||||
task.Status = "failed"
|
||||
task.Error = "created but not found in config"
|
||||
config.AddAuditLog(string(task.Type), task.Config.Name, "失败: "+task.Error, "admin")
|
||||
q.mu.Lock()
|
||||
q.persistTasks()
|
||||
q.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
task.ContainerID = c.ID
|
||||
task.ContainerName = c.Name
|
||||
|
||||
// 3) Start + initialize SSH/network in the same worker.
|
||||
// If init fails, destroy the container so no dead entry remains.
|
||||
startErr := startByRuntime(c.ID)
|
||||
if startErr != nil {
|
||||
if createdByTask {
|
||||
_ = destroyByRuntime(c.ID)
|
||||
}
|
||||
task.Status = "failed"
|
||||
task.Error = startErr.Error()
|
||||
config.AddAuditLog(string(task.Type), task.ContainerName, "初始化失败: "+startErr.Error(), "admin")
|
||||
} else {
|
||||
task.Status = "done"
|
||||
config.AddAuditLog(string(task.Type), task.ContainerName, "成功", "admin")
|
||||
}
|
||||
|
||||
q.mu.Lock()
|
||||
q.persistTasks()
|
||||
q.mu.Unlock()
|
||||
task := q.takeNextTask(true)
|
||||
go q.runCreateTask(task)
|
||||
}
|
||||
}
|
||||
|
||||
// opWorker handles all non-create tasks (start, stop, restart, delete, reinstall)
|
||||
// including the follow-up initialization after a create succeeds.
|
||||
func (q *TaskQueue) opWorker() {
|
||||
func (q *TaskQueue) opDispatcher() {
|
||||
for {
|
||||
q.mu.Lock()
|
||||
for len(q.opQueue) == 0 {
|
||||
q.opCond.Wait()
|
||||
}
|
||||
task := q.opQueue[0]
|
||||
q.opQueue = q.opQueue[1:]
|
||||
task.Status = "running"
|
||||
q.mu.Unlock()
|
||||
|
||||
var err error
|
||||
skipped := false
|
||||
err = resolveTaskContainer(task)
|
||||
// Block operations on expired or traffic-exceeded containers (except stop/delete)
|
||||
if err == nil && (task.Type == TaskStart || task.Type == TaskRestart || task.Type == TaskReinstall) {
|
||||
c := config.FindContainer(task.ContainerID)
|
||||
if c != nil {
|
||||
if lxc.IsExpired(*c) {
|
||||
err = fmt.Errorf("容器已到期,不允许此操作")
|
||||
} else if lxc.IsTrafficExceeded(*c) {
|
||||
err = fmt.Errorf("容器流量已超限,不允许此操作")
|
||||
}
|
||||
}
|
||||
}
|
||||
if err == nil && isSecurityStopTask(task) && !config.AppConfig.SecurityAutoShutdown {
|
||||
skipped = true
|
||||
}
|
||||
if err == nil {
|
||||
if !skipped {
|
||||
switch task.Type {
|
||||
case TaskStart:
|
||||
err = startByRuntime(task.ContainerID)
|
||||
case TaskStop:
|
||||
err = stopByRuntime(task.ContainerID)
|
||||
case TaskRestart:
|
||||
err = restartByRuntime(task.ContainerID)
|
||||
case TaskDelete:
|
||||
err = destroyByRuntime(task.ContainerID)
|
||||
if err == nil {
|
||||
time.Sleep(1 * time.Second)
|
||||
if config.FindContainer(task.ContainerID) != nil {
|
||||
err = fmt.Errorf("container still exists after delete: %d", task.ContainerID)
|
||||
}
|
||||
}
|
||||
case TaskReinstall:
|
||||
if lxc.HasSSHAuthOptions(task.Config) {
|
||||
err = reinstallByRuntime(task.ContainerID, task.TemplateID, task.Config)
|
||||
} else {
|
||||
err = reinstallByRuntime(task.ContainerID, task.TemplateID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
q.mu.Lock()
|
||||
auditUser := task.User
|
||||
if auditUser == "" {
|
||||
auditUser = "admin"
|
||||
}
|
||||
if err != nil {
|
||||
task.Status = "failed"
|
||||
task.Error = err.Error()
|
||||
config.AddAuditLogFull(string(task.Type), task.ContainerName, "失败: "+err.Error(), auditUser, task.IP, task.UserAgent, false, err.Error())
|
||||
} else if skipped {
|
||||
task.Status = "done"
|
||||
config.AddAuditLogFull(string(task.Type), task.ContainerName, "跳过: 安全告警自动关机已关闭", auditUser, task.IP, task.UserAgent, true, "")
|
||||
} else {
|
||||
task.Status = "done"
|
||||
config.AddAuditLogFull(string(task.Type), task.ContainerName, "成功", auditUser, task.IP, task.UserAgent, true, "")
|
||||
switch task.Type {
|
||||
case TaskStart:
|
||||
config.UpdateContainerStatus(task.ContainerID, "running")
|
||||
clearPolicyBlockAfterAdminRecovery(task)
|
||||
case TaskStop:
|
||||
config.UpdateContainerStatus(task.ContainerID, "stopped")
|
||||
case TaskRestart:
|
||||
config.UpdateContainerStatus(task.ContainerID, "running")
|
||||
clearPolicyBlockAfterAdminRecovery(task)
|
||||
case TaskReinstall:
|
||||
clearPolicyBlockAfterAdminRecovery(task)
|
||||
}
|
||||
}
|
||||
q.persistTasks()
|
||||
q.mu.Unlock()
|
||||
task := q.takeNextTask(false)
|
||||
go q.runOperationTask(task)
|
||||
}
|
||||
}
|
||||
|
||||
func (q *TaskQueue) takeNextTask(create bool) *Task {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
cond := q.opCond
|
||||
if create {
|
||||
cond = q.createCond
|
||||
}
|
||||
for {
|
||||
queue := q.opQueue
|
||||
if create {
|
||||
queue = q.createQueue
|
||||
}
|
||||
if q.activeTasks < q.maxConcurrency {
|
||||
if index := runnableTaskIndex(queue, q.activeTargets); index >= 0 {
|
||||
task := queue[index]
|
||||
queue = append(queue[:index], queue[index+1:]...)
|
||||
if create {
|
||||
q.createQueue = queue
|
||||
} else {
|
||||
q.opQueue = queue
|
||||
}
|
||||
task.Status = "running"
|
||||
task.Error = ""
|
||||
task.Stage = "preparing"
|
||||
task.StageDetail = "准备初始化环境"
|
||||
task.activeKey = taskConcurrencyKey(task)
|
||||
q.activeTargets[task.activeKey] = true
|
||||
q.activeTasks++
|
||||
q.persistTasks()
|
||||
return task
|
||||
}
|
||||
}
|
||||
cond.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
func runnableTaskIndex(queue []*Task, activeTargets map[string]bool) int {
|
||||
for index, task := range queue {
|
||||
if !activeTargets[taskConcurrencyKey(task)] {
|
||||
return index
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func taskConcurrencyKey(task *Task) string {
|
||||
if task == nil {
|
||||
return "task:nil"
|
||||
}
|
||||
name := strings.TrimSpace(task.ContainerName)
|
||||
if name == "" {
|
||||
name = strings.TrimSpace(task.Config.Name)
|
||||
}
|
||||
if name != "" {
|
||||
return "name:" + strings.ToLower(name)
|
||||
}
|
||||
if task.ContainerID > 0 {
|
||||
return fmt.Sprintf("id:%d", task.ContainerID)
|
||||
}
|
||||
return "task:" + task.ID
|
||||
}
|
||||
|
||||
func (q *TaskQueue) finishTask(task *Task, status string, taskErr error) {
|
||||
q.mu.Lock()
|
||||
task.Status = status
|
||||
if taskErr != nil {
|
||||
task.Error = taskErr.Error()
|
||||
if task.Type == TaskCreate {
|
||||
task.Stage = "failed"
|
||||
task.StageDetail = "初始化失败"
|
||||
}
|
||||
} else {
|
||||
task.Error = ""
|
||||
if task.Type == TaskCreate {
|
||||
task.Stage = "completed"
|
||||
task.StageDetail = "初始化完成"
|
||||
}
|
||||
}
|
||||
if task.activeKey != "" {
|
||||
delete(q.activeTargets, task.activeKey)
|
||||
task.activeKey = ""
|
||||
}
|
||||
if q.activeTasks > 0 {
|
||||
q.activeTasks--
|
||||
}
|
||||
q.persistTasks()
|
||||
q.signalDispatchers()
|
||||
q.mu.Unlock()
|
||||
}
|
||||
|
||||
func (q *TaskQueue) updateTaskStage(task *Task, stage, detail string) {
|
||||
q.mu.Lock()
|
||||
task.Stage = stage
|
||||
task.StageDetail = detail
|
||||
q.mu.Unlock()
|
||||
}
|
||||
|
||||
// runCreateTask handles lxc-create, resource setup, start, and SSH init. A
|
||||
// restored task resumes initialization when the same-name container exists.
|
||||
func (q *TaskQueue) runCreateTask(task *Task) {
|
||||
q.mu.Lock()
|
||||
createdByTask := false
|
||||
if task.Config.Name == "" {
|
||||
task.Config.Name = task.ContainerName
|
||||
}
|
||||
task.Config.NormalizeResourceAliases()
|
||||
cfg := task.Config
|
||||
q.mu.Unlock()
|
||||
cfg.Progress = func(stage, detail string) {
|
||||
q.updateTaskStage(task, stage, detail)
|
||||
}
|
||||
if cfg.Name == "" {
|
||||
err := fmt.Errorf("container name is required")
|
||||
config.AddAuditLog(string(task.Type), task.ContainerName, "failed: "+err.Error(), "admin")
|
||||
q.finishTask(task, "failed", err)
|
||||
return
|
||||
}
|
||||
c := config.FindContainerByName(cfg.Name)
|
||||
if c == nil {
|
||||
if err := createByRuntime(cfg); err != nil {
|
||||
config.AddAuditLog(string(task.Type), cfg.Name, "失败: "+err.Error(), "admin")
|
||||
q.finishTask(task, "failed", err)
|
||||
return
|
||||
}
|
||||
createdByTask = true
|
||||
c = config.FindContainerByName(cfg.Name)
|
||||
if c == nil {
|
||||
err := fmt.Errorf("created but not found in config")
|
||||
config.AddAuditLog(string(task.Type), task.Config.Name, "失败: "+err.Error(), "admin")
|
||||
q.finishTask(task, "failed", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
q.mu.Lock()
|
||||
task.ContainerID = c.ID
|
||||
task.ContainerName = c.Name
|
||||
q.mu.Unlock()
|
||||
startDetail := "启动容器并等待网络就绪"
|
||||
if strings.EqualFold(cfg.Virtualization, config.VirtualizationKVM) {
|
||||
startDetail = "启动虚拟机并等待网络就绪"
|
||||
}
|
||||
q.updateTaskStage(task, "starting", startDetail)
|
||||
if err := startByRuntime(c.ID); err != nil {
|
||||
if createdByTask {
|
||||
_ = destroyByRuntime(c.ID)
|
||||
}
|
||||
config.AddAuditLog(string(task.Type), task.ContainerName, "初始化失败: "+err.Error(), "admin")
|
||||
q.finishTask(task, "failed", err)
|
||||
return
|
||||
}
|
||||
config.AddAuditLog(string(task.Type), task.ContainerName, "成功", "admin")
|
||||
q.finishTask(task, "done", nil)
|
||||
}
|
||||
|
||||
func (q *TaskQueue) runOperationTask(task *Task) {
|
||||
q.mu.Lock()
|
||||
err := resolveTaskContainer(task)
|
||||
q.mu.Unlock()
|
||||
skipped := false
|
||||
if err == nil && (task.Type == TaskStart || task.Type == TaskRestart || task.Type == TaskReinstall) {
|
||||
c := config.FindContainer(task.ContainerID)
|
||||
if c != nil {
|
||||
if lxc.IsExpired(*c) {
|
||||
err = fmt.Errorf("容器已到期,不允许此操作")
|
||||
} else if lxc.IsTrafficExceeded(*c) {
|
||||
err = fmt.Errorf("容器流量已超限,不允许此操作")
|
||||
}
|
||||
}
|
||||
}
|
||||
if err == nil && isSecurityStopTask(task) && !config.AppConfig.SecurityAutoShutdown {
|
||||
skipped = true
|
||||
}
|
||||
if err == nil && !skipped {
|
||||
switch task.Type {
|
||||
case TaskStart:
|
||||
err = startByRuntime(task.ContainerID)
|
||||
case TaskStop:
|
||||
err = stopByRuntime(task.ContainerID)
|
||||
case TaskRestart:
|
||||
err = restartByRuntime(task.ContainerID)
|
||||
case TaskDelete:
|
||||
err = destroyByRuntime(task.ContainerID)
|
||||
if err == nil {
|
||||
time.Sleep(time.Second)
|
||||
if config.FindContainer(task.ContainerID) != nil {
|
||||
err = fmt.Errorf("container still exists after delete: %d", task.ContainerID)
|
||||
}
|
||||
}
|
||||
case TaskReinstall:
|
||||
if lxc.HasSSHAuthOptions(task.Config) {
|
||||
err = reinstallByRuntime(task.ContainerID, task.TemplateID, task.Config)
|
||||
} else {
|
||||
err = reinstallByRuntime(task.ContainerID, task.TemplateID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auditUser := task.User
|
||||
if auditUser == "" {
|
||||
auditUser = "admin"
|
||||
}
|
||||
if err != nil {
|
||||
config.AddAuditLogFull(string(task.Type), task.ContainerName, "失败: "+err.Error(), auditUser, task.IP, task.UserAgent, false, err.Error())
|
||||
q.finishTask(task, "failed", err)
|
||||
return
|
||||
}
|
||||
if skipped {
|
||||
config.AddAuditLogFull(string(task.Type), task.ContainerName, "跳过: 安全告警自动关机已关闭", auditUser, task.IP, task.UserAgent, true, "")
|
||||
q.finishTask(task, "done", nil)
|
||||
return
|
||||
}
|
||||
|
||||
config.AddAuditLogFull(string(task.Type), task.ContainerName, "成功", auditUser, task.IP, task.UserAgent, true, "")
|
||||
switch task.Type {
|
||||
case TaskStart:
|
||||
config.UpdateContainerStatus(task.ContainerID, "running")
|
||||
clearPolicyBlockAfterAdminRecovery(task)
|
||||
case TaskStop:
|
||||
config.UpdateContainerStatus(task.ContainerID, "stopped")
|
||||
case TaskRestart:
|
||||
config.UpdateContainerStatus(task.ContainerID, "running")
|
||||
clearPolicyBlockAfterAdminRecovery(task)
|
||||
case TaskReinstall:
|
||||
clearPolicyBlockAfterAdminRecovery(task)
|
||||
}
|
||||
q.finishTask(task, "done", nil)
|
||||
}
|
||||
|
||||
func isSecurityStopTask(task *Task) bool {
|
||||
return task != nil && task.Type == TaskStop && task.User == "system:security"
|
||||
}
|
||||
@@ -503,7 +630,8 @@ func (q *TaskQueue) GetTasks() []*Task {
|
||||
result := make([]*Task, 0, len(q.tasks))
|
||||
// Collect all task IDs, sort by creation time (extracted from ID number)
|
||||
for _, t := range q.tasks {
|
||||
result = append(result, t)
|
||||
copyTask := *t
|
||||
result = append(result, ©Task)
|
||||
}
|
||||
// Stable sort by ID number (task-N where N is sequential)
|
||||
for i := 0; i < len(result); i++ {
|
||||
@@ -660,6 +788,10 @@ func HandleBatchCreate(w http.ResponseWriter, r *http.Request) {
|
||||
if req.Containers[i].DiskGB < 1 {
|
||||
req.Containers[i].DiskGB = 5
|
||||
}
|
||||
if err := validateCreateStoragePool(&req.Containers[i]); err != nil {
|
||||
jsonResponse(w, http.StatusBadRequest, APIResponse{Success: false, Message: name + ": " + err.Error()})
|
||||
return
|
||||
}
|
||||
if !isImageEnabledAndDownloaded(req.Containers[i].TemplateID, req.Containers[i].Virtualization) {
|
||||
jsonResponse(w, http.StatusForbidden, APIResponse{Success: false, Message: name + ": template is not enabled or downloaded"})
|
||||
return
|
||||
@@ -879,6 +1011,8 @@ func HandleTasks(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// RestoreTasks restores task queue from config
|
||||
func RestoreTasks() {
|
||||
globalQueue.mu.Lock()
|
||||
defer globalQueue.mu.Unlock()
|
||||
for _, st := range config.AppConfig.Tasks {
|
||||
if st.Type == string(TaskStop) && st.User == "system:security" && !config.AppConfig.SecurityAutoShutdown {
|
||||
continue
|
||||
@@ -908,6 +1042,8 @@ func RestoreTasks() {
|
||||
ContainerName: containerName,
|
||||
Status: st.Status,
|
||||
Error: st.Error,
|
||||
Stage: "queued",
|
||||
StageDetail: "排队等待",
|
||||
CreatedAt: st.CreatedAt,
|
||||
TemplateID: st.TemplateID,
|
||||
Config: cfg,
|
||||
@@ -937,3 +1073,17 @@ func parseIDNum(id string) int {
|
||||
}
|
||||
return num
|
||||
}
|
||||
|
||||
func validateCreateStoragePool(cfg *lxc.ContainerConfig) error {
|
||||
required := config.StorageContentLXC
|
||||
if cfg.Virtualization == config.VirtualizationKVM {
|
||||
required = config.StorageContentKVM
|
||||
}
|
||||
requiredBytes := int64(cfg.DiskGB) * 1024 * 1024 * 1024
|
||||
pool, err := config.SelectStoragePoolForContent(required, cfg.StoragePoolID, requiredBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.StoragePoolID = pool.ID
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"clicd/internal/config"
|
||||
"clicd/internal/lxc"
|
||||
)
|
||||
|
||||
func TestRunnableTaskIndexSkipsActiveContainer(t *testing.T) {
|
||||
queue := []*Task{
|
||||
{ID: "task-1", Type: TaskStop, ContainerID: 1, ContainerName: "alpha"},
|
||||
{ID: "task-2", Type: TaskStart, ContainerID: 1, ContainerName: "alpha"},
|
||||
{ID: "task-3", Type: TaskStart, ContainerID: 2, ContainerName: "beta"},
|
||||
}
|
||||
active := map[string]bool{taskConcurrencyKey(queue[0]): true}
|
||||
|
||||
if got := runnableTaskIndex(queue[1:], active); got != 1 {
|
||||
t.Fatalf("runnableTaskIndex() = %d, want 1 for the other container", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskConcurrencyKeyUsesContainerName(t *testing.T) {
|
||||
create := &Task{ID: "task-1", Type: TaskCreate, Config: lxcConfigWithName("Example")}
|
||||
operation := &Task{ID: "task-2", Type: TaskDelete, ContainerID: 9, ContainerName: "example"}
|
||||
if taskConcurrencyKey(create) != taskConcurrencyKey(operation) {
|
||||
t.Fatalf("same container received different concurrency keys: %q and %q", taskConcurrencyKey(create), taskConcurrencyKey(operation))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskQueueSetConcurrencyNormalizesAndReports(t *testing.T) {
|
||||
q := newTaskQueue(config.DefaultTaskConcurrency)
|
||||
q.SetConcurrency(config.MaxTaskConcurrency + 10)
|
||||
if got := q.Settings().Concurrency; got != config.MaxTaskConcurrency {
|
||||
t.Fatalf("concurrency = %d, want %d", got, config.MaxTaskConcurrency)
|
||||
}
|
||||
q.SetConcurrency(0)
|
||||
if got := q.Settings().Concurrency; got != config.DefaultTaskConcurrency {
|
||||
t.Fatalf("concurrency = %d, want default %d", got, config.DefaultTaskConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskQueueUpdateTaskStage(t *testing.T) {
|
||||
q := newTaskQueue(config.DefaultTaskConcurrency)
|
||||
task := &Task{ID: "task-1", Type: TaskCreate, Status: "running"}
|
||||
|
||||
q.updateTaskStage(task, "rootfs", "下载模板并创建基础文件系统")
|
||||
|
||||
if task.Stage != "rootfs" || task.StageDetail != "下载模板并创建基础文件系统" {
|
||||
t.Fatalf("unexpected task stage: %q %q", task.Stage, task.StageDetail)
|
||||
}
|
||||
}
|
||||
|
||||
func lxcConfigWithName(name string) lxc.ContainerConfig {
|
||||
return lxc.ContainerConfig{Name: name}
|
||||
}
|
||||
Reference in New Issue
Block a user