Add custom image handling and access policy management

- Implement tests for custom KVM and LXC image creation, ensuring invalid sources and architecture mismatches are rejected.
- Introduce access policy management in CLI, allowing configuration of allowed sources and trusted proxies.
- Add NAT network configuration with validation for RFC1918 compliance and subnet parsing.
- Create panel access policy management, including normalization and evaluation of access decisions based on client IPs and forwarded headers.
- Develop middleware for enforcing access policies in the server, returning appropriate responses for allowed and denied requests.
- Enhance custom image downloading and validation, ensuring integrity and security of downloaded root filesystem archives.
- Include comprehensive tests for all new functionalities to ensure reliability and correctness.
This commit is contained in:
MengMengCode
2026-07-26 04:04:45 +08:00
parent 8283b88ded
commit 38debab1aa
49 changed files with 4504 additions and 246 deletions
+310
View File
@@ -0,0 +1,310 @@
package lxc
import (
"bufio"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"time"
)
type CustomImageDownloadProgress struct {
Stage string
DownloadedBytes int64
TotalBytes int64
Percent int
}
type CustomImageDownloadProgressFunc func(CustomImageDownloadProgress)
func CustomImagePath(id string) string {
template := FindTemplate(id)
if template == nil || !template.Custom {
return filepath.Join("/var/cache/lxc/download/custom", "__invalid_image_id__", "rootfs.tar")
}
return filepath.Join("/var/cache/lxc/download/custom", template.ID, "rootfs.tar")
}
func CustomImageDownloadedInfo(id string) (bool, int64) {
info, err := os.Stat(CustomImagePath(id))
if err != nil || info.IsDir() {
return false, 0
}
return true, info.Size()
}
func DeleteCustomImage(id string) error {
template := FindTemplate(id)
if template == nil || !template.Custom {
return fmt.Errorf("custom LXC image not found")
}
return os.RemoveAll(filepath.Dir(CustomImagePath(id)))
}
func DownloadCustomImageWithProgress(ctx context.Context, template Template, progress CustomImageDownloadProgressFunc) error {
if !template.Custom {
return fmt.Errorf("template is not a custom LXC image")
}
target := CustomImagePath(template.ID)
if ok, _ := CustomImageDownloadedInfo(template.ID); ok {
return nil
}
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
return err
}
tmp := target + ".tmp"
_ = os.Remove(tmp)
if err := downloadCustomRootfs(ctx, template.URL, tmp, progress); err != nil {
_ = os.Remove(tmp)
return err
}
if err := ctx.Err(); err != nil {
_ = os.Remove(tmp)
return err
}
if template.SHA256 != "" {
if err := verifyCustomRootfsSHA256(tmp, template.SHA256); err != nil {
_ = os.Remove(tmp)
return err
}
}
if progress != nil {
progress(CustomImageDownloadProgress{Stage: "validating", Percent: 100})
}
if err := ValidateCustomRootfsArchive(tmp); err != nil {
_ = os.Remove(tmp)
return err
}
if err := os.Rename(tmp, target); err != nil {
_ = os.Remove(tmp)
return err
}
return os.Chmod(target, 0644)
}
func downloadCustomRootfs(ctx context.Context, sourceURL, target string, progress CustomImageDownloadProgressFunc) error {
client := http.Client{
Timeout: 30 * time.Minute,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return fmt.Errorf("too many redirects")
}
return nil
},
}
request, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil)
if err != nil {
return err
}
request.Header.Set("User-Agent", "CLICD/1.0 LXC image downloader")
response, err := client.Do(request)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode < 200 || response.StatusCode >= 300 {
return fmt.Errorf("download failed: %s", response.Status)
}
file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
return err
}
defer file.Close()
total := response.ContentLength
buffer := make([]byte, 128*1024)
var downloaded int64
for {
count, readErr := response.Body.Read(buffer)
if count > 0 {
if _, err := file.Write(buffer[:count]); err != nil {
return err
}
downloaded += int64(count)
if progress != nil {
percent := 0
if total > 0 {
percent = int(downloaded * 100 / total)
if percent > 100 {
percent = 100
}
}
progress(CustomImageDownloadProgress{
Stage: "downloading",
DownloadedBytes: downloaded,
TotalBytes: total,
Percent: percent,
})
}
}
if readErr == io.EOF {
break
}
if readErr != nil {
return readErr
}
}
return file.Sync()
}
func verifyCustomRootfsSHA256(filePath, expected string) error {
file, err := os.Open(filePath)
if err != nil {
return err
}
defer file.Close()
hash := sha256.New()
if _, err := io.Copy(hash, file); err != nil {
return err
}
actual := hex.EncodeToString(hash.Sum(nil))
if !strings.EqualFold(actual, strings.TrimSpace(expected)) {
return fmt.Errorf("SHA-256 mismatch: expected %s, got %s", expected, actual)
}
return nil
}
func ValidateCustomRootfsArchive(archivePath string) error {
command := exec.Command("tar", "-tf", archivePath)
stdout, err := command.StdoutPipe()
if err != nil {
return err
}
var stderr strings.Builder
command.Stderr = &stderr
if err := command.Start(); err != nil {
return fmt.Errorf("failed to inspect rootfs archive: %v", err)
}
scanner := bufio.NewScanner(stdout)
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
entries := make([]string, 0, 4096)
for scanner.Scan() {
if len(entries) >= 2_000_000 {
_ = command.Process.Kill()
return fmt.Errorf("rootfs archive contains too many entries")
}
entries = append(entries, scanner.Text())
}
scanErr := scanner.Err()
waitErr := command.Wait()
if scanErr != nil {
return fmt.Errorf("failed to read rootfs archive: %v", scanErr)
}
if waitErr != nil {
return fmt.Errorf("invalid rootfs archive: %v, output: %s", waitErr, strings.TrimSpace(stderr.String()))
}
return validateCustomRootfsEntries(entries)
}
func validateCustomRootfsEntries(entries []string) error {
hasInit := false
for _, entry := range entries {
entry = strings.TrimSpace(strings.ReplaceAll(entry, "\\", "/"))
entry = strings.TrimPrefix(entry, "./")
if entry == "" || entry == "." {
continue
}
if strings.HasPrefix(entry, "/") {
return fmt.Errorf("rootfs archive contains an absolute path: %s", entry)
}
clean := path.Clean(entry)
if clean == ".." || strings.HasPrefix(clean, "../") {
return fmt.Errorf("rootfs archive contains path traversal: %s", entry)
}
switch strings.TrimSuffix(clean, "/") {
case "sbin/init", "usr/lib/systemd/systemd", "lib/systemd/systemd", "bin/busybox", "bin/sh":
hasInit = true
}
}
if len(entries) == 0 {
return fmt.Errorf("rootfs archive is empty")
}
if !hasInit {
return fmt.Errorf("rootfs archive does not contain a supported init")
}
return nil
}
func ExtractCustomRootfs(templateID, destination string) error {
template := FindTemplate(templateID)
if template == nil || !template.Custom {
return fmt.Errorf("custom LXC image not found: %s", templateID)
}
archive := CustomImagePath(template.ID)
if ok, _ := CustomImageDownloadedInfo(template.ID); !ok {
return fmt.Errorf("custom LXC image is not downloaded: %s", templateID)
}
if err := ValidateCustomRootfsArchive(archive); err != nil {
return err
}
if err := os.MkdirAll(destination, 0755); err != nil {
return err
}
output, err := exec.Command("tar", "-xpf", archive, "-C", destination).CombinedOutput()
if err != nil {
return fmt.Errorf("failed to extract custom LXC rootfs: %v, output: %s", err, strings.TrimSpace(string(output)))
}
if err := secureExtractedRootfs(destination); err != nil {
return err
}
if !rootfsHasInit(destination) {
return fmt.Errorf("extracted custom LXC rootfs is invalid: init not found")
}
return nil
}
func secureExtractedRootfs(root string) error {
root, err := filepath.Abs(root)
if err != nil {
return err
}
return filepath.WalkDir(root, func(filePath string, entry os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
info, err := entry.Info()
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink == 0 {
return nil
}
target, err := os.Readlink(filePath)
if err != nil {
return err
}
var resolved string
if filepath.IsAbs(target) {
resolved = filepath.Join(root, strings.TrimLeft(filepath.ToSlash(target), "/"))
relative, err := filepath.Rel(filepath.Dir(filePath), resolved)
if err != nil {
return err
}
if err := os.Remove(filePath); err != nil {
return err
}
if err := os.Symlink(relative, filePath); err != nil {
return err
}
} else {
resolved = filepath.Join(filepath.Dir(filePath), target)
}
relativeToRoot, err := filepath.Rel(root, filepath.Clean(resolved))
if err != nil {
return err
}
if relativeToRoot == ".." || strings.HasPrefix(relativeToRoot, ".."+string(os.PathSeparator)) {
return fmt.Errorf("rootfs symlink escapes the archive root: %s -> %s", filePath, target)
}
return nil
})
}
@@ -0,0 +1,61 @@
package lxc
import (
"path/filepath"
"runtime"
"testing"
"clicd/internal/config"
)
func TestGetTemplatesIncludesHostArchitectureCustomLXCImage(t *testing.T) {
previous := config.AppConfig
t.Cleanup(func() { config.AppConfig = previous })
config.AppConfig = &config.ClicdConfig{CustomLXCImages: []config.CustomLXCImage{
{
ID: "custom-lxc-host", Name: "Host Rootfs", Distro: "alpine",
Release: "3.21", Arch: runtime.GOARCH, URL: "https://example.test/rootfs.tar.xz",
},
{
ID: "custom-lxc-other", Name: "Other Rootfs", Distro: "alpine",
Release: "3.21", Arch: "not-" + runtime.GOARCH, URL: "https://example.test/other.tar.xz",
},
}}
template := FindTemplate("custom-lxc-host")
if template == nil || !template.Custom || template.URL == "" {
t.Fatalf("custom LXC template was not exposed correctly: %+v", template)
}
if FindTemplate("custom-lxc-other") != nil {
t.Fatal("custom LXC template for another architecture was exposed")
}
}
func TestCustomImagePathUsesAllowlistedID(t *testing.T) {
previous := config.AppConfig
t.Cleanup(func() { config.AppConfig = previous })
config.AppConfig = &config.ClicdConfig{}
for _, id := range []string{"", ".", "..", "../../etc/passwd", "/absolute", "unknown"} {
got := filepath.ToSlash(CustomImagePath(id))
if filepath.Base(filepath.Dir(got)) != "__invalid_image_id__" {
t.Fatalf("CustomImagePath(%q) = %q", id, got)
}
}
}
func TestValidateCustomRootfsEntries(t *testing.T) {
if err := validateCustomRootfsEntries([]string{"./etc/", "./bin/", "./bin/sh"}); err != nil {
t.Fatalf("valid rootfs entries failed: %v", err)
}
for _, entries := range [][]string{
{},
{"etc/passwd"},
{"/etc/passwd", "bin/sh"},
{"../../etc/passwd", "bin/sh"},
} {
if err := validateCustomRootfsEntries(entries); err == nil {
t.Fatalf("unsafe rootfs entries unexpectedly passed: %#v", entries)
}
}
}
+89 -26
View File
@@ -503,15 +503,30 @@ func (m *Manager) CreateContainer(cfg ContainerConfig) error {
lxcName, id, tmpl.Distro, tmpl.Release, tmpl.Arch)
cfg.ReportProgress("rootfs", "下载模板并创建基础文件系统")
args := []string{"-n", lxcName, "-t", "download", "--",
"-d", tmpl.Distro, "-r", tmpl.Release, "-a", tmpl.Arch}
if tmpl.Variant != "" {
args = append(args, "--variant", tmpl.Variant)
}
cmd := exec.Command("lxc-create", args...)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("lxc-create failed: %v, output: %s", err, string(output))
if tmpl.Custom {
output, err := exec.Command("lxc-create", "-n", lxcName, "-t", "none").CombinedOutput()
if err != nil {
return fmt.Errorf("lxc-create failed for custom rootfs: %v, output: %s", err, string(output))
}
if err := m.configureCustomLXCBase(lxcName, tmpl); err != nil {
_ = m.cleanupContainerStorage(lxcName)
return err
}
if err := ExtractCustomRootfs(tmpl.ID, filepath.Join(containerDir, "rootfs")); err != nil {
_ = m.cleanupContainerStorage(lxcName)
return err
}
} else {
args := []string{"-n", lxcName, "-t", "download", "--",
"-d", tmpl.Distro, "-r", tmpl.Release, "-a", tmpl.Arch}
if tmpl.Variant != "" {
args = append(args, "--variant", tmpl.Variant)
}
cmd := exec.Command("lxc-create", args...)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("lxc-create failed: %v, output: %s", err, string(output))
}
}
cfg.ReportProgress("storage", "复制容器数据到存储磁盘")
@@ -676,6 +691,41 @@ func (m *Manager) CreateContainer(cfg ContainerConfig) error {
return nil
}
func (m *Manager) configureCustomLXCBase(lxcName string, tmpl *Template) error {
rootfsPath, err := m.safeRootfsPath(filepath.Join(m.LxcPath, lxcName, "rootfs"))
if err != nil {
return fmt.Errorf("invalid custom LXC rootfs path: %v", err)
}
configFile := filepath.Join(filepath.Dir(rootfsPath), "config")
data, err := os.ReadFile(configFile)
if err != nil {
return fmt.Errorf("failed to read custom LXC base config: %v", err)
}
if _, err := os.Stat("/usr/share/lxc/config/common.conf"); err != nil {
return fmt.Errorf("LXC common configuration is unavailable: %v", err)
}
arch := "linux64"
switch strings.ToLower(strings.TrimSpace(tmpl.Arch)) {
case "amd64", "x86_64", "arm64", "aarch64":
default:
return fmt.Errorf("unsupported custom LXC architecture: %s", tmpl.Arch)
}
base := []string{
"# CLICD custom rootfs base configuration",
"lxc.include = /usr/share/lxc/config/common.conf",
"lxc.arch = " + arch,
"lxc.rootfs.path = dir:" + rootfsPath,
"lxc.uts.name = " + lxcName,
"",
}
if err := os.WriteFile(configFile, []byte(strings.Join(base, "\n")+string(data)), 0644); err != nil {
return fmt.Errorf("failed to write custom LXC base config: %v", err)
}
return nil
}
func (m *Manager) preconfigureNetwork(rootfsPath string, cfg ContainerConfig) {
templateID := cfg.TemplateID
osRelease := ""
@@ -1667,6 +1717,9 @@ func appArmorProfileForTemplate(templateID string) (string, error) {
func systemdTemplateNeedsUnconfinedAppArmor(templateID string) bool {
id := strings.ToLower(strings.TrimSpace(templateID))
if template := FindTemplate(templateID); template != nil {
id += " " + strings.ToLower(template.Distro+" "+template.Release)
}
if id == "" || strings.Contains(id, "alpine") {
return false
}
@@ -2564,7 +2617,7 @@ if [ -L /etc/resolv.conf ] 2>/dev/null; then
fi
# Also try resolvectl for systemd-resolved setups
if command -v resolvectl >/dev/null 2>&1; then
resolvectl dns eth0 10.0.3.1 2>/dev/null || true
resolvectl dns eth0 __CLICD_LXC_GATEWAY__ 2>/dev/null || true
resolvectl dns eth0 8.8.8.8 2>/dev/null || true
resolvectl domain eth0 '~.' 2>/dev/null || true
fi
@@ -2572,7 +2625,7 @@ fi
# Avoid the trap where systemd stub resolver puts "nameserver 127.0.0.53"
# but doesn't actually resolve anything.
if ! grep -q '^nameserver [1-9]' /etc/resolv.conf 2>/dev/null; then
echo "nameserver 10.0.3.1" > /etc/resolv.conf
echo "nameserver __CLICD_LXC_GATEWAY__" > /etc/resolv.conf
echo "nameserver 8.8.8.8" >> /etc/resolv.conf
fi
export DEBIAN_FRONTEND=noninteractive
@@ -2709,6 +2762,7 @@ ensure_sshd_runtime_dir
}
`
script = strings.ReplaceAll(script, "__CLICD_PUBKEY_AUTH__", pubkeyValue)
script = strings.ReplaceAll(script, "__CLICD_LXC_GATEWAY__", config.LXCNATNetwork().Gateway)
if !startService {
return script
}
@@ -3308,23 +3362,32 @@ func (m *Manager) replaceRootfsFromTemplate(lxcName string, tmpl *Template) erro
}
defer m.cleanupTemporaryContainer(tmpName)
args := []string{
"-n", tmpName,
"-t", "download",
"--",
"-d", tmpl.Distro,
"-r", tmpl.Release,
"-a", tmpl.Arch,
}
if tmpl.Variant != "" {
args = append(args, "--variant", tmpl.Variant)
}
output, err := exec.Command("lxc-create", args...).CombinedOutput()
if err != nil {
return fmt.Errorf("failed to download replacement rootfs: %v, output: %s", err, string(output))
tmpRootfs := filepath.Join(tmpDir, "rootfs")
if tmpl.Custom {
if err := os.MkdirAll(tmpRootfs, 0755); err != nil {
return err
}
if err := ExtractCustomRootfs(tmpl.ID, tmpRootfs); err != nil {
return err
}
} else {
args := []string{
"-n", tmpName,
"-t", "download",
"--",
"-d", tmpl.Distro,
"-r", tmpl.Release,
"-a", tmpl.Arch,
}
if tmpl.Variant != "" {
args = append(args, "--variant", tmpl.Variant)
}
output, err := exec.Command("lxc-create", args...).CombinedOutput()
if err != nil {
return fmt.Errorf("failed to download replacement rootfs: %v, output: %s", err, string(output))
}
}
tmpRootfs := filepath.Join(tmpDir, "rootfs")
if !rootfsHasInit(tmpRootfs) {
return fmt.Errorf("downloaded replacement rootfs is invalid: init not found")
}
+177
View File
@@ -122,6 +122,78 @@ func TestNormalizeCreateNATMappingsRejectsManagementPortConflict(t *testing.T) {
}
}
func TestTaggedRuleLineNumbersReturnsMatchingRulesDescending(t *testing.T) {
output := []byte(`Chain PREROUTING (policy ACCEPT)
num target prot opt source destination
2 DNAT tcp -- 0.0.0.0/0 0.0.0.0/0 tcp dpt:30080 /* clicd-c12-any-30080 */
7 DNAT tcp -- 0.0.0.0/0 0.0.0.0/0 tcp dpt:30081 /* clicd-c13-any-30081 */
11 DNAT tcp -- 0.0.0.0/0 0.0.0.0/0 tcp dpt:30082 /* clicd-c12-any-30082 */
`)
got := taggedRuleLineNumbers(output, "clicd-c12-")
want := []int{11, 2}
if !reflect.DeepEqual(got, want) {
t.Fatalf("taggedRuleLineNumbers() = %v, want %v", got, want)
}
}
func TestPortMappingConntrackDeleteArgs(t *testing.T) {
got := portMappingConntrackDeleteArgs(config.PortMapping{
HostIP: "203.0.113.10",
HostPort: 32022,
Protocol: "TCP",
})
want := []string{"-D", "-p", "tcp", "--dport", "32022", "--dst", "203.0.113.10"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("portMappingConntrackDeleteArgs() = %v, want %v", got, want)
}
if got := portMappingConntrackDeleteArgs(config.PortMapping{HostPort: 32022, Protocol: "icmp"}); got != nil {
t.Fatalf("unsupported protocol returned args: %v", got)
}
}
func TestUpdateSSHPortMappingKeepsIdentityAndSynchronizesSSHPort(t *testing.T) {
previous := config.AppConfig
t.Cleanup(func() { config.AppConfig = previous })
config.AppConfig = &config.ClicdConfig{
NATPortStart: 30000,
NATPortEnd: 65535,
Containers: []config.Container{{
ID: 12,
Name: "ct-test",
Status: "stopped",
SSHPort: 30022,
PortMappings: []config.PortMapping{{
HostPort: 30022,
ContainerPort: 22,
Protocol: "tcp",
Description: "SSH",
}},
}},
}
manager := NewManager()
mappings, err := manager.UpdatePortMapping(12, 0, config.PortMapping{
HostPort: 31022,
ContainerPort: 22,
Protocol: "tcp",
Description: "renamed",
})
if err != nil {
t.Fatal(err)
}
if len(mappings) != 1 || mappings[0].Description != "SSH" {
t.Fatalf("updated mappings = %+v", mappings)
}
container := config.FindContainer(12)
if container == nil || container.SSHPort != 31022 {
t.Fatalf("container after SSH update = %+v", container)
}
if _, err := manager.DeletePortMapping(12, 0); err == nil {
t.Fatal("updated SSH mapping became deletable")
}
}
func TestReserveCreateNATPortsProtectsConcurrentTasks(t *testing.T) {
previous := config.AppConfig
t.Cleanup(func() { config.AppConfig = previous })
@@ -133,10 +205,12 @@ func TestReserveCreateNATPortsProtectsConcurrentTasks(t *testing.T) {
createNATReservationMu.Lock()
createNATReservations = map[uint64][]config.PortMapping{}
queuedCreateNATReservations = map[string][]config.PortMapping{}
createNATReservationMu.Unlock()
t.Cleanup(func() {
createNATReservationMu.Lock()
createNATReservations = map[uint64][]config.PortMapping{}
queuedCreateNATReservations = map[string][]config.PortMapping{}
createNATReservationMu.Unlock()
})
@@ -181,6 +255,109 @@ func TestReserveCreateNATPortsProtectsConcurrentTasks(t *testing.T) {
}
}
func TestReserveBatchCreateNATPortsPlansAllAutomaticPorts(t *testing.T) {
previous := config.AppConfig
t.Cleanup(func() { config.AppConfig = previous })
config.AppConfig = &config.ClicdConfig{
NATPortStart: 30000,
NATPortEnd: 30010,
NextSSHPort: 30001,
}
createNATReservationMu.Lock()
createNATReservations = map[uint64][]config.PortMapping{}
queuedCreateNATReservations = map[string][]config.PortMapping{}
createNATReservationMu.Unlock()
t.Cleanup(func() {
createNATReservationMu.Lock()
createNATReservations = map[uint64][]config.PortMapping{}
queuedCreateNATReservations = map[string][]config.PortMapping{}
createNATReservationMu.Unlock()
})
configs := []ContainerConfig{
{Name: "batch-1", PortMappingCount: 2},
{Name: "batch-2", PortMappingCount: 2},
}
for i := range configs {
if err := configs[i].NormalizeCreateNATMappings(); err != nil {
t.Fatal(err)
}
}
planned, err := ReserveBatchCreateNATPorts(configs)
if err != nil {
t.Fatal(err)
}
used := map[int]string{}
for _, cfg := range planned {
if cfg.ManagementPort == 0 {
t.Fatalf("%s has no planned management port", cfg.Name)
}
if len(cfg.NATPortMappings) != 1 {
t.Fatalf("%s automatic mappings = %d, want 1", cfg.Name, len(cfg.NATPortMappings))
}
for _, port := range []int{cfg.ManagementPort, cfg.NATPortMappings[0].HostPort} {
if owner := used[port]; owner != "" {
t.Fatalf("planned port %d is shared by %s and %s", port, owner, cfg.Name)
}
used[port] = cfg.Name
}
}
for _, cfg := range planned {
port, release, err := ReserveCreateNATPorts(cfg)
if err != nil {
t.Fatalf("%s could not claim its queued reservation: %v", cfg.Name, err)
}
if port != cfg.ManagementPort {
t.Fatalf("%s claimed management port %d, want %d", cfg.Name, port, cfg.ManagementPort)
}
release()
}
if len(queuedCreateNATReservations) != 0 {
t.Fatalf("queued reservations remain after claim: %v", queuedCreateNATReservations)
}
}
func TestReserveBatchCreateNATPortsRejectsWholeConflictingBatch(t *testing.T) {
previous := config.AppConfig
t.Cleanup(func() { config.AppConfig = previous })
config.AppConfig = &config.ClicdConfig{
NATPortStart: 30000,
NATPortEnd: 30010,
NextSSHPort: 30001,
}
createNATReservationMu.Lock()
createNATReservations = map[uint64][]config.PortMapping{}
queuedCreateNATReservations = map[string][]config.PortMapping{}
createNATReservationMu.Unlock()
t.Cleanup(func() {
createNATReservationMu.Lock()
createNATReservations = map[uint64][]config.PortMapping{}
queuedCreateNATReservations = map[string][]config.PortMapping{}
createNATReservationMu.Unlock()
})
configs := []ContainerConfig{
{Name: "batch-1", NATPortMappings: []config.PortMapping{{HostPort: 30005, ContainerPort: 80, Protocol: "tcp"}}},
{Name: "batch-2", NATPortMappings: []config.PortMapping{{HostPort: 30005, ContainerPort: 8080, Protocol: "tcp"}}},
}
for i := range configs {
if err := configs[i].NormalizeCreateNATMappings(); err != nil {
t.Fatal(err)
}
}
if _, err := ReserveBatchCreateNATPorts(configs); err == nil {
t.Fatal("conflicting batch was accepted")
}
if len(queuedCreateNATReservations) != 0 {
t.Fatalf("conflicting batch left partial reservations: %v", queuedCreateNATReservations)
}
}
func TestRootfsCommandRejectsUnmanagedCommand(t *testing.T) {
base := t.TempDir()
rootfs := filepath.Join(base, "ct-1", "rootfs")
+376 -40
View File
@@ -1,9 +1,12 @@
package lxc
import (
"errors"
"fmt"
"net/netip"
"os/exec"
"regexp"
"sort"
"strconv"
"strings"
"sync"
@@ -12,9 +15,10 @@ import (
)
var (
createNATReservationMu sync.Mutex
createNATReservationNextID uint64
createNATReservations = map[uint64][]config.PortMapping{}
createNATReservationMu sync.Mutex
createNATReservationNextID uint64
createNATReservations = map[uint64][]config.PortMapping{}
queuedCreateNATReservations = map[string][]config.PortMapping{}
)
// ApplyPortMappings applies iptables DNAT rules for a container's port mappings
@@ -29,14 +33,16 @@ func (m *Manager) ApplyPortMappings(id int) error {
EnsureAssignedPublicIPv4s(c.PublicIPv4s)
tag := clicdTag(id)
bridge := "lxcbr0"
subnet := "10.0.3.0/24"
subnet := config.LXCNATNetwork().Subnet
if c.IsKVM() {
bridge = "virbr0"
subnet = "192.168.122.0/24"
subnet = config.KVMNATNetwork().Subnet
}
EnsureForwardRules(bridge)
m.CleanPortMappings(id)
if err := m.CleanPortMappings(id); err != nil {
return fmt.Errorf("clean existing port mappings for container %d: %w", id, err)
}
deleteBridgeMasquerade(subnet)
for _, pm := range c.PortMappings {
@@ -243,9 +249,13 @@ func clicdTag(id int) string { return "c" + strconv.Itoa(id) }
func EnsureAllRunningPortMappings() {
m := NewManager()
m.cleanOrphanedPortMappings()
for i := range config.AppConfig.Containers {
c := &config.AppConfig.Containers[i]
if c.Status != "running" || strings.TrimSpace(c.IP) == "" {
if err := m.CleanPortMappings(c.ID); err != nil {
fmt.Printf("Warning: failed to clean inactive port mappings for %s: %v\n", c.Name, err)
}
continue
}
if err := m.ApplyPortMappings(c.ID); err != nil {
@@ -254,6 +264,33 @@ func EnsureAllRunningPortMappings() {
}
}
var taggedContainerIDPattern = regexp.MustCompile(`clicd-c([0-9]+)-`)
func (m *Manager) cleanOrphanedPortMappings() {
output, err := exec.Command("iptables-save").Output()
if err != nil {
return
}
configured := make(map[int]bool, len(config.AppConfig.Containers))
for i := range config.AppConfig.Containers {
configured[config.AppConfig.Containers[i].ID] = true
}
seen := map[int]bool{}
for _, match := range taggedContainerIDPattern.FindAllSubmatch(output, -1) {
if len(match) < 2 {
continue
}
id, err := strconv.Atoi(string(match[1]))
if err != nil || configured[id] || seen[id] {
continue
}
seen[id] = true
if err := m.CleanPortMappings(id); err != nil {
fmt.Printf("Warning: failed to clean orphaned port mappings for container %d: %v\n", id, err)
}
}
}
// EnsureForwardRules makes sure iptables FORWARD chain allows bridge traffic.
func EnsureForwardRules(bridge string) {
if bridge == "" {
@@ -306,16 +343,108 @@ func ensureLibvirtForwardRules(bridge string) {
// CleanPortMappings removes all iptables rules for a container
func (m *Manager) CleanPortMappings(id int) error {
tag := clicdTag(id)
for _, chain := range []string{"PREROUTING", "POSTROUTING"} {
cmd := exec.Command("sh", "-c",
fmt.Sprintf("iptables -t nat -L %s -n --line-numbers 2>/dev/null | grep 'clicd-%s-' | awk '{print $1}' | sort -rn | while read num; do iptables -t nat -D %s $num; done", chain, tag, chain))
cmd.Run()
marker := "clicd-" + clicdTag(id) + "-"
var cleanupErrors []error
for _, target := range []struct {
table string
chain string
}{
{table: "nat", chain: "PREROUTING"},
{table: "nat", chain: "POSTROUTING"},
{chain: "FORWARD"},
} {
if err := deleteTaggedIPTablesRules(target.table, target.chain, marker); err != nil {
cleanupErrors = append(cleanupErrors, err)
}
}
cmd := exec.Command("sh", "-c",
fmt.Sprintf("iptables -S FORWARD 2>/dev/null | grep 'clicd-%s-' | sed 's/^-A /-D /' | while read rule; do iptables $rule; done", tag))
cmd.Run()
return nil
if c := config.FindContainer(id); c != nil {
for _, mapping := range c.PortMappings {
clearPortMappingConntrack(mapping)
}
}
return errors.Join(cleanupErrors...)
}
func deleteTaggedIPTablesRules(table, chain, marker string) error {
listArgs := []string{"-w", "5"}
if table != "" {
listArgs = append(listArgs, "-t", table)
}
listArgs = append(listArgs, "-L", chain, "-n", "--line-numbers")
output, err := exec.Command("iptables", listArgs...).CombinedOutput()
if err != nil {
return fmt.Errorf("list iptables %s/%s: %w: %s", tableName(table), chain, err, strings.TrimSpace(string(output)))
}
var deleteErrors []error
for _, lineNumber := range taggedRuleLineNumbers(output, marker) {
deleteArgs := []string{"-w", "5"}
if table != "" {
deleteArgs = append(deleteArgs, "-t", table)
}
deleteArgs = append(deleteArgs, "-D", chain, strconv.Itoa(lineNumber))
if output, err := exec.Command("iptables", deleteArgs...).CombinedOutput(); err != nil {
deleteErrors = append(deleteErrors, fmt.Errorf(
"delete iptables %s/%s rule %d: %w: %s",
tableName(table), chain, lineNumber, err, strings.TrimSpace(string(output)),
))
}
}
return errors.Join(deleteErrors...)
}
func taggedRuleLineNumbers(output []byte, marker string) []int {
lineNumbers := make([]int, 0)
for _, line := range strings.Split(string(output), "\n") {
if !strings.Contains(line, marker) {
continue
}
fields := strings.Fields(line)
if len(fields) == 0 {
continue
}
lineNumber, err := strconv.Atoi(fields[0])
if err == nil && lineNumber > 0 {
lineNumbers = append(lineNumbers, lineNumber)
}
}
sort.Sort(sort.Reverse(sort.IntSlice(lineNumbers)))
return lineNumbers
}
func tableName(table string) string {
if table == "" {
return "filter"
}
return table
}
func clearPortMappingConntrack(mapping config.PortMapping) {
args := portMappingConntrackDeleteArgs(mapping)
if len(args) == 0 {
return
}
// conntrack exits non-zero when no matching flow exists; that is already clean.
_ = exec.Command("conntrack", args...).Run()
}
func portMappingConntrackDeleteArgs(mapping config.PortMapping) []string {
protocol := strings.ToLower(strings.TrimSpace(mapping.Protocol))
if protocol != "tcp" && protocol != "udp" {
return nil
}
if mapping.HostPort < 1 || mapping.HostPort > 65535 {
return nil
}
args := []string{
"-D",
"-p", protocol,
"--dport", strconv.Itoa(mapping.HostPort),
}
if hostIP := strings.TrimSpace(mapping.HostIP); hostIP != "" {
args = append(args, "--dst", hostIP)
}
return args
}
// SetupDefaultPortMappings creates default port mappings
@@ -368,14 +497,19 @@ func (m *Manager) UpdatePortMapping(id int, index int, pm config.PortMapping) ([
if index < 0 || index >= len(c.PortMappings) {
return nil, fmt.Errorf("invalid port mapping index: %d", index)
}
existing := c.PortMappings[index]
normalized, err := normalizePortMapping(c, index, pm)
if err != nil {
return nil, err
}
if strings.EqualFold(existing.Description, "SSH") {
normalized.Description = "SSH"
}
c.PortMappings[index] = normalized
if err := persistAndReloadMappings(m, c); err != nil {
return nil, err
}
clearPortMappingConntrack(existing)
return c.PortMappings, nil
}
@@ -388,17 +522,20 @@ func (m *Manager) DeletePortMapping(id int, index int) ([]config.PortMapping, er
if index < 0 || index >= len(c.PortMappings) {
return nil, fmt.Errorf("invalid port mapping index: %d", index)
}
if c.PortMappings[index].Description == "SSH" {
removed := c.PortMappings[index]
if strings.EqualFold(removed.Description, "SSH") {
return nil, fmt.Errorf("SSH default mapping cannot be deleted")
}
c.PortMappings = append(c.PortMappings[:index], c.PortMappings[index+1:]...)
if err := persistAndReloadMappings(m, c); err != nil {
return nil, err
}
clearPortMappingConntrack(removed)
return c.PortMappings, nil
}
func persistAndReloadMappings(m *Manager, c *config.Container) error {
syncContainerSSHPort(c)
config.SaveConfig()
if c.Status == "running" && c.IP != "" {
return m.ApplyPortMappings(c.ID)
@@ -406,6 +543,18 @@ func persistAndReloadMappings(m *Manager, c *config.Container) error {
return nil
}
func syncContainerSSHPort(c *config.Container) {
if c == nil {
return
}
for _, mapping := range c.PortMappings {
if strings.EqualFold(mapping.Description, "SSH") {
c.SSHPort = mapping.HostPort
return
}
}
}
func (m *Manager) UpdatePublicIPv4Assignments(id int, requested []string, count int, auto bool) (*config.Container, error) {
c := config.FindContainer(id)
if c == nil {
@@ -553,32 +702,25 @@ func ReserveCreateNATPorts(cfg ContainerConfig) (int, func(), error) {
createNATReservationMu.Lock()
defer createNATReservationMu.Unlock()
owner := createNATReservationOwner(cfg.Name)
requestedReservations := createNATReservationMappings(cfg, cfg.ManagementPort)
if queued, ok := queuedCreateNATReservations[owner]; ok {
if !sameCreateNATReservations(queued, requestedReservations) {
return 0, nil, fmt.Errorf("queued NAT port plan for %s no longer matches the create task", cfg.Name)
}
delete(queuedCreateNATReservations, owner)
return activateCreateNATReservationLocked(cfg.ManagementPort, queued)
}
if err := ValidateCreateNATPortAvailability(cfg); err != nil {
return 0, nil, err
}
requestedReservations := append([]config.PortMapping(nil), cfg.NATPortMappings...)
if cfg.ManagementPort > 0 {
requestedReservations = append(requestedReservations, config.PortMapping{
HostPort: cfg.ManagementPort,
Protocol: "tcp",
})
}
for _, requested := range requestedReservations {
for _, reservations := range createNATReservations {
for _, reserved := range reservations {
if requested.HostPort == reserved.HostPort && protocolsOverlap(requested.Protocol, reserved.Protocol) {
return 0, nil, fmt.Errorf("NAT host port %d/%s is reserved by another create task", requested.HostPort, requested.Protocol)
}
}
}
if err := validateCreateNATReservationsAvailableLocked(requestedReservations, owner); err != nil {
return 0, nil, err
}
excluded := cfg.RequestedNATHostPorts()
for _, reservations := range createNATReservations {
for _, reserved := range reservations {
excluded = append(excluded, reserved.HostPort)
}
}
excluded = append(excluded, allReservedCreateNATHostPortsLocked(owner)...)
managementPort := cfg.ManagementPort
if managementPort == 0 {
var err error
@@ -588,12 +730,96 @@ func ReserveCreateNATPorts(cfg ContainerConfig) (int, func(), error) {
}
}
reservations := createNATReservationMappings(cfg, managementPort)
return activateCreateNATReservationLocked(managementPort, reservations)
}
// ReserveBatchCreateNATPorts resolves every automatic NAT port and reserves
// the complete batch before any create task is enqueued.
func ReserveBatchCreateNATPorts(configs []ContainerConfig) ([]ContainerConfig, error) {
createNATReservationMu.Lock()
defer createNATReservationMu.Unlock()
planned := append([]ContainerConfig(nil), configs...)
addedOwners := make([]string, 0, len(planned))
rollback := func() {
for _, owner := range addedOwners {
delete(queuedCreateNATReservations, owner)
}
}
for i := range planned {
cfg := &planned[i]
cfg.NATPortMappings = append([]config.PortMapping(nil), cfg.NATPortMappings...)
if !cfg.WantsNAT() {
continue
}
owner := createNATReservationOwner(cfg.Name)
if owner == "" {
rollback()
return nil, fmt.Errorf("container name is required for NAT port reservation")
}
if _, exists := queuedCreateNATReservations[owner]; exists {
rollback()
return nil, fmt.Errorf("container creation already has reserved NAT ports: %s", cfg.Name)
}
if err := ValidateCreateNATPortAvailability(*cfg); err != nil {
rollback()
return nil, fmt.Errorf("%s: %w", cfg.Name, err)
}
explicit := createNATReservationMappings(*cfg, cfg.ManagementPort)
if err := validateCreateNATReservationsAvailableLocked(explicit, owner); err != nil {
rollback()
return nil, fmt.Errorf("%s: %w", cfg.Name, err)
}
excluded := cfg.RequestedNATHostPorts()
excluded = append(excluded, allReservedCreateNATHostPortsLocked(owner)...)
if cfg.ManagementPort == 0 {
port, err := config.AllocateSSHPortExcluding(excluded)
if err != nil {
rollback()
return nil, fmt.Errorf("%s: %w", cfg.Name, err)
}
cfg.ManagementPort = port
}
if len(cfg.NATPortMappings) == 0 && cfg.PortMappingCount > 1 {
generated, err := planDefaultCreateNATMappingsLocked(*cfg, cfg.PortMappingCount-1, owner)
if err != nil {
rollback()
return nil, fmt.Errorf("%s: %w", cfg.Name, err)
}
cfg.NATPortMappings = generated
cfg.PortMappingCount = len(generated) + 1
}
reservations := createNATReservationMappings(*cfg, cfg.ManagementPort)
if err := validateCreateNATReservationsAvailableLocked(reservations, owner); err != nil {
rollback()
return nil, fmt.Errorf("%s: %w", cfg.Name, err)
}
queuedCreateNATReservations[owner] = reservations
addedOwners = append(addedOwners, owner)
}
return planned, nil
}
func ReleaseQueuedCreateNATPorts(name string) {
owner := createNATReservationOwner(name)
if owner == "" {
return
}
createNATReservationMu.Lock()
delete(queuedCreateNATReservations, owner)
createNATReservationMu.Unlock()
}
func activateCreateNATReservationLocked(managementPort int, reservations []config.PortMapping) (int, func(), error) {
createNATReservationNextID++
reservationID := createNATReservationNextID
reservations := make([]config.PortMapping, 0, len(cfg.NATPortMappings)+1)
reservations = append(reservations, config.PortMapping{HostPort: managementPort, Protocol: "tcp"})
reservations = append(reservations, cfg.NATPortMappings...)
createNATReservations[reservationID] = reservations
createNATReservations[reservationID] = append([]config.PortMapping(nil), reservations...)
var once sync.Once
release := func() {
@@ -606,6 +832,116 @@ func ReserveCreateNATPorts(cfg ContainerConfig) (int, func(), error) {
return managementPort, release, nil
}
func createNATReservationOwner(name string) string {
return strings.ToLower(strings.TrimSpace(name))
}
func createNATReservationMappings(cfg ContainerConfig, managementPort int) []config.PortMapping {
reservations := make([]config.PortMapping, 0, len(cfg.NATPortMappings)+1)
if managementPort > 0 {
reservations = append(reservations, config.PortMapping{HostPort: managementPort, Protocol: "tcp"})
}
reservations = append(reservations, cfg.NATPortMappings...)
return reservations
}
func validateCreateNATReservationsAvailableLocked(requested []config.PortMapping, exceptOwner string) error {
for _, candidate := range requested {
for _, reservations := range createNATReservations {
if conflictingCreateNATReservation(candidate, reservations) {
return fmt.Errorf("NAT host port %d/%s is reserved by another create task", candidate.HostPort, candidate.Protocol)
}
}
for owner, reservations := range queuedCreateNATReservations {
if owner == exceptOwner {
continue
}
if conflictingCreateNATReservation(candidate, reservations) {
return fmt.Errorf("NAT host port %d/%s is reserved by queued create task %s", candidate.HostPort, candidate.Protocol, owner)
}
}
}
return nil
}
func conflictingCreateNATReservation(candidate config.PortMapping, reservations []config.PortMapping) bool {
for _, reserved := range reservations {
if candidate.HostPort == reserved.HostPort && protocolsOverlap(candidate.Protocol, reserved.Protocol) {
return true
}
}
return false
}
func allReservedCreateNATHostPortsLocked(exceptOwner string) []int {
ports := make([]int, 0)
for _, reservations := range createNATReservations {
for _, reserved := range reservations {
ports = append(ports, reserved.HostPort)
}
}
for owner, reservations := range queuedCreateNATReservations {
if owner == exceptOwner {
continue
}
for _, reserved := range reservations {
ports = append(ports, reserved.HostPort)
}
}
return ports
}
func planDefaultCreateNATMappingsLocked(cfg ContainerConfig, count int, owner string) ([]config.PortMapping, error) {
if count <= 0 {
return nil, nil
}
unavailable := map[int]bool{cfg.ManagementPort: true}
for _, port := range allReservedCreateNATHostPortsLocked(owner) {
unavailable[port] = true
}
for _, mapping := range cfg.NATPortMappings {
unavailable[mapping.HostPort] = true
}
candidate := &config.Container{ID: -1}
start, end := config.NATPortRange()
mappings := make([]config.PortMapping, 0, count)
for port := start; port <= end && len(mappings) < count; port++ {
if unavailable[port] || !HostPortAvailable(candidate, "", port, "tcp") {
continue
}
unavailable[port] = true
mappings = append(mappings, config.PortMapping{
HostPort: port,
ContainerPort: port,
Protocol: "tcp",
Description: fmt.Sprintf("Port-%d", port),
})
}
if len(mappings) != count {
return nil, fmt.Errorf("not enough free NAT4 host ports for %d automatic mappings", count)
}
return mappings, nil
}
func sameCreateNATReservations(left, right []config.PortMapping) bool {
if len(left) != len(right) {
return false
}
counts := make(map[string]int, len(left))
for _, mapping := range left {
counts[fmt.Sprintf("%d/%s", mapping.HostPort, strings.ToLower(mapping.Protocol))]++
}
for _, mapping := range right {
key := fmt.Sprintf("%d/%s", mapping.HostPort, strings.ToLower(mapping.Protocol))
if counts[key] == 0 {
return false
}
counts[key]--
}
return true
}
func allocateDefaultEqualPorts(c *config.Container, count int) []int {
if count <= 0 {
return nil
+26 -2
View File
@@ -1,6 +1,10 @@
package lxc
import "runtime"
import (
"runtime"
"clicd/internal/config"
)
// Template represents an LXC image template
type Template struct {
@@ -11,12 +15,15 @@ type Template struct {
Arch string `json:"arch"`
Variant string `json:"variant"`
Description string `json:"description"`
URL string `json:"url,omitempty"`
SHA256 string `json:"sha256,omitempty"`
Custom bool `json:"custom,omitempty"`
}
// GetTemplates returns available LXC image templates (only verified working ones)
func GetTemplates() []Template {
arch := defaultTemplateArch()
return []Template{
templates := []Template{
{
ID: "ubuntu-noble", Name: "Ubuntu 24.04",
Distro: "ubuntu", Release: "noble", Arch: arch,
@@ -68,6 +75,23 @@ func GetTemplates() []Template {
Description: "Rocky Linux 10",
},
}
for _, custom := range config.ListCustomLXCImages() {
if custom.Arch != arch {
continue
}
templates = append(templates, Template{
ID: custom.ID,
Name: custom.Name,
Distro: custom.Distro,
Release: custom.Release,
Arch: custom.Arch,
Description: custom.Description,
URL: custom.URL,
SHA256: custom.SHA256,
Custom: true,
})
}
return templates
}
func defaultTemplateArch() string {