first commit
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
/incudal-agent
|
||||
/dist/*
|
||||
*.test
|
||||
coverage.out
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
# Incudal Host Agent
|
||||
|
||||
宿主机 Agent 客户端实现。
|
||||
|
||||
当前阶段负责读取配置、向面板上报 HMAC 签名心跳,并按面板心跳响应执行自动升级。
|
||||
|
||||
## 配置
|
||||
|
||||
默认配置路径:
|
||||
|
||||
```bash
|
||||
/etc/incudal-agent/config.yaml
|
||||
```
|
||||
|
||||
示例:
|
||||
|
||||
```yaml
|
||||
panel_url: "https://idev.bitpd.com"
|
||||
agent_id: "agt_xxx"
|
||||
agent_secret: "ias_xxx"
|
||||
heartbeat_interval_seconds: 30
|
||||
request_timeout_seconds: 10
|
||||
```
|
||||
|
||||
也可以使用环境变量覆盖:
|
||||
|
||||
```bash
|
||||
INCUDAL_PANEL_URL=
|
||||
INCUDAL_AGENT_ID=
|
||||
INCUDAL_AGENT_SECRET=
|
||||
INCUDAL_HEARTBEAT_INTERVAL_SECONDS=
|
||||
INCUDAL_REQUEST_TIMEOUT_SECONDS=
|
||||
```
|
||||
|
||||
## 运行
|
||||
|
||||
单次心跳测试:
|
||||
|
||||
```bash
|
||||
go run ./cmd/incudal-agent -config ./config.example.yaml -once
|
||||
```
|
||||
|
||||
循环心跳:
|
||||
|
||||
```bash
|
||||
go run ./cmd/incudal-agent -config /etc/incudal-agent/config.yaml
|
||||
```
|
||||
|
||||
## 当前能力
|
||||
|
||||
- 读取 Agent 配置
|
||||
- 采集 CPU 数量和内存总量
|
||||
- 探测常见 Incus/LXD Unix socket
|
||||
- 生成 canonical JSON body hash
|
||||
- 生成 HMAC-SHA256 签名
|
||||
- 调用 `POST /api/agent/heartbeat`
|
||||
- 读取心跳响应中的 `upgrade` 指令并自动升级自身
|
||||
|
||||
当前不执行实例创建、销毁、启停等下发任务。
|
||||
|
||||
## 自动升级
|
||||
|
||||
面板会在 Agent 心跳响应中返回升级指令:
|
||||
|
||||
```json
|
||||
{
|
||||
"upgrade": {
|
||||
"available": true,
|
||||
"version": "v1.0.1",
|
||||
"url": "https://<panel>/api/agent/binary/incudal-agent-linux-amd64?v=v1.0.1",
|
||||
"sha256": "<sha256>",
|
||||
"gzip": false
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Agent 只接受当前 `panel_url` 同源下载地址。下载后先校验 SHA-256,再解包、写入临时文件、备份旧二进制、原子替换并执行 `systemctl restart incudal-agent`。
|
||||
|
||||
`-once` 单次心跳测试模式不会执行自动升级,避免安装前置检测阶段替换正在测试的二进制。
|
||||
|
||||
旧版本 Agent 不包含升级执行器,首次启用自动升级时仍需要通过面板安装命令或重新安装按钮部署一次新版 Agent;之后才会按心跳响应自动升级。
|
||||
|
||||
## Release 构建与发布
|
||||
|
||||
本地构建双架构二进制:
|
||||
|
||||
```bash
|
||||
cd ..
|
||||
bash agent/scripts/build-release.sh
|
||||
```
|
||||
|
||||
Agent 版本统一从 `agent/VERSION` 读取,格式固定为 `vMAJOR.MINOR.PATCH`。
|
||||
需要发布新版 Agent 时,先递增 `agent/VERSION`,再构建 release 产物。
|
||||
|
||||
产物:
|
||||
|
||||
```text
|
||||
agent/dist/incudal-agent-linux-amd64
|
||||
agent/dist/incudal-agent-linux-arm64
|
||||
agent/dist/manifest.json
|
||||
```
|
||||
|
||||
`agent/dist` 是本地临时构建目录,不再纳入 Git。正式发布由 GitHub Actions `Agent Build & Release` 完成。
|
||||
|
||||
推送中只要 `agent/VERSION` 发生变化,Actions 会读取该版本号,构建并发布 GitHub Release:
|
||||
|
||||
```text
|
||||
tag: agent-v0.0.1
|
||||
assets:
|
||||
incudal-agent-x86_64-v0.0.1
|
||||
incudal-agent-aarch64-v0.0.1
|
||||
```
|
||||
|
||||
面板运行时不会读取本地 `agent/dist`。它会从 GitHub Release 查询最新 Agent 版本,动态生成 `/api/agent/manifest.json`,并通过 `/api/agent/binary/*` 代理下载对应 Release 资产。
|
||||
|
||||
默认 GitHub Release 仓库为 `qwer-xyz/incudal_classic`。如果部署到 fork 或私有仓库,可设置:
|
||||
|
||||
```bash
|
||||
INCUDAL_AGENT_RELEASE_REPOSITORY="owner/repo"
|
||||
INCUDAL_AGENT_RELEASE_TOKEN="github_pat_xxx" # 私有仓库需要
|
||||
```
|
||||
|
||||
## 安装脚本
|
||||
|
||||
面板提供通用安装脚本:
|
||||
|
||||
```bash
|
||||
curl -fsSL "$PANEL_URL/api/agent/install.sh" | sudo env \
|
||||
INCUDAL_PANEL_URL="$PANEL_URL" \
|
||||
INCUDAL_AGENT_INSTALL_TOKEN="$AGENT_INSTALL_TOKEN" \
|
||||
INCUDAL_AGENT_BINARY_URL="$BINARY_URL" \
|
||||
bash
|
||||
```
|
||||
|
||||
`INCUDAL_AGENT_INSTALL_TOKEN` 由面板生成,30 分钟内有效且只能使用一次。
|
||||
安装脚本会调用 `/api/agent/install-config/:token` 拉取 `agent_id` 和 `agent_secret`。
|
||||
旧的 `INCUDAL_AGENT_ID` / `INCUDAL_AGENT_SECRET` 直传方式仍保留兼容。
|
||||
|
||||
安装脚本会:
|
||||
|
||||
- 下载 `incudal-agent` 二进制
|
||||
- 写入 `/etc/incudal-agent/config.yaml`
|
||||
- 写入 systemd service
|
||||
- 执行一次心跳测试
|
||||
- 启动或重启 `incudal-agent.service`
|
||||
|
||||
如果没有传入 `INCUDAL_AGENT_BINARY_URL`,默认从当前面板下载:
|
||||
|
||||
```text
|
||||
https://<panel>/api/agent/binary/incudal-agent-linux-amd64
|
||||
https://<panel>/api/agent/binary/incudal-agent-linux-arm64
|
||||
```
|
||||
|
||||
默认下载会先读取面板的 manifest:
|
||||
|
||||
```text
|
||||
https://<panel>/api/agent/manifest.json
|
||||
```
|
||||
|
||||
安装脚本会按当前 OS/ARCH 取出文件名和 SHA-256,下载后先校验摘要,再解包安装。
|
||||
|
||||
如果手动传入 `INCUDAL_AGENT_BINARY_URL`,可同时传入 `INCUDAL_AGENT_BINARY_SHA256` 开启校验;未传 SHA-256 时仍保留兼容安装,但会输出 warning。
|
||||
|
||||
安装脚本会先下载到临时文件,再原子替换 `/usr/local/bin/incudal-agent`。
|
||||
重复安装或升级时,会执行 `systemctl restart incudal-agent` 确保立即切换到最新二进制。
|
||||
|
||||
dry-run 验证:
|
||||
|
||||
```bash
|
||||
INCUDAL_AGENT_DRY_RUN=1 \
|
||||
INCUDAL_PANEL_URL="http://127.0.0.1:8888" \
|
||||
INCUDAL_AGENT_INSTALL_TOKEN="ait_testtoken_abcdefghijklmnopqrstuvwxyz123456" \
|
||||
bash server/templates/agent-install.sh
|
||||
```
|
||||
@@ -0,0 +1 @@
|
||||
v0.0.1
|
||||
@@ -0,0 +1,110 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"incudal-agent/internal/config"
|
||||
"incudal-agent/internal/panel"
|
||||
"incudal-agent/internal/report"
|
||||
"incudal-agent/internal/upgrade"
|
||||
)
|
||||
|
||||
var version = "dev"
|
||||
|
||||
func main() {
|
||||
configPath := flag.String("config", "/etc/incudal-agent/config.yaml", "agent config file")
|
||||
once := flag.Bool("once", false, "send one heartbeat and exit")
|
||||
flag.Parse()
|
||||
|
||||
cfg, err := config.Load(*configPath)
|
||||
if err != nil {
|
||||
log.Fatalf("load config: %v", err)
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
client := panel.New(cfg)
|
||||
if *once {
|
||||
if _, err := sendHeartbeat(ctx, client, cfg.HeartbeatIntervalSeconds); err != nil {
|
||||
log.Fatalf("heartbeat: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf("incudal-agent started: panel=%s interval=%s", cfg.PanelURL, cfg.HeartbeatInterval)
|
||||
upgradeRunner := upgrade.DefaultRunner(cfg)
|
||||
var upgradeInProgress atomic.Bool
|
||||
if result, err := sendHeartbeat(ctx, client, cfg.HeartbeatIntervalSeconds); err != nil {
|
||||
log.Printf("heartbeat failed: %v", err)
|
||||
} else {
|
||||
scheduleAgentUpgrade(ctx, upgradeRunner, result, &upgradeInProgress)
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(cfg.HeartbeatInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Printf("incudal-agent stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
if result, err := sendHeartbeat(ctx, client, cfg.HeartbeatIntervalSeconds); err != nil {
|
||||
log.Printf("heartbeat failed: %v", err)
|
||||
} else {
|
||||
scheduleAgentUpgrade(ctx, upgradeRunner, result, &upgradeInProgress)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sendHeartbeat(ctx context.Context, client *panel.Client, heartbeatIntervalSeconds int) (panel.HeartbeatResult, error) {
|
||||
result, err := client.Heartbeat(ctx, report.HeartbeatPayload(version, heartbeatIntervalSeconds))
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
upgradeAvailable := result.Upgrade != nil && result.Upgrade.Available
|
||||
log.Printf("heartbeat ok: status=%d latencyMs=%d upgrade=%t", result.StatusCode, result.LatencyMs, upgradeAvailable)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func scheduleAgentUpgrade(ctx context.Context, runner *upgrade.Runner, result panel.HeartbeatResult, upgradeInProgress *atomic.Bool) {
|
||||
if result.Upgrade == nil || !result.Upgrade.Available {
|
||||
return
|
||||
}
|
||||
if !upgradeInProgress.CompareAndSwap(false, true) {
|
||||
log.Printf("agent upgrade already scheduled: version=%s", result.Upgrade.Version)
|
||||
return
|
||||
}
|
||||
|
||||
instruction := *result.Upgrade
|
||||
log.Printf("agent upgrade scheduled: version=%s", instruction.Version)
|
||||
go func() {
|
||||
defer upgradeInProgress.Store(false)
|
||||
|
||||
if delay := upgrade.RandomJitter(5 * time.Minute); delay > 0 {
|
||||
timer := time.NewTimer(delay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
|
||||
upgradeCtx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
if err := runner.Apply(upgradeCtx, instruction, version); err != nil {
|
||||
log.Printf("agent upgrade failed: version=%s error=%v", instruction.Version, err)
|
||||
return
|
||||
}
|
||||
log.Printf("agent upgrade applied: version=%s", instruction.Version)
|
||||
}()
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
# Incudal Host Agent minimal config.
|
||||
# 首版仅支持简单 key: value 格式,不支持嵌套 YAML。
|
||||
|
||||
panel_url: "https://idev.bitpd.com"
|
||||
agent_id: "agt_replace_me"
|
||||
agent_secret: "ias_replace_me"
|
||||
heartbeat_interval_seconds: 30
|
||||
request_timeout_seconds: 10
|
||||
@@ -0,0 +1,3 @@
|
||||
module incudal-agent
|
||||
|
||||
go 1.19
|
||||
@@ -0,0 +1,158 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultHeartbeatIntervalSeconds = 30
|
||||
MinHeartbeatIntervalSeconds = 5
|
||||
MaxHeartbeatIntervalSeconds = 3600
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
PanelURL string
|
||||
AgentID string
|
||||
AgentSecret string
|
||||
HeartbeatInterval time.Duration
|
||||
RequestTimeout time.Duration
|
||||
HeartbeatIntervalSeconds int
|
||||
RequestTimeoutSeconds int
|
||||
}
|
||||
|
||||
func Load(path string) (Config, error) {
|
||||
values := map[string]string{}
|
||||
if path != "" {
|
||||
fileValues, err := readKeyValueFile(path)
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return Config{}, err
|
||||
}
|
||||
for key, value := range fileValues {
|
||||
values[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
overlayEnv(values, "panel_url", "INCUDAL_PANEL_URL")
|
||||
overlayEnv(values, "agent_id", "INCUDAL_AGENT_ID")
|
||||
overlayEnv(values, "agent_secret", "INCUDAL_AGENT_SECRET")
|
||||
overlayEnv(values, "heartbeat_interval_seconds", "INCUDAL_HEARTBEAT_INTERVAL_SECONDS")
|
||||
overlayEnv(values, "request_timeout_seconds", "INCUDAL_REQUEST_TIMEOUT_SECONDS")
|
||||
|
||||
heartbeatSeconds := clampInt(
|
||||
parsePositiveInt(values["heartbeat_interval_seconds"], DefaultHeartbeatIntervalSeconds),
|
||||
MinHeartbeatIntervalSeconds,
|
||||
MaxHeartbeatIntervalSeconds,
|
||||
)
|
||||
timeoutSeconds := parsePositiveInt(values["request_timeout_seconds"], 10)
|
||||
cfg := Config{
|
||||
PanelURL: strings.TrimRight(values["panel_url"], "/"),
|
||||
AgentID: values["agent_id"],
|
||||
AgentSecret: values["agent_secret"],
|
||||
HeartbeatIntervalSeconds: heartbeatSeconds,
|
||||
RequestTimeoutSeconds: timeoutSeconds,
|
||||
HeartbeatInterval: time.Duration(heartbeatSeconds) * time.Second,
|
||||
RequestTimeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func (cfg Config) Validate() error {
|
||||
if cfg.PanelURL == "" {
|
||||
return errors.New("panel_url is required")
|
||||
}
|
||||
parsed, err := url.Parse(cfg.PanelURL)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
return fmt.Errorf("panel_url is invalid: %s", cfg.PanelURL)
|
||||
}
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return fmt.Errorf("panel_url scheme must be http or https: %s", parsed.Scheme)
|
||||
}
|
||||
if cfg.AgentID == "" {
|
||||
return errors.New("agent_id is required")
|
||||
}
|
||||
if cfg.AgentSecret == "" {
|
||||
return errors.New("agent_secret is required")
|
||||
}
|
||||
if cfg.HeartbeatInterval < time.Duration(MinHeartbeatIntervalSeconds)*time.Second {
|
||||
return fmt.Errorf("heartbeat interval must be at least %d seconds", MinHeartbeatIntervalSeconds)
|
||||
}
|
||||
if cfg.RequestTimeout < time.Second {
|
||||
return errors.New("request timeout must be at least 1 second")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readKeyValueFile(path string) (map[string]string, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
values := map[string]string{}
|
||||
scanner := bufio.NewScanner(file)
|
||||
lineNumber := 0
|
||||
for scanner.Scan() {
|
||||
lineNumber++
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key, value, ok := strings.Cut(line, ":")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid config line %d: expected key: value", lineNumber)
|
||||
}
|
||||
key = strings.TrimSpace(key)
|
||||
value = trimConfigValue(value)
|
||||
if key == "" {
|
||||
return nil, fmt.Errorf("invalid config line %d: empty key", lineNumber)
|
||||
}
|
||||
values[key] = value
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func trimConfigValue(value string) string {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
trimmed = strings.Trim(trimmed, `"`)
|
||||
trimmed = strings.Trim(trimmed, `'`)
|
||||
return trimmed
|
||||
}
|
||||
|
||||
func overlayEnv(values map[string]string, key string, envName string) {
|
||||
if value := strings.TrimSpace(os.Getenv(envName)); value != "" {
|
||||
values[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
func parsePositiveInt(value string, fallback int) int {
|
||||
parsed, err := strconv.Atoi(strings.TrimSpace(value))
|
||||
if err != nil || parsed <= 0 {
|
||||
return fallback
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func clampInt(value int, min int, max int) int {
|
||||
if value < min {
|
||||
return min
|
||||
}
|
||||
if value > max {
|
||||
return max
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLoadClampsHeartbeatInterval(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
value string
|
||||
expected int
|
||||
}{
|
||||
{name: "too low", value: "1", expected: MinHeartbeatIntervalSeconds},
|
||||
{name: "too high", value: "7200", expected: MaxHeartbeatIntervalSeconds},
|
||||
{name: "valid", value: "60", expected: 60},
|
||||
{name: "invalid", value: "invalid", expected: DefaultHeartbeatIntervalSeconds},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
configPath := writeTestConfig(t, tt.value)
|
||||
cfg, err := Load(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
if cfg.HeartbeatIntervalSeconds != tt.expected {
|
||||
t.Fatalf("heartbeat seconds mismatch: got=%d want=%d", cfg.HeartbeatIntervalSeconds, tt.expected)
|
||||
}
|
||||
if cfg.HeartbeatInterval != time.Duration(tt.expected)*time.Second {
|
||||
t.Fatalf("heartbeat interval mismatch: got=%s want=%s", cfg.HeartbeatInterval, time.Duration(tt.expected)*time.Second)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestConfig(t *testing.T, heartbeatInterval string) string {
|
||||
t.Helper()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||
content := "panel_url: \"https://panel.example\"\n" +
|
||||
"agent_id: \"agt_test\"\n" +
|
||||
"agent_secret: \"ias_test\"\n" +
|
||||
"heartbeat_interval_seconds: " + heartbeatInterval + "\n"
|
||||
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package panel
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"incudal-agent/internal/config"
|
||||
"incudal-agent/internal/protocol"
|
||||
)
|
||||
|
||||
const heartbeatPath = "/api/agent/heartbeat"
|
||||
|
||||
type Client struct {
|
||||
panelURL string
|
||||
agentID string
|
||||
agentSecret string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
type HeartbeatResult struct {
|
||||
StatusCode int
|
||||
Body string
|
||||
OK bool
|
||||
Upgrade *UpgradeInstruction
|
||||
LatencyMs int64
|
||||
}
|
||||
|
||||
type UpgradeInstruction struct {
|
||||
Available bool `json:"available"`
|
||||
Version string `json:"version"`
|
||||
URL string `json:"url"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Gzip bool `json:"gzip"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type heartbeatResponse struct {
|
||||
Upgrade *UpgradeInstruction `json:"upgrade"`
|
||||
}
|
||||
|
||||
func New(cfg config.Config) *Client {
|
||||
return &Client{
|
||||
panelURL: strings.TrimRight(cfg.PanelURL, "/"),
|
||||
agentID: cfg.AgentID,
|
||||
agentSecret: cfg.AgentSecret,
|
||||
httpClient: &http.Client{
|
||||
Timeout: cfg.RequestTimeout,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (client *Client) Heartbeat(ctx context.Context, payload map[string]any) (HeartbeatResult, error) {
|
||||
body, err := protocol.CanonicalJSON(payload)
|
||||
if err != nil {
|
||||
return HeartbeatResult{}, err
|
||||
}
|
||||
|
||||
timestamp := protocol.NewTimestamp()
|
||||
nonce, err := protocol.NewNonce()
|
||||
if err != nil {
|
||||
return HeartbeatResult{}, err
|
||||
}
|
||||
bodyHash := protocol.BodySHA256(body)
|
||||
signingPayload := protocol.SigningPayload(http.MethodPost, heartbeatPath, timestamp, nonce, bodyHash)
|
||||
signature := protocol.Signature(client.agentSecret, signingPayload)
|
||||
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPost, client.panelURL+heartbeatPath, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return HeartbeatResult{}, err
|
||||
}
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
request.Header.Set("x-incudal-agent-id", client.agentID)
|
||||
request.Header.Set("x-incudal-timestamp", timestamp)
|
||||
request.Header.Set("x-incudal-nonce", nonce)
|
||||
request.Header.Set("x-incudal-body-sha256", bodyHash)
|
||||
request.Header.Set("x-incudal-signature", signature)
|
||||
|
||||
startedAt := time.Now()
|
||||
response, err := client.httpClient.Do(request)
|
||||
if err != nil {
|
||||
return HeartbeatResult{}, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
responseBody, err := io.ReadAll(io.LimitReader(response.Body, 1024*1024))
|
||||
if err != nil {
|
||||
return HeartbeatResult{}, err
|
||||
}
|
||||
|
||||
result := HeartbeatResult{
|
||||
StatusCode: response.StatusCode,
|
||||
Body: string(responseBody),
|
||||
OK: response.StatusCode >= 200 && response.StatusCode < 300,
|
||||
LatencyMs: time.Since(startedAt).Milliseconds(),
|
||||
}
|
||||
if !result.OK {
|
||||
return result, fmt.Errorf("heartbeat failed: status=%d body=%s", response.StatusCode, result.Body)
|
||||
}
|
||||
|
||||
var parsedResponse heartbeatResponse
|
||||
if err := json.Unmarshal(responseBody, &parsedResponse); err == nil {
|
||||
result.Upgrade = parsedResponse.Upgrade
|
||||
}
|
||||
|
||||
var parsed map[string]any
|
||||
if err := json.Unmarshal(responseBody, &parsed); err == nil {
|
||||
parsed["latencyMs"] = result.LatencyMs
|
||||
if compact, err := json.Marshal(parsed); err == nil {
|
||||
result.Body = string(compact)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package protocol
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CanonicalJSON 使用 Go 标准库的 JSON 编码。
|
||||
// map key 会按字典序输出,必须与面板端 stableStringify 规则保持一致。
|
||||
func CanonicalJSON(value any) ([]byte, error) {
|
||||
return json.Marshal(value)
|
||||
}
|
||||
|
||||
func BodySHA256(body []byte) string {
|
||||
sum := sha256.Sum256(body)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func SigningPayload(method string, path string, timestamp string, nonce string, bodyHash string) string {
|
||||
return strings.Join([]string{
|
||||
strings.ToUpper(method),
|
||||
path,
|
||||
timestamp,
|
||||
nonce,
|
||||
strings.ToLower(bodyHash),
|
||||
}, "\n")
|
||||
}
|
||||
|
||||
func Signature(secret string, payload string) string {
|
||||
mac := hmac.New(sha256.New, []byte(secret))
|
||||
mac.Write([]byte(payload))
|
||||
return hex.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
|
||||
func NewTimestamp() string {
|
||||
return fmt.Sprintf("%d", time.Now().UnixMilli())
|
||||
}
|
||||
|
||||
func NewNonce() (string, error) {
|
||||
var raw [18]byte
|
||||
if _, err := rand.Read(raw[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(raw[:]), nil
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package protocol
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCanonicalJSONIsStableForMapOrder(t *testing.T) {
|
||||
bodyA := map[string]any{
|
||||
"version": "0.1.0",
|
||||
"resources": map[string]any{
|
||||
"memory": 1024,
|
||||
"cpu": 8,
|
||||
},
|
||||
"capabilities": []any{"heartbeat", "report"},
|
||||
}
|
||||
bodyB := map[string]any{
|
||||
"capabilities": []any{"heartbeat", "report"},
|
||||
"resources": map[string]any{
|
||||
"cpu": 8,
|
||||
"memory": 1024,
|
||||
},
|
||||
"version": "0.1.0",
|
||||
}
|
||||
|
||||
jsonA, err := CanonicalJSON(bodyA)
|
||||
if err != nil {
|
||||
t.Fatalf("canonical json A: %v", err)
|
||||
}
|
||||
jsonB, err := CanonicalJSON(bodyB)
|
||||
if err != nil {
|
||||
t.Fatalf("canonical json B: %v", err)
|
||||
}
|
||||
|
||||
if string(jsonA) != string(jsonB) {
|
||||
t.Fatalf("canonical json mismatch:\nA=%s\nB=%s", jsonA, jsonB)
|
||||
}
|
||||
if BodySHA256(jsonA) != BodySHA256(jsonB) {
|
||||
t.Fatalf("body hash mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignatureChangesWithPath(t *testing.T) {
|
||||
secret := "ias_test_secret"
|
||||
bodyHash := BodySHA256([]byte(`{"ok":true}`))
|
||||
payloadA := SigningPayload("POST", "/api/agent/heartbeat", "1777380000000", "nonce-123456", bodyHash)
|
||||
payloadB := SigningPayload("POST", "/api/agent/report", "1777380000000", "nonce-123456", bodyHash)
|
||||
|
||||
if Signature(secret, payloadA) == Signature(secret, payloadB) {
|
||||
t.Fatalf("signature should change when request path changes")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
package report
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
maxReportedIncusInstances = 1000
|
||||
incusStateConcurrency = 8
|
||||
)
|
||||
|
||||
var externalGuestInterfacePattern = regexp.MustCompile(`^(eth[0-9]+|en(?:o|p|s|x)[a-z0-9]+)$`)
|
||||
|
||||
type incusAPIResponse struct {
|
||||
Type string `json:"type"`
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error"`
|
||||
Metadata json.RawMessage `json:"metadata"`
|
||||
}
|
||||
|
||||
type incusInstanceSummary struct {
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
StatusCode int `json:"status_code"`
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
type incusInstanceState struct {
|
||||
Status string `json:"status"`
|
||||
Network map[string]incusNetworkDevice `json:"network"`
|
||||
}
|
||||
|
||||
type incusNetworkDevice struct {
|
||||
Addresses []incusNetworkAddress `json:"addresses"`
|
||||
Hwaddr string `json:"hwaddr"`
|
||||
Counters incusNetworkCounters `json:"counters"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
type incusNetworkAddress struct {
|
||||
Family string `json:"family"`
|
||||
Address string `json:"address"`
|
||||
}
|
||||
|
||||
type incusNetworkCounters struct {
|
||||
BytesReceived json.Number `json:"bytes_received"`
|
||||
BytesSent json.Number `json:"bytes_sent"`
|
||||
}
|
||||
|
||||
type trafficCounters struct {
|
||||
rx uint64
|
||||
tx uint64
|
||||
}
|
||||
|
||||
func collectIncusInstanceReport() map[string]any {
|
||||
reportedAt := time.Now().UTC().Format(time.RFC3339)
|
||||
socketPath, ok := detectIncusSocket()
|
||||
if !ok {
|
||||
return map[string]any{
|
||||
"available": false,
|
||||
"reportedAt": reportedAt,
|
||||
"total": 0,
|
||||
"items": []any{},
|
||||
}
|
||||
}
|
||||
|
||||
// 只通过本机 Unix socket 做只读采集,不要求宿主机开放额外 Agent 端口。
|
||||
client := newIncusUnixHTTPClient(socketPath)
|
||||
instances, err := listIncusInstances(client)
|
||||
if err != nil {
|
||||
return map[string]any{
|
||||
"available": false,
|
||||
"reportedAt": reportedAt,
|
||||
"total": 0,
|
||||
"items": []any{},
|
||||
"error": truncateReportError(err),
|
||||
}
|
||||
}
|
||||
|
||||
limitedInstances := instances
|
||||
if len(limitedInstances) > maxReportedIncusInstances {
|
||||
limitedInstances = limitedInstances[:maxReportedIncusInstances]
|
||||
}
|
||||
|
||||
items := make([]map[string]any, len(limitedInstances))
|
||||
var wg sync.WaitGroup
|
||||
semaphore := make(chan struct{}, incusStateConcurrency)
|
||||
|
||||
for index, instance := range limitedInstances {
|
||||
wg.Add(1)
|
||||
go func(index int, instance incusInstanceSummary) {
|
||||
defer wg.Done()
|
||||
semaphore <- struct{}{}
|
||||
defer func() { <-semaphore }()
|
||||
|
||||
items[index] = buildIncusInstanceReportItem(client, instance)
|
||||
}(index, instance)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
normalizedItems := make([]any, 0, len(items))
|
||||
for _, item := range items {
|
||||
if item != nil {
|
||||
normalizedItems = append(normalizedItems, item)
|
||||
}
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"available": true,
|
||||
"reportedAt": reportedAt,
|
||||
"total": len(instances),
|
||||
"truncated": len(instances) > len(limitedInstances),
|
||||
"items": normalizedItems,
|
||||
}
|
||||
}
|
||||
|
||||
func newIncusUnixHTTPClient(socketPath string) *http.Client {
|
||||
transport := &http.Transport{
|
||||
DialContext: func(ctx context.Context, _network string, _addr string) (net.Conn, error) {
|
||||
var dialer net.Dialer
|
||||
return dialer.DialContext(ctx, "unix", socketPath)
|
||||
},
|
||||
DisableCompression: true,
|
||||
DisableKeepAlives: true,
|
||||
}
|
||||
|
||||
return &http.Client{
|
||||
Transport: transport,
|
||||
Timeout: 8 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func listIncusInstances(client *http.Client) ([]incusInstanceSummary, error) {
|
||||
return incusRequest[[]incusInstanceSummary](client, "/1.0/instances?recursion=1")
|
||||
}
|
||||
|
||||
func getIncusInstanceState(client *http.Client, name string) (incusInstanceState, error) {
|
||||
return incusRequest[incusInstanceState](client, "/1.0/instances/"+url.PathEscape(name)+"/state")
|
||||
}
|
||||
|
||||
func incusRequest[T any](client *http.Client, path string) (T, error) {
|
||||
var zero T
|
||||
request, err := http.NewRequest(http.MethodGet, "http://incus"+path, nil)
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return zero, fmt.Errorf("incus request failed: path=%s status=%d", path, response.StatusCode)
|
||||
}
|
||||
|
||||
var envelope incusAPIResponse
|
||||
decoder := json.NewDecoder(response.Body)
|
||||
decoder.UseNumber()
|
||||
if err := decoder.Decode(&envelope); err != nil {
|
||||
return zero, err
|
||||
}
|
||||
if envelope.Type == "error" {
|
||||
if envelope.Error != "" {
|
||||
return zero, fmt.Errorf("incus error: %s", envelope.Error)
|
||||
}
|
||||
return zero, fmt.Errorf("incus error: status=%s", envelope.Status)
|
||||
}
|
||||
|
||||
metadataDecoder := json.NewDecoder(bytes.NewReader(envelope.Metadata))
|
||||
metadataDecoder.UseNumber()
|
||||
if err := metadataDecoder.Decode(&zero); err != nil {
|
||||
return zero, err
|
||||
}
|
||||
return zero, nil
|
||||
}
|
||||
|
||||
func buildIncusInstanceReportItem(client *http.Client, instance incusInstanceSummary) map[string]any {
|
||||
item := map[string]any{
|
||||
"name": instance.Name,
|
||||
"status": instance.Status,
|
||||
"statusCode": instance.StatusCode,
|
||||
"type": instance.Type,
|
||||
}
|
||||
|
||||
if instance.Name == "" {
|
||||
return item
|
||||
}
|
||||
|
||||
state, err := getIncusInstanceState(client, instance.Name)
|
||||
if err != nil {
|
||||
item["error"] = truncateReportError(err)
|
||||
return item
|
||||
}
|
||||
if state.Status != "" {
|
||||
item["status"] = state.Status
|
||||
}
|
||||
|
||||
counters := getTrafficCountersFromIncusState(instance.Name, state)
|
||||
item["traffic"] = map[string]any{
|
||||
"rxBytes": strconv.FormatUint(counters.rx, 10),
|
||||
"txBytes": strconv.FormatUint(counters.tx, 10),
|
||||
}
|
||||
|
||||
if ipv4, ipv6 := firstRoutableAddresses(state.Network); ipv4 != "" || ipv6 != "" {
|
||||
network := map[string]any{}
|
||||
if ipv4 != "" {
|
||||
network["ipv4"] = ipv4
|
||||
}
|
||||
if ipv6 != "" {
|
||||
network["ipv6"] = ipv6
|
||||
}
|
||||
item["network"] = network
|
||||
}
|
||||
|
||||
return item
|
||||
}
|
||||
|
||||
func getTrafficCountersFromIncusState(instanceName string, state incusInstanceState) trafficCounters {
|
||||
billableVmMacs := generateBillableVmMacs(instanceName)
|
||||
totals := trafficCounters{}
|
||||
fallbackInterfaces := make([]incusNetworkDevice, 0)
|
||||
hasStrictBillableInterface := false
|
||||
|
||||
// 与面板旧采集口径保持一致,避免 guest 内部 bridge/veth 被重复计费。
|
||||
for ifName, ifData := range state.Network {
|
||||
if isBillableNetworkInterface(ifName, ifData, billableVmMacs) {
|
||||
hasStrictBillableInterface = true
|
||||
addNetworkCounters(&totals, ifData.Counters)
|
||||
continue
|
||||
}
|
||||
|
||||
if isLikelyExternalGuestInterface(ifName) {
|
||||
fallbackInterfaces = append(fallbackInterfaces, ifData)
|
||||
}
|
||||
}
|
||||
|
||||
if !hasStrictBillableInterface {
|
||||
for _, ifData := range fallbackInterfaces {
|
||||
addNetworkCounters(&totals, ifData.Counters)
|
||||
}
|
||||
}
|
||||
|
||||
return totals
|
||||
}
|
||||
|
||||
func isBillableNetworkInterface(ifName string, ifData incusNetworkDevice, billableVmMacs map[string]struct{}) bool {
|
||||
if ifName == "lo" {
|
||||
return false
|
||||
}
|
||||
if ifName == "eth0" || ifName == "eth1" {
|
||||
return true
|
||||
}
|
||||
|
||||
hwaddr := strings.ToLower(strings.TrimSpace(ifData.Hwaddr))
|
||||
if hwaddr == "" {
|
||||
return false
|
||||
}
|
||||
_, ok := billableVmMacs[hwaddr]
|
||||
return ok
|
||||
}
|
||||
|
||||
func isLikelyExternalGuestInterface(ifName string) bool {
|
||||
return externalGuestInterfacePattern.MatchString(strings.ToLower(ifName))
|
||||
}
|
||||
|
||||
func addNetworkCounters(totals *trafficCounters, counters incusNetworkCounters) {
|
||||
totals.rx += jsonNumberToUint64(counters.BytesReceived)
|
||||
totals.tx += jsonNumberToUint64(counters.BytesSent)
|
||||
}
|
||||
|
||||
func jsonNumberToUint64(value json.Number) uint64 {
|
||||
raw := strings.TrimSpace(value.String())
|
||||
if raw == "" {
|
||||
return 0
|
||||
}
|
||||
parsed, err := strconv.ParseUint(raw, 10, 64)
|
||||
if err == nil {
|
||||
return parsed
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func generateBillableVmMacs(seed string) map[string]struct{} {
|
||||
return map[string]struct{}{
|
||||
generateVmNicMac(seed, "eth0"): {},
|
||||
generateVmNicMac(seed, "eth1"): {},
|
||||
}
|
||||
}
|
||||
|
||||
func generateVmNicMac(seed string, nicLabel string) string {
|
||||
hash := sha256.Sum256([]byte("incudal-vm-nic:" + seed + ":" + nicLabel))
|
||||
bytes := []byte{0x02, hash[0], hash[1], hash[2], hash[3], hash[4]}
|
||||
encoded := hex.EncodeToString(bytes)
|
||||
return strings.Join([]string{
|
||||
encoded[0:2],
|
||||
encoded[2:4],
|
||||
encoded[4:6],
|
||||
encoded[6:8],
|
||||
encoded[8:10],
|
||||
encoded[10:12],
|
||||
}, ":")
|
||||
}
|
||||
|
||||
func firstRoutableAddresses(network map[string]incusNetworkDevice) (string, string) {
|
||||
var ipv4 string
|
||||
var ipv6 string
|
||||
|
||||
for _, ifData := range network {
|
||||
for _, address := range ifData.Addresses {
|
||||
if address.Address == "" {
|
||||
continue
|
||||
}
|
||||
ip := net.ParseIP(address.Address)
|
||||
if ip == nil || !isRoutableGuestIP(ip) {
|
||||
continue
|
||||
}
|
||||
if ipv4 == "" && ip.To4() != nil && strings.EqualFold(address.Family, "inet") {
|
||||
ipv4 = address.Address
|
||||
continue
|
||||
}
|
||||
if ipv6 == "" && ip.To4() == nil && strings.EqualFold(address.Family, "inet6") {
|
||||
ipv6 = address.Address
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ipv4, ipv6
|
||||
}
|
||||
|
||||
func isRoutableGuestIP(ip net.IP) bool {
|
||||
return !ip.IsLoopback() &&
|
||||
!ip.IsUnspecified() &&
|
||||
!ip.IsLinkLocalUnicast() &&
|
||||
!ip.IsLinkLocalMulticast() &&
|
||||
!ip.IsMulticast()
|
||||
}
|
||||
|
||||
func truncateReportError(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
message := err.Error()
|
||||
if len(message) > 200 {
|
||||
return message[:200]
|
||||
}
|
||||
return message
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
package report
|
||||
|
||||
import (
|
||||
"os"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
var incusSocketCandidates = []string{
|
||||
"/var/lib/incus/unix.socket",
|
||||
"/var/snap/incus/common/lxd/unix.socket",
|
||||
"/var/lib/lxd/unix.socket",
|
||||
}
|
||||
|
||||
func HeartbeatPayload(version string, heartbeatIntervalSeconds int) map[string]any {
|
||||
return map[string]any{
|
||||
"version": version,
|
||||
"capabilities": []any{"heartbeat", "report", "host-metrics", "instance-status", "traffic-counters"},
|
||||
"runtime": map[string]any{
|
||||
"goos": runtime.GOOS,
|
||||
"goarch": runtime.GOARCH,
|
||||
},
|
||||
"incus": detectIncus(),
|
||||
"instances": collectIncusInstanceReport(),
|
||||
"resources": collectResources(),
|
||||
"metrics": collectMetrics(heartbeatIntervalSeconds),
|
||||
}
|
||||
}
|
||||
|
||||
func collectResources() map[string]any {
|
||||
resources := map[string]any{
|
||||
"cpuTotal": runtime.NumCPU(),
|
||||
}
|
||||
if cpuUsagePercent := readCPUUsagePercent(); cpuUsagePercent >= 0 {
|
||||
resources["cpuUsagePercent"] = cpuUsagePercent
|
||||
}
|
||||
for key, value := range readMemoryStats() {
|
||||
resources[key] = value
|
||||
}
|
||||
for key, value := range readDiskStats("/") {
|
||||
resources[key] = value
|
||||
}
|
||||
if processCount := readProcessCount(); processCount >= 0 {
|
||||
resources["processCount"] = processCount
|
||||
}
|
||||
return resources
|
||||
}
|
||||
|
||||
func collectMetrics(heartbeatIntervalSeconds int) map[string]any {
|
||||
switch {
|
||||
case heartbeatIntervalSeconds <= 0:
|
||||
heartbeatIntervalSeconds = 30
|
||||
case heartbeatIntervalSeconds < 5:
|
||||
heartbeatIntervalSeconds = 5
|
||||
case heartbeatIntervalSeconds > 3600:
|
||||
heartbeatIntervalSeconds = 3600
|
||||
}
|
||||
|
||||
metrics := map[string]any{
|
||||
"reportedAt": time.Now().UTC().Format(time.RFC3339),
|
||||
"heartbeatIntervalSeconds": heartbeatIntervalSeconds,
|
||||
}
|
||||
if uptimeSeconds := readUptimeSeconds(); uptimeSeconds > 0 {
|
||||
metrics["uptimeSeconds"] = uptimeSeconds
|
||||
}
|
||||
for key, value := range readLoadAverage() {
|
||||
metrics[key] = value
|
||||
}
|
||||
return metrics
|
||||
}
|
||||
|
||||
func detectIncus() map[string]any {
|
||||
socketPath, ok := detectIncusSocket()
|
||||
if ok {
|
||||
return map[string]any{
|
||||
"available": true,
|
||||
"socket": socketPath,
|
||||
}
|
||||
}
|
||||
return map[string]any{
|
||||
"available": false,
|
||||
"socket": "",
|
||||
}
|
||||
}
|
||||
|
||||
func detectIncusSocket() (string, bool) {
|
||||
for _, socketPath := range incusSocketCandidates {
|
||||
if info, err := os.Stat(socketPath); err == nil && !info.IsDir() {
|
||||
return socketPath, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
type cpuStat struct {
|
||||
idle uint64
|
||||
total uint64
|
||||
}
|
||||
|
||||
func readCPUUsagePercent() float64 {
|
||||
before, ok := readCPUStat()
|
||||
if !ok {
|
||||
return -1
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
after, ok := readCPUStat()
|
||||
if !ok {
|
||||
return -1
|
||||
}
|
||||
|
||||
totalDelta := after.total - before.total
|
||||
idleDelta := after.idle - before.idle
|
||||
if totalDelta == 0 || idleDelta > totalDelta {
|
||||
return -1
|
||||
}
|
||||
|
||||
return roundPercent(float64(totalDelta-idleDelta) / float64(totalDelta) * 100)
|
||||
}
|
||||
|
||||
func readCPUStat() (cpuStat, bool) {
|
||||
content, err := os.ReadFile("/proc/stat")
|
||||
if err != nil {
|
||||
return cpuStat{}, false
|
||||
}
|
||||
for _, line := range strings.Split(string(content), "\n") {
|
||||
if !strings.HasPrefix(line, "cpu ") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 5 {
|
||||
return cpuStat{}, false
|
||||
}
|
||||
|
||||
var values []uint64
|
||||
for _, field := range fields[1:] {
|
||||
value, err := strconv.ParseUint(field, 10, 64)
|
||||
if err != nil {
|
||||
return cpuStat{}, false
|
||||
}
|
||||
values = append(values, value)
|
||||
}
|
||||
|
||||
var total uint64
|
||||
for _, value := range values {
|
||||
total += value
|
||||
}
|
||||
idle := values[3]
|
||||
if len(values) > 4 {
|
||||
idle += values[4]
|
||||
}
|
||||
return cpuStat{idle: idle, total: total}, true
|
||||
}
|
||||
return cpuStat{}, false
|
||||
}
|
||||
|
||||
func readMemoryStats() map[string]any {
|
||||
meminfo := readMeminfoKB()
|
||||
stats := map[string]any{}
|
||||
memTotal := meminfo["MemTotal"]
|
||||
memAvailable := meminfo["MemAvailable"]
|
||||
if memTotal > 0 {
|
||||
memUsed := memTotal - memAvailable
|
||||
if memUsed < 0 {
|
||||
memUsed = 0
|
||||
}
|
||||
stats["memoryTotalMb"] = memTotal / 1024
|
||||
stats["memoryAvailableMb"] = memAvailable / 1024
|
||||
stats["memoryUsedMb"] = memUsed / 1024
|
||||
stats["memoryUsagePercent"] = roundPercent(float64(memUsed) / float64(memTotal) * 100)
|
||||
}
|
||||
|
||||
swapTotal := meminfo["SwapTotal"]
|
||||
swapFree := meminfo["SwapFree"]
|
||||
if swapTotal > 0 {
|
||||
swapUsed := swapTotal - swapFree
|
||||
if swapUsed < 0 {
|
||||
swapUsed = 0
|
||||
}
|
||||
stats["swapTotalMb"] = swapTotal / 1024
|
||||
stats["swapUsedMb"] = swapUsed / 1024
|
||||
stats["swapUsagePercent"] = roundPercent(float64(swapUsed) / float64(swapTotal) * 100)
|
||||
} else {
|
||||
stats["swapTotalMb"] = int64(0)
|
||||
stats["swapUsedMb"] = int64(0)
|
||||
stats["swapUsagePercent"] = float64(0)
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
func readMeminfoKB() map[string]int64 {
|
||||
content, err := os.ReadFile("/proc/meminfo")
|
||||
if err != nil {
|
||||
return map[string]int64{}
|
||||
}
|
||||
|
||||
values := map[string]int64{}
|
||||
for _, line := range strings.Split(string(content), "\n") {
|
||||
key, rest, ok := strings.Cut(line, ":")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(rest)
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
kb, err := strconv.ParseInt(fields[0], 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
values[key] = kb
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func readDiskStats(path string) map[string]any {
|
||||
var stat syscall.Statfs_t
|
||||
if err := syscall.Statfs(path, &stat); err != nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
|
||||
blockSize := uint64(stat.Bsize)
|
||||
total := stat.Blocks * blockSize
|
||||
free := stat.Bfree * blockSize
|
||||
if total == 0 || free > total {
|
||||
return map[string]any{}
|
||||
}
|
||||
|
||||
used := total - free
|
||||
return map[string]any{
|
||||
"diskMountpoint": path,
|
||||
"diskTotalBytes": total,
|
||||
"diskUsedBytes": used,
|
||||
"diskAvailableBytes": stat.Bavail * blockSize,
|
||||
"diskUsagePercent": roundPercent(float64(used) / float64(total) * 100),
|
||||
}
|
||||
}
|
||||
|
||||
func readLoadAverage() map[string]any {
|
||||
content, err := os.ReadFile("/proc/loadavg")
|
||||
if err != nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
fields := strings.Fields(string(content))
|
||||
if len(fields) < 3 {
|
||||
return map[string]any{}
|
||||
}
|
||||
|
||||
loads := map[string]any{}
|
||||
keys := []string{"load1", "load5", "load15"}
|
||||
for index, key := range keys {
|
||||
value, err := strconv.ParseFloat(fields[index], 64)
|
||||
if err == nil {
|
||||
loads[key] = value
|
||||
}
|
||||
}
|
||||
return loads
|
||||
}
|
||||
|
||||
func readProcessCount() int {
|
||||
entries, err := os.ReadDir("/proc")
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
count := 0
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
if _, err := strconv.Atoi(entry.Name()); err == nil {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func readUptimeSeconds() int64 {
|
||||
content, err := os.ReadFile("/proc/uptime")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
fields := strings.Fields(string(content))
|
||||
if len(fields) == 0 {
|
||||
return 0
|
||||
}
|
||||
value, err := strconv.ParseFloat(fields[0], 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return int64(value)
|
||||
}
|
||||
|
||||
func roundPercent(value float64) float64 {
|
||||
return float64(int(value*10+0.5)) / 10
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
package report
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestHeartbeatPayloadIncludesHostMetrics(t *testing.T) {
|
||||
payload := HeartbeatPayload("test-version", 30)
|
||||
|
||||
capabilities, ok := payload["capabilities"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("capabilities missing or invalid: %#v", payload["capabilities"])
|
||||
}
|
||||
if !containsCapability(capabilities, "host-metrics") {
|
||||
t.Fatalf("host-metrics capability missing: %#v", capabilities)
|
||||
}
|
||||
if !containsCapability(capabilities, "instance-status") {
|
||||
t.Fatalf("instance-status capability missing: %#v", capabilities)
|
||||
}
|
||||
if !containsCapability(capabilities, "traffic-counters") {
|
||||
t.Fatalf("traffic-counters capability missing: %#v", capabilities)
|
||||
}
|
||||
|
||||
instances, ok := payload["instances"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("instances report missing or invalid: %#v", payload["instances"])
|
||||
}
|
||||
for _, key := range []string{"available", "reportedAt", "total", "items"} {
|
||||
if _, ok := instances[key]; !ok {
|
||||
t.Fatalf("instances key %s missing: %#v", key, instances)
|
||||
}
|
||||
}
|
||||
|
||||
runtimeInfo, ok := payload["runtime"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("runtime missing or invalid: %#v", payload["runtime"])
|
||||
}
|
||||
for _, key := range []string{"goos", "goarch"} {
|
||||
if _, ok := runtimeInfo[key]; !ok {
|
||||
t.Fatalf("runtime key %s missing: %#v", key, runtimeInfo)
|
||||
}
|
||||
}
|
||||
|
||||
resources, ok := payload["resources"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("resources missing or invalid: %#v", payload["resources"])
|
||||
}
|
||||
for _, key := range []string{
|
||||
"cpuTotal",
|
||||
"cpuUsagePercent",
|
||||
"memoryTotalMb",
|
||||
"memoryUsedMb",
|
||||
"memoryUsagePercent",
|
||||
"swapTotalMb",
|
||||
"swapUsedMb",
|
||||
"swapUsagePercent",
|
||||
"diskTotalBytes",
|
||||
"diskUsedBytes",
|
||||
"diskUsagePercent",
|
||||
"processCount",
|
||||
} {
|
||||
if _, ok := resources[key]; !ok {
|
||||
t.Fatalf("resource key %s missing: %#v", key, resources)
|
||||
}
|
||||
}
|
||||
|
||||
metrics, ok := payload["metrics"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("metrics missing or invalid: %#v", payload["metrics"])
|
||||
}
|
||||
for _, key := range []string{"reportedAt", "heartbeatIntervalSeconds", "uptimeSeconds", "load1", "load5", "load15"} {
|
||||
if _, ok := metrics[key]; !ok {
|
||||
t.Fatalf("metric key %s missing: %#v", key, metrics)
|
||||
}
|
||||
}
|
||||
if metrics["heartbeatIntervalSeconds"] != 30 {
|
||||
t.Fatalf("heartbeat interval mismatch: %#v", metrics["heartbeatIntervalSeconds"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeatPayloadClampsHeartbeatInterval(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input int
|
||||
expected int
|
||||
}{
|
||||
{name: "zero falls back", input: 0, expected: 30},
|
||||
{name: "too low clamps to min", input: 1, expected: 5},
|
||||
{name: "too high clamps to max", input: 7200, expected: 3600},
|
||||
{name: "valid stays unchanged", input: 60, expected: 60},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
payload := HeartbeatPayload("test-version", tt.input)
|
||||
metrics, ok := payload["metrics"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("metrics missing or invalid: %#v", payload["metrics"])
|
||||
}
|
||||
if metrics["heartbeatIntervalSeconds"] != tt.expected {
|
||||
t.Fatalf("heartbeat interval mismatch: got=%#v want=%d", metrics["heartbeatIntervalSeconds"], tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrafficCountersFromIncusStateUsesBillableInterfaces(t *testing.T) {
|
||||
state := incusInstanceState{
|
||||
Network: map[string]incusNetworkDevice{
|
||||
"lo": {
|
||||
Counters: incusNetworkCounters{
|
||||
BytesReceived: "999",
|
||||
BytesSent: "999",
|
||||
},
|
||||
},
|
||||
"eth0": {
|
||||
Counters: incusNetworkCounters{
|
||||
BytesReceived: "100",
|
||||
BytesSent: "200",
|
||||
},
|
||||
},
|
||||
"docker0": {
|
||||
Counters: incusNetworkCounters{
|
||||
BytesReceived: "300",
|
||||
BytesSent: "400",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
counters := getTrafficCountersFromIncusState("vm-test", state)
|
||||
if counters.rx != 100 || counters.tx != 200 {
|
||||
t.Fatalf("traffic counters mismatch: got=%+v", counters)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrafficCountersFromIncusStateFallsBackToExternalInterfaces(t *testing.T) {
|
||||
state := incusInstanceState{
|
||||
Network: map[string]incusNetworkDevice{
|
||||
"lo": {
|
||||
Counters: incusNetworkCounters{
|
||||
BytesReceived: "999",
|
||||
BytesSent: "999",
|
||||
},
|
||||
},
|
||||
"enp5s0": {
|
||||
Counters: incusNetworkCounters{
|
||||
BytesReceived: "123",
|
||||
BytesSent: "456",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
counters := getTrafficCountersFromIncusState("vm-test", state)
|
||||
if counters.rx != 123 || counters.tx != 456 {
|
||||
t.Fatalf("traffic counters mismatch: got=%+v", counters)
|
||||
}
|
||||
}
|
||||
|
||||
func containsCapability(capabilities []any, expected string) bool {
|
||||
for _, capability := range capabilities {
|
||||
if capability == expected {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
package upgrade
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"incudal-agent/internal/config"
|
||||
"incudal-agent/internal/panel"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultServiceName = "incudal-agent"
|
||||
defaultMaxDownloadBytes = 64 * 1024 * 1024
|
||||
)
|
||||
|
||||
var ErrUpgradeInProgress = errors.New("agent upgrade already in progress")
|
||||
var systemdServiceNamePattern = regexp.MustCompile(`^[A-Za-z0-9_.@-]+$`)
|
||||
|
||||
type RestartFunc func(ctx context.Context, serviceName string) error
|
||||
|
||||
type Runner struct {
|
||||
BinaryPath string
|
||||
BackupPath string
|
||||
LockPath string
|
||||
ServiceName string
|
||||
AllowedBaseURL string
|
||||
HTTPClient *http.Client
|
||||
Restart RestartFunc
|
||||
MaxDownloadBytes int64
|
||||
}
|
||||
|
||||
func DefaultRunner(cfg config.Config) *Runner {
|
||||
binaryPath, err := os.Executable()
|
||||
if err != nil || binaryPath == "" {
|
||||
binaryPath = "/usr/local/bin/incudal-agent"
|
||||
}
|
||||
|
||||
return &Runner{
|
||||
BinaryPath: binaryPath,
|
||||
BackupPath: binaryPath + ".bak",
|
||||
LockPath: defaultLockPath(),
|
||||
ServiceName: defaultServiceName,
|
||||
AllowedBaseURL: cfg.PanelURL,
|
||||
HTTPClient: &http.Client{Timeout: cfg.RequestTimeout},
|
||||
Restart: restartSystemdService,
|
||||
MaxDownloadBytes: defaultMaxDownloadBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func RandomJitter(max time.Duration) time.Duration {
|
||||
if max <= 0 {
|
||||
return 0
|
||||
}
|
||||
limit := big.NewInt(int64(max))
|
||||
value, err := rand.Int(rand.Reader, limit)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return time.Duration(value.Int64())
|
||||
}
|
||||
|
||||
func (runner *Runner) Apply(ctx context.Context, instruction panel.UpgradeInstruction, currentVersion string) error {
|
||||
if !instruction.Available {
|
||||
return nil
|
||||
}
|
||||
if instruction.Version == "" {
|
||||
return errors.New("upgrade version is required")
|
||||
}
|
||||
if instruction.Version == currentVersion {
|
||||
return nil
|
||||
}
|
||||
if instruction.URL == "" {
|
||||
return errors.New("upgrade URL is required")
|
||||
}
|
||||
if instruction.SHA256 == "" {
|
||||
return errors.New("upgrade sha256 is required")
|
||||
}
|
||||
if err := runner.validateUpgradeURL(instruction.URL); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
unlock, err := acquireLock(runner.lockPath())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
packageBytes, err := runner.download(ctx, instruction.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verifySHA256(packageBytes, instruction.SHA256); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
binaryBytes := packageBytes
|
||||
if instruction.Gzip {
|
||||
binaryBytes, err = gunzip(packageBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
tempPath, err := runner.writeTempBinary(binaryBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runner.replaceBinary(tempPath); err != nil {
|
||||
_ = os.Remove(tempPath)
|
||||
return err
|
||||
}
|
||||
|
||||
if err := runner.restart(ctx); err != nil {
|
||||
return fmt.Errorf("restart agent after upgrade: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (runner *Runner) validateUpgradeURL(rawURL string) error {
|
||||
upgradeURL, err := url.Parse(rawURL)
|
||||
if err != nil || upgradeURL.Scheme == "" || upgradeURL.Host == "" {
|
||||
return fmt.Errorf("upgrade URL is invalid: %s", rawURL)
|
||||
}
|
||||
if upgradeURL.Scheme != "http" && upgradeURL.Scheme != "https" {
|
||||
return fmt.Errorf("upgrade URL scheme is not allowed: %s", upgradeURL.Scheme)
|
||||
}
|
||||
|
||||
baseURL, err := url.Parse(strings.TrimRight(runner.AllowedBaseURL, "/"))
|
||||
if err != nil || baseURL.Scheme == "" || baseURL.Host == "" {
|
||||
return fmt.Errorf("panel URL is invalid: %s", runner.AllowedBaseURL)
|
||||
}
|
||||
if !strings.EqualFold(upgradeURL.Scheme, baseURL.Scheme) || !strings.EqualFold(upgradeURL.Host, baseURL.Host) {
|
||||
return errors.New("upgrade URL is outside panel origin")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (runner *Runner) download(ctx context.Context, rawURL string) ([]byte, error) {
|
||||
client := runner.HTTPClient
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 30 * time.Second}
|
||||
}
|
||||
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return nil, fmt.Errorf("download upgrade failed: status=%d body=%s", response.StatusCode, string(body))
|
||||
}
|
||||
|
||||
limit := runner.MaxDownloadBytes
|
||||
if limit <= 0 {
|
||||
limit = defaultMaxDownloadBytes
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, limit+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(body)) > limit {
|
||||
return nil, fmt.Errorf("upgrade package exceeds %d bytes", limit)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (runner *Runner) writeTempBinary(binaryBytes []byte) (string, error) {
|
||||
binaryPath := runner.binaryPath()
|
||||
tempFile, err := os.CreateTemp(filepath.Dir(binaryPath), ".incudal-agent-upgrade-*")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
tempPath := tempFile.Name()
|
||||
defer tempFile.Close()
|
||||
|
||||
if _, err := tempFile.Write(binaryBytes); err != nil {
|
||||
_ = os.Remove(tempPath)
|
||||
return "", err
|
||||
}
|
||||
if err := tempFile.Chmod(0755); err != nil {
|
||||
_ = os.Remove(tempPath)
|
||||
return "", err
|
||||
}
|
||||
return tempPath, nil
|
||||
}
|
||||
|
||||
func (runner *Runner) replaceBinary(tempPath string) error {
|
||||
binaryPath := runner.binaryPath()
|
||||
backupPath := runner.backupPath()
|
||||
|
||||
if _, err := os.Stat(binaryPath); err == nil {
|
||||
_ = os.Remove(backupPath)
|
||||
if err := copyFile(binaryPath, backupPath); err != nil {
|
||||
return fmt.Errorf("backup current agent: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.Rename(tempPath, binaryPath); err != nil {
|
||||
return fmt.Errorf("replace agent binary: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (runner *Runner) rollback() error {
|
||||
backupPath := runner.backupPath()
|
||||
if _, err := os.Stat(backupPath); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(backupPath, runner.binaryPath())
|
||||
}
|
||||
|
||||
func (runner *Runner) restart(ctx context.Context) error {
|
||||
if runner.Restart == nil {
|
||||
return nil
|
||||
}
|
||||
serviceName := runner.ServiceName
|
||||
if serviceName == "" {
|
||||
serviceName = defaultServiceName
|
||||
}
|
||||
return runner.Restart(ctx, serviceName)
|
||||
}
|
||||
|
||||
func (runner *Runner) binaryPath() string {
|
||||
if runner.BinaryPath != "" {
|
||||
return runner.BinaryPath
|
||||
}
|
||||
return "/usr/local/bin/incudal-agent"
|
||||
}
|
||||
|
||||
func (runner *Runner) backupPath() string {
|
||||
if runner.BackupPath != "" {
|
||||
return runner.BackupPath
|
||||
}
|
||||
return runner.binaryPath() + ".bak"
|
||||
}
|
||||
|
||||
func (runner *Runner) lockPath() string {
|
||||
if runner.LockPath != "" {
|
||||
return runner.LockPath
|
||||
}
|
||||
return defaultLockPath()
|
||||
}
|
||||
|
||||
func defaultLockPath() string {
|
||||
if info, err := os.Stat("/run"); err == nil && info.IsDir() {
|
||||
return "/run/incudal-agent-upgrade.lock"
|
||||
}
|
||||
return filepath.Join(os.TempDir(), "incudal-agent-upgrade.lock")
|
||||
}
|
||||
|
||||
func acquireLock(lockPath string) (func(), error) {
|
||||
file, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrExist) {
|
||||
return nil, ErrUpgradeInProgress
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
_, _ = fmt.Fprintf(file, "%d\n", os.Getpid())
|
||||
_ = file.Close()
|
||||
|
||||
return func() {
|
||||
_ = os.Remove(lockPath)
|
||||
}, nil
|
||||
}
|
||||
|
||||
func verifySHA256(payload []byte, expected string) error {
|
||||
sum := sha256.Sum256(payload)
|
||||
actual := hex.EncodeToString(sum[:])
|
||||
if !strings.EqualFold(actual, expected) {
|
||||
return fmt.Errorf("upgrade sha256 mismatch: expected=%s actual=%s", expected, actual)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func gunzip(payload []byte) ([]byte, error) {
|
||||
reader, err := gzip.NewReader(bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer reader.Close()
|
||||
return io.ReadAll(reader)
|
||||
}
|
||||
|
||||
func copyFile(source string, target string) error {
|
||||
sourceFile, err := os.Open(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer sourceFile.Close()
|
||||
|
||||
info, err := sourceFile.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mode := info.Mode().Perm()
|
||||
if mode == 0 {
|
||||
mode = 0755
|
||||
}
|
||||
|
||||
targetFile, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer targetFile.Close()
|
||||
|
||||
if _, err := io.Copy(targetFile, sourceFile); err != nil {
|
||||
return err
|
||||
}
|
||||
return targetFile.Chmod(mode)
|
||||
}
|
||||
|
||||
func restartSystemdService(ctx context.Context, serviceName string) error {
|
||||
if !systemdServiceNamePattern.MatchString(serviceName) {
|
||||
return fmt.Errorf("invalid systemd service name: %s", serviceName)
|
||||
}
|
||||
|
||||
if err := scheduleSystemdRestart(ctx, serviceName); err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
systemctlPath, err := exec.LookPath("systemctl")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 不等待 systemctl 完成。Agent 正在重启自身,等待子进程会在服务停止时
|
||||
// 收到 SIGTERM,旧逻辑会误判失败并回滚已替换的新二进制。
|
||||
command := exec.CommandContext(ctx, systemctlPath, "restart", serviceName)
|
||||
return command.Start()
|
||||
}
|
||||
|
||||
func scheduleSystemdRestart(ctx context.Context, serviceName string) error {
|
||||
systemdRunPath, err := exec.LookPath("systemd-run")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
systemctlPath, err := exec.LookPath("systemctl")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
unitName := fmt.Sprintf("incudal-agent-restart-%d", os.Getpid())
|
||||
args := []string{
|
||||
"--unit", unitName,
|
||||
"--description", "Restart Incudal Agent after self-upgrade",
|
||||
"--on-active=2s",
|
||||
"--collect",
|
||||
systemctlPath, "restart", serviceName,
|
||||
}
|
||||
|
||||
command := exec.CommandContext(ctx, systemdRunPath, args...)
|
||||
output, err := command.CombinedOutput()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 老版本 systemd 可能不支持 --collect,降级重试一次。
|
||||
if strings.Contains(string(output), "unrecognized option '--collect'") ||
|
||||
strings.Contains(string(output), "Unknown option --collect") {
|
||||
args = []string{
|
||||
"--unit", unitName,
|
||||
"--description", "Restart Incudal Agent after self-upgrade",
|
||||
"--on-active=2s",
|
||||
systemctlPath, "restart", serviceName,
|
||||
}
|
||||
command = exec.CommandContext(ctx, systemdRunPath, args...)
|
||||
output, err = command.CombinedOutput()
|
||||
}
|
||||
if err != nil {
|
||||
trimmedOutput := strings.TrimSpace(string(output))
|
||||
if trimmedOutput == "" {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("%w: %s", err, trimmedOutput)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
package upgrade
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"incudal-agent/internal/panel"
|
||||
)
|
||||
|
||||
func TestApplyUpgradeReplacesBinaryAndRestarts(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
binaryPath := filepath.Join(tempDir, "incudal-agent")
|
||||
if err := os.WriteFile(binaryPath, []byte("old-binary"), 0755); err != nil {
|
||||
t.Fatalf("write current binary: %v", err)
|
||||
}
|
||||
|
||||
nextBinary := []byte("new-binary")
|
||||
packageBytes := gzipBytes(t, nextBinary)
|
||||
sha := sha256Hex(packageBytes)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
response.WriteHeader(http.StatusOK)
|
||||
_, _ = response.Write(packageBytes)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
restarted := false
|
||||
runner := Runner{
|
||||
BinaryPath: binaryPath,
|
||||
BackupPath: binaryPath + ".bak",
|
||||
LockPath: filepath.Join(tempDir, "upgrade.lock"),
|
||||
ServiceName: "incudal-agent",
|
||||
AllowedBaseURL: server.URL,
|
||||
HTTPClient: server.Client(),
|
||||
MaxDownloadBytes: 1024 * 1024,
|
||||
Restart: func(_ context.Context, serviceName string) error {
|
||||
if serviceName != "incudal-agent" {
|
||||
t.Fatalf("unexpected service name: %s", serviceName)
|
||||
}
|
||||
restarted = true
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
err := runner.Apply(context.Background(), panel.UpgradeInstruction{
|
||||
Available: true,
|
||||
Version: "v2",
|
||||
URL: server.URL + "/incudal-agent-linux-amd64.gz",
|
||||
SHA256: sha,
|
||||
Gzip: true,
|
||||
}, "v1")
|
||||
if err != nil {
|
||||
t.Fatalf("apply upgrade: %v", err)
|
||||
}
|
||||
if !restarted {
|
||||
t.Fatalf("restart was not called")
|
||||
}
|
||||
|
||||
actual, err := os.ReadFile(binaryPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read replaced binary: %v", err)
|
||||
}
|
||||
if string(actual) != string(nextBinary) {
|
||||
t.Fatalf("binary mismatch: %q", string(actual))
|
||||
}
|
||||
|
||||
backup, err := os.ReadFile(binaryPath + ".bak")
|
||||
if err != nil {
|
||||
t.Fatalf("read backup binary: %v", err)
|
||||
}
|
||||
if string(backup) != "old-binary" {
|
||||
t.Fatalf("backup mismatch: %q", string(backup))
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyUpgradeRejectsBadSHA(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
binaryPath := filepath.Join(tempDir, "incudal-agent")
|
||||
if err := os.WriteFile(binaryPath, []byte("old-binary"), 0755); err != nil {
|
||||
t.Fatalf("write current binary: %v", err)
|
||||
}
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
response.WriteHeader(http.StatusOK)
|
||||
_, _ = response.Write([]byte("payload"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
restarted := false
|
||||
runner := Runner{
|
||||
BinaryPath: binaryPath,
|
||||
BackupPath: binaryPath + ".bak",
|
||||
LockPath: filepath.Join(tempDir, "upgrade.lock"),
|
||||
AllowedBaseURL: server.URL,
|
||||
HTTPClient: server.Client(),
|
||||
MaxDownloadBytes: 1024 * 1024,
|
||||
Restart: func(context.Context, string) error {
|
||||
restarted = true
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
err := runner.Apply(context.Background(), panel.UpgradeInstruction{
|
||||
Available: true,
|
||||
Version: "v2",
|
||||
URL: server.URL + "/incudal-agent-linux-amd64.gz",
|
||||
SHA256: "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
Gzip: true,
|
||||
}, "v1")
|
||||
if err == nil {
|
||||
t.Fatalf("expected sha mismatch")
|
||||
}
|
||||
if restarted {
|
||||
t.Fatalf("restart should not be called")
|
||||
}
|
||||
|
||||
current, err := os.ReadFile(binaryPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read current binary: %v", err)
|
||||
}
|
||||
if string(current) != "old-binary" {
|
||||
t.Fatalf("current binary should stay unchanged: %q", string(current))
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyUpgradeDoesNotRollbackWhenSelfRestartIsInterrupted(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
binaryPath := filepath.Join(tempDir, "incudal-agent")
|
||||
if err := os.WriteFile(binaryPath, []byte("old-binary"), 0755); err != nil {
|
||||
t.Fatalf("write current binary: %v", err)
|
||||
}
|
||||
|
||||
nextBinary := []byte("new-binary")
|
||||
packageBytes := gzipBytes(t, nextBinary)
|
||||
sha := sha256Hex(packageBytes)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
response.WriteHeader(http.StatusOK)
|
||||
_, _ = response.Write(packageBytes)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
runner := Runner{
|
||||
BinaryPath: binaryPath,
|
||||
BackupPath: binaryPath + ".bak",
|
||||
LockPath: filepath.Join(tempDir, "upgrade.lock"),
|
||||
ServiceName: "incudal-agent",
|
||||
AllowedBaseURL: server.URL,
|
||||
HTTPClient: server.Client(),
|
||||
MaxDownloadBytes: 1024 * 1024,
|
||||
Restart: func(context.Context, string) error {
|
||||
return errors.New("signal: terminated")
|
||||
},
|
||||
}
|
||||
|
||||
err := runner.Apply(context.Background(), panel.UpgradeInstruction{
|
||||
Available: true,
|
||||
Version: "v2",
|
||||
URL: server.URL + "/incudal-agent-linux-amd64.gz",
|
||||
SHA256: sha,
|
||||
Gzip: true,
|
||||
}, "v1")
|
||||
if err == nil {
|
||||
t.Fatalf("expected restart error")
|
||||
}
|
||||
|
||||
actual, err := os.ReadFile(binaryPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read replaced binary: %v", err)
|
||||
}
|
||||
if string(actual) != string(nextBinary) {
|
||||
t.Fatalf("binary should stay replaced after restart interruption: %q", string(actual))
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyUpgradeRejectsDifferentOrigin(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
binaryPath := filepath.Join(tempDir, "incudal-agent")
|
||||
if err := os.WriteFile(binaryPath, []byte("old-binary"), 0755); err != nil {
|
||||
t.Fatalf("write current binary: %v", err)
|
||||
}
|
||||
|
||||
runner := Runner{
|
||||
BinaryPath: binaryPath,
|
||||
BackupPath: binaryPath + ".bak",
|
||||
LockPath: filepath.Join(tempDir, "upgrade.lock"),
|
||||
AllowedBaseURL: "https://panel.example",
|
||||
Restart: func(context.Context, string) error {
|
||||
t.Fatalf("restart should not be called")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
err := runner.Apply(context.Background(), panel.UpgradeInstruction{
|
||||
Available: true,
|
||||
Version: "v2",
|
||||
URL: "https://evil.example/incudal-agent-linux-amd64.gz",
|
||||
SHA256: "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
Gzip: true,
|
||||
}, "v1")
|
||||
if err == nil {
|
||||
t.Fatalf("expected origin validation error")
|
||||
}
|
||||
}
|
||||
|
||||
func gzipBytes(t *testing.T, payload []byte) []byte {
|
||||
t.Helper()
|
||||
|
||||
var buffer bytes.Buffer
|
||||
writer := gzip.NewWriter(&buffer)
|
||||
if _, err := writer.Write(payload); err != nil {
|
||||
t.Fatalf("gzip write: %v", err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatalf("gzip close: %v", err)
|
||||
}
|
||||
return buffer.Bytes()
|
||||
}
|
||||
|
||||
func sha256Hex(payload []byte) string {
|
||||
sum := sha256.Sum256(payload)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
DIST_DIR="${DIST_DIR:-${ROOT_DIR}/dist}"
|
||||
VERSION_FILE="${VERSION_FILE:-${ROOT_DIR}/VERSION}"
|
||||
VERSION_PATTERN='^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$'
|
||||
|
||||
default_version() {
|
||||
if [ ! -f "${VERSION_FILE}" ]; then
|
||||
echo "Agent version file not found: ${VERSION_FILE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tr -d '[:space:]' < "${VERSION_FILE}"
|
||||
}
|
||||
|
||||
VERSION="${VERSION:-$(default_version)}"
|
||||
if [[ ! "${VERSION}" =~ ${VERSION_PATTERN} ]]; then
|
||||
echo "Invalid Agent version: ${VERSION}" >&2
|
||||
echo "Expected format: vMAJOR.MINOR.PATCH or vMAJOR.MINOR.PATCH-suffix" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "${DIST_DIR}"
|
||||
|
||||
build_one() {
|
||||
local goarch="$1"
|
||||
local output="${DIST_DIR}/incudal-agent-linux-${goarch}"
|
||||
|
||||
echo "Building ${output} (version=${VERSION})"
|
||||
(
|
||||
cd "${ROOT_DIR}"
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH="${goarch}" \
|
||||
go build \
|
||||
-trimpath \
|
||||
-buildvcs=false \
|
||||
-gcflags "all=-l" \
|
||||
-ldflags "-s -w -X main.version=${VERSION}" \
|
||||
-o "${output}" \
|
||||
./cmd/incudal-agent
|
||||
)
|
||||
chmod +x "${output}"
|
||||
gzip -9 -c "${output}" > "${output}.gz"
|
||||
}
|
||||
|
||||
build_one amd64
|
||||
build_one arm64
|
||||
|
||||
sha256_file() {
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
}
|
||||
|
||||
size_file() {
|
||||
wc -c < "$1" | tr -d ' '
|
||||
}
|
||||
|
||||
GENERATED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
AMD64_GZ="${DIST_DIR}/incudal-agent-linux-amd64.gz"
|
||||
ARM64_GZ="${DIST_DIR}/incudal-agent-linux-arm64.gz"
|
||||
cat > "${DIST_DIR}/manifest.json" <<EOF_MANIFEST
|
||||
{
|
||||
"version": "${VERSION}",
|
||||
"generatedAt": "${GENERATED_AT}",
|
||||
"files": {
|
||||
"linux-amd64": {
|
||||
"name": "incudal-agent-linux-amd64.gz",
|
||||
"sha256": "$(sha256_file "${AMD64_GZ}")",
|
||||
"size": $(size_file "${AMD64_GZ}"),
|
||||
"gzip": true
|
||||
},
|
||||
"linux-arm64": {
|
||||
"name": "incudal-agent-linux-arm64.gz",
|
||||
"sha256": "$(sha256_file "${ARM64_GZ}")",
|
||||
"size": $(size_file "${ARM64_GZ}"),
|
||||
"gzip": true
|
||||
}
|
||||
}
|
||||
}
|
||||
EOF_MANIFEST
|
||||
|
||||
ls -lh "${DIST_DIR}"/incudal-agent-linux-* "${DIST_DIR}/manifest.json"
|
||||
Reference in New Issue
Block a user