Update docker CPU metrics and add OpenRC installer support (Refs #255)
This commit is contained in:
rcourtman
parent
6eb1a10d9b
commit
adda6eea38
2 changed files with 227 additions and 56 deletions
|
|
@ -81,26 +81,34 @@ const (
|
|||
|
||||
// Agent collects Docker metrics and posts them to Pulse.
|
||||
type Agent struct {
|
||||
cfg Config
|
||||
docker *client.Client
|
||||
daemonHost string
|
||||
runtime RuntimeKind
|
||||
runtimeVer string
|
||||
supportsSwarm bool
|
||||
httpClients map[bool]*http.Client
|
||||
logger zerolog.Logger
|
||||
machineID string
|
||||
hostName string
|
||||
cpuCount int
|
||||
targets []TargetConfig
|
||||
allowedStates map[string]struct{}
|
||||
stateFilters []string
|
||||
hostID string
|
||||
cfg Config
|
||||
docker *client.Client
|
||||
daemonHost string
|
||||
runtime RuntimeKind
|
||||
runtimeVer string
|
||||
supportsSwarm bool
|
||||
httpClients map[bool]*http.Client
|
||||
logger zerolog.Logger
|
||||
machineID string
|
||||
hostName string
|
||||
cpuCount int
|
||||
targets []TargetConfig
|
||||
allowedStates map[string]struct{}
|
||||
stateFilters []string
|
||||
hostID string
|
||||
prevContainerCPU map[string]cpuSample
|
||||
}
|
||||
|
||||
// ErrStopRequested indicates the agent should terminate gracefully after acknowledging a stop command.
|
||||
var ErrStopRequested = errors.New("docker host stop requested")
|
||||
|
||||
type cpuSample struct {
|
||||
totalUsage uint64
|
||||
systemUsage uint64
|
||||
onlineCPUs uint32
|
||||
read time.Time
|
||||
}
|
||||
|
||||
// New creates a new Docker agent instance.
|
||||
func New(cfg Config) (*Agent, error) {
|
||||
targets, err := normalizeTargets(cfg.Targets)
|
||||
|
|
@ -212,19 +220,20 @@ func New(cfg Config) (*Agent, error) {
|
|||
}
|
||||
|
||||
agent := &Agent{
|
||||
cfg: cfg,
|
||||
docker: dockerClient,
|
||||
daemonHost: dockerClient.DaemonHost(),
|
||||
runtime: runtimeKind,
|
||||
runtimeVer: info.ServerVersion,
|
||||
supportsSwarm: runtimeKind == RuntimeDocker,
|
||||
httpClients: httpClients,
|
||||
logger: *logger,
|
||||
machineID: machineID,
|
||||
hostName: hostName,
|
||||
targets: cfg.Targets,
|
||||
allowedStates: make(map[string]struct{}, len(stateFilters)),
|
||||
stateFilters: stateFilters,
|
||||
cfg: cfg,
|
||||
docker: dockerClient,
|
||||
daemonHost: dockerClient.DaemonHost(),
|
||||
runtime: runtimeKind,
|
||||
runtimeVer: info.ServerVersion,
|
||||
supportsSwarm: runtimeKind == RuntimeDocker,
|
||||
httpClients: httpClients,
|
||||
logger: *logger,
|
||||
machineID: machineID,
|
||||
hostName: hostName,
|
||||
targets: cfg.Targets,
|
||||
allowedStates: make(map[string]struct{}, len(stateFilters)),
|
||||
stateFilters: stateFilters,
|
||||
prevContainerCPU: make(map[string]cpuSample),
|
||||
}
|
||||
|
||||
for _, state := range stateFilters {
|
||||
|
|
@ -702,6 +711,7 @@ func (a *Agent) collectContainers(ctx context.Context) ([]agentsdocker.Container
|
|||
}
|
||||
|
||||
containers := make([]agentsdocker.Container, 0, len(list))
|
||||
active := make(map[string]struct{}, len(list))
|
||||
for _, summary := range list {
|
||||
if len(a.allowedStates) > 0 {
|
||||
if _, ok := a.allowedStates[strings.ToLower(summary.State)]; !ok {
|
||||
|
|
@ -709,6 +719,8 @@ func (a *Agent) collectContainers(ctx context.Context) ([]agentsdocker.Container
|
|||
}
|
||||
}
|
||||
|
||||
active[summary.ID] = struct{}{}
|
||||
|
||||
container, err := a.collectContainer(ctx, summary)
|
||||
if err != nil {
|
||||
a.logger.Warn().Str("container", strings.Join(summary.Names, ",")).Err(err).Msg("Failed to collect container stats")
|
||||
|
|
@ -716,9 +728,22 @@ func (a *Agent) collectContainers(ctx context.Context) ([]agentsdocker.Container
|
|||
}
|
||||
containers = append(containers, container)
|
||||
}
|
||||
a.pruneStaleCPUSamples(active)
|
||||
return containers, nil
|
||||
}
|
||||
|
||||
func (a *Agent) pruneStaleCPUSamples(active map[string]struct{}) {
|
||||
if len(a.prevContainerCPU) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for id := range a.prevContainerCPU {
|
||||
if _, ok := active[id]; !ok {
|
||||
delete(a.prevContainerCPU, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Agent) collectContainer(ctx context.Context, summary types.Container) (agentsdocker.Container, error) {
|
||||
const perContainerTimeout = 15 * time.Second
|
||||
|
||||
|
|
@ -751,9 +776,11 @@ func (a *Agent) collectContainer(ctx context.Context, summary types.Container) (
|
|||
return agentsdocker.Container{}, fmt.Errorf("decode stats: %w", err)
|
||||
}
|
||||
|
||||
cpuPercent = calculateCPUPercent(stats, a.cpuCount)
|
||||
cpuPercent = a.calculateContainerCPUPercent(summary.ID, stats)
|
||||
memUsage, memLimit, memPercent = calculateMemoryUsage(stats)
|
||||
blockIO = summarizeBlockIO(stats)
|
||||
} else {
|
||||
delete(a.prevContainerCPU, summary.ID)
|
||||
}
|
||||
|
||||
createdAt := time.Unix(summary.Created, 0)
|
||||
|
|
@ -1256,6 +1283,74 @@ func summarizeBlockIO(stats containertypes.StatsResponse) *agentsdocker.Containe
|
|||
}
|
||||
}
|
||||
|
||||
func (a *Agent) calculateContainerCPUPercent(id string, stats containertypes.StatsResponse) float64 {
|
||||
current := cpuSample{
|
||||
totalUsage: stats.CPUStats.CPUUsage.TotalUsage,
|
||||
systemUsage: stats.CPUStats.SystemUsage,
|
||||
onlineCPUs: stats.CPUStats.OnlineCPUs,
|
||||
read: stats.Read,
|
||||
}
|
||||
|
||||
percent := calculateCPUPercent(stats, a.cpuCount)
|
||||
if percent > 0 {
|
||||
a.prevContainerCPU[id] = current
|
||||
return percent
|
||||
}
|
||||
|
||||
prev, ok := a.prevContainerCPU[id]
|
||||
a.prevContainerCPU[id] = current
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
|
||||
var totalDelta float64
|
||||
if current.totalUsage >= prev.totalUsage {
|
||||
totalDelta = float64(current.totalUsage - prev.totalUsage)
|
||||
} else {
|
||||
// Counter likely reset (container restart); fall back to current reading.
|
||||
totalDelta = float64(current.totalUsage)
|
||||
}
|
||||
|
||||
if totalDelta <= 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
onlineCPUs := current.onlineCPUs
|
||||
if onlineCPUs == 0 {
|
||||
onlineCPUs = prev.onlineCPUs
|
||||
}
|
||||
if onlineCPUs == 0 && a.cpuCount > 0 {
|
||||
onlineCPUs = uint32(a.cpuCount)
|
||||
}
|
||||
if onlineCPUs == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
var systemDelta float64
|
||||
if current.systemUsage >= prev.systemUsage {
|
||||
systemDelta = float64(current.systemUsage - prev.systemUsage)
|
||||
} else if current.systemUsage > 0 {
|
||||
systemDelta = float64(current.systemUsage)
|
||||
}
|
||||
|
||||
if systemDelta > 0 {
|
||||
return safeFloat((totalDelta / systemDelta) * float64(onlineCPUs) * 100.0)
|
||||
}
|
||||
|
||||
if !prev.read.IsZero() && !current.read.IsZero() {
|
||||
elapsed := current.read.Sub(prev.read).Seconds()
|
||||
if elapsed > 0 {
|
||||
denominator := elapsed * float64(onlineCPUs) * 1e9
|
||||
if denominator > 0 {
|
||||
cpuPercent := (totalDelta / denominator) * 100.0
|
||||
return safeFloat(cpuPercent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
func calculateCPUPercent(stats containertypes.StatsResponse, hostCPUs int) float64 {
|
||||
totalDelta := float64(stats.CPUStats.CPUUsage.TotalUsage - stats.PreCPUStats.CPUUsage.TotalUsage)
|
||||
systemDelta := float64(stats.CPUStats.SystemUsage - stats.PreCPUStats.SystemUsage)
|
||||
|
|
|
|||
|
|
@ -1461,6 +1461,28 @@ allow_reenroll_if_needed() {
|
|||
|
||||
allow_reenroll_if_needed "$AGENT_IDENTIFIER"
|
||||
|
||||
# Determine whether to disable auto-update (development server)
|
||||
NO_AUTO_UPDATE_FLAG=""
|
||||
if command -v curl &> /dev/null || command -v wget &> /dev/null; then
|
||||
SERVER_INFO_URL="$PRIMARY_URL/api/server/info"
|
||||
SERVER_INFO=""
|
||||
if command -v curl &> /dev/null; then
|
||||
SERVER_INFO=$(curl -fsSL "$SERVER_INFO_URL" 2>/dev/null || true)
|
||||
else
|
||||
SERVER_INFO=$(wget -qO- "$SERVER_INFO_URL" 2>/dev/null || true)
|
||||
fi
|
||||
|
||||
if [[ -n "$SERVER_INFO" ]] && echo "$SERVER_INFO" | grep -q '"isDevelopment"[[:space:]]*:[[:space:]]*true'; then
|
||||
NO_AUTO_UPDATE_FLAG=" --no-auto-update"
|
||||
log_info 'Development server detected – auto-update disabled'
|
||||
fi
|
||||
|
||||
if [[ -n "$NO_AUTO_UPDATE_FLAG" ]] && ! "$AGENT_PATH" --help 2>&1 | grep -q -- '--no-auto-update'; then
|
||||
log_warn 'Agent binary lacks --no-auto-update flag; keeping auto-update enabled'
|
||||
NO_AUTO_UPDATE_FLAG=""
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check if systemd is available
|
||||
if ! command -v systemctl &> /dev/null || [ ! -d /etc/systemd/system ]; then
|
||||
printf '\n%s\n' '-- Systemd not detected; configuring alternative startup --'
|
||||
|
|
@ -1489,6 +1511,87 @@ EOF
|
|||
log_info 'Log file : /var/log/pulse-docker-agent.log'
|
||||
log_info 'Host visible in Pulse: ~30 seconds'
|
||||
exit 0
|
||||
elif command -v rc-service >/dev/null 2>&1 && { [ -x /sbin/openrc-run ] || [ -x /bin/openrc-run ] || [ -x /usr/bin/openrc-run ]; }; then
|
||||
log_info 'Detected OpenRC environment'
|
||||
|
||||
log_header 'Preparing service environment'
|
||||
ensure_service_user
|
||||
ensure_service_home
|
||||
ensure_docker_group_membership
|
||||
write_env_file
|
||||
|
||||
OPENRC_SERVICE="/etc/init.d/pulse-docker-agent"
|
||||
OPENRC_PIDFILE="/run/pulse-docker-agent.pid"
|
||||
OPENRC_PIDDIR=$(dirname "$OPENRC_PIDFILE")
|
||||
OPENRC_LOG="/var/log/pulse-docker-agent.log"
|
||||
OPENRC_ENV_FILE="$ENV_FILE"
|
||||
OPENRC_OWNER="$SERVICE_USER_ACTUAL:$SERVICE_GROUP_ACTUAL"
|
||||
|
||||
mkdir -p "$(dirname "$OPENRC_LOG")"
|
||||
if [[ ! -f "$OPENRC_LOG" ]]; then
|
||||
touch "$OPENRC_LOG"
|
||||
fi
|
||||
chown "$SERVICE_USER_ACTUAL":"$SERVICE_GROUP_ACTUAL" "$OPENRC_LOG" >/dev/null 2>&1 || true
|
||||
chmod 0640 "$OPENRC_LOG" >/dev/null 2>&1 || true
|
||||
|
||||
cat > "$OPENRC_SERVICE" <<EOF
|
||||
#!/sbin/openrc-run
|
||||
|
||||
description="Pulse Docker Agent"
|
||||
command="$AGENT_PATH"
|
||||
command_args="--url $PRIMARY_URL --interval $INTERVAL$NO_AUTO_UPDATE_FLAG"
|
||||
command_user="$SERVICE_USER_ACTUAL:$SERVICE_GROUP_ACTUAL"
|
||||
pidfile="$OPENRC_PIDFILE"
|
||||
supervisor="supervise-daemon"
|
||||
output_log="$OPENRC_LOG"
|
||||
error_log="$OPENRC_LOG"
|
||||
|
||||
env_file="$OPENRC_ENV_FILE"
|
||||
log_owner="$OPENRC_OWNER"
|
||||
pid_dir="$OPENRC_PIDDIR"
|
||||
|
||||
depend() {
|
||||
need localmount
|
||||
use net docker
|
||||
after docker
|
||||
}
|
||||
|
||||
start_pre() {
|
||||
if [ -f "\$env_file" ]; then
|
||||
set -a
|
||||
. "\$env_file"
|
||||
set +a
|
||||
fi
|
||||
checkpath --directory --mode 0755 "\$pid_dir"
|
||||
checkpath --file --owner "\$log_owner" --mode 0640 "$OPENRC_LOG"
|
||||
return 0
|
||||
}
|
||||
EOF
|
||||
|
||||
chmod +x "$OPENRC_SERVICE"
|
||||
|
||||
if rc-service pulse-docker-agent status >/dev/null 2>&1; then
|
||||
rc-service pulse-docker-agent stop >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
if rc-update add pulse-docker-agent default >/dev/null 2>&1; then
|
||||
log_success 'Added pulse-docker-agent to OpenRC default runlevel'
|
||||
else
|
||||
log_warn 'Failed to add service to default runlevel; run: rc-update add pulse-docker-agent default'
|
||||
fi
|
||||
|
||||
if rc-service pulse-docker-agent start >/dev/null 2>&1; then
|
||||
log_success 'Started pulse-docker-agent service'
|
||||
else
|
||||
log_warn 'Failed to start pulse-docker-agent service; run: rc-service pulse-docker-agent start'
|
||||
fi
|
||||
|
||||
log_header 'Installation complete'
|
||||
log_info 'Agent service enabled via OpenRC'
|
||||
log_info 'Check status : rc-service pulse-docker-agent status'
|
||||
log_info 'Follow logs : tail -f /var/log/pulse-docker-agent.log'
|
||||
log_info 'Host visible in Pulse : ~30 seconds'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log_info 'Manual startup environment detected'
|
||||
|
|
@ -1503,33 +1606,6 @@ EOF
|
|||
|
||||
fi
|
||||
|
||||
|
||||
# Check if server is in development mode
|
||||
NO_AUTO_UPDATE_FLAG=""
|
||||
if command -v curl &> /dev/null || command -v wget &> /dev/null; then
|
||||
SERVER_INFO_URL="$PRIMARY_URL/api/server/info"
|
||||
IS_DEV="false"
|
||||
|
||||
if command -v curl &> /dev/null; then
|
||||
SERVER_INFO=$(curl -fsSL "$SERVER_INFO_URL" 2>/dev/null || echo "")
|
||||
elif command -v wget &> /dev/null; then
|
||||
SERVER_INFO=$(wget -qO- "$SERVER_INFO_URL" 2>/dev/null || echo "")
|
||||
fi
|
||||
|
||||
if [[ -n "$SERVER_INFO" ]] && echo "$SERVER_INFO" | grep -q '"isDevelopment"[[:space:]]*:[[:space:]]*true'; then
|
||||
IS_DEV="true"
|
||||
NO_AUTO_UPDATE_FLAG=" --no-auto-update"
|
||||
log_info 'Development server detected – auto-update disabled'
|
||||
fi
|
||||
|
||||
if [[ -n "$NO_AUTO_UPDATE_FLAG" ]]; then
|
||||
if ! "$AGENT_PATH" --help 2>&1 | grep -q -- '--no-auto-update'; then
|
||||
log_warn 'Agent binary lacks --no-auto-update flag; keeping auto-update enabled'
|
||||
NO_AUTO_UPDATE_FLAG=""
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
log_header 'Preparing service environment'
|
||||
ensure_service_user
|
||||
ensure_service_home
|
||||
|
|
|
|||
Loading…
Reference in a new issue