Update docker CPU metrics and add OpenRC installer support (Refs #255)
This commit is contained in:
parent
6eb1a10d9b
commit
adda6eea38
2 changed files with 227 additions and 56 deletions
|
|
@ -81,26 +81,34 @@ const (
|
||||||
|
|
||||||
// Agent collects Docker metrics and posts them to Pulse.
|
// Agent collects Docker metrics and posts them to Pulse.
|
||||||
type Agent struct {
|
type Agent struct {
|
||||||
cfg Config
|
cfg Config
|
||||||
docker *client.Client
|
docker *client.Client
|
||||||
daemonHost string
|
daemonHost string
|
||||||
runtime RuntimeKind
|
runtime RuntimeKind
|
||||||
runtimeVer string
|
runtimeVer string
|
||||||
supportsSwarm bool
|
supportsSwarm bool
|
||||||
httpClients map[bool]*http.Client
|
httpClients map[bool]*http.Client
|
||||||
logger zerolog.Logger
|
logger zerolog.Logger
|
||||||
machineID string
|
machineID string
|
||||||
hostName string
|
hostName string
|
||||||
cpuCount int
|
cpuCount int
|
||||||
targets []TargetConfig
|
targets []TargetConfig
|
||||||
allowedStates map[string]struct{}
|
allowedStates map[string]struct{}
|
||||||
stateFilters []string
|
stateFilters []string
|
||||||
hostID string
|
hostID string
|
||||||
|
prevContainerCPU map[string]cpuSample
|
||||||
}
|
}
|
||||||
|
|
||||||
// ErrStopRequested indicates the agent should terminate gracefully after acknowledging a stop command.
|
// ErrStopRequested indicates the agent should terminate gracefully after acknowledging a stop command.
|
||||||
var ErrStopRequested = errors.New("docker host stop requested")
|
var ErrStopRequested = errors.New("docker host stop requested")
|
||||||
|
|
||||||
|
type cpuSample struct {
|
||||||
|
totalUsage uint64
|
||||||
|
systemUsage uint64
|
||||||
|
onlineCPUs uint32
|
||||||
|
read time.Time
|
||||||
|
}
|
||||||
|
|
||||||
// New creates a new Docker agent instance.
|
// New creates a new Docker agent instance.
|
||||||
func New(cfg Config) (*Agent, error) {
|
func New(cfg Config) (*Agent, error) {
|
||||||
targets, err := normalizeTargets(cfg.Targets)
|
targets, err := normalizeTargets(cfg.Targets)
|
||||||
|
|
@ -212,19 +220,20 @@ func New(cfg Config) (*Agent, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
agent := &Agent{
|
agent := &Agent{
|
||||||
cfg: cfg,
|
cfg: cfg,
|
||||||
docker: dockerClient,
|
docker: dockerClient,
|
||||||
daemonHost: dockerClient.DaemonHost(),
|
daemonHost: dockerClient.DaemonHost(),
|
||||||
runtime: runtimeKind,
|
runtime: runtimeKind,
|
||||||
runtimeVer: info.ServerVersion,
|
runtimeVer: info.ServerVersion,
|
||||||
supportsSwarm: runtimeKind == RuntimeDocker,
|
supportsSwarm: runtimeKind == RuntimeDocker,
|
||||||
httpClients: httpClients,
|
httpClients: httpClients,
|
||||||
logger: *logger,
|
logger: *logger,
|
||||||
machineID: machineID,
|
machineID: machineID,
|
||||||
hostName: hostName,
|
hostName: hostName,
|
||||||
targets: cfg.Targets,
|
targets: cfg.Targets,
|
||||||
allowedStates: make(map[string]struct{}, len(stateFilters)),
|
allowedStates: make(map[string]struct{}, len(stateFilters)),
|
||||||
stateFilters: stateFilters,
|
stateFilters: stateFilters,
|
||||||
|
prevContainerCPU: make(map[string]cpuSample),
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, state := range stateFilters {
|
for _, state := range stateFilters {
|
||||||
|
|
@ -702,6 +711,7 @@ func (a *Agent) collectContainers(ctx context.Context) ([]agentsdocker.Container
|
||||||
}
|
}
|
||||||
|
|
||||||
containers := make([]agentsdocker.Container, 0, len(list))
|
containers := make([]agentsdocker.Container, 0, len(list))
|
||||||
|
active := make(map[string]struct{}, len(list))
|
||||||
for _, summary := range list {
|
for _, summary := range list {
|
||||||
if len(a.allowedStates) > 0 {
|
if len(a.allowedStates) > 0 {
|
||||||
if _, ok := a.allowedStates[strings.ToLower(summary.State)]; !ok {
|
if _, ok := a.allowedStates[strings.ToLower(summary.State)]; !ok {
|
||||||
|
|
@ -709,6 +719,8 @@ func (a *Agent) collectContainers(ctx context.Context) ([]agentsdocker.Container
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
active[summary.ID] = struct{}{}
|
||||||
|
|
||||||
container, err := a.collectContainer(ctx, summary)
|
container, err := a.collectContainer(ctx, summary)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.logger.Warn().Str("container", strings.Join(summary.Names, ",")).Err(err).Msg("Failed to collect container stats")
|
a.logger.Warn().Str("container", strings.Join(summary.Names, ",")).Err(err).Msg("Failed to collect container stats")
|
||||||
|
|
@ -716,9 +728,22 @@ func (a *Agent) collectContainers(ctx context.Context) ([]agentsdocker.Container
|
||||||
}
|
}
|
||||||
containers = append(containers, container)
|
containers = append(containers, container)
|
||||||
}
|
}
|
||||||
|
a.pruneStaleCPUSamples(active)
|
||||||
return containers, nil
|
return containers, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (a *Agent) pruneStaleCPUSamples(active map[string]struct{}) {
|
||||||
|
if len(a.prevContainerCPU) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for id := range a.prevContainerCPU {
|
||||||
|
if _, ok := active[id]; !ok {
|
||||||
|
delete(a.prevContainerCPU, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (a *Agent) collectContainer(ctx context.Context, summary types.Container) (agentsdocker.Container, error) {
|
func (a *Agent) collectContainer(ctx context.Context, summary types.Container) (agentsdocker.Container, error) {
|
||||||
const perContainerTimeout = 15 * time.Second
|
const perContainerTimeout = 15 * time.Second
|
||||||
|
|
||||||
|
|
@ -751,9 +776,11 @@ func (a *Agent) collectContainer(ctx context.Context, summary types.Container) (
|
||||||
return agentsdocker.Container{}, fmt.Errorf("decode stats: %w", err)
|
return agentsdocker.Container{}, fmt.Errorf("decode stats: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
cpuPercent = calculateCPUPercent(stats, a.cpuCount)
|
cpuPercent = a.calculateContainerCPUPercent(summary.ID, stats)
|
||||||
memUsage, memLimit, memPercent = calculateMemoryUsage(stats)
|
memUsage, memLimit, memPercent = calculateMemoryUsage(stats)
|
||||||
blockIO = summarizeBlockIO(stats)
|
blockIO = summarizeBlockIO(stats)
|
||||||
|
} else {
|
||||||
|
delete(a.prevContainerCPU, summary.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
createdAt := time.Unix(summary.Created, 0)
|
createdAt := time.Unix(summary.Created, 0)
|
||||||
|
|
@ -1256,6 +1283,74 @@ func summarizeBlockIO(stats containertypes.StatsResponse) *agentsdocker.Containe
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (a *Agent) calculateContainerCPUPercent(id string, stats containertypes.StatsResponse) float64 {
|
||||||
|
current := cpuSample{
|
||||||
|
totalUsage: stats.CPUStats.CPUUsage.TotalUsage,
|
||||||
|
systemUsage: stats.CPUStats.SystemUsage,
|
||||||
|
onlineCPUs: stats.CPUStats.OnlineCPUs,
|
||||||
|
read: stats.Read,
|
||||||
|
}
|
||||||
|
|
||||||
|
percent := calculateCPUPercent(stats, a.cpuCount)
|
||||||
|
if percent > 0 {
|
||||||
|
a.prevContainerCPU[id] = current
|
||||||
|
return percent
|
||||||
|
}
|
||||||
|
|
||||||
|
prev, ok := a.prevContainerCPU[id]
|
||||||
|
a.prevContainerCPU[id] = current
|
||||||
|
if !ok {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
var totalDelta float64
|
||||||
|
if current.totalUsage >= prev.totalUsage {
|
||||||
|
totalDelta = float64(current.totalUsage - prev.totalUsage)
|
||||||
|
} else {
|
||||||
|
// Counter likely reset (container restart); fall back to current reading.
|
||||||
|
totalDelta = float64(current.totalUsage)
|
||||||
|
}
|
||||||
|
|
||||||
|
if totalDelta <= 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
onlineCPUs := current.onlineCPUs
|
||||||
|
if onlineCPUs == 0 {
|
||||||
|
onlineCPUs = prev.onlineCPUs
|
||||||
|
}
|
||||||
|
if onlineCPUs == 0 && a.cpuCount > 0 {
|
||||||
|
onlineCPUs = uint32(a.cpuCount)
|
||||||
|
}
|
||||||
|
if onlineCPUs == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
var systemDelta float64
|
||||||
|
if current.systemUsage >= prev.systemUsage {
|
||||||
|
systemDelta = float64(current.systemUsage - prev.systemUsage)
|
||||||
|
} else if current.systemUsage > 0 {
|
||||||
|
systemDelta = float64(current.systemUsage)
|
||||||
|
}
|
||||||
|
|
||||||
|
if systemDelta > 0 {
|
||||||
|
return safeFloat((totalDelta / systemDelta) * float64(onlineCPUs) * 100.0)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !prev.read.IsZero() && !current.read.IsZero() {
|
||||||
|
elapsed := current.read.Sub(prev.read).Seconds()
|
||||||
|
if elapsed > 0 {
|
||||||
|
denominator := elapsed * float64(onlineCPUs) * 1e9
|
||||||
|
if denominator > 0 {
|
||||||
|
cpuPercent := (totalDelta / denominator) * 100.0
|
||||||
|
return safeFloat(cpuPercent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
func calculateCPUPercent(stats containertypes.StatsResponse, hostCPUs int) float64 {
|
func calculateCPUPercent(stats containertypes.StatsResponse, hostCPUs int) float64 {
|
||||||
totalDelta := float64(stats.CPUStats.CPUUsage.TotalUsage - stats.PreCPUStats.CPUUsage.TotalUsage)
|
totalDelta := float64(stats.CPUStats.CPUUsage.TotalUsage - stats.PreCPUStats.CPUUsage.TotalUsage)
|
||||||
systemDelta := float64(stats.CPUStats.SystemUsage - stats.PreCPUStats.SystemUsage)
|
systemDelta := float64(stats.CPUStats.SystemUsage - stats.PreCPUStats.SystemUsage)
|
||||||
|
|
|
||||||
|
|
@ -1461,6 +1461,28 @@ allow_reenroll_if_needed() {
|
||||||
|
|
||||||
allow_reenroll_if_needed "$AGENT_IDENTIFIER"
|
allow_reenroll_if_needed "$AGENT_IDENTIFIER"
|
||||||
|
|
||||||
|
# Determine whether to disable auto-update (development server)
|
||||||
|
NO_AUTO_UPDATE_FLAG=""
|
||||||
|
if command -v curl &> /dev/null || command -v wget &> /dev/null; then
|
||||||
|
SERVER_INFO_URL="$PRIMARY_URL/api/server/info"
|
||||||
|
SERVER_INFO=""
|
||||||
|
if command -v curl &> /dev/null; then
|
||||||
|
SERVER_INFO=$(curl -fsSL "$SERVER_INFO_URL" 2>/dev/null || true)
|
||||||
|
else
|
||||||
|
SERVER_INFO=$(wget -qO- "$SERVER_INFO_URL" 2>/dev/null || true)
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$SERVER_INFO" ]] && echo "$SERVER_INFO" | grep -q '"isDevelopment"[[:space:]]*:[[:space:]]*true'; then
|
||||||
|
NO_AUTO_UPDATE_FLAG=" --no-auto-update"
|
||||||
|
log_info 'Development server detected – auto-update disabled'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$NO_AUTO_UPDATE_FLAG" ]] && ! "$AGENT_PATH" --help 2>&1 | grep -q -- '--no-auto-update'; then
|
||||||
|
log_warn 'Agent binary lacks --no-auto-update flag; keeping auto-update enabled'
|
||||||
|
NO_AUTO_UPDATE_FLAG=""
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# Check if systemd is available
|
# Check if systemd is available
|
||||||
if ! command -v systemctl &> /dev/null || [ ! -d /etc/systemd/system ]; then
|
if ! command -v systemctl &> /dev/null || [ ! -d /etc/systemd/system ]; then
|
||||||
printf '\n%s\n' '-- Systemd not detected; configuring alternative startup --'
|
printf '\n%s\n' '-- Systemd not detected; configuring alternative startup --'
|
||||||
|
|
@ -1489,6 +1511,87 @@ EOF
|
||||||
log_info 'Log file : /var/log/pulse-docker-agent.log'
|
log_info 'Log file : /var/log/pulse-docker-agent.log'
|
||||||
log_info 'Host visible in Pulse: ~30 seconds'
|
log_info 'Host visible in Pulse: ~30 seconds'
|
||||||
exit 0
|
exit 0
|
||||||
|
elif command -v rc-service >/dev/null 2>&1 && { [ -x /sbin/openrc-run ] || [ -x /bin/openrc-run ] || [ -x /usr/bin/openrc-run ]; }; then
|
||||||
|
log_info 'Detected OpenRC environment'
|
||||||
|
|
||||||
|
log_header 'Preparing service environment'
|
||||||
|
ensure_service_user
|
||||||
|
ensure_service_home
|
||||||
|
ensure_docker_group_membership
|
||||||
|
write_env_file
|
||||||
|
|
||||||
|
OPENRC_SERVICE="/etc/init.d/pulse-docker-agent"
|
||||||
|
OPENRC_PIDFILE="/run/pulse-docker-agent.pid"
|
||||||
|
OPENRC_PIDDIR=$(dirname "$OPENRC_PIDFILE")
|
||||||
|
OPENRC_LOG="/var/log/pulse-docker-agent.log"
|
||||||
|
OPENRC_ENV_FILE="$ENV_FILE"
|
||||||
|
OPENRC_OWNER="$SERVICE_USER_ACTUAL:$SERVICE_GROUP_ACTUAL"
|
||||||
|
|
||||||
|
mkdir -p "$(dirname "$OPENRC_LOG")"
|
||||||
|
if [[ ! -f "$OPENRC_LOG" ]]; then
|
||||||
|
touch "$OPENRC_LOG"
|
||||||
|
fi
|
||||||
|
chown "$SERVICE_USER_ACTUAL":"$SERVICE_GROUP_ACTUAL" "$OPENRC_LOG" >/dev/null 2>&1 || true
|
||||||
|
chmod 0640 "$OPENRC_LOG" >/dev/null 2>&1 || true
|
||||||
|
|
||||||
|
cat > "$OPENRC_SERVICE" <<EOF
|
||||||
|
#!/sbin/openrc-run
|
||||||
|
|
||||||
|
description="Pulse Docker Agent"
|
||||||
|
command="$AGENT_PATH"
|
||||||
|
command_args="--url $PRIMARY_URL --interval $INTERVAL$NO_AUTO_UPDATE_FLAG"
|
||||||
|
command_user="$SERVICE_USER_ACTUAL:$SERVICE_GROUP_ACTUAL"
|
||||||
|
pidfile="$OPENRC_PIDFILE"
|
||||||
|
supervisor="supervise-daemon"
|
||||||
|
output_log="$OPENRC_LOG"
|
||||||
|
error_log="$OPENRC_LOG"
|
||||||
|
|
||||||
|
env_file="$OPENRC_ENV_FILE"
|
||||||
|
log_owner="$OPENRC_OWNER"
|
||||||
|
pid_dir="$OPENRC_PIDDIR"
|
||||||
|
|
||||||
|
depend() {
|
||||||
|
need localmount
|
||||||
|
use net docker
|
||||||
|
after docker
|
||||||
|
}
|
||||||
|
|
||||||
|
start_pre() {
|
||||||
|
if [ -f "\$env_file" ]; then
|
||||||
|
set -a
|
||||||
|
. "\$env_file"
|
||||||
|
set +a
|
||||||
|
fi
|
||||||
|
checkpath --directory --mode 0755 "\$pid_dir"
|
||||||
|
checkpath --file --owner "\$log_owner" --mode 0640 "$OPENRC_LOG"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
chmod +x "$OPENRC_SERVICE"
|
||||||
|
|
||||||
|
if rc-service pulse-docker-agent status >/dev/null 2>&1; then
|
||||||
|
rc-service pulse-docker-agent stop >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
if rc-update add pulse-docker-agent default >/dev/null 2>&1; then
|
||||||
|
log_success 'Added pulse-docker-agent to OpenRC default runlevel'
|
||||||
|
else
|
||||||
|
log_warn 'Failed to add service to default runlevel; run: rc-update add pulse-docker-agent default'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if rc-service pulse-docker-agent start >/dev/null 2>&1; then
|
||||||
|
log_success 'Started pulse-docker-agent service'
|
||||||
|
else
|
||||||
|
log_warn 'Failed to start pulse-docker-agent service; run: rc-service pulse-docker-agent start'
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_header 'Installation complete'
|
||||||
|
log_info 'Agent service enabled via OpenRC'
|
||||||
|
log_info 'Check status : rc-service pulse-docker-agent status'
|
||||||
|
log_info 'Follow logs : tail -f /var/log/pulse-docker-agent.log'
|
||||||
|
log_info 'Host visible in Pulse : ~30 seconds'
|
||||||
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
log_info 'Manual startup environment detected'
|
log_info 'Manual startup environment detected'
|
||||||
|
|
@ -1503,33 +1606,6 @@ EOF
|
||||||
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
||||||
# Check if server is in development mode
|
|
||||||
NO_AUTO_UPDATE_FLAG=""
|
|
||||||
if command -v curl &> /dev/null || command -v wget &> /dev/null; then
|
|
||||||
SERVER_INFO_URL="$PRIMARY_URL/api/server/info"
|
|
||||||
IS_DEV="false"
|
|
||||||
|
|
||||||
if command -v curl &> /dev/null; then
|
|
||||||
SERVER_INFO=$(curl -fsSL "$SERVER_INFO_URL" 2>/dev/null || echo "")
|
|
||||||
elif command -v wget &> /dev/null; then
|
|
||||||
SERVER_INFO=$(wget -qO- "$SERVER_INFO_URL" 2>/dev/null || echo "")
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ -n "$SERVER_INFO" ]] && echo "$SERVER_INFO" | grep -q '"isDevelopment"[[:space:]]*:[[:space:]]*true'; then
|
|
||||||
IS_DEV="true"
|
|
||||||
NO_AUTO_UPDATE_FLAG=" --no-auto-update"
|
|
||||||
log_info 'Development server detected – auto-update disabled'
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ -n "$NO_AUTO_UPDATE_FLAG" ]]; then
|
|
||||||
if ! "$AGENT_PATH" --help 2>&1 | grep -q -- '--no-auto-update'; then
|
|
||||||
log_warn 'Agent binary lacks --no-auto-update flag; keeping auto-update enabled'
|
|
||||||
NO_AUTO_UPDATE_FLAG=""
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
log_header 'Preparing service environment'
|
log_header 'Preparing service environment'
|
||||||
ensure_service_user
|
ensure_service_user
|
||||||
ensure_service_home
|
ensure_service_home
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue