feat(runner): track per-sandbox host-side file descriptor usage #5001
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -11,6 +11,8 @@ import ( | |
| "errors" | ||
| "fmt" | ||
| "log/slog" | ||
| "runtime" | ||
| "strings" | ||
| "sync" | ||
| "time" | ||
|
|
||
|
|
@@ -30,6 +32,9 @@ type CollectorConfig struct { | |
| WindowSize int | ||
| CPUUsageSnapshotInterval time.Duration | ||
| AllocatedResourcesSnapshotInterval time.Duration | ||
| // SandboxFdUsageWarningThresholdPercent is the worst-process fd usage | ||
| // percentage at or above which a sandbox is warned about. | ||
| SandboxFdUsageWarningThresholdPercent float64 | ||
| } | ||
|
|
||
| // Collector collects system metrics | ||
|
|
@@ -47,6 +52,11 @@ type Collector struct { | |
| allocatedDiskGiB float32 | ||
| startedSandboxCount float32 | ||
|
|
||
| // Per-sandbox host-side fd usage sampling (Linux only) | ||
| fdSamplingEnabled bool | ||
| lastFdSandboxIds map[string]struct{} | ||
| fdWarn *fdWarnTracker | ||
|
|
||
| // Intervals for snapshotting metrics in seconds | ||
| cpuUsageSnapshotInterval time.Duration | ||
| allocatedResourcesSnapshotInterval time.Duration | ||
|
|
@@ -82,11 +92,18 @@ func NewCollector(cfg CollectorConfig) *Collector { | |
| cpuRing: ring.New(cfg.WindowSize), | ||
| cpuUsageSnapshotInterval: cfg.CPUUsageSnapshotInterval, | ||
| allocatedResourcesSnapshotInterval: cfg.AllocatedResourcesSnapshotInterval, | ||
| fdSamplingEnabled: runtime.GOOS == "linux", | ||
| lastFdSandboxIds: make(map[string]struct{}), | ||
| fdWarn: newFdWarnTracker(cfg.SandboxFdUsageWarningThresholdPercent), | ||
| } | ||
| } | ||
|
|
||
| // Start begins needed metrics collection processes | ||
| func (c *Collector) Start(ctx context.Context) { | ||
| if !c.fdSamplingEnabled { | ||
| c.log.InfoContext(ctx, "Per-sandbox file descriptor sampling disabled", "goos", runtime.GOOS) | ||
| } | ||
|
|
||
| go c.snapshotCPUUsage(ctx) | ||
| go c.snapshotAllocatedResources(ctx) | ||
| } | ||
|
|
@@ -254,8 +271,19 @@ func (c *Collector) snapshotAllocatedResources(ctx context.Context) { | |
| var totalAllocatedDiskGB float32 = 0 // Disk in GB | ||
| var startedSandboxCount float32 = 0 // Count of running containers | ||
|
|
||
| var seenFdSandboxIds map[string]struct{} | ||
| if c.fdSamplingEnabled { | ||
| seenFdSandboxIds = make(map[string]struct{}) | ||
| } | ||
|
|
||
| for _, ctr := range containers { | ||
| containerJSON, err := c.docker.ContainerInspect(ctx, ctr.ID) | ||
| if err != nil { | ||
| c.log.WarnContext(ctx, "Failed to get allocated resources for container", "container_id", ctr.ID, "error", err) | ||
| continue | ||
| } | ||
|
|
||
| cpu, memory, disk, err := getContainerAllocatedResources(containerJSON) | ||
| if err != nil { | ||
| c.log.WarnContext(ctx, "Failed to get allocated resources for container", "container_id", ctr.ID, "error", err) | ||
| continue | ||
|
|
@@ -270,6 +298,16 @@ func (c *Collector) snapshotAllocatedResources(ctx context.Context) { | |
|
|
||
| // For disk: count all containers (running and stopped) | ||
| totalAllocatedDiskGB += disk | ||
|
|
||
| if c.fdSamplingEnabled && ctr.State == "running" && containerJSON.State != nil && containerJSON.State.Pid > 0 { | ||
| c.sampleContainerFdUsage(ctx, containerJSON, seenFdSandboxIds) | ||
| } | ||
| } | ||
|
|
||
| if c.fdSamplingEnabled { | ||
| c.cleanupFdMetrics(seenFdSandboxIds) | ||
| c.fdWarn.prune(seenFdSandboxIds) | ||
| c.lastFdSandboxIds = seenFdSandboxIds | ||
| } | ||
|
|
||
| // Convert to original API units | ||
|
|
@@ -283,13 +321,85 @@ func (c *Collector) snapshotAllocatedResources(ctx context.Context) { | |
| } | ||
| } | ||
|
|
||
| // sampleContainerFdUsage samples host-side fd usage of a running sandbox | ||
| // container and updates the per-sandbox Prometheus gauges. The container name | ||
| // is the sandbox ID. | ||
| func (c *Collector) sampleContainerFdUsage(ctx context.Context, containerJSON *container.InspectResponse, seen map[string]struct{}) { | ||
| sandboxId := strings.TrimPrefix(containerJSON.Name, "/") | ||
|
|
||
| // The sandbox is present whether or not sampling succeeds below: | ||
| // recording it in seen up front keeps the warn tracker's rate-limit | ||
| // state across transient sampling failures, so recovery does not | ||
| // immediately re-warn. | ||
| seen[sandboxId] = struct{}{} | ||
|
|
||
| usage, err := sampleSandboxFdUsage(procRoot, cgroupRoot, containerJSON.State.Pid) | ||
| if err != nil { | ||
| c.log.DebugContext(ctx, "Failed to sample sandbox fd usage", "sandbox_id", sandboxId, "error", err) | ||
| // cleanupFdMetrics skips seen sandboxes, so drop this sandbox's | ||
| // gauges explicitly: no stale values for the failed tick. | ||
| deleteSandboxFdMetrics(sandboxId) | ||
| return | ||
| } | ||
|
|
||
| common.SandboxOpenFds.WithLabelValues(sandboxId).Set(float64(usage.OpenFds)) | ||
|
Comment on lines
+328
to
+345
Collaborator
Author
There was a problem hiding this comment. Fixed in 2df2ffe: presence is recorded before sampling, so a transient sampling failure no longer prunes warn state; gauges are dropped for the failed tick only. |
||
| common.SandboxFdLimit.WithLabelValues(sandboxId).Set(float64(usage.WorstFdLimit)) | ||
| common.SandboxFdUsagePercent.WithLabelValues(sandboxId).Set(usage.UsagePercent) | ||
|
|
||
| switch c.fdWarn.observe(sandboxId, usage.UsagePercent, time.Now()) { | ||
| case fdWarnEventWarn: | ||
| c.log.WarnContext(ctx, "Sandbox file descriptor usage above threshold", | ||
| "sandbox_id", sandboxId, | ||
| "open_fds", usage.OpenFds, | ||
| "worst_pid", usage.WorstPid, | ||
| "worst_open_fds", usage.WorstOpenFds, | ||
| "fd_limit", usage.WorstFdLimit, | ||
| "usage_percent", usage.UsagePercent, | ||
| "threshold_percent", c.fdWarn.thresholdPercent) | ||
| case fdWarnEventClear: | ||
| c.log.InfoContext(ctx, "Sandbox file descriptor usage recovered", | ||
| "sandbox_id", sandboxId, | ||
| "usage_percent", usage.UsagePercent, | ||
| "threshold_percent", c.fdWarn.thresholdPercent) | ||
| case fdWarnEventNone: | ||
| } | ||
|
|
||
| // Best-effort fd attribution for runtime helper processes (shim and | ||
| // friends) whenever a runtime name is present in HostConfig. dockerd | ||
| // backfills the default runtime name ("runc") at create, so this also | ||
| // runs for default-runtime containers; attribution is best-effort for | ||
| // any runtime. Never feeds the warning/percentage path; on failure the | ||
| // metric is simply absent. | ||
| if containerJSON.HostConfig != nil && containerJSON.HostConfig.Runtime != "" { | ||
| helperFds, err := sampleRuntimeHelperFds(procRoot, containerJSON.State.Pid, usage.memberPids) | ||
| if err != nil { | ||
| c.log.DebugContext(ctx, "Failed to sample runtime helper fd usage", "sandbox_id", sandboxId, "error", err) | ||
| common.SandboxRuntimeHelperOpenFds.DeleteLabelValues(sandboxId) | ||
| } else { | ||
| common.SandboxRuntimeHelperOpenFds.WithLabelValues(sandboxId).Set(float64(helperFds)) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // cleanupFdMetrics deletes gauge label sets for sandboxes that disappeared | ||
| // since the previous snapshot so no stale series linger. | ||
| func (c *Collector) cleanupFdMetrics(seen map[string]struct{}) { | ||
| for sandboxId := range c.lastFdSandboxIds { | ||
| if _, ok := seen[sandboxId]; !ok { | ||
| deleteSandboxFdMetrics(sandboxId) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // deleteSandboxFdMetrics drops every fd gauge label set of a sandbox. | ||
| func deleteSandboxFdMetrics(sandboxId string) { | ||
| common.SandboxOpenFds.DeleteLabelValues(sandboxId) | ||
| common.SandboxFdLimit.DeleteLabelValues(sandboxId) | ||
| common.SandboxFdUsagePercent.DeleteLabelValues(sandboxId) | ||
| common.SandboxRuntimeHelperOpenFds.DeleteLabelValues(sandboxId) | ||
| } | ||
|
|
||
| func getContainerAllocatedResources(containerJSON *container.InspectResponse) (float32, float32, float32, error) { | ||
| if containerJSON.HostConfig == nil { | ||
| return 0, 0, 0, nil | ||
| } | ||
|
|
@@ -313,7 +423,7 @@ func (c *Collector) getContainerAllocatedResources(ctx context.Context, containe | |
| // Disk allocation from StorageOpt (assuming xfs filesystem) | ||
| storageGB, err := common.ParseStorageOptSizeGB(containerJSON.HostConfig.StorageOpt) | ||
| if err != nil { | ||
| return allocatedCpu, allocatedMemory, 0, fmt.Errorf("error parsing storage quota for container %s: %v", containerJSON.ID, err) | ||
| } | ||
|
|
||
| if storageGB > 0 { | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.