fix(ci3): bind bench memory to its NUMA node to cut variance

Benches pin CPUs (docker --cpuset-cpus / taskset) but not memory. On the
dedicated bench box (m6a.32xlarge, 4 NUMA nodes of 16 cores) the MSM/commitment
phases of proving are memory-bandwidth-bound, so when their buffers land on a
remote node they run ~50% slower while the compute-bound phases (trace gen,
sumcheck) stay flat — the bimodal flip seen in bench history (e.g. avm_bulk
wire commitments 2.1s vs 3.2s).

Bind each bench's memory to its CPUs' NUMA node: --cpuset-mems on the docker
path, numactl --membind on the taskset path (new cpu_numa_nodes helper maps a
CPU list to node ids). Also disable automatic NUMA balancing during the bench
run so the kernel doesn't migrate pages mid-measurement.

Author: charlielye

ci3/bench_engine

@@ -21,6 +21,11 @@ function isolate_bench_cpus {
     sudo sh -c "echo 0 > /sys/devices/system/cpu/cpu$cpu/online" 2>/dev/null || true
   done
 
+  # Each bench's memory is bound to its CPUs' NUMA node (--cpuset-mems / numactl), so the
+  # kernel's automatic page migration only adds jitter to the memory-bandwidth-bound
+  # phases. Turn it off for the run.
+  sudo sh -c "echo 0 > /proc/sys/kernel/numa_balancing" 2>/dev/null || true
+
   export BENCH_CPU_COUNT=$physical
   echo "Benchmark CPU isolation: CPUs 0-$((physical - 1)) ($physical cores, hyperthreads off) for benchmarks."
 }
@@ -33,6 +38,7 @@ function unisolate_bench_cpus {
   for cpu in $(seq 1 $((total_cpus - 1))); do
     sudo sh -c "echo 1 > /sys/devices/system/cpu/cpu$cpu/online" 2>/dev/null || true
   done
+  sudo sh -c "echo 1 > /proc/sys/kernel/numa_balancing" 2>/dev/null || true
   echo "All CPUs re-enabled. Online CPUs: $(nproc)"
 }
 

ci3/docker_isolate

@@ -26,8 +26,14 @@ if [ -n "${NAME:-}" ]; then
   docker rm -f $name &>/dev/null || true
 fi
 
-# For pinning to given CPUs.
-[ -n "${CPU_LIST:-}" ] && cpuset_arg="--cpuset-cpus=$CPU_LIST"
+# For pinning to given CPUs, and binding memory to the same NUMA node(s) so that
+# memory-bandwidth-bound work (e.g. the MSM/commitment phases of proving) doesn't stream
+# from a remote node — the main source of run-to-run variance on the bench box.
+if [ -n "${CPU_LIST:-}" ]; then
+  cpuset_arg="--cpuset-cpus=$CPU_LIST"
+  mems=$(cpu_numa_nodes "$CPU_LIST")
+  [ -n "$mems" ] && cpuset_mems_arg="--cpuset-mems=$mems"
+fi
 
 # Env vars to inject into the container.
 arg_env_vars=()
@@ -60,6 +66,7 @@ cid=$(docker run -d \
   ${name_arg:-} \
   ${network_arg:-} \
   ${cpuset_arg:-} \
+  ${cpuset_mems_arg:-} \
   --cpus=$CPUS \
   --memory=$MEM \
   --user $(id -u):$(id -g) \

ci3/exec_test

@@ -52,7 +52,15 @@ if [ "${ISOLATE:-0}" -eq 1 ]; then
   docker_isolate "timeout -v $TIMEOUT bash -c '$test_cmd'" &> >(add_timestamps)
 else
   [ "${ONLY_TERM_PARENT:-0}" -eq 1 ] && fg_arg="--foreground"
-  taskset -c $CPU_LIST timeout ${fg_arg:-} -v $TIMEOUT bash -c "$test_cmd" &> >(add_timestamps)
+  # Bind memory to the CPU list's NUMA node(s), matching the docker path's --cpuset-mems,
+  # so memory-bandwidth-bound work doesn't stream from a remote node. Needs numactl; if it
+  # (or NUMA topology) is absent, fall back to plain taskset.
+  mems=$(cpu_numa_nodes "$CPU_LIST")
+  if [ -n "$mems" ] && command -v numactl &>/dev/null; then
+    numactl --membind=$mems taskset -c $CPU_LIST timeout ${fg_arg:-} -v $TIMEOUT bash -c "$test_cmd" &> >(add_timestamps)
+  else
+    taskset -c $CPU_LIST timeout ${fg_arg:-} -v $TIMEOUT bash -c "$test_cmd" &> >(add_timestamps)
+  fi
 fi
 code=$?
 set -e

ci3/source_stdlib

@@ -46,6 +46,24 @@ function get_num_cpus_max {
   echo $jobs
 }
 
+# Given a comma-separated CPU list, print the comma-separated set of NUMA node ids those
+# CPUs belong to (sorted, unique). Empty if the box exposes no NUMA topology. Used to bind
+# a pinned bench's memory to the same node(s) as its cores (--cpuset-mems / numactl), so
+# memory-bandwidth-bound work doesn't stream from a remote node.
+function cpu_numa_nodes {
+  local cpu_list="${1:-}"
+  [ -z "$cpu_list" ] && return
+  local cpu d nodes=()
+  for cpu in ${cpu_list//,/ }; do
+    for d in /sys/devices/system/cpu/cpu$cpu/node*; do
+      [ -e "$d" ] || continue
+      nodes+=("${d##*/node}")
+    done
+  done
+  [ "${#nodes[@]}" -eq 0 ] && return
+  printf '%s\n' "${nodes[@]}" | sort -un | paste -sd,
+}
+
 function aws_get_meta_data {
   curl -fs -H "X-aws-ec2-metadata-token: $AWS_TOKEN" http://169.254.169.254/latest/meta-data/$1 || true
 }