cgroup_init: sleepable, and a missing cgrp_ctx is not fatal

.clang-format

@@ -0,0 +1,70 @@
+# Linux kernel style, matching src/bpf/main.bpf.c as written.
+#
+# Without this file clangd falls back to LLVM style (2-space indent, braces on
+# the same line) and editor.formatOnSave rewrites the whole file on first save.
+# Derived from the kernel tree's own .clang-format, trimmed to what this
+# single-file BPF source actually needs.
+---
+Language: Cpp
+BasedOnStyle: LLVM
+
+# Indentation: hard tabs, 8 wide, as in the kernel.
+UseTab: Always
+TabWidth: 8
+IndentWidth: 8
+ContinuationIndentWidth: 8
+ConstructorInitializerIndentWidth: 8
+
+ColumnLimit: 0
+
+# Kernel brace placement: functions get a brace on the next line, everything
+# else keeps it on the same line.
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterFunction: true
+  AfterStruct: true
+  AfterUnion: true
+  AfterEnum: true
+  AfterControlStatement: Never
+  BeforeElse: false
+  BeforeWhile: false
+
+AlignEscapedNewlines: Left
+AlignAfterOpenBracket: Align
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AlwaysBreakBeforeMultilineStrings: false
+BinPackArguments: true
+BinPackParameters: true
+BreakBeforeBinaryOperators: None
+BreakBeforeTernaryOperators: false
+Cpp11BracedListStyle: false
+IndentCaseLabels: false
+IndentGotoLabels: false
+IndentPPDirectives: None
+KeepEmptyLinesAtTheStartOfBlocks: false
+MaxEmptyLinesToKeep: 1
+PointerAlignment: Right
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+SortIncludes: Never
+ReflowComments: false
+
+# BPF/kernel macros that take a block or act as control flow; without these
+# clang-format indents the bodies as if they were expressions.
+ForEachMacros:
+  - bpf_for
+  - bpf_for_each
+  - bpf_repeat
+  - list_for_each_entry
+  - list_for_each_entry_safe
+
+AlignTrailingComments: true
+AlignConsecutiveMacros: false

.envrc

@@ -0,0 +1 @@
+use flake

.gitignore

@@ -4,3 +4,6 @@
 src/bpf/vmlinux.h
 /nixos.qcow2
 /Cargo.lock.orig
+/compile_commands.json
+/.direnv/
+/.cache/

.vscode/settings.json

@@ -0,0 +1,13 @@
+{
+  // main.bpf.c is hand-formatted kernel style with tab-aligned trailing
+  // comments, which clang-format cannot reproduce. The global
+  // editor.formatOnSave would rewrite ~200 lines on every save, so it is off
+  // for C here. .clang-format still exists for deliberate formatting of new
+  // code (right-click -> Format Selection).
+  "[c]": {
+    "editor.formatOnSave": false
+  },
+  "[cpp]": {
+    "editor.formatOnSave": false
+  }
+}

docs/gpu-feeder.md

@@ -0,0 +1,109 @@
+# `--gpu-boost-cap`: don't deboost the render thread
+
+A GPU feeder is a task that submitted GPU work within the last 100 ms, as seen
+by a `tp_btf/i915_request_add` hook: a game's render thread, the compositor,
+Xwayland. BMQ deboosts a task for burning whole slices, which is exactly what a
+render thread does on a heavy frame, and strict priority then queues it behind
+every better level. `--gpu-boost-cap` stops a recent feeder from being
+deboosted past the given boost.
+
+**Result: `-1` removes most of the stutter, and is the default.**
+
+The idea came from reading the Infinity scheduler (galpt/infinity-scheduler),
+whose v4.8 couples CPU and GPU accounting. Its GPU half patches `drm_sched`,
+which i915 does not use, so none of that applies to Ivy Bridge. What carried
+over was the question of whether the CPU side lets the GPU's feeders wait.
+
+## What was run
+
+Half-Life 2 (native GL, Mesa crocus), the same 60 s camera pan each round, on
+the Ivy Bridge target (i5-3210M, 2c/4t, HD 4000, kernel 7.2.8). Frame times
+from MangoHud's CSV log; scheduler numbers from `SORYU_LOG` at one report per
+second. Both settings ran from the same binary, alternating off, on, off, on,
+so the knob is the only thing that differs.
+
+## Finding the cause
+
+The first instrument only saw wake->run, and feeders looked fine there: p99
+around 0.5-0.8 ms with or without load. The render thread's problem is not
+waking up. It rarely sleeps mid-frame; it burns its slice, stops still
+runnable, and waits to get the CPU back -- a path that never passes
+`ops.runnable` and so was invisible.
+
+Measuring that requeue wait, split by the feeder's boost at the moment it
+stopped (one 60 s pan, knob off):
+
+| | p99, median second | p99.9, worst second | seconds with p99.9 > 4 ms |
+| --- | --- | --- | --- |
+| boosted (boost < 0) | 0.92 ms | 3.1 ms | 0 of 72 |
+| deboosted (boost >= 0) | 1.0 ms | **25.2 ms** | **6 of 61** |
+
+Boosted, the wait never went past a few slices. Deboosted, it reached two
+frames at 85 fps. Feeder boost spent that run at a median of -7.6, so the
+render thread was crossing into the deboosted class constantly.
+
+## Frames, idle desktop
+
+| | median frame | p99.9 frame | frames > 2x median | frames > 25 ms | 0.1% low |
+| --- | --- | --- | --- | --- | --- |
+| off, round 1 | 13.2 ms | 44.0 ms | 15 | 27 | 22.6 fps |
+| **-1, round 1** | **12.5 ms** | **26.1 ms** | **6** | **6** | **37.8 fps** |
+| off, round 2 | 12.7 ms | 43.2 ms | 23 | 26 | 21.7 fps |
+| **-1, round 2** | **12.5 ms** | **25.7 ms** | **8** | **8** | **37.9 fps** |
+
+The two rounds of each setting agree closely. Frames over 25 ms fell by about
+70%.
+
+On the scheduler side, deboosted feeder requeues went from ~785/s to ~1/s,
+while the cap itself fired only ~35 times a second -- a small intervention.
+
+## Under load
+
+The same pan with `stress-ng --cpu 4 --timeout 60s` running alongside:
+
+| | avg fps | 1% low | 0.1% low | stress-ng ops/s (real) | ops/s (usr+sys) |
+| --- | --- | --- | --- | --- | --- |
+| off | 60.8 | 30.0 | 21.2 | 1026 | 778 |
+| **-1** | **73.3** | **46.6** | **36.8** | 750 | 753 |
+
+## The cost
+
+stress-ng lost 27% of its throughput. Its ops per CPU-second barely moved (778
+against 753), so it did not run slower -- it got less CPU, 60 CPU-seconds
+instead of 79 over the minute, and the game took the difference. That is the
+trade the knob exists to make.
+
+Nothing starved. The worst wait was ~130 jiffies in both rounds, which is the
+`starve_ns` bound itself, and `starve_serve` moved from 1.2/s to 1.5/s.
+
+On an idle desktop the other side of the trade shows up in the `deboosted`
+class: its wake->run p99 went from ~0.5 ms to 1.0-1.3 ms.
+
+The cap applies to anything that submits GPU work, including an OpenCL render
+(DaVinci, beignet/pocl). A build running alongside one will lose a similar
+share. That was judged acceptable for a desktop; `--gpu-boost-cap=12` turns it
+off.
+
+## Dropped: waking out of a GPU wait
+
+The first version also hooked `i915_request_wait_begin/end` and
+`dma_fence_wait_start/end`, to time feeders waking from a GPU wait. Over 20
+minutes of play that saw 1251 such wakeups against 434k submissions: crocus
+does not block in the kernel for its fences, so there was nothing there to
+schedule. The hooks were removed.
+
+## For reference: EEVDF
+
+With soryu unloaded, the same game under `stress-ng --cpu 4` ran at 8-11 fps
+(read off the overlay, one run), against 40-50 for soryu without the cap. On an
+idle desktop EEVDF's 1% low was 47.7 against 43.2 for soryu without the cap,
+which is the stutter this change removes; with it, soryu's 1% low is 46.6-47.3.
+
+## Open
+
+- One game. A Vulkan title through hasvk, and a game whose render thread
+  rarely burns a full slice, would say whether -1 is right in general or only
+  for this workload.
+- No AMD numbers. Without `/sys/kernel/btf/i915` the hook does not load and the
+  cap has no feeders to act on; an amdgpu equivalent would need its own
+  tracepoint.

src/main.rs

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 //
+//
 // scx_soryu loader.
 //
 // runs in the foreground and holds no state of its own: dropping
@@ -9,6 +10,7 @@
 mod bpf_skel;
 mod stats;
 use bpf_skel::*;
+use stats::fmt_ns;
 use stats::Metrics;
 
 use std::mem::MaybeUninit;
@@ -35,46 +37,170 @@ use scx_utils::UserExitInfo;
 
 const SCHEDULER_NAME: &str = "scx_soryu";
 
-/// mirrors `enum stat_idx` in main.bpf.c.
+/// mirrors `enum soryu_stat_idx` in main.bpf.c, in order.
-const STAT_NAMES: &[&str] = &[
+///
-    "enq_shared",
+/// The names and the indices come from the same list on purpose. They used to
-    "enq_local",
+/// be a `STAT_NAMES` array plus hand-written `g(0)`..`g(15)` calls in
-    "enq_reenq",
+/// `metrics()`, with nothing tying the two together: inserting one counter in
-    "enq_pinned",
+/// the middle of the BPF enum would have silently shifted every metric after
-    "dispatch",
+/// it, and the numbers would still have looked plausible.
-    "insert_fail",
+macro_rules! soryu_stats {
-    "cpu_release",
+    ($($name:ident),+ $(,)?) => {
-    "boost_sleep",
+        const STAT_NAMES: &[&str] = &[$(stringify!($name)),+];
-    "boost_wake",
+
-    "deboost",
+        #[allow(non_upper_case_globals)]
-    "slice_carry",
+        mod stat {
-    "fork_split",
+            soryu_stats!(@idx 0usize; $($name),+);
-    "preempt_kick",
+        }
-    "starve_serve",
+    };
-    "preempt_none",
+    (@idx $n:expr; $head:ident $(, $tail:ident)*) => {
-    "preempt_idle",
+        pub const $head: usize = $n;
-];
+        soryu_stats!(@idx $n + 1; $($tail),*);
+    };
+    (@idx $n:expr;) => {};
+}
+
+soryu_stats!(
+    enq_shared,
+    enq_local,
+    enq_reenq,
+    enq_pinned,
+    dispatch,
+    insert_fail,
+    cpu_release,
+    boost_sleep,
+    boost_wake,
+    deboost,
+    slice_carry,
+    fork_split,
+    preempt_kick,
+    starve_serve,
+    preempt_none,
+    preempt_idle,
+    dispatch_miss,
+    starve_blocked,
+    bitmap_repair,
+    wake_donate,
+    gpu_submit,
+    gpu_cap,
+);
 
 /// mirrors BOOST_RANGE in main.bpf.c.
 const BOOST_RANGE: i32 = 12;
 
-/// mirrors LAT_CLASSES / LAT_BUCKETS in main.bpf.c.
+/// mirrors ULE_BUCKETS / ULE_MAX in main.bpf.c.
+const ULE_BUCKETS: usize = 20;
+const ULE_MAX: usize = 100;
+
+/// mirrors LAT_CLASSES / LAT_OCTAVES / LAT_SUBS in main.bpf.c.
 const LAT_CLASSES: usize = 3;
-const LAT_BUCKETS: usize = 32;
+const LAT_OCTAVES: usize = 32;
+const LAT_SUBS: usize = 4;
+const LAT_BUCKETS: usize = LAT_OCTAVES * LAT_SUBS;
 const LAT_CLASS_NAMES: [&str; LAT_CLASSES] = ["boosted", "neutral", "deboosted"];
 
+/// mirrors GPU_POPS in main.bpf.c.
+const GPU_POPS: usize = 3;
+const GPU_POP_NAMES: [&str; GPU_POPS] = [
+    "gpu-feeder",
+    "gpu-requeue boosted",
+    "gpu-requeue deboosted",
+];
+
+/// Module BTF the i915 tracepoints resolve against. Absent means no i915 --
+/// the test VM, or an AMD machine -- and those programs must not be loaded,
+/// or the whole object fails with them.
+const I915_BTF: &str = "/sys/kernel/btf/i915";
+
+/// Lower edge of bucket @i in ns: mantissa (4 + sub) / 4, scaled by 2^octave.
+/// Inverse of lat_bucket() in main.bpf.c.
+fn bucket_ns(i: usize) -> u64 {
+    let oct = (i / LAT_SUBS) as u32;
+    let sub = (i % LAT_SUBS) as u64;
+
+    ((4 + sub) << oct) >> 2
+}
+
+/// First bucket whose lower edge is at or above @ns, for the tail report.
+fn bucket_at_least(ns: u64) -> usize {
+    (0..LAT_BUCKETS)
+        .find(|&i| bucket_ns(i) >= ns)
+        .unwrap_or(LAT_BUCKETS - 1)
+}
+
+/// Elementwise cur - prev, so a report describes its own interval rather than
+/// everything since load.
+fn hist_delta(cur: &[u64], prev: &[u64]) -> Vec<u64> {
+    cur.iter()
+        .enumerate()
+        .map(|(i, &c)| c.saturating_sub(prev.get(i).copied().unwrap_or(0)))
+        .collect()
+}
+
 #[derive(Parser, Debug, Clone)]
 #[command(about = "BMQ policy port for sched_ext", version)]
 struct Opts {
     #[arg(long, default_value_t = 5000, env = "SORYU_TIMEOUT_MS")]
     timeout_ms: u32,
-    #[arg(long, default_value_t = true, env = "SORYU_BOOST_ON_WAKE",
+    /// Boost on wakeup in proportion to time slept. Off by default: measured
+    /// worse (docs/boost-on-wake.md). Takes a value -- `--boost-on-wake=true`.
+    #[arg(long, default_value_t = false, env = "SORYU_BOOST_ON_WAKE",
           action = clap::ArgAction::Set)]
     boost_on_wake: bool,
+    /// Clear bitmap bits in ops.dispatch, with the repair sweep that makes it
+    /// safe. Off by default: measured worse (docs/bitmap-clear.md). Takes a
+    /// value -- `--clear-bitmap=true`.
+    #[arg(long, default_value_t = false, env = "SORYU_CLEAR_BITMAP",
+          action = clap::ArgAction::Set)]
+    clear_bitmap: bool,
+    /// Throw the local DSQ back through ops.enqueue when a higher sched class
+    /// takes the CPU. On by default. See main.bpf.c for the risk. Takes a
+    /// value -- `--reenqueue-on-release=false`.
+    #[arg(long, default_value_t = true, env = "SORYU_REENQUEUE_ON_RELEASE",
+          action = clap::ArgAction::Set)]
+    reenqueue_on_release: bool,
     #[arg(long, default_value_t = 1000, env = "SORYU_SLICE_US")]
     slice_us: u64,
     #[arg(long, default_value_t = 128, env = "SORYU_STARVE_MS")]
     starve_ms: u64,
+
+    /// Levels the FreeBSD ULE interactivity score may move a task, in each
+    /// direction, on top of BMQ's boost. 0 disables it and restores BMQ.
+    /// The score is measured either way -- see the ule line in the report.
+    #[arg(long, default_value_t = 0, env = "SORYU_ULE_WEIGHT")]
+    ule_weight: i32,
+
+    /// Combined run+sleep history the ULE score keeps before halving both.
+    /// ULE's SCHED_SLP_RUN_MAX.
+    #[arg(long, default_value_t = 5000, env = "SORYU_ULE_WINDOW_MS")]
+    ule_window_ms: u64,
+
+    /// Levels a cgroup's cpu.weight may move its tasks, in each direction.
+    /// 0 disables it, which is what this scheduler did until now -- systemd's
+    /// CPUWeight= had no effect at all. Changes fairness between groups, not
+    /// latency.
+    #[arg(long, default_value_t = 0, env = "SORYU_CGRP_WEIGHT_LEVELS")]
+    cgrp_weight_levels: i32,
+
+    /// Boost steps a waker may hand down to the task it wakes, capped by the
+    /// gap between them. 0 disables it.
+    #[arg(long, default_value_t = 0, env = "SORYU_WAKE_DONATE")]
+    wake_donate: i32,
+
+    /// Order each level DSQ by virtual time rather than FIFO -- within a level
+    /// only, so BMQ's strict priority between levels is untouched. Off by
+    /// default. Takes a value -- `--vtime-within-level=true`.
+    #[arg(long, default_value_t = false, env = "SORYU_VTIME_WITHIN_LEVEL",
+          action = clap::ArgAction::Set)]
+    vtime_within_level: bool,
+
+    /// Boost a recent GPU feeder may not be deboosted past, in [-12, 12].
+    /// The default -1 keeps feeders in the boosted class; 12 turns it off.
+    /// Measured in docs/gpu-feeder.md.
+    #[arg(long, default_value_t = -1, env = "SORYU_GPU_BOOST_CAP",
+          value_parser = clap::value_parser!(i32).range(-12..=12))]
+    gpu_boost_cap: i32,
+
     #[arg(long, default_value_t = 1, env = "SORYU_REPORT_SEC")]
     report_sec: u64,
     #[arg(long, env = "SORYU_LOG")]
@@ -100,30 +226,48 @@ fn percentiles(buckets: &[u64]) -> Option<(u64, u64, u64)> {
     }
 
     let at = |frac: f64| -> u64 {
-        let target = (total as f64 * frac) as u64;
+        // ceil, and never zero: truncating gave target = 0 for small n, and
+        // `seen >= 0` is true before any bucket has been added, so p50 of a
+        // single sample reported bucket 0 (1ns) no matter where the sample
+        // actually landed.
+        let target = ((total as f64 * frac).ceil() as u64).max(1);
         let mut seen = 0u64;
         for (i, &c) in buckets.iter().enumerate() {
             seen += c;
             if seen >= target {
-                return 1u64 << i;
+                return bucket_ns(i);
             }
         }
-        1u64 << (buckets.len() - 1)
+        bucket_ns(buckets.len() - 1)
     };
 
     Some((at(0.50), at(0.99), at(0.999)))
 }
 
-fn fmt_ns(ns: u64) -> String {
+/// Mean and occupied-bucket distribution of a boost histogram, None if empty.
-    if ns >= 1_000_000_000 {
+fn boost_summary(hist: &[u64]) -> Option<(f64, String)> {
-        format!("{:.1}s", ns as f64 / 1e9)
+    let total: u64 = hist.iter().sum();
-    } else if ns >= 1_000_000 {
+    if total == 0 {
-        format!("{}ms", ns / 1_000_000)
+        return None;
-    } else if ns >= 1_000 {
-        format!("{}us", ns / 1_000)
-    } else {
-        format!("{ns}ns")
     }
+
+    let occupied: Vec<String> = hist
+        .iter()
+        .enumerate()
+        .filter(|(_, &c)| c > 0)
+        .map(|(i, &c)| {
+            let b = i as i32 - BOOST_RANGE;
+            format!("{b:+}:{:.1}%", 100.0 * c as f64 / total as f64)
+        })
+        .collect();
+
+    let weighted: i64 = hist
+        .iter()
+        .enumerate()
+        .map(|(i, &c)| (i as i64 - BOOST_RANGE as i64) * c as i64)
+        .sum();
+
+    Some((weighted as f64 / total as f64, occupied.join(" ")))
 }
 
 struct Scheduler<'a> {
@@ -132,6 +276,28 @@ struct Scheduler<'a> {
     struct_ops: Option<libbpf_rs::Link>,
     stats_server: StatsServer<(), Metrics>,
     prev_waits: Vec<u64>,
+
+    /// Histogram snapshots from the previous report, so each line describes
+    /// its own interval.
+    ///
+    /// lat_hist and boost_hist accumulate from load and were printed raw, so a
+    /// report was the average of every phase the machine had been through. A
+    /// run that idled for 5 s and then got busy for 10 s printed one blended
+    /// p50 that described neither -- and it moved 16us -> 8us, which reads
+    /// like an improvement and is really just the mixture shifting. The
+    /// counters were already deltaed for `--monitor`; the histograms were not
+    /// deltaed anywhere.
+    prev_lat: Vec<u64>,
+    prev_boost: Vec<u64>,
+    prev_ule: Vec<u64>,
+    prev_bounce: Vec<u64>,
+    prev_release: Vec<u64>,
+    prev_gpu_lat: Vec<u64>,
+    prev_gpu_boost: Vec<u64>,
+
+    /// Separate snapshots for the stats server: it samples on its own schedule
+    /// and must not consume the report's intervals, or vice versa.
+    stats_prev_lat: Vec<u64>,
 }
 
 impl<'a> Scheduler<'a> {
@@ -153,27 +319,47 @@ impl<'a> Scheduler<'a> {
             .rodata_data
             .as_mut()
             .context("no rodata section in skeleton")?;
-        rodata.nr_cpu_ids = *scx_utils::NR_CPU_IDS as u32;
 
-        // SORYU_BOOST_ON_WAKE=0 reproduces BMQ as shipped (its wake boost is
+        // The default reproduces BMQ as shipped, whose wake boost is dead
-        // dead code); the default reproduces what BMQ meant to do.
+        // code. SORYU_BOOST_ON_WAKE=1 turns on what BMQ meant to do, which
+        // measured worse -- see docs/boost-on-wake.md.
         rodata.boost_on_wake = opts.boost_on_wake;
         rodata.starve_ns = opts.starve_ms * 1_000_000;
+        rodata.clear_bitmap = opts.clear_bitmap;
+        rodata.reenqueue_on_release = opts.reenqueue_on_release;
+        rodata.ule_weight = opts.ule_weight;
+        rodata.ule_window_ns = opts.ule_window_ms * 1_000_000;
+        rodata.cgrp_weight_levels = opts.cgrp_weight_levels;
+        rodata.wake_donate = opts.wake_donate;
+        rodata.vtime_within_level = opts.vtime_within_level;
+        rodata.gpu_boost_cap = opts.gpu_boost_cap;
 
         rodata.slice_ns = opts.slice_us * 1_000;
 
+        let have_i915 = std::path::Path::new(I915_BTF).exists();
+        if !have_i915 {
+            skel.progs.soryu_i915_request_add.set_autoload(false);
+            info!("no {I915_BTF}: no GPU feeders, gpu_boost_cap has nothing to act on");
+        }
+
         let mut skel = scx_ops_load!(skel, soryu_ops, uei)?;
         let struct_ops = Some(scx_ops_attach!(skel, soryu_ops)?);
 
         let rd = skel.maps.rodata_data.as_ref().unwrap();
         info!(
             "{} attached: {} CPUs, slice {} us, boost_on_wake={}, \
-             starve {} ms, timeout {} ms",
+             starve {} ms, clear_bitmap={}, ule_weight={} (window {} ms), \
+             vtime_within_level={}, gpu_boost_cap={}, timeout {} ms",
             SCHEDULER_NAME,
             *scx_utils::NR_CPU_IDS,
             rd.slice_ns / 1000,
             rd.boost_on_wake,
             rd.starve_ns / 1_000_000,
+            rd.clear_bitmap,
+            rd.ule_weight,
+            rd.ule_window_ns / 1_000_000,
+            rd.vtime_within_level,
+            rd.gpu_boost_cap,
             opts.timeout_ms
         );
 
@@ -185,6 +371,14 @@ impl<'a> Scheduler<'a> {
             struct_ops,
             stats_server,
             prev_waits: Vec::new(),
+            prev_lat: Vec::new(),
+            prev_boost: Vec::new(),
+            prev_ule: Vec::new(),
+            prev_bounce: Vec::new(),
+            prev_release: Vec::new(),
+            prev_gpu_lat: Vec::new(),
+            prev_gpu_boost: Vec::new(),
+            stats_prev_lat: Vec::new(),
         })
     }
 
@@ -217,55 +411,71 @@ impl<'a> Scheduler<'a> {
     }
 
     /// One snapshot for the stats server.
-    fn metrics(&self) -> Metrics {
+    ///
+    /// The counters go out cumulative and `Metrics::delta` differences them at
+    /// the far end; the latency percentiles cannot be differenced that way, so
+    /// they are computed here from the interval histogram directly.
+    fn metrics(&mut self) -> Metrics {
         let c = self.read_stats().unwrap_or_default();
         let g = |i: usize| c.get(i).copied().unwrap_or(0);
 
         let mut m = Metrics {
-            enq_shared: g(0),
+            enq_shared: g(stat::enq_shared),
-            enq_local: g(1),
+            enq_local: g(stat::enq_local),
-            enq_reenq: g(2),
+            enq_reenq: g(stat::enq_reenq),
-            enq_pinned: g(3),
+            enq_pinned: g(stat::enq_pinned),
-            insert_fail: g(5),
+            dispatch: g(stat::dispatch),
-            cpu_release: g(6),
+            insert_fail: g(stat::insert_fail),
-            boost_sleep: g(7),
+            cpu_release: g(stat::cpu_release),
-            boost_wake: g(8),
+            boost_sleep: g(stat::boost_sleep),
-            deboost: g(9),
+            boost_wake: g(stat::boost_wake),
-            slice_carry: g(10),
+            deboost: g(stat::deboost),
-            fork_split: g(11),
+            slice_carry: g(stat::slice_carry),
-            preempt_kick: g(12),
+            fork_split: g(stat::fork_split),
-            starve_serve: g(13),
+            preempt_kick: g(stat::preempt_kick),
-            preempt_none: g(14),
+            starve_serve: g(stat::starve_serve),
-            preempt_idle: g(15),
+            preempt_none: g(stat::preempt_none),
+            preempt_idle: g(stat::preempt_idle),
+            dispatch_miss: g(stat::dispatch_miss),
+            starve_blocked: g(stat::starve_blocked),
+            bitmap_repair: g(stat::bitmap_repair),
+            wake_donate: g(stat::wake_donate),
+            gpu_submit: g(stat::gpu_submit),
+            gpu_cap: g(stat::gpu_cap),
             ..Default::default()
         };
 
-        if let Some(bss) = self.skel.maps.bss_data.as_ref() {
+        let Some(bss) = self.skel.maps.bss_data.as_ref() else {
-            m.max_wait_j = bss.max_wait_jiffies;
+            return m;
-            m.lvl_bitmap = bss.lvl_bitmap;
+        };
 
-            let hist = &bss.boost_hist;
+        m.max_wait_j = bss.max_wait_jiffies;
-            let total: u64 = hist.iter().sum();
+        m.lvl_bitmap = bss.lvl_bitmap;
-            if total > 0 {
-                let weighted: i64 = hist
-                    .iter()
-                    .enumerate()
-                    .map(|(i, &n)| (i as i64 - BOOST_RANGE as i64) * n as i64)
-                    .sum();
-                m.boost_mean_x100 = weighted * 100 / total as i64;
-                m.boost_floor_pct = hist[0] * 100 / total;
-                m.boost_ceil_pct = hist[hist.len() - 1] * 100 / total;
-            }
 
-            // Class 0 is "boosted" -- the interactive tail, and the only class
+        let hist = &bss.boost_hist;
-            // with enough samples to mean anything (hogs barely ever wake).
+        let total: u64 = hist.iter().sum();
-            let boosted = &bss.lat_hist[0..LAT_BUCKETS];
+        if total > 0 {
-            m.lat_boosted_n = boosted.iter().sum();
+            let weighted: i64 = hist
-            if let Some((p50, p99, p999)) = percentiles(boosted) {
+                .iter()
-                m.lat_boosted_p50 = p50;
+                .enumerate()
-                m.lat_boosted_p99 = p99;
+                .map(|(i, &n)| (i as i64 - BOOST_RANGE as i64) * n as i64)
-                m.lat_boosted_p999 = p999;
+                .sum();
-            }
+            m.boost_mean_x100 = weighted * 100 / total as i64;
+            m.boost_floor_pct = hist[0] * 100 / total;
+            m.boost_ceil_pct = hist[hist.len() - 1] * 100 / total;
+        }
+
+        // Class 0 is "boosted" -- the interactive tail, and the only class
+        // with enough samples to mean anything (hogs barely ever wake).
+        let cur = bss.lat_hist[0..LAT_BUCKETS].to_vec();
+        let interval = hist_delta(&cur, &self.stats_prev_lat);
+        self.stats_prev_lat = cur;
+
+        m.lat_boosted_n = interval.iter().sum();
+        if let Some((p50, p99, p999)) = percentiles(&interval) {
+            m.lat_boosted_p50 = p50;
+            m.lat_boosted_p99 = p99;
+            m.lat_boosted_p999 = p999;
         }
 
         m
@@ -326,70 +536,165 @@ impl<'a> Scheduler<'a> {
             info!("  per-level worst wait: {}", waits.join(" "));
         }
 
-        if let Some(bss) = self.skel.maps.bss_data.as_ref() {
+        let Some(bss) = self.skel.maps.bss_data.as_ref() else {
-            let hist = &bss.boost_hist;
+            return;
-            let total: u64 = hist.iter().sum();
+        };
 
-            if total > 0 {
+        // Everything below describes this interval only. The cumulative view
-                let occupied: Vec<String> = hist
+        // is still available -- max_wait and lvl_max_wait above are running
-                    .iter()
+        // maxima, and the counters on the first line are totals since load.
-                    .enumerate()
+        let boost_cur = bss.boost_hist.to_vec();
-                    .filter(|(_, &c)| c > 0)
+        let lat_cur = bss.lat_hist.to_vec();
-                    .map(|(i, &c)| {
+        let ule_cur = bss.ule_hist.to_vec();
-                        let boost = i as i32 - BOOST_RANGE;
+        let bounce_cur = bss.bounce_hist.to_vec();
-                        format!("{boost:+}:{:.0}%", 100.0 * c as f64 / total as f64)
+        let release_cur = bss.release_reason.to_vec();
-                    })
+        let gpu_lat_cur = bss.gpu_lat_hist.to_vec();
-                    .collect();
+        let gpu_boost_cur = bss.gpu_boost_hist.to_vec();
 
-                let weighted: i64 = hist
+        let boost = hist_delta(&boost_cur, &self.prev_boost);
-                    .iter()
+        let lat = hist_delta(&lat_cur, &self.prev_lat);
-                    .enumerate()
+        let ule = hist_delta(&ule_cur, &self.prev_ule);
-                    .map(|(i, &c)| (i as i64 - BOOST_RANGE as i64) * c as i64)
+        let bounce = hist_delta(&bounce_cur, &self.prev_bounce);
-                    .sum();
+        let release = hist_delta(&release_cur, &self.prev_release);
-                let mean = weighted as f64 / total as f64;
+        let gpu_lat = hist_delta(&gpu_lat_cur, &self.prev_gpu_lat);
+        let gpu_boost = hist_delta(&gpu_boost_cur, &self.prev_gpu_boost);
 
-                info!("  boost mean={mean:+.2} dist: {}", occupied.join(" "));
+        self.prev_boost = boost_cur;
+        self.prev_lat = lat_cur;
+        self.prev_ule = ule_cur;
+        self.prev_bounce = bounce_cur;
+        self.prev_release = release_cur;
+        self.prev_gpu_lat = gpu_lat_cur;
+        self.prev_gpu_boost = gpu_boost_cur;
+
+        if let Some((mean, dist)) = boost_summary(&boost) {
+            info!("  boost mean={mean:+.2} dist: {dist}");
+        }
+
+        // Same shape as the line above, so the two read side by side: if
+        // feeders sit on the floor like everyone else, boost already has them.
+        if let Some((mean, dist)) = boost_summary(&gpu_boost) {
+            info!("  gpu-feeder boost mean={mean:+.2} dist: {dist}");
+        }
+
+        let step = ULE_MAX / ULE_BUCKETS;
+        let ule_all: u64 = ule.iter().sum();
+
+        for c in 0..LAT_CLASSES {
+            let slice = &ule[c * ULE_BUCKETS..(c + 1) * ULE_BUCKETS];
+            let n: u64 = slice.iter().sum();
+            if n == 0 {
+                continue;
             }
 
-            let lines: Vec<String> = (0..LAT_CLASSES)
+            // Bucket midpoint, so the mean is not biased low by truncation.
-                .filter_map(|c| {
+            let weighted: u64 = slice
-                    let slice = &bss.lat_hist[c * LAT_BUCKETS..(c + 1) * LAT_BUCKETS];
+                .iter()
-                    percentiles(slice).map(|(p50, p99, p999)| {
+                .enumerate()
-                        format!(
+                .map(|(i, &c)| (i * step + step / 2) as u64 * c)
-                            "{}: p50={} p99={} p99.9={} n={}",
+                .sum();
-                            LAT_CLASS_NAMES[c],
+
-                            fmt_ns(p50),
+            // Off both extremes, on a band comparable in width to the one
-                            fmt_ns(p99),
+            // used for boost. The flat number was first computed over 25-74,
-                            fmt_ns(p999),
+            // which is half the ULE range against 92% of the boost range --
-                            slice.iter().sum::<u64>()
+            // not a comparison, and it made ULE look worse than it is.
-                        )
+            let mid: u64 = slice[2..18].iter().sum();
-                    })
+
+            let occupied: Vec<String> = slice
+                .iter()
+                .enumerate()
+                .filter(|(_, &c)| c > 0)
+                .map(|(i, &c)| format!("{}:{:.1}%", i * step, 100.0 * c as f64 / n as f64))
+                .collect();
+
+            info!(
+                "  ule[{}] mean={:.1} mid(10-89)={:.1}% share={:.1}% dist: {}",
+                LAT_CLASS_NAMES[c],
+                weighted as f64 / n as f64,
+                100.0 * mid as f64 / n as f64,
+                100.0 * n as f64 / ule_all as f64,
+                occupied.join(" ")
+            );
+        }
+
+        for c in 0..LAT_CLASSES {
+            let slice = &lat[c * LAT_BUCKETS..(c + 1) * LAT_BUCKETS];
+            let Some((p50, p99, p999)) = percentiles(slice) else {
+                continue;
+            };
+
+            info!(
+                "  wake->run {}: p50={} p99={} p99.9={} n={}",
+                LAT_CLASS_NAMES[c],
+                fmt_ns(p50),
+                fmt_ns(p99),
+                fmt_ns(p999),
+                slice.iter().sum::<u64>()
+            );
+        }
+
+        // Subsets of the lines above, cut by what the task was doing with the
+        // GPU rather than by its boost.
+        for (p, name) in GPU_POP_NAMES.iter().enumerate() {
+            let slice = &gpu_lat[p * LAT_BUCKETS..(p + 1) * LAT_BUCKETS];
+            let Some((p50, p99, p999)) = percentiles(slice) else {
+                continue;
+            };
+
+            info!(
+                "  wake->run {}: p50={} p99={} p99.9={} n={}",
+                name,
+                fmt_ns(p50),
+                fmt_ns(p99),
+                fmt_ns(p999),
+                slice.iter().sum::<u64>()
+            );
+        }
+
+        // The population every other latency line here is blind to: bounced
+        // by ops.cpu_release, re-enqueued without ever passing through
+        // ops.runnable, so no wake_at and no wake->run sample.
+        if let Some((p50, p99, p999)) = percentiles(&bounce) {
+            let n: u64 = bounce.iter().sum();
+            let names = ["rt", "dl", "stop", "unknown"];
+            let by: Vec<String> = release
+                .iter()
+                .enumerate()
+                .filter(|(_, &c)| c > 0)
+                .map(|(i, &c)| format!("{}:{}", names.get(i).unwrap_or(&"?"), c))
+                .collect();
+
+            info!(
+                "  bounce->run p50={} p99={} p99.9={} n={}  taken by {}",
+                fmt_ns(p50),
+                fmt_ns(p99),
+                fmt_ns(p999),
+                n,
+                if by.is_empty() { "-".into() } else { by.join(" ") },
+            );
+        }
+
+        let boosted = &lat[0..LAT_BUCKETS];
+        let n: u64 = boosted.iter().sum();
+        if n > 0 {
+            // Thresholds in ns rather than bucket indices: the bucket layout
+            // is now an implementation detail of lat_bucket(), and hardcoded
+            // indices silently meant something else the moment it changed.
+            let tail: Vec<String> = [1_000_000u64, 4_000_000, 16_000_000, 64_000_000]
+                .iter()
+                .map(|&ns| {
+                    // Label with the bucket's own edge, not the threshold we
+                    // asked for. They differ by up to the ~19% bucket width,
+                    // and printing ">1.0ms" for a count that really starts at
+                    // 1.05ms is the kind of quiet 5% lie this whole change
+                    // exists to remove.
+                    let from = bucket_at_least(ns);
+                    let pct = 100.0 * boosted[from..].iter().sum::<u64>() as f64 / n as f64;
+                    format!(">{} {:.3}%", fmt_ns(bucket_ns(from)), pct)
                 })
                 .collect();
 
-            for l in lines {
+            info!("  wake->run boosted tail: {}", tail.join("  "));
-                info!("  wake->run {l}");
-            }
-
-            let boosted = &bss.lat_hist[0..LAT_BUCKETS];
-            let n: u64 = boosted.iter().sum();
-            if n > 0 {
-                let above = |bucket: usize| -> f64 {
-                    100.0 * boosted[bucket..].iter().sum::<u64>() as f64 / n as f64
-                };
-
-                info!(
-                    "  wake->run boosted tail: >{} {:.3}%  >{} {:.3}%  >{} {:.4}%  >{} {:.4}%",
-                    fmt_ns(1 << 20),
-                    above(20),
-                    fmt_ns(1 << 22),
-                    above(22),
-                    fmt_ns(1 << 24),
-                    above(24),
-                    fmt_ns(1 << 26),
-                    above(26),
-                );
-            }
         }
     }
 
@@ -398,11 +703,31 @@ impl<'a> Scheduler<'a> {
         let every = self.opts.report_sec.max(1);
         let mut ticks = 0u64;
 
+        // Telemetry is not the scheduler. A broken stats channel used to
+        // propagate out of run(), which main() reports as "ejected by the
+        // kernel" -- so a dead socket read as a stall in the journal, exited
+        // non-zero, and (under the systemd unit) fired the critical desktop
+        // notification. Meanwhile the actual policy was still running fine.
+        // Now it degrades: log once, stop serving stats, keep scheduling.
+        let mut stats_live = true;
+
         while !shutdown.load(Ordering::Relaxed) && !self.exited() {
+            if !stats_live {
+                std::thread::sleep(Duration::from_secs(1));
+                ticks += 1;
+                if ticks % every == 0 {
+                    self.report();
+                }
+                continue;
+            }
+
             match req_ch.recv_timeout(Duration::from_secs(1)) {
                 Ok(()) => {
                     let m = self.metrics();
-                    res_ch.send(m)?;
+                    if let Err(e) = res_ch.send(m) {
+                        warn!("stats channel closed ({e}); telemetry off, scheduler still running");
+                        stats_live = false;
+                    }
                 }
                 Err(RecvTimeoutError::Timeout) => {
                     ticks += 1;
@@ -410,7 +735,10 @@ impl<'a> Scheduler<'a> {
                         self.report();
                     }
                 }
-                Err(e) => Err(e)?,
+                Err(e) => {
+                    warn!("stats channel closed ({e}); telemetry off, scheduler still running");
+                    stats_live = false;
+                }
             }
         }
 
@@ -482,7 +810,7 @@ fn main() -> Result<()> {
             Ok(uei) => uei,
             Err(e) => {
                 error!("{SCHEDULER_NAME} was ejected by the kernel: {e:#}");
-                error!("all tasks are back on default kernel scheduler; the soryu is NOT running");
+                error!("all tasks are back on default kernel scheduler; the scheduler is NOT running");
                 return Err(e);
             }
         };

src/stats.rs

@@ -29,7 +29,9 @@ pub struct Metrics {
     pub enq_pinned: u64,
     #[stat(desc = "Tasks bounced back by ops.cpu_release")]
     pub enq_reenq: u64,
-    #[stat(desc = "DSQ inserts that failed")]
+    #[stat(desc = "Tasks moved from a level DSQ to a local one")]
+    pub dispatch: u64,
+    #[stat(desc = "DSQ inserts that failed and fell back to the global DSQ")]
     pub insert_fail: u64,
     #[stat(desc = "CPUs taken by a higher sched class (PipeWire RT, i915)")]
     pub cpu_release: u64,
@@ -53,6 +55,20 @@ pub struct Metrics {
     #[stat(desc = "Wakeups that kicked an idle CPU rather than preempting anyone")]
     pub preempt_idle: u64,
 
+    #[stat(desc = "Lit-but-empty level DSQs touched while placing a task")]
+    pub dispatch_miss: u64,
+    #[stat(desc = "Starving levels this CPU could not serve (affinity)")]
+    pub starve_blocked: u64,
+    #[stat(desc = "Bits relit after a level was found occupied but unadvertised")]
+    pub bitmap_repair: u64,
+    #[stat(desc = "Wakeups where the waker handed the wakee some of its boost")]
+    pub wake_donate: u64,
+
+    #[stat(desc = "GPU submissions (i915_request_add) by scx tasks")]
+    pub gpu_submit: u64,
+    #[stat(desc = "Deboosts skipped because the task is a recent GPU feeder")]
+    pub gpu_cap: u64,
+
     #[stat(desc = "Mean boost, x100 (negative is more interactive)")]
     pub boost_mean_x100: i64,
     #[stat(desc = "Percent of scheduling events at the -12 boost floor")]
@@ -71,30 +87,47 @@ pub struct Metrics {
 
     #[stat(desc = "Worst wait by any runnable task since load, jiffies")]
     pub max_wait_j: u64,
-    #[stat(desc = "Levels currently holding tasks")]
+    /// Meaning depends on --clear-bitmap, which defaults to false: set-only,
+    /// so "levels used since load" rather than a live occupancy map. Clearing
+    /// makes it live and measured worse (docs/bitmap-clear.md). With clearing
+    /// on, watch `bitmap_repair` -- a nonzero rate means the clear is racing
+    /// more than the argument in main.bpf.c predicts.
+    #[stat(desc = "Levels holding tasks (set-only unless --clear-bitmap)")]
     pub lvl_bitmap: u64,
 }
 
-impl Metrics {
+/// Single copy, shared with the loader's own report.
-    fn fmt_ns(ns: u64) -> String {
+///
-        if ns >= 1_000_000_000 {
+/// One significant decimal below 100, because the histogram now resolves ~19%
-            format!("{:.1}s", ns as f64 / 1e9)
+/// steps and truncating to whole units threw that away again -- 1.25 ms and
-        } else if ns >= 1_000_000 {
+/// 1.75 ms both printed as "1ms", which is the exact resolution problem the
-            format!("{}ms", ns / 1_000_000)
+/// finer buckets were added to fix.
-        } else if ns >= 1_000 {
+pub fn fmt_ns(ns: u64) -> String {
-            format!("{}us", ns / 1_000)
+    let (v, unit) = if ns >= 1_000_000_000 {
-        } else {
+        (ns as f64 / 1e9, "s")
-            format!("{ns}ns")
+    } else if ns >= 1_000_000 {
-        }
+        (ns as f64 / 1e6, "ms")
-    }
+    } else if ns >= 1_000 {
+        (ns as f64 / 1e3, "us")
+    } else {
+        return format!("{ns}ns");
+    };
 
+    if v >= 100.0 {
+        format!("{v:.0}{unit}")
+    } else {
+        format!("{v:.1}{unit}")
+    }
+}
+
+impl Metrics {
     pub fn format<W: Write>(&self, w: &mut W) -> Result<()> {
         writeln!(
             w,
             "wake->run interactive  p50 {:>7}  p99 {:>7}  p99.9 {:>7}  (n={})",
-            Self::fmt_ns(self.lat_boosted_p50),
+            fmt_ns(self.lat_boosted_p50),
-            Self::fmt_ns(self.lat_boosted_p99),
+            fmt_ns(self.lat_boosted_p99),
-            Self::fmt_ns(self.lat_boosted_p999),
+            fmt_ns(self.lat_boosted_p999),
             self.lat_boosted_n,
         )?;
         writeln!(
@@ -108,11 +141,12 @@ impl Metrics {
         )?;
         writeln!(
             w,
-            "enqueue    local {} shared {} pinned {} reenq {} fail {}",
+            "enqueue    local {} shared {} pinned {} reenq {} dispatch {} fail {}",
             self.enq_local,
             self.enq_shared,
             self.enq_pinned,
             self.enq_reenq,
+            self.dispatch,
             self.insert_fail,
         )?;
         writeln!(
@@ -126,34 +160,64 @@ impl Metrics {
         )?;
         writeln!(
             w,
-            "preempt    kick {}  idle {}  none {}  cpu_release {}  starve_serve {}",
+            "preempt    kick {}  idle {}  none {}  cpu_release {}",
-            self.preempt_kick,
+            self.preempt_kick, self.preempt_idle, self.preempt_none, self.cpu_release,
-            self.preempt_idle,
+        )?;
-            self.preempt_none,
+        writeln!(
-            self.cpu_release,
+            w,
+            "scan       dispatch_miss {} ({:.1} per dispatch)  repair {}  starve_serve {}  starve_blocked {}",
+            self.dispatch_miss,
+            if self.dispatch > 0 {
+                self.dispatch_miss as f64 / self.dispatch as f64
+            } else {
+                0.0
+            },
+            self.bitmap_repair,
             self.starve_serve,
+            self.starve_blocked,
+        )?;
+        writeln!(w, "donate     wake_donate {}", self.wake_donate)?;
+        writeln!(
+            w,
+            "gpu        submit {}  cap {}",
+            self.gpu_submit, self.gpu_cap,
         )?;
         Ok(())
     }
 
+    /// Counters are monotonic *within one attach*. They are not monotonic
+    /// across one: a hotplug reload rebuilds the maps from zero while a
+    /// `--monitor` client is still connected and still holding the old
+    /// snapshot as `prev`. Plain `-` wraps there (release builds have
+    /// overflow checks off), so a reload printed deltas near 1.8e19 rather
+    /// than the restart it actually was. saturating_sub prints 0 instead.
     fn delta(&self, rhs: &Self) -> Self {
         Self {
-            enq_local: self.enq_local - rhs.enq_local,
+            enq_local: self.enq_local.saturating_sub(rhs.enq_local),
-            enq_shared: self.enq_shared - rhs.enq_shared,
+            enq_shared: self.enq_shared.saturating_sub(rhs.enq_shared),
-            enq_pinned: self.enq_pinned - rhs.enq_pinned,
+            enq_pinned: self.enq_pinned.saturating_sub(rhs.enq_pinned),
-            enq_reenq: self.enq_reenq - rhs.enq_reenq,
+            enq_reenq: self.enq_reenq.saturating_sub(rhs.enq_reenq),
-            insert_fail: self.insert_fail - rhs.insert_fail,
+            dispatch: self.dispatch.saturating_sub(rhs.dispatch),
-            cpu_release: self.cpu_release - rhs.cpu_release,
+            insert_fail: self.insert_fail.saturating_sub(rhs.insert_fail),
-            boost_sleep: self.boost_sleep - rhs.boost_sleep,
+            cpu_release: self.cpu_release.saturating_sub(rhs.cpu_release),
-            boost_wake: self.boost_wake - rhs.boost_wake,
+            boost_sleep: self.boost_sleep.saturating_sub(rhs.boost_sleep),
-            deboost: self.deboost - rhs.deboost,
+            boost_wake: self.boost_wake.saturating_sub(rhs.boost_wake),
-            slice_carry: self.slice_carry - rhs.slice_carry,
+            deboost: self.deboost.saturating_sub(rhs.deboost),
-            fork_split: self.fork_split - rhs.fork_split,
+            slice_carry: self.slice_carry.saturating_sub(rhs.slice_carry),
-            preempt_kick: self.preempt_kick - rhs.preempt_kick,
+            fork_split: self.fork_split.saturating_sub(rhs.fork_split),
-            starve_serve: self.starve_serve - rhs.starve_serve,
+            preempt_kick: self.preempt_kick.saturating_sub(rhs.preempt_kick),
-            preempt_none: self.preempt_none - rhs.preempt_none,
+            starve_serve: self.starve_serve.saturating_sub(rhs.starve_serve),
-            preempt_idle: self.preempt_idle - rhs.preempt_idle,
+            preempt_none: self.preempt_none.saturating_sub(rhs.preempt_none),
-            lat_boosted_n: self.lat_boosted_n - rhs.lat_boosted_n,
+            preempt_idle: self.preempt_idle.saturating_sub(rhs.preempt_idle),
+            dispatch_miss: self.dispatch_miss.saturating_sub(rhs.dispatch_miss),
+            starve_blocked: self.starve_blocked.saturating_sub(rhs.starve_blocked),
+            bitmap_repair: self.bitmap_repair.saturating_sub(rhs.bitmap_repair),
+            wake_donate: self.wake_donate.saturating_sub(rhs.wake_donate),
+            gpu_submit: self.gpu_submit.saturating_sub(rhs.gpu_submit),
+            gpu_cap: self.gpu_cap.saturating_sub(rhs.gpu_cap),
+            // lat_boosted_* are already per-interval: the loader computes them
+            // from a differenced histogram, since percentiles cannot be
+            // subtracted. Differencing the count here too would zero it.
             ..self.clone()
         }
     }

tools/gen-compile-commands.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Regenerate compile_commands.json for src/bpf/main.bpf.c so clangd sees the
+# same flags the real BPF build uses.
+#
+# scx_cargo prints its computed cflags to the build-script output as a line
+# starting with `scx_utils:clang=`. The include paths in it point into
+# target/<profile>/build/scx_soryu-<hash>/out, and that hash changes whenever
+# the build-script inputs change, so this is generated rather than committed.
+#
+# Run inside the devShell after a build:
+#   cargo build --release && ./tools/gen-compile-commands.sh
+
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+root="$PWD"
+
+profile="${1:-release}"
+out=$(ls -td "target/$profile"/build/scx_soryu-*/output 2>/dev/null | head -1 || true)
+
+if [ -z "$out" ]; then
+  echo "error: no build-script output under target/$profile/build/." >&2
+  echo "       run 'cargo build --${profile/dev/}' first (inside the devShell)." >&2
+  exit 1
+fi
+
+line=$(grep -h '^scx_utils:clang=' "$out" | tail -1 || true)
+if [ -z "$line" ]; then
+  echo "error: no 'scx_utils:clang=' line in $out" >&2
+  echo "       the build script was cached; touch src/bpf/main.bpf.c and rebuild." >&2
+  exit 1
+fi
+
+# The cflags are the bracketed Rust Vec<String> debug print at the end of the
+# line: ["-g", "-O2", ...]. Pull the quoted strings out of that part only.
+flags=$(printf '%s' "$line" \
+  | sed 's/.*\(\[[^][]*\]\)$/\1/' \
+  | grep -o '"[^"]*"' \
+  | sed 's/^"//; s/"$//')
+
+if [ -z "$flags" ]; then
+  echo "error: could not parse cflags out of: $line" >&2
+  exit 1
+fi
+
+{
+  printf '[\n  {\n'
+  printf '    "directory": "%s",\n' "$root"
+  printf '    "file": "%s/src/bpf/main.bpf.c",\n' "$root"
+  printf '    "arguments": [\n'
+  printf '      "clang",\n'
+  printf '      "-xc",\n'
+  printf '      "-target", "bpf",\n'
+  while IFS= read -r f; do
+    printf '      %s,\n' "$(printf '%s' "$f" | sed 's/\\/\\\\/g; s/"/\\"/g; s/^/"/; s/$/"/')"
+  done <<<"$flags"
+  printf '      "-c",\n'
+  printf '      "%s/src/bpf/main.bpf.c"\n' "$root"
+  printf '    ]\n  }\n]\n'
+} >compile_commands.json
+
+echo "wrote compile_commands.json ($(wc -l <<<"$flags") flags, profile=$profile)"