changes

src/main.rs

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 //
+//
 // scx_soryu loader.
 //
 // runs in the foreground and holds no state of its own: dropping
@@ -9,6 +10,7 @@
 mod bpf_skel;
 mod stats;
 use bpf_skel::*;
+use stats::fmt_ns;
 use stats::Metrics;
 
 use std::mem::MaybeUninit;
@@ -35,46 +37,147 @@ use scx_utils::UserExitInfo;
 
 const SCHEDULER_NAME: &str = "scx_soryu";
 
-/// mirrors `enum stat_idx` in main.bpf.c.
+/// mirrors `enum soryu_stat_idx` in main.bpf.c, in order.
-const STAT_NAMES: &[&str] = &[
+///
-    "enq_shared",
+/// The names and the indices come from the same list on purpose. They used to
-    "enq_local",
+/// be a `STAT_NAMES` array plus hand-written `g(0)`..`g(15)` calls in
-    "enq_reenq",
+/// `metrics()`, with nothing tying the two together: inserting one counter in
-    "enq_pinned",
+/// the middle of the BPF enum would have silently shifted every metric after
-    "dispatch",
+/// it, and the numbers would still have looked plausible.
-    "insert_fail",
+macro_rules! soryu_stats {
-    "cpu_release",
+    ($($name:ident),+ $(,)?) => {
-    "boost_sleep",
+        const STAT_NAMES: &[&str] = &[$(stringify!($name)),+];
-    "boost_wake",
+
-    "deboost",
+        #[allow(non_upper_case_globals)]
-    "slice_carry",
+        mod stat {
-    "fork_split",
+            soryu_stats!(@idx 0usize; $($name),+);
-    "preempt_kick",
+        }
-    "starve_serve",
+    };
-    "preempt_none",
+    (@idx $n:expr; $head:ident $(, $tail:ident)*) => {
-    "preempt_idle",
+        pub const $head: usize = $n;
-];
+        soryu_stats!(@idx $n + 1; $($tail),*);
+    };
+    (@idx $n:expr;) => {};
+}
+
+soryu_stats!(
+    enq_shared,
+    enq_local,
+    enq_reenq,
+    enq_pinned,
+    dispatch,
+    insert_fail,
+    cpu_release,
+    boost_sleep,
+    boost_wake,
+    deboost,
+    slice_carry,
+    fork_split,
+    preempt_kick,
+    starve_serve,
+    preempt_none,
+    preempt_idle,
+    dispatch_miss,
+    starve_blocked,
+    bitmap_repair,
+    wake_donate,
+);
 
 /// mirrors BOOST_RANGE in main.bpf.c.
 const BOOST_RANGE: i32 = 12;
 
-/// mirrors LAT_CLASSES / LAT_BUCKETS in main.bpf.c.
+/// mirrors ULE_BUCKETS / ULE_MAX in main.bpf.c.
+const ULE_BUCKETS: usize = 20;
+const ULE_MAX: usize = 100;
+
+/// mirrors LAT_CLASSES / LAT_OCTAVES / LAT_SUBS in main.bpf.c.
 const LAT_CLASSES: usize = 3;
-const LAT_BUCKETS: usize = 32;
+const LAT_OCTAVES: usize = 32;
+const LAT_SUBS: usize = 4;
+const LAT_BUCKETS: usize = LAT_OCTAVES * LAT_SUBS;
 const LAT_CLASS_NAMES: [&str; LAT_CLASSES] = ["boosted", "neutral", "deboosted"];
 
+/// Lower edge of bucket @i in ns: mantissa (4 + sub) / 4, scaled by 2^octave.
+/// Inverse of lat_bucket() in main.bpf.c.
+fn bucket_ns(i: usize) -> u64 {
+    let oct = (i / LAT_SUBS) as u32;
+    let sub = (i % LAT_SUBS) as u64;
+
+    ((4 + sub) << oct) >> 2
+}
+
+/// First bucket whose lower edge is at or above @ns, for the tail report.
+fn bucket_at_least(ns: u64) -> usize {
+    (0..LAT_BUCKETS)
+        .find(|&i| bucket_ns(i) >= ns)
+        .unwrap_or(LAT_BUCKETS - 1)
+}
+
+/// Elementwise cur - prev, so a report describes its own interval rather than
+/// everything since load.
+fn hist_delta(cur: &[u64], prev: &[u64]) -> Vec<u64> {
+    cur.iter()
+        .enumerate()
+        .map(|(i, &c)| c.saturating_sub(prev.get(i).copied().unwrap_or(0)))
+        .collect()
+}
+
 #[derive(Parser, Debug, Clone)]
 #[command(about = "BMQ policy port for sched_ext", version)]
 struct Opts {
     #[arg(long, default_value_t = 5000, env = "SORYU_TIMEOUT_MS")]
     timeout_ms: u32,
-    #[arg(long, default_value_t = true, env = "SORYU_BOOST_ON_WAKE",
+    /// Boost on wakeup in proportion to time slept. Off by default: measured
+    /// worse (docs/boost-on-wake.md). Takes a value -- `--boost-on-wake=true`.
+    #[arg(long, default_value_t = false, env = "SORYU_BOOST_ON_WAKE",
           action = clap::ArgAction::Set)]
     boost_on_wake: bool,
+    /// Clear bitmap bits in ops.dispatch, with the repair sweep that makes it
+    /// safe. Off by default: measured worse (docs/bitmap-clear.md). Takes a
+    /// value -- `--clear-bitmap=true`.
+    #[arg(long, default_value_t = false, env = "SORYU_CLEAR_BITMAP",
+          action = clap::ArgAction::Set)]
+    clear_bitmap: bool,
+    /// Throw the local DSQ back through ops.enqueue when a higher sched class
+    /// takes the CPU. On by default. See main.bpf.c for the risk. Takes a
+    /// value -- `--reenqueue-on-release=false`.
+    #[arg(long, default_value_t = true, env = "SORYU_REENQUEUE_ON_RELEASE",
+          action = clap::ArgAction::Set)]
+    reenqueue_on_release: bool,
     #[arg(long, default_value_t = 1000, env = "SORYU_SLICE_US")]
     slice_us: u64,
     #[arg(long, default_value_t = 128, env = "SORYU_STARVE_MS")]
     starve_ms: u64,
+
+    /// Levels the FreeBSD ULE interactivity score may move a task, in each
+    /// direction, on top of BMQ's boost. 0 disables it and restores BMQ.
+    /// The score is measured either way -- see the ule line in the report.
+    #[arg(long, default_value_t = 0, env = "SORYU_ULE_WEIGHT")]
+    ule_weight: i32,
+
+    /// Combined run+sleep history the ULE score keeps before halving both.
+    /// ULE's SCHED_SLP_RUN_MAX.
+    #[arg(long, default_value_t = 5000, env = "SORYU_ULE_WINDOW_MS")]
+    ule_window_ms: u64,
+
+    /// Levels a cgroup's cpu.weight may move its tasks, in each direction.
+    /// 0 disables it, which is what this scheduler did until now -- systemd's
+    /// CPUWeight= had no effect at all. Changes fairness between groups, not
+    /// latency.
+    #[arg(long, default_value_t = 0, env = "SORYU_CGRP_WEIGHT_LEVELS")]
+    cgrp_weight_levels: i32,
+
+    /// Boost steps a waker may hand down to the task it wakes, capped by the
+    /// gap between them. 0 disables it.
+    #[arg(long, default_value_t = 0, env = "SORYU_WAKE_DONATE")]
+    wake_donate: i32,
+
+    /// Order each level DSQ by virtual time rather than FIFO -- within a level
+    /// only, so BMQ's strict priority between levels is untouched. Off by
+    /// default. Takes a value -- `--vtime-within-level=true`.
+    #[arg(long, default_value_t = false, env = "SORYU_VTIME_WITHIN_LEVEL",
+          action = clap::ArgAction::Set)]
+    vtime_within_level: bool,
     #[arg(long, default_value_t = 1, env = "SORYU_REPORT_SEC")]
     report_sec: u64,
     #[arg(long, env = "SORYU_LOG")]
@@ -100,38 +203,50 @@ fn percentiles(buckets: &[u64]) -> Option<(u64, u64, u64)> {
     }
 
     let at = |frac: f64| -> u64 {
-        let target = (total as f64 * frac) as u64;
+        // ceil, and never zero: truncating gave target = 0 for small n, and
+        // `seen >= 0` is true before any bucket has been added, so p50 of a
+        // single sample reported bucket 0 (1ns) no matter where the sample
+        // actually landed.
+        let target = ((total as f64 * frac).ceil() as u64).max(1);
         let mut seen = 0u64;
         for (i, &c) in buckets.iter().enumerate() {
             seen += c;
             if seen >= target {
-                return 1u64 << i;
+                return bucket_ns(i);
             }
         }
-        1u64 << (buckets.len() - 1)
+        bucket_ns(buckets.len() - 1)
     };
 
     Some((at(0.50), at(0.99), at(0.999)))
 }
 
-fn fmt_ns(ns: u64) -> String {
-    if ns >= 1_000_000_000 {
-        format!("{:.1}s", ns as f64 / 1e9)
-    } else if ns >= 1_000_000 {
-        format!("{}ms", ns / 1_000_000)
-    } else if ns >= 1_000 {
-        format!("{}us", ns / 1_000)
-    } else {
-        format!("{ns}ns")
-    }
-}
-
 struct Scheduler<'a> {
     opts: Opts,
     skel: BpfSkel<'a>,
     struct_ops: Option<libbpf_rs::Link>,
     stats_server: StatsServer<(), Metrics>,
     prev_waits: Vec<u64>,
+
+    /// Histogram snapshots from the previous report, so each line describes
+    /// its own interval.
+    ///
+    /// lat_hist and boost_hist accumulate from load and were printed raw, so a
+    /// report was the average of every phase the machine had been through. A
+    /// run that idled for 5 s and then got busy for 10 s printed one blended
+    /// p50 that described neither -- and it moved 16us -> 8us, which reads
+    /// like an improvement and is really just the mixture shifting. The
+    /// counters were already deltaed for `--monitor`; the histograms were not
+    /// deltaed anywhere.
+    prev_lat: Vec<u64>,
+    prev_boost: Vec<u64>,
+    prev_ule: Vec<u64>,
+    prev_bounce: Vec<u64>,
+    prev_release: Vec<u64>,
+
+    /// Separate snapshots for the stats server: it samples on its own schedule
+    /// and must not consume the report's intervals, or vice versa.
+    stats_prev_lat: Vec<u64>,
 }
 
 impl<'a> Scheduler<'a> {
@@ -153,12 +268,19 @@ impl<'a> Scheduler<'a> {
             .rodata_data
             .as_mut()
             .context("no rodata section in skeleton")?;
-        rodata.nr_cpu_ids = *scx_utils::NR_CPU_IDS as u32;
 
-        // SORYU_BOOST_ON_WAKE=0 reproduces BMQ as shipped (its wake boost is
+        // The default reproduces BMQ as shipped, whose wake boost is dead
-        // dead code); the default reproduces what BMQ meant to do.
+        // code. SORYU_BOOST_ON_WAKE=1 turns on what BMQ meant to do, which
+        // measured worse -- see docs/boost-on-wake.md.
         rodata.boost_on_wake = opts.boost_on_wake;
         rodata.starve_ns = opts.starve_ms * 1_000_000;
+        rodata.clear_bitmap = opts.clear_bitmap;
+        rodata.reenqueue_on_release = opts.reenqueue_on_release;
+        rodata.ule_weight = opts.ule_weight;
+        rodata.ule_window_ns = opts.ule_window_ms * 1_000_000;
+        rodata.cgrp_weight_levels = opts.cgrp_weight_levels;
+        rodata.wake_donate = opts.wake_donate;
+        rodata.vtime_within_level = opts.vtime_within_level;
 
         rodata.slice_ns = opts.slice_us * 1_000;
 
@@ -168,12 +290,17 @@ impl<'a> Scheduler<'a> {
         let rd = skel.maps.rodata_data.as_ref().unwrap();
         info!(
             "{} attached: {} CPUs, slice {} us, boost_on_wake={}, \
-             starve {} ms, timeout {} ms",
+             starve {} ms, clear_bitmap={}, ule_weight={} (window {} ms), \
+             vtime_within_level={}, timeout {} ms",
             SCHEDULER_NAME,
             *scx_utils::NR_CPU_IDS,
             rd.slice_ns / 1000,
             rd.boost_on_wake,
             rd.starve_ns / 1_000_000,
+            rd.clear_bitmap,
+            rd.ule_weight,
+            rd.ule_window_ns / 1_000_000,
+            rd.vtime_within_level,
             opts.timeout_ms
         );
 
@@ -185,6 +312,12 @@ impl<'a> Scheduler<'a> {
             struct_ops,
             stats_server,
             prev_waits: Vec::new(),
+            prev_lat: Vec::new(),
+            prev_boost: Vec::new(),
+            prev_ule: Vec::new(),
+            prev_bounce: Vec::new(),
+            prev_release: Vec::new(),
+            stats_prev_lat: Vec::new(),
         })
     }
 
@@ -217,30 +350,42 @@ impl<'a> Scheduler<'a> {
     }
 
     /// One snapshot for the stats server.
-    fn metrics(&self) -> Metrics {
+    ///
+    /// The counters go out cumulative and `Metrics::delta` differences them at
+    /// the far end; the latency percentiles cannot be differenced that way, so
+    /// they are computed here from the interval histogram directly.
+    fn metrics(&mut self) -> Metrics {
         let c = self.read_stats().unwrap_or_default();
         let g = |i: usize| c.get(i).copied().unwrap_or(0);
 
         let mut m = Metrics {
-            enq_shared: g(0),
+            enq_shared: g(stat::enq_shared),
-            enq_local: g(1),
+            enq_local: g(stat::enq_local),
-            enq_reenq: g(2),
+            enq_reenq: g(stat::enq_reenq),
-            enq_pinned: g(3),
+            enq_pinned: g(stat::enq_pinned),
-            insert_fail: g(5),
+            dispatch: g(stat::dispatch),
-            cpu_release: g(6),
+            insert_fail: g(stat::insert_fail),
-            boost_sleep: g(7),
+            cpu_release: g(stat::cpu_release),
-            boost_wake: g(8),
+            boost_sleep: g(stat::boost_sleep),
-            deboost: g(9),
+            boost_wake: g(stat::boost_wake),
-            slice_carry: g(10),
+            deboost: g(stat::deboost),
-            fork_split: g(11),
+            slice_carry: g(stat::slice_carry),
-            preempt_kick: g(12),
+            fork_split: g(stat::fork_split),
-            starve_serve: g(13),
+            preempt_kick: g(stat::preempt_kick),
-            preempt_none: g(14),
+            starve_serve: g(stat::starve_serve),
-            preempt_idle: g(15),
+            preempt_none: g(stat::preempt_none),
+            preempt_idle: g(stat::preempt_idle),
+            dispatch_miss: g(stat::dispatch_miss),
+            starve_blocked: g(stat::starve_blocked),
+            bitmap_repair: g(stat::bitmap_repair),
+            wake_donate: g(stat::wake_donate),
             ..Default::default()
         };
 
-        if let Some(bss) = self.skel.maps.bss_data.as_ref() {
+        let Some(bss) = self.skel.maps.bss_data.as_ref() else {
+            return m;
+        };
+
         m.max_wait_j = bss.max_wait_jiffies;
         m.lvl_bitmap = bss.lvl_bitmap;
 
@@ -259,14 +404,16 @@ impl<'a> Scheduler<'a> {
 
         // Class 0 is "boosted" -- the interactive tail, and the only class
         // with enough samples to mean anything (hogs barely ever wake).
-            let boosted = &bss.lat_hist[0..LAT_BUCKETS];
+        let cur = bss.lat_hist[0..LAT_BUCKETS].to_vec();
-            m.lat_boosted_n = boosted.iter().sum();
+        let interval = hist_delta(&cur, &self.stats_prev_lat);
-            if let Some((p50, p99, p999)) = percentiles(boosted) {
+        self.stats_prev_lat = cur;
+
+        m.lat_boosted_n = interval.iter().sum();
+        if let Some((p50, p99, p999)) = percentiles(&interval) {
             m.lat_boosted_p50 = p50;
             m.lat_boosted_p99 = p99;
             m.lat_boosted_p999 = p999;
         }
-        }
 
         m
     }
@@ -326,22 +473,44 @@ impl<'a> Scheduler<'a> {
             info!("  per-level worst wait: {}", waits.join(" "));
         }
 
-        if let Some(bss) = self.skel.maps.bss_data.as_ref() {
+        let Some(bss) = self.skel.maps.bss_data.as_ref() else {
-            let hist = &bss.boost_hist;
+            return;
-            let total: u64 = hist.iter().sum();
+        };
 
+        // Everything below describes this interval only. The cumulative view
+        // is still available -- max_wait and lvl_max_wait above are running
+        // maxima, and the counters on the first line are totals since load.
+        let boost_cur = bss.boost_hist.to_vec();
+        let lat_cur = bss.lat_hist.to_vec();
+        let ule_cur = bss.ule_hist.to_vec();
+        let bounce_cur = bss.bounce_hist.to_vec();
+        let release_cur = bss.release_reason.to_vec();
+
+        let boost = hist_delta(&boost_cur, &self.prev_boost);
+        let lat = hist_delta(&lat_cur, &self.prev_lat);
+        let ule = hist_delta(&ule_cur, &self.prev_ule);
+        let bounce = hist_delta(&bounce_cur, &self.prev_bounce);
+        let release = hist_delta(&release_cur, &self.prev_release);
+
+        self.prev_boost = boost_cur;
+        self.prev_lat = lat_cur;
+        self.prev_ule = ule_cur;
+        self.prev_bounce = bounce_cur;
+        self.prev_release = release_cur;
+
+        let total: u64 = boost.iter().sum();
         if total > 0 {
-                let occupied: Vec<String> = hist
+            let occupied: Vec<String> = boost
                 .iter()
                 .enumerate()
                 .filter(|(_, &c)| c > 0)
                 .map(|(i, &c)| {
-                        let boost = i as i32 - BOOST_RANGE;
+                    let b = i as i32 - BOOST_RANGE;
-                        format!("{boost:+}:{:.0}%", 100.0 * c as f64 / total as f64)
+                    format!("{b:+}:{:.1}%", 100.0 * c as f64 / total as f64)
                 })
                 .collect();
 
-                let weighted: i64 = hist
+            let weighted: i64 = boost
                 .iter()
                 .enumerate()
                 .map(|(i, &c)| (i as i64 - BOOST_RANGE as i64) * c as i64)
@@ -351,45 +520,106 @@ impl<'a> Scheduler<'a> {
             info!("  boost mean={mean:+.2} dist: {}", occupied.join(" "));
         }
 
-            let lines: Vec<String> = (0..LAT_CLASSES)
+        let step = ULE_MAX / ULE_BUCKETS;
-                .filter_map(|c| {
+        let ule_all: u64 = ule.iter().sum();
-                    let slice = &bss.lat_hist[c * LAT_BUCKETS..(c + 1) * LAT_BUCKETS];
+
-                    percentiles(slice).map(|(p50, p99, p999)| {
+        for c in 0..LAT_CLASSES {
-                        format!(
+            let slice = &ule[c * ULE_BUCKETS..(c + 1) * ULE_BUCKETS];
-                            "{}: p50={} p99={} p99.9={} n={}",
+            let n: u64 = slice.iter().sum();
+            if n == 0 {
+                continue;
+            }
+
+            // Bucket midpoint, so the mean is not biased low by truncation.
+            let weighted: u64 = slice
+                .iter()
+                .enumerate()
+                .map(|(i, &c)| (i * step + step / 2) as u64 * c)
+                .sum();
+
+            // Off both extremes, on a band comparable in width to the one
+            // used for boost. The flat number was first computed over 25-74,
+            // which is half the ULE range against 92% of the boost range --
+            // not a comparison, and it made ULE look worse than it is.
+            let mid: u64 = slice[2..18].iter().sum();
+
+            let occupied: Vec<String> = slice
+                .iter()
+                .enumerate()
+                .filter(|(_, &c)| c > 0)
+                .map(|(i, &c)| format!("{}:{:.1}%", i * step, 100.0 * c as f64 / n as f64))
+                .collect();
+
+            info!(
+                "  ule[{}] mean={:.1} mid(10-89)={:.1}% share={:.1}% dist: {}",
+                LAT_CLASS_NAMES[c],
+                weighted as f64 / n as f64,
+                100.0 * mid as f64 / n as f64,
+                100.0 * n as f64 / ule_all as f64,
+                occupied.join(" ")
+            );
+        }
+
+        for c in 0..LAT_CLASSES {
+            let slice = &lat[c * LAT_BUCKETS..(c + 1) * LAT_BUCKETS];
+            let Some((p50, p99, p999)) = percentiles(slice) else {
+                continue;
+            };
+
+            info!(
+                "  wake->run {}: p50={} p99={} p99.9={} n={}",
                 LAT_CLASS_NAMES[c],
                 fmt_ns(p50),
                 fmt_ns(p99),
                 fmt_ns(p999),
                 slice.iter().sum::<u64>()
-                        )
+            );
-                    })
+        }
+
+        // The population every other latency line here is blind to: bounced
+        // by ops.cpu_release, re-enqueued without ever passing through
+        // ops.runnable, so no wake_at and no wake->run sample.
+        if let Some((p50, p99, p999)) = percentiles(&bounce) {
+            let n: u64 = bounce.iter().sum();
+            let names = ["rt", "dl", "stop", "unknown"];
+            let by: Vec<String> = release
+                .iter()
+                .enumerate()
+                .filter(|(_, &c)| c > 0)
+                .map(|(i, &c)| format!("{}:{}", names.get(i).unwrap_or(&"?"), c))
+                .collect();
+
+            info!(
+                "  bounce->run p50={} p99={} p99.9={} n={}  taken by {}",
+                fmt_ns(p50),
+                fmt_ns(p99),
+                fmt_ns(p999),
+                n,
+                if by.is_empty() { "-".into() } else { by.join(" ") },
+            );
+        }
+
+        let boosted = &lat[0..LAT_BUCKETS];
+        let n: u64 = boosted.iter().sum();
+        if n > 0 {
+            // Thresholds in ns rather than bucket indices: the bucket layout
+            // is now an implementation detail of lat_bucket(), and hardcoded
+            // indices silently meant something else the moment it changed.
+            let tail: Vec<String> = [1_000_000u64, 4_000_000, 16_000_000, 64_000_000]
+                .iter()
+                .map(|&ns| {
+                    // Label with the bucket's own edge, not the threshold we
+                    // asked for. They differ by up to the ~19% bucket width,
+                    // and printing ">1.0ms" for a count that really starts at
+                    // 1.05ms is the kind of quiet 5% lie this whole change
+                    // exists to remove.
+                    let from = bucket_at_least(ns);
+                    let pct = 100.0 * boosted[from..].iter().sum::<u64>() as f64 / n as f64;
+                    format!(">{} {:.3}%", fmt_ns(bucket_ns(from)), pct)
                 })
                 .collect();
 
-            for l in lines {
+            info!("  wake->run boosted tail: {}", tail.join("  "));
-                info!("  wake->run {l}");
-            }
-
-            let boosted = &bss.lat_hist[0..LAT_BUCKETS];
-            let n: u64 = boosted.iter().sum();
-            if n > 0 {
-                let above = |bucket: usize| -> f64 {
-                    100.0 * boosted[bucket..].iter().sum::<u64>() as f64 / n as f64
-                };
-
-                info!(
-                    "  wake->run boosted tail: >{} {:.3}%  >{} {:.3}%  >{} {:.4}%  >{} {:.4}%",
-                    fmt_ns(1 << 20),
-                    above(20),
-                    fmt_ns(1 << 22),
-                    above(22),
-                    fmt_ns(1 << 24),
-                    above(24),
-                    fmt_ns(1 << 26),
-                    above(26),
-                );
-            }
         }
     }
 
@@ -398,11 +628,31 @@ impl<'a> Scheduler<'a> {
         let every = self.opts.report_sec.max(1);
         let mut ticks = 0u64;
 
+        // Telemetry is not the scheduler. A broken stats channel used to
+        // propagate out of run(), which main() reports as "ejected by the
+        // kernel" -- so a dead socket read as a stall in the journal, exited
+        // non-zero, and (under the systemd unit) fired the critical desktop
+        // notification. Meanwhile the actual policy was still running fine.
+        // Now it degrades: log once, stop serving stats, keep scheduling.
+        let mut stats_live = true;
+
         while !shutdown.load(Ordering::Relaxed) && !self.exited() {
+            if !stats_live {
+                std::thread::sleep(Duration::from_secs(1));
+                ticks += 1;
+                if ticks % every == 0 {
+                    self.report();
+                }
+                continue;
+            }
+
             match req_ch.recv_timeout(Duration::from_secs(1)) {
                 Ok(()) => {
                     let m = self.metrics();
-                    res_ch.send(m)?;
+                    if let Err(e) = res_ch.send(m) {
+                        warn!("stats channel closed ({e}); telemetry off, scheduler still running");
+                        stats_live = false;
+                    }
                 }
                 Err(RecvTimeoutError::Timeout) => {
                     ticks += 1;
@@ -410,7 +660,10 @@ impl<'a> Scheduler<'a> {
                         self.report();
                     }
                 }
-                Err(e) => Err(e)?,
+                Err(e) => {
+                    warn!("stats channel closed ({e}); telemetry off, scheduler still running");
+                    stats_live = false;
+                }
             }
         }
 
@@ -482,7 +735,7 @@ fn main() -> Result<()> {
             Ok(uei) => uei,
             Err(e) => {
                 error!("{SCHEDULER_NAME} was ejected by the kernel: {e:#}");
-                error!("all tasks are back on default kernel scheduler; the soryu is NOT running");
+                error!("all tasks are back on default kernel scheduler; the scheduler is NOT running");
                 return Err(e);
             }
         };

src/stats.rs

@@ -29,7 +29,9 @@ pub struct Metrics {
     pub enq_pinned: u64,
     #[stat(desc = "Tasks bounced back by ops.cpu_release")]
     pub enq_reenq: u64,
-    #[stat(desc = "DSQ inserts that failed")]
+    #[stat(desc = "Tasks moved from a level DSQ to a local one")]
+    pub dispatch: u64,
+    #[stat(desc = "DSQ inserts that failed and fell back to the global DSQ")]
     pub insert_fail: u64,
     #[stat(desc = "CPUs taken by a higher sched class (PipeWire RT, i915)")]
     pub cpu_release: u64,
@@ -53,6 +55,15 @@ pub struct Metrics {
     #[stat(desc = "Wakeups that kicked an idle CPU rather than preempting anyone")]
     pub preempt_idle: u64,
 
+    #[stat(desc = "Lit-but-empty level DSQs touched while placing a task")]
+    pub dispatch_miss: u64,
+    #[stat(desc = "Starving levels this CPU could not serve (affinity)")]
+    pub starve_blocked: u64,
+    #[stat(desc = "Bits relit after a level was found occupied but unadvertised")]
+    pub bitmap_repair: u64,
+    #[stat(desc = "Wakeups where the waker handed the wakee some of its boost")]
+    pub wake_donate: u64,
+
     #[stat(desc = "Mean boost, x100 (negative is more interactive)")]
     pub boost_mean_x100: i64,
     #[stat(desc = "Percent of scheduling events at the -12 boost floor")]
@@ -71,30 +82,47 @@ pub struct Metrics {
 
     #[stat(desc = "Worst wait by any runnable task since load, jiffies")]
     pub max_wait_j: u64,
-    #[stat(desc = "Levels currently holding tasks")]
+    /// Meaning depends on --clear-bitmap, which defaults to false: set-only,
+    /// so "levels used since load" rather than a live occupancy map. Clearing
+    /// makes it live and measured worse (docs/bitmap-clear.md). With clearing
+    /// on, watch `bitmap_repair` -- a nonzero rate means the clear is racing
+    /// more than the argument in main.bpf.c predicts.
+    #[stat(desc = "Levels holding tasks (set-only unless --clear-bitmap)")]
     pub lvl_bitmap: u64,
 }
 
-impl Metrics {
+/// Single copy, shared with the loader's own report.
-    fn fmt_ns(ns: u64) -> String {
+///
-        if ns >= 1_000_000_000 {
+/// One significant decimal below 100, because the histogram now resolves ~19%
-            format!("{:.1}s", ns as f64 / 1e9)
+/// steps and truncating to whole units threw that away again -- 1.25 ms and
+/// 1.75 ms both printed as "1ms", which is the exact resolution problem the
+/// finer buckets were added to fix.
+pub fn fmt_ns(ns: u64) -> String {
+    let (v, unit) = if ns >= 1_000_000_000 {
+        (ns as f64 / 1e9, "s")
     } else if ns >= 1_000_000 {
-            format!("{}ms", ns / 1_000_000)
+        (ns as f64 / 1e6, "ms")
     } else if ns >= 1_000 {
-            format!("{}us", ns / 1_000)
+        (ns as f64 / 1e3, "us")
     } else {
-            format!("{ns}ns")
+        return format!("{ns}ns");
+    };
+
+    if v >= 100.0 {
+        format!("{v:.0}{unit}")
+    } else {
+        format!("{v:.1}{unit}")
     }
 }
 
+impl Metrics {
     pub fn format<W: Write>(&self, w: &mut W) -> Result<()> {
         writeln!(
             w,
             "wake->run interactive  p50 {:>7}  p99 {:>7}  p99.9 {:>7}  (n={})",
-            Self::fmt_ns(self.lat_boosted_p50),
+            fmt_ns(self.lat_boosted_p50),
-            Self::fmt_ns(self.lat_boosted_p99),
+            fmt_ns(self.lat_boosted_p99),
-            Self::fmt_ns(self.lat_boosted_p999),
+            fmt_ns(self.lat_boosted_p999),
             self.lat_boosted_n,
         )?;
         writeln!(
@@ -108,11 +136,12 @@ impl Metrics {
         )?;
         writeln!(
             w,
-            "enqueue    local {} shared {} pinned {} reenq {} fail {}",
+            "enqueue    local {} shared {} pinned {} reenq {} dispatch {} fail {}",
             self.enq_local,
             self.enq_shared,
             self.enq_pinned,
             self.enq_reenq,
+            self.dispatch,
             self.insert_fail,
         )?;
         writeln!(
@@ -126,34 +155,57 @@ impl Metrics {
         )?;
         writeln!(
             w,
-            "preempt    kick {}  idle {}  none {}  cpu_release {}  starve_serve {}",
+            "preempt    kick {}  idle {}  none {}  cpu_release {}",
-            self.preempt_kick,
+            self.preempt_kick, self.preempt_idle, self.preempt_none, self.cpu_release,
-            self.preempt_idle,
-            self.preempt_none,
-            self.cpu_release,
-            self.starve_serve,
         )?;
+        writeln!(
+            w,
+            "scan       dispatch_miss {} ({:.1} per dispatch)  repair {}  starve_serve {}  starve_blocked {}",
+            self.dispatch_miss,
+            if self.dispatch > 0 {
+                self.dispatch_miss as f64 / self.dispatch as f64
+            } else {
+                0.0
+            },
+            self.bitmap_repair,
+            self.starve_serve,
+            self.starve_blocked,
+        )?;
+        writeln!(w, "donate     wake_donate {}", self.wake_donate)?;
         Ok(())
     }
 
+    /// Counters are monotonic *within one attach*. They are not monotonic
+    /// across one: a hotplug reload rebuilds the maps from zero while a
+    /// `--monitor` client is still connected and still holding the old
+    /// snapshot as `prev`. Plain `-` wraps there (release builds have
+    /// overflow checks off), so a reload printed deltas near 1.8e19 rather
+    /// than the restart it actually was. saturating_sub prints 0 instead.
     fn delta(&self, rhs: &Self) -> Self {
         Self {
-            enq_local: self.enq_local - rhs.enq_local,
+            enq_local: self.enq_local.saturating_sub(rhs.enq_local),
-            enq_shared: self.enq_shared - rhs.enq_shared,
+            enq_shared: self.enq_shared.saturating_sub(rhs.enq_shared),
-            enq_pinned: self.enq_pinned - rhs.enq_pinned,
+            enq_pinned: self.enq_pinned.saturating_sub(rhs.enq_pinned),
-            enq_reenq: self.enq_reenq - rhs.enq_reenq,
+            enq_reenq: self.enq_reenq.saturating_sub(rhs.enq_reenq),
-            insert_fail: self.insert_fail - rhs.insert_fail,
+            dispatch: self.dispatch.saturating_sub(rhs.dispatch),
-            cpu_release: self.cpu_release - rhs.cpu_release,
+            insert_fail: self.insert_fail.saturating_sub(rhs.insert_fail),
-            boost_sleep: self.boost_sleep - rhs.boost_sleep,
+            cpu_release: self.cpu_release.saturating_sub(rhs.cpu_release),
-            boost_wake: self.boost_wake - rhs.boost_wake,
+            boost_sleep: self.boost_sleep.saturating_sub(rhs.boost_sleep),
-            deboost: self.deboost - rhs.deboost,
+            boost_wake: self.boost_wake.saturating_sub(rhs.boost_wake),
-            slice_carry: self.slice_carry - rhs.slice_carry,
+            deboost: self.deboost.saturating_sub(rhs.deboost),
-            fork_split: self.fork_split - rhs.fork_split,
+            slice_carry: self.slice_carry.saturating_sub(rhs.slice_carry),
-            preempt_kick: self.preempt_kick - rhs.preempt_kick,
+            fork_split: self.fork_split.saturating_sub(rhs.fork_split),
-            starve_serve: self.starve_serve - rhs.starve_serve,
+            preempt_kick: self.preempt_kick.saturating_sub(rhs.preempt_kick),
-            preempt_none: self.preempt_none - rhs.preempt_none,
+            starve_serve: self.starve_serve.saturating_sub(rhs.starve_serve),
-            preempt_idle: self.preempt_idle - rhs.preempt_idle,
+            preempt_none: self.preempt_none.saturating_sub(rhs.preempt_none),
-            lat_boosted_n: self.lat_boosted_n - rhs.lat_boosted_n,
+            preempt_idle: self.preempt_idle.saturating_sub(rhs.preempt_idle),
+            dispatch_miss: self.dispatch_miss.saturating_sub(rhs.dispatch_miss),
+            starve_blocked: self.starve_blocked.saturating_sub(rhs.starve_blocked),
+            bitmap_repair: self.bitmap_repair.saturating_sub(rhs.bitmap_repair),
+            wake_donate: self.wake_donate.saturating_sub(rhs.wake_donate),
+            // lat_boosted_* are already per-interval: the loader computes them
+            // from a differenced histogram, since percentiles cannot be
+            // subtracted. Differencing the count here too would zero it.
             ..self.clone()
         }
     }