diff --git a/src/bpf/main.bpf.c b/src/bpf/main.bpf.c index 171bd3c..27bab15 100644 --- a/src/bpf/main.bpf.c +++ b/src/bpf/main.bpf.c @@ -41,8 +41,17 @@ UEI_DEFINE(uei); #define DSQ_BASE 0x736f7279UL /* "sory" */ #define dsq_id(lvl) (DSQ_BASE + (lvl)) +/* + * Always overwritten by the loader, whose default is 1000 us. BMQ's own 4 ms + * base slice measured worse here: the same compile took 4m49s at 4194 us + * against 4m39s at both 1000 and 500, so 1000 was hardcoded. The value below + * is only what a hand-loaded object would get. + * + * Note this shifts what RESCHED_NS means: at 4 ms it was a 2.4% remainder, + * at 1 ms it is 10%. The carry/deboost threshold is coarser than BMQ's by + * that factor. + */ const volatile u64 slice_ns = 4194304ULL; /* 4 << 20, BMQ sysctl_sched_base_slice */ -const volatile u32 nr_cpu_ids = 1; /* filled in by the loader */ /* * BMQ's sched_task_ttwu() boosts on wakeup in proportion to how long the task @@ -55,9 +64,19 @@ const volatile u32 nr_cpu_ids = 1; /* filled in by the loader */ * it 0, and the whole function is dead code in every BMQ build shipped. * * Fixing it means running behaviour nobody has field-tested; leaving it means - * reproducing a typo. So it is a knob, measured both ways in stage 2. + * reproducing a typo. So it was made a knob and measured both ways. + * + * The typo wins. Two paired runs of the same compile, 68 ten-second intervals + * (docs/boost-on-wake.md): with the wake boost off the interactive tail past + * 1 ms is roughly half, in 63 of 68 intervals, sign test p = 8e-14. Compile + * time is unchanged -- the two configurations traded places between rounds. + * + * So the default is off: BMQ as shipped, typo intact. That is both what the + * measurement says and what a decade of field use already ran. The knob stays + * because the mechanism behind the difference is not understood -- see the + * open question at the end of that doc. */ -const volatile bool boost_on_wake = true; +const volatile bool boost_on_wake = false; /* * Anti-starvation, and a deliberate deviation from BMQ. @@ -69,7 +88,7 @@ const volatile bool boost_on_wake = true; * one dies. Under sched_ext the watchdog calls it a fatal stall and ejects the * whole scheduler, which is what happened on 1 Sep: helium:disk$2 (nice 19, * level 25) waited 7.115s behind seven compilers at level 22 and took us down - * with it. See docs/stage6-stall.md. + * with it. * * So: a level that has held tasks this long without being served gets a turn, * once, ahead of better levels. One task per level per starve_ns, which works @@ -80,6 +99,182 @@ const volatile bool boost_on_wake = true; * 0 disables it and restores BMQ's semantics exactly. */ const volatile u64 starve_ns = 128ULL << 20; /* ~134 ms */ + +/* + * Whether ops.dispatch clears bitmap bits, and with them runs the repair + * sweep that makes clearing survivable. Both or neither: with the clear off, + * no bit can be lost, so the sweep has nothing to repair and its per-dispatch + * nr_queued call is pure cost. + * + * This is a knob and not a decision because the measurement is unresolved. + * Clearing does what it was meant to do -- 55 DSQ probes per dispatch down to + * 0.52 -- but the same build measured a worse interactive tail than the runs + * before it (0.114% past 1 ms against 0.075%, 28 of 34 paired intervals), and + * disabling anti-starvation entirely did not move that number, so the cost is + * not the priority inversions it was first blamed on. + * + * The remaining suspect is this: set-only left lvl_bitmap effectively + * read-only after warmup, and the scan touched 25 different DSQ cache lines + * with unlocked reads that every CPU can hold shared. Clearing turns one u64 + * into a write-hot line -- ~595k atomic RMWs on top of the ~1.1M the enqueue + * side already does -- that four threads must take exclusive turns owning. + * Trading 25 shared reads for one contended line is not obviously a win. + * + * They did. Two rounds, back to back, order reversed between them: clearing + * loses 46 of 69 paired intervals, p = 0.008, a median tail ratio of 0.84 and + * one bucket on p99. Default off. + * + * The asymmetry is what settles it. Clearing buys 37x fewer DSQ probes per + * dispatch -- and that scan never had a measurable latency cost to begin with, + * because consume_dispatch_q() tests list_empty() unlocked (ext.c:2517) and 25 + * shared cache lines are cheaper to touch than one exclusively owned u64. + * Paying a measured cost for an unmeasured saving is the wrong trade. + * + * What clearing looked like it also bought -- a working starvation probe -- + * turned out to be a separate thing that was only accidentally coupled to it. + * See the probe below. + */ +const volatile bool clear_bitmap = false; + +/* + * Whether ops.cpu_release throws the local DSQ back through ops.enqueue. + * + * scx_bpf_reenqueue_local() is the conventional thing to do here, and it is + * right when the CPU is about to be busy for a while: the tasks queued on it + * should go find another one rather than wait. It is waste when the higher + * class is a short burst, because the task pays a full re-enqueue -- level + * recomputed, DSQ reinserted, possibly migrated -- to avoid a wait it would + * have finished sooner than the trip. + * + * Measured: on a desktop, 38916 releases over five minutes, 99.6% of them RT, + * about 130 a second, each bouncing 0.98 tasks. Those bounces then take twice + * as long to reach a CPU as an ordinary wakeup does (p99 1.3 ms against 655 + * us). The RT in question is an audio callback, tens of microseconds. + * + * Risk, stated plainly: with this off, tasks sitting in the local DSQ of a CPU + * that a long-running RT or DL task has taken cannot migrate away. Nothing in + * this scheduler covers that -- starve_ns walks the shared level DSQs, not + * local ones. The backstop is the 5 s watchdog, which ejects rather than + * hangs. Short audio bursts are safe; an RT hog is not. + */ +const volatile bool reenqueue_on_release = true; + +/* + * cgroup cpu.weight, in levels. + * + * Until now this scheduler was completely deaf to it. ops.cgroup_set_weight + * was not implemented, so systemd's CPUWeight=, and with it the whole + * system.slice / user.slice / app.slice arrangement every desktop already has, + * did nothing at all. nice worked; cgroups did not. + * + * cpu.weight is [1, 10000] with 100 the default, and it is a geometric scale: + * one nice step is a factor of 1.25, so an octave of weight is log(2)/log(1.25) + * = 3.1 nice steps. Rounded to 3, which keeps the arithmetic integer and errs + * toward giving cgroups slightly less authority than nice rather than more. + * + * cgrp_weight_levels caps how far that may move a task in each direction; 0 + * disables it. Unlike nice, this is a knob without a measurement behind it + * yet -- and note that what it changes is fairness between groups, not + * latency. Do not expect the interactive tail to move. It is here because a + * desktop scheduler ignoring cpu.weight is a gap, not because it is expected + * to win an A/B. + */ +const volatile s32 cgrp_weight_levels = 0; + +/* + * Wake-chain priority donation. + * + * XNU propagates priority through blocking chains with turnstiles; Linux does + * the same for rt_mutex, but only inside the RT class. From BPF the lock graph + * is not visible -- but the wake graph is, and it is a decent proxy: if A + * wakes B, B is usually about to do something on A's behalf, so B inheriting + * some of A's urgency is closer to right than B keeping its own. + * + * This is deliberately the opposite kind of idea from the ULE ratio, which + * failed. ULE graded more finely and the tail did not move, which said the + * problem is not priority resolution. Donation does not grade at all; it + * changes *who* is urgent. That question has not been tested. + * + * wake_donate is how many boost steps a waker may hand down, capped by the gap + * between them so a wakee is never pushed past its waker. 0 disables it. + */ +const volatile s32 wake_donate = 0; + +/* + * Order each level DSQ by virtual time instead of FIFO. + * + * Every mature sched_ext scheduler does this -- bpfland, flash, lavd, p2dq, + * cake, mitosis, beerland, forge all insert with a vtime. soryu was the + * outlier, inserting plain FIFO. + * + * It is worth trying here specifically because it is not another grading + * scheme. The ULE ratio graded more finely, reached the level arithmetic, and + * moved the tail by nothing (p = 1.0), which said the problem is not priority + * resolution. This changes ordering *within* a band, an axis nothing has + * touched yet. + * + * And FIFO's worst case has exactly the shape of the tail that is left. Arrive + * at a level behind k tasks and wait k slices; at slice_ns = 1 ms, k = 5 is + * 5 ms, which is the >4.2 ms band. Roughly half of all events sit on the boost + * floor, so that queue is the deep one and FIFO is at its worst there. + * + * Unweighted on purpose. Priority already lives in the level -- tasks sharing + * a level share a (nice + boost) by construction -- so within one, vtime means + * only "who has used least CPU goes first". BMQ's strict priority between + * levels is untouched. + * + * A DSQ must be used consistently as FIFO or vtime, never both. This knob is + * read-only after load, and the built-in DSQs the other paths use + * (SCX_DSQ_LOCAL, LOCAL_ON, GLOBAL) are separate, so the level DSQs stay + * consistent either way. + */ +const volatile bool vtime_within_level = false; + +/* + * The clock vtime is measured against: the largest vtime seen at ops.running. + * A sleeping task's vtime stands still while this advances, which is the + * credit that lets it jump the queue on waking -- bounded to one slice by the + * clamp in ops.enqueue, or a task that slept for a minute would own the level. + */ +u64 vtime_now; + +/* + * FreeBSD ULE's interactivity score, bolted alongside BMQ's boost rather than + * replacing it. + * + * BMQ grades a task with a counter: +1 better per sleep, +1 worse per burnt + * slice. Measured under compile load, that counter is not a grade at all -- + * it is a one-bit classifier. The distribution sits bimodal at roughly 54% on + * the -12 floor and 39% on the +12 ceiling, with about 5% spread across the + * twenty-five values in between. A task that both sleeps a lot and burns a lot + * -- a render thread, a compositor, a decoder -- oscillates around the middle + * and never settles anywhere useful. + * + * ULE asks a different question. Not "how many times did you sleep" but "what + * fraction of your recent life was sleep", over a decaying window: + * + * sched_ule.c, sched_interact_score() + * runtime > slptime: 50 + (50 - slptime / (runtime / 50)) -> (50, 100] + * slptime > runtime: runtime / (slptime / 50) -> [0, 50) + * equal: 50 + * + * 0 is a pure sleeper, 100 a pure hog. That is a continuum by construction, + * which is the point: it can put the render thread somewhere, and BMQ's + * counter cannot. + * + * ule_weight is the level adjustment in each direction, so the score moves a + * task by at most +/- ule_weight levels on top of whatever boost did. 0 + * disables it entirely and restores BMQ, which is the default -- this is an + * experiment with a knob, not a decision, and every decision on this branch so + * far was made by running the knob both ways back to back. + */ +#define ULE_MAX 100 +#define ULE_HALF (ULE_MAX / 2) + +const volatile s32 ule_weight = 0; + +/* ULE's SCHED_SLP_RUN_MAX: how much combined history is kept before halving. */ +const volatile u64 ule_window_ns = 5000000000ULL; /* 5 s */ u64 lvl_bitmap; /* Level each CPU is currently running, LEVELS when idle. Used by stage 4. */ @@ -98,11 +293,93 @@ u32 probe_seq; */ u64 boost_hist[2 * BOOST_RANGE + 1]; -#define LAT_CLASSES 3 /* 0 boosted, 1 neutral, 2 deboosted */ -#define LAT_BUCKETS 32 /* log2(ns): bucket 10 is ~1us, 30 is ~1s */ +#define LAT_CLASSES 3 /* 0 boosted, 1 neutral, 2 deboosted */ + +/* + * Latency histogram, log2 with a mantissa. + * + * Plain log2 buckets step by a factor of two, which turned out to be too + * coarse to answer any of the questions the histogram exists for. Measured + * 1000 us against 500 us of slice: the compile differed by 0 s and every + * printed percentile was bit-identical, because a real 20-40% change in + * latency does not move a value across a 2x boundary. The instrument could + * only ever say "same". + * + * So keep the octave and add the top two mantissa bits: buckets land on + * 1.00, 1.25, 1.50 and 1.75 times each power of two, a ~19% step. Same + * unbounded range, 384 u64 of .bss instead of 96. + */ +#define LAT_OCTAVES 32 +#define LAT_SUBS 4 /* mantissa buckets per octave */ +#define LAT_BUCKETS (LAT_OCTAVES * LAT_SUBS) u64 lat_hist[LAT_CLASSES * LAT_BUCKETS]; +/* + * ULE score at ops.running, in buckets of 5, split by what BMQ's boost thinks + * of the same task at the same moment. Kept even when ule_weight is 0: the + * first thing worth knowing is whether the score separates tasks that boost + * cannot, and that is answerable without letting it touch a decision. + * + * Split, because the flat version produced a contradiction. On the same + * compile, boost put 43.6% of events at its +12 ceiling -- fully deboosted, + * hogs by BMQ's reckoning -- while ULE put 1.9% at score >= 75. Two metrics + * over one population disagreeing that sharply about who the hogs are means + * one of them is wrong, or they measure different things, and an A/B on + * ule_weight is uninterpretable until that is settled. + * + * Classes match lat_hist: 0 boosted (boost < 0), 1 neutral, 2 deboosted. + */ +#define ULE_BUCKETS 20 +u64 ule_hist[LAT_CLASSES * ULE_BUCKETS]; + +/* + * Time from being bounced by ops.cpu_release to running again, same buckets + * as lat_hist. + * + * This exists because the tail everything else is measured against cannot see + * this population at all. scx_bpf_reenqueue_local() throws the local DSQ back + * through ops.enqueue with SCX_ENQ_REENQ; ops.runnable is not part of that + * path, so wake_at is never stamped, and ops.running records a latency sample + * only when wake_at is set. Every task bounced by a higher sched class spends + * that trip invisible. + * + * Measured at ~110 bounces per second on a desktop against ~3 during a + * compile, while the desktop tail was three times worse. That is a + * correlation across two different workloads and proves nothing, but it is + * the one concrete lead in a day of nulls, and the instrument that would have + * settled it was blind by construction. + */ +u64 bounce_hist[LAT_BUCKETS]; + +/* + * Which sched class took the CPU, indexed by enum scx_cpu_preempt_reason: + * 0 RT, 1 DL, 2 stop, 3 unknown. + */ +#define RELEASE_REASONS 4 +u64 release_reason[RELEASE_REASONS]; + +/* + * Bucket index for @ns: octave * LAT_SUBS + the two bits below the leading + * one. Values under 4 ns have no mantissa bits to take and all land in + * bucket 0, which is fine -- nothing real is measured there. + */ +static u32 lat_bucket(u64 ns) +{ + u32 oct, sub; + + if (!ns) + return 0; + + oct = 63 - __builtin_clzll(ns); + sub = oct >= 2 ? (u32)(ns >> (oct - 2)) & (LAT_SUBS - 1) : 0; + + if (oct >= LAT_OCTAVES) + return LAT_BUCKETS - 1; + + return oct * LAT_SUBS + sub; +} + struct task_ctx { s32 boost; /* [-BOOST_RANGE, +BOOST_RANGE], negative = better */ @@ -111,8 +388,33 @@ struct task_ctx u64 wake_at; /* scx_bpf_now() at ops.runnable, for lat_hist */ u64 slice_given; /* what p->scx.slice held at ops.running */ u64 slice_left; + u64 runtime; /* ns run, decaying -- ULE ts_runtime */ + u64 slptime; /* ns slept, decaying -- ULE ts_slptime */ + u64 bounce_at; /* scx_bpf_now() when ops.cpu_release threw it back */ + s32 cgrp_adj; /* cached level adjustment from cpu.weight */ + u64 cgrp_seen; /* cgrp_gen this adjustment was computed against */ }; +struct cgrp_ctx +{ + u32 weight; +}; + +struct +{ + __uint(type, BPF_MAP_TYPE_CGRP_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct cgrp_ctx); +} cgrp_ctxs SEC(".maps"); + +/* + * Bumped whenever any cgroup's weight changes. Tasks cache their own + * adjustment and recheck it against this, because ops.cgroup_set_weight + * affects every task in a cgroup and there is no way to walk them. + */ +u64 cgrp_gen; + struct { __uint(type, BPF_MAP_TYPE_TASK_STORAGE); @@ -139,6 +441,10 @@ enum soryu_stat_idx SORYU_STAT_STARVE_SERVE, SORYU_STAT_PREEMPT_NONE, SORYU_STAT_PREEMPT_IDLE, + SORYU_STAT_DISPATCH_MISS, + SORYU_STAT_STARVE_BLOCKED, + SORYU_STAT_BITMAP_REPAIR, + SORYU_STAT_WAKE_DONATE, SORYU_NR_STATS, }; @@ -198,6 +504,136 @@ static void deboost_task(struct task_struct *p, struct task_ctx *tctx) tctx->boost++; } +/* + * sched_ule.c, sched_interact_score(). Integer division by a value derived + * from the larger term, so the result stays in [0, 100] without a clamp: + * in each branch the numerator is the smaller of the two. + * + * BPF division by zero yields zero rather than faulting, but div is forced to + * at least 1 anyway so the arithmetic matches ULE's max(1, ...). + */ +static u32 ule_score(struct task_ctx *tctx) +{ + u64 r = tctx->runtime, sl = tctx->slptime, div; + + if (r > sl) + { + div = r / ULE_HALF; + if (!div) + div = 1; + return ULE_HALF + (ULE_HALF - (u32)(sl / div)); + } + + if (sl > r) + { + div = sl / ULE_HALF; + if (!div) + div = 1; + return (u32)(r / div); + } + + return ULE_HALF; +} + +/* + * sched_ule.c, sched_interact_update(). Halve both terms once their sum + * outgrows the window, so the score describes recent behaviour rather than + * everything since fork. A task that has been away for much longer than the + * window is pinned to whichever extreme it was at instead of decayed -- ULE + * does this so a thread that slept for a minute wakes up interactive rather + * than gradually earning it back. + */ +static void ule_decay(struct task_ctx *tctx) +{ + u64 sum = tctx->runtime + tctx->slptime; + + if (sum < ule_window_ns) + return; + + if (sum > ule_window_ns * 2) + { + if (tctx->runtime > tctx->slptime) + { + tctx->runtime = ule_window_ns; + tctx->slptime = 1; + } + else + { + tctx->slptime = ule_window_ns; + tctx->runtime = 1; + } + return; + } + + tctx->runtime >>= 1; + tctx->slptime >>= 1; +} + +/* [-ule_weight, +ule_weight] levels: negative for sleepers, positive for hogs. */ +static s32 ule_adjust(struct task_ctx *tctx) +{ + if (!ule_weight || !tctx) + return 0; + + return ((s32)ule_score(tctx) - ULE_HALF) * ule_weight / ULE_HALF; +} + +/* + * cpu.weight -> level adjustment. Geometric in, linear out: an octave of + * weight is worth 3 nice steps (log 2 / log 1.25 = 3.106, rounded down), and + * 100 is the neutral point because that is cpu.weight's default. + * + * Negative is better, matching boost: a heavier cgroup should outrank a + * lighter one. + */ +static s32 cgrp_weight_adj(u32 weight) +{ + s32 oct, adj; + + if (!cgrp_weight_levels || !weight) + return 0; + + /* ilog2(weight) - ilog2(100); 100 lands on 6. */ + oct = (s32)(31 - __builtin_clz(weight)) - 6; + adj = -oct * 3; + + if (adj > cgrp_weight_levels) + adj = cgrp_weight_levels; + if (adj < -cgrp_weight_levels) + adj = -cgrp_weight_levels; + + return adj; +} + +/* + * Refresh the cached cgroup adjustment if any weight has changed since it was + * computed. Called from ops.enqueue rather than from task_level(), which runs + * on several hot paths and would otherwise pay an acquire/release per call for + * a value that changes when someone edits a systemd unit. + */ +static void cgrp_refresh(struct task_struct *p, struct task_ctx *tctx) +{ + struct cgroup *cgrp; + struct cgrp_ctx *cgc; + u64 gen = READ_ONCE(cgrp_gen); + + if (!cgrp_weight_levels || !tctx || tctx->cgrp_seen == gen) + return; + + tctx->cgrp_seen = gen; + tctx->cgrp_adj = 0; + + cgrp = scx_bpf_task_cgroup(p); + if (!cgrp) + return; + + cgc = bpf_cgrp_storage_get(&cgrp_ctxs, cgrp, 0, 0); + if (cgc) + tctx->cgrp_adj = cgrp_weight_adj(cgc->weight); + + bpf_cgroup_release(cgrp); +} + static u32 task_level(struct task_struct *p, struct task_ctx *tctx) { s32 v; @@ -207,13 +643,26 @@ static u32 task_level(struct task_struct *p, struct task_ctx *tctx) /* * static_prio in [100, 139], boost in [-12, 12], so v lands in [0, 63] - * and never goes negative — no clamp needed, and >> 1 == / 2. + * and >> 1 == / 2. * * static_prio rather than p->scx.weight on purpose: weight is a * logarithmic thing derived from nice, and the level math wants nice * itself. ops.set_weight is deliberately not implemented. + * + * The clamp is cheap insurance, not arithmetic we expect to need. It + * only matters if scx ever hands us a task with static_prio < 100 (an + * RT prio), and there the unclamped path is genuinely dangerous: v goes + * negative, (u32)v >> 1 is enormous, dsq_id(lvl) names a DSQ that was + * never created, and the insert fails -- which used to mean a lost task + * and an ejection. Two instructions to not depend on that. */ - v = (s32)p->static_prio - 88 + (tctx ? tctx->boost : 0); + v = (s32)p->static_prio - 88 + (tctx ? tctx->boost : 0) + ule_adjust(tctx) + + (tctx ? tctx->cgrp_adj : 0); + + if (v < 0) + v = 0; + if (v > 2 * (IDLE_LEVEL - 1) + 1) + v = 2 * (IDLE_LEVEL - 1) + 1; return (u32)v >> 1; } @@ -226,6 +675,25 @@ static u64 task_slice(struct task_ctx *tctx) return slice_ns; } +/* + * A failed insert used to be counted and dropped. That is not a lost sample, + * it is a lost task: it sits in no DSQ, ops.dispatch has nothing to find it + * by, and the watchdog calls that a runnable task stall and ejects us. So + * insert_fail > 0 meant "ejection in at most timeout_ms", which is a strange + * thing for a counter to mean. + * + * SCX_DSQ_GLOBAL is the escape hatch: the core consumes it before entering + * ops.dispatch and again inside the dispatch loop (ext.c, consume_global_dsq), + * so a task parked there runs without soryu having to know about it. The cost + * is that it runs out of level order -- one priority inversion instead of the + * whole scheduler. + */ +static void insert_fallback(struct task_struct *p, u64 slice, u64 enq_flags) +{ + stat_inc(SORYU_STAT_INSERT_FAIL); + scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, slice, enq_flags); +} + /* * Kick the CPU running the worst thing this task could displace. * @@ -249,7 +717,15 @@ static void try_preempt(struct task_struct *p, u32 lvl, u64 enq_flags) { const struct cpumask *allowed = p->cpus_ptr; s32 worst_cpu = -1; - u32 worst = lvl + 1; /* never IPI an equal or better level */ + /* + * Seed with @lvl so the `cl > worst` test below means "strictly worse + * than the waker" -- an equal or better level is never IPIed. Seeding + * with lvl + 1 instead skipped CPUs running exactly one level worse, + * and one level is two nice steps under BMQ's compression, so a nice 0 + * wakeup would not preempt a nice 4 hog. The pinned path below always + * used the correct test (cpu_level[cpu] > lvl); the two now agree. + */ + u32 worst = lvl; u32 nr = scx_bpf_nr_cpu_ids(); u32 cpu; @@ -342,7 +818,7 @@ s32 BPF_STRUCT_OPS(soryu_select_cpu, struct task_struct *p, s32 prev_cpu, if (scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, task_slice(tctx), 0)) stat_inc(SORYU_STAT_ENQ_LOCAL); else - stat_inc(SORYU_STAT_INSERT_FAIL); + insert_fallback(p, task_slice(tctx), 0); return cpu; } @@ -350,15 +826,26 @@ s32 BPF_STRUCT_OPS(soryu_select_cpu, struct task_struct *p, s32 prev_cpu, void BPF_STRUCT_OPS(soryu_enqueue, struct task_struct *p, u64 enq_flags) { struct task_ctx *tctx = lookup_task_ctx(p); - u32 lvl = task_level(p, tctx); + u32 lvl; + + cgrp_refresh(p, tctx); + lvl = task_level(p, tctx); /* * Tasks bounced back by ops.cpu_release did not stop through any fault * of their own; stage 2 must not read this as a scheduling signal. */ if (enq_flags & SCX_ENQ_REENQ) + { stat_inc(SORYU_STAT_ENQ_REENQ); + /* Start of the trip nothing else measures. Stamped here rather + * than in ops.cpu_release because that callback does not know + * which tasks scx_bpf_reenqueue_local() will hand back. */ + if (tctx) + tctx->bounce_at = scx_bpf_now(); + } + /* * Single-CPU tasks gain nothing from a shared queue, and they make every * other CPU pay: consume_dispatch_q() walks the whole DSQ list under @@ -393,14 +880,35 @@ void BPF_STRUCT_OPS(soryu_enqueue, struct task_struct *p, u64 enq_flags) } else { - stat_inc(SORYU_STAT_INSERT_FAIL); + insert_fallback(p, task_slice(tctx), enq_flags); } return; } - if (!scx_bpf_dsq_insert(p, dsq_id(lvl), task_slice(tctx), enq_flags)) + if (vtime_within_level) { - stat_inc(SORYU_STAT_INSERT_FAIL); + u64 now_v = READ_ONCE(vtime_now); + u64 floor_v = now_v - slice_ns; + u64 v = p->scx.dsq_vtime; + + /* One slice of credit, no more. time_before() handles the wrap + * that a plain `<` would get wrong once vtime_now passes 2^63. */ + if (time_before(v, floor_v)) + { + v = floor_v; + scx_bpf_task_set_dsq_vtime(p, v); + } + + if (!scx_bpf_dsq_insert_vtime(p, dsq_id(lvl), task_slice(tctx), + v, enq_flags)) + { + insert_fallback(p, task_slice(tctx), enq_flags); + return; + } + } + else if (!scx_bpf_dsq_insert(p, dsq_id(lvl), task_slice(tctx), enq_flags)) + { + insert_fallback(p, task_slice(tctx), enq_flags); return; } stat_inc(SORYU_STAT_ENQ_SHARED); @@ -441,24 +949,72 @@ void BPF_STRUCT_OPS(soryu_dispatch, s32 cpu, struct task_struct *prev) { u64 mask = READ_ONCE(lvl_bitmap); u64 now = scx_bpf_now(); + + /* + * One level per dispatch, round robin. Two jobs share this cursor: the + * starvation probe below, and the bitmap repair sweep here. + * + * Repair is what makes clearing bits survivable. The clear in the scan + * below races against ops.enqueue, and while the handshake there is + * ordered correctly on x86, a bit lost anyway would strand its tasks + * forever -- which is the LFBMQ failure exactly. So: if a level has + * queued tasks but no bit, light it. The cursor covers all 33 levels + * every 33 dispatches, which at the measured ~100k dispatches/s is + * well under a millisecond, against a 5 s watchdog. A lost bit becomes + * a delayed dispatch instead of a stalled task. + * + * Cost is one unlocked nr_queued read per dispatch, and only when the + * bit is already clear. Against the 55 probes per dispatch this scan + * used to cost, that is not a trade worth thinking about. + */ + u32 probe = READ_ONCE(starve_cursor); + + probe = probe < LEVELS - 1 ? probe + 1 : 0; + WRITE_ONCE(starve_cursor, probe); + + if (clear_bitmap && !(mask & (1ULL << probe)) && + scx_bpf_dsq_nr_queued(dsq_id(probe)) > 0) + { + __sync_fetch_and_or(&lvl_bitmap, 1ULL << probe); + mask |= 1ULL << probe; + stat_inc(SORYU_STAT_BITMAP_REPAIR); + } + if (starve_ns) { - u32 probe = READ_ONCE(starve_cursor); - - probe = probe < LEVELS - 1 ? probe + 1 : 0; - WRITE_ONCE(starve_cursor, probe); - /* * Telemetry, once every 1024 dispatches -- tens of times a * second, which is three orders of magnitude finer than a * multi-second watchdog needs, and cheap enough to ignore. * (1024 mod 33 = 1, so consecutive samples walk every level.) */ - if (!(probe_seq++ & 1023) && (mask & (1ULL << probe))) + if (!(probe_seq++ & 1023) && scx_bpf_dsq_nr_queued(dsq_id(probe)) > 0) sample_head_wait(probe); - if ((mask & (1ULL << probe)) && - now - READ_ONCE(lvl_last_run[probe]) > starve_ns) + /* + * Deliberately not gated on lvl_bitmap. + * + * It used to be, and that quietly made the whole mechanism + * depend on the bitmap being accurate: with set-only bits most + * probes landed on levels that had been empty for minutes, + * failed the move, stamped lvl_last_run and moved on. Measured + * 27 serves against 244 for the same workload with bits being + * cleared -- the stall protection was mostly spending itself on + * stale bits. + * + * That looked like an argument for clearing. It was not; it was + * an argument for not asking the bitmap. nr_queued answers the + * question directly with one unlocked read, and clearing turned + * out to cost ~16% of the interactive tail (46 of 69 paired + * intervals over two rounds, p = 0.008) by making one u64 a + * write-hot line four threads take turns owning. + * + * Cheap test first: the timestamp compare is free and passing + * it is rare, so nr_queued only runs when a level really has + * gone starve_ns without service. + */ + if (now - READ_ONCE(lvl_last_run[probe]) > starve_ns && + scx_bpf_dsq_nr_queued(dsq_id(probe)) > 0) { if (scx_bpf_dsq_move_to_local(dsq_id(probe), 0)) { @@ -469,14 +1025,34 @@ void BPF_STRUCT_OPS(soryu_dispatch, s32 cpu, struct task_struct *prev) } /* - * Empty, or nothing here can run on this CPU. Either - * way it is not starving -- it was offered a turn and - * had nothing to take. Stamping it keeps a level that - * has been idle for hours (the bitmap is set-only, so - * its bit stays lit forever) from claiming a free pass - * on every probe. + * The move failed, and the two reasons it can fail are + * not the same thing. + * + * Empty: the level really was offered a turn and had + * nothing to take. Stamp it -- the bitmap is set-only, + * so a level idle for hours keeps its bit lit forever + * and would otherwise claim a free pass on every probe. + * + * Not empty, but nothing here can run on this CPU: the + * level IS starving, this CPU just cannot help. Stamping + * it here was a silent hole in the only stall protection + * this scheduler has. Probes land on a given level about + * every 8 ms under load (measured: ~1.4M dispatches over + * 33 levels in 5.5 min), so a level reset every 8 ms can + * never reach a 128 ms starve_ns -- the protection was + * permanently off for any task whose cpumask excludes the + * probing CPU. Reachable: nr_cpus_allowed == 1 goes to + * LOCAL_ON, but restricted-and-multi-CPU masks land here, + * which is exactly the class ops.enqueue documents as + * still using the shared queue. + * + * So leave the clock running and let a CPU that can + * actually take the task find it starving. */ - WRITE_ONCE(lvl_last_run[probe], now); + if (!scx_bpf_dsq_nr_queued(dsq_id(probe))) + WRITE_ONCE(lvl_last_run[probe], now); + else + stat_inc(SORYU_STAT_STARVE_BLOCKED); } } @@ -498,6 +1074,58 @@ void BPF_STRUCT_OPS(soryu_dispatch, s32 cpu, struct task_struct *prev) return; } + /* + * A miss on a lit-but-empty level. + * + * Measured before this clear existed: a median of 55 probes to + * place one task, peaking at 279, against 33 levels. Higher + * than the level count because most of the scanning happens on + * dispatch calls that find nothing at all -- those walk every + * lit level and never increment `dispatch`. A CPU going idle + * was touching 25 DSQs to learn it had no work. + * + * So clear the bit. This is the handshake LFBMQ gets wrong at + * alt_core.c:4260, and getting it wrong strands tasks forever, + * so it is written out in full: + * + * enqueue: insert into the DSQ, THEN set the bit + * dispatch: observe empty, clear the bit, THEN re-check + * + * Both orders matter, and both need the store to land before + * the following load. LFBMQ uses smp_rmb() for the second one, + * which orders loads against loads and cannot order a store + * against a later load -- and store/load is precisely the one + * reordering x86 permits. Here __sync_fetch_and_and() is a + * seq_cst builtin and JITs to a lock-prefixed RMW on x86, so + * the clear is a full barrier and the re-check cannot float + * above it. + * + * Every interleaving is then covered: + * + * insert lands before our first nr_queued read -> we see it, + * no clear + * insert lands between clear and re-check -> re-check + * sees it, + * we re-set + * insert lands after re-check -> enqueue's + * own set + * + * That argument holds on x86. It is thinner on a weakly ordered + * arch, where a BPF atomic without a fetch may not imply a full + * barrier. The repair sweep at the top of this function is the + * reason that is survivable rather than fatal: a bit lost to any + * hole in the reasoning above is relit within 33 dispatches. + */ + stat_inc(SORYU_STAT_DISPATCH_MISS); + + if (clear_bitmap && !scx_bpf_dsq_nr_queued(dsq_id(lvl))) + { + __sync_fetch_and_and(&lvl_bitmap, ~bit); + + if (scx_bpf_dsq_nr_queued(dsq_id(lvl)) > 0) + __sync_fetch_and_or(&lvl_bitmap, bit); + } + mask &= ~bit; } } @@ -518,10 +1146,57 @@ void BPF_STRUCT_OPS(soryu_runnable, struct task_struct *p, u64 enq_flags) tctx->wake_at = now; - if (!boost_on_wake || !tctx->sleep_at || !slice_ns) + /* + * Donate down the wake chain. ops.runnable runs in the waker's context + * for a synchronous wakeup, so bpf_get_current_task_btf() is the waker. + * Not true for every wakeup path -- a remote IPI delivers elsewhere -- + * which makes this an approximation, and the same approximation + * scx_lavd works from. + * + * Capped by the gap: a wakee is pulled toward its waker, never past it. + * Without that a chain of wakeups would ratchet everyone to the floor, + * which is the saturation that boost already suffers from and the last + * thing worth adding more of. + */ + if (wake_donate && (enq_flags & SCX_ENQ_WAKEUP)) + { + struct task_struct *waker = bpf_get_current_task_btf(); + + if (waker && waker != p) + { + struct task_ctx *wtctx = lookup_task_ctx(waker); + + if (wtctx && wtctx->boost < tctx->boost) + { + s32 gap = tctx->boost - wtctx->boost; + s32 give = gap < wake_donate ? gap : wake_donate; + + boost_task(p, tctx, (u32)give); + stat_inc(SORYU_STAT_WAKE_DONATE); + } + } + } + + /* + * slptime accounting is deliberately outside the boost_on_wake gate. + * The ULE score has to see every sleep whether or not BMQ's wake boost + * is enabled, or the two knobs would not be independent and the A/B + * would measure their product. + */ + if (tctx->sleep_at) + { + slept = now > tctx->sleep_at ? now - tctx->sleep_at : 0; + tctx->sleep_at = 0; + tctx->slptime += slept; + ule_decay(tctx); + } + else + { + slept = 0; + } + + if (!boost_on_wake || !slept || !slice_ns) return; - slept = now > tctx->sleep_at ? now - tctx->sleep_at : 0; - tctx->sleep_at = 0; steps = slept / slice_ns; if (!steps) @@ -550,6 +1225,32 @@ void BPF_STRUCT_OPS(soryu_running, struct task_struct *p) if (idx < 2 * BOOST_RANGE + 1) boost_hist[idx]++; + /* + * Clamp, do not drop. ule_score() returns a closed [0, 100], + * and 100 is not a rounding artifact -- it is the score of a + * task with slptime == 0, i.e. every pure CPU hog. Dividing by + * the bucket width puts those at index 20 in a 20-wide array, + * and the obvious `if (sc < ULE_BUCKETS)` guard then discarded + * them silently: 46.5% of events by boost's own count, the + * entire compiler population, missing from the histogram. + * + * It read as ULE and BMQ disagreeing about who the hogs were. + * They did not; the top bucket was on the floor. lat_bucket() + * clamps for the same reason. + */ + u32 sc = ule_score(tctx) / (ULE_MAX / ULE_BUCKETS); + u32 ucls = tctx->boost < 0 ? 0 : (tctx->boost ? 2 : 1); + + if (sc >= ULE_BUCKETS) + sc = ULE_BUCKETS - 1; + + ule_hist[ucls * ULE_BUCKETS + sc]++; + + /* Advance the reference clock. Racy across CPUs and benign: + * the worst case is a slightly stale floor for one enqueue. */ + if (vtime_within_level && time_before(READ_ONCE(vtime_now), p->scx.dsq_vtime)) + WRITE_ONCE(vtime_now, p->scx.dsq_vtime); + tctx->run_at = scx_bpf_now(); tctx->slice_given = p->scx.slice; @@ -559,18 +1260,28 @@ void BPF_STRUCT_OPS(soryu_running, struct task_struct *p) ? tctx->run_at - tctx->wake_at : 0; u32 cls = tctx->boost < 0 ? 0 : (tctx->boost ? 2 : 1); - u32 b = lat ? 63 - __builtin_clzll(lat) : 0; - u32 slot; - - if (b >= LAT_BUCKETS) - b = LAT_BUCKETS - 1; - - slot = cls * LAT_BUCKETS + b; + u32 slot = cls * LAT_BUCKETS + lat_bucket(lat); if (slot < LAT_CLASSES * LAT_BUCKETS) lat_hist[slot]++; tctx->wake_at = 0; } + + /* + * Recorded independently of wake_at, not instead of it. A task + * that woke and was then bounced belongs in both: wake->run is + * what the workload experiences, bounce->run is what the bounce + * added to it. + */ + if (tctx->bounce_at) + { + u64 bl = tctx->run_at > tctx->bounce_at + ? tctx->run_at - tctx->bounce_at + : 0; + + bounce_hist[lat_bucket(bl)]++; + tctx->bounce_at = 0; + } } runnable_at = p->scx.runnable_at; @@ -599,17 +1310,28 @@ void BPF_STRUCT_OPS(soryu_stopping, struct task_struct *p, bool runnable) if (!tctx) return; + now = scx_bpf_now(); + ran = now > tctx->run_at ? now - tctx->run_at : 0; + + /* Same reasoning as slptime in ops.runnable: every scrap of CPU counts + * toward the score, on the sleeping path as much as the preempted one. */ + tctx->runtime += ran; + ule_decay(tctx); + + /* Unweighted: the level already carries priority, so inside one this is + * only "who has used least CPU goes first". */ + if (vtime_within_level) + scx_bpf_task_set_dsq_vtime(p, p->scx.dsq_vtime + ran); + if (!runnable) { boost_task(p, tctx, 1); - tctx->sleep_at = scx_bpf_now(); + tctx->sleep_at = now; tctx->slice_left = 0; /* sleeping earns a fresh slice */ stat_inc(SORYU_STAT_BOOST_SLEEP); return; } - now = scx_bpf_now(); - ran = now > tctx->run_at ? now - tctx->run_at : 0; left = tctx->slice_given > ran ? tctx->slice_given - ran : 0; if (left < RESCHED_NS) @@ -627,11 +1349,17 @@ void BPF_STRUCT_OPS(soryu_stopping, struct task_struct *p, bool runnable) void BPF_STRUCT_OPS(soryu_cpu_release, s32 cpu, struct scx_cpu_release_args *args) { - scx_bpf_reenqueue_local(); + u32 reason = args ? (u32)args->reason : RELEASE_REASONS - 1; + + if (reenqueue_on_release) + scx_bpf_reenqueue_local(); if (cpu >= 0 && cpu < MAX_CPUS) cpu_level[cpu] = LEVELS; + if (reason < RELEASE_REASONS) + release_reason[reason]++; + stat_inc(SORYU_STAT_CPU_RELEASE); } @@ -651,6 +1379,16 @@ s32 BPF_STRUCT_OPS(soryu_init_task, struct task_struct *p, tctx->wake_at = 0; tctx->slice_given = 0; tctx->slice_left = 0; + tctx->runtime = 0; + tctx->slptime = 0; + tctx->bounce_at = 0; + tctx->cgrp_adj = 0; + tctx->cgrp_seen = 0; /* forces a refresh on the first enqueue */ + + /* Starting at 0 would read as an enormous credit against vtime_now and + * hand every new task the head of its level. */ + if (vtime_within_level) + scx_bpf_task_set_dsq_vtime(p, READ_ONCE(vtime_now)); /* * BMQ (alt_core.c:2874, sched_cgroup_fork): @@ -689,6 +1427,48 @@ s32 BPF_STRUCT_OPS(soryu_init_task, struct task_struct *p, return 0; } +s32 BPF_STRUCT_OPS(soryu_cgroup_init, struct cgroup *cgrp, + struct scx_cgroup_init_args *args) +{ + struct cgrp_ctx *cgc; + + cgc = bpf_cgrp_storage_get(&cgrp_ctxs, cgrp, 0, + BPF_LOCAL_STORAGE_GET_F_CREATE); + if (!cgc) + return -ENOMEM; + + cgc->weight = args->weight; + __sync_fetch_and_add(&cgrp_gen, 1); + + return 0; +} + +void BPF_STRUCT_OPS(soryu_cgroup_set_weight, struct cgroup *cgrp, u32 weight) +{ + struct cgrp_ctx *cgc = bpf_cgrp_storage_get(&cgrp_ctxs, cgrp, 0, 0); + + if (cgc) + cgc->weight = weight; + + /* + * Every task in @cgrp now has a stale cached adjustment and there is no + * way to walk them, so invalidate everyone's. Weights change when + * somebody edits a unit file, so the refresh this triggers is a handful + * of lookups spread over the next enqueue of each running task. + */ + __sync_fetch_and_add(&cgrp_gen, 1); +} + +void BPF_STRUCT_OPS(soryu_cgroup_move, struct task_struct *p, + struct cgroup *from, struct cgroup *to) +{ + struct task_ctx *tctx = lookup_task_ctx(p); + + /* Same effect, one task: force a refresh on its next enqueue. */ + if (tctx) + tctx->cgrp_seen = 0; +} + void BPF_STRUCT_OPS(soryu_dump, struct scx_dump_ctx *dctx) { u64 mask = READ_ONCE(lvl_bitmap); @@ -702,9 +1482,12 @@ void BPF_STRUCT_OPS(soryu_dump, struct scx_dump_ctx *dctx) { s32 nr = scx_bpf_dsq_nr_queued(dsq_id(lvl)); - /* [BIT CLEAR!] is an invariant violation: a level holding tasks - * whose bit is unset can never be dispatched. Must never fire - * now that bits are set-only. */ + /* [BIT CLEAR!] is a level holding tasks whose bit is unset -- + * it cannot be dispatched by the scan until the repair sweep + * relights it. Transient and self-healing within 33 dispatches; + * seeing it in a crash dump means the clear handshake lost a + * race, which is worth knowing about even though the sweep + * covers it. */ if (nr > 0) scx_bpf_dump(" lvl %2u: %d queued%s\n", lvl, nr, (mask & (1ULL << lvl)) ? "" : " [BIT CLEAR!]"); @@ -742,6 +1525,9 @@ SCX_OPS_DEFINE(soryu_ops, .running = (void *)soryu_running, .stopping = (void *)soryu_stopping, .cpu_release = (void *)soryu_cpu_release, + .cgroup_init = (void *)soryu_cgroup_init, + .cgroup_set_weight = (void *)soryu_cgroup_set_weight, + .cgroup_move = (void *)soryu_cgroup_move, .init_task = (void *)soryu_init_task, .dump = (void *)soryu_dump, .init = (void *)soryu_init, diff --git a/src/main.rs b/src/main.rs index 307dff8..48d7c54 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 // +// // scx_soryu loader. // // runs in the foreground and holds no state of its own: dropping @@ -9,6 +10,7 @@ mod bpf_skel; mod stats; use bpf_skel::*; +use stats::fmt_ns; use stats::Metrics; use std::mem::MaybeUninit; @@ -35,46 +37,147 @@ use scx_utils::UserExitInfo; const SCHEDULER_NAME: &str = "scx_soryu"; -/// mirrors `enum stat_idx` in main.bpf.c. -const STAT_NAMES: &[&str] = &[ - "enq_shared", - "enq_local", - "enq_reenq", - "enq_pinned", - "dispatch", - "insert_fail", - "cpu_release", - "boost_sleep", - "boost_wake", - "deboost", - "slice_carry", - "fork_split", - "preempt_kick", - "starve_serve", - "preempt_none", - "preempt_idle", -]; +/// mirrors `enum soryu_stat_idx` in main.bpf.c, in order. +/// +/// The names and the indices come from the same list on purpose. They used to +/// be a `STAT_NAMES` array plus hand-written `g(0)`..`g(15)` calls in +/// `metrics()`, with nothing tying the two together: inserting one counter in +/// the middle of the BPF enum would have silently shifted every metric after +/// it, and the numbers would still have looked plausible. +macro_rules! soryu_stats { + ($($name:ident),+ $(,)?) => { + const STAT_NAMES: &[&str] = &[$(stringify!($name)),+]; + + #[allow(non_upper_case_globals)] + mod stat { + soryu_stats!(@idx 0usize; $($name),+); + } + }; + (@idx $n:expr; $head:ident $(, $tail:ident)*) => { + pub const $head: usize = $n; + soryu_stats!(@idx $n + 1; $($tail),*); + }; + (@idx $n:expr;) => {}; +} + +soryu_stats!( + enq_shared, + enq_local, + enq_reenq, + enq_pinned, + dispatch, + insert_fail, + cpu_release, + boost_sleep, + boost_wake, + deboost, + slice_carry, + fork_split, + preempt_kick, + starve_serve, + preempt_none, + preempt_idle, + dispatch_miss, + starve_blocked, + bitmap_repair, + wake_donate, +); /// mirrors BOOST_RANGE in main.bpf.c. const BOOST_RANGE: i32 = 12; -/// mirrors LAT_CLASSES / LAT_BUCKETS in main.bpf.c. +/// mirrors ULE_BUCKETS / ULE_MAX in main.bpf.c. +const ULE_BUCKETS: usize = 20; +const ULE_MAX: usize = 100; + +/// mirrors LAT_CLASSES / LAT_OCTAVES / LAT_SUBS in main.bpf.c. const LAT_CLASSES: usize = 3; -const LAT_BUCKETS: usize = 32; +const LAT_OCTAVES: usize = 32; +const LAT_SUBS: usize = 4; +const LAT_BUCKETS: usize = LAT_OCTAVES * LAT_SUBS; const LAT_CLASS_NAMES: [&str; LAT_CLASSES] = ["boosted", "neutral", "deboosted"]; +/// Lower edge of bucket @i in ns: mantissa (4 + sub) / 4, scaled by 2^octave. +/// Inverse of lat_bucket() in main.bpf.c. +fn bucket_ns(i: usize) -> u64 { + let oct = (i / LAT_SUBS) as u32; + let sub = (i % LAT_SUBS) as u64; + + ((4 + sub) << oct) >> 2 +} + +/// First bucket whose lower edge is at or above @ns, for the tail report. +fn bucket_at_least(ns: u64) -> usize { + (0..LAT_BUCKETS) + .find(|&i| bucket_ns(i) >= ns) + .unwrap_or(LAT_BUCKETS - 1) +} + +/// Elementwise cur - prev, so a report describes its own interval rather than +/// everything since load. +fn hist_delta(cur: &[u64], prev: &[u64]) -> Vec { + cur.iter() + .enumerate() + .map(|(i, &c)| c.saturating_sub(prev.get(i).copied().unwrap_or(0))) + .collect() +} + #[derive(Parser, Debug, Clone)] #[command(about = "BMQ policy port for sched_ext", version)] struct Opts { #[arg(long, default_value_t = 5000, env = "SORYU_TIMEOUT_MS")] timeout_ms: u32, - #[arg(long, default_value_t = true, env = "SORYU_BOOST_ON_WAKE", + /// Boost on wakeup in proportion to time slept. Off by default: measured + /// worse (docs/boost-on-wake.md). Takes a value -- `--boost-on-wake=true`. + #[arg(long, default_value_t = false, env = "SORYU_BOOST_ON_WAKE", action = clap::ArgAction::Set)] boost_on_wake: bool, + /// Clear bitmap bits in ops.dispatch, with the repair sweep that makes it + /// safe. Off by default: measured worse (docs/bitmap-clear.md). Takes a + /// value -- `--clear-bitmap=true`. + #[arg(long, default_value_t = false, env = "SORYU_CLEAR_BITMAP", + action = clap::ArgAction::Set)] + clear_bitmap: bool, + /// Throw the local DSQ back through ops.enqueue when a higher sched class + /// takes the CPU. On by default. See main.bpf.c for the risk. Takes a + /// value -- `--reenqueue-on-release=false`. + #[arg(long, default_value_t = true, env = "SORYU_REENQUEUE_ON_RELEASE", + action = clap::ArgAction::Set)] + reenqueue_on_release: bool, #[arg(long, default_value_t = 1000, env = "SORYU_SLICE_US")] slice_us: u64, #[arg(long, default_value_t = 128, env = "SORYU_STARVE_MS")] starve_ms: u64, + + /// Levels the FreeBSD ULE interactivity score may move a task, in each + /// direction, on top of BMQ's boost. 0 disables it and restores BMQ. + /// The score is measured either way -- see the ule line in the report. + #[arg(long, default_value_t = 0, env = "SORYU_ULE_WEIGHT")] + ule_weight: i32, + + /// Combined run+sleep history the ULE score keeps before halving both. + /// ULE's SCHED_SLP_RUN_MAX. + #[arg(long, default_value_t = 5000, env = "SORYU_ULE_WINDOW_MS")] + ule_window_ms: u64, + + /// Levels a cgroup's cpu.weight may move its tasks, in each direction. + /// 0 disables it, which is what this scheduler did until now -- systemd's + /// CPUWeight= had no effect at all. Changes fairness between groups, not + /// latency. + #[arg(long, default_value_t = 0, env = "SORYU_CGRP_WEIGHT_LEVELS")] + cgrp_weight_levels: i32, + + /// Boost steps a waker may hand down to the task it wakes, capped by the + /// gap between them. 0 disables it. + #[arg(long, default_value_t = 0, env = "SORYU_WAKE_DONATE")] + wake_donate: i32, + + /// Order each level DSQ by virtual time rather than FIFO -- within a level + /// only, so BMQ's strict priority between levels is untouched. Off by + /// default. Takes a value -- `--vtime-within-level=true`. + #[arg(long, default_value_t = false, env = "SORYU_VTIME_WITHIN_LEVEL", + action = clap::ArgAction::Set)] + vtime_within_level: bool, #[arg(long, default_value_t = 1, env = "SORYU_REPORT_SEC")] report_sec: u64, #[arg(long, env = "SORYU_LOG")] @@ -100,38 +203,50 @@ fn percentiles(buckets: &[u64]) -> Option<(u64, u64, u64)> { } let at = |frac: f64| -> u64 { - let target = (total as f64 * frac) as u64; + // ceil, and never zero: truncating gave target = 0 for small n, and + // `seen >= 0` is true before any bucket has been added, so p50 of a + // single sample reported bucket 0 (1ns) no matter where the sample + // actually landed. + let target = ((total as f64 * frac).ceil() as u64).max(1); let mut seen = 0u64; for (i, &c) in buckets.iter().enumerate() { seen += c; if seen >= target { - return 1u64 << i; + return bucket_ns(i); } } - 1u64 << (buckets.len() - 1) + bucket_ns(buckets.len() - 1) }; Some((at(0.50), at(0.99), at(0.999))) } -fn fmt_ns(ns: u64) -> String { - if ns >= 1_000_000_000 { - format!("{:.1}s", ns as f64 / 1e9) - } else if ns >= 1_000_000 { - format!("{}ms", ns / 1_000_000) - } else if ns >= 1_000 { - format!("{}us", ns / 1_000) - } else { - format!("{ns}ns") - } -} - struct Scheduler<'a> { opts: Opts, skel: BpfSkel<'a>, struct_ops: Option, stats_server: StatsServer<(), Metrics>, prev_waits: Vec, + + /// Histogram snapshots from the previous report, so each line describes + /// its own interval. + /// + /// lat_hist and boost_hist accumulate from load and were printed raw, so a + /// report was the average of every phase the machine had been through. A + /// run that idled for 5 s and then got busy for 10 s printed one blended + /// p50 that described neither -- and it moved 16us -> 8us, which reads + /// like an improvement and is really just the mixture shifting. The + /// counters were already deltaed for `--monitor`; the histograms were not + /// deltaed anywhere. + prev_lat: Vec, + prev_boost: Vec, + prev_ule: Vec, + prev_bounce: Vec, + prev_release: Vec, + + /// Separate snapshots for the stats server: it samples on its own schedule + /// and must not consume the report's intervals, or vice versa. + stats_prev_lat: Vec, } impl<'a> Scheduler<'a> { @@ -153,12 +268,19 @@ impl<'a> Scheduler<'a> { .rodata_data .as_mut() .context("no rodata section in skeleton")?; - rodata.nr_cpu_ids = *scx_utils::NR_CPU_IDS as u32; - // SORYU_BOOST_ON_WAKE=0 reproduces BMQ as shipped (its wake boost is - // dead code); the default reproduces what BMQ meant to do. + // The default reproduces BMQ as shipped, whose wake boost is dead + // code. SORYU_BOOST_ON_WAKE=1 turns on what BMQ meant to do, which + // measured worse -- see docs/boost-on-wake.md. rodata.boost_on_wake = opts.boost_on_wake; rodata.starve_ns = opts.starve_ms * 1_000_000; + rodata.clear_bitmap = opts.clear_bitmap; + rodata.reenqueue_on_release = opts.reenqueue_on_release; + rodata.ule_weight = opts.ule_weight; + rodata.ule_window_ns = opts.ule_window_ms * 1_000_000; + rodata.cgrp_weight_levels = opts.cgrp_weight_levels; + rodata.wake_donate = opts.wake_donate; + rodata.vtime_within_level = opts.vtime_within_level; rodata.slice_ns = opts.slice_us * 1_000; @@ -168,12 +290,17 @@ impl<'a> Scheduler<'a> { let rd = skel.maps.rodata_data.as_ref().unwrap(); info!( "{} attached: {} CPUs, slice {} us, boost_on_wake={}, \ - starve {} ms, timeout {} ms", + starve {} ms, clear_bitmap={}, ule_weight={} (window {} ms), \ + vtime_within_level={}, timeout {} ms", SCHEDULER_NAME, *scx_utils::NR_CPU_IDS, rd.slice_ns / 1000, rd.boost_on_wake, rd.starve_ns / 1_000_000, + rd.clear_bitmap, + rd.ule_weight, + rd.ule_window_ns / 1_000_000, + rd.vtime_within_level, opts.timeout_ms ); @@ -185,6 +312,12 @@ impl<'a> Scheduler<'a> { struct_ops, stats_server, prev_waits: Vec::new(), + prev_lat: Vec::new(), + prev_boost: Vec::new(), + prev_ule: Vec::new(), + prev_bounce: Vec::new(), + prev_release: Vec::new(), + stats_prev_lat: Vec::new(), }) } @@ -217,55 +350,69 @@ impl<'a> Scheduler<'a> { } /// One snapshot for the stats server. - fn metrics(&self) -> Metrics { + /// + /// The counters go out cumulative and `Metrics::delta` differences them at + /// the far end; the latency percentiles cannot be differenced that way, so + /// they are computed here from the interval histogram directly. + fn metrics(&mut self) -> Metrics { let c = self.read_stats().unwrap_or_default(); let g = |i: usize| c.get(i).copied().unwrap_or(0); let mut m = Metrics { - enq_shared: g(0), - enq_local: g(1), - enq_reenq: g(2), - enq_pinned: g(3), - insert_fail: g(5), - cpu_release: g(6), - boost_sleep: g(7), - boost_wake: g(8), - deboost: g(9), - slice_carry: g(10), - fork_split: g(11), - preempt_kick: g(12), - starve_serve: g(13), - preempt_none: g(14), - preempt_idle: g(15), + enq_shared: g(stat::enq_shared), + enq_local: g(stat::enq_local), + enq_reenq: g(stat::enq_reenq), + enq_pinned: g(stat::enq_pinned), + dispatch: g(stat::dispatch), + insert_fail: g(stat::insert_fail), + cpu_release: g(stat::cpu_release), + boost_sleep: g(stat::boost_sleep), + boost_wake: g(stat::boost_wake), + deboost: g(stat::deboost), + slice_carry: g(stat::slice_carry), + fork_split: g(stat::fork_split), + preempt_kick: g(stat::preempt_kick), + starve_serve: g(stat::starve_serve), + preempt_none: g(stat::preempt_none), + preempt_idle: g(stat::preempt_idle), + dispatch_miss: g(stat::dispatch_miss), + starve_blocked: g(stat::starve_blocked), + bitmap_repair: g(stat::bitmap_repair), + wake_donate: g(stat::wake_donate), ..Default::default() }; - if let Some(bss) = self.skel.maps.bss_data.as_ref() { - m.max_wait_j = bss.max_wait_jiffies; - m.lvl_bitmap = bss.lvl_bitmap; + let Some(bss) = self.skel.maps.bss_data.as_ref() else { + return m; + }; - let hist = &bss.boost_hist; - let total: u64 = hist.iter().sum(); - if total > 0 { - let weighted: i64 = hist - .iter() - .enumerate() - .map(|(i, &n)| (i as i64 - BOOST_RANGE as i64) * n as i64) - .sum(); - m.boost_mean_x100 = weighted * 100 / total as i64; - m.boost_floor_pct = hist[0] * 100 / total; - m.boost_ceil_pct = hist[hist.len() - 1] * 100 / total; - } + m.max_wait_j = bss.max_wait_jiffies; + m.lvl_bitmap = bss.lvl_bitmap; - // Class 0 is "boosted" -- the interactive tail, and the only class - // with enough samples to mean anything (hogs barely ever wake). - let boosted = &bss.lat_hist[0..LAT_BUCKETS]; - m.lat_boosted_n = boosted.iter().sum(); - if let Some((p50, p99, p999)) = percentiles(boosted) { - m.lat_boosted_p50 = p50; - m.lat_boosted_p99 = p99; - m.lat_boosted_p999 = p999; - } + let hist = &bss.boost_hist; + let total: u64 = hist.iter().sum(); + if total > 0 { + let weighted: i64 = hist + .iter() + .enumerate() + .map(|(i, &n)| (i as i64 - BOOST_RANGE as i64) * n as i64) + .sum(); + m.boost_mean_x100 = weighted * 100 / total as i64; + m.boost_floor_pct = hist[0] * 100 / total; + m.boost_ceil_pct = hist[hist.len() - 1] * 100 / total; + } + + // Class 0 is "boosted" -- the interactive tail, and the only class + // with enough samples to mean anything (hogs barely ever wake). + let cur = bss.lat_hist[0..LAT_BUCKETS].to_vec(); + let interval = hist_delta(&cur, &self.stats_prev_lat); + self.stats_prev_lat = cur; + + m.lat_boosted_n = interval.iter().sum(); + if let Some((p50, p99, p999)) = percentiles(&interval) { + m.lat_boosted_p50 = p50; + m.lat_boosted_p99 = p99; + m.lat_boosted_p999 = p999; } m @@ -326,70 +473,153 @@ impl<'a> Scheduler<'a> { info!(" per-level worst wait: {}", waits.join(" ")); } - if let Some(bss) = self.skel.maps.bss_data.as_ref() { - let hist = &bss.boost_hist; - let total: u64 = hist.iter().sum(); + let Some(bss) = self.skel.maps.bss_data.as_ref() else { + return; + }; - if total > 0 { - let occupied: Vec = hist - .iter() - .enumerate() - .filter(|(_, &c)| c > 0) - .map(|(i, &c)| { - let boost = i as i32 - BOOST_RANGE; - format!("{boost:+}:{:.0}%", 100.0 * c as f64 / total as f64) - }) - .collect(); + // Everything below describes this interval only. The cumulative view + // is still available -- max_wait and lvl_max_wait above are running + // maxima, and the counters on the first line are totals since load. + let boost_cur = bss.boost_hist.to_vec(); + let lat_cur = bss.lat_hist.to_vec(); + let ule_cur = bss.ule_hist.to_vec(); + let bounce_cur = bss.bounce_hist.to_vec(); + let release_cur = bss.release_reason.to_vec(); - let weighted: i64 = hist - .iter() - .enumerate() - .map(|(i, &c)| (i as i64 - BOOST_RANGE as i64) * c as i64) - .sum(); - let mean = weighted as f64 / total as f64; + let boost = hist_delta(&boost_cur, &self.prev_boost); + let lat = hist_delta(&lat_cur, &self.prev_lat); + let ule = hist_delta(&ule_cur, &self.prev_ule); + let bounce = hist_delta(&bounce_cur, &self.prev_bounce); + let release = hist_delta(&release_cur, &self.prev_release); - info!(" boost mean={mean:+.2} dist: {}", occupied.join(" ")); - } + self.prev_boost = boost_cur; + self.prev_lat = lat_cur; + self.prev_ule = ule_cur; + self.prev_bounce = bounce_cur; + self.prev_release = release_cur; - let lines: Vec = (0..LAT_CLASSES) - .filter_map(|c| { - let slice = &bss.lat_hist[c * LAT_BUCKETS..(c + 1) * LAT_BUCKETS]; - percentiles(slice).map(|(p50, p99, p999)| { - format!( - "{}: p50={} p99={} p99.9={} n={}", - LAT_CLASS_NAMES[c], - fmt_ns(p50), - fmt_ns(p99), - fmt_ns(p999), - slice.iter().sum::() - ) - }) + let total: u64 = boost.iter().sum(); + if total > 0 { + let occupied: Vec = boost + .iter() + .enumerate() + .filter(|(_, &c)| c > 0) + .map(|(i, &c)| { + let b = i as i32 - BOOST_RANGE; + format!("{b:+}:{:.1}%", 100.0 * c as f64 / total as f64) }) .collect(); - for l in lines { - info!(" wake->run {l}"); + let weighted: i64 = boost + .iter() + .enumerate() + .map(|(i, &c)| (i as i64 - BOOST_RANGE as i64) * c as i64) + .sum(); + let mean = weighted as f64 / total as f64; + + info!(" boost mean={mean:+.2} dist: {}", occupied.join(" ")); + } + + let step = ULE_MAX / ULE_BUCKETS; + let ule_all: u64 = ule.iter().sum(); + + for c in 0..LAT_CLASSES { + let slice = &ule[c * ULE_BUCKETS..(c + 1) * ULE_BUCKETS]; + let n: u64 = slice.iter().sum(); + if n == 0 { + continue; } - let boosted = &bss.lat_hist[0..LAT_BUCKETS]; - let n: u64 = boosted.iter().sum(); - if n > 0 { - let above = |bucket: usize| -> f64 { - 100.0 * boosted[bucket..].iter().sum::() as f64 / n as f64 - }; + // Bucket midpoint, so the mean is not biased low by truncation. + let weighted: u64 = slice + .iter() + .enumerate() + .map(|(i, &c)| (i * step + step / 2) as u64 * c) + .sum(); - info!( - " wake->run boosted tail: >{} {:.3}% >{} {:.3}% >{} {:.4}% >{} {:.4}%", - fmt_ns(1 << 20), - above(20), - fmt_ns(1 << 22), - above(22), - fmt_ns(1 << 24), - above(24), - fmt_ns(1 << 26), - above(26), - ); - } + // Off both extremes, on a band comparable in width to the one + // used for boost. The flat number was first computed over 25-74, + // which is half the ULE range against 92% of the boost range -- + // not a comparison, and it made ULE look worse than it is. + let mid: u64 = slice[2..18].iter().sum(); + + let occupied: Vec = slice + .iter() + .enumerate() + .filter(|(_, &c)| c > 0) + .map(|(i, &c)| format!("{}:{:.1}%", i * step, 100.0 * c as f64 / n as f64)) + .collect(); + + info!( + " ule[{}] mean={:.1} mid(10-89)={:.1}% share={:.1}% dist: {}", + LAT_CLASS_NAMES[c], + weighted as f64 / n as f64, + 100.0 * mid as f64 / n as f64, + 100.0 * n as f64 / ule_all as f64, + occupied.join(" ") + ); + } + + for c in 0..LAT_CLASSES { + let slice = &lat[c * LAT_BUCKETS..(c + 1) * LAT_BUCKETS]; + let Some((p50, p99, p999)) = percentiles(slice) else { + continue; + }; + + info!( + " wake->run {}: p50={} p99={} p99.9={} n={}", + LAT_CLASS_NAMES[c], + fmt_ns(p50), + fmt_ns(p99), + fmt_ns(p999), + slice.iter().sum::() + ); + } + + // The population every other latency line here is blind to: bounced + // by ops.cpu_release, re-enqueued without ever passing through + // ops.runnable, so no wake_at and no wake->run sample. + if let Some((p50, p99, p999)) = percentiles(&bounce) { + let n: u64 = bounce.iter().sum(); + let names = ["rt", "dl", "stop", "unknown"]; + let by: Vec = release + .iter() + .enumerate() + .filter(|(_, &c)| c > 0) + .map(|(i, &c)| format!("{}:{}", names.get(i).unwrap_or(&"?"), c)) + .collect(); + + info!( + " bounce->run p50={} p99={} p99.9={} n={} taken by {}", + fmt_ns(p50), + fmt_ns(p99), + fmt_ns(p999), + n, + if by.is_empty() { "-".into() } else { by.join(" ") }, + ); + } + + let boosted = &lat[0..LAT_BUCKETS]; + let n: u64 = boosted.iter().sum(); + if n > 0 { + // Thresholds in ns rather than bucket indices: the bucket layout + // is now an implementation detail of lat_bucket(), and hardcoded + // indices silently meant something else the moment it changed. + let tail: Vec = [1_000_000u64, 4_000_000, 16_000_000, 64_000_000] + .iter() + .map(|&ns| { + // Label with the bucket's own edge, not the threshold we + // asked for. They differ by up to the ~19% bucket width, + // and printing ">1.0ms" for a count that really starts at + // 1.05ms is the kind of quiet 5% lie this whole change + // exists to remove. + let from = bucket_at_least(ns); + let pct = 100.0 * boosted[from..].iter().sum::() as f64 / n as f64; + format!(">{} {:.3}%", fmt_ns(bucket_ns(from)), pct) + }) + .collect(); + + info!(" wake->run boosted tail: {}", tail.join(" ")); } } @@ -398,11 +628,31 @@ impl<'a> Scheduler<'a> { let every = self.opts.report_sec.max(1); let mut ticks = 0u64; + // Telemetry is not the scheduler. A broken stats channel used to + // propagate out of run(), which main() reports as "ejected by the + // kernel" -- so a dead socket read as a stall in the journal, exited + // non-zero, and (under the systemd unit) fired the critical desktop + // notification. Meanwhile the actual policy was still running fine. + // Now it degrades: log once, stop serving stats, keep scheduling. + let mut stats_live = true; + while !shutdown.load(Ordering::Relaxed) && !self.exited() { + if !stats_live { + std::thread::sleep(Duration::from_secs(1)); + ticks += 1; + if ticks % every == 0 { + self.report(); + } + continue; + } + match req_ch.recv_timeout(Duration::from_secs(1)) { Ok(()) => { let m = self.metrics(); - res_ch.send(m)?; + if let Err(e) = res_ch.send(m) { + warn!("stats channel closed ({e}); telemetry off, scheduler still running"); + stats_live = false; + } } Err(RecvTimeoutError::Timeout) => { ticks += 1; @@ -410,7 +660,10 @@ impl<'a> Scheduler<'a> { self.report(); } } - Err(e) => Err(e)?, + Err(e) => { + warn!("stats channel closed ({e}); telemetry off, scheduler still running"); + stats_live = false; + } } } @@ -482,7 +735,7 @@ fn main() -> Result<()> { Ok(uei) => uei, Err(e) => { error!("{SCHEDULER_NAME} was ejected by the kernel: {e:#}"); - error!("all tasks are back on default kernel scheduler; the soryu is NOT running"); + error!("all tasks are back on default kernel scheduler; the scheduler is NOT running"); return Err(e); } }; diff --git a/src/stats.rs b/src/stats.rs index 429597a..69f667d 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -29,7 +29,9 @@ pub struct Metrics { pub enq_pinned: u64, #[stat(desc = "Tasks bounced back by ops.cpu_release")] pub enq_reenq: u64, - #[stat(desc = "DSQ inserts that failed")] + #[stat(desc = "Tasks moved from a level DSQ to a local one")] + pub dispatch: u64, + #[stat(desc = "DSQ inserts that failed and fell back to the global DSQ")] pub insert_fail: u64, #[stat(desc = "CPUs taken by a higher sched class (PipeWire RT, i915)")] pub cpu_release: u64, @@ -53,6 +55,15 @@ pub struct Metrics { #[stat(desc = "Wakeups that kicked an idle CPU rather than preempting anyone")] pub preempt_idle: u64, + #[stat(desc = "Lit-but-empty level DSQs touched while placing a task")] + pub dispatch_miss: u64, + #[stat(desc = "Starving levels this CPU could not serve (affinity)")] + pub starve_blocked: u64, + #[stat(desc = "Bits relit after a level was found occupied but unadvertised")] + pub bitmap_repair: u64, + #[stat(desc = "Wakeups where the waker handed the wakee some of its boost")] + pub wake_donate: u64, + #[stat(desc = "Mean boost, x100 (negative is more interactive)")] pub boost_mean_x100: i64, #[stat(desc = "Percent of scheduling events at the -12 boost floor")] @@ -71,30 +82,47 @@ pub struct Metrics { #[stat(desc = "Worst wait by any runnable task since load, jiffies")] pub max_wait_j: u64, - #[stat(desc = "Levels currently holding tasks")] + /// Meaning depends on --clear-bitmap, which defaults to false: set-only, + /// so "levels used since load" rather than a live occupancy map. Clearing + /// makes it live and measured worse (docs/bitmap-clear.md). With clearing + /// on, watch `bitmap_repair` -- a nonzero rate means the clear is racing + /// more than the argument in main.bpf.c predicts. + #[stat(desc = "Levels holding tasks (set-only unless --clear-bitmap)")] pub lvl_bitmap: u64, } -impl Metrics { - fn fmt_ns(ns: u64) -> String { - if ns >= 1_000_000_000 { - format!("{:.1}s", ns as f64 / 1e9) - } else if ns >= 1_000_000 { - format!("{}ms", ns / 1_000_000) - } else if ns >= 1_000 { - format!("{}us", ns / 1_000) - } else { - format!("{ns}ns") - } - } +/// Single copy, shared with the loader's own report. +/// +/// One significant decimal below 100, because the histogram now resolves ~19% +/// steps and truncating to whole units threw that away again -- 1.25 ms and +/// 1.75 ms both printed as "1ms", which is the exact resolution problem the +/// finer buckets were added to fix. +pub fn fmt_ns(ns: u64) -> String { + let (v, unit) = if ns >= 1_000_000_000 { + (ns as f64 / 1e9, "s") + } else if ns >= 1_000_000 { + (ns as f64 / 1e6, "ms") + } else if ns >= 1_000 { + (ns as f64 / 1e3, "us") + } else { + return format!("{ns}ns"); + }; + if v >= 100.0 { + format!("{v:.0}{unit}") + } else { + format!("{v:.1}{unit}") + } +} + +impl Metrics { pub fn format(&self, w: &mut W) -> Result<()> { writeln!( w, "wake->run interactive p50 {:>7} p99 {:>7} p99.9 {:>7} (n={})", - Self::fmt_ns(self.lat_boosted_p50), - Self::fmt_ns(self.lat_boosted_p99), - Self::fmt_ns(self.lat_boosted_p999), + fmt_ns(self.lat_boosted_p50), + fmt_ns(self.lat_boosted_p99), + fmt_ns(self.lat_boosted_p999), self.lat_boosted_n, )?; writeln!( @@ -108,11 +136,12 @@ impl Metrics { )?; writeln!( w, - "enqueue local {} shared {} pinned {} reenq {} fail {}", + "enqueue local {} shared {} pinned {} reenq {} dispatch {} fail {}", self.enq_local, self.enq_shared, self.enq_pinned, self.enq_reenq, + self.dispatch, self.insert_fail, )?; writeln!( @@ -126,34 +155,57 @@ impl Metrics { )?; writeln!( w, - "preempt kick {} idle {} none {} cpu_release {} starve_serve {}", - self.preempt_kick, - self.preempt_idle, - self.preempt_none, - self.cpu_release, - self.starve_serve, + "preempt kick {} idle {} none {} cpu_release {}", + self.preempt_kick, self.preempt_idle, self.preempt_none, self.cpu_release, )?; + writeln!( + w, + "scan dispatch_miss {} ({:.1} per dispatch) repair {} starve_serve {} starve_blocked {}", + self.dispatch_miss, + if self.dispatch > 0 { + self.dispatch_miss as f64 / self.dispatch as f64 + } else { + 0.0 + }, + self.bitmap_repair, + self.starve_serve, + self.starve_blocked, + )?; + writeln!(w, "donate wake_donate {}", self.wake_donate)?; Ok(()) } + /// Counters are monotonic *within one attach*. They are not monotonic + /// across one: a hotplug reload rebuilds the maps from zero while a + /// `--monitor` client is still connected and still holding the old + /// snapshot as `prev`. Plain `-` wraps there (release builds have + /// overflow checks off), so a reload printed deltas near 1.8e19 rather + /// than the restart it actually was. saturating_sub prints 0 instead. fn delta(&self, rhs: &Self) -> Self { Self { - enq_local: self.enq_local - rhs.enq_local, - enq_shared: self.enq_shared - rhs.enq_shared, - enq_pinned: self.enq_pinned - rhs.enq_pinned, - enq_reenq: self.enq_reenq - rhs.enq_reenq, - insert_fail: self.insert_fail - rhs.insert_fail, - cpu_release: self.cpu_release - rhs.cpu_release, - boost_sleep: self.boost_sleep - rhs.boost_sleep, - boost_wake: self.boost_wake - rhs.boost_wake, - deboost: self.deboost - rhs.deboost, - slice_carry: self.slice_carry - rhs.slice_carry, - fork_split: self.fork_split - rhs.fork_split, - preempt_kick: self.preempt_kick - rhs.preempt_kick, - starve_serve: self.starve_serve - rhs.starve_serve, - preempt_none: self.preempt_none - rhs.preempt_none, - preempt_idle: self.preempt_idle - rhs.preempt_idle, - lat_boosted_n: self.lat_boosted_n - rhs.lat_boosted_n, + enq_local: self.enq_local.saturating_sub(rhs.enq_local), + enq_shared: self.enq_shared.saturating_sub(rhs.enq_shared), + enq_pinned: self.enq_pinned.saturating_sub(rhs.enq_pinned), + enq_reenq: self.enq_reenq.saturating_sub(rhs.enq_reenq), + dispatch: self.dispatch.saturating_sub(rhs.dispatch), + insert_fail: self.insert_fail.saturating_sub(rhs.insert_fail), + cpu_release: self.cpu_release.saturating_sub(rhs.cpu_release), + boost_sleep: self.boost_sleep.saturating_sub(rhs.boost_sleep), + boost_wake: self.boost_wake.saturating_sub(rhs.boost_wake), + deboost: self.deboost.saturating_sub(rhs.deboost), + slice_carry: self.slice_carry.saturating_sub(rhs.slice_carry), + fork_split: self.fork_split.saturating_sub(rhs.fork_split), + preempt_kick: self.preempt_kick.saturating_sub(rhs.preempt_kick), + starve_serve: self.starve_serve.saturating_sub(rhs.starve_serve), + preempt_none: self.preempt_none.saturating_sub(rhs.preempt_none), + preempt_idle: self.preempt_idle.saturating_sub(rhs.preempt_idle), + dispatch_miss: self.dispatch_miss.saturating_sub(rhs.dispatch_miss), + starve_blocked: self.starve_blocked.saturating_sub(rhs.starve_blocked), + bitmap_repair: self.bitmap_repair.saturating_sub(rhs.bitmap_repair), + wake_donate: self.wake_donate.saturating_sub(rhs.wake_donate), + // lat_boosted_* are already per-interval: the loader computes them + // from a differenced histogram, since percentiles cannot be + // subtracted. Differencing the count here too would zero it. ..self.clone() } }