cgroup_init: sleepable, and a missing cgrp_ctx is not fatal

.clang-format

@@ -0,0 +1,70 @@
+# Linux kernel style, matching src/bpf/main.bpf.c as written.
+#
+# Without this file clangd falls back to LLVM style (2-space indent, braces on
+# the same line) and editor.formatOnSave rewrites the whole file on first save.
+# Derived from the kernel tree's own .clang-format, trimmed to what this
+# single-file BPF source actually needs.
+---
+Language: Cpp
+BasedOnStyle: LLVM
+
+# Indentation: hard tabs, 8 wide, as in the kernel.
+UseTab: Always
+TabWidth: 8
+IndentWidth: 8
+ContinuationIndentWidth: 8
+ConstructorInitializerIndentWidth: 8
+
+ColumnLimit: 0
+
+# Kernel brace placement: functions get a brace on the next line, everything
+# else keeps it on the same line.
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterFunction: true
+  AfterStruct: true
+  AfterUnion: true
+  AfterEnum: true
+  AfterControlStatement: Never
+  BeforeElse: false
+  BeforeWhile: false
+
+AlignEscapedNewlines: Left
+AlignAfterOpenBracket: Align
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AlwaysBreakBeforeMultilineStrings: false
+BinPackArguments: true
+BinPackParameters: true
+BreakBeforeBinaryOperators: None
+BreakBeforeTernaryOperators: false
+Cpp11BracedListStyle: false
+IndentCaseLabels: false
+IndentGotoLabels: false
+IndentPPDirectives: None
+KeepEmptyLinesAtTheStartOfBlocks: false
+MaxEmptyLinesToKeep: 1
+PointerAlignment: Right
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+SortIncludes: Never
+ReflowComments: false
+
+# BPF/kernel macros that take a block or act as control flow; without these
+# clang-format indents the bodies as if they were expressions.
+ForEachMacros:
+  - bpf_for
+  - bpf_for_each
+  - bpf_repeat
+  - list_for_each_entry
+  - list_for_each_entry_safe
+
+AlignTrailingComments: true
+AlignConsecutiveMacros: false

.envrc

@@ -0,0 +1 @@
+use flake

.gitignore

@@ -4,3 +4,6 @@
 src/bpf/vmlinux.h
 /nixos.qcow2
 /Cargo.lock.orig
+/compile_commands.json
+/.direnv/
+/.cache/

.vscode/settings.json

@@ -0,0 +1,13 @@
+{
+  // main.bpf.c is hand-formatted kernel style with tab-aligned trailing
+  // comments, which clang-format cannot reproduce. The global
+  // editor.formatOnSave would rewrite ~200 lines on every save, so it is off
+  // for C here. .clang-format still exists for deliberate formatting of new
+  // code (right-click -> Format Selection).
+  "[c]": {
+    "editor.formatOnSave": false
+  },
+  "[cpp]": {
+    "editor.formatOnSave": false
+  }
+}

docs/gpu-feeder.md

@@ -0,0 +1,109 @@
+# `--gpu-boost-cap`: don't deboost the render thread
+
+A GPU feeder is a task that submitted GPU work within the last 100 ms, as seen
+by a `tp_btf/i915_request_add` hook: a game's render thread, the compositor,
+Xwayland. BMQ deboosts a task for burning whole slices, which is exactly what a
+render thread does on a heavy frame, and strict priority then queues it behind
+every better level. `--gpu-boost-cap` stops a recent feeder from being
+deboosted past the given boost.
+
+**Result: `-1` removes most of the stutter, and is the default.**
+
+The idea came from reading the Infinity scheduler (galpt/infinity-scheduler),
+whose v4.8 couples CPU and GPU accounting. Its GPU half patches `drm_sched`,
+which i915 does not use, so none of that applies to Ivy Bridge. What carried
+over was the question of whether the CPU side lets the GPU's feeders wait.
+
+## What was run
+
+Half-Life 2 (native GL, Mesa crocus), the same 60 s camera pan each round, on
+the Ivy Bridge target (i5-3210M, 2c/4t, HD 4000, kernel 7.2.8). Frame times
+from MangoHud's CSV log; scheduler numbers from `SORYU_LOG` at one report per
+second. Both settings ran from the same binary, alternating off, on, off, on,
+so the knob is the only thing that differs.
+
+## Finding the cause
+
+The first instrument only saw wake->run, and feeders looked fine there: p99
+around 0.5-0.8 ms with or without load. The render thread's problem is not
+waking up. It rarely sleeps mid-frame; it burns its slice, stops still
+runnable, and waits to get the CPU back -- a path that never passes
+`ops.runnable` and so was invisible.
+
+Measuring that requeue wait, split by the feeder's boost at the moment it
+stopped (one 60 s pan, knob off):
+
+| | p99, median second | p99.9, worst second | seconds with p99.9 > 4 ms |
+| --- | --- | --- | --- |
+| boosted (boost < 0) | 0.92 ms | 3.1 ms | 0 of 72 |
+| deboosted (boost >= 0) | 1.0 ms | **25.2 ms** | **6 of 61** |
+
+Boosted, the wait never went past a few slices. Deboosted, it reached two
+frames at 85 fps. Feeder boost spent that run at a median of -7.6, so the
+render thread was crossing into the deboosted class constantly.
+
+## Frames, idle desktop
+
+| | median frame | p99.9 frame | frames > 2x median | frames > 25 ms | 0.1% low |
+| --- | --- | --- | --- | --- | --- |
+| off, round 1 | 13.2 ms | 44.0 ms | 15 | 27 | 22.6 fps |
+| **-1, round 1** | **12.5 ms** | **26.1 ms** | **6** | **6** | **37.8 fps** |
+| off, round 2 | 12.7 ms | 43.2 ms | 23 | 26 | 21.7 fps |
+| **-1, round 2** | **12.5 ms** | **25.7 ms** | **8** | **8** | **37.9 fps** |
+
+The two rounds of each setting agree closely. Frames over 25 ms fell by about
+70%.
+
+On the scheduler side, deboosted feeder requeues went from ~785/s to ~1/s,
+while the cap itself fired only ~35 times a second -- a small intervention.
+
+## Under load
+
+The same pan with `stress-ng --cpu 4 --timeout 60s` running alongside:
+
+| | avg fps | 1% low | 0.1% low | stress-ng ops/s (real) | ops/s (usr+sys) |
+| --- | --- | --- | --- | --- | --- |
+| off | 60.8 | 30.0 | 21.2 | 1026 | 778 |
+| **-1** | **73.3** | **46.6** | **36.8** | 750 | 753 |
+
+## The cost
+
+stress-ng lost 27% of its throughput. Its ops per CPU-second barely moved (778
+against 753), so it did not run slower -- it got less CPU, 60 CPU-seconds
+instead of 79 over the minute, and the game took the difference. That is the
+trade the knob exists to make.
+
+Nothing starved. The worst wait was ~130 jiffies in both rounds, which is the
+`starve_ns` bound itself, and `starve_serve` moved from 1.2/s to 1.5/s.
+
+On an idle desktop the other side of the trade shows up in the `deboosted`
+class: its wake->run p99 went from ~0.5 ms to 1.0-1.3 ms.
+
+The cap applies to anything that submits GPU work, including an OpenCL render
+(DaVinci, beignet/pocl). A build running alongside one will lose a similar
+share. That was judged acceptable for a desktop; `--gpu-boost-cap=12` turns it
+off.
+
+## Dropped: waking out of a GPU wait
+
+The first version also hooked `i915_request_wait_begin/end` and
+`dma_fence_wait_start/end`, to time feeders waking from a GPU wait. Over 20
+minutes of play that saw 1251 such wakeups against 434k submissions: crocus
+does not block in the kernel for its fences, so there was nothing there to
+schedule. The hooks were removed.
+
+## For reference: EEVDF
+
+With soryu unloaded, the same game under `stress-ng --cpu 4` ran at 8-11 fps
+(read off the overlay, one run), against 40-50 for soryu without the cap. On an
+idle desktop EEVDF's 1% low was 47.7 against 43.2 for soryu without the cap,
+which is the stutter this change removes; with it, soryu's 1% low is 46.6-47.3.
+
+## Open
+
+- One game. A Vulkan title through hasvk, and a game whose render thread
+  rarely burns a full slice, would say whether -1 is right in general or only
+  for this workload.
+- No AMD numbers. Without `/sys/kernel/btf/i915` the hook does not load and the
+  cap has no feeders to act on; an amdgpu equivalent would need its own
+  tracepoint.

src/bpf/main.bpf.c

@@ -359,6 +359,75 @@ u64 bounce_hist[LAT_BUCKETS];
 #define RELEASE_REASONS 4
 u64 release_reason[RELEASE_REASONS];
 
+/*
+ * GPU feeders: tasks that submitted GPU work within the last GPU_FEEDER_NS,
+ * as seen by the i915_request_add hook at the bottom of this file.
+ *
+ * The thread that hands work to the GPU (a game's render thread, the
+ * compositor, Xwayland) is the one whose lateness costs a frame: if it runs
+ * late, the batch reaches the ring late, and on Gen7 there is no GPU-side
+ * scheduling to make that back up -- legacy ringbuffer submission, no
+ * preemption. Infinity v4.8 coupled CPU and GPU accounting for this reason.
+ *
+ * Three populations, all against lat_hist's buckets:
+ *
+ *   0  feeder: wake->run of a feeder
+ *   1  feeder requeue, boosted: stopped still runnable (slice end, kick,
+ *      bounce) with boost < 0, until it runs again
+ *   2  feeder requeue, not boosted: same, boost >= 0
+ *
+ * 1 and 2 are the population wake->run cannot see. A render thread that burns
+ * its slice mid-frame does not sleep, so it never passes ops.runnable and
+ * never gets a wake_at; the wait it pays to get the CPU back was invisible.
+ * The split is by boost at the moment it stopped, because the question was
+ * whether being deboosted is what makes that wait long. It was -- see
+ * gpu_boost_cap below and docs/gpu-feeder.md.
+ *
+ * Feeders waking out of a GPU wait were measured too, through the i915 and
+ * dma_fence wait tracepoints, and dropped: 1251 such wakeups in 20 minutes of
+ * play. Mesa's crocus does not block in the kernel for its fences, so there
+ * was no population there to schedule.
+ *
+ * 100 ms is three frames at 30 fps: generous on purpose, so the population is
+ * every thread that is plausibly feeding a frame and not only the ones caught
+ * mid-burst.
+ */
+#define GPU_FEEDER_NS (100ULL * 1000 * 1000)
+#define GPU_POPS 3
+
+/*
+ * A recent feeder is not deboosted past gpu_boost_cap. BOOST_RANGE (12) turns
+ * it off, since boost never exceeds it anyway.
+ *
+ * BMQ deboosts a render thread for doing exactly what a heavy frame needs,
+ * burning whole slices, and strict priority then queues it behind every
+ * better level. One 60 s camera pan in Half-Life 2, no background load: a
+ * feeder that stopped still runnable while boosted got the CPU back within
+ * 3.1 ms at p99.9 in every second of the run. Deboosted, it passed 4 ms in 6
+ * of 61 seconds and reached 25 ms -- two frames at 85 fps. -1 keeps feeders
+ * in the boosted class, where that wait never went long.
+ *
+ * Measured both ways, same binary, alternating (docs/gpu-feeder.md):
+ *
+ *   idle desktop, two rounds each   frames > 25 ms  26-27 -> 6-8
+ *                                   p99.9 frametime 44 ms -> 26 ms
+ *   under stress-ng --cpu 4         average fps     60.8 -> 73.3
+ *                                   1% low          30.0 -> 46.6
+ *
+ * The cost is that anything submitting to the GPU keeps its standing however
+ * much CPU it burns -- an OpenCL render included -- and outranks nice-0 hogs
+ * while it does. Measured: stress-ng lost 27% of its throughput to the game,
+ * at unchanged ops per CPU-second, so it ran less rather than slower, and the
+ * worst wait stayed at the starve_ns bound either way. That is the trade a
+ * desktop scheduler should make, so it is the default.
+ */
+const volatile s32 gpu_boost_cap = -1;
+
+u64 gpu_lat_hist[GPU_POPS * LAT_BUCKETS];
+
+/* boost_hist, restricted to feeders. */
+u64 gpu_boost_hist[2 * BOOST_RANGE + 1];
+
 /*
  * Bucket index for @ns: octave * LAT_SUBS + the two bits below the leading
  * one. Values under 4 ns have no mantissa bits to take and all land in
@@ -393,6 +462,9 @@ struct task_ctx
 	u64 bounce_at; /* scx_bpf_now() when ops.cpu_release threw it back */
 	s32 cgrp_adj;  /* cached level adjustment from cpu.weight */
 	u64 cgrp_seen; /* cgrp_gen this adjustment was computed against */
+	u64 gpu_submit_at; /* scx_bpf_now() at its last i915_request_add */
+	u64 requeue_at;	   /* scx_bpf_now() when a feeder stopped still runnable */
+	bool requeue_boosted; /* its boost was < 0 at that moment */
 };
 
 struct cgrp_ctx
@@ -445,6 +517,8 @@ enum soryu_stat_idx
 	SORYU_STAT_STARVE_BLOCKED,
 	SORYU_STAT_BITMAP_REPAIR,
 	SORYU_STAT_WAKE_DONATE,
+	SORYU_STAT_GPU_SUBMIT,
+	SORYU_STAT_GPU_CAP,
 	SORYU_NR_STATS,
 };
 
@@ -470,6 +544,16 @@ static struct task_ctx *lookup_task_ctx(struct task_struct *p)
 	return bpf_task_storage_get(&task_ctxs, p, 0, 0);
 }
 
+/*
+ * Signed, so a submit stamped on another CPU a hair ahead of this one's clock
+ * still reads as recent rather than as 2^64 ns ago.
+ */
+static bool gpu_feeder(struct task_ctx *tctx, u64 now)
+{
+	return tctx->gpu_submit_at &&
+		   (s64)(now - tctx->gpu_submit_at) < (s64)GPU_FEEDER_NS;
+}
+
 /*
  * BMQ boost_task(): the floor depends on policy. SCHED_NORMAL may go all the
  * way to -12; SCHED_BATCH may only climb back to 0, so a batch task can recover
@@ -1221,9 +1305,15 @@ void BPF_STRUCT_OPS(soryu_running, struct task_struct *p)
 	if (tctx)
 	{
 		u32 idx = (u32)(tctx->boost + BOOST_RANGE);
+		u64 now = scx_bpf_now();
+		bool feeder = gpu_feeder(tctx, now);
 
 		if (idx < 2 * BOOST_RANGE + 1)
+		{
 			boost_hist[idx]++;
+			if (feeder)
+				gpu_boost_hist[idx]++;
+		}
 
 		/*
 		 * Clamp, do not drop. ule_score() returns a closed [0, 100],
@@ -1251,7 +1341,7 @@ void BPF_STRUCT_OPS(soryu_running, struct task_struct *p)
 		if (vtime_within_level && time_before(READ_ONCE(vtime_now), p->scx.dsq_vtime))
 			WRITE_ONCE(vtime_now, p->scx.dsq_vtime);
 
-		tctx->run_at = scx_bpf_now();
+		tctx->run_at = now;
 		tctx->slice_given = p->scx.slice;
 
 		if (tctx->wake_at)
@@ -1260,13 +1350,30 @@ void BPF_STRUCT_OPS(soryu_running, struct task_struct *p)
 						  ? tctx->run_at - tctx->wake_at
 						  : 0;
 			u32 cls = tctx->boost < 0 ? 0 : (tctx->boost ? 2 : 1);
-			u32 slot = cls * LAT_BUCKETS + lat_bucket(lat);
+			u32 b = lat_bucket(lat);
+			u32 slot = cls * LAT_BUCKETS + b;
 			if (slot < LAT_CLASSES * LAT_BUCKETS)
 				lat_hist[slot]++;
 
+			/* Every sample here is also in lat_hist above; this is a
+			 * subset of it, not a separate measurement. */
+			if (feeder && b < LAT_BUCKETS)
+				gpu_lat_hist[b]++;
+
 			tctx->wake_at = 0;
 		}
 
+		if (tctx->requeue_at)
+		{
+			u64 rl = now > tctx->requeue_at ? now - tctx->requeue_at : 0;
+			u32 pop = tctx->requeue_boosted ? 1 : 2;
+			u32 slot = pop * LAT_BUCKETS + lat_bucket(rl);
+
+			if (slot < GPU_POPS * LAT_BUCKETS)
+				gpu_lat_hist[slot]++;
+			tctx->requeue_at = 0;
+		}
+
 		/*
 		 * Recorded independently of wake_at, not instead of it. A task
 		 * that woke and was then bounced belongs in both: wake->run is
@@ -1332,11 +1439,24 @@ void BPF_STRUCT_OPS(soryu_stopping, struct task_struct *p, bool runnable)
 		return;
 	}
 
+	/* Boost as it was while running, before the deboost below can move it. */
+	if (gpu_feeder(tctx, now))
+	{
+		tctx->requeue_at = now;
+		tctx->requeue_boosted = tctx->boost < 0;
+	}
+
 	left = tctx->slice_given > ran ? tctx->slice_given - ran : 0;
 
 	if (left < RESCHED_NS)
 	{
-		deboost_task(p, tctx);
+		/* The first test keeps gpu_cap at zero when the knob is off,
+		 * rather than counting deboosts that were no-ops at +12. */
+		if (gpu_boost_cap < BOOST_RANGE && tctx->boost >= gpu_boost_cap &&
+			gpu_feeder(tctx, now))
+			stat_inc(SORYU_STAT_GPU_CAP);
+		else
+			deboost_task(p, tctx);
 		tctx->slice_left = 0;
 		stat_inc(SORYU_STAT_DEBOOST);
 	}
@@ -1384,6 +1504,9 @@ s32 BPF_STRUCT_OPS(soryu_init_task, struct task_struct *p,
 	tctx->bounce_at = 0;
 	tctx->cgrp_adj = 0;
 	tctx->cgrp_seen = 0; /* forces a refresh on the first enqueue */
+	tctx->gpu_submit_at = 0;
+	tctx->requeue_at = 0;
+	tctx->requeue_boosted = false;
 
 	/* Starting at 0 would read as an enormous credit against vtime_now and
 	 * hand every new task the head of its level. */
@@ -1427,15 +1550,29 @@ s32 BPF_STRUCT_OPS(soryu_init_task, struct task_struct *p,
 	return 0;
 }
 
-s32 BPF_STRUCT_OPS(soryu_cgroup_init, struct cgroup *cgrp,
-				   struct scx_cgroup_init_args *args)
+/*
+ * Sleepable, and never fatal.
+ *
+ * This used to return -ENOMEM when the storage could not be created, and an
+ * error from ops.cgroup_init ejects the scheduler. It did, at boot, once per
+ * boot on 7-11 Oct: "ops.cgroup_init() failed (-12)" while systemd was
+ * creating cgroups, then systemd restarted us 5 s later -- 5 s of EEVDF every
+ * boot. Non-sleepable, the create is a GFP_ATOMIC allocation, and that is the
+ * likeliest way for it to fail there; sleepable makes it GFP_KERNEL.
+ *
+ * And a missing cgrp_ctx is not worth an ejection anyway: cgrp_refresh() and
+ * ops.cgroup_set_weight both already treat it as "no weight known", so the
+ * cost is that one cgroup's tasks run at the neutral adjustment.
+ */
+s32 BPF_STRUCT_OPS_SLEEPABLE(soryu_cgroup_init, struct cgroup *cgrp,
+							 struct scx_cgroup_init_args *args)
 {
 	struct cgrp_ctx *cgc;
 
 	cgc = bpf_cgrp_storage_get(&cgrp_ctxs, cgrp, 0,
 							   BPF_LOCAL_STORAGE_GET_F_CREATE);
 	if (!cgc)
-		return -ENOMEM;
+		return 0;
 
 	cgc->weight = args->weight;
 	__sync_fetch_and_add(&cgrp_gen, 1);
@@ -1517,6 +1654,39 @@ void BPF_STRUCT_OPS(soryu_exit, struct scx_exit_info *ei)
 	UEI_RECORD(uei, ei);
 }
 
+/*
+ * The feeder hook. i915_request_add fires in the context of the task doing
+ * the submitting, so current is the task we want.
+ *
+ * Kernel threads are skipped: i915 submits its own requests from workers
+ * (heartbeats, retires, display), and a kworker that once touched the ring is
+ * not a frame feeder. A task scx has not initialised yet has no tctx and is
+ * skipped too.
+ *
+ * The tracepoint lives in i915's module BTF. The loader turns this program
+ * off when /sys/kernel/btf/i915 is absent -- the test VM, the AMD machines --
+ * so the scheduler still loads there, with no feeders and gpu_boost_cap
+ * inert. The request pointer is never dereferenced, which is also why it is
+ * typed void: nothing here depends on i915's layout.
+ */
+SEC("tp_btf/i915_request_add")
+int BPF_PROG(soryu_i915_request_add, void *rq)
+{
+	struct task_struct *p = bpf_get_current_task_btf();
+	struct task_ctx *tctx;
+
+	if (!p || (p->flags & PF_KTHREAD))
+		return 0;
+
+	tctx = lookup_task_ctx(p);
+	if (tctx)
+	{
+		tctx->gpu_submit_at = scx_bpf_now();
+		stat_inc(SORYU_STAT_GPU_SUBMIT);
+	}
+	return 0;
+}
+
 SCX_OPS_DEFINE(soryu_ops,
 			   .select_cpu = (void *)soryu_select_cpu,
 			   .enqueue = (void *)soryu_enqueue,

src/main.rs

@@ -81,6 +81,8 @@ soryu_stats!(
     starve_blocked,
     bitmap_repair,
     wake_donate,
+    gpu_submit,
+    gpu_cap,
 );
 
 /// mirrors BOOST_RANGE in main.bpf.c.
@@ -97,6 +99,19 @@ const LAT_SUBS: usize = 4;
 const LAT_BUCKETS: usize = LAT_OCTAVES * LAT_SUBS;
 const LAT_CLASS_NAMES: [&str; LAT_CLASSES] = ["boosted", "neutral", "deboosted"];
 
+/// mirrors GPU_POPS in main.bpf.c.
+const GPU_POPS: usize = 3;
+const GPU_POP_NAMES: [&str; GPU_POPS] = [
+    "gpu-feeder",
+    "gpu-requeue boosted",
+    "gpu-requeue deboosted",
+];
+
+/// Module BTF the i915 tracepoints resolve against. Absent means no i915 --
+/// the test VM, or an AMD machine -- and those programs must not be loaded,
+/// or the whole object fails with them.
+const I915_BTF: &str = "/sys/kernel/btf/i915";
+
 /// Lower edge of bucket @i in ns: mantissa (4 + sub) / 4, scaled by 2^octave.
 /// Inverse of lat_bucket() in main.bpf.c.
 fn bucket_ns(i: usize) -> u64 {
@@ -178,6 +193,14 @@ struct Opts {
     #[arg(long, default_value_t = false, env = "SORYU_VTIME_WITHIN_LEVEL",
           action = clap::ArgAction::Set)]
     vtime_within_level: bool,
+
+    /// Boost a recent GPU feeder may not be deboosted past, in [-12, 12].
+    /// The default -1 keeps feeders in the boosted class; 12 turns it off.
+    /// Measured in docs/gpu-feeder.md.
+    #[arg(long, default_value_t = -1, env = "SORYU_GPU_BOOST_CAP",
+          value_parser = clap::value_parser!(i32).range(-12..=12))]
+    gpu_boost_cap: i32,
+
     #[arg(long, default_value_t = 1, env = "SORYU_REPORT_SEC")]
     report_sec: u64,
     #[arg(long, env = "SORYU_LOG")]
@@ -221,6 +244,32 @@ fn percentiles(buckets: &[u64]) -> Option<(u64, u64, u64)> {
     Some((at(0.50), at(0.99), at(0.999)))
 }
 
+/// Mean and occupied-bucket distribution of a boost histogram, None if empty.
+fn boost_summary(hist: &[u64]) -> Option<(f64, String)> {
+    let total: u64 = hist.iter().sum();
+    if total == 0 {
+        return None;
+    }
+
+    let occupied: Vec<String> = hist
+        .iter()
+        .enumerate()
+        .filter(|(_, &c)| c > 0)
+        .map(|(i, &c)| {
+            let b = i as i32 - BOOST_RANGE;
+            format!("{b:+}:{:.1}%", 100.0 * c as f64 / total as f64)
+        })
+        .collect();
+
+    let weighted: i64 = hist
+        .iter()
+        .enumerate()
+        .map(|(i, &c)| (i as i64 - BOOST_RANGE as i64) * c as i64)
+        .sum();
+
+    Some((weighted as f64 / total as f64, occupied.join(" ")))
+}
+
 struct Scheduler<'a> {
     opts: Opts,
     skel: BpfSkel<'a>,
@@ -243,6 +292,8 @@ struct Scheduler<'a> {
     prev_ule: Vec<u64>,
     prev_bounce: Vec<u64>,
     prev_release: Vec<u64>,
+    prev_gpu_lat: Vec<u64>,
+    prev_gpu_boost: Vec<u64>,
 
     /// Separate snapshots for the stats server: it samples on its own schedule
     /// and must not consume the report's intervals, or vice versa.
@@ -281,9 +332,16 @@ impl<'a> Scheduler<'a> {
         rodata.cgrp_weight_levels = opts.cgrp_weight_levels;
         rodata.wake_donate = opts.wake_donate;
         rodata.vtime_within_level = opts.vtime_within_level;
+        rodata.gpu_boost_cap = opts.gpu_boost_cap;
 
         rodata.slice_ns = opts.slice_us * 1_000;
 
+        let have_i915 = std::path::Path::new(I915_BTF).exists();
+        if !have_i915 {
+            skel.progs.soryu_i915_request_add.set_autoload(false);
+            info!("no {I915_BTF}: no GPU feeders, gpu_boost_cap has nothing to act on");
+        }
+
         let mut skel = scx_ops_load!(skel, soryu_ops, uei)?;
         let struct_ops = Some(scx_ops_attach!(skel, soryu_ops)?);
 
@@ -291,7 +349,7 @@ impl<'a> Scheduler<'a> {
         info!(
             "{} attached: {} CPUs, slice {} us, boost_on_wake={}, \
              starve {} ms, clear_bitmap={}, ule_weight={} (window {} ms), \
-             vtime_within_level={}, timeout {} ms",
+             vtime_within_level={}, gpu_boost_cap={}, timeout {} ms",
             SCHEDULER_NAME,
             *scx_utils::NR_CPU_IDS,
             rd.slice_ns / 1000,
@@ -301,6 +359,7 @@ impl<'a> Scheduler<'a> {
             rd.ule_weight,
             rd.ule_window_ns / 1_000_000,
             rd.vtime_within_level,
+            rd.gpu_boost_cap,
             opts.timeout_ms
         );
 
@@ -317,6 +376,8 @@ impl<'a> Scheduler<'a> {
             prev_ule: Vec::new(),
             prev_bounce: Vec::new(),
             prev_release: Vec::new(),
+            prev_gpu_lat: Vec::new(),
+            prev_gpu_boost: Vec::new(),
             stats_prev_lat: Vec::new(),
         })
     }
@@ -379,6 +440,8 @@ impl<'a> Scheduler<'a> {
             starve_blocked: g(stat::starve_blocked),
             bitmap_repair: g(stat::bitmap_repair),
             wake_donate: g(stat::wake_donate),
+            gpu_submit: g(stat::gpu_submit),
+            gpu_cap: g(stat::gpu_cap),
             ..Default::default()
         };
 
@@ -485,39 +548,33 @@ impl<'a> Scheduler<'a> {
         let ule_cur = bss.ule_hist.to_vec();
         let bounce_cur = bss.bounce_hist.to_vec();
         let release_cur = bss.release_reason.to_vec();
+        let gpu_lat_cur = bss.gpu_lat_hist.to_vec();
+        let gpu_boost_cur = bss.gpu_boost_hist.to_vec();
 
         let boost = hist_delta(&boost_cur, &self.prev_boost);
         let lat = hist_delta(&lat_cur, &self.prev_lat);
         let ule = hist_delta(&ule_cur, &self.prev_ule);
         let bounce = hist_delta(&bounce_cur, &self.prev_bounce);
         let release = hist_delta(&release_cur, &self.prev_release);
+        let gpu_lat = hist_delta(&gpu_lat_cur, &self.prev_gpu_lat);
+        let gpu_boost = hist_delta(&gpu_boost_cur, &self.prev_gpu_boost);
 
         self.prev_boost = boost_cur;
         self.prev_lat = lat_cur;
         self.prev_ule = ule_cur;
         self.prev_bounce = bounce_cur;
         self.prev_release = release_cur;
+        self.prev_gpu_lat = gpu_lat_cur;
+        self.prev_gpu_boost = gpu_boost_cur;
 
-        let total: u64 = boost.iter().sum();
-        if total > 0 {
-            let occupied: Vec<String> = boost
-                .iter()
-                .enumerate()
-                .filter(|(_, &c)| c > 0)
-                .map(|(i, &c)| {
-                    let b = i as i32 - BOOST_RANGE;
-                    format!("{b:+}:{:.1}%", 100.0 * c as f64 / total as f64)
-                })
-                .collect();
+        if let Some((mean, dist)) = boost_summary(&boost) {
+            info!("  boost mean={mean:+.2} dist: {dist}");
+        }
 
-            let weighted: i64 = boost
-                .iter()
-                .enumerate()
-                .map(|(i, &c)| (i as i64 - BOOST_RANGE as i64) * c as i64)
-                .sum();
-            let mean = weighted as f64 / total as f64;
-
-            info!("  boost mean={mean:+.2} dist: {}", occupied.join(" "));
+        // Same shape as the line above, so the two read side by side: if
+        // feeders sit on the floor like everyone else, boost already has them.
+        if let Some((mean, dist)) = boost_summary(&gpu_boost) {
+            info!("  gpu-feeder boost mean={mean:+.2} dist: {dist}");
         }
 
         let step = ULE_MAX / ULE_BUCKETS;
@@ -576,6 +633,24 @@ impl<'a> Scheduler<'a> {
             );
         }
 
+        // Subsets of the lines above, cut by what the task was doing with the
+        // GPU rather than by its boost.
+        for (p, name) in GPU_POP_NAMES.iter().enumerate() {
+            let slice = &gpu_lat[p * LAT_BUCKETS..(p + 1) * LAT_BUCKETS];
+            let Some((p50, p99, p999)) = percentiles(slice) else {
+                continue;
+            };
+
+            info!(
+                "  wake->run {}: p50={} p99={} p99.9={} n={}",
+                name,
+                fmt_ns(p50),
+                fmt_ns(p99),
+                fmt_ns(p999),
+                slice.iter().sum::<u64>()
+            );
+        }
+
         // The population every other latency line here is blind to: bounced
         // by ops.cpu_release, re-enqueued without ever passing through
         // ops.runnable, so no wake_at and no wake->run sample.

src/stats.rs

@@ -64,6 +64,11 @@ pub struct Metrics {
     #[stat(desc = "Wakeups where the waker handed the wakee some of its boost")]
     pub wake_donate: u64,
 
+    #[stat(desc = "GPU submissions (i915_request_add) by scx tasks")]
+    pub gpu_submit: u64,
+    #[stat(desc = "Deboosts skipped because the task is a recent GPU feeder")]
+    pub gpu_cap: u64,
+
     #[stat(desc = "Mean boost, x100 (negative is more interactive)")]
     pub boost_mean_x100: i64,
     #[stat(desc = "Percent of scheduling events at the -12 boost floor")]
@@ -172,6 +177,11 @@ impl Metrics {
             self.starve_blocked,
         )?;
         writeln!(w, "donate     wake_donate {}", self.wake_donate)?;
+        writeln!(
+            w,
+            "gpu        submit {}  cap {}",
+            self.gpu_submit, self.gpu_cap,
+        )?;
         Ok(())
     }
 
@@ -203,6 +213,8 @@ impl Metrics {
             starve_blocked: self.starve_blocked.saturating_sub(rhs.starve_blocked),
             bitmap_repair: self.bitmap_repair.saturating_sub(rhs.bitmap_repair),
             wake_donate: self.wake_donate.saturating_sub(rhs.wake_donate),
+            gpu_submit: self.gpu_submit.saturating_sub(rhs.gpu_submit),
+            gpu_cap: self.gpu_cap.saturating_sub(rhs.gpu_cap),
             // lat_boosted_* are already per-interval: the loader computes them
             // from a differenced histogram, since percentiles cannot be
             // subtracted. Differencing the count here too would zero it.

tools/gen-compile-commands.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Regenerate compile_commands.json for src/bpf/main.bpf.c so clangd sees the
+# same flags the real BPF build uses.
+#
+# scx_cargo prints its computed cflags to the build-script output as a line
+# starting with `scx_utils:clang=`. The include paths in it point into
+# target/<profile>/build/scx_soryu-<hash>/out, and that hash changes whenever
+# the build-script inputs change, so this is generated rather than committed.
+#
+# Run inside the devShell after a build:
+#   cargo build --release && ./tools/gen-compile-commands.sh
+
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+root="$PWD"
+
+profile="${1:-release}"
+out=$(ls -td "target/$profile"/build/scx_soryu-*/output 2>/dev/null | head -1 || true)
+
+if [ -z "$out" ]; then
+  echo "error: no build-script output under target/$profile/build/." >&2
+  echo "       run 'cargo build --${profile/dev/}' first (inside the devShell)." >&2
+  exit 1
+fi
+
+line=$(grep -h '^scx_utils:clang=' "$out" | tail -1 || true)
+if [ -z "$line" ]; then
+  echo "error: no 'scx_utils:clang=' line in $out" >&2
+  echo "       the build script was cached; touch src/bpf/main.bpf.c and rebuild." >&2
+  exit 1
+fi
+
+# The cflags are the bracketed Rust Vec<String> debug print at the end of the
+# line: ["-g", "-O2", ...]. Pull the quoted strings out of that part only.
+flags=$(printf '%s' "$line" \
+  | sed 's/.*\(\[[^][]*\]\)$/\1/' \
+  | grep -o '"[^"]*"' \
+  | sed 's/^"//; s/"$//')
+
+if [ -z "$flags" ]; then
+  echo "error: could not parse cflags out of: $line" >&2
+  exit 1
+fi
+
+{
+  printf '[\n  {\n'
+  printf '    "directory": "%s",\n' "$root"
+  printf '    "file": "%s/src/bpf/main.bpf.c",\n' "$root"
+  printf '    "arguments": [\n'
+  printf '      "clang",\n'
+  printf '      "-xc",\n'
+  printf '      "-target", "bpf",\n'
+  while IFS= read -r f; do
+    printf '      %s,\n' "$(printf '%s' "$f" | sed 's/\\/\\\\/g; s/"/\\"/g; s/^/"/; s/$/"/')"
+  done <<<"$flags"
+  printf '      "-c",\n'
+  printf '      "%s/src/bpf/main.bpf.c"\n' "$root"
+  printf '    ]\n  }\n]\n'
+} >compile_commands.json
+
+echo "wrote compile_commands.json ($(wc -l <<<"$flags") flags, profile=$profile)"