From b64e40d88d1b9716937721cc73b878fbb52ff57a Mon Sep 17 00:00:00 2001 From: elwrcl Date: Wed, 2 Sep 2026 20:00:01 +0300 Subject: [PATCH] first commit --- .gitignore | 6 + Cargo.lock | 1540 ++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 27 + README.md | 38 ++ build.rs | 7 + docs/service.md | 99 +++ flake.lock | 61 ++ flake.nix | 331 ++++++++++ src/bpf/main.bpf.c | 751 +++++++++++++++++++++ src/bpf_skel.rs | 7 + src/main.rs | 502 +++++++++++++++ src/stats.rs | 197 ++++++ 12 files changed, 3566 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 README.md create mode 100644 build.rs create mode 100644 docs/service.md create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 src/bpf/main.bpf.c create mode 100644 src/bpf_skel.rs create mode 100644 src/main.rs create mode 100644 src/stats.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e18f329 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/target/ +/result +/result-* +src/bpf/vmlinux.h +/nixos.qcow2 +/Cargo.lock.orig diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..47af2f8 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1540 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn 2.0.119", +] + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "serde", + "tap", + "wyz", +] + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "camino" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb1307f12aa967b5a58416e87b3653360e0fd614a016b6e970db08fecbb1b80d" +dependencies = [ + "serde_core", +] + +[[package]] +name = "cargo-platform" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35af189006b9c0f00a064685c727031e3ed2d8020f7ba284d78cc2671bd36ea" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd5eb614ed4c27c5d706420e4320fbe3216ab31fa1c33cd8246ac36dae4479ba" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", + "terminal_size", + "unicase", + "unicode-width", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "convert_case" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "ctrlc" +version = "3.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" +dependencies = [ + "dispatch2", + "nix", + "windows-sys", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags", + "block2", + "libc", + "objc2", +] + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libbpf-cargo" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e63283df9689af55b85f91e99dc8b213ae89fc3e1d9bb29ab7f75eb8a8a783" +dependencies = [ + "anyhow", + "cargo_metadata", + "clap", + "libbpf-rs", + "memmap2", + "serde", + "serde_json", + "tempfile", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "libbpf-rs" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd8e8db46a2d5967b5b3dba7aba32a875dc38f222395040607045635ec1f9b63" +dependencies = [ + "bitflags", + "libbpf-sys", + "libc", + "vsprintf", +] + +[[package]] +name = "libbpf-sys" +version = "1.7.0+v1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a109478760b2900aa2a6f2087e9d0de1d9c535b1758602af2845d5d2ccfaed7c" +dependencies = [ + "cc", + "nix", + "pkg-config", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ruzstd" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7c1c839d570d835527c9a5e4db7cb2198683a988cb9d7293fc8674e6bd58fc8" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scx_cargo" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1befd874ce76a94bc696b587ca52c123be345c930a1c9f8ea2d52dd3375fe2b5" +dependencies = [ + "anyhow", + "bindgen", + "glob", + "lazy_static", + "libbpf-cargo", + "libbpf-rs", + "ruzstd", + "sscanf", + "tar", + "tempfile", + "tracing", + "tracing-subscriber", + "version-compare", + "walkdir", +] + +[[package]] +name = "scx_soryu" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "crossbeam", + "ctrlc", + "libbpf-rs", + "libc", + "log", + "scx_cargo", + "scx_stats", + "scx_stats_derive", + "scx_utils", + "serde", + "simplelog", +] + +[[package]] +name = "scx_stats" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56631808bc2cba14c8f0be00e0aea8e70c27f3ed6b4630e892ab62c6cfd85e37" +dependencies = [ + "anyhow", + "crossbeam", + "libc", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.119", +] + +[[package]] +name = "scx_stats_derive" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5708dfb29b0be460e0fb9aa704c67193fad64398bceeab00083c11450b0c39ec" +dependencies = [ + "proc-macro2", + "quote", + "scx_stats", + "serde_json", + "syn 2.0.119", +] + +[[package]] +name = "scx_utils" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53eb9590123e91af1593c138aa45428dd13fb525b045cc80d7ea601fbf67b0fa" +dependencies = [ + "anyhow", + "bindgen", + "bitvec", + "clap", + "glob", + "hex", + "lazy_static", + "libbpf-cargo", + "libbpf-rs", + "libc", + "log", + "nix", + "num", + "paste", + "regex", + "ruzstd", + "scx_cargo", + "scx_stats", + "serde", + "sscanf", + "tar", + "tempfile", + "tracing", + "tracing-subscriber", + "version-compare", + "walkdir", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simplelog" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16257adbfaef1ee58b1363bdc0664c9b8e1e30aed86049635fb5f147d065a9c0" +dependencies = [ + "log", + "termcolor", + "time", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "sscanf" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40717d6e75d650a68c4ce1c4247407dc2b70e77e1b04eb0819ea604bac7d5b58" +dependencies = [ + "regex-automata", + "regex-syntax", + "sscanf_macro", +] + +[[package]] +name = "sscanf_macro" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604bbbe9f013054db6976eed3108230ceae6f10cd84d59d196e96d4aa32838d1" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "regex-syntax", + "strsim", + "syn 2.0.119", + "unicode-width", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "chrono", + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "twox-hash" +version = "2.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version-compare" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c2856837ef78f57382f06b2b8563a2f512f7185d732608fd9176cb3b8edf0e" + +[[package]] +name = "vsprintf" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aec2f81b75ca063294776b4f7e8da71d1d5ae81c2b1b149c8d89969230265d63" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..275f54f --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "scx_soryu" +version = "0.1.0" +edition = "2021" +description = "BMQ policy port for sched_ext" +license = "GPL-2.0-only" + +[dependencies] +anyhow = "1.0" +clap = { version = "4", features = ["derive", "env", "wrap_help"] } +ctrlc = { version = "3.4", features = ["termination"] } +libbpf-rs = "=0.26.2" +libc = "0.2" +log = "0.4" +scx_stats = "1.1.3" +scx_stats_derive = "1.1.3" +scx_utils = "1.1.3" +serde = { version = "1", features = ["derive"] } +crossbeam = "0.8" +simplelog = "0.12" + +[build-dependencies] +scx_cargo = "1.1.3" + +[profile.release] +lto = "thin" +panic = "abort" diff --git a/README.md b/README.md new file mode 100644 index 0000000..ec6daf4 --- /dev/null +++ b/README.md @@ -0,0 +1,38 @@ +# scx_soryu + +a port of the BMQ/LFBMQ scheduling policy onto [sched_ext](https://docs.kernel.org/scheduler/sched-ext.html). +the policy is pure BPF C; userspace is a thin Rust loader and more safer than normal scheduler. + +## tested on + +| machine | cpu | status | +| --- | --- | --- | +| target, tested | Intel i5-3210m (Ivy Bridge, 2c/4t) | **verified** — bare metal, extended game + build sessions | +| target, untested | Ryzen 5 PRO 4650u (Renoir, 6c/12t) | not yet run | +| target, untested | Ryzen 5 7500F (ZEN4, 6c/12t) | not yet run | + +## development + +```sh +nix develop # clang, bpftool, rust, libbpf; kernel drift guard +``` + +run it by hand: + +```sh +sudo ./target/release/scx_soryu +``` + +or as a NixOS service — see [`docs/service.md`](docs/service.md). + +test VM (a smoke test, not a benchmark — measure on bare metal): + +```sh +nix build .#nixosConfigurations.testvm.config.system.build.vm +./result/bin/run-nixos-vm +``` + +## kernel + +audited series is **7.1.x**. the devShell refuses to open outside it. +`vmlinux.h` is generated from the running kernel's BTF and is not committed. diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..579d54d --- /dev/null +++ b/build.rs @@ -0,0 +1,7 @@ +fn main() { + scx_cargo::BpfBuilder::new() + .unwrap() + .enable_skel("src/bpf/main.bpf.c", "bpf") + .build() + .unwrap(); +} diff --git a/docs/service.md b/docs/service.md new file mode 100644 index 0000000..3bf3ee3 --- /dev/null +++ b/docs/service.md @@ -0,0 +1,99 @@ +# the NixOS service + +the scheduler is now also a flake package and a NixOS module. running +`sudo ./target/release/scx_soryu` by hand still works — the service is an +addition, not a replacement. + +## adding it to copland + +`flake.nix` inputs: + +```nix +scx-soryu.url = "github:elwrcl/scx_soryu/bmq"; +``` + +in the configuration: + +```nix +imports = [ inputs.scx-soryu.nixosModules.default ]; + +services.scx-soryu = { + enable = true; + notify.user = "elars"; # desktop notification on ejection +}; +``` + +that's it. the defaults are measured values: `sliceUs = 1000` +([`slice.md`](slice.md)), `starveMs = 128` ([`stage6-starve.md`](stage6-starve.md)), +`timeoutMs = 5000`, `reportSec = 60`. + +to turn it off: `services.scx-soryu.enable = false;` or +`systemctl stop scx-soryu`. either one falls back to EEVDF immediately. + +## "something happened and nobody noticed" + +an ejection is a silent event: the kernel drops the scheduler, every task +falls back to default kernel scheduler, and nothing on the desktop says so. three layers were added to fix that. + +**1. the loader says so, loudly.** ejection used to be a bare +`Error: ...` line printed by anyhow — easy to scroll past in a terminal, +unlabeled and unsearchable in the journal. now it's two ERROR-level lines: + +``` +scx_soryu was ejected by the kernel: EXIT: runnable task stall (...) +all tasks are back on default kernel scheduler; the scheduler is NOT running +``` + +**2. systemd retries three times, then gives up.** `Restart=on-failure`, +3 starts in 5 minutes. a transient load spike recovers on its own; a +persistent problem doesn't flap forever — it lands in `failed`. + +**3. `failed` triggers a desktop notification.** a second unit bound with +`OnFailure=` finds the user's session bus and calls +`notify-send --urgency=critical`. if there's no session, it exits quietly +(no notification, no error either). + +either way, the full story is in the journal: + +``` +journalctl -u scx-soryu -n 200 +``` + +## telemetry noise + +the loader prints four lines a second — fine in a terminal, a disaster in +the journal. `--report-sec` was added; the service uses 60. shutdown is +still checked once a second regardless, so `systemctl stop` isn't slowed +down. + +to see live status while the service is running: + +``` +sudo scx_soryu --monitor 3 +``` + +## verified in a VM + +`nixosConfigurations.servicevm` — the same test VM, but running the +scheduler as a service instead of by hand. `tools/servicevm.sh`: + +| check | result | +| --- | --- | +| comes up at boot | `active`, `sched_ext=enabled` | +| clean `systemctl stop` | `inactive`, `sched_ext=disabled`, **not** `failed` | +| restarts | `active`, `enabled` | +| repeated SIGKILL | 2 restarts, then `failed` — no flapping | +| journal | `Start request repeated too quickly` + `Failed with result 'start-limit-hit'` | + +a clean `systemctl stop` mattered: SIGTERM reaches the loader and drops the +struct_ops link, so unload is clean and the unit isn't marked `failed`. + +## packaging note + +`packages.default` builds inside the sandbox: the BPF object is compiled +against `scx_utils`'s vendored `vmlinux.h`, then CO-RE-relocated against +the running kernel's BTF at load time. so the build doesn't depend on the +running kernel — `/sys/kernel/btf/vmlinux` is only needed at **run** time. + +(the `src/bpf/vmlinux.h` sitting in the repo turned out unused — deleting +it, the build still passes the same way. it was already gitignored.) diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..bf0b8fd --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1784007870, + "narHash": "sha256-djcLt/JJphyNt4eDY9XTly+/WbCK5lqWq9lSgCmJkkQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "18b9261cb3294b6d2a06d03f96872827b8fe2698", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "18b9261cb3294b6d2a06d03f96872827b8fe2698", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..c5e58e3 --- /dev/null +++ b/flake.nix @@ -0,0 +1,331 @@ +{ + description = "scx_soryu — BMQ/LFBMQ policy port for sched_ext"; + + # The test VM's kernel is the same derivation as + # the host's booted kernel. See docs/KERNEL.md. + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/18b9261cb3294b6d2a06d03f96872827b8fe2698"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils }: + let + system = "x86_64-linux"; + pkgs = nixpkgs.legacyPackages.${system}; + kernelPackages = pkgs.linuxPackages_7_1; + kernelVersion = kernelPackages.kernel.modDirVersion; + + auditedSeries = "7.1"; + vmBase = { pkgs, lib, ... }: { + boot.kernelPackages = kernelPackages; + + # SysRq-S (sched_ext reset) and SysRq-D (dump) are gated behind + # SYSRQ_ENABLE_RTNICE (0x100). NixOS's default of 184 omits it, so + # the emergency escape hatch is dead unless we set this. + boot.kernel.sysctl."kernel.sysrq" = 440; + + virtualisation = { + cores = 4; + memorySize = 4096; + graphics = false; + # the host: 2 physical cores x 2 threads, one LLC. Without + # this QEMU reports 4 sockets and scx_bpf_select_cpu_dfl()'s SMT + # walk has nothing real to walk. + qemu.options = [ "-cpu host" "-smp 4,sockets=1,cores=2,threads=2" ]; + sharedDirectories.soryu = { + source = "/home/elars/Projects/scx_soryu"; + target = "/mnt/soryu"; + }; + }; + + services.getty.autologinUser = "root"; + users.users.root.password = ""; + + # rt-tests carries both cyclictest and hackbench (M1 harness). + environment.systemPackages = with pkgs; [ stress-ng rt-tests ]; + + system.stateVersion = "25.05"; + }; + in + { + devShells.${system}.default = pkgs.mkShell { + name = "scx_soryu"; + + nativeBuildInputs = with pkgs; [ + clang + llvm + bpftools + pkg-config + rustc + cargo + rustfmt + clippy + rust-analyzer + ]; + + buildInputs = with pkgs; [ + libbpf + elfutils + zlib + zstd + ]; + + # scx_utils' build script runs bindgen, which dlopens libclang. + LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; + + # The nix cc-wrapper injects -fzero-call-used-regs, which clang rejects + # for -target bpf. nixpkgs' own scx derivation disables the same knob. + hardeningDisable = [ "zerocallusedregs" ]; + + RUSTFLAGS = builtins.concatStringsSep " " [ + "-C relocation-model=pic" + "-C link-args=-lelf" + "-C link-args=-lz" + "-C link-args=-lzstd" + ]; + + shellHook = '' + expected="${kernelVersion}" + series="${auditedSeries}" + running="$(uname -r)" + running_series="$(echo "$running" | cut -d. -f1,2)" + + if [ "$running_series" != "$series" ]; then + cat >&2 <&2 <&2 + return 1 + fi + + echo "scx_soryu devShell — kernel $running, BTF ok" + ''; + }; + + packages.${system}.default = pkgs.rustPlatform.buildRustPackage { + pname = "scx_soryu"; + version = "0.1.0"; + + # target/ is a gigabyte and nixos.qcow2 is the VM's disk; neither + # belongs in the store path, and both would invalidate the hash on + # every build. + src = pkgs.lib.cleanSourceWith { + src = ./.; + filter = path: _type: + !(builtins.elem (baseNameOf path) [ "target" "result" "nixos.qcow2" ]); + }; + + cargoLock.lockFile = ./Cargo.lock; + + nativeBuildInputs = with pkgs; [ + clang + llvm + bpftools + pkg-config + rustPlatform.bindgenHook # scx_utils runs bindgen; needs libclang + ]; + + buildInputs = with pkgs; [ libbpf elfutils zlib zstd ]; + + # Same two workarounds the devShell needs, for the same reasons. + hardeningDisable = [ "zerocallusedregs" ]; + RUSTFLAGS = builtins.concatStringsSep " " [ + "-C relocation-model=pic" + "-C link-args=-lelf" + "-C link-args=-lz" + "-C link-args=-lzstd" + ]; + + doCheck = false; # the tests that matter need a kernel, not a sandbox + + meta = { + description = "BMQ/LFBMQ policy port for sched_ext"; + mainProgram = "scx_soryu"; + }; + }; + + nixosModules.default = { config, lib, pkgs, ... }: + let + cfg = config.services.scx-soryu; + in + { + options.services.scx-soryu = { + enable = lib.mkEnableOption "the scx_soryu sched_ext scheduler"; + + package = lib.mkOption { + type = lib.types.package; + default = self.packages.${pkgs.stdenv.hostPlatform.system}.default; + description = "scx_soryu build to run."; + }; + + sliceUs = lib.mkOption { + type = lib.types.ints.positive; + default = 1000; + description = '' + Timeslice in microseconds. 1000 is the measured knee: see + docs/slice.md. 4194 restores BMQ's own value. + ''; + }; + + starveMs = lib.mkOption { + type = lib.types.ints.unsigned; + default = 128; + description = '' + Anti-starvation deadline. 0 restores BMQ's semantics. + ''; + }; + + timeoutMs = lib.mkOption { + type = lib.types.ints.positive; + default = 5000; + description = "Kernel watchdog threshold for a stalled task."; + }; + + reportSec = lib.mkOption { + type = lib.types.ints.positive; + default = 60; + description = '' + Seconds between telemetry lines in the journal. The loader's + own default is 1, which is right for a terminal and far too + chatty for a service. + ''; + }; + + extraArgs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ "--boost-on-wake" "false" ]; + }; + + notify = { + enable = lib.mkOption { + type = lib.types.bool; + default = cfg.notify.user != null; + defaultText = lib.literalExpression "notify.user != null"; + description = '' + Send a desktop notification when the scheduler gives up. + + Being ejected is silent: the kernel puts every task back on + default scheduler and nothing on the desktop says so. + Without this the only trace is the journal. + ''; + }; + + user = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "elars"; + description = "Desktop user to notify."; + }; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [{ + assertion = !cfg.notify.enable || cfg.notify.user != null; + message = "services.scx-soryu.notify.enable needs notify.user."; + }]; + + systemd.services.scx-soryu = { + description = "scx_soryu sched_ext scheduler"; + documentation = [ "https://github.com/elwrcl/scx_soryu" ]; + wantedBy = [ "multi-user.target" ]; + after = [ "systemd-udev-settle.service" ]; + startLimitIntervalSec = 300; + startLimitBurst = 3; + + onFailure = lib.mkIf cfg.notify.enable [ "scx-soryu-notify.service" ]; + + serviceConfig = { + ExecStart = lib.escapeShellArgs ([ + (lib.getExe cfg.package) + "--slice-us" (toString cfg.sliceUs) + "--starve-ms" (toString cfg.starveMs) + "--timeout-ms" (toString cfg.timeoutMs) + "--report-sec" (toString cfg.reportSec) + ] ++ cfg.extraArgs); + + Type = "simple"; + Restart = "on-failure"; + RestartSec = 5; + KillSignal = "SIGTERM"; + TimeoutStopSec = 15; + + # Losing the scheduler to the OOM killer would be a strange + # way to lose the scheduler. + OOMScoreAdjust = -500; + }; + }; + + systemd.services.scx-soryu-notify = lib.mkIf cfg.notify.enable { + description = "Report that scx_soryu gave up"; + serviceConfig = { + Type = "oneshot"; + ExecStart = pkgs.writeShellScript "scx-soryu-notify" '' + set -u + echo "soryu panic!; the system is running default scheduler" >&2 + echo "journalctl -u scx-soryu -n 200 has the exit reason" >&2 + + uid=$(${pkgs.coreutils}/bin/id -u ${cfg.notify.user}) || exit 0 + bus=/run/user/$uid/bus + [ -S "$bus" ] || exit 0 + + ${pkgs.util-linux}/bin/runuser -u ${cfg.notify.user} -- ${pkgs.coreutils}/bin/env "DBUS_SESSION_BUS_ADDRESS=unix:path=$bus" ${pkgs.libnotify}/bin/notify-send --urgency=critical --icon=dialog-error "scx_soryu durdu" "Sistem EEVDF'e döndü. journalctl -u scx-soryu" || true + ''; + }; + }; + }; + }; + + nixosConfigurations.testvm = nixpkgs.lib.nixosSystem { + inherit system; + modules = [ + "${nixpkgs}/nixos/modules/virtualisation/qemu-vm.nix" + vmBase + ]; + }; + + nixosConfigurations.servicevm = nixpkgs.lib.nixosSystem { + inherit system; + modules = [ + "${nixpkgs}/nixos/modules/virtualisation/qemu-vm.nix" + vmBase + self.nixosModules.default + { + services.scx-soryu = { + enable = true; + reportSec = 5; + # No desktop session in here, so nothing to notify. + notify.enable = false; + }; + } + ]; + }; + }; +} diff --git a/src/bpf/main.bpf.c b/src/bpf/main.bpf.c new file mode 100644 index 0000000..171bd3c --- /dev/null +++ b/src/bpf/main.bpf.c @@ -0,0 +1,751 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * scx_soryu — BMQ policy on sched_ext. + */ + +#include + +char _license[] SEC("license") = "GPL"; + +UEI_DEFINE(uei); + +/* + * BMQ compresses (nice + boost) by two, so adjacent nice levels share a queue. + * That halves nice resolution and in exchange gives boost twice the leverage + * per step. Keeping the compression is the point of porting BMQ rather than + * inventing a lookalike. + * + * BMQ: 25 + (prio + boost - 128) / 2, where prio = 160 + nice + * 0-based: (32 + nice + boost) / 2 + * mainline: (static_prio - 88 + boost) / 2, nice = static_prio - 120 + */ +#define BOOST_RANGE 12 /* BMQ MAX_PRIORITY_ADJ */ +#define NORMAL_LEVELS 32 /* (12 + 40 + 12) / 2 */ +#define LEVELS (NORMAL_LEVELS + 1) /* last level is SCHED_IDLE */ +#define IDLE_LEVEL (LEVELS - 1) + +#define MAX_CPUS 1024 + +/* + * BMQ's RESCHED_NS (alt_core.c:94). A remainder below this is treated as spent: + * handing back a sliver just makes the task come straight back, so BMQ refills + * instead. Same threshold, same reason. + */ +#define RESCHED_NS (100 << 10) /* ~102 us */ + +/* uapi/linux/sched.h; vmlinux.h carries the enum but not these defines. */ +#define SCHED_NORMAL 0 +#define SCHED_BATCH 3 +#define SCHED_IDLE 5 + +#define DSQ_BASE 0x736f7279UL /* "sory" */ +#define dsq_id(lvl) (DSQ_BASE + (lvl)) + +const volatile u64 slice_ns = 4194304ULL; /* 4 << 20, BMQ sysctl_sched_base_slice */ +const volatile u32 nr_cpu_ids = 1; /* filled in by the loader */ + +/* + * BMQ's sched_task_ttwu() boosts on wakeup in proportion to how long the task + * slept, which is what makes sleeping for 1us worth nothing and closes the + * sleep-gaming hole. Except the line reads + * + * s64 delta = this_rq()->clock_task > p->last_ran; + * + * -- a comparison, not a subtraction. delta is always 0 or 1, the shift makes + * it 0, and the whole function is dead code in every BMQ build shipped. + * + * Fixing it means running behaviour nobody has field-tested; leaving it means + * reproducing a typo. So it is a knob, measured both ways in stage 2. + */ +const volatile bool boost_on_wake = true; + +/* + * Anti-starvation, and a deliberate deviation from BMQ. + * + * BMQ has none. bmq.h is forty lines: boost comes from sleeping, deboost from + * burning a slice, and nothing whatsoever happens to a task that is merely + * waiting. Strict priority, no aging, no deadline. On mainline that is + * survivable -- a nice-19 thread stuck behind a compile is just slow, and no + * one dies. Under sched_ext the watchdog calls it a fatal stall and ejects the + * whole scheduler, which is what happened on 1 Sep: helium:disk$2 (nice 19, + * level 25) waited 7.115s behind seven compilers at level 22 and took us down + * with it. See docs/stage6-stall.md. + * + * So: a level that has held tasks this long without being served gets a turn, + * once, ahead of better levels. One task per level per starve_ns, which works + * out to roughly 3% of a CPU for a starving band -- close to what nice 19 is + * worth under CFS anyway (weight 15/1024). Strict priority is preserved + * everywhere else. + * + * 0 disables it and restores BMQ's semantics exactly. + */ +const volatile u64 starve_ns = 128ULL << 20; /* ~134 ms */ +u64 lvl_bitmap; + +/* Level each CPU is currently running, LEVELS when idle. Used by stage 4. */ +static u8 cpu_level[MAX_CPUS]; + +u64 max_wait_jiffies; +u64 lvl_max_wait[LEVELS]; +u64 lvl_last_run[LEVELS]; +u32 starve_cursor; +u32 probe_seq; + +/* + * How many times each boost value was seen at ops.running. The single most + * useful stage 2 number: it says whether boost is actually moving, and in which + * direction the workload pushes it. Index is boost + BOOST_RANGE. + */ +u64 boost_hist[2 * BOOST_RANGE + 1]; + +#define LAT_CLASSES 3 /* 0 boosted, 1 neutral, 2 deboosted */ +#define LAT_BUCKETS 32 /* log2(ns): bucket 10 is ~1us, 30 is ~1s */ + +u64 lat_hist[LAT_CLASSES * LAT_BUCKETS]; + +struct task_ctx +{ + s32 boost; /* [-BOOST_RANGE, +BOOST_RANGE], negative = better */ + u64 sleep_at; /* scx_bpf_now() when the task went to sleep */ + u64 run_at; /* scx_bpf_now() when it last started running */ + u64 wake_at; /* scx_bpf_now() at ops.runnable, for lat_hist */ + u64 slice_given; /* what p->scx.slice held at ops.running */ + u64 slice_left; +}; + +struct +{ + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct task_ctx); +} task_ctxs SEC(".maps"); + +enum soryu_stat_idx +{ + SORYU_STAT_ENQ_SHARED, + SORYU_STAT_ENQ_LOCAL, + SORYU_STAT_ENQ_REENQ, + SORYU_STAT_ENQ_PINNED, + SORYU_STAT_DISPATCH, + SORYU_STAT_INSERT_FAIL, + SORYU_STAT_CPU_RELEASE, + SORYU_STAT_BOOST_SLEEP, + SORYU_STAT_BOOST_WAKE, + SORYU_STAT_DEBOOST, + SORYU_STAT_SLICE_CARRY, + SORYU_STAT_FORK_SPLIT, + SORYU_STAT_PREEMPT_KICK, + SORYU_STAT_STARVE_SERVE, + SORYU_STAT_PREEMPT_NONE, + SORYU_STAT_PREEMPT_IDLE, + SORYU_NR_STATS, +}; + +struct +{ + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u64)); + __uint(max_entries, SORYU_NR_STATS); +} stats SEC(".maps"); + +static void stat_inc(enum soryu_stat_idx idx) +{ + u32 key = idx; + u64 *cnt = bpf_map_lookup_elem(&stats, &key); + + if (cnt) + (*cnt)++; +} + +static struct task_ctx *lookup_task_ctx(struct task_struct *p) +{ + return bpf_task_storage_get(&task_ctxs, p, 0, 0); +} + +/* + * BMQ boost_task(): the floor depends on policy. SCHED_NORMAL may go all the + * way to -12; SCHED_BATCH may only climb back to 0, so a batch task can recover + * from a deboost but never outrank a neutral one; anything else (SCHED_IDLE) + * never boosts at all. + */ +static void boost_task(struct task_struct *p, struct task_ctx *tctx, u32 n) +{ + s32 limit, want; + + switch (p->policy) + { + case SCHED_NORMAL: + limit = -BOOST_RANGE; + break; + case SCHED_BATCH: + limit = 0; + break; + default: + return; + } + + want = (s32)tctx->boost - (s32)n; + tctx->boost = want < limit ? limit : want; +} + +/* BMQ deboost_task(): NORMAL and BATCH both decay, SCHED_IDLE does not. */ +static void deboost_task(struct task_struct *p, struct task_ctx *tctx) +{ + if ((p->policy == SCHED_NORMAL || p->policy == SCHED_BATCH) && + tctx->boost < BOOST_RANGE) + tctx->boost++; +} + +static u32 task_level(struct task_struct *p, struct task_ctx *tctx) +{ + s32 v; + + if (p->policy == SCHED_IDLE) + return IDLE_LEVEL; + + /* + * static_prio in [100, 139], boost in [-12, 12], so v lands in [0, 63] + * and never goes negative — no clamp needed, and >> 1 == / 2. + * + * static_prio rather than p->scx.weight on purpose: weight is a + * logarithmic thing derived from nice, and the level math wants nice + * itself. ops.set_weight is deliberately not implemented. + */ + v = (s32)p->static_prio - 88 + (tctx ? tctx->boost : 0); + + return (u32)v >> 1; +} + +static u64 task_slice(struct task_ctx *tctx) +{ + if (tctx && tctx->slice_left >= RESCHED_NS) + return tctx->slice_left; + + return slice_ns; +} + +/* + * Kick the CPU running the worst thing this task could displace. + * + * BMQ does this on every wakeup: if the woken task outranks what some CPU is + * running, send a resched IPI. With a shared queue there is no single "target + * CPU", so we track what level each CPU is running (cpu_level[]) and search. + * + * Two deliberate differences from LFBMQ (alt_core.c:4225): + * + * - It kicks *every* eligible CPU; we kick only the worst one. Broadcasting + * costs an IPI per CPU per wakeup, and stage 1 measured the shared queue + * taking 82% of wakeups under game load -- that is a lot of wakeups to + * multiply. One kick is enough: the preempted CPU re-enters dispatch and + * pulls the lowest level from the bitmap anyway. + * + * - It respects p->cpus_ptr and so do we. The spec originally dropped this; + * kicking a CPU the task cannot run on wastes the IPI and hands the slot to + * somebody else. + */ +static void try_preempt(struct task_struct *p, u32 lvl, u64 enq_flags) +{ + const struct cpumask *allowed = p->cpus_ptr; + s32 worst_cpu = -1; + u32 worst = lvl + 1; /* never IPI an equal or better level */ + u32 nr = scx_bpf_nr_cpu_ids(); + u32 cpu; + + if (!allowed) + return; + + /* + * Wakeups only. BMQ checks for preemption in wakeup_srq_task() + * (alt_core.c:4211) -- on the wakeup path, not on every enqueue. + * + * Running it on every enqueue closes a loop: the task we preempt is + * re-enqueued, that re-enqueue runs the check again, which preempts + * somebody else, and so on. Measured at 3.2M kicks in 32s (~100k + * IPIs/sec) on four spinners plus one 100Hz waker -- three orders of + * magnitude more than the wakeups that caused it. + */ + if (!(enq_flags & SCX_ENQ_WAKEUP)) + return; + + bpf_for(cpu, 0, nr) + { + u32 cl; + + if (cpu >= MAX_CPUS) + break; + if (!bpf_cpumask_test_cpu(cpu, allowed)) + continue; + + cl = cpu_level[cpu]; + if (cl > worst) + { + worst = cl; + worst_cpu = cpu; + } + } + + if (worst_cpu >= 0) + { + scx_bpf_kick_cpu(worst_cpu, SCX_KICK_PREEMPT); + + /* + * Two very different events were hiding behind one counter. + * + * cpu_level is set to LEVELS when a CPU stops running an scx + * task, so an idle CPU always looks "worse" than any level and + * always wins the search. Most of what preempt_kick counted was + * therefore *waking an idle CPU*, not preempting anybody -- + * measured in the VM at 11:1 against real preemption with eight + * sleepers. Necessary work, but it says nothing about + * preemption pressure, and the two must not share a number. + */ + if (worst >= LEVELS) + stat_inc(SORYU_STAT_PREEMPT_IDLE); + else + stat_inc(SORYU_STAT_PREEMPT_KICK); + return; + } + + /* + * Nobody worse to kick, so this wakeup waits for a slice to end on its + * own -- up to slice_ns. + * + * This is the measurement behind the slice question. When the boost + * range saturates, every task sits at the same level, `worst` never + * exceeds `lvl`, and preemption switches itself off: the scheduler + * silently degrades to FIFO-with-a-4ms-quantum exactly in the workload + * (gaming, 97% at the floor) where responsiveness is the whole point. + * If this counter dwarfs preempt_kick there, the tail is a slice-length + * problem and no amount of boost tuning will touch it. + */ + stat_inc(SORYU_STAT_PREEMPT_NONE); +} + +s32 BPF_STRUCT_OPS(soryu_select_cpu, struct task_struct *p, s32 prev_cpu, + u64 wake_flags) +{ + struct task_ctx *tctx; + bool is_idle = false; + s32 cpu; + + cpu = scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, &is_idle); + if (!is_idle) + return cpu; + + /* + * Hot path: an idle CPU was found, so skip the shared queue entirely. + * ops.enqueue is not called for tasks inserted from here. + */ + tctx = lookup_task_ctx(p); + if (scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, task_slice(tctx), 0)) + stat_inc(SORYU_STAT_ENQ_LOCAL); + else + stat_inc(SORYU_STAT_INSERT_FAIL); + + return cpu; +} + +void BPF_STRUCT_OPS(soryu_enqueue, struct task_struct *p, u64 enq_flags) +{ + struct task_ctx *tctx = lookup_task_ctx(p); + u32 lvl = task_level(p, tctx); + + /* + * Tasks bounced back by ops.cpu_release did not stop through any fault + * of their own; stage 2 must not read this as a scheduling signal. + */ + if (enq_flags & SCX_ENQ_REENQ) + stat_inc(SORYU_STAT_ENQ_REENQ); + + /* + * Single-CPU tasks gain nothing from a shared queue, and they make every + * other CPU pay: consume_dispatch_q() walks the whole DSQ list under + * dsq->lock skipping tasks it cannot migrate, so a pile of pinned tasks + * turns each remote dispatch into a futile scan. Measured at 64 pinned + * tasks: 6x slower than EEVDF before this, and it is exactly the + * live-lock the kernel comments about. Per-CPU kthreads are in this + * class too, so it is not an exotic case. + * + * Restricted-but-multi-CPU masks still go to the shared queue; they are + * rarer and the fix for them is per-CPU DSQs, which is a bigger change. + */ + if (p->nr_cpus_allowed == 1) + { + s32 cpu = scx_bpf_task_cpu(p); + + if (scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cpu, + task_slice(tctx), enq_flags)) + { + stat_inc(SORYU_STAT_ENQ_PINNED); + + /* Only one CPU can ever run it, so the search above is + * pointless -- just check that one. Wakeups only, same + * reason as try_preempt(). */ + if ((enq_flags & SCX_ENQ_WAKEUP) && + cpu >= 0 && cpu < MAX_CPUS && + cpu_level[cpu] > lvl) + { + scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); + stat_inc(SORYU_STAT_PREEMPT_KICK); + } + } + else + { + stat_inc(SORYU_STAT_INSERT_FAIL); + } + return; + } + + if (!scx_bpf_dsq_insert(p, dsq_id(lvl), task_slice(tctx), enq_flags)) + { + stat_inc(SORYU_STAT_INSERT_FAIL); + return; + } + stat_inc(SORYU_STAT_ENQ_SHARED); + + /* Insert first, then advertise: a bit set late is a missed dispatch + * round, a bit set early is a dispatch that finds nothing. */ + __sync_fetch_and_or(&lvl_bitmap, 1ULL << lvl); + + try_preempt(p, lvl, enq_flags); +} + +static void sample_head_wait(u32 lvl) +{ + struct task_struct *p; + u64 now = bpf_jiffies64(); + + if (lvl >= LEVELS) + return; + + bpf_for_each(scx_dsq, p, dsq_id(lvl), 0) + { + u64 runnable_at = p->scx.runnable_at; + + if (runnable_at) + { + u64 waited = time_delta(now, runnable_at); + + if (waited > READ_ONCE(max_wait_jiffies)) + WRITE_ONCE(max_wait_jiffies, waited); + if (waited > READ_ONCE(lvl_max_wait[lvl])) + WRITE_ONCE(lvl_max_wait[lvl], waited); + } + break; + } +} + +void BPF_STRUCT_OPS(soryu_dispatch, s32 cpu, struct task_struct *prev) +{ + u64 mask = READ_ONCE(lvl_bitmap); + u64 now = scx_bpf_now(); + if (starve_ns) + { + u32 probe = READ_ONCE(starve_cursor); + + probe = probe < LEVELS - 1 ? probe + 1 : 0; + WRITE_ONCE(starve_cursor, probe); + + /* + * Telemetry, once every 1024 dispatches -- tens of times a + * second, which is three orders of magnitude finer than a + * multi-second watchdog needs, and cheap enough to ignore. + * (1024 mod 33 = 1, so consecutive samples walk every level.) + */ + if (!(probe_seq++ & 1023) && (mask & (1ULL << probe))) + sample_head_wait(probe); + + if ((mask & (1ULL << probe)) && + now - READ_ONCE(lvl_last_run[probe]) > starve_ns) + { + if (scx_bpf_dsq_move_to_local(dsq_id(probe), 0)) + { + WRITE_ONCE(lvl_last_run[probe], now); + stat_inc(SORYU_STAT_DISPATCH); + stat_inc(SORYU_STAT_STARVE_SERVE); + return; + } + + /* + * Empty, or nothing here can run on this CPU. Either + * way it is not starving -- it was offered a turn and + * had nothing to take. Stamping it keeps a level that + * has been idle for hours (the bitmap is set-only, so + * its bit stays lit forever) from claiming a free pass + * on every probe. + */ + WRITE_ONCE(lvl_last_run[probe], now); + } + } + + bpf_repeat(LEVELS) + { + u32 lvl; + u64 bit; + + if (!mask) + break; + + lvl = __builtin_ctzll(mask); + bit = 1ULL << lvl; + + if (scx_bpf_dsq_move_to_local(dsq_id(lvl), 0)) + { + WRITE_ONCE(lvl_last_run[lvl], now); + stat_inc(SORYU_STAT_DISPATCH); + return; + } + + mask &= ~bit; + } +} + +void BPF_STRUCT_OPS(soryu_runnable, struct task_struct *p, u64 enq_flags) +{ + struct task_ctx *tctx; + u64 now, slept, steps; + + if (enq_flags & SCX_ENQ_REENQ) + return; + + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + now = scx_bpf_now(); + + tctx->wake_at = now; + + if (!boost_on_wake || !tctx->sleep_at || !slice_ns) + return; + slept = now > tctx->sleep_at ? now - tctx->sleep_at : 0; + tctx->sleep_at = 0; + + steps = slept / slice_ns; + if (!steps) + return; + + if (steps > 2 * BOOST_RANGE) + steps = 2 * BOOST_RANGE; + + boost_task(p, tctx, (u32)steps); + stat_inc(SORYU_STAT_BOOST_WAKE); +} + +void BPF_STRUCT_OPS(soryu_running, struct task_struct *p) +{ + struct task_ctx *tctx = lookup_task_ctx(p); + s32 cpu = scx_bpf_task_cpu(p); + u64 runnable_at, waited; + + if (cpu >= 0 && cpu < MAX_CPUS) + cpu_level[cpu] = task_level(p, tctx); + + if (tctx) + { + u32 idx = (u32)(tctx->boost + BOOST_RANGE); + + if (idx < 2 * BOOST_RANGE + 1) + boost_hist[idx]++; + + tctx->run_at = scx_bpf_now(); + tctx->slice_given = p->scx.slice; + + if (tctx->wake_at) + { + u64 lat = tctx->run_at > tctx->wake_at + ? tctx->run_at - tctx->wake_at + : 0; + u32 cls = tctx->boost < 0 ? 0 : (tctx->boost ? 2 : 1); + u32 b = lat ? 63 - __builtin_clzll(lat) : 0; + u32 slot; + + if (b >= LAT_BUCKETS) + b = LAT_BUCKETS - 1; + + slot = cls * LAT_BUCKETS + b; + if (slot < LAT_CLASSES * LAT_BUCKETS) + lat_hist[slot]++; + + tctx->wake_at = 0; + } + } + + runnable_at = p->scx.runnable_at; + if (runnable_at) + { + u32 lvl = task_level(p, tctx); + + waited = time_delta(bpf_jiffies64(), runnable_at); + if (waited > READ_ONCE(max_wait_jiffies)) + WRITE_ONCE(max_wait_jiffies, waited); + + if (lvl < LEVELS && waited > READ_ONCE(lvl_max_wait[lvl])) + WRITE_ONCE(lvl_max_wait[lvl], waited); + } +} + +void BPF_STRUCT_OPS(soryu_stopping, struct task_struct *p, bool runnable) +{ + struct task_ctx *tctx = lookup_task_ctx(p); + s32 cpu = scx_bpf_task_cpu(p); + u64 now, ran, left; + + if (cpu >= 0 && cpu < MAX_CPUS) + cpu_level[cpu] = LEVELS; + + if (!tctx) + return; + + if (!runnable) + { + boost_task(p, tctx, 1); + tctx->sleep_at = scx_bpf_now(); + tctx->slice_left = 0; /* sleeping earns a fresh slice */ + stat_inc(SORYU_STAT_BOOST_SLEEP); + return; + } + + now = scx_bpf_now(); + ran = now > tctx->run_at ? now - tctx->run_at : 0; + left = tctx->slice_given > ran ? tctx->slice_given - ran : 0; + + if (left < RESCHED_NS) + { + deboost_task(p, tctx); + tctx->slice_left = 0; + stat_inc(SORYU_STAT_DEBOOST); + } + else + { + tctx->slice_left = left; + stat_inc(SORYU_STAT_SLICE_CARRY); + } +} + +void BPF_STRUCT_OPS(soryu_cpu_release, s32 cpu, struct scx_cpu_release_args *args) +{ + scx_bpf_reenqueue_local(); + + if (cpu >= 0 && cpu < MAX_CPUS) + cpu_level[cpu] = LEVELS; + + stat_inc(SORYU_STAT_CPU_RELEASE); +} + +s32 BPF_STRUCT_OPS(soryu_init_task, struct task_struct *p, + struct scx_init_task_args *args) +{ + struct task_ctx *tctx; + + tctx = bpf_task_storage_get(&task_ctxs, p, 0, + BPF_LOCAL_STORAGE_GET_F_CREATE); + if (!tctx) + return -ENOMEM; + + tctx->boost = 0; + tctx->sleep_at = 0; + tctx->run_at = 0; + tctx->wake_at = 0; + tctx->slice_given = 0; + tctx->slice_left = 0; + + /* + * BMQ (alt_core.c:2874, sched_cgroup_fork): + * + * rq->curr->time_slice /= 2; + * p->time_slice = rq->curr->time_slice; + * + * with the comment "the total amount of pending timeslices in the + * system doesn't change". Without it every fork mints a fresh full + * slice out of nothing, so anything that forks in a loop -- a build, + * most obviously -- gains CPU share against everything else simply by + * forking. Splitting closes that. + * + */ + if (args->fork) + { + struct task_struct *parent = bpf_get_current_task_btf(); + struct task_ctx *pctx; + u64 half; + + if (parent) + { + half = parent->scx.slice / 2; + + scx_bpf_task_set_slice(parent, half); + + pctx = bpf_task_storage_get(&task_ctxs, parent, 0, 0); + if (pctx) + pctx->slice_left = half; + + tctx->slice_left = half >= RESCHED_NS ? half : 0; + stat_inc(SORYU_STAT_FORK_SPLIT); + } + } + + return 0; +} + +void BPF_STRUCT_OPS(soryu_dump, struct scx_dump_ctx *dctx) +{ + u64 mask = READ_ONCE(lvl_bitmap); + u32 lvl; + + scx_bpf_dump_header(); + scx_bpf_dump("soryu: bitmap=0x%llx max_wait=%llu jiffies\n", + mask, READ_ONCE(max_wait_jiffies)); + + bpf_for(lvl, 0, LEVELS) + { + s32 nr = scx_bpf_dsq_nr_queued(dsq_id(lvl)); + + /* [BIT CLEAR!] is an invariant violation: a level holding tasks + * whose bit is unset can never be dispatched. Must never fire + * now that bits are set-only. */ + if (nr > 0) + scx_bpf_dump(" lvl %2u: %d queued%s\n", lvl, nr, + (mask & (1ULL << lvl)) ? "" : " [BIT CLEAR!]"); + } +} + +s32 BPF_STRUCT_OPS_SLEEPABLE(soryu_init) +{ + u32 lvl; + s32 ret; + + bpf_for(lvl, 0, LEVELS) + { + ret = scx_bpf_create_dsq(dsq_id(lvl), -1); + if (ret) + return ret; + } + + bpf_for(lvl, 0, MAX_CPUS) + cpu_level[lvl] = LEVELS; + + return 0; +} + +void BPF_STRUCT_OPS(soryu_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + +SCX_OPS_DEFINE(soryu_ops, + .select_cpu = (void *)soryu_select_cpu, + .enqueue = (void *)soryu_enqueue, + .dispatch = (void *)soryu_dispatch, + .runnable = (void *)soryu_runnable, + .running = (void *)soryu_running, + .stopping = (void *)soryu_stopping, + .cpu_release = (void *)soryu_cpu_release, + .init_task = (void *)soryu_init_task, + .dump = (void *)soryu_dump, + .init = (void *)soryu_init, + .exit = (void *)soryu_exit, + .flags = SCX_OPS_KEEP_BUILTIN_IDLE, + .timeout_ms = 5000, + .name = "soryu"); diff --git a/src/bpf_skel.rs b/src/bpf_skel.rs new file mode 100644 index 0000000..ad8f9ac --- /dev/null +++ b/src/bpf_skel.rs @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// libbpf-cargo generates the skeleton into OUT_DIR; this module just pulls it +// in. Kept separate so the generated code's lint profile stays isolated. +#![allow(dead_code)] +#![allow(clippy::all)] +include!(concat!(env!("OUT_DIR"), "/bpf_skel.rs")); diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..307dff8 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,502 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// scx_soryu loader. +// +// runs in the foreground and holds no state of its own: dropping +// the struct_ops link is what unloads the scheduler, so Ctrl-C is the emergency +// exit during development. + +mod bpf_skel; +mod stats; +use bpf_skel::*; +use stats::Metrics; + +use std::mem::MaybeUninit; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; +use clap::Parser; +use crossbeam::channel::RecvTimeoutError; +use scx_stats::prelude::*; +use libbpf_rs::OpenObject; +use libbpf_rs::MapCore; +use log::error; +use log::info; +use log::warn; +use scx_utils::scx_ops_attach; +use scx_utils::scx_ops_load; +use scx_utils::scx_ops_open; +use scx_utils::uei_exited; +use scx_utils::uei_report; +use scx_utils::UserExitInfo; + +const SCHEDULER_NAME: &str = "scx_soryu"; + +/// mirrors `enum stat_idx` in main.bpf.c. +const STAT_NAMES: &[&str] = &[ + "enq_shared", + "enq_local", + "enq_reenq", + "enq_pinned", + "dispatch", + "insert_fail", + "cpu_release", + "boost_sleep", + "boost_wake", + "deboost", + "slice_carry", + "fork_split", + "preempt_kick", + "starve_serve", + "preempt_none", + "preempt_idle", +]; + +/// mirrors BOOST_RANGE in main.bpf.c. +const BOOST_RANGE: i32 = 12; + +/// mirrors LAT_CLASSES / LAT_BUCKETS in main.bpf.c. +const LAT_CLASSES: usize = 3; +const LAT_BUCKETS: usize = 32; +const LAT_CLASS_NAMES: [&str; LAT_CLASSES] = ["boosted", "neutral", "deboosted"]; + +#[derive(Parser, Debug, Clone)] +#[command(about = "BMQ policy port for sched_ext", version)] +struct Opts { + #[arg(long, default_value_t = 5000, env = "SORYU_TIMEOUT_MS")] + timeout_ms: u32, + #[arg(long, default_value_t = true, env = "SORYU_BOOST_ON_WAKE", + action = clap::ArgAction::Set)] + boost_on_wake: bool, + #[arg(long, default_value_t = 1000, env = "SORYU_SLICE_US")] + slice_us: u64, + #[arg(long, default_value_t = 128, env = "SORYU_STARVE_MS")] + starve_ms: u64, + #[arg(long, default_value_t = 1, env = "SORYU_REPORT_SEC")] + report_sec: u64, + #[arg(long, env = "SORYU_LOG")] + log: Option, + + /// Print libbpf's output, including the verifier log on a load failure. + #[arg(long, default_value_t = false, env = "SORYU_DEBUG")] + debug: bool, + + /// Bytes of scheduler state captured when the kernel ejects us. + #[arg(long, default_value_t = 65536)] + exit_dump_len: u32, + + /// Attach to a running scx_soryu and print its telemetry every N seconds. + /// Starts no scheduler of its own. + #[arg(long)] + monitor: Option, +} +fn percentiles(buckets: &[u64]) -> Option<(u64, u64, u64)> { + let total: u64 = buckets.iter().sum(); + if total == 0 { + return None; + } + + let at = |frac: f64| -> u64 { + let target = (total as f64 * frac) as u64; + let mut seen = 0u64; + for (i, &c) in buckets.iter().enumerate() { + seen += c; + if seen >= target { + return 1u64 << i; + } + } + 1u64 << (buckets.len() - 1) + }; + + Some((at(0.50), at(0.99), at(0.999))) +} + +fn fmt_ns(ns: u64) -> String { + if ns >= 1_000_000_000 { + format!("{:.1}s", ns as f64 / 1e9) + } else if ns >= 1_000_000 { + format!("{}ms", ns / 1_000_000) + } else if ns >= 1_000 { + format!("{}us", ns / 1_000) + } else { + format!("{ns}ns") + } +} + +struct Scheduler<'a> { + opts: Opts, + skel: BpfSkel<'a>, + struct_ops: Option, + stats_server: StatsServer<(), Metrics>, + prev_waits: Vec, +} + +impl<'a> Scheduler<'a> { + fn init(opts: Opts, open_object: &'a mut MaybeUninit) -> Result { + let mut skel_builder = BpfSkelBuilder::default(); + // SORYU_DEBUG=1 turns on libbpf's output, which is the only way to see + // the verifier's rejection log when a program fails to load. + skel_builder.obj_builder.debug(opts.debug); + + let open_opts = scx_utils::libbpf_clap_opts::LibbpfOpts::default().into_bpf_open_opts(); + let mut skel = scx_ops_open!(skel_builder, open_object, soryu_ops, open_opts)?; + + // Dump the scheduler's own state when we get ejected; ops.dump fills it. + skel.struct_ops.soryu_ops_mut().exit_dump_len = opts.exit_dump_len; + skel.struct_ops.soryu_ops_mut().timeout_ms = opts.timeout_ms; + + let rodata = skel + .maps + .rodata_data + .as_mut() + .context("no rodata section in skeleton")?; + rodata.nr_cpu_ids = *scx_utils::NR_CPU_IDS as u32; + + // SORYU_BOOST_ON_WAKE=0 reproduces BMQ as shipped (its wake boost is + // dead code); the default reproduces what BMQ meant to do. + rodata.boost_on_wake = opts.boost_on_wake; + rodata.starve_ns = opts.starve_ms * 1_000_000; + + rodata.slice_ns = opts.slice_us * 1_000; + + let mut skel = scx_ops_load!(skel, soryu_ops, uei)?; + let struct_ops = Some(scx_ops_attach!(skel, soryu_ops)?); + + let rd = skel.maps.rodata_data.as_ref().unwrap(); + info!( + "{} attached: {} CPUs, slice {} us, boost_on_wake={}, \ + starve {} ms, timeout {} ms", + SCHEDULER_NAME, + *scx_utils::NR_CPU_IDS, + rd.slice_ns / 1000, + rd.boost_on_wake, + rd.starve_ns / 1_000_000, + opts.timeout_ms + ); + + let stats_server = StatsServer::new(stats::server_data()).launch()?; + + Ok(Self { + opts, + skel, + struct_ops, + stats_server, + prev_waits: Vec::new(), + }) + } + + fn exited(&mut self) -> bool { + uei_exited!(&self.skel, uei) + } + + /// Per-CPU counters summed across CPUs. + fn read_stats(&self) -> Result> { + let nr_cpus = *scx_utils::NR_CPU_IDS; + let mut out = vec![0u64; STAT_NAMES.len()]; + + for (idx, slot) in out.iter_mut().enumerate() { + let key = (idx as u32).to_ne_bytes(); + let per_cpu = self + .skel + .maps + .stats + .lookup_percpu(&key, libbpf_rs::MapFlags::ANY)? + .unwrap_or_default(); + + *slot = per_cpu + .iter() + .take(nr_cpus) + .map(|v| u64::from_ne_bytes(v[..8].try_into().unwrap_or([0; 8]))) + .sum(); + } + + Ok(out) + } + + /// One snapshot for the stats server. + fn metrics(&self) -> Metrics { + let c = self.read_stats().unwrap_or_default(); + let g = |i: usize| c.get(i).copied().unwrap_or(0); + + let mut m = Metrics { + enq_shared: g(0), + enq_local: g(1), + enq_reenq: g(2), + enq_pinned: g(3), + insert_fail: g(5), + cpu_release: g(6), + boost_sleep: g(7), + boost_wake: g(8), + deboost: g(9), + slice_carry: g(10), + fork_split: g(11), + preempt_kick: g(12), + starve_serve: g(13), + preempt_none: g(14), + preempt_idle: g(15), + ..Default::default() + }; + + if let Some(bss) = self.skel.maps.bss_data.as_ref() { + m.max_wait_j = bss.max_wait_jiffies; + m.lvl_bitmap = bss.lvl_bitmap; + + let hist = &bss.boost_hist; + let total: u64 = hist.iter().sum(); + if total > 0 { + let weighted: i64 = hist + .iter() + .enumerate() + .map(|(i, &n)| (i as i64 - BOOST_RANGE as i64) * n as i64) + .sum(); + m.boost_mean_x100 = weighted * 100 / total as i64; + m.boost_floor_pct = hist[0] * 100 / total; + m.boost_ceil_pct = hist[hist.len() - 1] * 100 / total; + } + + // Class 0 is "boosted" -- the interactive tail, and the only class + // with enough samples to mean anything (hogs barely ever wake). + let boosted = &bss.lat_hist[0..LAT_BUCKETS]; + m.lat_boosted_n = boosted.iter().sum(); + if let Some((p50, p99, p999)) = percentiles(boosted) { + m.lat_boosted_p50 = p50; + m.lat_boosted_p99 = p99; + m.lat_boosted_p999 = p999; + } + } + + m + } + + fn report(&mut self) { + let stats = match self.read_stats() { + Ok(s) => s, + Err(e) => { + warn!("stat read failed: {e}"); + return; + } + }; + + let bss = self.skel.maps.bss_data.as_ref(); + let (bitmap, max_wait) = bss + .map(|b| (b.lvl_bitmap, b.max_wait_jiffies)) + .unwrap_or((0, 0)); + + let raw: Vec = self + .skel + .maps + .bss_data + .as_ref() + .map(|b| b.lvl_max_wait.to_vec()) + .unwrap_or_default(); + + let waits: Vec = raw + .iter() + .enumerate() + .filter(|(_, &w)| w > 0) + .map(|(lvl, &w)| { + let rising = self.prev_waits.get(lvl).map(|&p| w > p).unwrap_or(false); + let mark = if rising { "*" } else { "" }; + match lvl { + 32 => format!("idle:{w}j{mark}"), + 10 => format!("l10*n0boosted:{w}j{mark}"), + 16 => format!("l16*n0neutral:{w}j{mark}"), + 22 => format!("l22*n0hog:{w}j{mark}"), + _ => format!("l{lvl}:{w}j{mark}"), + } + }) + .collect(); + + if !raw.is_empty() { + self.prev_waits = raw; + } + + let line = STAT_NAMES + .iter() + .zip(stats.iter()) + .map(|(n, v)| format!("{n}={v}")) + .collect::>() + .join(" "); + + info!("{line} bitmap=0x{bitmap:x} max_wait={max_wait}j"); + if !waits.is_empty() { + info!(" per-level worst wait: {}", waits.join(" ")); + } + + if let Some(bss) = self.skel.maps.bss_data.as_ref() { + let hist = &bss.boost_hist; + let total: u64 = hist.iter().sum(); + + if total > 0 { + let occupied: Vec = hist + .iter() + .enumerate() + .filter(|(_, &c)| c > 0) + .map(|(i, &c)| { + let boost = i as i32 - BOOST_RANGE; + format!("{boost:+}:{:.0}%", 100.0 * c as f64 / total as f64) + }) + .collect(); + + let weighted: i64 = hist + .iter() + .enumerate() + .map(|(i, &c)| (i as i64 - BOOST_RANGE as i64) * c as i64) + .sum(); + let mean = weighted as f64 / total as f64; + + info!(" boost mean={mean:+.2} dist: {}", occupied.join(" ")); + } + + let lines: Vec = (0..LAT_CLASSES) + .filter_map(|c| { + let slice = &bss.lat_hist[c * LAT_BUCKETS..(c + 1) * LAT_BUCKETS]; + percentiles(slice).map(|(p50, p99, p999)| { + format!( + "{}: p50={} p99={} p99.9={} n={}", + LAT_CLASS_NAMES[c], + fmt_ns(p50), + fmt_ns(p99), + fmt_ns(p999), + slice.iter().sum::() + ) + }) + }) + .collect(); + + for l in lines { + info!(" wake->run {l}"); + } + + let boosted = &bss.lat_hist[0..LAT_BUCKETS]; + let n: u64 = boosted.iter().sum(); + if n > 0 { + let above = |bucket: usize| -> f64 { + 100.0 * boosted[bucket..].iter().sum::() as f64 / n as f64 + }; + + info!( + " wake->run boosted tail: >{} {:.3}% >{} {:.3}% >{} {:.4}% >{} {:.4}%", + fmt_ns(1 << 20), + above(20), + fmt_ns(1 << 22), + above(22), + fmt_ns(1 << 24), + above(24), + fmt_ns(1 << 26), + above(26), + ); + } + } + } + + fn run(&mut self, shutdown: Arc) -> Result { + let (res_ch, req_ch) = self.stats_server.channels(); + let every = self.opts.report_sec.max(1); + let mut ticks = 0u64; + + while !shutdown.load(Ordering::Relaxed) && !self.exited() { + match req_ch.recv_timeout(Duration::from_secs(1)) { + Ok(()) => { + let m = self.metrics(); + res_ch.send(m)?; + } + Err(RecvTimeoutError::Timeout) => { + ticks += 1; + if ticks % every == 0 { + self.report(); + } + } + Err(e) => Err(e)?, + } + } + + // Dropping the link is what detaches the scheduler. + let _ = self.struct_ops.take(); + uei_report!(&self.skel, uei) + } +} + +impl Drop for Scheduler<'_> { + fn drop(&mut self) { + info!("{SCHEDULER_NAME} unloaded, tasks returned to default kernel scheduler"); + } +} + +fn main() -> Result<()> { + let opts = Opts::parse(); + + let cfg = simplelog::ConfigBuilder::new() + .set_time_offset_to_local() + .expect("failed to set local time offset") + .build(); + + // SORYU_LOG= writes the same output to a file as well. + let mut sinks: Vec> = vec![simplelog::TermLogger::new( + simplelog::LevelFilter::Info, + cfg.clone(), + simplelog::TerminalMode::Stderr, + simplelog::ColorChoice::Auto, + )]; + + if let Some(path) = opts.log.clone() { + let file = std::fs::File::create(&path) + .with_context(|| format!("cannot open SORYU_LOG file {path:?}"))?; + + sinks.push(simplelog::WriteLogger::new( + simplelog::LevelFilter::Info, + cfg, + file, + )); + } + + simplelog::CombinedLogger::init(sinks)?; + + let shutdown = Arc::new(AtomicBool::new(false)); + let shutdown_handler = shutdown.clone(); + ctrlc::set_handler(move || { + shutdown_handler.store(true, Ordering::Relaxed); + })?; + + if let Some(intv) = opts.monitor { + return stats::monitor(Duration::from_secs_f64(intv), shutdown); + } + + info!( + "pid {} — Ctrl-C to unload, or from another terminal: sudo kill -INT {}", + std::process::id(), + std::process::id() + ); + + let mut open_object = MaybeUninit::uninit(); + loop { + if shutdown.load(Ordering::Relaxed) { + break; + } + + let mut sched = Scheduler::init(opts.clone(), &mut open_object)?; + let uei = match sched.run(shutdown.clone()) { + Ok(uei) => uei, + Err(e) => { + error!("{SCHEDULER_NAME} was ejected by the kernel: {e:#}"); + error!("all tasks are back on default kernel scheduler; the soryu is NOT running"); + return Err(e); + } + }; + + if !uei.should_restart() { + break; + } + + if shutdown.load(Ordering::Relaxed) { + break; + } + info!("hotplug detected, reloading"); + } + + info!("{SCHEDULER_NAME} exiting"); + Ok(()) +} diff --git a/src/stats.rs b/src/stats.rs new file mode 100644 index 0000000..429597a --- /dev/null +++ b/src/stats.rs @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Live telemetry, readable from a second terminal with `scx_soryu --monitor` +// while the scheduler runs. + +use std::io::Write; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; +use scx_stats::prelude::*; +use scx_stats_derive::stat_doc; +use scx_stats_derive::Stats; +use serde::Deserialize; +use serde::Serialize; + +#[stat_doc] +#[derive(Clone, Debug, Default, Serialize, Deserialize, Stats)] +#[stat(top)] +pub struct Metrics { + #[stat(desc = "Wakeups that found an idle CPU and skipped the shared queue")] + pub enq_local: u64, + #[stat(desc = "Enqueues onto a shared level DSQ")] + pub enq_shared: u64, + #[stat(desc = "Enqueues of single-CPU tasks, sent straight to that CPU")] + pub enq_pinned: u64, + #[stat(desc = "Tasks bounced back by ops.cpu_release")] + pub enq_reenq: u64, + #[stat(desc = "DSQ inserts that failed")] + pub insert_fail: u64, + #[stat(desc = "CPUs taken by a higher sched class (PipeWire RT, i915)")] + pub cpu_release: u64, + + #[stat(desc = "Boosts from going to sleep")] + pub boost_sleep: u64, + #[stat(desc = "Boosts from waking after sleeping at least one slice")] + pub boost_wake: u64, + #[stat(desc = "Deboosts from burning a whole slice")] + pub deboost: u64, + #[stat(desc = "Preemptions that banked an unused slice remainder")] + pub slice_carry: u64, + #[stat(desc = "Forks that split the parent's remaining slice")] + pub fork_split: u64, + #[stat(desc = "Preemption IPIs sent")] + pub preempt_kick: u64, + #[stat(desc = "Dispatches that jumped priority to unstarve a level")] + pub starve_serve: u64, + #[stat(desc = "Wakeups with nobody worse to preempt -- they wait out a slice")] + pub preempt_none: u64, + #[stat(desc = "Wakeups that kicked an idle CPU rather than preempting anyone")] + pub preempt_idle: u64, + + #[stat(desc = "Mean boost, x100 (negative is more interactive)")] + pub boost_mean_x100: i64, + #[stat(desc = "Percent of scheduling events at the -12 boost floor")] + pub boost_floor_pct: u64, + #[stat(desc = "Percent at the +12 ceiling")] + pub boost_ceil_pct: u64, + + #[stat(desc = "Interactive wakeup->run p50, ns")] + pub lat_boosted_p50: u64, + #[stat(desc = "Interactive wakeup->run p99, ns -- the number that matters")] + pub lat_boosted_p99: u64, + #[stat(desc = "Interactive wakeup->run p99.9, ns")] + pub lat_boosted_p999: u64, + #[stat(desc = "Interactive wakeup samples")] + pub lat_boosted_n: u64, + + #[stat(desc = "Worst wait by any runnable task since load, jiffies")] + pub max_wait_j: u64, + #[stat(desc = "Levels currently holding tasks")] + pub lvl_bitmap: u64, +} + +impl Metrics { + fn fmt_ns(ns: u64) -> String { + if ns >= 1_000_000_000 { + format!("{:.1}s", ns as f64 / 1e9) + } else if ns >= 1_000_000 { + format!("{}ms", ns / 1_000_000) + } else if ns >= 1_000 { + format!("{}us", ns / 1_000) + } else { + format!("{ns}ns") + } + } + + pub fn format(&self, w: &mut W) -> Result<()> { + writeln!( + w, + "wake->run interactive p50 {:>7} p99 {:>7} p99.9 {:>7} (n={})", + Self::fmt_ns(self.lat_boosted_p50), + Self::fmt_ns(self.lat_boosted_p99), + Self::fmt_ns(self.lat_boosted_p999), + self.lat_boosted_n, + )?; + writeln!( + w, + "boost mean {:+.2} floor {}% ceiling {}% max_wait {}j bitmap 0x{:x}", + self.boost_mean_x100 as f64 / 100.0, + self.boost_floor_pct, + self.boost_ceil_pct, + self.max_wait_j, + self.lvl_bitmap, + )?; + writeln!( + w, + "enqueue local {} shared {} pinned {} reenq {} fail {}", + self.enq_local, + self.enq_shared, + self.enq_pinned, + self.enq_reenq, + self.insert_fail, + )?; + writeln!( + w, + "signal boost_sleep {} boost_wake {} deboost {} carry {} fork_split {}", + self.boost_sleep, + self.boost_wake, + self.deboost, + self.slice_carry, + self.fork_split, + )?; + writeln!( + w, + "preempt kick {} idle {} none {} cpu_release {} starve_serve {}", + self.preempt_kick, + self.preempt_idle, + self.preempt_none, + self.cpu_release, + self.starve_serve, + )?; + Ok(()) + } + + fn delta(&self, rhs: &Self) -> Self { + Self { + enq_local: self.enq_local - rhs.enq_local, + enq_shared: self.enq_shared - rhs.enq_shared, + enq_pinned: self.enq_pinned - rhs.enq_pinned, + enq_reenq: self.enq_reenq - rhs.enq_reenq, + insert_fail: self.insert_fail - rhs.insert_fail, + cpu_release: self.cpu_release - rhs.cpu_release, + boost_sleep: self.boost_sleep - rhs.boost_sleep, + boost_wake: self.boost_wake - rhs.boost_wake, + deboost: self.deboost - rhs.deboost, + slice_carry: self.slice_carry - rhs.slice_carry, + fork_split: self.fork_split - rhs.fork_split, + preempt_kick: self.preempt_kick - rhs.preempt_kick, + starve_serve: self.starve_serve - rhs.starve_serve, + preempt_none: self.preempt_none - rhs.preempt_none, + preempt_idle: self.preempt_idle - rhs.preempt_idle, + lat_boosted_n: self.lat_boosted_n - rhs.lat_boosted_n, + ..self.clone() + } + } +} + +pub fn server_data() -> StatsServerData<(), Metrics> { + let open: Box> = Box::new(move |(req_ch, res_ch)| { + req_ch.send(())?; + let mut prev = res_ch.recv()?; + + let read: Box> = Box::new(move |_args, (req_ch, res_ch)| { + req_ch.send(())?; + let cur = res_ch.recv()?; + let delta = cur.delta(&prev); + prev = cur; + delta.to_json() + }); + + Ok(read) + }); + + StatsServerData::new() + .add_meta(Metrics::meta()) + .add_ops("top", StatsOps { open, close: None }) +} + +pub fn monitor(intv: Duration, shutdown: Arc) -> Result<()> { + scx_utils::monitor_stats::( + &[], + intv, + || shutdown.load(Ordering::Relaxed), + |metrics| metrics.format(&mut std::io::stdout()), + ) + .with_context(|| { + format!( + "cannot reach the {} stats socket \ + (/var/run/scx/root/stats); is it running, and did you use sudo?", + crate::SCHEDULER_NAME + ) + }) +}