scx_pandemonium/
main.rs

1// PANDEMONIUM -- SCHED_EXT KERNEL SCHEDULER
2// ADAPTIVE DESKTOP SCHEDULING FOR LINUX
3//
4// SCHEDULING DECISIONS HAPPEN IN BPF (ZERO KERNEL-USERSPACE ROUND TRIPS)
5// RUST USERSPACE HANDLES: ADAPTIVE CONTROL LOOP, MONITORING, BENCHMARKING
6
7#[allow(non_upper_case_globals)]
8#[allow(non_camel_case_types)]
9#[allow(non_snake_case)]
10mod bpf_skel;
11
12mod bpf_intf;
13
14#[macro_use]
15mod log;
16mod adaptive;
17mod chaos;
18mod cli;
19mod scheduler;
20mod topology;
21mod tuning;
22mod watchdog;
23
24use std::mem::MaybeUninit;
25use std::sync::atomic::{AtomicBool, Ordering};
26use std::time::Duration;
27
28use anyhow::Result;
29use clap::{Parser, Subcommand};
30
31use scheduler::Scheduler;
32use scx_utils::build_id;
33
34static SHUTDOWN: AtomicBool = AtomicBool::new(false);
35
36#[derive(Parser)]
37#[command(name = "scx_pandemonium")]
38#[command(
39    version,
40    disable_version_flag = true,
41    about = "PANDEMONIUM -- ADAPTIVE LINUX SCHEDULER"
42)]
43struct Cli {
44    #[command(subcommand)]
45    command: Option<SubCmd>,
46
47    #[arg(short, long)]
48    verbose: bool,
49
50    /// Print scheduler version and exit.
51    #[arg(long)]
52    version: bool,
53
54    /// Internal: dump in-memory ring log on shutdown
55    #[arg(long, hide = true)]
56    dump_log: bool,
57
58    /// Internal: override CPU count for scaling formulas (test harness use)
59    #[arg(long, hide = true)]
60    nr_cpus: Option<u64>,
61
62    /// Run BPF scheduler only, disable Rust adaptive control loop
63    #[arg(long)]
64    no_adaptive: bool,
65}
66
67#[derive(Subcommand)]
68enum SubCmd {
69    /// Internal: interactive wakeup probe (Python test harness use)
70    #[command(hide = true)]
71    Probe,
72
73    /// Internal: CPU-pinned stress worker (Python test harness use)
74    #[command(hide = true)]
75    StressWorker(StressWorkerArgs),
76}
77
78#[derive(Parser)]
79struct StressWorkerArgs {
80    /// CPU to pin the stress worker to
81    #[arg(long)]
82    cpu: u32,
83}
84
85fn main() -> Result<()> {
86    let cli = Cli::parse();
87
88    let verbose = cli.verbose;
89    let dump_log = cli.dump_log;
90    let nr_cpus = cli.nr_cpus;
91    let no_adaptive = cli.no_adaptive;
92
93    if cli.version {
94        println!(
95            "scx_pandemonium {}",
96            build_id::full_version(env!("CARGO_PKG_VERSION"))
97        );
98        return Ok(());
99    }
100
101    match cli.command {
102        None => run_scheduler(verbose, dump_log, nr_cpus, no_adaptive),
103        Some(SubCmd::Probe) => {
104            cli::probe::run_probe();
105            Ok(())
106        }
107        Some(SubCmd::StressWorker(args)) => {
108            cli::stress::run_stress_worker(args.cpu);
109            Ok(())
110        }
111    }
112}
113
114fn run_scheduler(
115    verbose: bool,
116    dump_log: bool,
117    nr_cpus: Option<u64>,
118    no_adaptive: bool,
119) -> Result<()> {
120    ctrlc::set_handler(move || {
121        SHUTDOWN.store(true, Ordering::Relaxed);
122    })?;
123
124    // WATCHDOG: ABORTS IF THE CONTROL LOOP STALLS FOR MORE THAN 10 SECONDS.
125    // LIBBPF MAP OPERATIONS CAN HANG ON KERNEL STALL / VERIFIER RELOAD /
126    // PERCPU CONTENTION; WITHOUT THIS, TELEMETRY AND KNOB WRITES STOP SILENTLY.
127    watchdog::spawn(&SHUTDOWN, Duration::from_secs(10));
128
129    let nr_cpus_display =
130        nr_cpus.unwrap_or_else(|| libbpf_rs::num_possible_cpus().unwrap_or(1) as u64);
131    let governor = std::fs::read_to_string("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor")
132        .unwrap_or_default()
133        .trim()
134        .to_string();
135
136    let smt_on = std::fs::read_to_string("/sys/devices/system/cpu/smt/active")
137        .map(|s| s.trim() == "1")
138        .unwrap_or(false);
139
140    log_info!(
141        "scx_pandemonium {} SMT {}",
142        build_id::full_version(env!("CARGO_PKG_VERSION")),
143        if smt_on { "on" } else { "off" }
144    );
145    log_info!(
146        "CPUS: {} (governor: {})",
147        nr_cpus_display,
148        if governor.is_empty() {
149            "unknown"
150        } else {
151            &governor
152        }
153    );
154    log_info!("VERBOSE: {}", verbose);
155
156    let mut is_restart = false;
157    loop {
158        // ON RESTART, WAIT FOR KERNEL STRUCT_OPS CLEANUP.
159        // DETACH IS ASYNCHRONOUS -- UNDER HEAVY LOAD (12C SATURATED),
160        // THE KERNEL NEEDS TIME TO FULLY UNREGISTER THE OLD SCHEDULER.
161        if is_restart {
162            std::thread::sleep(Duration::from_secs(2));
163        }
164
165        let mut open_object = MaybeUninit::uninit();
166        let mut sched = Scheduler::init(&mut open_object, nr_cpus)?;
167
168        // POPULATE CACHE TOPOLOGY AT STARTUP -- the one detect-and-populate
169        // sequence (topology.rs owns it: all computes before any map write,
170        // tuning knobs written last as the "go" signal). The SAME sequence
171        // re-runs on hotplug via CpuTopology::poll_hotplug from both control
172        // loops, so a CPU broken at boot self-corrects and the R_eff/phi/
173        // domain tables track the live width.
174        let mut last_online = topology::CpuTopology::online_cpu_count();
175        if let Err(e) =
176            topology::CpuTopology::detect_and_populate(&mut sched, nr_cpus_display as usize)
177        {
178            log_warn!("CACHE TOPOLOGY DETECT FAILED: {}", e);
179        }
180
181        let should_restart = if no_adaptive {
182            // BPF-ONLY MODE: SCHEDULER RUNS WITH DEFAULT KNOBS, NO RUST TUNING
183            // STILL PRINTS STATS SO BENCHMARKS GET TELEMETRY FOR BOTH PHASES
184            log_info!("PANDEMONIUM IS ACTIVE (BPF ONLY, CTRL+C TO EXIT)");
185            let mut prev = scheduler::PandemoniumStats::default();
186            while !SHUTDOWN.load(Ordering::Relaxed) && !sched.exited() {
187                watchdog::LOOP_HEARTBEAT.fetch_add(1, Ordering::Relaxed);
188                std::thread::sleep(Duration::from_secs(1));
189
190                // HOTPLUG: re-derive topology when the online set changes
191                // (BPF-only mode has no adaptive loop to carry the poll).
192                topology::CpuTopology::poll_hotplug(
193                    &mut sched,
194                    nr_cpus_display as usize,
195                    &mut last_online,
196                );
197
198                let stats = sched.read_stats();
199
200                let delta_d = stats.nr_dispatches.wrapping_sub(prev.nr_dispatches);
201                let delta_idle = stats.nr_idle_hits.wrapping_sub(prev.nr_idle_hits);
202                let delta_shared = stats.nr_shared.wrapping_sub(prev.nr_shared);
203                let delta_preempt = stats.nr_preempt.wrapping_sub(prev.nr_preempt);
204                let delta_keep = stats.nr_keep_running.wrapping_sub(prev.nr_keep_running);
205                let delta_parks = stats.nr_osc_park.wrapping_sub(prev.nr_osc_park);
206                let delta_wake_sum = stats.wake_lat_sum.wrapping_sub(prev.wake_lat_sum);
207                let delta_wake_samples = stats.wake_lat_samples.wrapping_sub(prev.wake_lat_samples);
208                let delta_hard = stats.nr_hard_kicks.wrapping_sub(prev.nr_hard_kicks);
209                let delta_soft = stats.nr_soft_kicks.wrapping_sub(prev.nr_soft_kicks);
210                let delta_steal = stats.nr_steal.wrapping_sub(prev.nr_steal);
211                let delta_enq_wake = stats.nr_enq_wakeup.wrapping_sub(prev.nr_enq_wakeup);
212                let delta_enq_requeue = stats.nr_enq_requeue.wrapping_sub(prev.nr_enq_requeue);
213                let wake_avg_us = if delta_wake_samples > 0 {
214                    delta_wake_sum / delta_wake_samples / 1000
215                } else {
216                    0
217                };
218
219                let d_idle_sum = stats.wake_lat_idle_sum.wrapping_sub(prev.wake_lat_idle_sum);
220                let d_idle_cnt = stats.wake_lat_idle_cnt.wrapping_sub(prev.wake_lat_idle_cnt);
221                let d_kick_sum = stats.wake_lat_kick_sum.wrapping_sub(prev.wake_lat_kick_sum);
222                let d_kick_cnt = stats.wake_lat_kick_cnt.wrapping_sub(prev.wake_lat_kick_cnt);
223                let lat_idle_us = if d_idle_cnt > 0 {
224                    d_idle_sum / d_idle_cnt / 1000
225                } else {
226                    0
227                };
228                let lat_kick_us = if d_kick_cnt > 0 {
229                    d_kick_sum / d_kick_cnt / 1000
230                } else {
231                    0
232                };
233                let delta_reenq = stats.nr_reenqueue.wrapping_sub(prev.nr_reenqueue);
234
235                // L2 CACHE AFFINITY DELTAS
236                let dl2_hb = stats.nr_l2_hit_batch.wrapping_sub(prev.nr_l2_hit_batch);
237                let dl2_mb = stats.nr_l2_miss_batch.wrapping_sub(prev.nr_l2_miss_batch);
238                let dl2_hi = stats
239                    .nr_l2_hit_interactive
240                    .wrapping_sub(prev.nr_l2_hit_interactive);
241                let dl2_mi = stats
242                    .nr_l2_miss_interactive
243                    .wrapping_sub(prev.nr_l2_miss_interactive);
244                let l2_pct_b = if dl2_hb + dl2_mb > 0 {
245                    dl2_hb * 100 / (dl2_hb + dl2_mb)
246                } else {
247                    0
248                };
249                let l2_pct_i = if dl2_hi + dl2_mi > 0 {
250                    dl2_hi * 100 / (dl2_hi + dl2_mi)
251                } else {
252                    0
253                };
254
255                let idle_pct = if delta_d > 0 {
256                    delta_idle * 100 / delta_d
257                } else {
258                    0
259                };
260
261                let sojourn_ms = stats.batch_sojourn_ns / 1_000_000;
262                let longrun_label = if stats.longrun_mode_active > 0 {
263                    " LONGRUN"
264                } else {
265                    ""
266                };
267
268                if verbose {
269                    println!(
270                        "d/s: {:<8} idle: {}% shared: {:<6} preempt: {:<4} keep: {:<4} kick: H={:<4} S={:<4} enq: W={:<4} R={:<4} wake: {}us lat_idle: {}us lat_kick: {}us reenq: {} sjrn: {}ms l2: B={}% I={}% [BPF{}]",
271                        delta_d,
272                        idle_pct,
273                        delta_shared,
274                        delta_preempt,
275                        delta_keep,
276                        delta_hard,
277                        delta_soft,
278                        delta_enq_wake,
279                        delta_enq_requeue,
280                        wake_avg_us,
281                        lat_idle_us,
282                        lat_kick_us,
283                        delta_reenq,
284                        sojourn_ms,
285                        l2_pct_b,
286                        l2_pct_i,
287                        longrun_label,
288                    );
289                }
290
291                sched.log.snapshot(
292                    delta_d,
293                    delta_idle,
294                    delta_shared,
295                    delta_preempt,
296                    delta_keep,
297                    delta_parks,
298                    wake_avg_us,
299                    delta_hard,
300                    delta_soft,
301                    lat_idle_us,
302                    lat_kick_us,
303                    delta_steal,
304                );
305
306                prev = stats;
307            }
308
309            // KNOBS SUMMARY: CAPTURED BY TEST HARNESS FOR ARCHIVE
310            let knobs = sched.read_tuning_knobs();
311            let final_stats = sched.read_stats();
312            let l2_total_b = final_stats.nr_l2_hit_batch + final_stats.nr_l2_miss_batch;
313            let l2_total_i = final_stats.nr_l2_hit_interactive + final_stats.nr_l2_miss_interactive;
314            let l2_cum_b = if l2_total_b > 0 {
315                final_stats.nr_l2_hit_batch * 100 / l2_total_b
316            } else {
317                0
318            };
319            let l2_cum_i = if l2_total_i > 0 {
320                final_stats.nr_l2_hit_interactive * 100 / l2_total_i
321            } else {
322                0
323            };
324            // CROSS-DOMAIN SCATTER ATTRIBUTION (PER XDOM_* PATH), ON THE [KNOBS]
325            // LINE SO THE BENCH SUITE CAPTURES IT UNIFORMLY ACROSS BPF/ADAPTIVE
326            // (LETS THE SUITE COMPARE SCATTER BETWEEN MODES). scatter_pct IS THE
327            // PLACEMENT-SIDE FRACTION (idx 0..6).
328            let x = &final_stats.nr_cross_domain;
329            let x_scatter: u64 = x[0..6].iter().sum();
330            let x_scatter_pct = if final_stats.nr_dispatches > 0 {
331                x_scatter * 100 / final_stats.nr_dispatches
332            } else {
333                0
334            };
335            println!(
336                "[KNOBS] regime=BPF slice_ns={} batch_ns={} preempt_ns={} l2_hit=B:{}%/I:{}% cross_domain_scatter_pct={} cross_domain_sel_tight={} cross_domain_sel_sync={} cross_domain_sel_normal={} cross_domain_sel_dfl={} cross_domain_enq_t1={} cross_domain_enq_t2={} cross_domain_steal={} cross_domain_step5={} steal={} spill={} kick_declined={} stay_cost_held={} stay_move_taken={} dispatches={}",
337                knobs.slice_ns,
338                knobs.batch_slice_ns,
339                knobs.preempt_thresh_ns,
340                l2_cum_b,
341                l2_cum_i,
342                x_scatter_pct,
343                x[0],
344                x[1],
345                x[2],
346                x[3],
347                x[4],
348                x[5],
349                x[6],
350                x[7],
351                final_stats.nr_steal,
352                final_stats.nr_spill_kick_preempt,
353                final_stats.nr_kick_declined,
354                final_stats.nr_stay_cost_held,
355                final_stats.nr_stay_move_taken,
356                final_stats.nr_dispatches,
357            );
358
359            sched.read_exit_info()
360        } else {
361            // ADAPTIVE MODE: BPF + SINGLE-THREAD MONITOR LOOP
362            log_info!("PANDEMONIUM IS ACTIVE (CTRL+C TO EXIT)");
363            adaptive::monitor_loop(&mut sched, &SHUTDOWN, verbose, nr_cpus_display)?
364        };
365
366        log_info!("PANDEMONIUM IS SHUTTING DOWN");
367
368        if dump_log {
369            sched.log.dump();
370        }
371        sched.log.summary();
372
373        if !should_restart || SHUTDOWN.load(Ordering::Relaxed) {
374            break;
375        }
376
377        // RESET SHUTDOWN FOR RESTART
378        SHUTDOWN.store(false, Ordering::Relaxed);
379        log_info!("RESTARTING PANDEMONIUM...");
380        is_restart = true;
381    }
382
383    log_info!("Shutdown complete");
384    Ok(())
385}
scx_pandemonium/main.rs

scx_pandemonium/
main.rs
