scx_mlfq/
stats.rs

1// SPDX-License-Identifier: GPL-2.0
2//
3// Copyright (c) 2026 Galih Tama <galpt@v.recipes>
4//
5// This software may be used and distributed according to the terms of the GNU
6// General Public License version 2.
7
8//! Stats server, `#[derive(Stats)]` metrics and monitor loop.
9//!
10//! `Metrics` corresponds to the BPF-side `struct mlfq_stats`: the type is
11//! defined in `src/bpf/intf.h` and the `volatile` instance lives in
12//! `src/bpf/main.bpf.c`, plus a userspace uptime gauge. Field names match
13//! the BPF struct 1:1. The `top` stats op reports deltas over the poll
14//! interval.
15
16use std::io::Write;
17use std::sync::atomic::AtomicBool;
18use std::sync::atomic::Ordering;
19use std::sync::Arc;
20use std::time::Duration;
21
22use anyhow::Result;
23use scx_stats::prelude::*;
24use scx_stats_derive::stat_doc;
25use scx_stats_derive::Stats;
26use serde::Deserialize;
27use serde::Serialize;
28
29#[stat_doc]
30#[derive(Clone, Debug, Default, Serialize, Deserialize, Stats)]
31#[stat(top)]
32pub struct Metrics {
33    #[stat(desc = "Tasks currently executing on a CPU")]
34    pub on_cpu: u64,
35    #[stat(desc = "Total CPU runtime in ns")]
36    pub total_runtime: u64,
37    #[stat(desc = "Scheduler uptime (wall clock since attach)")]
38    pub uptime_ns: u64,
39    #[stat(desc = "Tasks placed in Q1")]
40    pub q1_placements: u64,
41    #[stat(desc = "Tasks placed in Q2")]
42    pub q2_placements: u64,
43    #[stat(desc = "Tasks placed in Q3")]
44    pub q3_placements: u64,
45    #[stat(desc = "Queue promotions")]
46    pub promotions: u64,
47    #[stat(desc = "Queue demotions")]
48    pub demotions: u64,
49    #[stat(desc = "Aging boosts to Q1")]
50    pub aging_boosts: u64,
51    #[stat(desc = "Short-sleep and I/O wakeup boosts")]
52    pub short_sleep_boosts: u64,
53    #[stat(desc = "Wakeup preemption kicks")]
54    pub preemption_kicks: u64,
55    #[stat(desc = "Q1 cpuperf target boosts set on running")]
56    pub cpuperf_boosts: u64,
57    #[stat(desc = "Dispatch moves from remote queue DSQs")]
58    pub steals: u64,
59    /*
60     * The dispatch steal is split by cache domain (see the two-tier
61     * steal scan). The same-LLC tier moves work inside a domain, the
62     * cross-LLC tier moves it between domains. steals == steals_same_llc +
63     * steals_cross_llc whenever the LLC data is populated (mlfq_nr_llcs
64     * > 0). A machine with LLC awareness disabled leaves the split
65     * counters at 0 and all moves count as `steals`.
66     */
67    #[stat(desc = "Dispatch moves from remote queue DSQs within the same LLC")]
68    pub steals_same_llc: u64,
69    #[stat(desc = "Dispatch moves from remote queue DSQs across LLC domains")]
70    pub steals_cross_llc: u64,
71    #[stat(desc = "Solo-task keep-running grants on empty dispatch")]
72    pub keep_running: u64,
73    #[stat(desc = "Enqueues dropped when task state cannot be allocated")]
74    pub enq_no_tctx: u64,
75    #[stat(desc = "Enqueues dropped for bad weight")]
76    pub enq_bad_weight: u64,
77    #[stat(desc = "Enqueues dropped for missing placement")]
78    pub enq_no_deadline: u64,
79    #[stat(desc = "Fast-path enqueues")]
80    pub enq_fastpath: u64,
81    #[stat(desc = "Regular-path enqueues")]
82    pub enq_regular: u64,
83    #[stat(desc = "Pinned enqueues to idle CPUs")]
84    pub enq_pinned_idle: u64,
85    #[stat(desc = "Pinned enqueues to busy CPUs")]
86    pub enq_pinned_busy: u64,
87    #[stat(desc = "Pinned enqueues to the global DSQ")]
88    pub enq_pinned_global: u64,
89    #[stat(desc = "MLFQ tree inference walks run")]
90    pub tree_inference: u64,
91    #[stat(desc = "Classifications served by the EMA fallback while untrained")]
92    pub tree_fallback: u64,
93    #[stat(desc = "Tree queue mappings that disagree with the base EMA mapping")]
94    pub tree_disagree: u64,
95    #[stat(desc = "Training samples emitted to the daemon")]
96    pub tree_samples_emitted: u64,
97    #[stat(desc = "Training samples dropped (ring buffer full)")]
98    pub tree_samples_dropped: u64,
99    #[stat(desc = "Realtime/DL/stop takeovers of SCX CPUs observed by the sched_switch hook")]
100    pub rt_takeovers: u64,
101    #[stat(desc = "DSQ evacuation passes that ran on realtime takeovers")]
102    pub rt_evacuations: u64,
103    #[stat(desc = "Placements redirected off realtime-occupied CPUs")]
104    pub rt_redirects: u64,
105    #[stat(desc = "SCX_ENQ_REENQ re-enqueues counted at enqueue")]
106    pub rt_reenqs: u64,
107    #[stat(
108        desc = "Per-op callback latency histogram, 4 ops x 8 buckets in microseconds (stopping, dispatch, enqueue, cpu_release)"
109    )]
110    pub op_lat: Vec<u64>,
111    #[stat(desc = "Training samples dropped by the per-pid window cap")]
112    pub tree_samples_cap_dropped: u64,
113    #[stat(desc = "Committed tree model generation, 0 while untrained")]
114    pub tree_model_generation: u64,
115    #[stat(desc = "Training samples behind the committed tree model")]
116    pub tree_model_samples: u64,
117    #[stat(desc = "Nodes of the committed tree model")]
118    pub tree_model_nodes: u64,
119    #[stat(
120        desc = "Committed tree MAE in microseconds on the held-out slice of its training window"
121    )]
122    pub tree_mae_tree_us: u64,
123    #[stat(desc = "Exact EMA-baseline MAE in microseconds on the same held-out slice")]
124    pub tree_mae_ema_us: u64,
125    /* Pearson correlation of the committed model's burst predictions
126     * with the labels on its held-out slice, scaled by 1000. Web UI
127     * only, so it carries no stat description. */
128    pub tree_corr_milli: i64,
129    /*
130     * System wakeup gauges and the effective adaptation state. The
131     * gauges and the effective values are instantaneous pass-throughs
132     * (the top op reports them as-is, like uptime_ns). The two counters
133     * are interval deltas like the placement counters.
134     */
135    #[stat(desc = "System wakeup-latency gauge (1s half-life EMA), microseconds")]
136    pub sys_lat_ema_us: u64,
137    #[stat(desc = "System wakeup-rate gauge (1s half-life EMA), fixed point; >> 8 = wakeups/s")]
138    pub sys_rate_ema: u64,
139    #[stat(desc = "Effective EMA Q1/Q2 band edge, microseconds")]
140    pub t_l_eff_us: u64,
141    #[stat(desc = "Effective EMA Q2/Q3 band edge, microseconds")]
142    pub t_h_eff_us: u64,
143    #[stat(desc = "Effective tree Q1/Q2 band edge, microseconds")]
144    pub t_int_eff_us: u64,
145    #[stat(desc = "Effective tree Q2/Q3 band edge, microseconds")]
146    pub t_bnd_eff_us: u64,
147    #[stat(desc = "Effective same-queue preemption residency guard, microseconds")]
148    pub guard_eff_us: u64,
149    #[stat(desc = "Adaptation shift, fixed point (FP_ONE = 1.0)")]
150    pub adapt_shift: i64,
151    #[stat(desc = "Wakeup arrivals (interval delta)")]
152    pub wakeup_total: u64,
153    #[stat(desc = "Adaptation steps run (interval delta)")]
154    pub adapt_steps: u64,
155}
156
157/// One entry of the web UI's per-CPU card grid.
158///
159/// The static fields (`freq_khz`, `llc_id`, `smt`) are seeded once at
160/// attach from the host topology (`topology::web_cpu_static`); the
161/// dynamic fields (`running_queue`, `running_pid`, `rt_occupied`,
162/// `running_gpu_submit`) are refreshed from the BPF per-CPU maps on
163/// every web-metrics poll. The carriers do not take part in the stats
164/// server's `delta()` accounting.
165#[derive(Clone, Debug, Default, Serialize, Deserialize)]
166pub struct PerCpuMetrics {
167    /// CPU id.
168    pub id: u32,
169    /// Maximum operating frequency of the CPU, in kHz.
170    pub freq_khz: u64,
171    /// Current operating frequency of the CPU, in kHz, refreshed from
172    /// sysfs at most once per second while the web UI serves stats. 0
173    /// when the cpufreq driver exposes no current frequency.
174    pub cur_freq_khz: u64,
175    /// LLC domain id of the CPU (0 when the domain is unknown).
176    pub llc_id: u32,
177    /// True when the CPU is the non-primary thread of an SMT core, the
178    /// virtual sibling. Display-only, see `topology::web_cpu_static`.
179    pub smt: bool,
180    /// Queue of the currently running task (0 = idle, 1..3).
181    pub running_queue: i32,
182    /// PID of the currently running task, 0 when idle.
183    pub running_pid: u32,
184    /// True when a realtime-class task currently occupies the CPU.
185    pub rt_occupied: bool,
186    /// The `gpu_submit` value of the task now on this CPU, 0..4 in steps
187    /// of FP_SHIFT. This is a snapshot of `mlfq_cpu_state.running_gpu_submit`
188    /// (which mirrors `task_ctx.gpu_submit` at `ops.running()`), not a
189    /// counter. It decays by one on long idle and is bumped at most once
190    /// per 10 ms, so a burst of trace hits for one job counts as one.
191    /// The system-wide lifetime total is `WebMetrics.gpu_submit_total`.
192    #[serde(alias = "gpu_submit")]
193    pub running_gpu_submit: u32,
194}
195
196/// Snapshot served by the web UI's `/api/stats` endpoint.
197///
198/// The run loop pushes one of these every iteration (both the stats
199/// request and the idle-timeout branches); the webui thread keeps the
200/// newest snapshot behind a mutex for the HTTP handlers. All fields are
201/// instantaneous gauges, not interval deltas: the gauges bypass the
202/// stats server's `delta()` entirely.
203#[derive(Clone, Debug, Default, Serialize, Deserialize)]
204pub struct WebMetrics {
205    /// The scheduler-wide counters, raw (no delta applied).
206    pub stats: Metrics,
207    /// One entry per online CPU.
208    pub per_cpu: Vec<PerCpuMetrics>,
209    /// Tracked runnable tasks per queue; index 0 unused, 1..3 = Q1..Q3.
210    pub queue_runnable: Vec<u64>,
211    /// Tracked runnable tasks per LLC domain.
212    pub llc_runnable: Vec<u64>,
213    /// Lifetime count of deduped gpu_submit bumps since attach, one per
214    /// 10 ms per-task window. This is the global hit counter
215    /// (`mlfq_gpu_submit_total`), not the per-task 0..4 quant. Each
216    /// logical GPU job that fires 2-3 tracepoints in a burst counts as
217    /// one here, same window as `PerCpuMetrics.running_gpu_submit`.
218    pub gpu_submit_total: u64,
219    /// Bitmask of GPU tracepoints that are attached on this run (bit0
220    /// amdgpu_cs, bit1 amdgpu_cs_ioctl, bit2 gpu_sched). Like
221    /// `gpu_submit_total` this is a system gauge, not a per-task value.
222    pub gpu_trace_mask: u32,
223}
224
225/// Bucket edges of the op-latency histogram, matching `enum
226/// mlfq_op_lat_consts` in `src/bpf/intf.h`, in microseconds.
227const OP_LAT_EDGES_US: [u64; 7] = [
228    crate::bpf_intf::mlfq_op_lat_consts_MLFQ_OP_LAT_EDGE_2 as u64,
229    crate::bpf_intf::mlfq_op_lat_consts_MLFQ_OP_LAT_EDGE_5 as u64,
230    crate::bpf_intf::mlfq_op_lat_consts_MLFQ_OP_LAT_EDGE_10 as u64,
231    crate::bpf_intf::mlfq_op_lat_consts_MLFQ_OP_LAT_EDGE_20 as u64,
232    crate::bpf_intf::mlfq_op_lat_consts_MLFQ_OP_LAT_EDGE_50 as u64,
233    crate::bpf_intf::mlfq_op_lat_consts_MLFQ_OP_LAT_EDGE_100 as u64,
234    crate::bpf_intf::mlfq_op_lat_consts_MLFQ_OP_LAT_EDGE_250 as u64,
235];
236
237/// Format one op's eight bucket counts with their microsecond edges as
238/// "low-high=count" pairs. A histogram shorter than eight buckets (the
239/// untracked default) formats as "n/a".
240fn fmt_op_lat(op: &[u64]) -> String {
241    if op.len() < 8 {
242        return "n/a".to_string();
243    }
244    let mut parts = Vec::new();
245    parts.push(format!("0-{}={}", OP_LAT_EDGES_US[0], op[0]));
246    for (i, edge) in OP_LAT_EDGES_US.iter().enumerate().skip(1) {
247        parts.push(format!("{}-{}={}", OP_LAT_EDGES_US[i - 1], edge, op[i]));
248    }
249    parts.push(format!("{}+={}", OP_LAT_EDGES_US[6], op[7]));
250    parts.join(" ")
251}
252
253impl Metrics {
254    fn format<W: Write>(&self, w: &mut W) -> Result<()> {
255        writeln!(
256            w,
257            "[{}] run={} runtime_ns={} uptime_ns={} \
258             placements: Q1={} Q2={} Q3={} \
259             promotions={} demotions={} aging_boosts={} short_sleep_boosts={} \
260             preemption_kicks={} cpuperf_boosts={}",
261            crate::SCHEDULER_NAME,
262            self.on_cpu,
263            self.total_runtime,
264            self.uptime_ns,
265            self.q1_placements,
266            self.q2_placements,
267            self.q3_placements,
268            self.promotions,
269            self.demotions,
270            self.aging_boosts,
271            self.short_sleep_boosts,
272            self.preemption_kicks,
273            self.cpuperf_boosts,
274        )?;
275        writeln!(
276            w,
277            "[{}] tree: gen={} nodes={} samples={} mae_tree={}us mae_ema={}us \
278             inf={} fallback={} disagree={} emitted={} dropped={} cap_dropped={}",
279            crate::SCHEDULER_NAME,
280            self.tree_model_generation,
281            self.tree_model_nodes,
282            self.tree_model_samples,
283            self.tree_mae_tree_us,
284            self.tree_mae_ema_us,
285            self.tree_inference,
286            self.tree_fallback,
287            self.tree_disagree,
288            self.tree_samples_emitted,
289            self.tree_samples_dropped,
290            self.tree_samples_cap_dropped,
291        )?;
292        writeln!(
293            w,
294            "[{}] op_lat_us: stopping[{}] dispatch[{}]",
295            crate::SCHEDULER_NAME,
296            fmt_op_lat(self.op_lat.get(0..8).unwrap_or(&[])),
297            fmt_op_lat(self.op_lat.get(8..16).unwrap_or(&[])),
298        )?;
299        writeln!(
300            w,
301            "[{}] op_lat_us: enqueue[{}] cpu_release[{}]",
302            crate::SCHEDULER_NAME,
303            fmt_op_lat(self.op_lat.get(16..24).unwrap_or(&[])),
304            fmt_op_lat(self.op_lat.get(24..32).unwrap_or(&[])),
305        )?;
306        writeln!(
307            w,
308            "[{}] adapt: lat_ema={}us rate_ema={}w/s shift={}% T_L_eff={}us T_H_eff={}us T_INT_eff={}us T_BND_eff={}us guard_eff={}us wakeups={} steps={}",
309            crate::SCHEDULER_NAME,
310            self.sys_lat_ema_us,
311            self.sys_rate_ema >> crate::bpf_intf::mlfq_consts_FP_SHIFT,
312            self.adapt_shift * 100 / crate::bpf_intf::mlfq_consts_FP_ONE as i64,
313            self.t_l_eff_us,
314            self.t_h_eff_us,
315            self.t_int_eff_us,
316            self.t_bnd_eff_us,
317            self.guard_eff_us,
318            self.wakeup_total,
319            self.adapt_steps,
320        )?;
321        Ok(())
322    }
323
324    /// Interval delta. Counters are wrapping deltas over the poll interval.
325    /// Gauges (`on_cpu`, `uptime_ns`, the tree model metadata) pass through
326    /// as instantaneous values.
327    pub fn delta(&self, rhs: &Self) -> Self {
328        Self {
329            on_cpu: self.on_cpu,
330            total_runtime: self.total_runtime.wrapping_sub(rhs.total_runtime),
331            uptime_ns: self.uptime_ns,
332            q1_placements: self.q1_placements.wrapping_sub(rhs.q1_placements),
333            q2_placements: self.q2_placements.wrapping_sub(rhs.q2_placements),
334            q3_placements: self.q3_placements.wrapping_sub(rhs.q3_placements),
335            promotions: self.promotions.wrapping_sub(rhs.promotions),
336            demotions: self.demotions.wrapping_sub(rhs.demotions),
337            aging_boosts: self.aging_boosts.wrapping_sub(rhs.aging_boosts),
338            short_sleep_boosts: self.short_sleep_boosts.wrapping_sub(rhs.short_sleep_boosts),
339            preemption_kicks: self.preemption_kicks.wrapping_sub(rhs.preemption_kicks),
340            cpuperf_boosts: self.cpuperf_boosts.wrapping_sub(rhs.cpuperf_boosts),
341            steals: self.steals.wrapping_sub(rhs.steals),
342            steals_same_llc: self.steals_same_llc.wrapping_sub(rhs.steals_same_llc),
343            steals_cross_llc: self.steals_cross_llc.wrapping_sub(rhs.steals_cross_llc),
344            keep_running: self.keep_running.wrapping_sub(rhs.keep_running),
345            enq_no_tctx: self.enq_no_tctx.wrapping_sub(rhs.enq_no_tctx),
346            enq_bad_weight: self.enq_bad_weight.wrapping_sub(rhs.enq_bad_weight),
347            enq_no_deadline: self.enq_no_deadline.wrapping_sub(rhs.enq_no_deadline),
348            enq_fastpath: self.enq_fastpath.wrapping_sub(rhs.enq_fastpath),
349            enq_regular: self.enq_regular.wrapping_sub(rhs.enq_regular),
350            enq_pinned_idle: self.enq_pinned_idle.wrapping_sub(rhs.enq_pinned_idle),
351            enq_pinned_busy: self.enq_pinned_busy.wrapping_sub(rhs.enq_pinned_busy),
352            enq_pinned_global: self.enq_pinned_global.wrapping_sub(rhs.enq_pinned_global),
353            tree_inference: self.tree_inference.wrapping_sub(rhs.tree_inference),
354            tree_fallback: self.tree_fallback.wrapping_sub(rhs.tree_fallback),
355            tree_disagree: self.tree_disagree.wrapping_sub(rhs.tree_disagree),
356            tree_samples_emitted: self
357                .tree_samples_emitted
358                .wrapping_sub(rhs.tree_samples_emitted),
359            tree_samples_dropped: self
360                .tree_samples_dropped
361                .wrapping_sub(rhs.tree_samples_dropped),
362            rt_takeovers: self.rt_takeovers.wrapping_sub(rhs.rt_takeovers),
363            rt_evacuations: self.rt_evacuations.wrapping_sub(rhs.rt_evacuations),
364            rt_redirects: self.rt_redirects.wrapping_sub(rhs.rt_redirects),
365            rt_reenqs: self.rt_reenqs.wrapping_sub(rhs.rt_reenqs),
366            /* The histogram is a per-interval delta like the counters. */
367            op_lat: self
368                .op_lat
369                .iter()
370                .zip(rhs.op_lat.iter())
371                .map(|(lhs, rhs)| lhs.wrapping_sub(*rhs))
372                .collect(),
373            tree_samples_cap_dropped: self
374                .tree_samples_cap_dropped
375                .wrapping_sub(rhs.tree_samples_cap_dropped),
376            /* Model metadata is a gauge: the currently committed model. */
377            tree_model_generation: self.tree_model_generation,
378            tree_model_samples: self.tree_model_samples,
379            tree_model_nodes: self.tree_model_nodes,
380            tree_mae_tree_us: self.tree_mae_tree_us,
381            tree_mae_ema_us: self.tree_mae_ema_us,
382            tree_corr_milli: self.tree_corr_milli,
383            /* Gauges pass through; the adaptation counters are deltas. */
384            sys_lat_ema_us: self.sys_lat_ema_us,
385            sys_rate_ema: self.sys_rate_ema,
386            t_l_eff_us: self.t_l_eff_us,
387            t_h_eff_us: self.t_h_eff_us,
388            t_int_eff_us: self.t_int_eff_us,
389            t_bnd_eff_us: self.t_bnd_eff_us,
390            guard_eff_us: self.guard_eff_us,
391            adapt_shift: self.adapt_shift,
392            wakeup_total: self.wakeup_total.wrapping_sub(rhs.wakeup_total),
393            adapt_steps: self.adapt_steps.wrapping_sub(rhs.adapt_steps),
394        }
395    }
396}
397
398/// The stats server definition. A single `top` op reporting interval deltas.
399pub fn server_data() -> StatsServerData<(), Metrics> {
400    let open: Box<dyn StatsOpener<(), Metrics>> = Box::new(move |(req_ch, res_ch)| {
401        req_ch.send(())?;
402        let mut prev = res_ch.recv()?;
403
404        let read: Box<dyn StatsReader<(), Metrics>> = Box::new(move |_args, (req_ch, res_ch)| {
405            req_ch.send(())?;
406            let cur = res_ch.recv()?;
407            let delta = cur.delta(&prev);
408            prev = cur;
409            delta.to_json()
410        });
411
412        Ok(read)
413    });
414
415    StatsServerData::new()
416        .add_meta(Metrics::meta())
417        .add_ops("top", StatsOps { open, close: None })
418}
419
420/// The monitor loop. It periodically polls the stats server and prints a
421/// one-line summary. It runs in its own thread (see `main.rs`) and exits
422/// on shutdown.
423pub fn monitor(intv: Duration, shutdown: Arc<AtomicBool>) -> Result<()> {
424    scx_utils::monitor_stats::<Metrics>(
425        &[],
426        intv,
427        || shutdown.load(Ordering::Relaxed),
428        |metrics| metrics.format(&mut std::io::stdout()),
429    )
430}
scx_mlfq/stats.rs

scx_mlfq/
stats.rs
