1use std::collections::BTreeMap;
2use std::io::Write;
3use std::sync::atomic::AtomicBool;
4use std::sync::atomic::Ordering;
5use std::sync::Arc;
6use std::thread::current;
7use std::thread::ThreadId;
8use std::time::Duration;
9use std::time::SystemTime;
10use std::time::UNIX_EPOCH;
11
12use anyhow::bail;
13use anyhow::Result;
14use chrono::DateTime;
15use chrono::Local;
16use scx_stats::prelude::*;
17use scx_stats_derive::stat_doc;
18use scx_stats_derive::Stats;
19use scx_utils::Cpumask;
20use scx_utils::Topology;
21use serde::Deserialize;
22use serde::Serialize;
23use tracing::warn;
24
25use crate::bpf_intf;
26use crate::BpfStats;
27use crate::Layer;
28use crate::LayerKind;
29use crate::Stats;
30use crate::LAYER_USAGE_OPEN;
31use crate::LAYER_USAGE_PROTECTED;
32use crate::LAYER_USAGE_PROTECTED_PREEMPT;
33use crate::LAYER_USAGE_SUM_UPTO;
34
35const GSTAT_EXCL_IDLE: usize = bpf_intf::global_stat_id_GSTAT_EXCL_IDLE as usize;
36const GSTAT_EXCL_WAKEUP: usize = bpf_intf::global_stat_id_GSTAT_EXCL_WAKEUP as usize;
37const GSTAT_HI_FB_EVENTS: usize = bpf_intf::global_stat_id_GSTAT_HI_FB_EVENTS as usize;
38const GSTAT_HI_FB_USAGE: usize = bpf_intf::global_stat_id_GSTAT_HI_FB_USAGE as usize;
39const GSTAT_LO_FB_EVENTS: usize = bpf_intf::global_stat_id_GSTAT_LO_FB_EVENTS as usize;
40const GSTAT_LO_FB_USAGE: usize = bpf_intf::global_stat_id_GSTAT_LO_FB_USAGE as usize;
41const GSTAT_FB_CPU_USAGE: usize = bpf_intf::global_stat_id_GSTAT_FB_CPU_USAGE as usize;
42const GSTAT_ANTISTALL: usize = bpf_intf::global_stat_id_GSTAT_ANTISTALL as usize;
43const GSTAT_SKIP_PREEMPT: usize = bpf_intf::global_stat_id_GSTAT_SKIP_PREEMPT as usize;
44const GSTAT_FIXUP_VTIME: usize = bpf_intf::global_stat_id_GSTAT_FIXUP_VTIME as usize;
45const GSTAT_PREEMPTING_MISMATCH: usize =
46 bpf_intf::global_stat_id_GSTAT_PREEMPTING_MISMATCH as usize;
47
48const LSTAT_SEL_LOCAL: usize = bpf_intf::layer_stat_id_LSTAT_SEL_LOCAL as usize;
49const LSTAT_ENQ_LOCAL: usize = bpf_intf::layer_stat_id_LSTAT_ENQ_LOCAL as usize;
50const LSTAT_ENQ_WAKEUP: usize = bpf_intf::layer_stat_id_LSTAT_ENQ_WAKEUP as usize;
51const LSTAT_ENQ_EXPIRE: usize = bpf_intf::layer_stat_id_LSTAT_ENQ_EXPIRE as usize;
52const LSTAT_ENQ_REENQ: usize = bpf_intf::layer_stat_id_LSTAT_ENQ_REENQ as usize;
53const LSTAT_ENQ_DSQ: usize = bpf_intf::layer_stat_id_LSTAT_ENQ_DSQ as usize;
54const LSTAT_MIN_EXEC: usize = bpf_intf::layer_stat_id_LSTAT_MIN_EXEC as usize;
55const LSTAT_MIN_EXEC_NS: usize = bpf_intf::layer_stat_id_LSTAT_MIN_EXEC_NS as usize;
56const LSTAT_OPEN_IDLE: usize = bpf_intf::layer_stat_id_LSTAT_OPEN_IDLE as usize;
57const LSTAT_AFFN_VIOL: usize = bpf_intf::layer_stat_id_LSTAT_AFFN_VIOL as usize;
58const LSTAT_KEEP: usize = bpf_intf::layer_stat_id_LSTAT_KEEP as usize;
59const LSTAT_KEEP_FAIL_MAX_EXEC: usize = bpf_intf::layer_stat_id_LSTAT_KEEP_FAIL_MAX_EXEC as usize;
60const LSTAT_KEEP_FAIL_BUSY: usize = bpf_intf::layer_stat_id_LSTAT_KEEP_FAIL_BUSY as usize;
61const LSTAT_PREEMPT: usize = bpf_intf::layer_stat_id_LSTAT_PREEMPT as usize;
62const LSTAT_PREEMPT_FIRST: usize = bpf_intf::layer_stat_id_LSTAT_PREEMPT_FIRST as usize;
63const LSTAT_PREEMPT_XLLC: usize = bpf_intf::layer_stat_id_LSTAT_PREEMPT_XLLC as usize;
64const LSTAT_PREEMPT_XNUMA: usize = bpf_intf::layer_stat_id_LSTAT_PREEMPT_XNUMA as usize;
65const LSTAT_PREEMPT_IDLE: usize = bpf_intf::layer_stat_id_LSTAT_PREEMPT_IDLE as usize;
66const LSTAT_PREEMPT_FAIL: usize = bpf_intf::layer_stat_id_LSTAT_PREEMPT_FAIL as usize;
67const LSTAT_EXCL_COLLISION: usize = bpf_intf::layer_stat_id_LSTAT_EXCL_COLLISION as usize;
68const LSTAT_EXCL_PREEMPT: usize = bpf_intf::layer_stat_id_LSTAT_EXCL_PREEMPT as usize;
69const LSTAT_YIELD: usize = bpf_intf::layer_stat_id_LSTAT_YIELD as usize;
70const LSTAT_YIELD_IGNORE: usize = bpf_intf::layer_stat_id_LSTAT_YIELD_IGNORE as usize;
71const LSTAT_MIGRATION: usize = bpf_intf::layer_stat_id_LSTAT_MIGRATION as usize;
72const LSTAT_XNUMA_MIGRATION: usize = bpf_intf::layer_stat_id_LSTAT_XNUMA_MIGRATION as usize;
73const LSTAT_XLLC_MIGRATION: usize = bpf_intf::layer_stat_id_LSTAT_XLLC_MIGRATION as usize;
74const LSTAT_XLLC_MIGRATION_SKIP: usize = bpf_intf::layer_stat_id_LSTAT_XLLC_MIGRATION_SKIP as usize;
75const LSTAT_XLAYER_WAKE: usize = bpf_intf::layer_stat_id_LSTAT_XLAYER_WAKE as usize;
76const LSTAT_XLAYER_REWAKE: usize = bpf_intf::layer_stat_id_LSTAT_XLAYER_REWAKE as usize;
77const LSTAT_LLC_DRAIN_TRY: usize = bpf_intf::layer_stat_id_LSTAT_LLC_DRAIN_TRY as usize;
78const LSTAT_LLC_DRAIN: usize = bpf_intf::layer_stat_id_LSTAT_LLC_DRAIN as usize;
79const LSTAT_SKIP_REMOTE_NODE: usize = bpf_intf::layer_stat_id_LSTAT_SKIP_REMOTE_NODE as usize;
80
81const LSTAT_RUNQ_LAT_BASE: usize = bpf_intf::layer_stat_id_LSTAT_RUNQ_LAT_BASE as usize;
82const NR_RUNQ_LAT_BUCKETS: usize = bpf_intf::consts_NR_RUNQ_LAT_BUCKETS as usize;
83
84const LLC_LSTAT_LAT: usize = bpf_intf::llc_layer_stat_id_LLC_LSTAT_LAT as usize;
85const LLC_LSTAT_CNT: usize = bpf_intf::llc_layer_stat_id_LLC_LSTAT_CNT as usize;
86
87fn calc_frac(a: f64, b: f64) -> f64 {
88 if b != 0.0 {
89 a / b * 100.0
90 } else {
91 0.0
92 }
93}
94
95fn fmt_pct(v: f64) -> String {
96 if v >= 99.95 {
97 format!("{:4.0}", v)
98 } else if v >= 10.0 {
99 format!("{:4.1}", v)
100 } else if v > 0.0 && v < 0.01 {
101 format!("{:4.2}", 0.01)
102 } else {
103 format!("{:4.2}", v)
104 }
105}
106
107fn fmt_duration_ms(ms: f64) -> String {
108 if ms >= 60_000.0 {
109 let min = ms / 60_000.0;
110 if min >= 100.0 {
111 format!("{:.0}min", min)
112 } else {
113 format!("{:.1}min", min)
114 }
115 } else if ms >= 1_000.0 {
116 let s = ms / 1_000.0;
117 if s >= 100.0 {
118 format!("{:.0}s", s)
119 } else {
120 format!("{:.1}s", s)
121 }
122 } else if ms >= 10.0 {
123 format!("{:.0}ms", ms)
124 } else {
125 format!("{:.1}ms", ms)
126 }
127}
128
129fn fmt_num(v: u64) -> String {
130 if v > 1_000_000 {
131 format!("{:5.1}m", v as f64 / 1_000_000.0)
132 } else if v > 1_000 {
133 format!("{:5.1}k", v as f64 / 1_000.0)
134 } else {
135 format!("{:5.0} ", v)
136 }
137}
138
139#[stat_doc]
140#[derive(Clone, Debug, Default, Serialize, Deserialize, Stats)]
141#[stat(_om_prefix = "l_", _om_label = "layer_name")]
142pub struct LayerStats {
143 #[stat(desc = "index", _om_skip)]
144 pub index: usize,
145 #[stat(desc = "Total CPU utilization (100% means one full CPU)")]
146 pub util: f64,
147 #[stat(desc = "Compensated CPU utilization (adjusted for irq/softirq/stolen)")]
148 pub util_compensated: f64,
149 #[stat(desc = "Protected CPU utilization %")]
150 pub util_protected_frac: f64,
151 #[stat(desc = "Preempt-protected CPU utilization %")]
152 pub util_protected_preempt_frac: f64,
153 #[stat(desc = "Open CPU utilization %")]
154 pub util_open_frac: f64,
155 #[stat(desc = "fraction of total CPU utilization")]
156 pub util_frac: f64,
157 #[stat(desc = "number of tasks")]
158 pub tasks: u32,
159 #[stat(desc = "count of sched events during the period")]
160 pub total: u64,
161 #[stat(desc = "% dispatched into idle CPU from select_cpu")]
162 pub sel_local: f64,
163 #[stat(desc = "% dispatched into idle CPU from enqueue")]
164 pub enq_local: f64,
165 #[stat(desc = "% enqueued after wakeup")]
166 pub enq_wakeup: f64,
167 #[stat(desc = "% enqueued after slice expiration")]
168 pub enq_expire: f64,
169 #[stat(desc = "% re-enqueued due to RT preemption")]
170 pub enq_reenq: f64,
171 #[stat(desc = "% enqueued into the layer's LLC DSQs")]
172 pub enq_dsq: f64,
173 #[stat(desc = "count of times exec duration < min_exec_us")]
174 pub min_exec: f64,
175 #[stat(desc = "total exec durations extended due to min_exec_us")]
176 pub min_exec_us: u64,
177 #[stat(desc = "% dispatched into idle CPUs occupied by other layers")]
178 pub open_idle: f64,
179 #[stat(desc = "% preempted other tasks")]
180 pub preempt: f64,
181 #[stat(desc = "% preempted XLLC tasks")]
182 pub preempt_xllc: f64,
183 #[stat(desc = "% preempted across NUMA nodes")]
184 pub preempt_xnuma: f64,
185 #[stat(desc = "% first-preempted other tasks")]
186 pub preempt_first: f64,
187 #[stat(desc = "% idle-preempted other tasks")]
188 pub preempt_idle: f64,
189 #[stat(desc = "% attempted to preempt other tasks but failed")]
190 pub preempt_fail: f64,
191 #[stat(desc = "% violated config due to CPU affinity")]
192 pub affn_viol: f64,
193 #[stat(desc = "% continued executing after slice expiration")]
194 pub keep: f64,
195 #[stat(desc = "% disallowed to continue executing due to max_exec")]
196 pub keep_fail_max_exec: f64,
197 #[stat(desc = "% disallowed to continue executing due to other tasks")]
198 pub keep_fail_busy: f64,
199 #[stat(desc = "whether is exclusive", _om_skip)]
200 pub is_excl: u32,
201 #[stat(desc = "count of times an excl task skipped a CPU as the sibling was also excl")]
202 pub excl_collision: f64,
203 #[stat(desc = "% a sibling CPU was preempted for an exclusive task")]
204 pub excl_preempt: f64,
205 #[stat(desc = "% yielded")]
206 pub yielded: f64,
207 #[stat(desc = "count of times yield was ignored")]
208 pub yield_ignore: u64,
209 #[stat(desc = "% migrated across CPUs")]
210 pub migration: f64,
211 #[stat(desc = "% migrated across NUMA nodes")]
212 pub xnuma_migration: f64,
213 #[stat(desc = "% migrated across LLCs")]
214 pub xllc_migration: f64,
215 #[stat(desc = "% migration skipped across LLCs due to xllc_mig_min_us")]
216 pub xllc_migration_skip: f64,
217 #[stat(desc = "% wakers across layers")]
218 pub xlayer_wake: f64,
219 #[stat(desc = "% rewakers across layers where waker has waken the task previously")]
220 pub xlayer_rewake: f64,
221 #[stat(desc = "% LLC draining tried")]
222 pub llc_drain_try: f64,
223 #[stat(desc = "% LLC draining succeeded")]
224 pub llc_drain: f64,
225 #[stat(desc = "% skip LLC dispatch on remote node")]
226 pub skip_remote_node: f64,
227 #[stat(desc = "mask of allocated CPUs", _om_skip)]
228 pub cpus: Vec<u64>,
229 #[stat(desc = "count of CPUs assigned")]
230 pub cur_nr_cpus: u32,
231 #[stat(desc = "minimum # of CPUs assigned")]
232 pub min_nr_cpus: u32,
233 #[stat(desc = "maximum # of CPUs assigned")]
234 pub max_nr_cpus: u32,
235 #[stat(desc = "count of CPUs assigned per LLC")]
236 pub nr_llc_cpus: Vec<u32>,
237 #[stat(desc = "slice duration config")]
238 pub slice_us: u64,
239 #[stat(desc = "Per-LLC scheduling event fractions")]
240 pub llc_fracs: Vec<f64>,
241 #[stat(desc = "Per-LLC average latency")]
242 pub llc_lats: Vec<f64>,
243 #[stat(desc = "Layer memory bandwidth as a % of total allowed (0 for \"no limit\"")]
244 pub membw_pct: f64,
245 #[stat(desc = "DSQ insertion ratio EWMA (10s window)")]
246 pub dsq_insert_ewma: f64,
247 #[stat(desc = "Per-node layer utilization (100% = one full CPU)")]
248 pub node_utils: Vec<f64>,
249 #[stat(desc = "Per-node pinned task utilization (100% = one full CPU)")]
250 pub node_pinned_utils: Vec<f64>,
251 #[stat(desc = "Per-node pinned task counts")]
252 pub node_pinned_tasks: Vec<u64>,
253 #[stat(desc = "Per-node load (100% = one full CPU, from duty cycle sum)")]
254 pub node_loads: Vec<f64>,
255 #[stat(desc = "Whether xnuma gating is active for this layer (0/1)")]
256 pub xnuma_active: u32,
257 #[stat(desc = "runqueue latency histogram, log2 us buckets 1us..32s, per stats interval")]
258 pub l_runq_lat_hist: Vec<u64>,
259}
260
261impl LayerStats {
262 pub fn new(
263 lidx: usize,
264 layer: &Layer,
265 stats: &Stats,
266 bstats: &BpfStats,
267 nr_cpus_range: (usize, usize),
268 xnuma_active: bool,
269 ) -> Self {
270 let lstat = |sidx| bstats.lstats[lidx][sidx];
271 let ltotal = lstat(LSTAT_SEL_LOCAL)
272 + lstat(LSTAT_ENQ_LOCAL)
273 + lstat(LSTAT_ENQ_WAKEUP)
274 + lstat(LSTAT_ENQ_EXPIRE)
275 + lstat(LSTAT_ENQ_REENQ)
276 + lstat(LSTAT_KEEP);
277 let lstat_pct = |sidx| {
278 if ltotal != 0 {
279 lstat(sidx) as f64 / ltotal as f64 * 100.0
280 } else {
281 0.0
282 }
283 };
284
285 let util_sum = stats.layer_utils[lidx]
286 .iter()
287 .take(LAYER_USAGE_SUM_UPTO + 1)
288 .sum::<f64>();
289
290 let util_comp_sum = stats.layer_utils_compensated[lidx]
291 .iter()
292 .take(LAYER_USAGE_SUM_UPTO + 1)
293 .sum::<f64>();
294
295 let membw_frac = match &layer.kind {
296 LayerKind::Open { .. } => 0.0,
298 LayerKind::Confined { membw_gb, .. } | LayerKind::Grouped { membw_gb, .. } => {
299 if let Some(membw_limit_gb) = membw_gb {
301 stats.layer_membws[lidx]
302 .iter()
303 .take(LAYER_USAGE_SUM_UPTO + 1)
304 .sum::<f64>()
305 / (*membw_limit_gb * (1024_u64.pow(3) as f64))
306 } else {
307 0.0
308 }
309 }
310 };
311
312 Self {
313 index: lidx,
314 util: util_sum * 100.0,
315 util_compensated: util_comp_sum * 100.0,
316 util_open_frac: calc_frac(stats.layer_utils[lidx][LAYER_USAGE_OPEN], util_sum),
317 util_protected_frac: calc_frac(
318 stats.layer_utils[lidx][LAYER_USAGE_PROTECTED],
319 util_sum,
320 ),
321 util_protected_preempt_frac: calc_frac(
322 stats.layer_utils[lidx][LAYER_USAGE_PROTECTED_PREEMPT],
323 util_sum,
324 ),
325 util_frac: calc_frac(util_sum, stats.total_util),
326 tasks: stats.nr_layer_tasks[lidx] as u32,
327 total: ltotal,
328 sel_local: lstat_pct(LSTAT_SEL_LOCAL),
329 enq_local: lstat_pct(LSTAT_ENQ_LOCAL),
330 enq_wakeup: lstat_pct(LSTAT_ENQ_WAKEUP),
331 enq_expire: lstat_pct(LSTAT_ENQ_EXPIRE),
332 enq_reenq: lstat_pct(LSTAT_ENQ_REENQ),
333 enq_dsq: lstat_pct(LSTAT_ENQ_DSQ),
334 min_exec: lstat_pct(LSTAT_MIN_EXEC),
335 min_exec_us: lstat(LSTAT_MIN_EXEC_NS) / 1000,
336 open_idle: lstat_pct(LSTAT_OPEN_IDLE),
337 preempt: lstat_pct(LSTAT_PREEMPT),
338 preempt_xllc: lstat_pct(LSTAT_PREEMPT_XLLC),
339 preempt_xnuma: lstat_pct(LSTAT_PREEMPT_XNUMA),
340 preempt_first: lstat_pct(LSTAT_PREEMPT_FIRST),
341 preempt_idle: lstat_pct(LSTAT_PREEMPT_IDLE),
342 preempt_fail: lstat_pct(LSTAT_PREEMPT_FAIL),
343 affn_viol: lstat_pct(LSTAT_AFFN_VIOL),
344 keep: lstat_pct(LSTAT_KEEP),
345 keep_fail_max_exec: lstat_pct(LSTAT_KEEP_FAIL_MAX_EXEC),
346 keep_fail_busy: lstat_pct(LSTAT_KEEP_FAIL_BUSY),
347 is_excl: layer.kind.common().exclusive as u32,
348 excl_collision: lstat_pct(LSTAT_EXCL_COLLISION),
349 excl_preempt: lstat_pct(LSTAT_EXCL_PREEMPT),
350 yielded: lstat_pct(LSTAT_YIELD),
351 yield_ignore: lstat(LSTAT_YIELD_IGNORE),
352 migration: lstat_pct(LSTAT_MIGRATION),
353 xnuma_migration: lstat_pct(LSTAT_XNUMA_MIGRATION),
354 xlayer_wake: lstat_pct(LSTAT_XLAYER_WAKE),
355 xlayer_rewake: lstat_pct(LSTAT_XLAYER_REWAKE),
356 xllc_migration: lstat_pct(LSTAT_XLLC_MIGRATION),
357 xllc_migration_skip: lstat_pct(LSTAT_XLLC_MIGRATION_SKIP),
358 llc_drain_try: lstat_pct(LSTAT_LLC_DRAIN_TRY),
359 llc_drain: lstat_pct(LSTAT_LLC_DRAIN),
360 skip_remote_node: lstat_pct(LSTAT_SKIP_REMOTE_NODE),
361 cpus: layer.cpus.as_raw_slice().to_vec(),
362 cur_nr_cpus: layer.cpus.weight() as u32,
363 min_nr_cpus: nr_cpus_range.0 as u32,
364 max_nr_cpus: nr_cpus_range.1 as u32,
365 nr_llc_cpus: layer.nr_llc_cpus.iter().map(|&v| v as u32).collect(),
366 slice_us: stats.layer_slice_us[lidx],
367 llc_fracs: {
368 let sid = LLC_LSTAT_CNT;
369 let sum = bstats.llc_lstats[lidx]
370 .iter()
371 .map(|lstats| lstats[sid])
372 .sum::<u64>() as f64;
373 bstats.llc_lstats[lidx]
374 .iter()
375 .map(|lstats| calc_frac(lstats[sid] as f64, sum))
376 .collect()
377 },
378 llc_lats: bstats.llc_lstats[lidx]
379 .iter()
380 .map(|lstats| lstats[LLC_LSTAT_LAT] as f64 / 1_000_000_000.0)
381 .collect(),
382 membw_pct: membw_frac * 100.0,
383 dsq_insert_ewma: stats.layer_dsq_insert_ewma[lidx] * 100.0,
384 node_utils: stats.layer_node_utils[lidx]
385 .iter()
386 .map(|u| u * 100.0)
387 .collect(),
388 node_pinned_utils: stats.layer_node_pinned_utils[lidx]
389 .iter()
390 .map(|u| u * 100.0)
391 .collect(),
392 node_pinned_tasks: stats.layer_nr_node_pinned_tasks[lidx].clone(),
393 node_loads: stats.layer_node_duty_sums[lidx]
394 .iter()
395 .map(|l| l * 100.0)
396 .collect(),
397 xnuma_active: if xnuma_active { 1 } else { 0 },
398 l_runq_lat_hist: (0..NR_RUNQ_LAT_BUCKETS)
399 .map(|b| lstat(LSTAT_RUNQ_LAT_BASE + b))
400 .collect(),
401 }
402 }
403
404 pub fn format<W: Write>(
405 &self,
406 w: &mut W,
407 name: &str,
408 topo: Option<&Topology>,
409 max_width: usize,
410 no_llc: bool,
411 ) -> Result<()> {
412 let comp_str = if self.util > 0.1 && (self.util_compensated - self.util).abs() > 0.1 {
414 let overhead_pct = (1.0 - self.util / self.util_compensated) * 100.0;
415 format!(" comp_overhead={:.1}%", overhead_pct)
416 } else {
417 String::new()
418 };
419 writeln!(
420 w,
421 "\n\u{25B6} {} \u{2500} util/open/frac={:6.1}/{}/{:7.1}{} prot/prot_preempt={}/{} tasks={:6}",
422 name,
423 self.util,
424 fmt_pct(self.util_open_frac),
425 self.util_frac,
426 comp_str,
427 fmt_pct(self.util_protected_frac),
428 fmt_pct(self.util_protected_preempt_frac),
429 self.tasks,
430 )?;
431
432 writeln!(
434 w,
435 " {:<7} tot={} dd_sel/enq={}/{} dsq/10s={}/{} wake/exp/re={}/{}/{}",
436 "sched",
437 fmt_num(self.total),
438 fmt_pct(self.sel_local),
439 fmt_pct(self.enq_local),
440 fmt_pct(self.enq_dsq),
441 fmt_pct(self.dsq_insert_ewma),
442 fmt_pct(self.enq_wakeup),
443 fmt_pct(self.enq_expire),
444 fmt_pct(self.enq_reenq),
445 )?;
446
447 writeln!(
449 w,
450 " {:<7} keep/max/busy={}/{}/{} yield/ign={}/{} slc={} min_ex={}/{}",
451 "exec",
452 fmt_pct(self.keep),
453 fmt_pct(self.keep_fail_max_exec),
454 fmt_pct(self.keep_fail_busy),
455 fmt_pct(self.yielded),
456 fmt_num(self.yield_ignore),
457 fmt_duration_ms(self.slice_us as f64 / 1000.0),
458 fmt_pct(self.min_exec),
459 fmt_duration_ms(self.min_exec_us as f64 / 1000.0),
460 )?;
461
462 writeln!(
464 w,
465 " {:<7} mig={} xnuma={} xllc/skip={}/{} open_idle={} affn_viol={}",
466 "mig",
467 fmt_pct(self.migration),
468 fmt_pct(self.xnuma_migration),
469 fmt_pct(self.xllc_migration),
470 fmt_pct(self.xllc_migration_skip),
471 fmt_pct(self.open_idle),
472 fmt_pct(self.affn_viol),
473 )?;
474
475 writeln!(
477 w,
478 " {:<7} preempt/first/xllc/xnuma/idle/fail={}/{}/{}/{}/{}/{}",
479 "preempt",
480 fmt_pct(self.preempt),
481 fmt_pct(self.preempt_first),
482 fmt_pct(self.preempt_xllc),
483 fmt_pct(self.preempt_xnuma),
484 fmt_pct(self.preempt_idle),
485 fmt_pct(self.preempt_fail),
486 )?;
487
488 writeln!(
490 w,
491 " {:<7} wake/re={}/{} llc_drain/try={}/{} skip_rnode={}",
492 "xlayer",
493 fmt_pct(self.xlayer_wake),
494 fmt_pct(self.xlayer_rewake),
495 fmt_pct(self.llc_drain),
496 fmt_pct(self.llc_drain_try),
497 fmt_pct(self.skip_remote_node),
498 )?;
499
500 if self.node_utils.len() > 1 {
502 let prefix = " node pin/ut/ld ";
503 let cell_width = 25;
505 let usable = if max_width > prefix.len() {
506 max_width - prefix.len()
507 } else {
508 60
509 };
510 let cells_per_row = (usable / cell_width).max(1);
511
512 for nid in 0..self.node_utils.len() {
513 let util = self.node_utils[nid];
514 let load = self.node_loads.get(nid).copied().unwrap_or(0.0);
515 let pin = self.node_pinned_utils.get(nid).copied().unwrap_or(0.0);
516 if nid % cells_per_row == 0 {
517 if nid > 0 {
518 writeln!(w)?;
519 }
520 write!(w, "{prefix}")?;
521 } else {
522 write!(w, " ")?;
523 }
524 write!(w, "N{}={:5.1}/{:5.1}/{:7.1}", nid, pin, util, load)?;
525 }
526 writeln!(w)?;
527 }
528
529 let cpumask = Cpumask::from_vec(self.cpus.clone());
531
532 if let Some(topo) = topo {
533 let header = topo.format_cpumask_header(&cpumask, self.min_nr_cpus, self.max_nr_cpus);
534 writeln!(w, " {}", header)?;
535 if cpumask.weight() > 0 {
536 topo.format_cpumask_grid(w, &cpumask, " ", max_width)?;
537 }
538 } else {
539 writeln!(
540 w,
541 " cpus={:3} [{:3},{:3}] {}",
542 self.cur_nr_cpus, self.min_nr_cpus, self.max_nr_cpus, cpumask,
543 )?;
544 }
545
546 if self.is_excl != 0 {
548 writeln!(
549 w,
550 " excl_coll={} excl_preempt={}",
551 fmt_pct(self.excl_collision),
552 fmt_pct(self.excl_preempt),
553 )?;
554 } else if self.excl_collision != 0.0 || self.excl_preempt != 0.0 {
555 warn!(
556 "{}: exclusive is off but excl_coll={} excl_preempt={}",
557 name,
558 fmt_pct(self.excl_collision),
559 fmt_pct(self.excl_preempt),
560 );
561 }
562
563 if !no_llc {
565 let active_llcs: Vec<(usize, f64, f64)> = self
567 .llc_fracs
568 .iter()
569 .zip(self.llc_lats.iter())
570 .enumerate()
571 .filter(|(i, (&frac, _))| {
572 let nr_cpus = self.nr_llc_cpus.get(*i).copied().unwrap_or(0);
573 nr_cpus > 0 || frac > 0.0
574 })
575 .map(|(i, (&frac, &lat))| (i, frac, lat))
576 .collect();
577
578 if !active_llcs.is_empty() {
579 let indent = " ";
580 writeln!(w, "{indent}LLC sched%/lat_ms")?;
581 let cell_width = 14;
583 let usable = if max_width > indent.len() {
584 max_width - indent.len()
585 } else {
586 60
587 };
588 let cells_per_row = (usable / cell_width).max(1);
589
590 for (col, &(llc_id, frac, lat)) in active_llcs.iter().enumerate() {
591 if col % cells_per_row == 0 {
592 if col > 0 {
593 writeln!(w)?;
594 }
595 write!(w, "{indent}")?;
596 } else {
597 write!(w, " ")?;
598 }
599 write!(w, "[{:02}]{}/{:4.1}", llc_id, fmt_pct(frac), lat * 1_000.0)?;
600 }
601 writeln!(w)?;
602 }
603 }
604
605 Ok(())
606 }
607}
608
609#[stat_doc]
610#[derive(Clone, Debug, Default, Serialize, Deserialize, Stats)]
611#[stat(top)]
612pub struct SysStats {
613 #[stat(desc = "timestamp", _om_skip)]
614 pub at: f64,
615 #[stat(desc = "# of NUMA nodes")]
616 pub nr_nodes: usize,
617 #[stat(desc = "# sched events during the period")]
618 pub total: u64,
619 #[stat(desc = "% dispatched directly into an idle CPU from select_cpu")]
620 pub local_sel: f64,
621 #[stat(desc = "% dispatched directly into an idle CPU from enqueue")]
622 pub local_enq: f64,
623 #[stat(desc = "% open layer tasks scheduled into allocated but idle CPUs")]
624 pub open_idle: f64,
625 #[stat(desc = "% violated config due to CPU affinity")]
626 pub affn_viol: f64,
627 #[stat(desc = "% sent to hi fallback DSQs")]
628 pub hi_fb: f64,
629 #[stat(desc = "% sent to lo fallback DSQs")]
630 pub lo_fb: f64,
631 #[stat(desc = "count of times an excl task skipped a CPU as the sibling was also excl")]
632 pub excl_collision: f64,
633 #[stat(desc = "count of times a sibling CPU was preempted for an excl task")]
634 pub excl_preempt: f64,
635 #[stat(desc = "count of times a CPU skipped dispatching due to an excl task on the sibling")]
636 pub excl_idle: f64,
637 #[stat(
638 desc = "count of times an idle sibling CPU was woken up after an excl task is finished"
639 )]
640 pub excl_wakeup: f64,
641 #[stat(desc = "CPU time this binary consumed during the period")]
642 pub proc_ms: u64,
643 #[stat(desc = "CPU busy % (100% means all CPU)")]
644 pub busy: f64,
645 #[stat(desc = "CPU util % (100% means one CPU)")]
646 pub util: f64,
647 #[stat(desc = "CPU util % used by hi fallback DSQs")]
648 pub hi_fb_util: f64,
649 #[stat(desc = "CPU util % used by lo fallback DSQs")]
650 pub lo_fb_util: f64,
651 #[stat(desc = "Number of tasks dispatched via antistall")]
652 pub antistall: u64,
653 #[stat(desc = "Number of times preemptions of non-scx tasks were avoided")]
654 pub skip_preempt: u64,
655 #[stat(desc = "Number of times vtime was out of range and fixed up")]
656 pub fixup_vtime: u64,
657 #[stat(desc = "Number of times cpuc->preempting_task didn't come on the CPU")]
658 pub preempting_mismatch: u64,
659 #[stat(desc = "per-node fallback CPUs")]
660 pub fallback_cpus: BTreeMap<u32, u32>,
661 #[stat(desc = "per-layer statistics")]
662 pub fallback_cpu_util: f64,
663 #[stat(desc = "fallback CPU util %")]
664 pub layers: BTreeMap<String, LayerStats>,
665 #[stat(desc = "Number of gpu tasks affinitized since scheduler start")]
666 pub gpu_tasks_affinitized: u64,
667 #[stat(desc = "Time (in ms) of last affinitization run.")]
668 pub gpu_task_affinitization_ms: u64,
669 #[stat(desc = "System CPU utilization EWMA (10s window)")]
670 pub system_cpu_util_ewma: f64,
671}
672
673impl SysStats {
674 pub fn new(
675 stats: &Stats,
676 bstats: &BpfStats,
677 fallback_cpus: &BTreeMap<usize, usize>,
678 ) -> Result<Self> {
679 let lsum = |idx| stats.bpf_stats.lstats_sums[idx];
680 let total = lsum(LSTAT_SEL_LOCAL)
681 + lsum(LSTAT_ENQ_LOCAL)
682 + lsum(LSTAT_ENQ_WAKEUP)
683 + lsum(LSTAT_ENQ_EXPIRE)
684 + lsum(LSTAT_ENQ_REENQ)
685 + lsum(LSTAT_KEEP);
686 let lsum_pct = |idx| {
687 if total != 0 {
688 lsum(idx) as f64 / total as f64 * 100.0
689 } else {
690 0.0
691 }
692 };
693
694 let elapsed_ns = stats.elapsed.as_nanos();
695
696 Ok(Self {
697 at: SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs_f64(),
698 nr_nodes: stats.topo.nodes.len(),
699 total,
700 local_sel: lsum_pct(LSTAT_SEL_LOCAL),
701 local_enq: lsum_pct(LSTAT_ENQ_LOCAL),
702 open_idle: lsum_pct(LSTAT_OPEN_IDLE),
703 affn_viol: lsum_pct(LSTAT_AFFN_VIOL),
704 hi_fb: calc_frac(
705 stats.bpf_stats.gstats[GSTAT_HI_FB_EVENTS] as f64,
706 total as f64,
707 ),
708 lo_fb: calc_frac(
709 stats.bpf_stats.gstats[GSTAT_LO_FB_EVENTS] as f64,
710 total as f64,
711 ),
712 excl_collision: lsum_pct(LSTAT_EXCL_COLLISION),
713 excl_preempt: lsum_pct(LSTAT_EXCL_PREEMPT),
714 excl_idle: bstats.gstats[GSTAT_EXCL_IDLE] as f64 / total as f64,
715 excl_wakeup: bstats.gstats[GSTAT_EXCL_WAKEUP] as f64 / total as f64,
716 proc_ms: stats.processing_dur.as_millis() as u64,
717 busy: stats.cpu_busy * 100.0,
718 util: stats.total_util * 100.0,
719 hi_fb_util: stats.bpf_stats.gstats[GSTAT_HI_FB_USAGE] as f64 / elapsed_ns as f64
720 * 100.0,
721 lo_fb_util: stats.bpf_stats.gstats[GSTAT_LO_FB_USAGE] as f64 / elapsed_ns as f64
722 * 100.0,
723 antistall: stats.bpf_stats.gstats[GSTAT_ANTISTALL],
724 skip_preempt: stats.bpf_stats.gstats[GSTAT_SKIP_PREEMPT],
725 fixup_vtime: stats.bpf_stats.gstats[GSTAT_FIXUP_VTIME],
726 preempting_mismatch: stats.bpf_stats.gstats[GSTAT_PREEMPTING_MISMATCH],
727 fallback_cpus: fallback_cpus
728 .iter()
729 .map(|(&k, &v)| (k as u32, v as u32))
730 .collect(),
731 fallback_cpu_util: stats.bpf_stats.gstats[GSTAT_FB_CPU_USAGE] as f64
732 / elapsed_ns as f64
733 * 100.0,
734 layers: BTreeMap::new(),
735 gpu_tasks_affinitized: stats.gpu_tasks_affinitized,
736 gpu_task_affinitization_ms: stats.gpu_task_affinitization_ms,
737 system_cpu_util_ewma: stats.system_cpu_util_ewma * 100.0,
738 })
739 }
740
741 pub fn format<W: Write>(&self, w: &mut W) -> Result<()> {
742 writeln!(
743 w,
744 "tot={:7} local_sel/enq={}/{} open_idle={} affn_viol={} hi/lo={}/{}",
745 self.total,
746 fmt_pct(self.local_sel),
747 fmt_pct(self.local_enq),
748 fmt_pct(self.open_idle),
749 fmt_pct(self.affn_viol),
750 fmt_pct(self.hi_fb),
751 fmt_pct(self.lo_fb),
752 )?;
753
754 let single_node = self.fallback_cpus.len() == 1;
755 let fb_cpus_str: Vec<String> = self
756 .fallback_cpus
757 .iter()
758 .map(|(n, c)| {
759 if single_node {
760 format!("{}", c)
761 } else {
762 format!("N{}:{}", n, c)
763 }
764 })
765 .collect();
766 writeln!(
767 w,
768 "busy={:5.1} util/hi/lo={:7.1}/{}/{} fb_cpus=[{}]/util={:4.1} proc={}ms sys_util_10s={:5.1}",
769 self.busy,
770 self.util,
771 fmt_pct(self.hi_fb_util),
772 fmt_pct(self.lo_fb_util),
773 fb_cpus_str.join(","),
774 self.fallback_cpu_util,
775 self.proc_ms,
776 self.system_cpu_util_ewma,
777 )?;
778
779 writeln!(
780 w,
781 "excl_coll={:.2} excl_preempt={:.2} excl_idle={:.2} excl_wakeup={:.2}",
782 self.excl_collision, self.excl_preempt, self.excl_idle, self.excl_wakeup
783 )?;
784
785 writeln!(
786 w,
787 "skip_preempt={} antistall={} fixup_vtime={} preempting_mismatch={}",
788 self.skip_preempt, self.antistall, self.fixup_vtime, self.preempting_mismatch
789 )?;
790
791 writeln!(
792 w,
793 "gpu_tasks_affinitized={} gpu_task_affinitization_time={}",
794 self.gpu_tasks_affinitized, self.gpu_task_affinitization_ms
795 )?;
796
797 Ok(())
798 }
799
800 pub fn format_all<W: Write>(
801 &self,
802 w: &mut W,
803 topo: Option<&Topology>,
804 max_width: usize,
805 no_llc: bool,
806 ) -> Result<()> {
807 self.format(w)?;
808
809 let mut idx_to_name: Vec<(usize, &String)> =
810 self.layers.iter().map(|(k, v)| (v.index, k)).collect();
811
812 idx_to_name.sort();
813
814 for (_idx, name) in &idx_to_name {
815 self.layers[*name].format(w, name, topo, max_width, no_llc)?;
816 }
817
818 Ok(())
819 }
820}
821
822#[derive(Debug)]
823pub enum StatsReq {
824 Hello(ThreadId),
825 Refresh(ThreadId, Box<Stats>),
826 Bye(ThreadId),
827}
828
829#[derive(Debug)]
830pub enum StatsRes {
831 Hello(Box<Stats>),
832 Refreshed(Box<(Stats, SysStats)>),
833 Bye,
834}
835
836pub fn server_data() -> StatsServerData<StatsReq, StatsRes> {
837 let open: Box<dyn StatsOpener<StatsReq, StatsRes>> = Box::new(move |(req_ch, res_ch)| {
838 let tid = current().id();
839 req_ch.send(StatsReq::Hello(tid))?;
840 let mut stats = Some(match res_ch.recv()? {
841 StatsRes::Hello(v) => *v,
842 res => bail!("invalid response to Hello: {:?}", res),
843 });
844
845 let read: Box<dyn StatsReader<StatsReq, StatsRes>> =
846 Box::new(move |_args, (req_ch, res_ch)| {
847 req_ch.send(StatsReq::Refresh(tid, Box::new(stats.take().unwrap())))?;
848 let (new_stats, sys_stats) = match res_ch.recv()? {
849 StatsRes::Refreshed(v) => *v,
850 res => bail!("invalid response to Refresh: {:?}", res),
851 };
852 stats = Some(new_stats);
853 sys_stats.to_json()
854 });
855
856 Ok(read)
857 });
858
859 let close: Box<dyn StatsCloser<StatsReq, StatsRes>> = Box::new(move |(req_ch, res_ch)| {
860 req_ch.send(StatsReq::Bye(current().id())).unwrap();
861 match res_ch.recv().unwrap() {
862 StatsRes::Bye => {}
863 res => panic!("invalid response to Bye: {:?}", res),
864 }
865 });
866
867 StatsServerData::new()
868 .add_meta(LayerStats::meta())
869 .add_meta(SysStats::meta())
870 .add_ops(
871 "top",
872 StatsOps {
873 open,
874 close: Some(close),
875 },
876 )
877}
878
879pub fn monitor(
880 intv: Duration,
881 shutdown: Arc<AtomicBool>,
882 max_width: usize,
883 no_llc: bool,
884) -> Result<()> {
885 let topo = Topology::new().ok();
886 scx_utils::monitor_stats::<SysStats>(
887 &[],
888 intv,
889 || shutdown.load(Ordering::Relaxed),
890 |sst| {
891 let dt = DateTime::<Local>::from(UNIX_EPOCH + Duration::from_secs_f64(sst.at));
892 let header = format!("\u{2501}\u{2501} {} ", dt.to_rfc2822());
893 let pad = max_width.saturating_sub(header.chars().count());
894 println!("{}{}", header, "\u{2501}".repeat(pad));
895 sst.format_all(&mut std::io::stdout(), topo.as_ref(), max_width, no_llc)
896 },
897 )
898}