1use std::collections::BTreeMap;
2use std::io::Write;
3use std::sync::Arc;
4use std::sync::atomic::AtomicBool;
5use std::sync::atomic::Ordering;
6use std::thread::ThreadId;
7use std::thread::current;
8use std::time::Duration;
9use std::time::SystemTime;
10use std::time::UNIX_EPOCH;
11
12use anyhow::Result;
13use anyhow::bail;
14use chrono::DateTime;
15use chrono::Local;
16use scx_stats::prelude::*;
17use scx_stats_derive::Stats;
18use scx_stats_derive::stat_doc;
19use scx_utils::Cpumask;
20use scx_utils::Topology;
21use serde::Deserialize;
22use serde::Serialize;
23use tracing::warn;
24
25use crate::BpfStats;
26use crate::LAYER_USAGE_OPEN;
27use crate::LAYER_USAGE_PROTECTED;
28use crate::LAYER_USAGE_PROTECTED_PREEMPT;
29use crate::LAYER_USAGE_SUM_UPTO;
30use crate::Layer;
31use crate::LayerKind;
32use crate::Stats;
33use crate::bpf_intf;
34
35const GSTAT_EXCL_IDLE: usize = bpf_intf::global_stat_id_GSTAT_EXCL_IDLE as usize;
36const GSTAT_EXCL_WAKEUP: usize = bpf_intf::global_stat_id_GSTAT_EXCL_WAKEUP as usize;
37const GSTAT_HI_FB_EVENTS: usize = bpf_intf::global_stat_id_GSTAT_HI_FB_EVENTS as usize;
38const GSTAT_HI_FB_USAGE: usize = bpf_intf::global_stat_id_GSTAT_HI_FB_USAGE as usize;
39const GSTAT_LO_FB_EVENTS: usize = bpf_intf::global_stat_id_GSTAT_LO_FB_EVENTS as usize;
40const GSTAT_LO_FB_USAGE: usize = bpf_intf::global_stat_id_GSTAT_LO_FB_USAGE as usize;
41const GSTAT_FB_CPU_USAGE: usize = bpf_intf::global_stat_id_GSTAT_FB_CPU_USAGE as usize;
42const GSTAT_ANTISTALL: usize = bpf_intf::global_stat_id_GSTAT_ANTISTALL as usize;
43const GSTAT_SKIP_PREEMPT: usize = bpf_intf::global_stat_id_GSTAT_SKIP_PREEMPT as usize;
44const GSTAT_FIXUP_VTIME: usize = bpf_intf::global_stat_id_GSTAT_FIXUP_VTIME as usize;
45const GSTAT_PREEMPTING_MISMATCH: usize =
46 bpf_intf::global_stat_id_GSTAT_PREEMPTING_MISMATCH as usize;
47
48const LSTAT_SEL_LOCAL: usize = bpf_intf::layer_stat_id_LSTAT_SEL_LOCAL as usize;
49const LSTAT_ENQ_LOCAL: usize = bpf_intf::layer_stat_id_LSTAT_ENQ_LOCAL as usize;
50const LSTAT_ENQ_WAKEUP: usize = bpf_intf::layer_stat_id_LSTAT_ENQ_WAKEUP as usize;
51const LSTAT_ENQ_EXPIRE: usize = bpf_intf::layer_stat_id_LSTAT_ENQ_EXPIRE as usize;
52const LSTAT_ENQ_REENQ: usize = bpf_intf::layer_stat_id_LSTAT_ENQ_REENQ as usize;
53const LSTAT_ENQ_DSQ: usize = bpf_intf::layer_stat_id_LSTAT_ENQ_DSQ as usize;
54const LSTAT_MIN_EXEC: usize = bpf_intf::layer_stat_id_LSTAT_MIN_EXEC as usize;
55const LSTAT_MIN_EXEC_NS: usize = bpf_intf::layer_stat_id_LSTAT_MIN_EXEC_NS as usize;
56const LSTAT_OPEN_IDLE: usize = bpf_intf::layer_stat_id_LSTAT_OPEN_IDLE as usize;
57const LSTAT_AFFN_VIOL: usize = bpf_intf::layer_stat_id_LSTAT_AFFN_VIOL as usize;
58const LSTAT_KEEP: usize = bpf_intf::layer_stat_id_LSTAT_KEEP as usize;
59const LSTAT_KEEP_FAIL_MAX_EXEC: usize = bpf_intf::layer_stat_id_LSTAT_KEEP_FAIL_MAX_EXEC as usize;
60const LSTAT_KEEP_FAIL_BUSY: usize = bpf_intf::layer_stat_id_LSTAT_KEEP_FAIL_BUSY as usize;
61const LSTAT_PREEMPT: usize = bpf_intf::layer_stat_id_LSTAT_PREEMPT as usize;
62const LSTAT_PREEMPT_FIRST: usize = bpf_intf::layer_stat_id_LSTAT_PREEMPT_FIRST as usize;
63const LSTAT_PREEMPT_XLLC: usize = bpf_intf::layer_stat_id_LSTAT_PREEMPT_XLLC as usize;
64const LSTAT_PREEMPT_XNUMA: usize = bpf_intf::layer_stat_id_LSTAT_PREEMPT_XNUMA as usize;
65const LSTAT_PREEMPT_IDLE: usize = bpf_intf::layer_stat_id_LSTAT_PREEMPT_IDLE as usize;
66const LSTAT_PREEMPT_FAIL: usize = bpf_intf::layer_stat_id_LSTAT_PREEMPT_FAIL as usize;
67const LSTAT_EXCL_COLLISION: usize = bpf_intf::layer_stat_id_LSTAT_EXCL_COLLISION as usize;
68const LSTAT_EXCL_PREEMPT: usize = bpf_intf::layer_stat_id_LSTAT_EXCL_PREEMPT as usize;
69const LSTAT_YIELD: usize = bpf_intf::layer_stat_id_LSTAT_YIELD as usize;
70const LSTAT_YIELD_IGNORE: usize = bpf_intf::layer_stat_id_LSTAT_YIELD_IGNORE as usize;
71const LSTAT_MIGRATION: usize = bpf_intf::layer_stat_id_LSTAT_MIGRATION as usize;
72const LSTAT_XNUMA_MIGRATION: usize = bpf_intf::layer_stat_id_LSTAT_XNUMA_MIGRATION as usize;
73const LSTAT_XLLC_MIGRATION: usize = bpf_intf::layer_stat_id_LSTAT_XLLC_MIGRATION as usize;
74const LSTAT_XLLC_MIGRATION_SKIP: usize = bpf_intf::layer_stat_id_LSTAT_XLLC_MIGRATION_SKIP as usize;
75const LSTAT_XLAYER_WAKE: usize = bpf_intf::layer_stat_id_LSTAT_XLAYER_WAKE as usize;
76const LSTAT_XLAYER_REWAKE: usize = bpf_intf::layer_stat_id_LSTAT_XLAYER_REWAKE as usize;
77const LSTAT_LLC_DRAIN_TRY: usize = bpf_intf::layer_stat_id_LSTAT_LLC_DRAIN_TRY as usize;
78const LSTAT_LLC_DRAIN: usize = bpf_intf::layer_stat_id_LSTAT_LLC_DRAIN as usize;
79const LSTAT_SKIP_REMOTE_NODE: usize = bpf_intf::layer_stat_id_LSTAT_SKIP_REMOTE_NODE as usize;
80
81const LSTAT_RUNQ_LAT_BASE: usize = bpf_intf::layer_stat_id_LSTAT_RUNQ_LAT_BASE as usize;
82const NR_RUNQ_LAT_BUCKETS: usize = bpf_intf::consts_NR_RUNQ_LAT_BUCKETS as usize;
83
84const LLC_LSTAT_LAT: usize = bpf_intf::llc_layer_stat_id_LLC_LSTAT_LAT as usize;
85const LLC_LSTAT_CNT: usize = bpf_intf::llc_layer_stat_id_LLC_LSTAT_CNT as usize;
86
87fn calc_frac(a: f64, b: f64) -> f64 {
88 if b != 0.0 { a / b * 100.0 } else { 0.0 }
89}
90
91fn fmt_pct(v: f64) -> String {
92 if v >= 99.95 {
93 format!("{:4.0}", v)
94 } else if v >= 10.0 {
95 format!("{:4.1}", v)
96 } else if v > 0.0 && v < 0.01 {
97 format!("{:4.2}", 0.01)
98 } else {
99 format!("{:4.2}", v)
100 }
101}
102
103fn fmt_duration_ms(ms: f64) -> String {
104 if ms >= 60_000.0 {
105 let min = ms / 60_000.0;
106 if min >= 100.0 {
107 format!("{:.0}min", min)
108 } else {
109 format!("{:.1}min", min)
110 }
111 } else if ms >= 1_000.0 {
112 let s = ms / 1_000.0;
113 if s >= 100.0 {
114 format!("{:.0}s", s)
115 } else {
116 format!("{:.1}s", s)
117 }
118 } else if ms >= 10.0 {
119 format!("{:.0}ms", ms)
120 } else {
121 format!("{:.1}ms", ms)
122 }
123}
124
125fn fmt_num(v: u64) -> String {
126 if v > 1_000_000 {
127 format!("{:5.1}m", v as f64 / 1_000_000.0)
128 } else if v > 1_000 {
129 format!("{:5.1}k", v as f64 / 1_000.0)
130 } else {
131 format!("{:5.0} ", v)
132 }
133}
134
135#[stat_doc]
136#[derive(Clone, Debug, Default, Serialize, Deserialize, Stats)]
137#[stat(_om_prefix = "l_", _om_label = "layer_name")]
138pub struct LayerStats {
139 #[stat(desc = "index", _om_skip)]
140 pub index: usize,
141 #[stat(desc = "Total CPU utilization (100% means one full CPU)")]
142 pub util: f64,
143 #[stat(desc = "Compensated CPU utilization (adjusted for irq/softirq/stolen)")]
144 pub util_compensated: f64,
145 #[stat(desc = "Protected CPU utilization %")]
146 pub util_protected_frac: f64,
147 #[stat(desc = "Preempt-protected CPU utilization %")]
148 pub util_protected_preempt_frac: f64,
149 #[stat(desc = "Open CPU utilization %")]
150 pub util_open_frac: f64,
151 #[stat(desc = "fraction of total CPU utilization")]
152 pub util_frac: f64,
153 #[stat(desc = "number of tasks")]
154 pub tasks: u32,
155 #[stat(desc = "count of sched events during the period")]
156 pub total: u64,
157 #[stat(desc = "% dispatched into idle CPU from select_cpu")]
158 pub sel_local: f64,
159 #[stat(desc = "% dispatched into idle CPU from enqueue")]
160 pub enq_local: f64,
161 #[stat(desc = "% enqueued after wakeup")]
162 pub enq_wakeup: f64,
163 #[stat(desc = "% enqueued after slice expiration")]
164 pub enq_expire: f64,
165 #[stat(desc = "% re-enqueued due to RT preemption")]
166 pub enq_reenq: f64,
167 #[stat(desc = "% enqueued into the layer's LLC DSQs")]
168 pub enq_dsq: f64,
169 #[stat(desc = "count of times exec duration < min_exec_us")]
170 pub min_exec: f64,
171 #[stat(desc = "total exec durations extended due to min_exec_us")]
172 pub min_exec_us: u64,
173 #[stat(desc = "% dispatched into idle CPUs occupied by other layers")]
174 pub open_idle: f64,
175 #[stat(desc = "% preempted other tasks")]
176 pub preempt: f64,
177 #[stat(desc = "% preempted XLLC tasks")]
178 pub preempt_xllc: f64,
179 #[stat(desc = "% preempted across NUMA nodes")]
180 pub preempt_xnuma: f64,
181 #[stat(desc = "% first-preempted other tasks")]
182 pub preempt_first: f64,
183 #[stat(desc = "% idle-preempted other tasks")]
184 pub preempt_idle: f64,
185 #[stat(desc = "% attempted to preempt other tasks but failed")]
186 pub preempt_fail: f64,
187 #[stat(desc = "% violated config due to CPU affinity")]
188 pub affn_viol: f64,
189 #[stat(desc = "% continued executing after slice expiration")]
190 pub keep: f64,
191 #[stat(desc = "% disallowed to continue executing due to max_exec")]
192 pub keep_fail_max_exec: f64,
193 #[stat(desc = "% disallowed to continue executing due to other tasks")]
194 pub keep_fail_busy: f64,
195 #[stat(desc = "whether is exclusive", _om_skip)]
196 pub is_excl: u32,
197 #[stat(desc = "count of times an excl task skipped a CPU as the sibling was also excl")]
198 pub excl_collision: f64,
199 #[stat(desc = "% a sibling CPU was preempted for an exclusive task")]
200 pub excl_preempt: f64,
201 #[stat(desc = "% yielded")]
202 pub yielded: f64,
203 #[stat(desc = "count of times yield was ignored")]
204 pub yield_ignore: u64,
205 #[stat(desc = "% migrated across CPUs")]
206 pub migration: f64,
207 #[stat(desc = "% migrated across NUMA nodes")]
208 pub xnuma_migration: f64,
209 #[stat(desc = "% migrated across LLCs")]
210 pub xllc_migration: f64,
211 #[stat(desc = "% migration skipped across LLCs due to xllc_mig_min_us")]
212 pub xllc_migration_skip: f64,
213 #[stat(desc = "% wakers across layers")]
214 pub xlayer_wake: f64,
215 #[stat(desc = "% rewakers across layers where waker has waken the task previously")]
216 pub xlayer_rewake: f64,
217 #[stat(desc = "% LLC draining tried")]
218 pub llc_drain_try: f64,
219 #[stat(desc = "% LLC draining succeeded")]
220 pub llc_drain: f64,
221 #[stat(desc = "% skip LLC dispatch on remote node")]
222 pub skip_remote_node: f64,
223 #[stat(desc = "mask of allocated CPUs", _om_skip)]
224 pub cpus: Vec<u64>,
225 #[stat(desc = "count of CPUs assigned")]
226 pub cur_nr_cpus: u32,
227 #[stat(desc = "minimum # of CPUs assigned")]
228 pub min_nr_cpus: u32,
229 #[stat(desc = "maximum # of CPUs assigned")]
230 pub max_nr_cpus: u32,
231 #[stat(desc = "count of CPUs assigned per LLC")]
232 pub nr_llc_cpus: Vec<u32>,
233 #[stat(desc = "slice duration config")]
234 pub slice_us: u64,
235 #[stat(desc = "Per-LLC scheduling event fractions")]
236 pub llc_fracs: Vec<f64>,
237 #[stat(desc = "Per-LLC average latency")]
238 pub llc_lats: Vec<f64>,
239 #[stat(desc = "Layer memory bandwidth as a % of total allowed (0 for \"no limit\"")]
240 pub membw_pct: f64,
241 #[stat(desc = "DSQ insertion ratio EWMA (10s window)")]
242 pub dsq_insert_ewma: f64,
243 #[stat(desc = "Per-node layer utilization (100% = one full CPU)")]
244 pub node_utils: Vec<f64>,
245 #[stat(desc = "Per-node pinned task utilization (100% = one full CPU)")]
246 pub node_pinned_utils: Vec<f64>,
247 #[stat(desc = "Per-node pinned task counts")]
248 pub node_pinned_tasks: Vec<u64>,
249 #[stat(desc = "Per-node load (100% = one full CPU, from duty cycle sum)")]
250 pub node_loads: Vec<f64>,
251 #[stat(desc = "Whether xnuma gating is active for this layer (0/1)")]
252 pub xnuma_active: u32,
253 #[stat(desc = "runqueue latency histogram, log2 us buckets 1us..32s, per stats interval")]
254 pub l_runq_lat_hist: Vec<u64>,
255}
256
257impl LayerStats {
258 pub fn new(
259 lidx: usize,
260 layer: &Layer,
261 stats: &Stats,
262 bstats: &BpfStats,
263 nr_cpus_range: (usize, usize),
264 xnuma_active: bool,
265 ) -> Self {
266 let lstat = |sidx| bstats.lstats[lidx][sidx];
267 let ltotal = lstat(LSTAT_SEL_LOCAL)
268 + lstat(LSTAT_ENQ_LOCAL)
269 + lstat(LSTAT_ENQ_WAKEUP)
270 + lstat(LSTAT_ENQ_EXPIRE)
271 + lstat(LSTAT_ENQ_REENQ)
272 + lstat(LSTAT_KEEP);
273 let lstat_pct = |sidx| {
274 if ltotal != 0 {
275 lstat(sidx) as f64 / ltotal as f64 * 100.0
276 } else {
277 0.0
278 }
279 };
280
281 let util_sum = stats.layer_utils[lidx]
282 .iter()
283 .take(LAYER_USAGE_SUM_UPTO + 1)
284 .sum::<f64>();
285
286 let util_comp_sum = stats.layer_utils_compensated[lidx]
287 .iter()
288 .take(LAYER_USAGE_SUM_UPTO + 1)
289 .sum::<f64>();
290
291 let membw_frac = match &layer.kind {
292 LayerKind::Open { .. } => 0.0,
294 LayerKind::Confined { membw_gb, .. } | LayerKind::Grouped { membw_gb, .. } => {
295 if let Some(membw_limit_gb) = membw_gb {
297 stats.layer_membws[lidx]
298 .iter()
299 .take(LAYER_USAGE_SUM_UPTO + 1)
300 .sum::<f64>()
301 / (*membw_limit_gb * (1024_u64.pow(3) as f64))
302 } else {
303 0.0
304 }
305 }
306 };
307
308 Self {
309 index: lidx,
310 util: util_sum * 100.0,
311 util_compensated: util_comp_sum * 100.0,
312 util_open_frac: calc_frac(stats.layer_utils[lidx][LAYER_USAGE_OPEN], util_sum),
313 util_protected_frac: calc_frac(
314 stats.layer_utils[lidx][LAYER_USAGE_PROTECTED],
315 util_sum,
316 ),
317 util_protected_preempt_frac: calc_frac(
318 stats.layer_utils[lidx][LAYER_USAGE_PROTECTED_PREEMPT],
319 util_sum,
320 ),
321 util_frac: calc_frac(util_sum, stats.total_util),
322 tasks: stats.nr_layer_tasks[lidx] as u32,
323 total: ltotal,
324 sel_local: lstat_pct(LSTAT_SEL_LOCAL),
325 enq_local: lstat_pct(LSTAT_ENQ_LOCAL),
326 enq_wakeup: lstat_pct(LSTAT_ENQ_WAKEUP),
327 enq_expire: lstat_pct(LSTAT_ENQ_EXPIRE),
328 enq_reenq: lstat_pct(LSTAT_ENQ_REENQ),
329 enq_dsq: lstat_pct(LSTAT_ENQ_DSQ),
330 min_exec: lstat_pct(LSTAT_MIN_EXEC),
331 min_exec_us: lstat(LSTAT_MIN_EXEC_NS) / 1000,
332 open_idle: lstat_pct(LSTAT_OPEN_IDLE),
333 preempt: lstat_pct(LSTAT_PREEMPT),
334 preempt_xllc: lstat_pct(LSTAT_PREEMPT_XLLC),
335 preempt_xnuma: lstat_pct(LSTAT_PREEMPT_XNUMA),
336 preempt_first: lstat_pct(LSTAT_PREEMPT_FIRST),
337 preempt_idle: lstat_pct(LSTAT_PREEMPT_IDLE),
338 preempt_fail: lstat_pct(LSTAT_PREEMPT_FAIL),
339 affn_viol: lstat_pct(LSTAT_AFFN_VIOL),
340 keep: lstat_pct(LSTAT_KEEP),
341 keep_fail_max_exec: lstat_pct(LSTAT_KEEP_FAIL_MAX_EXEC),
342 keep_fail_busy: lstat_pct(LSTAT_KEEP_FAIL_BUSY),
343 is_excl: layer.kind.common().exclusive as u32,
344 excl_collision: lstat_pct(LSTAT_EXCL_COLLISION),
345 excl_preempt: lstat_pct(LSTAT_EXCL_PREEMPT),
346 yielded: lstat_pct(LSTAT_YIELD),
347 yield_ignore: lstat(LSTAT_YIELD_IGNORE),
348 migration: lstat_pct(LSTAT_MIGRATION),
349 xnuma_migration: lstat_pct(LSTAT_XNUMA_MIGRATION),
350 xlayer_wake: lstat_pct(LSTAT_XLAYER_WAKE),
351 xlayer_rewake: lstat_pct(LSTAT_XLAYER_REWAKE),
352 xllc_migration: lstat_pct(LSTAT_XLLC_MIGRATION),
353 xllc_migration_skip: lstat_pct(LSTAT_XLLC_MIGRATION_SKIP),
354 llc_drain_try: lstat_pct(LSTAT_LLC_DRAIN_TRY),
355 llc_drain: lstat_pct(LSTAT_LLC_DRAIN),
356 skip_remote_node: lstat_pct(LSTAT_SKIP_REMOTE_NODE),
357 cpus: layer.cpus.as_raw_slice().to_vec(),
358 cur_nr_cpus: layer.cpus.weight() as u32,
359 min_nr_cpus: nr_cpus_range.0 as u32,
360 max_nr_cpus: nr_cpus_range.1 as u32,
361 nr_llc_cpus: layer.nr_llc_cpus.iter().map(|&v| v as u32).collect(),
362 slice_us: stats.layer_slice_us[lidx],
363 llc_fracs: {
364 let sid = LLC_LSTAT_CNT;
365 let sum = bstats.llc_lstats[lidx]
366 .iter()
367 .map(|lstats| lstats[sid])
368 .sum::<u64>() as f64;
369 bstats.llc_lstats[lidx]
370 .iter()
371 .map(|lstats| calc_frac(lstats[sid] as f64, sum))
372 .collect()
373 },
374 llc_lats: bstats.llc_lstats[lidx]
375 .iter()
376 .map(|lstats| lstats[LLC_LSTAT_LAT] as f64 / 1_000_000_000.0)
377 .collect(),
378 membw_pct: membw_frac * 100.0,
379 dsq_insert_ewma: stats.layer_dsq_insert_ewma[lidx] * 100.0,
380 node_utils: stats.layer_node_utils[lidx]
381 .iter()
382 .map(|u| u * 100.0)
383 .collect(),
384 node_pinned_utils: stats.layer_node_pinned_utils[lidx]
385 .iter()
386 .map(|u| u * 100.0)
387 .collect(),
388 node_pinned_tasks: stats.layer_nr_node_pinned_tasks[lidx].clone(),
389 node_loads: stats.layer_node_duty_sums[lidx]
390 .iter()
391 .map(|l| l * 100.0)
392 .collect(),
393 xnuma_active: if xnuma_active { 1 } else { 0 },
394 l_runq_lat_hist: (0..NR_RUNQ_LAT_BUCKETS)
395 .map(|b| lstat(LSTAT_RUNQ_LAT_BASE + b))
396 .collect(),
397 }
398 }
399
400 pub fn format<W: Write>(
401 &self,
402 w: &mut W,
403 name: &str,
404 topo: Option<&Topology>,
405 max_width: usize,
406 no_llc: bool,
407 ) -> Result<()> {
408 let comp_str = if self.util > 0.1 && (self.util_compensated - self.util).abs() > 0.1 {
410 let overhead_pct = (1.0 - self.util / self.util_compensated) * 100.0;
411 format!(" comp_overhead={:.1}%", overhead_pct)
412 } else {
413 String::new()
414 };
415 writeln!(
416 w,
417 "\n\u{25B6} {} \u{2500} util/open/frac={:6.1}/{}/{:7.1}{} prot/prot_preempt={}/{} tasks={:6}",
418 name,
419 self.util,
420 fmt_pct(self.util_open_frac),
421 self.util_frac,
422 comp_str,
423 fmt_pct(self.util_protected_frac),
424 fmt_pct(self.util_protected_preempt_frac),
425 self.tasks,
426 )?;
427
428 writeln!(
430 w,
431 " {:<7} tot={} dd_sel/enq={}/{} dsq/10s={}/{} wake/exp/re={}/{}/{}",
432 "sched",
433 fmt_num(self.total),
434 fmt_pct(self.sel_local),
435 fmt_pct(self.enq_local),
436 fmt_pct(self.enq_dsq),
437 fmt_pct(self.dsq_insert_ewma),
438 fmt_pct(self.enq_wakeup),
439 fmt_pct(self.enq_expire),
440 fmt_pct(self.enq_reenq),
441 )?;
442
443 writeln!(
445 w,
446 " {:<7} keep/max/busy={}/{}/{} yield/ign={}/{} slc={} min_ex={}/{}",
447 "exec",
448 fmt_pct(self.keep),
449 fmt_pct(self.keep_fail_max_exec),
450 fmt_pct(self.keep_fail_busy),
451 fmt_pct(self.yielded),
452 fmt_num(self.yield_ignore),
453 fmt_duration_ms(self.slice_us as f64 / 1000.0),
454 fmt_pct(self.min_exec),
455 fmt_duration_ms(self.min_exec_us as f64 / 1000.0),
456 )?;
457
458 writeln!(
460 w,
461 " {:<7} mig={} xnuma={} xllc/skip={}/{} open_idle={} affn_viol={}",
462 "mig",
463 fmt_pct(self.migration),
464 fmt_pct(self.xnuma_migration),
465 fmt_pct(self.xllc_migration),
466 fmt_pct(self.xllc_migration_skip),
467 fmt_pct(self.open_idle),
468 fmt_pct(self.affn_viol),
469 )?;
470
471 writeln!(
473 w,
474 " {:<7} preempt/first/xllc/xnuma/idle/fail={}/{}/{}/{}/{}/{}",
475 "preempt",
476 fmt_pct(self.preempt),
477 fmt_pct(self.preempt_first),
478 fmt_pct(self.preempt_xllc),
479 fmt_pct(self.preempt_xnuma),
480 fmt_pct(self.preempt_idle),
481 fmt_pct(self.preempt_fail),
482 )?;
483
484 writeln!(
486 w,
487 " {:<7} wake/re={}/{} llc_drain/try={}/{} skip_rnode={}",
488 "xlayer",
489 fmt_pct(self.xlayer_wake),
490 fmt_pct(self.xlayer_rewake),
491 fmt_pct(self.llc_drain),
492 fmt_pct(self.llc_drain_try),
493 fmt_pct(self.skip_remote_node),
494 )?;
495
496 if self.node_utils.len() > 1 {
498 let prefix = " node pin/ut/ld ";
499 let cell_width = 25;
501 let usable = if max_width > prefix.len() {
502 max_width - prefix.len()
503 } else {
504 60
505 };
506 let cells_per_row = (usable / cell_width).max(1);
507
508 for nid in 0..self.node_utils.len() {
509 let util = self.node_utils[nid];
510 let load = self.node_loads.get(nid).copied().unwrap_or(0.0);
511 let pin = self.node_pinned_utils.get(nid).copied().unwrap_or(0.0);
512 if nid % cells_per_row == 0 {
513 if nid > 0 {
514 writeln!(w)?;
515 }
516 write!(w, "{prefix}")?;
517 } else {
518 write!(w, " ")?;
519 }
520 write!(w, "N{}={:5.1}/{:5.1}/{:7.1}", nid, pin, util, load)?;
521 }
522 writeln!(w)?;
523 }
524
525 let cpumask = Cpumask::from_vec(self.cpus.clone());
527
528 if let Some(topo) = topo {
529 let header = topo.format_cpumask_header(&cpumask, self.min_nr_cpus, self.max_nr_cpus);
530 writeln!(w, " {}", header)?;
531 if cpumask.weight() > 0 {
532 topo.format_cpumask_grid(w, &cpumask, " ", max_width)?;
533 }
534 } else {
535 writeln!(
536 w,
537 " cpus={:3} [{:3},{:3}] {}",
538 self.cur_nr_cpus, self.min_nr_cpus, self.max_nr_cpus, cpumask,
539 )?;
540 }
541
542 if self.is_excl != 0 {
544 writeln!(
545 w,
546 " excl_coll={} excl_preempt={}",
547 fmt_pct(self.excl_collision),
548 fmt_pct(self.excl_preempt),
549 )?;
550 } else if self.excl_collision != 0.0 || self.excl_preempt != 0.0 {
551 warn!(
552 "{}: exclusive is off but excl_coll={} excl_preempt={}",
553 name,
554 fmt_pct(self.excl_collision),
555 fmt_pct(self.excl_preempt),
556 );
557 }
558
559 if !no_llc {
561 let active_llcs: Vec<(usize, f64, f64)> = self
563 .llc_fracs
564 .iter()
565 .zip(self.llc_lats.iter())
566 .enumerate()
567 .filter(|&(i, (&frac, _))| {
568 let nr_cpus = self.nr_llc_cpus.get(i).copied().unwrap_or(0);
569 nr_cpus > 0 || frac > 0.0
570 })
571 .map(|(i, (&frac, &lat))| (i, frac, lat))
572 .collect();
573
574 if !active_llcs.is_empty() {
575 let indent = " ";
576 writeln!(w, "{indent}LLC sched%/lat_ms")?;
577 let cell_width = 14;
579 let usable = if max_width > indent.len() {
580 max_width - indent.len()
581 } else {
582 60
583 };
584 let cells_per_row = (usable / cell_width).max(1);
585
586 for (col, &(llc_id, frac, lat)) in active_llcs.iter().enumerate() {
587 if col % cells_per_row == 0 {
588 if col > 0 {
589 writeln!(w)?;
590 }
591 write!(w, "{indent}")?;
592 } else {
593 write!(w, " ")?;
594 }
595 write!(w, "[{:02}]{}/{:4.1}", llc_id, fmt_pct(frac), lat * 1_000.0)?;
596 }
597 writeln!(w)?;
598 }
599 }
600
601 Ok(())
602 }
603}
604
605#[stat_doc]
606#[derive(Clone, Debug, Default, Serialize, Deserialize, Stats)]
607#[stat(top)]
608pub struct SysStats {
609 #[stat(desc = "timestamp", _om_skip)]
610 pub at: f64,
611 #[stat(desc = "# of NUMA nodes")]
612 pub nr_nodes: usize,
613 #[stat(desc = "# sched events during the period")]
614 pub total: u64,
615 #[stat(desc = "% dispatched directly into an idle CPU from select_cpu")]
616 pub local_sel: f64,
617 #[stat(desc = "% dispatched directly into an idle CPU from enqueue")]
618 pub local_enq: f64,
619 #[stat(desc = "% open layer tasks scheduled into allocated but idle CPUs")]
620 pub open_idle: f64,
621 #[stat(desc = "% violated config due to CPU affinity")]
622 pub affn_viol: f64,
623 #[stat(desc = "% sent to hi fallback DSQs")]
624 pub hi_fb: f64,
625 #[stat(desc = "% sent to lo fallback DSQs")]
626 pub lo_fb: f64,
627 #[stat(desc = "count of times an excl task skipped a CPU as the sibling was also excl")]
628 pub excl_collision: f64,
629 #[stat(desc = "count of times a sibling CPU was preempted for an excl task")]
630 pub excl_preempt: f64,
631 #[stat(desc = "count of times a CPU skipped dispatching due to an excl task on the sibling")]
632 pub excl_idle: f64,
633 #[stat(
634 desc = "count of times an idle sibling CPU was woken up after an excl task is finished"
635 )]
636 pub excl_wakeup: f64,
637 #[stat(desc = "CPU time this binary consumed during the period")]
638 pub proc_ms: u64,
639 #[stat(desc = "CPU busy % (100% means all CPU)")]
640 pub busy: f64,
641 #[stat(desc = "CPU util % (100% means one CPU)")]
642 pub util: f64,
643 #[stat(desc = "CPU util % used by hi fallback DSQs")]
644 pub hi_fb_util: f64,
645 #[stat(desc = "CPU util % used by lo fallback DSQs")]
646 pub lo_fb_util: f64,
647 #[stat(desc = "Number of tasks dispatched via antistall")]
648 pub antistall: u64,
649 #[stat(desc = "Number of times preemptions of non-scx tasks were avoided")]
650 pub skip_preempt: u64,
651 #[stat(desc = "Number of times vtime was out of range and fixed up")]
652 pub fixup_vtime: u64,
653 #[stat(desc = "Number of times cpuc->preempting_task didn't come on the CPU")]
654 pub preempting_mismatch: u64,
655 #[stat(desc = "per-node fallback CPUs")]
656 pub fallback_cpus: BTreeMap<u32, u32>,
657 #[stat(desc = "per-layer statistics")]
658 pub fallback_cpu_util: f64,
659 #[stat(desc = "fallback CPU util %")]
660 pub layers: BTreeMap<String, LayerStats>,
661 #[stat(desc = "Number of gpu tasks affinitized since scheduler start")]
662 pub gpu_tasks_affinitized: u64,
663 #[stat(desc = "Time (in ms) of last affinitization run.")]
664 pub gpu_task_affinitization_ms: u64,
665 #[stat(desc = "System CPU utilization EWMA (10s window)")]
666 pub system_cpu_util_ewma: f64,
667}
668
669impl SysStats {
670 pub fn new(
671 stats: &Stats,
672 bstats: &BpfStats,
673 fallback_cpus: &BTreeMap<usize, usize>,
674 ) -> Result<Self> {
675 let lsum = |idx| stats.bpf_stats.lstats_sums[idx];
676 let total = lsum(LSTAT_SEL_LOCAL)
677 + lsum(LSTAT_ENQ_LOCAL)
678 + lsum(LSTAT_ENQ_WAKEUP)
679 + lsum(LSTAT_ENQ_EXPIRE)
680 + lsum(LSTAT_ENQ_REENQ)
681 + lsum(LSTAT_KEEP);
682 let lsum_pct = |idx| {
683 if total != 0 {
684 lsum(idx) as f64 / total as f64 * 100.0
685 } else {
686 0.0
687 }
688 };
689
690 let elapsed_ns = stats.elapsed.as_nanos();
691
692 Ok(Self {
693 at: SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs_f64(),
694 nr_nodes: stats.topo.nodes.len(),
695 total,
696 local_sel: lsum_pct(LSTAT_SEL_LOCAL),
697 local_enq: lsum_pct(LSTAT_ENQ_LOCAL),
698 open_idle: lsum_pct(LSTAT_OPEN_IDLE),
699 affn_viol: lsum_pct(LSTAT_AFFN_VIOL),
700 hi_fb: calc_frac(
701 stats.bpf_stats.gstats[GSTAT_HI_FB_EVENTS] as f64,
702 total as f64,
703 ),
704 lo_fb: calc_frac(
705 stats.bpf_stats.gstats[GSTAT_LO_FB_EVENTS] as f64,
706 total as f64,
707 ),
708 excl_collision: lsum_pct(LSTAT_EXCL_COLLISION),
709 excl_preempt: lsum_pct(LSTAT_EXCL_PREEMPT),
710 excl_idle: bstats.gstats[GSTAT_EXCL_IDLE] as f64 / total as f64,
711 excl_wakeup: bstats.gstats[GSTAT_EXCL_WAKEUP] as f64 / total as f64,
712 proc_ms: stats.processing_dur.as_millis() as u64,
713 busy: stats.cpu_busy * 100.0,
714 util: stats.total_util * 100.0,
715 hi_fb_util: stats.bpf_stats.gstats[GSTAT_HI_FB_USAGE] as f64 / elapsed_ns as f64
716 * 100.0,
717 lo_fb_util: stats.bpf_stats.gstats[GSTAT_LO_FB_USAGE] as f64 / elapsed_ns as f64
718 * 100.0,
719 antistall: stats.bpf_stats.gstats[GSTAT_ANTISTALL],
720 skip_preempt: stats.bpf_stats.gstats[GSTAT_SKIP_PREEMPT],
721 fixup_vtime: stats.bpf_stats.gstats[GSTAT_FIXUP_VTIME],
722 preempting_mismatch: stats.bpf_stats.gstats[GSTAT_PREEMPTING_MISMATCH],
723 fallback_cpus: fallback_cpus
724 .iter()
725 .map(|(&k, &v)| (k as u32, v as u32))
726 .collect(),
727 fallback_cpu_util: stats.bpf_stats.gstats[GSTAT_FB_CPU_USAGE] as f64
728 / elapsed_ns as f64
729 * 100.0,
730 layers: BTreeMap::new(),
731 gpu_tasks_affinitized: stats.gpu_tasks_affinitized,
732 gpu_task_affinitization_ms: stats.gpu_task_affinitization_ms,
733 system_cpu_util_ewma: stats.system_cpu_util_ewma * 100.0,
734 })
735 }
736
737 pub fn format<W: Write>(&self, w: &mut W) -> Result<()> {
738 writeln!(
739 w,
740 "tot={:7} local_sel/enq={}/{} open_idle={} affn_viol={} hi/lo={}/{}",
741 self.total,
742 fmt_pct(self.local_sel),
743 fmt_pct(self.local_enq),
744 fmt_pct(self.open_idle),
745 fmt_pct(self.affn_viol),
746 fmt_pct(self.hi_fb),
747 fmt_pct(self.lo_fb),
748 )?;
749
750 let single_node = self.fallback_cpus.len() == 1;
751 let fb_cpus_str: Vec<String> = self
752 .fallback_cpus
753 .iter()
754 .map(|(n, c)| {
755 if single_node {
756 format!("{}", c)
757 } else {
758 format!("N{}:{}", n, c)
759 }
760 })
761 .collect();
762 writeln!(
763 w,
764 "busy={:5.1} util/hi/lo={:7.1}/{}/{} fb_cpus=[{}]/util={:4.1} proc={}ms sys_util_10s={:5.1}",
765 self.busy,
766 self.util,
767 fmt_pct(self.hi_fb_util),
768 fmt_pct(self.lo_fb_util),
769 fb_cpus_str.join(","),
770 self.fallback_cpu_util,
771 self.proc_ms,
772 self.system_cpu_util_ewma,
773 )?;
774
775 writeln!(
776 w,
777 "excl_coll={:.2} excl_preempt={:.2} excl_idle={:.2} excl_wakeup={:.2}",
778 self.excl_collision, self.excl_preempt, self.excl_idle, self.excl_wakeup
779 )?;
780
781 writeln!(
782 w,
783 "skip_preempt={} antistall={} fixup_vtime={} preempting_mismatch={}",
784 self.skip_preempt, self.antistall, self.fixup_vtime, self.preempting_mismatch
785 )?;
786
787 writeln!(
788 w,
789 "gpu_tasks_affinitized={} gpu_task_affinitization_time={}",
790 self.gpu_tasks_affinitized, self.gpu_task_affinitization_ms
791 )?;
792
793 Ok(())
794 }
795
796 pub fn format_all<W: Write>(
797 &self,
798 w: &mut W,
799 topo: Option<&Topology>,
800 max_width: usize,
801 no_llc: bool,
802 ) -> Result<()> {
803 self.format(w)?;
804
805 let mut idx_to_name: Vec<(usize, &String)> =
806 self.layers.iter().map(|(k, v)| (v.index, k)).collect();
807
808 idx_to_name.sort();
809
810 for (_idx, name) in &idx_to_name {
811 self.layers[*name].format(w, name, topo, max_width, no_llc)?;
812 }
813
814 Ok(())
815 }
816}
817
818#[derive(Debug)]
819pub enum StatsReq {
820 Hello(ThreadId),
821 Refresh(ThreadId, Box<Stats>),
822 Bye(ThreadId),
823}
824
825#[derive(Debug)]
826pub enum StatsRes {
827 Hello(Box<Stats>),
828 Refreshed(Box<(Stats, SysStats)>),
829 Bye,
830}
831
832pub fn server_data() -> StatsServerData<StatsReq, StatsRes> {
833 let open: Box<dyn StatsOpener<StatsReq, StatsRes>> = Box::new(move |(req_ch, res_ch)| {
834 let tid = current().id();
835 req_ch.send(StatsReq::Hello(tid))?;
836 let mut stats = Some(match res_ch.recv()? {
837 StatsRes::Hello(v) => *v,
838 res => bail!("invalid response to Hello: {:?}", res),
839 });
840
841 let read: Box<dyn StatsReader<StatsReq, StatsRes>> =
842 Box::new(move |_args, (req_ch, res_ch)| {
843 req_ch.send(StatsReq::Refresh(tid, Box::new(stats.take().unwrap())))?;
844 let (new_stats, sys_stats) = match res_ch.recv()? {
845 StatsRes::Refreshed(v) => *v,
846 res => bail!("invalid response to Refresh: {:?}", res),
847 };
848 stats = Some(new_stats);
849 sys_stats.to_json()
850 });
851
852 Ok(read)
853 });
854
855 let close: Box<dyn StatsCloser<StatsReq, StatsRes>> = Box::new(move |(req_ch, res_ch)| {
856 req_ch.send(StatsReq::Bye(current().id())).unwrap();
857 match res_ch.recv().unwrap() {
858 StatsRes::Bye => {}
859 res => panic!("invalid response to Bye: {:?}", res),
860 }
861 });
862
863 StatsServerData::new()
864 .add_meta(LayerStats::meta())
865 .add_meta(SysStats::meta())
866 .add_ops(
867 "top",
868 StatsOps {
869 open,
870 close: Some(close),
871 },
872 )
873}
874
875pub fn monitor(
876 intv: Duration,
877 shutdown: Arc<AtomicBool>,
878 max_width: usize,
879 no_llc: bool,
880) -> Result<()> {
881 let topo = Topology::new().ok();
882 scx_utils::monitor_stats::<SysStats>(
883 &[],
884 intv,
885 || shutdown.load(Ordering::Relaxed),
886 |sst| {
887 let dt = DateTime::<Local>::from(UNIX_EPOCH + Duration::from_secs_f64(sst.at));
888 let header = format!("\u{2501}\u{2501} {} ", dt.to_rfc2822());
889 let pad = max_width.saturating_sub(header.chars().count());
890 println!("{}{}", header, "\u{2501}".repeat(pad));
891 sst.format_all(&mut std::io::stdout(), topo.as_ref(), max_width, no_llc)
892 },
893 )
894}