1use std::collections::BTreeMap;
2use std::io::Write;
3use std::sync::atomic::AtomicBool;
4use std::sync::atomic::Ordering;
5use std::sync::Arc;
6use std::time::Duration;
7
8use anyhow::Result;
9use serde::Deserialize;
10use serde::Serialize;
11
12use scx_stats::prelude::*;
13use scx_stats_derive::stat_doc;
14use scx_stats_derive::Stats;
15
16use crate::DistributionStats;
17
18#[stat_doc]
19#[derive(Clone, Debug, Default, Serialize, Deserialize, Stats)]
20#[stat(_om_prefix = "c_")]
21#[stat(top)]
22pub struct CellMetrics {
23 #[stat(desc = "Number of cpus")]
24 pub num_cpus: u32,
25 #[stat(
26 desc = "Cgroup path of this cell, relative to the cgroup root (\"/\" for the root cell)",
27 _om_skip
28 )]
29 pub cgroup_path: String,
30 #[stat(desc = "Local queue %")]
31 pub local_q_pct: f64,
32 #[stat(desc = "CPU queue %")]
33 pub cpu_q_pct: f64,
34 #[stat(desc = "Cell queue %")]
35 pub cell_q_pct: f64,
36 #[stat(desc = "Borrowed CPU %")]
37 pub borrowed_pct: f64,
38 #[stat(desc = "Affinity violations % of global")]
39 pub affn_violations_pct: f64,
40 #[stat(desc = "Steal %")]
41 pub steal_pct: f64,
42 #[stat(desc = "Orphaned LLC DSQ drain events")]
43 pub drain_cnt: u64,
44 #[stat(desc = "Pin reject skipped %")]
45 pub pin_skip_pct: f64,
46 #[stat(desc = "Slice shrink events")]
47 pub slice_shrink: u64,
48 #[stat(desc = "Slice shrink at max")]
49 pub slice_shrink_max: u64,
50 #[stat(desc = "Slice shrink proportional")]
51 pub slice_shrink_proportional: u64,
52 #[stat(desc = "Slice shrink at min")]
53 pub slice_shrink_min: u64,
54 #[stat(desc = "Decision share % of global")]
55 pub share_of_decisions_pct: f64,
56 #[stat(desc = "Cell scheduling decisions")]
57 total_decisions: u64,
58 #[stat(desc = "CPU utilization %")]
59 pub util_pct: f64,
60 #[stat(desc = "Borrowed CPU time % of running")]
61 pub demand_borrow_pct: f64,
62 #[stat(desc = "Lent CPU time %")]
63 pub lent_pct: f64,
64 #[stat(desc = "EWMA-smoothed utilization %")]
65 pub smoothed_util_pct: f64,
66}
67
68impl CellMetrics {
69 pub fn update(&mut self, ds: &DistributionStats) {
70 self.local_q_pct = ds.local_q_pct;
71 self.cpu_q_pct = ds.cpu_q_pct;
72 self.cell_q_pct = ds.cell_q_pct;
73 self.borrowed_pct = ds.borrowed_pct;
74 self.affn_violations_pct = ds.affn_viol_pct;
75 self.steal_pct = ds.steal_pct;
76 self.pin_skip_pct = ds.pin_skip_pct;
77 self.share_of_decisions_pct = ds.share_of_decisions_pct;
78 self.total_decisions = ds.total_decisions;
79 }
80
81 pub fn update_demand(&mut self, util_pct: f64, demand_borrow_pct: f64, lent_pct: f64) {
82 self.util_pct = util_pct;
83 self.demand_borrow_pct = demand_borrow_pct;
84 self.lent_pct = lent_pct;
85 }
86}
87
88#[stat_doc]
89#[derive(Clone, Debug, Default, Serialize, Deserialize, Stats)]
90#[stat(top)]
91pub struct Metrics {
92 #[stat(desc = "Number of cells")]
93 pub num_cells: u32,
94 #[stat(desc = "Local queue %")]
95 pub local_q_pct: f64,
96 #[stat(desc = "CPU queue %")]
97 pub cpu_q_pct: f64,
98 #[stat(desc = "Cell queue %")]
99 pub cell_q_pct: f64,
100 #[stat(desc = "Borrowed CPU %")]
101 pub borrowed_pct: f64,
102 #[stat(desc = "Affinity violations % of global")]
103 pub affn_violations_pct: f64,
104 #[stat(desc = "Steal %")]
105 pub steal_pct: f64,
106 #[stat(desc = "Orphaned LLC DSQ drain events")]
107 pub drain_cnt: u64,
108 #[stat(desc = "Pin reject skipped %")]
109 pub pin_skip_pct: f64,
110 #[stat(desc = "Slice shrink events")]
111 pub slice_shrink: u64,
112 #[stat(desc = "Slice shrink at max")]
113 pub slice_shrink_max: u64,
114 #[stat(desc = "Slice shrink proportional")]
115 pub slice_shrink_proportional: u64,
116 #[stat(desc = "Slice shrink at min")]
117 pub slice_shrink_min: u64,
118 #[stat(desc = "Decision share % of global")]
119 pub share_of_decisions_pct: f64,
120 #[stat(desc = "Cell scheduling decisions")]
121 total_decisions: u64,
122 #[stat(desc = "CPU utilization %")]
123 pub util_pct: f64,
124 #[stat(desc = "Borrowed CPU time % of running")]
125 pub demand_borrow_pct: f64,
126 #[stat(desc = "Lent CPU time %")]
127 pub lent_pct: f64,
128 #[stat(desc = "Number of rebalancing events")]
129 pub rebalance_count: u64,
130 #[stat(
131 desc = "1 if the cell-0 holdout has taken a CPU already claimed by a workload cell, else 0"
132 )]
133 pub enforced_holdout: u64,
134 #[stat(desc = "Per-cell metrics")]
135 pub cells: BTreeMap<u32, CellMetrics>,
136}
137
138impl Metrics {
139 pub fn update(&mut self, ds: &DistributionStats) {
140 self.local_q_pct = ds.local_q_pct;
141 self.cpu_q_pct = ds.cpu_q_pct;
142 self.cell_q_pct = ds.cell_q_pct;
143 self.borrowed_pct = ds.borrowed_pct;
144 self.affn_violations_pct = ds.affn_viol_pct;
145 self.steal_pct = ds.steal_pct;
146 self.pin_skip_pct = ds.pin_skip_pct;
147 self.share_of_decisions_pct = ds.share_of_decisions_pct;
148 self.total_decisions = ds.total_decisions;
149 }
150
151 pub fn update_demand(&mut self, util_pct: f64, demand_borrow_pct: f64, lent_pct: f64) {
152 self.util_pct = util_pct;
153 self.demand_borrow_pct = demand_borrow_pct;
154 self.lent_pct = lent_pct;
155 }
156
157 fn delta(&self, _: &Self) -> Self {
158 Self { ..self.clone() }
159 }
160
161 fn format<W: Write>(&self, w: &mut W) -> Result<()> {
162 writeln!(w, "{}", serde_json::to_string_pretty(self)?)?;
163 Ok(())
164 }
165}
166
167pub fn server_data() -> StatsServerData<(), Metrics> {
168 let open: Box<dyn StatsOpener<(), Metrics>> = Box::new(move |(req_ch, res_ch)| {
169 req_ch.send(())?;
170 let mut prev = res_ch.recv()?;
171
172 let read: Box<dyn StatsReader<(), Metrics>> = Box::new(move |_args, (req_ch, res_ch)| {
173 req_ch.send(())?;
174 let cur = res_ch.recv()?;
175 let delta = cur.delta(&prev);
176 prev = cur;
177 delta.to_json()
178 });
179
180 Ok(read)
181 });
182
183 StatsServerData::new()
184 .add_meta(Metrics::meta())
185 .add_meta(CellMetrics::meta())
186 .add_ops("top", StatsOps { open, close: None })
187}
188
189pub fn monitor(intv: Duration, shutdown: Arc<AtomicBool>) -> Result<()> {
190 scx_utils::monitor_stats::<Metrics>(
191 &[],
192 intv,
193 || shutdown.load(Ordering::Relaxed),
194 |metrics| metrics.format(&mut std::io::stdout()),
195 )
196}