1use std::mem::MaybeUninit;
6
7use anyhow::Result;
8use libbpf_rs::skel::{OpenSkel, SkelBuilder};
9use libbpf_rs::MapCore;
10
11use crate::bpf_skel::*;
12use crate::tuning::{OscillatorState, TuningKnobs};
13use scx_pandemonium::event::EventLog;
14
15const SCX_EXIT_NONE: i32 = 0;
17const SCX_ECODE_RST_MASK: u64 = 1 << 16;
18
19const SCX_DSQ_FLAG_BUILTIN: u64 = 1u64 << 63;
21const SCX_DSQ_FLAG_LOCAL_ON: u64 = 1u64 << 62;
22
23#[repr(C)]
25#[derive(Default, Clone, Copy)]
26pub struct PandemoniumStats {
27 pub nr_dispatches: u64,
28 pub nr_idle_hits: u64,
29 pub nr_shared: u64,
30 pub nr_preempt: u64,
31 pub wake_lat_sum: u64,
32 pub wake_lat_samples: u64,
33 pub nr_keep_running: u64,
34 pub nr_hard_kicks: u64,
35 pub nr_soft_kicks: u64,
36 pub nr_enq_wakeup: u64,
37 pub nr_enq_requeue: u64,
38 pub wake_lat_idle_sum: u64,
39 pub wake_lat_idle_cnt: u64,
40 pub wake_lat_kick_sum: u64,
41 pub wake_lat_kick_cnt: u64,
42 pub nr_l2_hit_batch: u64,
43 pub nr_l2_miss_batch: u64,
44 pub nr_l2_hit_interactive: u64,
45 pub nr_l2_miss_interactive: u64,
46 pub nr_l2_hit_lat_crit: u64,
47 pub nr_l2_miss_lat_crit: u64,
48 pub nr_reenqueue: u64,
49 pub batch_sojourn_ns: u64,
50 pub longrun_mode_active: u64,
51 pub nr_overflow_rescue: u64,
52 pub nr_cross_domain: [u64; 8],
55 pub nr_osc_park: u64,
57 pub nr_spill_kick_preempt: u64,
60}
61
62const _: () = assert!(std::mem::size_of::<PandemoniumStats>() == 280);
65const _: () = assert!(std::mem::size_of::<TuningKnobs>() == 80);
66
67const _: () = assert!(crate::bpf_intf::MAX_AFFINITY_CANDIDATES == crate::bpf_intf::MAX_CPUS >> 3);
72
73const KNOBS_PIN: &str = "/sys/fs/bpf/pandemonium/tuning_knobs";
76
77pub struct Scheduler<'a> {
78 skel: MainSkel<'a>,
79 _link: libbpf_rs::Link,
80 pub log: EventLog,
81}
82
83impl<'a> Scheduler<'a> {
84 pub fn init(
85 open_object: &'a mut MaybeUninit<libbpf_rs::OpenObject>,
86 nr_cpus_override: Option<u64>,
87 ) -> Result<Self> {
88 let builder = MainSkelBuilder::default();
90 let mut open_skel = builder.open(open_object)?;
91
92 {
94 let ops = open_skel.struct_ops.pandemonium_ops_mut();
95 let name_field = &mut ops.name;
96 let version_suffix = scx_utils::build_id::ops_version_suffix(env!("CARGO_PKG_VERSION"));
97 let bytes = version_suffix.as_bytes();
98 let mut i = 0;
99 let mut bytes_idx = 0;
100 let mut found_null = false;
101 while i < name_field.len() - 1 {
102 found_null |= name_field[i] == 0;
103 if !found_null {
104 i += 1;
105 continue;
106 }
107 if bytes_idx < bytes.len() {
108 name_field[i] = bytes[bytes_idx] as i8;
109 bytes_idx += 1;
110 } else {
111 break;
112 }
113 i += 1;
114 }
115 name_field[i] = 0;
116 }
117
118 let rodata = open_skel.maps.rodata_data.as_mut().unwrap();
120
121 let possible = libbpf_rs::num_possible_cpus()? as u64;
122 rodata.nr_cpu_ids = nr_cpus_override.unwrap_or(possible);
123
124 rodata.__SCX_DSQ_FLAG_BUILTIN = SCX_DSQ_FLAG_BUILTIN;
126 rodata.__SCX_DSQ_FLAG_LOCAL_ON = SCX_DSQ_FLAG_LOCAL_ON;
127 rodata.__SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN;
128 rodata.__SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1;
129 rodata.__SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON;
130 rodata.__SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON | 1;
131 rodata.__SCX_DSQ_LOCAL_CPU_MASK = 0xFFFFFFFF;
132
133 rodata.__SCX_KICK_IDLE = 1;
135 rodata.__SCX_KICK_PREEMPT = 2;
136 rodata.__SCX_KICK_WAIT = 4;
137
138 let mut skel = open_skel.load()?;
140
141 let link = skel.maps.pandemonium_ops.attach_struct_ops()?;
143
144 let pin_dir = "/sys/fs/bpf/pandemonium";
146 let bpffs_ok = std::fs::create_dir_all(pin_dir).is_ok();
147 if bpffs_ok {
148 std::fs::remove_file(KNOBS_PIN).ok();
149 skel.maps.tuning_knobs_map.pin(KNOBS_PIN).ok();
150
151 let cache_pin = "/sys/fs/bpf/pandemonium/cache_domain";
152 std::fs::remove_file(cache_pin).ok();
153 skel.maps.cache_domain.pin(cache_pin).ok();
154
155 let observe_pin = "/sys/fs/bpf/pandemonium/task_class_observe";
156 std::fs::remove_file(observe_pin).ok();
157 skel.maps.task_class_observe.pin(observe_pin).ok();
158
159 let init_pin = "/sys/fs/bpf/pandemonium/task_class_init";
160 std::fs::remove_file(init_pin).ok();
161 skel.maps.task_class_init.pin(init_pin).ok();
162 } else {
163 log_warn!("BPFFS NOT AVAILABLE: map pinning skipped (scheduler still functional)");
164 }
165
166 Ok(Self {
167 skel,
168 _link: link,
169 log: EventLog::new(),
170 })
171 }
172
173 pub fn read_stats(&self) -> PandemoniumStats {
175 let key = 0u32.to_ne_bytes();
176 let mut total = PandemoniumStats::default();
177
178 let percpu_vals = match self
179 .skel
180 .maps
181 .stats_map
182 .lookup_percpu(&key, libbpf_rs::MapFlags::ANY)
183 {
184 Ok(Some(v)) => v,
185 _ => return total,
186 };
187
188 for cpu_val in &percpu_vals {
189 if cpu_val.len() >= std::mem::size_of::<PandemoniumStats>() {
190 let stats: PandemoniumStats = unsafe {
191 std::ptr::read_unaligned(cpu_val.as_ptr() as *const PandemoniumStats)
192 };
193 total.nr_dispatches += stats.nr_dispatches;
194 total.nr_idle_hits += stats.nr_idle_hits;
195 total.nr_shared += stats.nr_shared;
196 total.nr_preempt += stats.nr_preempt;
197 total.wake_lat_sum += stats.wake_lat_sum;
198 total.wake_lat_samples += stats.wake_lat_samples;
199 total.nr_keep_running += stats.nr_keep_running;
200 total.nr_hard_kicks += stats.nr_hard_kicks;
201 total.nr_soft_kicks += stats.nr_soft_kicks;
202 total.nr_enq_wakeup += stats.nr_enq_wakeup;
203 total.nr_enq_requeue += stats.nr_enq_requeue;
204 total.wake_lat_idle_sum += stats.wake_lat_idle_sum;
205 total.wake_lat_idle_cnt += stats.wake_lat_idle_cnt;
206 total.wake_lat_kick_sum += stats.wake_lat_kick_sum;
207 total.wake_lat_kick_cnt += stats.wake_lat_kick_cnt;
208 total.nr_l2_hit_batch += stats.nr_l2_hit_batch;
209 total.nr_l2_miss_batch += stats.nr_l2_miss_batch;
210 total.nr_l2_hit_interactive += stats.nr_l2_hit_interactive;
211 total.nr_l2_miss_interactive += stats.nr_l2_miss_interactive;
212 total.nr_l2_hit_lat_crit += stats.nr_l2_hit_lat_crit;
213 total.nr_l2_miss_lat_crit += stats.nr_l2_miss_lat_crit;
214 total.nr_reenqueue += stats.nr_reenqueue;
215 if stats.batch_sojourn_ns > total.batch_sojourn_ns {
216 total.batch_sojourn_ns = stats.batch_sojourn_ns;
217 }
218 if stats.longrun_mode_active > total.longrun_mode_active {
219 total.longrun_mode_active = stats.longrun_mode_active;
220 }
221 total.nr_overflow_rescue += stats.nr_overflow_rescue;
222 for i in 0..8 {
223 total.nr_cross_domain[i] += stats.nr_cross_domain[i];
224 }
225 total.nr_osc_park += stats.nr_osc_park;
226 total.nr_spill_kick_preempt += stats.nr_spill_kick_preempt;
227 }
228 }
229
230 total
231 }
232
233 pub fn write_tuning_knobs(&self, knobs: &TuningKnobs) -> Result<()> {
235 let key = 0u32.to_ne_bytes();
236 let value = unsafe {
237 std::slice::from_raw_parts(
238 knobs as *const TuningKnobs as *const u8,
239 std::mem::size_of::<TuningKnobs>(),
240 )
241 };
242 self.skel
243 .maps
244 .tuning_knobs_map
245 .update(&key, value, libbpf_rs::MapFlags::ANY)?;
246 Ok(())
247 }
248
249 pub fn write_topology_fields(&self, tau_ns: u64, codel_eq_ns: u64) -> Result<()> {
255 let mut knobs = self.read_tuning_knobs();
256 knobs.topology_tau_ns = tau_ns;
257 knobs.codel_eq_ns = codel_eq_ns;
258 self.write_tuning_knobs(&knobs)
259 }
260
261 pub fn read_oscillator_state(&self) -> OscillatorState {
265 let bss = match self.skel.maps.bss_data.as_ref() {
266 Some(b) => b,
267 None => return OscillatorState::default(),
268 };
269 let data = match self.skel.maps.data_data.as_ref() {
270 Some(d) => d,
271 None => return OscillatorState::default(),
272 };
273 OscillatorState {
274 codel_target_ns: bss.codel_target_ns,
275 codel_target_floor_ns: bss.codel_target_floor_ns,
276 codel_target_max_ns: data.codel_target_max_ns,
277 home_dist_extra_ns: self.read_reff_value(0, 0) as u64,
281 }
282 }
283
284 pub fn read_tuning_knobs(&self) -> TuningKnobs {
286 let key = 0u32.to_ne_bytes();
287 match self
288 .skel
289 .maps
290 .tuning_knobs_map
291 .lookup(&key, libbpf_rs::MapFlags::ANY)
292 {
293 Ok(Some(v)) if v.len() >= std::mem::size_of::<TuningKnobs>() => unsafe {
294 std::ptr::read_unaligned(v.as_ptr() as *const TuningKnobs)
295 },
296 _ => TuningKnobs::default(),
297 }
298 }
299
300 pub fn read_wake_lat_hist(&self) -> [[u64; 12]; 3] {
303 let mut result = [[0u64; 12]; 3];
304 for key_idx in 0u32..36 {
305 let key = key_idx.to_ne_bytes();
306 if let Ok(Some(percpu_vals)) = self
307 .skel
308 .maps
309 .wake_lat_hist
310 .lookup_percpu(&key, libbpf_rs::MapFlags::ANY)
311 {
312 let tier = (key_idx / 12) as usize;
313 let bucket = (key_idx % 12) as usize;
314 for cpu_val in &percpu_vals {
315 if cpu_val.len() >= std::mem::size_of::<u64>() {
316 let val: u64 =
317 unsafe { std::ptr::read_unaligned(cpu_val.as_ptr() as *const u64) };
318 result[tier][bucket] += val;
319 }
320 }
321 }
322 }
323 result
324 }
325
326 pub fn read_sleep_hist(&self) -> [u64; 4] {
329 let mut result = [0u64; 4];
330 for key_idx in 0u32..4 {
331 let key = key_idx.to_ne_bytes();
332 if let Ok(Some(percpu_vals)) = self
333 .skel
334 .maps
335 .sleep_hist
336 .lookup_percpu(&key, libbpf_rs::MapFlags::ANY)
337 {
338 for cpu_val in &percpu_vals {
339 if cpu_val.len() >= std::mem::size_of::<u64>() {
340 let val: u64 =
341 unsafe { std::ptr::read_unaligned(cpu_val.as_ptr() as *const u64) };
342 result[key_idx as usize] += val;
343 }
344 }
345 }
346 }
347 result
348 }
349
350 pub fn write_cache_domain(&self, cpu: u32, l2_group: u32) -> Result<()> {
352 let key = cpu.to_ne_bytes();
353 let val = l2_group.to_ne_bytes();
354 self.skel
355 .maps
356 .cache_domain
357 .update(&key, &val, libbpf_rs::MapFlags::ANY)?;
358 Ok(())
359 }
360
361 pub fn write_cpu_domain(&self, cpu: u32, domain: u32) -> Result<()> {
364 let key = cpu.to_ne_bytes();
365 let val = domain.to_ne_bytes();
366 self.skel
367 .maps
368 .cpu_domain
369 .update(&key, &val, libbpf_rs::MapFlags::ANY)?;
370 Ok(())
371 }
372
373 pub fn write_nr_overflow_domains(&mut self, nr_overflow_domains: u32) {
377 if let Some(data) = self.skel.maps.data_data.as_mut() {
378 data.nr_overflow_domains = nr_overflow_domains;
379 }
380 }
381
382 pub fn write_l2_sibling(&self, group_id: u32, slot: u32, cpu: u32) -> Result<()> {
384 let key = (group_id * 8 + slot).to_ne_bytes();
385 let val = cpu.to_ne_bytes();
386 self.skel
387 .maps
388 .l2_siblings
389 .update(&key, &val, libbpf_rs::MapFlags::ANY)?;
390 Ok(())
391 }
392
393 pub fn write_affinity_rank(&self, cpu: u32, slot: u32, target_cpu: u32) -> Result<()> {
397 let stride = crate::bpf_intf::MAX_AFFINITY_CANDIDATES;
401 let key = (cpu * stride + slot).to_ne_bytes();
402 let val = target_cpu.to_ne_bytes();
403 self.skel
404 .maps
405 .affinity_rank
406 .update(&key, &val, libbpf_rs::MapFlags::ANY)?;
407 Ok(())
408 }
409
410 pub fn write_reff_value(&self, cpu: u32, slot: u32, value: u32) -> Result<()> {
413 let stride = crate::bpf_intf::MAX_AFFINITY_CANDIDATES;
414 let key = (cpu * stride + slot).to_ne_bytes();
415 let val = value.to_ne_bytes();
416 self.skel
417 .maps
418 .reff_value
419 .update(&key, &val, libbpf_rs::MapFlags::ANY)?;
420 Ok(())
421 }
422
423 pub fn write_domain_phi(&self, cpu: u32, slot: u32, value: u32) -> Result<()> {
427 let stride = crate::bpf_intf::MAX_AFFINITY_CANDIDATES;
428 let key = (cpu * stride + slot).to_ne_bytes();
429 let val = value.to_ne_bytes();
430 self.skel
431 .maps
432 .domain_phi
433 .update(&key, &val, libbpf_rs::MapFlags::ANY)?;
434 Ok(())
435 }
436
437 pub fn read_reff_value(&self, cpu: u32, slot: u32) -> u32 {
441 let stride = crate::bpf_intf::MAX_AFFINITY_CANDIDATES;
442 let key = (cpu * stride + slot).to_ne_bytes();
443 match self
444 .skel
445 .maps
446 .reff_value
447 .lookup(&key, libbpf_rs::MapFlags::ANY)
448 {
449 Ok(Some(bytes)) if bytes.len() >= 4 => {
450 let v = u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]);
451 if v == u32::MAX {
452 0
453 } else {
454 v
455 }
456 }
457 _ => 0,
458 }
459 }
460
461 pub fn read_exit_info(&self) -> bool {
463 let data = self.skel.maps.data_data.as_ref().unwrap();
464 let kind = data.uei.kind;
465 let exit_code = data.uei.exit_code;
466
467 if kind != SCX_EXIT_NONE {
468 let reason_bytes: &[u8] =
469 unsafe { std::slice::from_raw_parts(data.uei.reason.as_ptr() as *const u8, 128) };
470 let msg_bytes: &[u8] =
471 unsafe { std::slice::from_raw_parts(data.uei.msg.as_ptr() as *const u8, 1024) };
472
473 let reason = std::str::from_utf8(reason_bytes)
474 .unwrap_or("unknown")
475 .trim_end_matches('\0');
476 let msg = std::str::from_utf8(msg_bytes)
477 .unwrap_or("")
478 .trim_end_matches('\0');
479
480 log_warn!("BPF exit: kind={} code={}", kind, exit_code);
481 if !reason.is_empty() {
482 log_warn!("BPF exit reason: {}", reason);
483 }
484 if !msg.is_empty() {
485 log_warn!("BPF exit msg: {}", msg);
486 }
487 }
488
489 (exit_code as u64 & SCX_ECODE_RST_MASK) != 0
490 }
491
492 pub fn exited(&self) -> bool {
493 self.skel.maps.data_data.as_ref().unwrap().uei.kind != SCX_EXIT_NONE
494 }
495}
496
497impl Drop for Scheduler<'_> {
498 fn drop(&mut self) {
499 let _ = self.skel.maps.tuning_knobs_map.unpin(KNOBS_PIN);
500 let _ = self
501 .skel
502 .maps
503 .cache_domain
504 .unpin("/sys/fs/bpf/pandemonium/cache_domain");
505 let _ = self
506 .skel
507 .maps
508 .task_class_observe
509 .unpin("/sys/fs/bpf/pandemonium/task_class_observe");
510 let _ = self
511 .skel
512 .maps
513 .task_class_init
514 .unpin("/sys/fs/bpf/pandemonium/task_class_init");
515 let _ = std::fs::remove_dir("/sys/fs/bpf/pandemonium");
516 }
517}