1use anyhow::{anyhow, bail, Context, Result};
7use libbpf_rs::libbpf_sys::*;
8use libbpf_rs::{AsRawLibbpf, OpenProgramImpl, ProgramImpl};
9use log::warn;
10use std::env;
11use std::ffi::c_void;
12use std::ffi::CStr;
13use std::ffi::CString;
14use std::io;
15use std::io::BufRead;
16use std::io::BufReader;
17use std::mem::size_of;
18use std::slice::from_raw_parts;
19
20const PROCFS_MOUNTS: &str = "/proc/mounts";
21const TRACEFS: &str = "tracefs";
22const DEBUGFS: &str = "debugfs";
23
24lazy_static::lazy_static! {
25 pub static ref SCX_OPS_KEEP_BUILTIN_IDLE: u64 =
26 read_enum("scx_ops_flags", "SCX_OPS_KEEP_BUILTIN_IDLE").unwrap_or(0);
27 pub static ref SCX_OPS_ENQ_LAST: u64 =
28 read_enum("scx_ops_flags", "SCX_OPS_ENQ_LAST").unwrap_or(0);
29 pub static ref SCX_OPS_ENQ_EXITING: u64 =
30 read_enum("scx_ops_flags", "SCX_OPS_ENQ_EXITING").unwrap_or(0);
31 pub static ref SCX_OPS_SWITCH_PARTIAL: u64 =
32 read_enum("scx_ops_flags", "SCX_OPS_SWITCH_PARTIAL").unwrap_or(0);
33 pub static ref SCX_OPS_ENQ_MIGRATION_DISABLED: u64 =
34 read_enum("scx_ops_flags", "SCX_OPS_ENQ_MIGRATION_DISABLED").unwrap_or(0);
35 pub static ref SCX_OPS_ALLOW_QUEUED_WAKEUP: u64 =
36 read_enum("scx_ops_flags", "SCX_OPS_ALLOW_QUEUED_WAKEUP").unwrap_or(0);
37 pub static ref SCX_OPS_BUILTIN_IDLE_PER_NODE: u64 =
38 read_enum("scx_ops_flags", "SCX_OPS_BUILTIN_IDLE_PER_NODE").unwrap_or(0);
39 pub static ref SCX_OPS_ALWAYS_ENQ_IMMED: u64 =
40 read_enum("scx_ops_flags", "SCX_OPS_ALWAYS_ENQ_IMMED").unwrap_or(0);
41
42 pub static ref SCX_PICK_IDLE_CORE: u64 =
43 read_enum("scx_pick_idle_cpu_flags", "SCX_PICK_IDLE_CORE").unwrap_or(0);
44 pub static ref SCX_PICK_IDLE_IN_NODE: u64 =
45 read_enum("scx_pick_idle_cpu_flags", "SCX_PICK_IDLE_IN_NODE").unwrap_or(0);
46
47 pub static ref ROOT_PREFIX: String =
48 env::var("SCX_SYSFS_PREFIX").unwrap_or("".to_string());
49}
50
51fn load_vmlinux_btf() -> &'static mut btf {
52 let btf = unsafe { btf__load_vmlinux_btf() };
53 if btf.is_null() {
54 panic!("btf__load_vmlinux_btf() returned NULL, was CONFIG_DEBUG_INFO_BTF enabled?")
55 }
56 unsafe { &mut *btf }
57}
58
59lazy_static::lazy_static! {
60 static ref VMLINUX_BTF: &'static mut btf = load_vmlinux_btf();
61}
62
63fn btf_kind(t: &btf_type) -> u32 {
64 (t.info >> 24) & 0x1f
65}
66
67fn btf_vlen(t: &btf_type) -> u32 {
68 t.info & 0xffff
69}
70
71fn btf_type_plus_1(t: &btf_type) -> *const c_void {
72 let ptr_val = t as *const btf_type as usize;
73 (ptr_val + size_of::<btf_type>()) as *const c_void
74}
75
76fn btf_enum(t: &btf_type) -> &[btf_enum] {
77 let ptr = btf_type_plus_1(t);
78 unsafe { from_raw_parts(ptr as *const btf_enum, btf_vlen(t) as usize) }
79}
80
81fn btf_enum64(t: &btf_type) -> &[btf_enum64] {
82 let ptr = btf_type_plus_1(t);
83 unsafe { from_raw_parts(ptr as *const btf_enum64, btf_vlen(t) as usize) }
84}
85
86fn btf_members(t: &btf_type) -> &[btf_member] {
87 let ptr = btf_type_plus_1(t);
88 unsafe { from_raw_parts(ptr as *const btf_member, btf_vlen(t) as usize) }
89}
90
91fn btf_params(t: &btf_type) -> &[btf_param] {
92 let ptr = btf_type_plus_1(t);
93 unsafe { from_raw_parts(ptr as *const btf_param, btf_vlen(t) as usize) }
94}
95
96fn btf_name_str_by_offset(btf: &btf, name_off: u32) -> Result<&str> {
97 let n = unsafe { btf__name_by_offset(btf, name_off) };
98 if n.is_null() {
99 bail!("btf__name_by_offset() returned NULL");
100 }
101 Ok(unsafe { CStr::from_ptr(n) }
102 .to_str()
103 .with_context(|| format!("Failed to convert {:?} to string", n))?)
104}
105
106pub fn read_enum(type_name: &str, name: &str) -> Result<u64> {
107 let btf: &btf = *VMLINUX_BTF;
108
109 let type_name = CString::new(type_name).unwrap();
110 let tid = unsafe { btf__find_by_name(btf, type_name.as_ptr()) };
111 if tid < 0 {
112 bail!("type {:?} doesn't exist, ret={}", type_name, tid);
113 }
114
115 let t = unsafe { btf__type_by_id(btf, tid as _) };
116 if t.is_null() {
117 bail!("btf__type_by_id({}) returned NULL", tid);
118 }
119 let t = unsafe { &*t };
120
121 match btf_kind(t) {
122 BTF_KIND_ENUM => {
123 for e in btf_enum(t).iter() {
124 if btf_name_str_by_offset(btf, e.name_off)? == name {
125 return Ok(e.val as u64);
126 }
127 }
128 }
129 BTF_KIND_ENUM64 => {
130 for e in btf_enum64(t).iter() {
131 if btf_name_str_by_offset(btf, e.name_off)? == name {
132 return Ok(((e.val_hi32 as u64) << 32) | (e.val_lo32) as u64);
133 }
134 }
135 }
136 _ => (),
137 }
138
139 Err(anyhow!("{:?} doesn't exist in {:?}", name, type_name))
140}
141
142pub fn read_enum_any(type_names: &[&str], name: &str) -> Result<u64> {
144 let mut errors = Vec::new();
145
146 for type_name in type_names {
147 match read_enum(type_name, name) {
148 Ok(val) => return Ok(val),
149 Err(err) => errors.push(format!("{}: {:#}", type_name, err)),
150 }
151 }
152
153 bail!(
154 "{:?} doesn't exist in any of {:?}: {}",
155 name,
156 type_names,
157 errors.join("; ")
158 )
159}
160
161pub fn struct_has_field(type_name: &str, field: &str) -> Result<bool> {
162 let btf: &btf = *VMLINUX_BTF;
163
164 let type_name = CString::new(type_name).unwrap();
165 let tid = unsafe { btf__find_by_name_kind(btf, type_name.as_ptr(), BTF_KIND_STRUCT) };
166 if tid < 0 {
167 bail!("type {:?} doesn't exist, ret={}", type_name, tid);
168 }
169
170 let t = unsafe { btf__type_by_id(btf, tid as _) };
171 if t.is_null() {
172 bail!("btf__type_by_id({}) returned NULL", tid);
173 }
174 let t = unsafe { &*t };
175
176 for m in btf_members(t).iter() {
177 if btf_name_str_by_offset(btf, m.name_off)? == field {
178 return Ok(true);
179 }
180 }
181
182 Ok(false)
183}
184
185pub fn ksym_exists(ksym: &str) -> Result<bool> {
186 let btf: &btf = *VMLINUX_BTF;
187
188 let ksym_name = CString::new(ksym).unwrap();
189 let tid = unsafe { btf__find_by_name(btf, ksym_name.as_ptr()) };
190 Ok(tid >= 0)
191}
192
193pub fn malformed_scx_kfuncs() -> Vec<String> {
203 let btf: &btf = *VMLINUX_BTF;
204 let mut bad = Vec::new();
205 let cnt = unsafe { btf__type_cnt(btf) };
206
207 for id in 1..cnt {
208 let t = unsafe { btf__type_by_id(btf, id) };
209 if t.is_null() {
210 continue;
211 }
212 let t = unsafe { &*t };
213 if btf_kind(t) != BTF_KIND_FUNC {
214 continue;
215 }
216
217 let Ok(name) = btf_name_str_by_offset(btf, t.name_off) else {
218 continue;
219 };
220 if !(name.starts_with("scx_bpf_") || name.starts_with("__scx_bpf_")) {
221 continue;
222 }
223 if name.ends_with("_impl") {
229 continue;
230 }
231
232 let proto_id = unsafe { t.__bindgen_anon_1.type_ };
234 let pt = unsafe { btf__type_by_id(btf, proto_id) };
235 if pt.is_null() {
236 continue;
237 }
238 let pt = unsafe { &*pt };
239 if btf_kind(pt) != BTF_KIND_FUNC_PROTO {
240 continue;
241 }
242
243 if btf_params(pt).iter().any(|p| {
244 p.name_off != 0 && matches!(btf_name_str_by_offset(btf, p.name_off), Ok("aux"))
245 }) {
246 bad.push(name.to_string());
247 }
248 }
249
250 bad
251}
252
253pub fn in_kallsyms(ksym: &str) -> Result<bool> {
254 let file = std::fs::File::open("/proc/kallsyms")?;
255 let reader = std::io::BufReader::new(file);
256
257 for line in reader.lines() {
258 for sym in line.unwrap().split_whitespace() {
259 if ksym == sym {
260 return Ok(true);
261 }
262 }
263 }
264
265 Ok(false)
266}
267
268pub fn get_fs_mount(mount_type: &str) -> Result<Vec<std::path::PathBuf>> {
270 let proc_mounts_path = std::path::Path::new(PROCFS_MOUNTS);
271
272 let file = std::fs::File::open(proc_mounts_path)
273 .with_context(|| format!("Failed to open {}", proc_mounts_path.display()))?;
274
275 let reader = BufReader::new(file);
276
277 let mut mounts = Vec::new();
278 for line in reader.lines() {
279 let line = line.context("Failed to read line from /proc/mounts")?;
280 let mount_info: Vec<&str> = line.split_whitespace().collect();
281
282 if mount_info.len() > 3 && mount_info[2] == mount_type {
283 let mount_path = std::path::PathBuf::from(mount_info[1]);
284 mounts.push(mount_path);
285 }
286 }
287
288 Ok(mounts)
289}
290
291pub fn tracefs_mount() -> Result<std::path::PathBuf> {
293 let mounts = get_fs_mount(TRACEFS)?;
294 mounts.into_iter().next().context("No tracefs mount found")
295}
296
297pub fn debugfs_mount() -> Result<std::path::PathBuf> {
299 let mounts = get_fs_mount(DEBUGFS)?;
300 mounts.into_iter().next().context("No debugfs mount found")
301}
302
303pub fn tracer_available(tracer: &str) -> Result<bool> {
304 let base_path = tracefs_mount().unwrap_or_else(|_| debugfs_mount().unwrap().join("tracing"));
305 let file = match std::fs::File::open(base_path.join("available_tracers")) {
306 Ok(f) => f,
307 Err(_) => return Ok(false),
308 };
309 let reader = std::io::BufReader::new(file);
310
311 for line in reader.lines() {
312 for tc in line.unwrap().split_whitespace() {
313 if tracer == tc {
314 return Ok(true);
315 }
316 }
317 }
318
319 Ok(false)
320}
321
322pub fn tracepoint_exists(tracepoint: &str) -> Result<bool> {
323 let base_path = tracefs_mount().unwrap_or_else(|_| debugfs_mount().unwrap().join("tracing"));
324 let file = match std::fs::File::open(base_path.join("available_events")) {
325 Ok(f) => f,
326 Err(_) => return Ok(false),
327 };
328 let reader = std::io::BufReader::new(file);
329
330 for line in reader.lines() {
331 for tp in line.unwrap().split_whitespace() {
332 if tracepoint == tp {
333 return Ok(true);
334 }
335 }
336 }
337
338 Ok(false)
339}
340
341pub fn cond_kprobe_enable<T>(sym: &str, prog_ptr: &OpenProgramImpl<T>) -> Result<bool> {
342 if in_kallsyms(sym)? {
343 unsafe {
344 bpf_program__set_autoload(prog_ptr.as_libbpf_object().as_ptr(), true);
345 }
346 return Ok(true);
347 } else {
348 warn!("symbol {sym} is missing, kprobe not loaded");
349 }
350
351 Ok(false)
352}
353
354pub fn cond_kprobes_enable<T>(kprobes: Vec<(&str, &OpenProgramImpl<T>)>) -> Result<bool> {
355 for (sym, _) in kprobes.iter() {
357 if in_kallsyms(sym)? == false {
358 warn!("symbol {sym} is missing, kprobe not loaded");
359 return Ok(false);
360 }
361 }
362
363 for (_, ptr) in kprobes.iter() {
365 unsafe {
366 bpf_program__set_autoload(ptr.as_libbpf_object().as_ptr(), true);
367 }
368 }
369
370 Ok(true)
371}
372
373pub fn cond_kprobe_load<T>(sym: &str, prog_ptr: &OpenProgramImpl<T>) -> Result<bool> {
374 if in_kallsyms(sym)? {
375 unsafe {
376 bpf_program__set_autoload(prog_ptr.as_libbpf_object().as_ptr(), true);
377 bpf_program__set_autoattach(prog_ptr.as_libbpf_object().as_ptr(), false);
378 }
379 return Ok(true);
380 } else {
381 warn!("symbol {sym} is missing, kprobe not loaded");
382 }
383
384 Ok(false)
385}
386
387pub fn cond_kprobe_attach<T>(sym: &str, prog_ptr: &ProgramImpl<T>) -> Result<bool> {
388 if in_kallsyms(sym)? {
389 unsafe {
390 bpf_program__attach(prog_ptr.as_libbpf_object().as_ptr());
391 }
392 return Ok(true);
393 } else {
394 warn!("symbol {sym} is missing, kprobe not loaded");
395 }
396
397 Ok(false)
398}
399
400pub fn cond_tracepoint_enable<T>(tracepoint: &str, prog_ptr: &OpenProgramImpl<T>) -> Result<bool> {
401 if tracepoint_exists(tracepoint)? {
402 unsafe {
403 bpf_program__set_autoload(prog_ptr.as_libbpf_object().as_ptr(), true);
404 }
405 return Ok(true);
406 } else {
407 warn!("tracepoint {tracepoint} is missing, tracepoint not loaded");
408 }
409
410 Ok(false)
411}
412
413pub fn cond_tracepoints_enable<T>(tracepoints: Vec<(&str, &OpenProgramImpl<T>)>) -> Result<bool> {
414 for (tp, _) in tracepoints.iter() {
416 if tracepoint_exists(tp)? == false {
417 warn!("tracepoint {tp} is missing, tracepoint not loaded");
418 return Ok(false);
419 }
420 }
421
422 for (_, ptr) in tracepoints.iter() {
424 unsafe {
425 bpf_program__set_autoload(ptr.as_libbpf_object().as_ptr(), true);
426 }
427 }
428
429 Ok(true)
430}
431
432pub fn is_sched_ext_enabled() -> io::Result<bool> {
433 let content = std::fs::read_to_string("/sys/kernel/sched_ext/state")?;
434
435 match content.trim() {
436 "enabled" => Ok(true),
437 "disabled" => Ok(false),
438 _ => {
439 Err(io::Error::new(
441 io::ErrorKind::InvalidData,
442 "Unexpected content in /sys/kernel/sched_ext/state",
443 ))
444 }
445 }
446}
447
448#[macro_export]
449macro_rules! unwrap_or_break {
450 ($expr: expr, $label: lifetime) => {{
451 match $expr {
452 Ok(val) => val,
453 Err(e) => break $label Err(e),
454 }
455 }};
456}
457
458pub fn check_min_requirements() -> Result<()> {
459 if let Ok(false) | Err(_) = struct_has_field("sched_ext_ops", "dump") {
462 bail!("sched_ext_ops.dump() missing, kernel too old?");
463 }
464 Ok(())
465}
466
467#[rustfmt::skip]
472#[macro_export]
473macro_rules! scx_ops_open {
474 ($builder: expr, $obj_ref: expr, $ops: ident, $open_opts: expr) => { 'block: {
475 scx_utils::paste! {
476 scx_utils::unwrap_or_break!(scx_utils::compat::check_min_requirements(), 'block);
477 use ::anyhow::Context;
478 use ::libbpf_rs::skel::SkelBuilder;
479
480 let mut skel = match $open_opts {
481 Some(opts_ref) => { match $builder.open_opts(opts_ref, $obj_ref).context("Failed to open BPF program with options") {
483 Ok(val) => val,
484 Err(e) => break 'block Err(e),
485 }
486 }
487 None => {
488 match $builder.open($obj_ref).context("Failed to open BPF program") {
489 Ok(val) => val,
490 Err(e) => break 'block Err(e),
491 }
492 }
493 };
494
495 let ops = skel.struct_ops.[<$ops _mut>]();
496 let path = std::path::Path::new("/sys/kernel/sched_ext/hotplug_seq");
497
498 let val = match std::fs::read_to_string(&path) {
499 Ok(val) => val,
500 Err(_) => {
501 break 'block Err(anyhow::anyhow!("Failed to open or read file {:?}", path));
502 }
503 };
504
505 ops.hotplug_seq = match val.trim().parse::<u64>() {
506 Ok(parsed) => parsed,
507 Err(_) => {
508 break 'block Err(anyhow::anyhow!("Failed to parse hotplug seq {}", val));
509 }
510 };
511
512 if let Ok(s) = ::std::env::var("SCX_TIMEOUT_MS") {
513 skel.struct_ops.[<$ops _mut>]().timeout_ms = match s.parse::<u32>() {
514 Ok(ms) => {
515 ::scx_utils::info!("Setting timeout_ms to {} based on environment", ms);
516 ms
517 },
518 Err(e) => {
519 break 'block anyhow::Result::Err(e).context("SCX_TIMEOUT_MS has invalid value");
520 },
521 };
522 }
523
524 {
525 let ops = skel.struct_ops.[<$ops _mut>]();
526
527 let name_field = &mut ops.name;
528
529 let version_suffix = ::scx_utils::build_id::ops_version_suffix(env!("CARGO_PKG_VERSION"));
530 let bytes = version_suffix.as_bytes();
531 let mut i = 0;
532 let mut bytes_idx = 0;
533 let mut found_null = false;
534
535 while i < name_field.len() - 1 {
536 found_null |= name_field[i] == 0;
537 if !found_null {
538 i += 1;
539 continue;
540 }
541
542 if bytes_idx < bytes.len() {
543 name_field[i] = bytes[bytes_idx] as i8;
544 bytes_idx += 1;
545 } else {
546 break;
547 }
548 i += 1;
549 }
550 name_field[i] = 0;
551 }
552
553 $crate::import_enums!(skel);
554
555 let result = ::anyhow::Result::Ok(skel);
556
557 result
558 }
559 }};
560}
561
562#[rustfmt::skip]
567#[macro_export]
568macro_rules! scx_ops_load {
569 ($skel: expr, $ops: ident, $uei: ident) => { 'block: {
570 scx_utils::paste! {
571 use ::anyhow::Context;
572 use ::libbpf_rs::skel::OpenSkel;
573
574 {
575 let ops = $skel.struct_ops.[<$ops _mut>]();
576 if ops.sub_cgroup_id > 0 {
577 if let Ok(false) | Err(_) = scx_utils::compat::struct_has_field("sched_ext_ops", "sub_cgroup_id") {
578 ::scx_utils::warn!("kernel doesn't support ops.sub_cgroup_id");
579 ops.sub_cgroup_id = 0;
580 }
581 }
582 }
583
584 scx_utils::uei_set_size!($skel, $ops, $uei);
585 $skel.load().context("Failed to load BPF program").map_err(|e| {
586 let bad = scx_utils::compat::malformed_scx_kfuncs();
587 if bad.is_empty() {
588 e
589 } else {
590 e.context(format!(
591 "the running kernel's BTF has malformed scx kfunc prototype(s): {}.\n\
592 \n\
593 These kfuncs are KF_IMPLICIT_ARGS but their public BTF prototype\n\
594 still carries the implicit 'struct bpf_prog_aux *' argument, which\n\
595 makes BPF programs fail to load with 'func_proto incompatible with\n\
596 vmlinux'. This happens when the kernel was built with pahole < 1.26.\n\
597 \n\
598 Fix: boot a kernel whose BTF was generated with pahole >= 1.26.\n\
599 Affected distros include Ubuntu 24.04 LTS. See kernel commit\n\
600 9edd04c4189e (\"docs: Raise minimum pahole version to 1.26 for\n\
601 KF_IMPLICIT_ARGS kfuncs\").",
602 bad.join(", ")
603 ))
604 }
605 })
606 }
607 }};
608}
609
610#[rustfmt::skip]
612#[macro_export]
613macro_rules! scx_ops_attach {
614 ($skel: expr, $ops: ident) => {
615 scx_ops_attach!($skel, $ops, false)
616 };
617 ($skel: expr, $ops: ident, $is_subsched: expr) => { 'block: {
618 use ::anyhow::Context;
619 use ::libbpf_rs::skel::Skel;
620
621 if !$is_subsched && scx_utils::compat::is_sched_ext_enabled().unwrap_or(false) {
622 break 'block Err(anyhow::anyhow!(
623 "another sched_ext scheduler is already running"
624 ));
625 }
626 $skel
627 .attach()
628 .context("Failed to attach non-struct_ops BPF programs")
629 .and_then(|_| {
630 $skel
631 .maps
632 .$ops
633 .attach_struct_ops()
634 .context("Failed to attach struct_ops BPF programs")
635 })
636 }};
637}
638
639#[cfg(test)]
640mod tests {
641 #[test]
642 fn test_read_enum() {
643 assert_eq!(super::read_enum("pid_type", "PIDTYPE_TGID").unwrap(), 1);
644 }
645
646 #[test]
647 fn test_read_enum_any() {
648 assert_eq!(
649 super::read_enum_any(&["NO_SUCH_TYPE", "pid_type"], "PIDTYPE_TGID").unwrap(),
650 1
651 );
652 assert!(super::read_enum_any(&["NO_SUCH_TYPE", "pid_type"], "NO_SUCH_ENUM").is_err());
653 }
654
655 #[test]
656 fn test_struct_has_field() {
657 assert!(super::struct_has_field("task_struct", "flags").unwrap());
658 assert!(!super::struct_has_field("task_struct", "NO_SUCH_FIELD").unwrap());
659 assert!(super::struct_has_field("NO_SUCH_STRUCT", "NO_SUCH_FIELD").is_err());
660 }
661
662 #[test]
663 fn test_ksym_exists() {
664 assert!(super::ksym_exists("bpf_task_acquire").unwrap());
665 assert!(!super::ksym_exists("NO_SUCH_KFUNC").unwrap());
666 }
667
668 #[test]
669 fn test_malformed_scx_kfuncs() {
670 let bad = super::malformed_scx_kfuncs();
674 assert!(bad.iter().all(|n| !n.ends_with("_impl")));
675 }
676}