1use anyhow::{anyhow, bail, Context, Result};
7use libbpf_rs::libbpf_sys::*;
8use libbpf_rs::{AsRawLibbpf, OpenProgramImpl};
9use log::warn;
10use std::env;
11use std::ffi::c_void;
12use std::ffi::CStr;
13use std::ffi::CString;
14use std::io;
15use std::io::BufRead;
16use std::io::BufReader;
17use std::mem::size_of;
18use std::slice::from_raw_parts;
19
20const PROCFS_MOUNTS: &str = "/proc/mounts";
21const TRACEFS: &str = "tracefs";
22const DEBUGFS: &str = "debugfs";
23
24lazy_static::lazy_static! {
25 pub static ref SCX_OPS_KEEP_BUILTIN_IDLE: u64 =
26 read_enum("scx_ops_flags", "SCX_OPS_KEEP_BUILTIN_IDLE").unwrap_or(0);
27 pub static ref SCX_OPS_ENQ_LAST: u64 =
28 read_enum("scx_ops_flags", "SCX_OPS_ENQ_LAST").unwrap_or(0);
29 pub static ref SCX_OPS_ENQ_EXITING: u64 =
30 read_enum("scx_ops_flags", "SCX_OPS_ENQ_EXITING").unwrap_or(0);
31 pub static ref SCX_OPS_SWITCH_PARTIAL: u64 =
32 read_enum("scx_ops_flags", "SCX_OPS_SWITCH_PARTIAL").unwrap_or(0);
33 pub static ref SCX_OPS_ENQ_MIGRATION_DISABLED: u64 =
34 read_enum("scx_ops_flags", "SCX_OPS_ENQ_MIGRATION_DISABLED").unwrap_or(0);
35 pub static ref SCX_OPS_ALLOW_QUEUED_WAKEUP: u64 =
36 read_enum("scx_ops_flags", "SCX_OPS_ALLOW_QUEUED_WAKEUP").unwrap_or(0);
37 pub static ref SCX_OPS_BUILTIN_IDLE_PER_NODE: u64 =
38 read_enum("scx_ops_flags", "SCX_OPS_BUILTIN_IDLE_PER_NODE").unwrap_or(0);
39
40 pub static ref SCX_PICK_IDLE_CORE: u64 =
41 read_enum("scx_pick_idle_cpu_flags", "SCX_PICK_IDLE_CORE").unwrap_or(0);
42 pub static ref SCX_PICK_IDLE_IN_NODE: u64 =
43 read_enum("scx_pick_idle_cpu_flags", "SCX_PICK_IDLE_IN_NODE").unwrap_or(0);
44
45 pub static ref ROOT_PREFIX: String =
46 env::var("SCX_SYSFS_PREFIX").unwrap_or("".to_string());
47}
48
49fn load_vmlinux_btf() -> &'static mut btf {
50 let btf = unsafe { btf__load_vmlinux_btf() };
51 if btf.is_null() {
52 panic!("btf__load_vmlinux_btf() returned NULL, was CONFIG_DEBUG_INFO_BTF enabled?")
53 }
54 unsafe { &mut *btf }
55}
56
57lazy_static::lazy_static! {
58 static ref VMLINUX_BTF: &'static mut btf = load_vmlinux_btf();
59}
60
61fn btf_kind(t: &btf_type) -> u32 {
62 (t.info >> 24) & 0x1f
63}
64
65fn btf_vlen(t: &btf_type) -> u32 {
66 t.info & 0xffff
67}
68
69fn btf_type_plus_1(t: &btf_type) -> *const c_void {
70 let ptr_val = t as *const btf_type as usize;
71 (ptr_val + size_of::<btf_type>()) as *const c_void
72}
73
74fn btf_enum(t: &btf_type) -> &[btf_enum] {
75 let ptr = btf_type_plus_1(t);
76 unsafe { from_raw_parts(ptr as *const btf_enum, btf_vlen(t) as usize) }
77}
78
79fn btf_enum64(t: &btf_type) -> &[btf_enum64] {
80 let ptr = btf_type_plus_1(t);
81 unsafe { from_raw_parts(ptr as *const btf_enum64, btf_vlen(t) as usize) }
82}
83
84fn btf_members(t: &btf_type) -> &[btf_member] {
85 let ptr = btf_type_plus_1(t);
86 unsafe { from_raw_parts(ptr as *const btf_member, btf_vlen(t) as usize) }
87}
88
89fn btf_name_str_by_offset(btf: &btf, name_off: u32) -> Result<&str> {
90 let n = unsafe { btf__name_by_offset(btf, name_off) };
91 if n.is_null() {
92 bail!("btf__name_by_offset() returned NULL");
93 }
94 Ok(unsafe { CStr::from_ptr(n) }
95 .to_str()
96 .with_context(|| format!("Failed to convert {:?} to string", n))?)
97}
98
99pub fn read_enum(type_name: &str, name: &str) -> Result<u64> {
100 let btf: &btf = *VMLINUX_BTF;
101
102 let type_name = CString::new(type_name).unwrap();
103 let tid = unsafe { btf__find_by_name(btf, type_name.as_ptr()) };
104 if tid < 0 {
105 bail!("type {:?} doesn't exist, ret={}", type_name, tid);
106 }
107
108 let t = unsafe { btf__type_by_id(btf, tid as _) };
109 if t.is_null() {
110 bail!("btf__type_by_id({}) returned NULL", tid);
111 }
112 let t = unsafe { &*t };
113
114 match btf_kind(t) {
115 BTF_KIND_ENUM => {
116 for e in btf_enum(t).iter() {
117 if btf_name_str_by_offset(btf, e.name_off)? == name {
118 return Ok(e.val as u64);
119 }
120 }
121 }
122 BTF_KIND_ENUM64 => {
123 for e in btf_enum64(t).iter() {
124 if btf_name_str_by_offset(btf, e.name_off)? == name {
125 return Ok(((e.val_hi32 as u64) << 32) | (e.val_lo32) as u64);
126 }
127 }
128 }
129 _ => (),
130 }
131
132 Err(anyhow!("{:?} doesn't exist in {:?}", name, type_name))
133}
134
135pub fn struct_has_field(type_name: &str, field: &str) -> Result<bool> {
136 let btf: &btf = *VMLINUX_BTF;
137
138 let type_name = CString::new(type_name).unwrap();
139 let tid = unsafe { btf__find_by_name_kind(btf, type_name.as_ptr(), BTF_KIND_STRUCT) };
140 if tid < 0 {
141 bail!("type {:?} doesn't exist, ret={}", type_name, tid);
142 }
143
144 let t = unsafe { btf__type_by_id(btf, tid as _) };
145 if t.is_null() {
146 bail!("btf__type_by_id({}) returned NULL", tid);
147 }
148 let t = unsafe { &*t };
149
150 for m in btf_members(t).iter() {
151 if btf_name_str_by_offset(btf, m.name_off)? == field {
152 return Ok(true);
153 }
154 }
155
156 Ok(false)
157}
158
159pub fn ksym_exists(ksym: &str) -> Result<bool> {
160 let btf: &btf = *VMLINUX_BTF;
161
162 let ksym_name = CString::new(ksym).unwrap();
163 let tid = unsafe { btf__find_by_name(btf, ksym_name.as_ptr()) };
164 Ok(tid >= 0)
165}
166
167pub fn in_kallsyms(ksym: &str) -> Result<bool> {
168 let file = std::fs::File::open("/proc/kallsyms")?;
169 let reader = std::io::BufReader::new(file);
170
171 for line in reader.lines() {
172 for sym in line.unwrap().split_whitespace() {
173 if ksym == sym {
174 return Ok(true);
175 }
176 }
177 }
178
179 Ok(false)
180}
181
182pub fn get_fs_mount(mount_type: &str) -> Result<Vec<std::path::PathBuf>> {
184 let proc_mounts_path = std::path::Path::new(PROCFS_MOUNTS);
185
186 let file = std::fs::File::open(proc_mounts_path)
187 .with_context(|| format!("Failed to open {}", proc_mounts_path.display()))?;
188
189 let reader = BufReader::new(file);
190
191 let mut mounts = Vec::new();
192 for line in reader.lines() {
193 let line = line.context("Failed to read line from /proc/mounts")?;
194 let mount_info: Vec<&str> = line.split_whitespace().collect();
195
196 if mount_info.len() > 3 && mount_info[2] == mount_type {
197 let mount_path = std::path::PathBuf::from(mount_info[1]);
198 mounts.push(mount_path);
199 }
200 }
201
202 Ok(mounts)
203}
204
205pub fn tracefs_mount() -> Result<std::path::PathBuf> {
207 let mounts = get_fs_mount(TRACEFS)?;
208 mounts.into_iter().next().context("No tracefs mount found")
209}
210
211pub fn debugfs_mount() -> Result<std::path::PathBuf> {
213 let mounts = get_fs_mount(DEBUGFS)?;
214 mounts.into_iter().next().context("No debugfs mount found")
215}
216
217pub fn tracepoint_exists(tracepoint: &str) -> Result<bool> {
218 let base_path = tracefs_mount().unwrap_or_else(|_| debugfs_mount().unwrap().join("tracing"));
219 let file = std::fs::File::open(base_path.join("available_events"))?;
220 let reader = std::io::BufReader::new(file);
221
222 for line in reader.lines() {
223 for tp in line.unwrap().split_whitespace() {
224 if tracepoint == tp {
225 return Ok(true);
226 }
227 }
228 }
229
230 Ok(false)
231}
232
233pub fn cond_kprobe_enable<T>(sym: &str, prog_ptr: &OpenProgramImpl<T>) -> Result<bool> {
234 if in_kallsyms(sym)? {
235 unsafe {
236 bpf_program__set_autoload(prog_ptr.as_libbpf_object().as_ptr(), true);
237 }
238 return Ok(true);
239 } else {
240 warn!("symbol {} is missing, kprobe not loaded", sym);
241 }
242
243 Ok(false)
244}
245
246pub fn cond_kprobes_enable<T>(kprobes: Vec<(&str, &OpenProgramImpl<T>)>) -> Result<bool> {
247 for (sym, _) in kprobes.iter() {
249 if in_kallsyms(sym)? == false {
250 warn!("symbol {} is missing, kprobe not loaded", sym);
251 return Ok(false);
252 }
253 }
254
255 for (_, ptr) in kprobes.iter() {
257 unsafe {
258 bpf_program__set_autoload(ptr.as_libbpf_object().as_ptr(), true);
259 }
260 }
261
262 Ok(true)
263}
264
265pub fn cond_tracepoint_enable<T>(tracepoint: &str, prog_ptr: &OpenProgramImpl<T>) -> Result<bool> {
266 if tracepoint_exists(tracepoint)? {
267 unsafe {
268 bpf_program__set_autoload(prog_ptr.as_libbpf_object().as_ptr(), true);
269 }
270 return Ok(true);
271 } else {
272 warn!(
273 "tradepoint {} is missing, tracepoint not loaded",
274 tracepoint
275 );
276 }
277
278 Ok(false)
279}
280
281pub fn cond_tracepoints_enable<T>(tracepoints: Vec<(&str, &OpenProgramImpl<T>)>) -> Result<bool> {
282 for (tp, _) in tracepoints.iter() {
284 if tracepoint_exists(tp)? == false {
285 warn!("tradepoint {} is missing, tracepoint not loaded", tp);
286 return Ok(false);
287 }
288 }
289
290 for (_, ptr) in tracepoints.iter() {
292 unsafe {
293 bpf_program__set_autoload(ptr.as_libbpf_object().as_ptr(), true);
294 }
295 }
296
297 Ok(true)
298}
299
300pub fn is_sched_ext_enabled() -> io::Result<bool> {
301 let content = std::fs::read_to_string("/sys/kernel/sched_ext/state")?;
302
303 match content.trim() {
304 "enabled" => Ok(true),
305 "disabled" => Ok(false),
306 _ => {
307 Err(io::Error::new(
309 io::ErrorKind::InvalidData,
310 "Unexpected content in /sys/kernel/sched_ext/state",
311 ))
312 }
313 }
314}
315
316#[macro_export]
317macro_rules! unwrap_or_break {
318 ($expr: expr, $label: lifetime) => {{
319 match $expr {
320 Ok(val) => val,
321 Err(e) => break $label Err(e),
322 }
323 }};
324}
325
326pub fn check_min_requirements() -> Result<()> {
327 if let Ok(false) | Err(_) = struct_has_field("sched_ext_ops", "dump") {
330 bail!("sched_ext_ops.dump() missing, kernel too old?");
331 }
332 Ok(())
333}
334
335#[rustfmt::skip]
340#[macro_export]
341macro_rules! scx_ops_open {
342 ($builder: expr, $obj_ref: expr, $ops: ident) => { 'block: {
343 scx_utils::paste! {
344 scx_utils::unwrap_or_break!(scx_utils::compat::check_min_requirements(), 'block);
345 use ::anyhow::Context;
346 use ::libbpf_rs::skel::SkelBuilder;
347
348 let mut skel = match $builder.open($obj_ref).context("Failed to open BPF program") {
349 Ok(val) => val,
350 Err(e) => break 'block Err(e),
351 };
352
353 let ops = skel.struct_ops.[<$ops _mut>]();
354 let path = std::path::Path::new("/sys/kernel/sched_ext/hotplug_seq");
355
356 let val = match std::fs::read_to_string(&path) {
357 Ok(val) => val,
358 Err(_) => {
359 break 'block Err(anyhow::anyhow!("Failed to open or read file {:?}", path));
360 }
361 };
362
363 ops.hotplug_seq = match val.trim().parse::<u64>() {
364 Ok(parsed) => parsed,
365 Err(_) => {
366 break 'block Err(anyhow::anyhow!("Failed to parse hotplug seq {}", val));
367 }
368 };
369
370 if let Ok(s) = ::std::env::var("SCX_TIMEOUT_MS") {
371 skel.struct_ops.[<$ops _mut>]().timeout_ms = match s.parse::<u32>() {
372 Ok(ms) => {
373 ::scx_utils::info!("Setting timeout_ms to {} based on environment", ms);
374 ms
375 },
376 Err(e) => {
377 break 'block anyhow::Result::Err(e).context("SCX_TIMEOUT_MS has invalid value");
378 },
379 };
380 }
381
382 $crate::import_enums!(skel);
383
384 let result = ::anyhow::Result::Ok(skel);
385
386 result
387 }
388 }};
389}
390
391#[rustfmt::skip]
396#[macro_export]
397macro_rules! scx_ops_load {
398 ($skel: expr, $ops: ident, $uei: ident) => { 'block: {
399 scx_utils::paste! {
400 use ::anyhow::Context;
401 use ::libbpf_rs::skel::OpenSkel;
402
403 scx_utils::uei_set_size!($skel, $ops, $uei);
404 $skel.load().context("Failed to load BPF program")
405 }
406 }};
407}
408
409#[rustfmt::skip]
411#[macro_export]
412macro_rules! scx_ops_attach {
413 ($skel: expr, $ops: ident) => { 'block: {
414 use ::anyhow::Context;
415 use ::libbpf_rs::skel::Skel;
416
417 if scx_utils::compat::is_sched_ext_enabled().unwrap_or(false) {
418 break 'block Err(anyhow::anyhow!(
419 "another sched_ext scheduler is already running"
420 ));
421 }
422 $skel
423 .attach()
424 .context("Failed to attach non-struct_ops BPF programs")
425 .and_then(|_| {
426 $skel
427 .maps
428 .$ops
429 .attach_struct_ops()
430 .context("Failed to attach struct_ops BPF programs")
431 })
432 }};
433}
434
435#[cfg(test)]
436mod tests {
437 #[test]
438 fn test_read_enum() {
439 assert_eq!(super::read_enum("pid_type", "PIDTYPE_TGID").unwrap(), 1);
440 }
441
442 #[test]
443 fn test_struct_has_field() {
444 assert!(super::struct_has_field("task_struct", "flags").unwrap());
445 assert!(!super::struct_has_field("task_struct", "NO_SUCH_FIELD").unwrap());
446 assert!(super::struct_has_field("NO_SUCH_STRUCT", "NO_SUCH_FIELD").is_err());
447 }
448
449 #[test]
450 fn test_ksym_exists() {
451 assert!(super::ksym_exists("bpf_task_acquire").unwrap());
452 assert!(!super::ksym_exists("NO_SUCH_KFUNC").unwrap());
453 }
454}