scx_utils/
compat.rs

1// Copyright (c) Meta Platforms, Inc. and affiliates.
2//
3// This software may be used and distributed according to the terms of the
4// GNU General Public License version 2.
5
6use anyhow::{anyhow, bail, Context, Result};
7use libbpf_rs::libbpf_sys::*;
8use libbpf_rs::{AsRawLibbpf, OpenProgramImpl};
9use log::warn;
10use std::env;
11use std::ffi::c_void;
12use std::ffi::CStr;
13use std::ffi::CString;
14use std::io;
15use std::io::BufRead;
16use std::io::BufReader;
17use std::mem::size_of;
18use std::slice::from_raw_parts;
19
20const PROCFS_MOUNTS: &str = "/proc/mounts";
21const TRACEFS: &str = "tracefs";
22const DEBUGFS: &str = "debugfs";
23
24lazy_static::lazy_static! {
25    pub static ref SCX_OPS_KEEP_BUILTIN_IDLE: u64 =
26        read_enum("scx_ops_flags", "SCX_OPS_KEEP_BUILTIN_IDLE").unwrap_or(0);
27    pub static ref SCX_OPS_ENQ_LAST: u64 =
28        read_enum("scx_ops_flags", "SCX_OPS_ENQ_LAST").unwrap_or(0);
29    pub static ref SCX_OPS_ENQ_EXITING: u64 =
30        read_enum("scx_ops_flags", "SCX_OPS_ENQ_EXITING").unwrap_or(0);
31    pub static ref SCX_OPS_SWITCH_PARTIAL: u64 =
32        read_enum("scx_ops_flags", "SCX_OPS_SWITCH_PARTIAL").unwrap_or(0);
33    pub static ref SCX_OPS_ENQ_MIGRATION_DISABLED: u64 =
34        read_enum("scx_ops_flags", "SCX_OPS_ENQ_MIGRATION_DISABLED").unwrap_or(0);
35    pub static ref SCX_OPS_ALLOW_QUEUED_WAKEUP: u64 =
36        read_enum("scx_ops_flags", "SCX_OPS_ALLOW_QUEUED_WAKEUP").unwrap_or(0);
37    pub static ref SCX_OPS_BUILTIN_IDLE_PER_NODE: u64 =
38        read_enum("scx_ops_flags", "SCX_OPS_BUILTIN_IDLE_PER_NODE").unwrap_or(0);
39
40    pub static ref SCX_PICK_IDLE_CORE: u64 =
41        read_enum("scx_pick_idle_cpu_flags", "SCX_PICK_IDLE_CORE").unwrap_or(0);
42    pub static ref SCX_PICK_IDLE_IN_NODE: u64 =
43        read_enum("scx_pick_idle_cpu_flags", "SCX_PICK_IDLE_IN_NODE").unwrap_or(0);
44
45    pub static ref ROOT_PREFIX: String =
46        env::var("SCX_SYSFS_PREFIX").unwrap_or("".to_string());
47}
48
49fn load_vmlinux_btf() -> &'static mut btf {
50    let btf = unsafe { btf__load_vmlinux_btf() };
51    if btf.is_null() {
52        panic!("btf__load_vmlinux_btf() returned NULL, was CONFIG_DEBUG_INFO_BTF enabled?")
53    }
54    unsafe { &mut *btf }
55}
56
57lazy_static::lazy_static! {
58    static ref VMLINUX_BTF: &'static mut btf = load_vmlinux_btf();
59}
60
61fn btf_kind(t: &btf_type) -> u32 {
62    (t.info >> 24) & 0x1f
63}
64
65fn btf_vlen(t: &btf_type) -> u32 {
66    t.info & 0xffff
67}
68
69fn btf_type_plus_1(t: &btf_type) -> *const c_void {
70    let ptr_val = t as *const btf_type as usize;
71    (ptr_val + size_of::<btf_type>()) as *const c_void
72}
73
74fn btf_enum(t: &btf_type) -> &[btf_enum] {
75    let ptr = btf_type_plus_1(t);
76    unsafe { from_raw_parts(ptr as *const btf_enum, btf_vlen(t) as usize) }
77}
78
79fn btf_enum64(t: &btf_type) -> &[btf_enum64] {
80    let ptr = btf_type_plus_1(t);
81    unsafe { from_raw_parts(ptr as *const btf_enum64, btf_vlen(t) as usize) }
82}
83
84fn btf_members(t: &btf_type) -> &[btf_member] {
85    let ptr = btf_type_plus_1(t);
86    unsafe { from_raw_parts(ptr as *const btf_member, btf_vlen(t) as usize) }
87}
88
89fn btf_name_str_by_offset(btf: &btf, name_off: u32) -> Result<&str> {
90    let n = unsafe { btf__name_by_offset(btf, name_off) };
91    if n.is_null() {
92        bail!("btf__name_by_offset() returned NULL");
93    }
94    Ok(unsafe { CStr::from_ptr(n) }
95        .to_str()
96        .with_context(|| format!("Failed to convert {:?} to string", n))?)
97}
98
99pub fn read_enum(type_name: &str, name: &str) -> Result<u64> {
100    let btf: &btf = *VMLINUX_BTF;
101
102    let type_name = CString::new(type_name).unwrap();
103    let tid = unsafe { btf__find_by_name(btf, type_name.as_ptr()) };
104    if tid < 0 {
105        bail!("type {:?} doesn't exist, ret={}", type_name, tid);
106    }
107
108    let t = unsafe { btf__type_by_id(btf, tid as _) };
109    if t.is_null() {
110        bail!("btf__type_by_id({}) returned NULL", tid);
111    }
112    let t = unsafe { &*t };
113
114    match btf_kind(t) {
115        BTF_KIND_ENUM => {
116            for e in btf_enum(t).iter() {
117                if btf_name_str_by_offset(btf, e.name_off)? == name {
118                    return Ok(e.val as u64);
119                }
120            }
121        }
122        BTF_KIND_ENUM64 => {
123            for e in btf_enum64(t).iter() {
124                if btf_name_str_by_offset(btf, e.name_off)? == name {
125                    return Ok(((e.val_hi32 as u64) << 32) | (e.val_lo32) as u64);
126                }
127            }
128        }
129        _ => (),
130    }
131
132    Err(anyhow!("{:?} doesn't exist in {:?}", name, type_name))
133}
134
135pub fn struct_has_field(type_name: &str, field: &str) -> Result<bool> {
136    let btf: &btf = *VMLINUX_BTF;
137
138    let type_name = CString::new(type_name).unwrap();
139    let tid = unsafe { btf__find_by_name_kind(btf, type_name.as_ptr(), BTF_KIND_STRUCT) };
140    if tid < 0 {
141        bail!("type {:?} doesn't exist, ret={}", type_name, tid);
142    }
143
144    let t = unsafe { btf__type_by_id(btf, tid as _) };
145    if t.is_null() {
146        bail!("btf__type_by_id({}) returned NULL", tid);
147    }
148    let t = unsafe { &*t };
149
150    for m in btf_members(t).iter() {
151        if btf_name_str_by_offset(btf, m.name_off)? == field {
152            return Ok(true);
153        }
154    }
155
156    Ok(false)
157}
158
159pub fn ksym_exists(ksym: &str) -> Result<bool> {
160    let btf: &btf = *VMLINUX_BTF;
161
162    let ksym_name = CString::new(ksym).unwrap();
163    let tid = unsafe { btf__find_by_name(btf, ksym_name.as_ptr()) };
164    Ok(tid >= 0)
165}
166
167pub fn in_kallsyms(ksym: &str) -> Result<bool> {
168    let file = std::fs::File::open("/proc/kallsyms")?;
169    let reader = std::io::BufReader::new(file);
170
171    for line in reader.lines() {
172        for sym in line.unwrap().split_whitespace() {
173            if ksym == sym {
174                return Ok(true);
175            }
176        }
177    }
178
179    Ok(false)
180}
181
182/// Returns the mount point for a filesystem type.
183pub fn get_fs_mount(mount_type: &str) -> Result<Vec<std::path::PathBuf>> {
184    let proc_mounts_path = std::path::Path::new(PROCFS_MOUNTS);
185
186    let file = std::fs::File::open(proc_mounts_path)
187        .with_context(|| format!("Failed to open {}", proc_mounts_path.display()))?;
188
189    let reader = BufReader::new(file);
190
191    let mut mounts = Vec::new();
192    for line in reader.lines() {
193        let line = line.context("Failed to read line from /proc/mounts")?;
194        let mount_info: Vec<&str> = line.split_whitespace().collect();
195
196        if mount_info.len() > 3 && mount_info[2] == mount_type {
197            let mount_path = std::path::PathBuf::from(mount_info[1]);
198            mounts.push(mount_path);
199        }
200    }
201
202    Ok(mounts)
203}
204
205/// Returns the tracefs mount point.
206pub fn tracefs_mount() -> Result<std::path::PathBuf> {
207    let mounts = get_fs_mount(TRACEFS)?;
208    mounts.into_iter().next().context("No tracefs mount found")
209}
210
211/// Returns the debugfs mount point.
212pub fn debugfs_mount() -> Result<std::path::PathBuf> {
213    let mounts = get_fs_mount(DEBUGFS)?;
214    mounts.into_iter().next().context("No debugfs mount found")
215}
216
217pub fn tracepoint_exists(tracepoint: &str) -> Result<bool> {
218    let base_path = tracefs_mount().unwrap_or_else(|_| debugfs_mount().unwrap().join("tracing"));
219    let file = std::fs::File::open(base_path.join("available_events"))?;
220    let reader = std::io::BufReader::new(file);
221
222    for line in reader.lines() {
223        for tp in line.unwrap().split_whitespace() {
224            if tracepoint == tp {
225                return Ok(true);
226            }
227        }
228    }
229
230    Ok(false)
231}
232
233pub fn cond_kprobe_enable<T>(sym: &str, prog_ptr: &OpenProgramImpl<T>) -> Result<bool> {
234    if in_kallsyms(sym)? {
235        unsafe {
236            bpf_program__set_autoload(prog_ptr.as_libbpf_object().as_ptr(), true);
237        }
238        return Ok(true);
239    } else {
240        warn!("symbol {sym} is missing, kprobe not loaded");
241    }
242
243    Ok(false)
244}
245
246pub fn cond_kprobes_enable<T>(kprobes: Vec<(&str, &OpenProgramImpl<T>)>) -> Result<bool> {
247    // Check if all the symbols exist.
248    for (sym, _) in kprobes.iter() {
249        if in_kallsyms(sym)? == false {
250            warn!("symbol {sym} is missing, kprobe not loaded");
251            return Ok(false);
252        }
253    }
254
255    // Enable all the tracepoints.
256    for (_, ptr) in kprobes.iter() {
257        unsafe {
258            bpf_program__set_autoload(ptr.as_libbpf_object().as_ptr(), true);
259        }
260    }
261
262    Ok(true)
263}
264
265pub fn cond_tracepoint_enable<T>(tracepoint: &str, prog_ptr: &OpenProgramImpl<T>) -> Result<bool> {
266    if tracepoint_exists(tracepoint)? {
267        unsafe {
268            bpf_program__set_autoload(prog_ptr.as_libbpf_object().as_ptr(), true);
269        }
270        return Ok(true);
271    } else {
272        warn!("tradepoint {tracepoint} is missing, tracepoint not loaded");
273    }
274
275    Ok(false)
276}
277
278pub fn cond_tracepoints_enable<T>(tracepoints: Vec<(&str, &OpenProgramImpl<T>)>) -> Result<bool> {
279    // Check if all the tracepoints exist.
280    for (tp, _) in tracepoints.iter() {
281        if tracepoint_exists(tp)? == false {
282            warn!("tradepoint {tp} is missing, tracepoint not loaded");
283            return Ok(false);
284        }
285    }
286
287    // Enable all the tracepoints.
288    for (_, ptr) in tracepoints.iter() {
289        unsafe {
290            bpf_program__set_autoload(ptr.as_libbpf_object().as_ptr(), true);
291        }
292    }
293
294    Ok(true)
295}
296
297pub fn is_sched_ext_enabled() -> io::Result<bool> {
298    let content = std::fs::read_to_string("/sys/kernel/sched_ext/state")?;
299
300    match content.trim() {
301        "enabled" => Ok(true),
302        "disabled" => Ok(false),
303        _ => {
304            // Error if the content is neither "enabled" nor "disabled"
305            Err(io::Error::new(
306                io::ErrorKind::InvalidData,
307                "Unexpected content in /sys/kernel/sched_ext/state",
308            ))
309        }
310    }
311}
312
313#[macro_export]
314macro_rules! unwrap_or_break {
315    ($expr: expr, $label: lifetime) => {{
316        match $expr {
317            Ok(val) => val,
318            Err(e) => break $label Err(e),
319        }
320    }};
321}
322
323pub fn check_min_requirements() -> Result<()> {
324    // ec7e3b0463e1 ("implement-ops") in https://github.com/sched-ext/sched_ext
325    // is the current minimum required kernel version.
326    if let Ok(false) | Err(_) = struct_has_field("sched_ext_ops", "dump") {
327        bail!("sched_ext_ops.dump() missing, kernel too old?");
328    }
329    Ok(())
330}
331
332/// struct sched_ext_ops can change over time. If compat.bpf.h::SCX_OPS_DEFINE()
333/// is used to define ops, and scx_ops_open!(), scx_ops_load!(), and
334/// scx_ops_attach!() are used to open, load and attach it, backward
335/// compatibility is automatically maintained where reasonable.
336#[rustfmt::skip]
337#[macro_export]
338macro_rules! scx_ops_open {
339    ($builder: expr, $obj_ref: expr, $ops: ident, $open_opts: expr) => { 'block: {
340        scx_utils::paste! {
341        scx_utils::unwrap_or_break!(scx_utils::compat::check_min_requirements(), 'block);
342            use ::anyhow::Context;
343            use ::libbpf_rs::skel::SkelBuilder;
344
345            let mut skel = match $open_opts {
346                Some(opts_ref) => { // Match a reference directly
347                    match $builder.open_opts(opts_ref, $obj_ref).context("Failed to open BPF program with options") {
348                        Ok(val) => val,
349                        Err(e) => break 'block Err(e),
350                    }
351                }
352                None => {
353                    match $builder.open($obj_ref).context("Failed to open BPF program") {
354                        Ok(val) => val,
355                        Err(e) => break 'block Err(e),
356                    }
357                }
358            };
359
360            let ops = skel.struct_ops.[<$ops _mut>]();
361            let path = std::path::Path::new("/sys/kernel/sched_ext/hotplug_seq");
362
363            let val = match std::fs::read_to_string(&path) {
364                Ok(val) => val,
365                Err(_) => {
366                    break 'block Err(anyhow::anyhow!("Failed to open or read file {:?}", path));
367                }
368            };
369
370            ops.hotplug_seq = match val.trim().parse::<u64>() {
371                Ok(parsed) => parsed,
372                Err(_) => {
373                    break 'block Err(anyhow::anyhow!("Failed to parse hotplug seq {}", val));
374                }
375            };
376
377            if let Ok(s) = ::std::env::var("SCX_TIMEOUT_MS") {
378                skel.struct_ops.[<$ops _mut>]().timeout_ms = match s.parse::<u32>() {
379                    Ok(ms) => {
380                        ::scx_utils::info!("Setting timeout_ms to {} based on environment", ms);
381                        ms
382                    },
383                    Err(e) => {
384                        break 'block anyhow::Result::Err(e).context("SCX_TIMEOUT_MS has invalid value");
385                    },
386                };
387            }
388
389            {
390                let ops = skel.struct_ops.[<$ops _mut>]();
391
392                let name_field = &mut ops.name;
393
394                let version_suffix = ::scx_utils::build_id::ops_version_suffix(env!("CARGO_PKG_VERSION"));
395                let bytes = version_suffix.as_bytes();
396                let mut i = 0;
397                let mut bytes_idx = 0;
398                let mut found_null = false;
399
400                while i < name_field.len() - 1 {
401                    found_null |= name_field[i] == 0;
402                    if !found_null {
403                        i += 1;
404                        continue;
405                    }
406
407                    if bytes_idx < bytes.len() {
408                        name_field[i] = bytes[bytes_idx] as i8;
409                        bytes_idx += 1;
410                    } else {
411                        break;
412                    }
413                    i += 1;
414                }
415                name_field[i] = 0;
416            }
417
418            $crate::import_enums!(skel);
419
420            let result = ::anyhow::Result::Ok(skel);
421
422            result
423        }
424    }};
425}
426
427/// struct sched_ext_ops can change over time. If compat.bpf.h::SCX_OPS_DEFINE()
428/// is used to define ops, and scx_ops_open!(), scx_ops_load!(), and
429/// scx_ops_attach!() are used to open, load and attach it, backward
430/// compatibility is automatically maintained where reasonable.
431#[rustfmt::skip]
432#[macro_export]
433macro_rules! scx_ops_load {
434    ($skel: expr, $ops: ident, $uei: ident) => { 'block: {
435        scx_utils::paste! {
436            use ::anyhow::Context;
437            use ::libbpf_rs::skel::OpenSkel;
438
439            scx_utils::uei_set_size!($skel, $ops, $uei);
440            $skel.load().context("Failed to load BPF program")
441        }
442    }};
443}
444
445/// Must be used together with scx_ops_load!(). See there.
446#[rustfmt::skip]
447#[macro_export]
448macro_rules! scx_ops_attach {
449    ($skel: expr, $ops: ident) => { 'block: {
450        use ::anyhow::Context;
451        use ::libbpf_rs::skel::Skel;
452
453        if scx_utils::compat::is_sched_ext_enabled().unwrap_or(false) {
454            break 'block Err(anyhow::anyhow!(
455                "another sched_ext scheduler is already running"
456            ));
457        }
458        $skel
459            .attach()
460            .context("Failed to attach non-struct_ops BPF programs")
461            .and_then(|_| {
462                $skel
463                    .maps
464                    .$ops
465                    .attach_struct_ops()
466                    .context("Failed to attach struct_ops BPF programs")
467            })
468    }};
469}
470
471#[cfg(test)]
472mod tests {
473    #[test]
474    fn test_read_enum() {
475        assert_eq!(super::read_enum("pid_type", "PIDTYPE_TGID").unwrap(), 1);
476    }
477
478    #[test]
479    fn test_struct_has_field() {
480        assert!(super::struct_has_field("task_struct", "flags").unwrap());
481        assert!(!super::struct_has_field("task_struct", "NO_SUCH_FIELD").unwrap());
482        assert!(super::struct_has_field("NO_SUCH_STRUCT", "NO_SUCH_FIELD").is_err());
483    }
484
485    #[test]
486    fn test_ksym_exists() {
487        assert!(super::ksym_exists("bpf_task_acquire").unwrap());
488        assert!(!super::ksym_exists("NO_SUCH_KFUNC").unwrap());
489    }
490}
scx_utils/compat.rs

scx_utils/
compat.rs
