scx_utils/
compat.rs

1// Copyright (c) Meta Platforms, Inc. and affiliates.
2//
3// This software may be used and distributed according to the terms of the
4// GNU General Public License version 2.
5
6use anyhow::{anyhow, bail, Context, Result};
7use libbpf_rs::libbpf_sys::*;
8use libbpf_rs::{AsRawLibbpf, OpenProgramImpl};
9use log::warn;
10use std::env;
11use std::ffi::c_void;
12use std::ffi::CStr;
13use std::ffi::CString;
14use std::io;
15use std::io::BufRead;
16use std::io::BufReader;
17use std::mem::size_of;
18use std::slice::from_raw_parts;
19
20const PROCFS_MOUNTS: &str = "/proc/mounts";
21const TRACEFS: &str = "tracefs";
22const DEBUGFS: &str = "debugfs";
23
24lazy_static::lazy_static! {
25    pub static ref SCX_OPS_KEEP_BUILTIN_IDLE: u64 =
26        read_enum("scx_ops_flags", "SCX_OPS_KEEP_BUILTIN_IDLE").unwrap_or(0);
27    pub static ref SCX_OPS_ENQ_LAST: u64 =
28        read_enum("scx_ops_flags", "SCX_OPS_ENQ_LAST").unwrap_or(0);
29    pub static ref SCX_OPS_ENQ_EXITING: u64 =
30        read_enum("scx_ops_flags", "SCX_OPS_ENQ_EXITING").unwrap_or(0);
31    pub static ref SCX_OPS_SWITCH_PARTIAL: u64 =
32        read_enum("scx_ops_flags", "SCX_OPS_SWITCH_PARTIAL").unwrap_or(0);
33    pub static ref SCX_OPS_ENQ_MIGRATION_DISABLED: u64 =
34        read_enum("scx_ops_flags", "SCX_OPS_ENQ_MIGRATION_DISABLED").unwrap_or(0);
35    pub static ref SCX_OPS_ALLOW_QUEUED_WAKEUP: u64 =
36        read_enum("scx_ops_flags", "SCX_OPS_ALLOW_QUEUED_WAKEUP").unwrap_or(0);
37    pub static ref SCX_OPS_BUILTIN_IDLE_PER_NODE: u64 =
38        read_enum("scx_ops_flags", "SCX_OPS_BUILTIN_IDLE_PER_NODE").unwrap_or(0);
39
40    pub static ref SCX_PICK_IDLE_CORE: u64 =
41        read_enum("scx_pick_idle_cpu_flags", "SCX_PICK_IDLE_CORE").unwrap_or(0);
42    pub static ref SCX_PICK_IDLE_IN_NODE: u64 =
43        read_enum("scx_pick_idle_cpu_flags", "SCX_PICK_IDLE_IN_NODE").unwrap_or(0);
44
45    pub static ref ROOT_PREFIX: String =
46        env::var("SCX_SYSFS_PREFIX").unwrap_or("".to_string());
47}
48
49fn load_vmlinux_btf() -> &'static mut btf {
50    let btf = unsafe { btf__load_vmlinux_btf() };
51    if btf.is_null() {
52        panic!("btf__load_vmlinux_btf() returned NULL, was CONFIG_DEBUG_INFO_BTF enabled?")
53    }
54    unsafe { &mut *btf }
55}
56
57lazy_static::lazy_static! {
58    static ref VMLINUX_BTF: &'static mut btf = load_vmlinux_btf();
59}
60
61fn btf_kind(t: &btf_type) -> u32 {
62    (t.info >> 24) & 0x1f
63}
64
65fn btf_vlen(t: &btf_type) -> u32 {
66    t.info & 0xffff
67}
68
69fn btf_type_plus_1(t: &btf_type) -> *const c_void {
70    let ptr_val = t as *const btf_type as usize;
71    (ptr_val + size_of::<btf_type>()) as *const c_void
72}
73
74fn btf_enum(t: &btf_type) -> &[btf_enum] {
75    let ptr = btf_type_plus_1(t);
76    unsafe { from_raw_parts(ptr as *const btf_enum, btf_vlen(t) as usize) }
77}
78
79fn btf_enum64(t: &btf_type) -> &[btf_enum64] {
80    let ptr = btf_type_plus_1(t);
81    unsafe { from_raw_parts(ptr as *const btf_enum64, btf_vlen(t) as usize) }
82}
83
84fn btf_members(t: &btf_type) -> &[btf_member] {
85    let ptr = btf_type_plus_1(t);
86    unsafe { from_raw_parts(ptr as *const btf_member, btf_vlen(t) as usize) }
87}
88
89fn btf_name_str_by_offset(btf: &btf, name_off: u32) -> Result<&str> {
90    let n = unsafe { btf__name_by_offset(btf, name_off) };
91    if n.is_null() {
92        bail!("btf__name_by_offset() returned NULL");
93    }
94    Ok(unsafe { CStr::from_ptr(n) }
95        .to_str()
96        .with_context(|| format!("Failed to convert {:?} to string", n))?)
97}
98
99pub fn read_enum(type_name: &str, name: &str) -> Result<u64> {
100    let btf: &btf = *VMLINUX_BTF;
101
102    let type_name = CString::new(type_name).unwrap();
103    let tid = unsafe { btf__find_by_name(btf, type_name.as_ptr()) };
104    if tid < 0 {
105        bail!("type {:?} doesn't exist, ret={}", type_name, tid);
106    }
107
108    let t = unsafe { btf__type_by_id(btf, tid as _) };
109    if t.is_null() {
110        bail!("btf__type_by_id({}) returned NULL", tid);
111    }
112    let t = unsafe { &*t };
113
114    match btf_kind(t) {
115        BTF_KIND_ENUM => {
116            for e in btf_enum(t).iter() {
117                if btf_name_str_by_offset(btf, e.name_off)? == name {
118                    return Ok(e.val as u64);
119                }
120            }
121        }
122        BTF_KIND_ENUM64 => {
123            for e in btf_enum64(t).iter() {
124                if btf_name_str_by_offset(btf, e.name_off)? == name {
125                    return Ok(((e.val_hi32 as u64) << 32) | (e.val_lo32) as u64);
126                }
127            }
128        }
129        _ => (),
130    }
131
132    Err(anyhow!("{:?} doesn't exist in {:?}", name, type_name))
133}
134
135pub fn struct_has_field(type_name: &str, field: &str) -> Result<bool> {
136    let btf: &btf = *VMLINUX_BTF;
137
138    let type_name = CString::new(type_name).unwrap();
139    let tid = unsafe { btf__find_by_name_kind(btf, type_name.as_ptr(), BTF_KIND_STRUCT) };
140    if tid < 0 {
141        bail!("type {:?} doesn't exist, ret={}", type_name, tid);
142    }
143
144    let t = unsafe { btf__type_by_id(btf, tid as _) };
145    if t.is_null() {
146        bail!("btf__type_by_id({}) returned NULL", tid);
147    }
148    let t = unsafe { &*t };
149
150    for m in btf_members(t).iter() {
151        if btf_name_str_by_offset(btf, m.name_off)? == field {
152            return Ok(true);
153        }
154    }
155
156    Ok(false)
157}
158
159pub fn ksym_exists(ksym: &str) -> Result<bool> {
160    let btf: &btf = *VMLINUX_BTF;
161
162    let ksym_name = CString::new(ksym).unwrap();
163    let tid = unsafe { btf__find_by_name(btf, ksym_name.as_ptr()) };
164    Ok(tid >= 0)
165}
166
167pub fn in_kallsyms(ksym: &str) -> Result<bool> {
168    let file = std::fs::File::open("/proc/kallsyms")?;
169    let reader = std::io::BufReader::new(file);
170
171    for line in reader.lines() {
172        for sym in line.unwrap().split_whitespace() {
173            if ksym == sym {
174                return Ok(true);
175            }
176        }
177    }
178
179    Ok(false)
180}
181
182/// Returns the mount point for a filesystem type.
183pub fn get_fs_mount(mount_type: &str) -> Result<Vec<std::path::PathBuf>> {
184    let proc_mounts_path = std::path::Path::new(PROCFS_MOUNTS);
185
186    let file = std::fs::File::open(proc_mounts_path)
187        .with_context(|| format!("Failed to open {}", proc_mounts_path.display()))?;
188
189    let reader = BufReader::new(file);
190
191    let mut mounts = Vec::new();
192    for line in reader.lines() {
193        let line = line.context("Failed to read line from /proc/mounts")?;
194        let mount_info: Vec<&str> = line.split_whitespace().collect();
195
196        if mount_info.len() > 3 && mount_info[2] == mount_type {
197            let mount_path = std::path::PathBuf::from(mount_info[1]);
198            mounts.push(mount_path);
199        }
200    }
201
202    Ok(mounts)
203}
204
205/// Returns the tracefs mount point.
206pub fn tracefs_mount() -> Result<std::path::PathBuf> {
207    let mounts = get_fs_mount(TRACEFS)?;
208    mounts.into_iter().next().context("No tracefs mount found")
209}
210
211/// Returns the debugfs mount point.
212pub fn debugfs_mount() -> Result<std::path::PathBuf> {
213    let mounts = get_fs_mount(DEBUGFS)?;
214    mounts.into_iter().next().context("No debugfs mount found")
215}
216
217pub fn tracepoint_exists(tracepoint: &str) -> Result<bool> {
218    let base_path = tracefs_mount().unwrap_or_else(|_| debugfs_mount().unwrap().join("tracing"));
219    let file = std::fs::File::open(base_path.join("available_events"))?;
220    let reader = std::io::BufReader::new(file);
221
222    for line in reader.lines() {
223        for tp in line.unwrap().split_whitespace() {
224            if tracepoint == tp {
225                return Ok(true);
226            }
227        }
228    }
229
230    Ok(false)
231}
232
233pub fn cond_kprobe_enable<T>(sym: &str, prog_ptr: &OpenProgramImpl<T>) -> Result<bool> {
234    if in_kallsyms(sym)? {
235        unsafe {
236            bpf_program__set_autoload(prog_ptr.as_libbpf_object().as_ptr(), true);
237        }
238        return Ok(true);
239    } else {
240        warn!("symbol {} is missing, kprobe not loaded", sym);
241    }
242
243    Ok(false)
244}
245
246pub fn cond_kprobes_enable<T>(kprobes: Vec<(&str, &OpenProgramImpl<T>)>) -> Result<bool> {
247    // Check if all the symbols exist.
248    for (sym, _) in kprobes.iter() {
249        if in_kallsyms(sym)? == false {
250            warn!("symbol {} is missing, kprobe not loaded", sym);
251            return Ok(false);
252        }
253    }
254
255    // Enable all the tracepoints.
256    for (_, ptr) in kprobes.iter() {
257        unsafe {
258            bpf_program__set_autoload(ptr.as_libbpf_object().as_ptr(), true);
259        }
260    }
261
262    Ok(true)
263}
264
265pub fn cond_tracepoint_enable<T>(tracepoint: &str, prog_ptr: &OpenProgramImpl<T>) -> Result<bool> {
266    if tracepoint_exists(tracepoint)? {
267        unsafe {
268            bpf_program__set_autoload(prog_ptr.as_libbpf_object().as_ptr(), true);
269        }
270        return Ok(true);
271    } else {
272        warn!(
273            "tradepoint {} is missing, tracepoint not loaded",
274            tracepoint
275        );
276    }
277
278    Ok(false)
279}
280
281pub fn cond_tracepoints_enable<T>(tracepoints: Vec<(&str, &OpenProgramImpl<T>)>) -> Result<bool> {
282    // Check if all the tracepoints exist.
283    for (tp, _) in tracepoints.iter() {
284        if tracepoint_exists(tp)? == false {
285            warn!("tradepoint {} is missing, tracepoint not loaded", tp);
286            return Ok(false);
287        }
288    }
289
290    // Enable all the tracepoints.
291    for (_, ptr) in tracepoints.iter() {
292        unsafe {
293            bpf_program__set_autoload(ptr.as_libbpf_object().as_ptr(), true);
294        }
295    }
296
297    Ok(true)
298}
299
300pub fn is_sched_ext_enabled() -> io::Result<bool> {
301    let content = std::fs::read_to_string("/sys/kernel/sched_ext/state")?;
302
303    match content.trim() {
304        "enabled" => Ok(true),
305        "disabled" => Ok(false),
306        _ => {
307            // Error if the content is neither "enabled" nor "disabled"
308            Err(io::Error::new(
309                io::ErrorKind::InvalidData,
310                "Unexpected content in /sys/kernel/sched_ext/state",
311            ))
312        }
313    }
314}
315
316#[macro_export]
317macro_rules! unwrap_or_break {
318    ($expr: expr, $label: lifetime) => {{
319        match $expr {
320            Ok(val) => val,
321            Err(e) => break $label Err(e),
322        }
323    }};
324}
325
326pub fn check_min_requirements() -> Result<()> {
327    // ec7e3b0463e1 ("implement-ops") in https://github.com/sched-ext/sched_ext
328    // is the current minimum required kernel version.
329    if let Ok(false) | Err(_) = struct_has_field("sched_ext_ops", "dump") {
330        bail!("sched_ext_ops.dump() missing, kernel too old?");
331    }
332    Ok(())
333}
334
335/// struct sched_ext_ops can change over time. If compat.bpf.h::SCX_OPS_DEFINE()
336/// is used to define ops, and scx_ops_open!(), scx_ops_load!(), and
337/// scx_ops_attach!() are used to open, load and attach it, backward
338/// compatibility is automatically maintained where reasonable.
339#[rustfmt::skip]
340#[macro_export]
341macro_rules! scx_ops_open {
342    ($builder: expr, $obj_ref: expr, $ops: ident) => { 'block: {
343        scx_utils::paste! {
344	    scx_utils::unwrap_or_break!(scx_utils::compat::check_min_requirements(), 'block);
345            use ::anyhow::Context;
346            use ::libbpf_rs::skel::SkelBuilder;
347
348            let mut skel = match $builder.open($obj_ref).context("Failed to open BPF program") {
349                Ok(val) => val,
350                Err(e) => break 'block Err(e),
351            };
352
353            let ops = skel.struct_ops.[<$ops _mut>]();
354            let path = std::path::Path::new("/sys/kernel/sched_ext/hotplug_seq");
355
356            let val = match std::fs::read_to_string(&path) {
357                Ok(val) => val,
358                Err(_) => {
359                    break 'block Err(anyhow::anyhow!("Failed to open or read file {:?}", path));
360                }
361            };
362
363            ops.hotplug_seq = match val.trim().parse::<u64>() {
364                Ok(parsed) => parsed,
365                Err(_) => {
366                    break 'block Err(anyhow::anyhow!("Failed to parse hotplug seq {}", val));
367                }
368            };
369
370            if let Ok(s) = ::std::env::var("SCX_TIMEOUT_MS") {
371                skel.struct_ops.[<$ops _mut>]().timeout_ms = match s.parse::<u32>() {
372                    Ok(ms) => {
373                        ::scx_utils::info!("Setting timeout_ms to {} based on environment", ms);
374                        ms
375                    },
376                    Err(e) => {
377                        break 'block anyhow::Result::Err(e).context("SCX_TIMEOUT_MS has invalid value");
378                    },
379                };
380            }
381
382            $crate::import_enums!(skel);
383
384            let result = ::anyhow::Result::Ok(skel);
385
386            result
387        }
388    }};
389}
390
391/// struct sched_ext_ops can change over time. If compat.bpf.h::SCX_OPS_DEFINE()
392/// is used to define ops, and scx_ops_open!(), scx_ops_load!(), and
393/// scx_ops_attach!() are used to open, load and attach it, backward
394/// compatibility is automatically maintained where reasonable.
395#[rustfmt::skip]
396#[macro_export]
397macro_rules! scx_ops_load {
398    ($skel: expr, $ops: ident, $uei: ident) => { 'block: {
399        scx_utils::paste! {
400            use ::anyhow::Context;
401            use ::libbpf_rs::skel::OpenSkel;
402
403            scx_utils::uei_set_size!($skel, $ops, $uei);
404            $skel.load().context("Failed to load BPF program")
405        }
406    }};
407}
408
409/// Must be used together with scx_ops_load!(). See there.
410#[rustfmt::skip]
411#[macro_export]
412macro_rules! scx_ops_attach {
413    ($skel: expr, $ops: ident) => { 'block: {
414        use ::anyhow::Context;
415        use ::libbpf_rs::skel::Skel;
416
417        if scx_utils::compat::is_sched_ext_enabled().unwrap_or(false) {
418            break 'block Err(anyhow::anyhow!(
419                "another sched_ext scheduler is already running"
420            ));
421        }
422        $skel
423            .attach()
424            .context("Failed to attach non-struct_ops BPF programs")
425            .and_then(|_| {
426                $skel
427                    .maps
428                    .$ops
429                    .attach_struct_ops()
430                    .context("Failed to attach struct_ops BPF programs")
431            })
432    }};
433}
434
435#[cfg(test)]
436mod tests {
437    #[test]
438    fn test_read_enum() {
439        assert_eq!(super::read_enum("pid_type", "PIDTYPE_TGID").unwrap(), 1);
440    }
441
442    #[test]
443    fn test_struct_has_field() {
444        assert!(super::struct_has_field("task_struct", "flags").unwrap());
445        assert!(!super::struct_has_field("task_struct", "NO_SUCH_FIELD").unwrap());
446        assert!(super::struct_has_field("NO_SUCH_STRUCT", "NO_SUCH_FIELD").is_err());
447    }
448
449    #[test]
450    fn test_ksym_exists() {
451        assert!(super::ksym_exists("bpf_task_acquire").unwrap());
452        assert!(!super::ksym_exists("NO_SUCH_KFUNC").unwrap());
453    }
454}