scx_utils/
perf.rs

1#![allow(non_upper_case_globals)]
2#![allow(non_camel_case_types)]
3#![allow(non_snake_case)]
4
5use libc::pid_t;
6use std::os::raw::{c_int, c_ulong};
7
8/// The `perf_event_open` system call.
9///
10/// See the [`perf_event_open(2) man page`][man] for details.
11///
12/// On error, this returns -1, and the C `errno` value (accessible via
13/// `std::io::Error::last_os_error`) is set to indicate the error.
14///
15/// Note: The `attrs` argument needs to be a `*mut` because if the `size` field
16/// is too small or too large, the kernel writes the size it was expecting back
17/// into that field. It might do other things as well.
18///
19/// # Safety
20///
21/// The `attrs` argument must point to a properly initialized
22/// `perf_event_attr` struct. The measurements and other behaviors its
23/// contents request must be safe.
24///
25/// [man]: https://www.mankier.com/2/perf_event_open
26pub unsafe fn perf_event_open(
27    attrs: *mut bindings::perf_event_attr,
28    pid: pid_t,
29    cpu: c_int,
30    group_fd: c_int,
31    flags: c_ulong,
32) -> c_int {
33    unsafe {
34        libc::syscall(
35            bindings::__NR_perf_event_open as libc::c_long,
36            attrs as *const bindings::perf_event_attr,
37            pid,
38            cpu,
39            group_fd,
40            flags,
41        ) as c_int
42    }
43}
44
45pub mod bindings {
46    include!(concat!(env!("OUT_DIR"), "/perf_bindings.rs"));
47}
48
49pub mod ioctls {
50    use crate::perf;
51    use std::os::raw::{c_int, c_uint};
52
53    #[allow(clippy::missing_safety_doc)]
54    pub unsafe fn enable(fd: c_int, arg: c_uint) -> c_int {
55        unsafe { libc::ioctl(fd, perf::bindings::ENABLE as libc::Ioctl, arg) }
56    }
57
58    #[allow(clippy::missing_safety_doc)]
59    pub unsafe fn reset(fd: c_int, arg: c_uint) -> c_int {
60        unsafe { libc::ioctl(fd, perf::bindings::RESET as libc::Ioctl, arg) }
61    }
62}
63
64use anyhow::Context as _;
65use anyhow::Result;
66use libbpf_rs::MapCore;
67use libbpf_rs::MapFlags;
68
69/// Must match lib/pmu.bpf.c SCX_PMU_STRIDE for the perf_events map key layout.
70pub const PERF_MAP_STRIDE: u32 = 4096;
71
72/// Perf event specification: either hex (0xN) or a symbolic name (e.g.
73/// cache-misses). `event_id` is the opaque id written to BPF rodata (it must
74/// match between install and read); `type_`/`config` drive perf_event_open.
75#[derive(Clone, Debug)]
76pub struct PerfEventSpec {
77    /// Opaque id for BPF (must match between install and read).
78    pub event_id: u64,
79    /// perf_event_attr.type (PERF_TYPE_RAW, PERF_TYPE_HARDWARE, etc.).
80    pub type_: u32,
81    /// perf_event_attr.config.
82    pub config: u64,
83    /// Original string for error messages.
84    pub display_name: String,
85}
86
87fn parse_hardware_event(s: &str) -> Option<u64> {
88    match s {
89        "cpu-cycles" | "cycles" => Some(0),
90        "instructions" => Some(1),
91        "cache-references" => Some(2),
92        "cache-misses" => Some(3),
93        "branch-instructions" | "branches" => Some(4),
94        "branch-misses" => Some(5),
95        "bus-cycles" => Some(6),
96        "stalled-cycles-frontend" | "idle-cycles-frontend" => Some(7),
97        "stalled-cycles-backend" | "idle-cycles-backend" => Some(8),
98        "ref-cycles" => Some(9),
99        _ => None,
100    }
101}
102
103fn parse_software_event(s: &str) -> Option<u64> {
104    match s {
105        "cpu-clock" => Some(0),
106        "task-clock" => Some(1),
107        "page-faults" | "faults" => Some(2),
108        "context-switches" | "cs" => Some(3),
109        "cpu-migrations" | "migrations" => Some(4),
110        "minor-faults" => Some(5),
111        "major-faults" => Some(6),
112        "alignment-faults" => Some(7),
113        "emulation-faults" => Some(8),
114        "dummy" => Some(9),
115        "bpf-output" => Some(10),
116        _ => None,
117    }
118}
119
120fn parse_hw_cache_event(s: &str) -> Option<u64> {
121    let (cache_id, prefix_len) = if s.starts_with("L1-dcache-") {
122        (0, 10)
123    } else if s.starts_with("L1-icache-") {
124        (1, 10)
125    } else if s.starts_with("LLC-") {
126        (2, 4)
127    } else if s.starts_with("dTLB-") {
128        (3, 5)
129    } else if s.starts_with("iTLB-") {
130        (4, 5)
131    } else if s.starts_with("branch-") {
132        (5, 7)
133    } else if s.starts_with("node-") {
134        (6, 5)
135    } else {
136        return None;
137    };
138
139    let suffix = &s[prefix_len..];
140    let (op_id, result_id) = match suffix {
141        "loads" => (0, 0),
142        "load-misses" => (0, 1),
143        "stores" => (1, 0),
144        "store-misses" => (1, 1),
145        "prefetches" => (2, 0),
146        "prefetch-misses" => (2, 1),
147        _ => return None,
148    };
149
150    Some((result_id << 16) | (op_id << 8) | cache_id)
151}
152
153/// Parse a perf event value: hex (0xN) or a symbolic name (e.g. cache-misses,
154/// LLC-load-misses, page-faults). Intended for use as a clap `value_parser`.
155pub fn parse_perf_event(s: &str) -> Result<PerfEventSpec, String> {
156    let s = s.trim();
157    if s.is_empty() || s == "0" || s.eq_ignore_ascii_case("0x0") {
158        return Ok(PerfEventSpec {
159            event_id: 0,
160            type_: bindings::PERF_TYPE_RAW,
161            config: 0,
162            display_name: s.to_string(),
163        });
164    }
165
166    if let Some(hex_str) = s.strip_prefix("0x").or_else(|| s.strip_prefix("0X")) {
167        if let Ok(config) = u64::from_str_radix(hex_str, 16) {
168            return Ok(PerfEventSpec {
169                event_id: config,
170                type_: bindings::PERF_TYPE_RAW,
171                config,
172                display_name: s.to_string(),
173            });
174        }
175    }
176
177    if let Some(config) = parse_hardware_event(s) {
178        let event_id = (bindings::PERF_TYPE_HARDWARE as u64) << 32 | config;
179        return Ok(PerfEventSpec {
180            event_id,
181            type_: bindings::PERF_TYPE_HARDWARE,
182            config,
183            display_name: s.to_string(),
184        });
185    }
186
187    if let Some(config) = parse_software_event(s) {
188        let event_id = (bindings::PERF_TYPE_SOFTWARE as u64) << 32 | config;
189        return Ok(PerfEventSpec {
190            event_id,
191            type_: bindings::PERF_TYPE_SOFTWARE,
192            config,
193            display_name: s.to_string(),
194        });
195    }
196
197    if let Some(config) = parse_hw_cache_event(s) {
198        let event_id = (bindings::PERF_TYPE_HW_CACHE as u64) << 32 | config;
199        return Ok(PerfEventSpec {
200            event_id,
201            type_: bindings::PERF_TYPE_HW_CACHE,
202            config,
203            display_name: s.to_string(),
204        });
205    }
206
207    Err(format!(
208        "Invalid perf event '{}': use hex (0xN) or a symbolic name (e.g. cache-misses, LLC-load-misses, page-faults)",
209        s
210    ))
211}
212
213/// Open a perf event on @cpu and register its fd in the given BPF PMU map (the
214/// scheduler's `scx_pmu_map`) at the slot for (@cpu, @counter_idx). counter_idx
215/// 0 is the migration event, 1 the sticky event, matching the PMU library
216/// install order.
217pub fn setup_perf_events(
218    map: &impl MapCore,
219    cpu: i32,
220    spec: &PerfEventSpec,
221    counter_idx: u32,
222) -> Result<()> {
223    if spec.event_id == 0 {
224        return Ok(());
225    }
226
227    // `disabled` and `inherit` default to 0 via Default::default().
228    let mut attrs = bindings::perf_event_attr {
229        type_: spec.type_,
230        config: spec.config,
231        size: std::mem::size_of::<bindings::perf_event_attr>() as u32,
232        ..Default::default()
233    };
234
235    let fd = unsafe { perf_event_open(&mut attrs, -1, cpu, -1, 0) };
236
237    if fd < 0 {
238        let err = std::io::Error::last_os_error();
239        return Err(anyhow::anyhow!(
240            "Failed to open perf event '{}' on CPU {}: {}",
241            spec.display_name,
242            cpu,
243            err
244        ));
245    }
246
247    let key = cpu as u32 + counter_idx * PERF_MAP_STRIDE;
248
249    map.update(&key.to_ne_bytes(), &fd.to_ne_bytes(), MapFlags::ANY)
250        .with_context(|| "Failed to update perf_events map")?;
251
252    Ok(())
253}
scx_utils/perf.rs

scx_utils/
perf.rs
