1use std::{env, ffi::CString, fs, path::PathBuf, process::Command};
10
11use libbpf_rs::libbpf_sys;
12
13pub struct Kfunc {
17 pub name: &'static str,
18 pub params: &'static [(&'static str, Ty)],
19 pub ret: Ty,
20 pub weak: bool,
21 pub sized: bool,
22}
23
24#[derive(Clone, Copy)]
27#[allow(dead_code)]
28pub enum Ty {
29 Void,
30 I32,
31 U32,
32 U64,
33 Bool,
34 TaskPtr,
35 VoidPtr,
36 CpumaskPtr,
37 TopoPtr,
38 CgroupPtr,
39 BytePtr,
40 U64Ptr,
41}
42
43#[doc(hidden)]
44#[macro_export]
45macro_rules! kfunc_ret {
46 () => {
47 $crate::Ty::Void
48 };
49 ($ret:ident) => {
50 $crate::Ty::$ret
51 };
52}
53
54#[doc(hidden)]
55#[macro_export]
56macro_rules! kfunc_attr {
57 ($want:ident) => {
58 false
59 };
60 ($want:ident $attr:ident $($rest:ident)*) => {
61 stringify!($want) == stringify!($attr) || $crate::kfunc_attr!($want $($rest)*)
62 };
63}
64
65#[macro_export]
68macro_rules! kfuncs {
69 ($( $(#[$attr:ident])* fn $name:ident ( $($param:ident : $type:ident),* ) $(-> $ret:ident)? ; )*) => {
70 fn kfuncs() -> Vec<$crate::Kfunc> {
71 vec![$($crate::Kfunc {
72 name: stringify!($name),
73 params: &[$((stringify!($param), $crate::Ty::$type)),*],
74 ret: $crate::kfunc_ret!($($ret)?),
75 weak: $crate::kfunc_attr!(weak $($attr)*),
76 sized: $crate::kfunc_attr!(sized $($attr)*),
77 },)*]
78 }
79 };
80}
81
82pub struct Spec<'a> {
85 pub package: &'a str,
87 pub manifest: &'a str,
89 pub catalog: &'a str,
90 pub crate_name: &'a str,
91 pub kfuncs: Vec<Kfunc>,
92 pub ksyms: &'a [(&'a std::ffi::CStr, u8, u32)],
95 pub variants: &'a [(&'a str, bool)],
98}
99
100pub const KSYM_INT: u8 = 0;
103pub const KSYM_DOMAIN_POINTER: u8 = 1;
104pub const KSYM_RQ: u8 = 2;
105pub const RUNTIME_KSYMS: [(&std::ffi::CStr, u8, u32); 6] = [
107 (c"sched_core_priority", KSYM_INT, 4),
108 (c"sd_asym_packing", KSYM_DOMAIN_POINTER, 8),
109 (c"runqueues", KSYM_RQ, 0),
110 (c"sd_llc", KSYM_DOMAIN_POINTER, 8),
111 (c"sd_llc_id", KSYM_INT, 4),
112 (c"sd_share_id", KSYM_INT, 4),
113];
114
115mod catalog {
116 include!("../../scx_rustsched/kfuncs.rs");
117
118 pub fn runtime() -> Vec<crate::Kfunc> {
119 kfuncs()
120 }
121}
122
123pub fn runtime_kfuncs() -> Vec<Kfunc> {
125 catalog::runtime()
126}
127
128pub fn build_default() {
134 let catalog =
135 std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../scx_rustsched/kfuncs.rs");
136 let package = env::var("CARGO_PKG_NAME").expect("run from a build script");
137 let manifest = fs::read_to_string("bpf/Cargo.toml").expect("read bpf/Cargo.toml");
140 let crate_name = manifest
141 .lines()
142 .find_map(|line| {
143 let value = line.trim().strip_prefix("name")?.trim_start();
144 Some(value.strip_prefix('=')?.trim().trim_matches('"').to_owned())
145 })
146 .expect("find the BPF crate's name in bpf/Cargo.toml");
147 build(&Spec {
148 package: &package,
149 manifest: "bpf/Cargo.toml",
150 catalog: catalog.to_str().unwrap(),
151 crate_name: &crate_name,
152 kfuncs: runtime_kfuncs(),
153 ksyms: &RUNTIME_KSYMS,
154 variants: &[("legacy", false), ("sized", true)],
155 });
156}
157
158fn undefined_symbols(object: &std::path::Path) -> std::collections::HashSet<String> {
159 let output = Command::new("llvm-nm")
160 .args(["--undefined-only", "--format=just-symbols"])
161 .arg(object)
162 .output()
163 .expect("run llvm-nm on the Rust BPF object");
164 assert!(output.status.success(), "list Rust BPF object symbols");
165 String::from_utf8_lossy(&output.stdout)
166 .lines()
167 .map(|line| line.trim().to_owned())
168 .collect()
169}
170
171unsafe fn resolved(btf: *const libbpf_sys::btf, mut id: u32) -> (u32, *const libbpf_sys::btf_type) {
173 loop {
174 let ty = unsafe { libbpf_sys::btf__type_by_id(btf, id) };
175 assert!(!ty.is_null(), "resolve BTF type");
176 let kind = unsafe { (*ty).info >> 24 & 0x1f };
177 if !matches!(
178 kind,
179 libbpf_sys::BTF_KIND_TYPEDEF
180 | libbpf_sys::BTF_KIND_CONST
181 | libbpf_sys::BTF_KIND_VOLATILE
182 | libbpf_sys::BTF_KIND_RESTRICT
183 ) {
184 return (kind, ty);
185 }
186 id = unsafe { (*ty).__bindgen_anon_1.type_ };
187 }
188}
189
190unsafe fn trusted_arguments(btf: *const libbpf_sys::btf) -> Vec<(i32, i32)> {
199 let mut arguments = Vec::new();
200 unsafe {
201 for id in 1..libbpf_sys::btf__type_cnt(btf) {
202 let function = libbpf_sys::btf__type_by_id(btf, id);
203 if (*function).info >> 24 & 0x1f != libbpf_sys::BTF_KIND_FUNC
204 || (*function).info & 0xffff != libbpf_sys::BTF_FUNC_GLOBAL
205 {
206 continue;
207 }
208 let (kind, prototype) = resolved(btf, (*function).__bindgen_anon_1.type_);
209 if kind != libbpf_sys::BTF_KIND_FUNC_PROTO {
210 continue;
211 }
212 let parameters = std::slice::from_raw_parts(
213 prototype.add(1).cast::<libbpf_sys::btf_param>(),
214 ((*prototype).info & 0xffff) as usize,
215 );
216 for (index, parameter) in parameters.iter().enumerate() {
217 let (kind, pointer) = resolved(btf, parameter.type_);
218 if kind != libbpf_sys::BTF_KIND_PTR {
219 continue;
220 }
221 let (kind, target) = resolved(btf, (*pointer).__bindgen_anon_1.type_);
222 if kind == libbpf_sys::BTF_KIND_STRUCT
223 && std::ffi::CStr::from_ptr(libbpf_sys::btf__name_by_offset(
224 btf,
225 (*target).name_off,
226 )) == c"task_struct"
227 {
228 arguments.push((id as i32, index as i32));
229 }
230 }
231 }
232 }
233 arguments
234}
235
236fn checked(result: i32) -> i32 {
237 assert!(result >= 0, "BTF operation failed: {result}");
238 result
239}
240
241fn add_btf(spec: &Spec, object: &std::path::Path, output: &std::path::Path, sized_topology: bool) {
245 let path = CString::new(object.as_os_str().as_encoded_bytes()).unwrap();
246 let referenced = undefined_symbols(object);
250 unsafe {
251 let btf = libbpf_sys::btf__parse_elf(path.as_ptr(), std::ptr::null_mut());
252 assert!(!btf.is_null(), "parse Rust BPF BTF");
253 let task = checked(libbpf_sys::btf__find_by_name_kind(
254 btf,
255 c"task_struct".as_ptr(),
256 libbpf_sys::BTF_KIND_STRUCT,
257 ));
258 let task_pointer = checked(libbpf_sys::btf__add_ptr(btf, task));
259 let cpumask = checked(libbpf_sys::btf__add_struct(btf, c"cpumask".as_ptr(), 1024));
260 let cpumask_pointer = checked(libbpf_sys::btf__add_ptr(btf, cpumask));
261 let integer = checked(libbpf_sys::btf__find_by_name_kind(
262 btf,
263 c"u64".as_ptr(),
264 libbpf_sys::BTF_KIND_INT,
265 ));
266 let signed = checked(libbpf_sys::btf__find_by_name_kind(
267 btf,
268 c"i32".as_ptr(),
269 libbpf_sys::BTF_KIND_INT,
270 ));
271 let topology = checked(libbpf_sys::btf__add_struct(
272 btf,
273 c"scx_cid_topo".as_ptr(),
274 if sized_topology { 40 } else { 32 },
275 ));
276 for (index, name) in [
277 c"core_cid",
278 c"core_idx",
279 c"llc_cid",
280 c"llc_idx",
281 c"node_cid",
282 c"node_idx",
283 c"shard_cid",
284 c"shard_idx",
285 c"cluster_cid",
286 c"cluster_idx",
287 ]
288 .into_iter()
289 .take(if sized_topology { 10 } else { 8 })
290 .enumerate()
291 {
292 checked(libbpf_sys::btf__add_field(
293 btf,
294 name.as_ptr(),
295 signed,
296 (index * 32) as u32,
297 0,
298 ));
299 }
300 let topology_pointer = checked(libbpf_sys::btf__add_ptr(btf, topology));
301 let boolean = checked(libbpf_sys::btf__find_by_name_kind(
302 btf,
303 c"bool".as_ptr(),
304 libbpf_sys::BTF_KIND_INT,
305 ));
306 let void_pointer = checked(libbpf_sys::btf__add_ptr(btf, 0));
307 let unsigned = checked(libbpf_sys::btf__find_by_name_kind(
308 btf,
309 c"u32".as_ptr(),
310 libbpf_sys::BTF_KIND_INT,
311 ));
312 let cgroup = match libbpf_sys::btf__find_by_name_kind(
315 btf,
316 c"cgroup".as_ptr(),
317 libbpf_sys::BTF_KIND_STRUCT,
318 ) {
319 id if id > 0 => id,
320 _ => checked(libbpf_sys::btf__add_struct(btf, c"cgroup".as_ptr(), 0)),
321 };
322 let cgroup_pointer = checked(libbpf_sys::btf__add_ptr(btf, cgroup));
323 let byte =
325 match libbpf_sys::btf__find_by_name_kind(btf, c"u8".as_ptr(), libbpf_sys::BTF_KIND_INT)
326 {
327 id if id > 0 => id,
328 _ => checked(libbpf_sys::btf__add_int(btf, c"char".as_ptr(), 1, 0)),
329 };
330 let byte_pointer = checked(libbpf_sys::btf__add_ptr(btf, byte));
331 let integer_pointer = checked(libbpf_sys::btf__add_ptr(btf, integer));
332 let id = |ty: Ty| match ty {
333 Ty::Void => 0,
334 Ty::I32 => signed,
335 Ty::U32 => unsigned,
336 Ty::U64 => integer,
337 Ty::Bool => boolean,
338 Ty::TaskPtr => task_pointer,
339 Ty::VoidPtr => void_pointer,
340 Ty::CpumaskPtr => cpumask_pointer,
341 Ty::TopoPtr => topology_pointer,
342 Ty::CgroupPtr => cgroup_pointer,
343 Ty::BytePtr => byte_pointer,
344 Ty::U64Ptr => integer_pointer,
345 };
346 let mut functions = Vec::new();
347 for kfunc in spec
348 .kfuncs
349 .iter()
350 .filter(|kfunc| referenced.contains(kfunc.name))
351 {
352 let prototype = checked(libbpf_sys::btf__add_func_proto(btf, id(kfunc.ret)));
353 for (parameter, ty) in kfunc.params {
354 let parameter = CString::new(*parameter).unwrap();
355 checked(libbpf_sys::btf__add_func_param(
356 btf,
357 parameter.as_ptr(),
358 id(*ty),
359 ));
360 }
361 if kfunc.sized && sized_topology {
362 checked(libbpf_sys::btf__add_func_param(
363 btf,
364 c"size".as_ptr(),
365 integer,
366 ));
367 }
368 let name = CString::new(kfunc.name).unwrap();
369 functions.push(checked(libbpf_sys::btf__add_func(
370 btf,
371 name.as_ptr(),
372 libbpf_sys::BTF_FUNC_EXTERN,
373 prototype,
374 )));
375 }
376 let domain = checked(libbpf_sys::btf__add_struct(
380 btf,
381 c"sched_domain".as_ptr(),
382 0,
383 ));
384 let domain_pointer = checked(libbpf_sys::btf__add_ptr(btf, domain));
385 let runqueue = checked(libbpf_sys::btf__add_struct(btf, c"rq".as_ptr(), 0));
386 let mut variables = Vec::new();
387 for (name, kind, size) in spec
388 .ksyms
389 .iter()
390 .copied()
391 .filter(|(name, _, _)| referenced.contains(name.to_str().unwrap()))
392 {
393 let kind = match kind {
394 KSYM_DOMAIN_POINTER => domain_pointer,
395 KSYM_RQ => runqueue,
396 _ => signed,
397 };
398 variables.push((
399 checked(libbpf_sys::btf__add_var(
400 btf,
401 name.as_ptr(),
402 libbpf_sys::BTF_VAR_GLOBAL_EXTERN as i32,
403 kind,
404 )),
405 size,
406 ));
407 }
408 checked(libbpf_sys::btf__add_datasec(btf, c".ksyms".as_ptr(), 0));
409 for function in functions {
410 checked(libbpf_sys::btf__add_datasec_var_info(btf, function, 0, 0));
411 }
412 for (variable, size) in variables {
413 checked(libbpf_sys::btf__add_datasec_var_info(
414 btf, variable, 0, size,
415 ));
416 }
417 for (function, argument) in trusted_arguments(btf) {
418 checked(libbpf_sys::btf__add_decl_tag(
419 btf,
420 c"arg:trusted".as_ptr(),
421 function,
422 argument,
423 ));
424 }
425 let mut size = 0;
426 let data = libbpf_sys::btf__raw_data(btf, &mut size);
427 assert!(!data.is_null(), "serialize BTF");
428 fs::write(
429 output,
430 std::slice::from_raw_parts(data.cast::<u8>(), size as usize),
431 )
432 .expect("write kfunc BTF");
433 libbpf_sys::btf__free(btf);
434 }
435 let status = Command::new("llvm-objcopy")
436 .args(
437 spec.ksyms
438 .iter()
439 .map(|(name, _, _)| name.to_str().unwrap())
440 .chain(
441 spec.kfuncs
442 .iter()
443 .filter(|kfunc| kfunc.weak)
444 .map(|kfunc| kfunc.name),
445 )
446 .map(|name| format!("--weaken-symbol={name}")),
447 )
448 .arg("--update-section")
449 .arg(format!(".BTF={}", output.display()))
450 .arg(object)
451 .status()
452 .expect("run llvm-objcopy to update BTF");
453 assert!(status.success(), "update Rust BPF BTF");
454}
455
456fn runs(program: &str, args: &[&str]) -> bool {
458 Command::new(program)
459 .args(args)
460 .output()
461 .is_ok_and(|output| output.status.success())
462}
463
464fn missing_tools(toolchain: &Option<String>) -> Vec<String> {
467 let mut missing = Vec::new();
468 let sysroot = match toolchain {
469 Some(toolchain) => Command::new("rustup")
470 .args(["run", toolchain, "rustc", "--print", "sysroot"])
471 .output(),
472 None => Command::new("rustc").args(["--print", "sysroot"]).output(),
473 };
474 match sysroot {
475 Ok(output) if output.status.success() => {
476 let sysroot = PathBuf::from(String::from_utf8_lossy(&output.stdout).trim());
477 if !sysroot.join("lib/rustlib/src/rust/library/core").is_dir() {
478 missing.push("rust-src".into());
479 }
480 }
481 _ => missing.push(format!(
482 "the {} toolchain",
483 toolchain.as_deref().unwrap_or("Rust")
484 )),
485 }
486 for tool in ["bpf-linker", "llvm-objcopy", "llvm-nm"] {
487 if !runs(tool, &["--version"]) {
488 missing.push(tool.into());
489 }
490 }
491 missing
492}
493
494pub fn build(spec: &Spec) {
498 let bpf = std::path::Path::new(spec.manifest).parent().unwrap();
499 println!("cargo:rerun-if-changed={}", spec.manifest);
500 println!(
501 "cargo:rerun-if-changed={}",
502 bpf.join("Cargo.lock").display()
503 );
504 println!("cargo:rerun-if-changed={}", bpf.join("src").display());
505 if let Some(scheduler) = bpf.parent() {
507 println!("cargo:rerun-if-changed={}", scheduler.join("src").display());
508 }
509 println!("cargo:rerun-if-changed={}", spec.catalog);
510 for runtime in ["scx_rustsched", "scx_rustsched_abi", "scx_rustsched_macros"] {
514 let runtime = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
515 .join("..")
516 .join(runtime);
517 for path in ["Cargo.toml", "src"] {
518 println!("cargo:rerun-if-changed={}", runtime.join(path).display());
519 }
520 }
521 println!("cargo:rerun-if-env-changed=BPF_RUST_TOOLCHAIN");
522 println!("cargo:rerun-if-env-changed=PATH");
523 let toolchain = env::var("BPF_RUST_TOOLCHAIN").ok();
529 let output = PathBuf::from(env::var_os("OUT_DIR").unwrap());
530 let missing = missing_tools(&toolchain);
534 if !missing.is_empty() {
535 println!(
536 "cargo:warning={}: no Rust BPF toolchain ({} missing), \
537 building a binary that cannot load the scheduler",
538 spec.package,
539 missing.join(", ")
540 );
541 for (variant, _) in spec.variants {
542 fs::write(output.join(format!("scheduler-{variant}.bpf.o")), [])
543 .expect("write empty BPF object");
544 }
545 return;
546 }
547 let target = match env::var("CARGO_CFG_TARGET_ENDIAN").unwrap().as_str() {
548 "little" => "bpfel-unknown-none",
549 "big" => "bpfeb-unknown-none",
550 other => panic!("unsupported endianness: {other}"),
551 };
552 for (variant, sized_topology) in spec.variants.iter().copied() {
553 let mut command = match &toolchain {
554 Some(toolchain) => {
555 let mut command = Command::new("rustup");
556 command.args(["run", toolchain, "cargo"]);
557 command
558 }
559 None => Command::new(env::var_os("CARGO").unwrap_or("cargo".into())),
560 };
561 command
564 .env("RUSTC_BOOTSTRAP", "1")
565 .args(["build", "--release", "--locked"])
566 .args(["--manifest-path", spec.manifest, "--target", target])
567 .args(["-Z", "build-std=core"])
568 .arg("--target-dir")
569 .arg(output.join("bpf-target"))
570 .env_remove("CARGO_ENCODED_RUSTFLAGS")
571 .env_remove("RUSTC")
572 .env_remove("RUSTC_WORKSPACE_WRAPPER")
573 .env(
574 "RUSTFLAGS",
575 format!(
576 "-C target-cpu=v3 -C debuginfo=2 -C link-arg=--btf -C link-arg=--export={}",
577 spec.kfuncs
578 .iter()
579 .map(|kfunc| kfunc.name)
580 .collect::<Vec<_>>()
581 .join(",")
582 ),
583 );
584 if sized_topology {
585 command.args(["--features", "topo_size"]);
586 }
587 let status = command
588 .status()
589 .expect("run the BPF cargo build; install rust-src and bpf-linker");
590 assert!(status.success(), "Rust BPF {variant} build failed");
591 let object = output.join(format!("scheduler-{variant}.bpf.o"));
592 fs::copy(
593 output
594 .join("bpf-target")
595 .join(target)
596 .join("release")
597 .join(spec.crate_name),
598 &object,
599 )
600 .expect("copy Rust BPF object");
601 add_btf(
602 spec,
603 &object,
604 &output.join(format!("scheduler-{variant}.btf")),
605 sized_topology,
606 );
607 }
608}
609
610#[cfg(test)]
611mod tests {
612 use super::*;
613
614 unsafe fn function(
616 btf: *mut libbpf_sys::btf,
617 name: &std::ffi::CStr,
618 linkage: u32,
619 parameters: &[i32],
620 ) -> i32 {
621 unsafe {
622 let prototype = checked(libbpf_sys::btf__add_func_proto(btf, 0));
623 for parameter in parameters {
624 checked(libbpf_sys::btf__add_func_param(
625 btf,
626 c"p".as_ptr(),
627 *parameter,
628 ));
629 }
630 checked(libbpf_sys::btf__add_func(
631 btf,
632 name.as_ptr(),
633 linkage,
634 prototype,
635 ))
636 }
637 }
638
639 #[test]
640 fn task_parameters_of_global_functions_are_trusted() {
641 unsafe {
642 let btf = libbpf_sys::btf__new_empty();
643 assert!(!btf.is_null());
644 let int = checked(libbpf_sys::btf__add_int(
645 btf,
646 c"int".as_ptr(),
647 4,
648 libbpf_sys::BTF_INT_SIGNED as i32,
649 ));
650 let task = checked(libbpf_sys::btf__add_struct(btf, c"task_struct".as_ptr(), 0));
651 let task_pointer = checked(libbpf_sys::btf__add_ptr(btf, task));
652 let constant = checked(libbpf_sys::btf__add_const(btf, task));
654 let constant_pointer = checked(libbpf_sys::btf__add_ptr(btf, constant));
655 let alias = checked(libbpf_sys::btf__add_typedef(
656 btf,
657 c"task_ref".as_ptr(),
658 constant_pointer,
659 ));
660 let other = checked(libbpf_sys::btf__add_struct(btf, c"cpumask".as_ptr(), 0));
661 let other_pointer = checked(libbpf_sys::btf__add_ptr(btf, other));
662
663 let scan = function(
664 btf,
665 c"scan",
666 libbpf_sys::BTF_FUNC_GLOBAL,
667 &[task_pointer, int, alias],
668 );
669 function(btf, c"local", libbpf_sys::BTF_FUNC_STATIC, &[task_pointer]);
670 function(
671 btf,
672 c"mask",
673 libbpf_sys::BTF_FUNC_GLOBAL,
674 &[other_pointer, int],
675 );
676
677 assert_eq!(trusted_arguments(btf), vec![(scan, 0), (scan, 2)]);
678 libbpf_sys::btf__free(btf);
679 }
680 }
681}