1mod bpf_skel;
9pub use bpf_skel::*;
10pub mod bpf_intf;
11pub use bpf_intf::*;
12
13mod stats;
14use std::mem::MaybeUninit;
15use std::sync::atomic::AtomicBool;
16use std::sync::atomic::Ordering;
17use std::sync::Arc;
18use std::time::Duration;
19use std::time::Instant;
20
21use anyhow::Result;
22use clap::CommandFactory;
23use clap::Parser;
24use clap_complete::generate;
25use clap_complete::Shell;
26use crossbeam::channel::RecvTimeoutError;
27use libbpf_rs::MapCore;
28use log::info;
29use scx_stats::prelude::*;
30use scx_utils::build_id;
31use scx_utils::compat;
32use scx_utils::libbpf_clap_opts::LibbpfOpts;
33use scx_utils::scx_ops_attach;
34use scx_utils::scx_ops_load;
35use scx_utils::scx_ops_open;
36use scx_utils::try_set_rlimit_infinity;
37use scx_utils::uei_exited;
38use scx_utils::uei_report;
39use scx_utils::UserExitInfo;
40
41use stats::Metrics;
42
43const SCHEDULER_NAME: &str = "scx_flow";
44
45fn full_version() -> String {
46 build_id::full_version(env!("CARGO_PKG_VERSION"))
47}
48
49#[derive(Debug, Parser)]
50#[command(name = SCHEDULER_NAME, version, disable_version_flag = true)]
51struct Opts {
52 #[clap(long)]
54 stats: Option<f64>,
55
56 #[clap(long)]
58 monitor: Option<f64>,
59
60 #[clap(short, long, action = clap::ArgAction::SetTrue)]
62 debug: bool,
63
64 #[clap(short = 'V', long, action = clap::ArgAction::SetTrue)]
66 version: bool,
67
68 #[clap(long, action = clap::ArgAction::SetTrue)]
70 no_autotune: bool,
71
72 #[clap(long, value_name = "SHELL", hide = true)]
74 completions: Option<Shell>,
75
76 #[clap(flatten, next_help_heading = "Libbpf Options")]
77 libbpf: LibbpfOpts,
78}
79
80struct Scheduler<'a> {
81 skel: BpfSkel<'a>,
82 struct_ops: Option<libbpf_rs::Link>,
83 stats_server: StatsServer<(), Metrics>,
84}
85
86#[derive(Clone, Copy, Debug, Eq, PartialEq)]
87enum AutoTuneMode {
88 Balanced,
89 Latency,
90 Throughput,
91}
92
93impl AutoTuneMode {
94 fn as_u64(self) -> u64 {
95 match self {
96 Self::Balanced => 0,
97 Self::Latency => 1,
98 Self::Throughput => 2,
99 }
100 }
101
102 fn as_str(self) -> &'static str {
103 match self {
104 Self::Balanced => "balanced",
105 Self::Latency => "latency",
106 Self::Throughput => "throughput",
107 }
108 }
109}
110
111#[derive(Clone, Copy, Debug, Eq, PartialEq)]
112struct RuntimeTunables {
113 reserved_max_ns: u64,
114 shared_slice_ns: u64,
115 interactive_floor_ns: u64,
116 preempt_budget_min_ns: u64,
117 preempt_refill_min_ns: u64,
118 latency_credit_grant: u64,
119 latency_credit_decay: u64,
120 latency_debt_urgent_min: u64,
121 urgent_latency_burst_max: u64,
122 reserved_quota_burst_max: u64,
123 reserved_lane_burst_max: u64,
124 contained_starvation_max: u64,
125 shared_starvation_max: u64,
126 local_fast_nr_running_max: u64,
127 local_reserved_burst_max: u64,
128}
129
130impl Default for RuntimeTunables {
131 fn default() -> Self {
132 Self {
133 reserved_max_ns: u64::from(consts_FLOW_SLICE_RESERVED_MAX_NS),
134 shared_slice_ns: u64::from(consts_FLOW_SLICE_SHARED_NS),
135 interactive_floor_ns: u64::from(consts_FLOW_INTERACTIVE_FLOOR_NS),
136 preempt_budget_min_ns: u64::from(consts_FLOW_PREEMPT_BUDGET_MIN_NS),
137 preempt_refill_min_ns: u64::from(consts_FLOW_PREEMPT_REFILL_MIN_NS),
138 latency_credit_grant: u64::from(consts_FLOW_LATENCY_CREDIT_GRANT),
139 latency_credit_decay: u64::from(consts_FLOW_LATENCY_CREDIT_DECAY),
140 latency_debt_urgent_min: u64::from(consts_FLOW_LATENCY_DEBT_URGENT_MIN),
141 urgent_latency_burst_max: u64::from(consts_FLOW_URGENT_LATENCY_BURST_MAX),
142 reserved_quota_burst_max: u64::from(consts_FLOW_RESERVED_QUOTA_BURST_MAX),
143 reserved_lane_burst_max: u64::from(consts_FLOW_RESERVED_LANE_BURST_MAX),
144 contained_starvation_max: u64::from(consts_FLOW_CONTAINED_STARVATION_MAX),
145 shared_starvation_max: u64::from(consts_FLOW_SHARED_STARVATION_MAX),
146 local_fast_nr_running_max: u64::from(consts_FLOW_LOCAL_FAST_NR_RUNNING_MAX),
147 local_reserved_burst_max: u64::from(consts_FLOW_LOCAL_RESERVED_BURST_MAX),
148 }
149 }
150}
151
152impl RuntimeTunables {
153 fn clamp(self) -> Self {
154 Self {
155 reserved_max_ns: self.reserved_max_ns.clamp(
156 u64::from(consts_FLOW_SLICE_MIN_NS),
157 u64::from(consts_FLOW_SLICE_RESERVED_TUNE_MAX_NS),
158 ),
159 shared_slice_ns: self.shared_slice_ns.clamp(
160 u64::from(consts_FLOW_SLICE_SHARED_MIN_NS),
161 u64::from(consts_FLOW_SLICE_SHARED_MAX_NS),
162 ),
163 interactive_floor_ns: self.interactive_floor_ns.clamp(
164 u64::from(consts_FLOW_INTERACTIVE_FLOOR_MIN_NS),
165 u64::from(consts_FLOW_INTERACTIVE_FLOOR_MAX_NS),
166 ),
167 preempt_budget_min_ns: self.preempt_budget_min_ns.clamp(
168 u64::from(consts_FLOW_PREEMPT_BUDGET_MIN_NS),
169 u64::from(consts_FLOW_PREEMPT_BUDGET_MAX_NS),
170 ),
171 preempt_refill_min_ns: self.preempt_refill_min_ns.clamp(
172 u64::from(consts_FLOW_PREEMPT_REFILL_MIN_NS),
173 u64::from(consts_FLOW_PREEMPT_REFILL_MAX_NS),
174 ),
175 latency_credit_grant: self.latency_credit_grant.clamp(
176 u64::from(consts_FLOW_LATENCY_CREDIT_GRANT_MIN),
177 u64::from(consts_FLOW_LATENCY_CREDIT_GRANT_MAX),
178 ),
179 latency_credit_decay: self.latency_credit_decay.clamp(
180 u64::from(consts_FLOW_LATENCY_CREDIT_DECAY_MIN),
181 u64::from(consts_FLOW_LATENCY_CREDIT_DECAY_MAX),
182 ),
183 latency_debt_urgent_min: self.latency_debt_urgent_min.clamp(
184 u64::from(consts_FLOW_LATENCY_DEBT_URGENT_MIN_MIN),
185 u64::from(consts_FLOW_LATENCY_DEBT_URGENT_MIN_MAX),
186 ),
187 urgent_latency_burst_max: self.urgent_latency_burst_max.clamp(
188 u64::from(consts_FLOW_URGENT_LATENCY_BURST_MIN),
189 u64::from(consts_FLOW_URGENT_LATENCY_BURST_MAX_TUNE),
190 ),
191 reserved_quota_burst_max: self.reserved_quota_burst_max.clamp(
192 u64::from(consts_FLOW_RESERVED_QUOTA_BURST_MIN),
193 u64::from(consts_FLOW_RESERVED_QUOTA_BURST_MAX_TUNE),
194 ),
195 reserved_lane_burst_max: self.reserved_lane_burst_max.clamp(
196 u64::from(consts_FLOW_RESERVED_LANE_BURST_MIN),
197 u64::from(consts_FLOW_RESERVED_LANE_BURST_MAX_TUNE),
198 ),
199 contained_starvation_max: self.contained_starvation_max.clamp(
200 u64::from(consts_FLOW_CONTAINED_STARVATION_MIN),
201 u64::from(consts_FLOW_CONTAINED_STARVATION_MAX_TUNE),
202 ),
203 shared_starvation_max: self.shared_starvation_max.clamp(
204 u64::from(consts_FLOW_SHARED_STARVATION_MIN),
205 u64::from(consts_FLOW_SHARED_STARVATION_MAX_TUNE),
206 ),
207 local_fast_nr_running_max: self.local_fast_nr_running_max.clamp(
208 u64::from(consts_FLOW_LOCAL_FAST_NR_RUNNING_MIN),
209 u64::from(consts_FLOW_LOCAL_FAST_NR_RUNNING_MAX_TUNE),
210 ),
211 local_reserved_burst_max: self.local_reserved_burst_max.clamp(
212 u64::from(consts_FLOW_LOCAL_RESERVED_BURST_MIN),
213 u64::from(consts_FLOW_LOCAL_RESERVED_BURST_MAX_TUNE),
214 ),
215 }
216 }
217
218 fn target_for(mode: AutoTuneMode) -> Self {
219 match mode {
220 AutoTuneMode::Balanced => Self::default(),
221 AutoTuneMode::Latency => Self {
222 reserved_max_ns: 300 * 1000,
223 shared_slice_ns: 900 * 1000,
224 interactive_floor_ns: 140 * 1000,
225 preempt_budget_min_ns: 225 * 1000,
226 preempt_refill_min_ns: 250 * 1000,
227 latency_credit_grant: u64::from(consts_FLOW_LATENCY_CREDIT_GRANT),
228 latency_credit_decay: u64::from(consts_FLOW_LATENCY_CREDIT_DECAY),
229 latency_debt_urgent_min: u64::from(consts_FLOW_LATENCY_DEBT_URGENT_MIN),
230 urgent_latency_burst_max: 3,
231 reserved_quota_burst_max: u64::from(consts_FLOW_RESERVED_QUOTA_BURST_MAX),
232 reserved_lane_burst_max: 4,
233 contained_starvation_max: u64::from(consts_FLOW_CONTAINED_STARVATION_MAX),
234 shared_starvation_max: 10,
235 local_fast_nr_running_max: u64::from(consts_FLOW_LOCAL_FAST_NR_RUNNING_MAX),
236 local_reserved_burst_max: 3,
237 }
238 .clamp(),
239 AutoTuneMode::Throughput => Self {
240 reserved_max_ns: 200 * 1000,
241 shared_slice_ns: 1200 * 1000,
242 interactive_floor_ns: 80 * 1000,
243 preempt_budget_min_ns: 300 * 1000,
244 preempt_refill_min_ns: 325 * 1000,
245 latency_credit_grant: u64::from(consts_FLOW_LATENCY_CREDIT_GRANT),
246 latency_credit_decay: u64::from(consts_FLOW_LATENCY_CREDIT_DECAY),
247 latency_debt_urgent_min: 2,
248 urgent_latency_burst_max: 1,
249 reserved_quota_burst_max: 3,
250 reserved_lane_burst_max: 6,
251 contained_starvation_max: u64::from(consts_FLOW_CONTAINED_STARVATION_MAX),
252 shared_starvation_max: u64::from(consts_FLOW_SHARED_STARVATION_MAX),
253 local_fast_nr_running_max: u64::from(consts_FLOW_LOCAL_FAST_NR_RUNNING_MAX),
254 local_reserved_burst_max: 3,
255 }
256 .clamp(),
257 }
258 }
259
260 fn step_towards(&mut self, target: Self) -> bool {
261 let mut changed = false;
262
263 changed |= step_u64(&mut self.reserved_max_ns, target.reserved_max_ns, 25 * 1000);
264 changed |= step_u64(
265 &mut self.shared_slice_ns,
266 target.shared_slice_ns,
267 100 * 1000,
268 );
269 changed |= step_u64(
270 &mut self.interactive_floor_ns,
271 target.interactive_floor_ns,
272 20 * 1000,
273 );
274 changed |= step_u64(
275 &mut self.preempt_budget_min_ns,
276 target.preempt_budget_min_ns,
277 25 * 1000,
278 );
279 changed |= step_u64(
280 &mut self.preempt_refill_min_ns,
281 target.preempt_refill_min_ns,
282 25 * 1000,
283 );
284 changed |= step_u64(
285 &mut self.latency_credit_grant,
286 target.latency_credit_grant,
287 1,
288 );
289 changed |= step_u64(
290 &mut self.latency_credit_decay,
291 target.latency_credit_decay,
292 1,
293 );
294 changed |= step_u64(
295 &mut self.latency_debt_urgent_min,
296 target.latency_debt_urgent_min,
297 1,
298 );
299 changed |= step_u64(
300 &mut self.urgent_latency_burst_max,
301 target.urgent_latency_burst_max,
302 1,
303 );
304 changed |= step_u64(
305 &mut self.reserved_quota_burst_max,
306 target.reserved_quota_burst_max,
307 1,
308 );
309 changed |= step_u64(
310 &mut self.reserved_lane_burst_max,
311 target.reserved_lane_burst_max,
312 1,
313 );
314 changed |= step_u64(
315 &mut self.contained_starvation_max,
316 target.contained_starvation_max,
317 1,
318 );
319 changed |= step_u64(
320 &mut self.shared_starvation_max,
321 target.shared_starvation_max,
322 1,
323 );
324 changed |= step_u64(
325 &mut self.local_fast_nr_running_max,
326 target.local_fast_nr_running_max,
327 1,
328 );
329 changed |= step_u64(
330 &mut self.local_reserved_burst_max,
331 target.local_reserved_burst_max,
332 1,
333 );
334
335 *self = self.clamp();
336 changed
337 }
338}
339
340fn step_u64(value: &mut u64, target: u64, step: u64) -> bool {
341 if *value == target {
342 return false;
343 }
344
345 if *value < target {
346 *value = (*value + step).min(target);
347 } else {
348 *value = value.saturating_sub(step).max(target);
349 }
350
351 true
352}
353
354#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
355struct CpuPolicyStateAgg {
356 urgent_latency_burst_rounds: u64,
357 high_priority_burst_rounds: u64,
358 local_reserved_burst_rounds: u64,
359 local_reserved_fast_grants: u64,
360 local_reserved_burst_continuations: u64,
361 reserved_lane_burst_rounds: u64,
362 contained_starvation_rounds: u64,
363 shared_starvation_rounds: u64,
364 budget_refill_events: u64,
365 budget_exhaustions: u64,
366 runnable_wakeups: u64,
367 urgent_latency_dispatches: u64,
368 urgent_latency_burst_grants: u64,
369 urgent_latency_burst_continuations: u64,
370 urgent_latency_enqueues: u64,
371 urgent_latency_misses: u64,
372 latency_dispatches: u64,
373 latency_debt_raises: u64,
374 latency_debt_decays: u64,
375 latency_debt_urgent_enqueues: u64,
376 reserved_dispatches: u64,
377 shared_dispatches: u64,
378 contained_dispatches: u64,
379 contained_rescue_dispatches: u64,
380 shared_rescue_dispatches: u64,
381 local_fast_dispatches: u64,
382 wake_preempt_dispatches: u64,
383 cpu_stability_biases: u64,
384 last_cpu_matches: u64,
385 latency_lane_candidates: u64,
386 latency_lane_enqueues: u64,
387 latency_candidate_local_enqueues: u64,
388 latency_candidate_hog_blocks: u64,
389 positive_budget_wakeups: u64,
390 rt_sensitive_wakeups: u64,
391 reserved_local_enqueues: u64,
392 reserved_global_enqueues: u64,
393 reserved_quota_skips: u64,
394 quota_shared_forces: u64,
395 quota_contained_forces: u64,
396 reserved_lane_grants: u64,
397 reserved_lane_burst_continuations: u64,
398 reserved_lane_skips: u64,
399 reserved_lane_shared_forces: u64,
400 reserved_lane_contained_forces: u64,
401 reserved_lane_shared_misses: u64,
402 reserved_lane_contained_misses: u64,
403 shared_wakeup_enqueues: u64,
404 shared_starved_head_enqueues: u64,
405 local_quota_skips: u64,
406 rt_sensitive_local_enqueues: u64,
407 rt_sensitive_preempts: u64,
408 direct_local_candidates: u64,
409 direct_local_enqueues: u64,
410 direct_local_rejections: u64,
411 direct_local_mismatches: u64,
412 ipc_wake_candidates: u64,
413 ipc_local_enqueues: u64,
414 ipc_score_raises: u64,
415 ipc_boosts: u64,
416 contained_enqueues: u64,
417 contained_starved_head_enqueues: u64,
418 hog_containment_enqueues: u64,
419 hog_recoveries: u64,
420 cpu_migrations: u64,
421}
422
423#[derive(Debug)]
424struct AutoTuner {
425 tunables: RuntimeTunables,
426 mode: AutoTuneMode,
427 pending_mode: AutoTuneMode,
428 pending_steps: u8,
429 latency_cooldown: u8,
430 generation: u64,
431 prev_metrics: Metrics,
432}
433
434impl AutoTuner {
435 fn new(initial_metrics: Metrics) -> Self {
436 Self {
437 tunables: RuntimeTunables::default(),
438 mode: AutoTuneMode::Balanced,
439 pending_mode: AutoTuneMode::Balanced,
440 pending_steps: 0,
441 latency_cooldown: 0,
442 generation: 0,
443 prev_metrics: initial_metrics,
444 }
445 }
446
447 fn evaluate_mode(&self, current: &Metrics, delta: &Metrics) -> AutoTuneMode {
448 let positive = delta.positive_budget_wakeups;
449 let shared_wake = delta.shared_wakeup_enqueues;
450 let reserved_local = delta.reserved_local_enqueues;
451 let reserved_global = delta.reserved_global_enqueues;
452 let reserved_dispatches = delta.reserved_dispatches;
453 let latency_dispatches = delta.latency_dispatches;
454 let contained_enqueues = delta.contained_enqueues;
455 let contained_dispatches = delta.contained_dispatches;
456 let contained_rescues = delta.contained_rescue_dispatches;
457 let shared_rescues = delta.shared_rescue_dispatches;
458 let wake_preempt = delta.wake_preempt_dispatches;
459 let exhaustions = delta.budget_exhaustions;
460 let runnable = delta.runnable_wakeups;
461 let direct_candidates = delta.direct_local_candidates;
462 let direct_rejections = delta.direct_local_rejections;
463 let direct_mismatches = delta.direct_local_mismatches;
464 let cpu_biases = delta.cpu_stability_biases;
465 let reserved_total = reserved_local + reserved_global;
466 let lane_events = positive + shared_wake + contained_enqueues;
467 let urgent_latency_dispatches = delta.urgent_latency_dispatches;
468 let total_latency_dispatches = latency_dispatches + urgent_latency_dispatches;
469 let dispatch_total = total_latency_dispatches
470 + reserved_dispatches
471 + contained_dispatches
472 + delta.shared_dispatches;
473
474 if lane_events < 3 && reserved_total + contained_dispatches < 2 {
475 return self.mode;
476 }
477
478 let shared_ratio = shared_wake as f64 / (positive + shared_wake).max(1) as f64;
479 let global_ratio = reserved_global as f64 / reserved_total.max(1) as f64;
480 let preempt_ratio = wake_preempt as f64 / reserved_local.max(1) as f64;
481 let exhaustion_ratio = exhaustions as f64 / positive.max(1) as f64;
482 let contained_ratio = contained_enqueues as f64 / positive.max(1) as f64;
483 let direct_reject_ratio = direct_rejections as f64 / direct_candidates.max(1) as f64;
484 let direct_mismatch_attempts = cpu_biases.saturating_add(direct_mismatches);
485 let direct_mismatch_ratio =
486 direct_mismatches as f64 / direct_mismatch_attempts.max(1) as f64;
487 let rescue_total = contained_rescues + shared_rescues;
488 let rescue_ratio = rescue_total as f64 / dispatch_total.max(1) as f64;
489 let latency_dispatch_ratio = total_latency_dispatches as f64 / dispatch_total.max(1) as f64;
490 let rescue_pressure = rescue_total >= 8 && rescue_ratio > 0.08;
491 let keep_latency_mode = self.mode == AutoTuneMode::Latency
492 && current.nr_running >= 1
493 && !rescue_pressure
494 && (wake_preempt > 0
495 || latency_dispatch_ratio > 0.45
496 || shared_ratio > 0.45
497 || global_ratio > 0.35
498 || exhaustion_ratio > 0.30
499 || runnable > 64);
500 let keep_throughput_mode = self.mode == AutoTuneMode::Throughput
501 && current.nr_running >= 2
502 && (contained_enqueues > 0 || contained_dispatches > 0)
503 && shared_ratio < 0.45
504 && global_ratio < 0.30;
505 let should_enter_throughput_mode = current.nr_running >= 3
506 && ((shared_ratio < 0.45
507 && global_ratio < 0.30
508 && ((contained_dispatches > 0 && contained_ratio > 0.12)
509 || (direct_candidates > 0
510 && direct_reject_ratio > 0.20
511 && direct_mismatch_ratio > 0.20)))
512 || (reserved_local >= 2
513 && preempt_ratio > 0.65
514 && shared_ratio < 0.35
515 && global_ratio < 0.20));
516 let should_enter_latency_mode = latency_dispatch_ratio > 0.40
517 || wake_preempt > 0
518 || shared_ratio > 0.45
519 || global_ratio > 0.35
520 || exhaustion_ratio > 0.30;
521 let should_rebalance_mode = rescue_pressure
522 && latency_dispatch_ratio < 0.40
523 && shared_ratio < 0.45
524 && global_ratio < 0.35
525 && exhaustion_ratio < 0.30;
526
527 if keep_latency_mode || (should_enter_latency_mode && !should_rebalance_mode) {
528 AutoTuneMode::Latency
529 } else if should_rebalance_mode {
530 AutoTuneMode::Balanced
531 } else if keep_throughput_mode || should_enter_throughput_mode {
532 AutoTuneMode::Throughput
533 } else {
534 AutoTuneMode::Balanced
535 }
536 }
537
538 fn update(&mut self, current: &Metrics) -> Option<(AutoTuneMode, RuntimeTunables, u64)> {
539 let delta = current.delta(&self.prev_metrics);
540 self.prev_metrics = current.clone();
541
542 let desired_mode = self.evaluate_mode(current, &delta);
543 let mut next_mode = self.mode;
544 let leave_latency =
545 self.mode == AutoTuneMode::Latency && desired_mode != AutoTuneMode::Latency;
546
547 if desired_mode == AutoTuneMode::Latency {
548 self.latency_cooldown = 3;
549 } else if self.latency_cooldown > 0 {
550 self.latency_cooldown -= 1;
551 }
552
553 if desired_mode == self.mode {
554 self.pending_mode = self.mode;
555 self.pending_steps = 0;
556 } else {
557 if desired_mode == self.pending_mode {
558 self.pending_steps = self.pending_steps.saturating_add(1);
559 } else {
560 self.pending_mode = desired_mode;
561 self.pending_steps = 1;
562 }
563
564 let required_steps = if leave_latency || self.latency_cooldown > 0 {
565 4
566 } else {
567 2
568 };
569
570 if self.pending_steps >= required_steps {
571 next_mode = desired_mode;
572 self.pending_mode = desired_mode;
573 self.pending_steps = 0;
574 }
575 }
576
577 let target = RuntimeTunables::target_for(next_mode);
578 let mode_changed = next_mode != self.mode;
579 let tunables_changed = self.tunables.step_towards(target);
580
581 if !mode_changed && !tunables_changed {
582 return None;
583 }
584
585 self.mode = next_mode;
586 self.generation += 1;
587 Some((self.mode, self.tunables, self.generation))
588 }
589}
590
591impl<'a> Scheduler<'a> {
592 fn read_cpu_policy_state(&self) -> CpuPolicyStateAgg {
593 let key = 0u32.to_ne_bytes();
594 let mut agg = CpuPolicyStateAgg::default();
595
596 let percpu_vals: Vec<Vec<u8>> = match self
597 .skel
598 .maps
599 .cpu_state
600 .lookup_percpu(&key, libbpf_rs::MapFlags::ANY)
601 {
602 Ok(Some(vals)) => vals,
603 _ => return agg,
604 };
605
606 for cpu_val in percpu_vals.iter() {
607 if cpu_val.len() < std::mem::size_of::<bpf_intf::flow_cpu_state>() {
608 continue;
609 }
610
611 let state = unsafe {
612 std::ptr::read_unaligned(cpu_val.as_ptr() as *const bpf_intf::flow_cpu_state)
613 };
614
615 agg.urgent_latency_burst_rounds = agg
616 .urgent_latency_burst_rounds
617 .max(state.urgent_latency_burst_rounds);
618 agg.high_priority_burst_rounds = agg
619 .high_priority_burst_rounds
620 .max(state.high_priority_burst_rounds);
621 agg.local_reserved_burst_rounds = agg
622 .local_reserved_burst_rounds
623 .max(state.local_reserved_burst_rounds);
624 agg.local_reserved_fast_grants = agg
625 .local_reserved_fast_grants
626 .saturating_add(state.local_reserved_fast_grants);
627 agg.local_reserved_burst_continuations = agg
628 .local_reserved_burst_continuations
629 .saturating_add(state.local_reserved_burst_continuations);
630 agg.reserved_lane_burst_rounds = agg
631 .reserved_lane_burst_rounds
632 .max(state.reserved_lane_burst_rounds);
633 agg.contained_starvation_rounds = agg
634 .contained_starvation_rounds
635 .max(state.contained_starvation_rounds);
636 agg.shared_starvation_rounds = agg
637 .shared_starvation_rounds
638 .max(state.shared_starvation_rounds);
639 agg.budget_refill_events = agg
640 .budget_refill_events
641 .saturating_add(state.budget_refill_events);
642 agg.budget_exhaustions = agg
643 .budget_exhaustions
644 .saturating_add(state.budget_exhaustions);
645 agg.runnable_wakeups = agg.runnable_wakeups.saturating_add(state.runnable_wakeups);
646 agg.urgent_latency_dispatches = agg
647 .urgent_latency_dispatches
648 .saturating_add(state.urgent_latency_dispatches);
649 agg.urgent_latency_burst_grants = agg
650 .urgent_latency_burst_grants
651 .saturating_add(state.urgent_latency_burst_grants);
652 agg.urgent_latency_burst_continuations = agg
653 .urgent_latency_burst_continuations
654 .saturating_add(state.urgent_latency_burst_continuations);
655 agg.urgent_latency_enqueues = agg
656 .urgent_latency_enqueues
657 .saturating_add(state.urgent_latency_enqueues);
658 agg.urgent_latency_misses = agg
659 .urgent_latency_misses
660 .saturating_add(state.urgent_latency_misses);
661 agg.latency_dispatches = agg
662 .latency_dispatches
663 .saturating_add(state.latency_dispatches);
664 agg.latency_debt_raises = agg
665 .latency_debt_raises
666 .saturating_add(state.latency_debt_raises);
667 agg.latency_debt_decays = agg
668 .latency_debt_decays
669 .saturating_add(state.latency_debt_decays);
670 agg.latency_debt_urgent_enqueues = agg
671 .latency_debt_urgent_enqueues
672 .saturating_add(state.latency_debt_urgent_enqueues);
673 agg.reserved_dispatches = agg
674 .reserved_dispatches
675 .saturating_add(state.reserved_dispatches);
676 agg.shared_dispatches = agg
677 .shared_dispatches
678 .saturating_add(state.shared_dispatches);
679 agg.contained_dispatches = agg
680 .contained_dispatches
681 .saturating_add(state.contained_dispatches);
682 agg.contained_rescue_dispatches = agg
683 .contained_rescue_dispatches
684 .saturating_add(state.contained_rescue_dispatches);
685 agg.shared_rescue_dispatches = agg
686 .shared_rescue_dispatches
687 .saturating_add(state.shared_rescue_dispatches);
688 agg.local_fast_dispatches = agg
689 .local_fast_dispatches
690 .saturating_add(state.local_fast_dispatches);
691 agg.wake_preempt_dispatches = agg
692 .wake_preempt_dispatches
693 .saturating_add(state.wake_preempt_dispatches);
694 agg.cpu_stability_biases = agg
695 .cpu_stability_biases
696 .saturating_add(state.cpu_stability_biases);
697 agg.last_cpu_matches = agg.last_cpu_matches.saturating_add(state.last_cpu_matches);
698 agg.latency_lane_candidates = agg
699 .latency_lane_candidates
700 .saturating_add(state.latency_lane_candidates);
701 agg.latency_lane_enqueues = agg
702 .latency_lane_enqueues
703 .saturating_add(state.latency_lane_enqueues);
704 agg.latency_candidate_local_enqueues = agg
705 .latency_candidate_local_enqueues
706 .saturating_add(state.latency_candidate_local_enqueues);
707 agg.latency_candidate_hog_blocks = agg
708 .latency_candidate_hog_blocks
709 .saturating_add(state.latency_candidate_hog_blocks);
710 agg.positive_budget_wakeups = agg
711 .positive_budget_wakeups
712 .saturating_add(state.positive_budget_wakeups);
713 agg.rt_sensitive_wakeups = agg
714 .rt_sensitive_wakeups
715 .saturating_add(state.rt_sensitive_wakeups);
716 agg.reserved_local_enqueues = agg
717 .reserved_local_enqueues
718 .saturating_add(state.reserved_local_enqueues);
719 agg.reserved_global_enqueues = agg
720 .reserved_global_enqueues
721 .saturating_add(state.reserved_global_enqueues);
722 agg.reserved_quota_skips = agg
723 .reserved_quota_skips
724 .saturating_add(state.reserved_quota_skips);
725 agg.quota_shared_forces = agg
726 .quota_shared_forces
727 .saturating_add(state.quota_shared_forces);
728 agg.quota_contained_forces = agg
729 .quota_contained_forces
730 .saturating_add(state.quota_contained_forces);
731 agg.reserved_lane_grants = agg
732 .reserved_lane_grants
733 .saturating_add(state.reserved_lane_grants);
734 agg.reserved_lane_burst_continuations = agg
735 .reserved_lane_burst_continuations
736 .saturating_add(state.reserved_lane_burst_continuations);
737 agg.reserved_lane_skips = agg
738 .reserved_lane_skips
739 .saturating_add(state.reserved_lane_skips);
740 agg.reserved_lane_shared_forces = agg
741 .reserved_lane_shared_forces
742 .saturating_add(state.reserved_lane_shared_forces);
743 agg.reserved_lane_contained_forces = agg
744 .reserved_lane_contained_forces
745 .saturating_add(state.reserved_lane_contained_forces);
746 agg.reserved_lane_shared_misses = agg
747 .reserved_lane_shared_misses
748 .saturating_add(state.reserved_lane_shared_misses);
749 agg.reserved_lane_contained_misses = agg
750 .reserved_lane_contained_misses
751 .saturating_add(state.reserved_lane_contained_misses);
752 agg.shared_wakeup_enqueues = agg
753 .shared_wakeup_enqueues
754 .saturating_add(state.shared_wakeup_enqueues);
755 agg.shared_starved_head_enqueues = agg
756 .shared_starved_head_enqueues
757 .saturating_add(state.shared_starved_head_enqueues);
758 agg.local_quota_skips = agg
759 .local_quota_skips
760 .saturating_add(state.local_quota_skips);
761 agg.rt_sensitive_local_enqueues = agg
762 .rt_sensitive_local_enqueues
763 .saturating_add(state.rt_sensitive_local_enqueues);
764 agg.rt_sensitive_preempts = agg
765 .rt_sensitive_preempts
766 .saturating_add(state.rt_sensitive_preempts);
767 agg.direct_local_candidates = agg
768 .direct_local_candidates
769 .saturating_add(state.direct_local_candidates);
770 agg.direct_local_enqueues = agg
771 .direct_local_enqueues
772 .saturating_add(state.direct_local_enqueues);
773 agg.direct_local_rejections = agg
774 .direct_local_rejections
775 .saturating_add(state.direct_local_rejections);
776 agg.direct_local_mismatches = agg
777 .direct_local_mismatches
778 .saturating_add(state.direct_local_mismatches);
779 agg.ipc_wake_candidates = agg
780 .ipc_wake_candidates
781 .saturating_add(state.ipc_wake_candidates);
782 agg.ipc_local_enqueues = agg
783 .ipc_local_enqueues
784 .saturating_add(state.ipc_local_enqueues);
785 agg.ipc_score_raises = agg.ipc_score_raises.saturating_add(state.ipc_score_raises);
786 agg.ipc_boosts = agg.ipc_boosts.saturating_add(state.ipc_boosts);
787 agg.contained_enqueues = agg
788 .contained_enqueues
789 .saturating_add(state.contained_enqueues);
790 agg.contained_starved_head_enqueues = agg
791 .contained_starved_head_enqueues
792 .saturating_add(state.contained_starved_head_enqueues);
793 agg.hog_containment_enqueues = agg
794 .hog_containment_enqueues
795 .saturating_add(state.hog_containment_enqueues);
796 agg.hog_recoveries = agg.hog_recoveries.saturating_add(state.hog_recoveries);
797 agg.cpu_migrations = agg.cpu_migrations.saturating_add(state.cpu_migrations);
798 }
799
800 agg
801 }
802
803 fn init(
804 opts: &'a Opts,
805 open_object: &'a mut MaybeUninit<libbpf_rs::OpenObject>,
806 ) -> Result<Self> {
807 try_set_rlimit_infinity();
808
809 let mut skel_builder = BpfSkelBuilder::default();
810 skel_builder.obj_builder.debug(opts.debug);
811
812 let open_opts = opts.libbpf.clone().into_bpf_open_opts();
813 let mut skel = scx_ops_open!(skel_builder, open_object, flow_ops, open_opts)?;
814
815 skel.struct_ops.flow_ops_mut().flags = *compat::SCX_OPS_ENQ_EXITING
816 | *compat::SCX_OPS_ENQ_LAST
817 | *compat::SCX_OPS_ENQ_MIGRATION_DISABLED
818 | *compat::SCX_OPS_ALLOW_QUEUED_WAKEUP;
819
820 let mut skel = scx_ops_load!(skel, flow_ops, uei)?;
821 Self::write_runtime_tunables(
822 &mut skel,
823 RuntimeTunables::default(),
824 AutoTuneMode::Balanced,
825 0,
826 );
827
828 let struct_ops = scx_ops_attach!(skel, flow_ops)?;
829
830 let stats_server = StatsServer::new(stats::server_data()).launch()?;
832
833 Ok(Self {
834 skel,
835 struct_ops: Some(struct_ops),
836 stats_server,
837 })
838 }
839
840 fn get_metrics(&self) -> Metrics {
841 let bss_data = self.skel.maps.bss_data.as_ref().unwrap();
842 let data = self.skel.maps.data_data.as_ref().unwrap();
843 let cpu_policy_state = self.read_cpu_policy_state();
844 Metrics {
845 nr_running: bss_data.nr_running,
846 total_runtime: bss_data.total_runtime,
847 reserved_dispatches: bss_data.reserved_dispatches
848 + cpu_policy_state.reserved_dispatches,
849 urgent_latency_dispatches: bss_data.urgent_latency_dispatches
850 + cpu_policy_state.urgent_latency_dispatches,
851 urgent_latency_burst_grants: bss_data.urgent_latency_burst_grants
852 + cpu_policy_state.urgent_latency_burst_grants,
853 urgent_latency_burst_continuations: bss_data.urgent_latency_burst_continuations
854 + cpu_policy_state.urgent_latency_burst_continuations,
855 latency_dispatches: bss_data.latency_dispatches + cpu_policy_state.latency_dispatches,
856 shared_dispatches: bss_data.shared_dispatches + cpu_policy_state.shared_dispatches,
857 contained_dispatches: bss_data.contained_dispatches
858 + cpu_policy_state.contained_dispatches,
859 local_fast_dispatches: bss_data.local_fast_dispatches
860 + cpu_policy_state.local_fast_dispatches,
861 wake_preempt_dispatches: bss_data.wake_preempt_dispatches
862 + cpu_policy_state.wake_preempt_dispatches,
863 budget_refill_events: bss_data.budget_refill_events
864 + cpu_policy_state.budget_refill_events,
865 budget_exhaustions: bss_data.budget_exhaustions + cpu_policy_state.budget_exhaustions,
866 positive_budget_wakeups: bss_data.positive_budget_wakeups
867 + cpu_policy_state.positive_budget_wakeups,
868 urgent_latency_enqueues: bss_data.urgent_latency_enqueues
869 + cpu_policy_state.urgent_latency_enqueues,
870 latency_lane_enqueues: bss_data.latency_lane_enqueues
871 + cpu_policy_state.latency_lane_enqueues,
872 latency_lane_candidates: bss_data.latency_lane_candidates
873 + cpu_policy_state.latency_lane_candidates,
874 latency_candidate_local_enqueues: bss_data.latency_candidate_local_enqueues
875 + cpu_policy_state.latency_candidate_local_enqueues,
876 latency_candidate_hog_blocks: bss_data.latency_candidate_hog_blocks
877 + cpu_policy_state.latency_candidate_hog_blocks,
878 latency_debt_raises: bss_data.latency_debt_raises
879 + cpu_policy_state.latency_debt_raises,
880 latency_debt_decays: bss_data.latency_debt_decays
881 + cpu_policy_state.latency_debt_decays,
882 latency_debt_urgent_enqueues: bss_data.latency_debt_urgent_enqueues
883 + cpu_policy_state.latency_debt_urgent_enqueues,
884 urgent_latency_misses: bss_data.urgent_latency_misses
885 + cpu_policy_state.urgent_latency_misses,
886 reserved_local_enqueues: bss_data.reserved_local_enqueues
887 + cpu_policy_state.reserved_local_enqueues,
888 reserved_global_enqueues: bss_data.reserved_global_enqueues
889 + cpu_policy_state.reserved_global_enqueues,
890 shared_wakeup_enqueues: bss_data.shared_wakeup_enqueues
891 + cpu_policy_state.shared_wakeup_enqueues,
892 runnable_wakeups: bss_data.runnable_wakeups + cpu_policy_state.runnable_wakeups,
893 cpu_release_reenqueues: bss_data.cpu_release_reenqueues,
894 urgent_latency_burst_rounds: cpu_policy_state.urgent_latency_burst_rounds,
895 high_priority_burst_rounds: cpu_policy_state.high_priority_burst_rounds,
896 local_reserved_burst_rounds: cpu_policy_state.local_reserved_burst_rounds,
897 local_reserved_fast_grants: bss_data.local_reserved_fast_grants
898 + cpu_policy_state.local_reserved_fast_grants,
899 local_reserved_burst_continuations: bss_data.local_reserved_burst_continuations
900 + cpu_policy_state.local_reserved_burst_continuations,
901 local_quota_skips: bss_data.local_quota_skips + cpu_policy_state.local_quota_skips,
902 reserved_quota_skips: bss_data.reserved_quota_skips
903 + cpu_policy_state.reserved_quota_skips,
904 quota_shared_forces: bss_data.quota_shared_forces
905 + cpu_policy_state.quota_shared_forces,
906 quota_contained_forces: bss_data.quota_contained_forces
907 + cpu_policy_state.quota_contained_forces,
908 init_task_events: bss_data.init_task_events,
909 enable_events: bss_data.enable_events,
910 exit_task_events: bss_data.exit_task_events,
911 cpu_stability_biases: bss_data.cpu_stability_biases
912 + cpu_policy_state.cpu_stability_biases,
913 last_cpu_matches: bss_data.last_cpu_matches + cpu_policy_state.last_cpu_matches,
914 cpu_migrations: bss_data.cpu_migrations + cpu_policy_state.cpu_migrations,
915 rt_sensitive_wakeups: bss_data.rt_sensitive_wakeups
916 + cpu_policy_state.rt_sensitive_wakeups,
917 rt_sensitive_local_enqueues: bss_data.rt_sensitive_local_enqueues
918 + cpu_policy_state.rt_sensitive_local_enqueues,
919 rt_sensitive_preempts: bss_data.rt_sensitive_preempts
920 + cpu_policy_state.rt_sensitive_preempts,
921 reserved_lane_burst_rounds: cpu_policy_state.reserved_lane_burst_rounds,
922 reserved_lane_grants: bss_data.reserved_lane_grants
923 + cpu_policy_state.reserved_lane_grants,
924 reserved_lane_burst_continuations: bss_data.reserved_lane_burst_continuations
925 + cpu_policy_state.reserved_lane_burst_continuations,
926 reserved_lane_skips: bss_data.reserved_lane_skips
927 + cpu_policy_state.reserved_lane_skips,
928 reserved_lane_shared_forces: bss_data.reserved_lane_shared_forces
929 + cpu_policy_state.reserved_lane_shared_forces,
930 reserved_lane_contained_forces: bss_data.reserved_lane_contained_forces
931 + cpu_policy_state.reserved_lane_contained_forces,
932 reserved_lane_shared_misses: bss_data.reserved_lane_shared_misses
933 + cpu_policy_state.reserved_lane_shared_misses,
934 reserved_lane_contained_misses: bss_data.reserved_lane_contained_misses
935 + cpu_policy_state.reserved_lane_contained_misses,
936 contained_starved_head_enqueues: bss_data.contained_starved_head_enqueues
937 + cpu_policy_state.contained_starved_head_enqueues,
938 shared_starved_head_enqueues: bss_data.shared_starved_head_enqueues
939 + cpu_policy_state.shared_starved_head_enqueues,
940 direct_local_candidates: bss_data.direct_local_candidates
941 + cpu_policy_state.direct_local_candidates,
942 direct_local_enqueues: bss_data.direct_local_enqueues
943 + cpu_policy_state.direct_local_enqueues,
944 direct_local_rejections: bss_data.direct_local_rejections
945 + cpu_policy_state.direct_local_rejections,
946 direct_local_mismatches: bss_data.direct_local_mismatches
947 + cpu_policy_state.direct_local_mismatches,
948 ipc_wake_candidates: bss_data.ipc_wake_candidates
949 + cpu_policy_state.ipc_wake_candidates,
950 ipc_local_enqueues: bss_data.ipc_local_enqueues + cpu_policy_state.ipc_local_enqueues,
951 ipc_score_raises: bss_data.ipc_score_raises + cpu_policy_state.ipc_score_raises,
952 ipc_boosts: bss_data.ipc_boosts + cpu_policy_state.ipc_boosts,
953 contained_enqueues: bss_data.contained_enqueues + cpu_policy_state.contained_enqueues,
954 hog_containment_enqueues: bss_data.hog_containment_enqueues
955 + cpu_policy_state.hog_containment_enqueues,
956 hog_recoveries: bss_data.hog_recoveries + cpu_policy_state.hog_recoveries,
957 contained_starvation_rounds: cpu_policy_state.contained_starvation_rounds,
958 shared_starvation_rounds: cpu_policy_state.shared_starvation_rounds,
959 contained_rescue_dispatches: bss_data.contained_rescue_dispatches
960 + cpu_policy_state.contained_rescue_dispatches,
961 shared_rescue_dispatches: bss_data.shared_rescue_dispatches
962 + cpu_policy_state.shared_rescue_dispatches,
963 tune_latency_credit_grant: data.tune_latency_credit_grant,
964 tune_latency_credit_decay: data.tune_latency_credit_decay,
965 tune_latency_debt_urgent_min: data.tune_latency_debt_urgent_min,
966 tune_urgent_latency_burst_max: data.tune_urgent_latency_burst_max,
967 tune_reserved_quota_burst_max: data.tune_reserved_quota_burst_max,
968 tune_contained_starvation_max: data.tune_contained_starvation_max,
969 tune_shared_starvation_max: data.tune_shared_starvation_max,
970 tune_local_fast_nr_running_max: data.tune_local_fast_nr_running_max,
971 tune_local_reserved_burst_max: data.tune_local_reserved_burst_max,
972 tune_reserved_lane_burst_max: data.tune_reserved_lane_burst_max,
973 autotune_generation: bss_data.autotune_generation,
974 autotune_mode: bss_data.autotune_mode,
975 tune_reserved_max_ns: data.tune_reserved_max_ns,
976 tune_shared_slice_ns: data.tune_shared_slice_ns,
977 tune_interactive_floor_ns: data.tune_interactive_floor_ns,
978 tune_preempt_budget_min_ns: data.tune_preempt_budget_min_ns,
979 tune_preempt_refill_min_ns: data.tune_preempt_refill_min_ns,
980 }
981 }
982
983 fn write_runtime_tunables(
984 skel: &mut BpfSkel<'a>,
985 tunables: RuntimeTunables,
986 mode: AutoTuneMode,
987 generation: u64,
988 ) {
989 let data = skel.maps.data_data.as_mut().unwrap();
990 data.tune_reserved_max_ns = tunables.reserved_max_ns;
991 data.tune_shared_slice_ns = tunables.shared_slice_ns;
992 data.tune_interactive_floor_ns = tunables.interactive_floor_ns;
993 data.tune_preempt_budget_min_ns = tunables.preempt_budget_min_ns;
994 data.tune_preempt_refill_min_ns = tunables.preempt_refill_min_ns;
995 data.tune_latency_credit_grant = tunables.latency_credit_grant;
996 data.tune_latency_credit_decay = tunables.latency_credit_decay;
997 data.tune_latency_debt_urgent_min = tunables.latency_debt_urgent_min;
998 data.tune_urgent_latency_burst_max = tunables.urgent_latency_burst_max;
999 data.tune_reserved_quota_burst_max = tunables.reserved_quota_burst_max;
1000 data.tune_contained_starvation_max = tunables.contained_starvation_max;
1001 data.tune_shared_starvation_max = tunables.shared_starvation_max;
1002 data.tune_local_fast_nr_running_max = tunables.local_fast_nr_running_max;
1003 data.tune_local_reserved_burst_max = tunables.local_reserved_burst_max;
1004 data.tune_reserved_lane_burst_max = tunables.reserved_lane_burst_max;
1005
1006 let bss_data = skel.maps.bss_data.as_mut().unwrap();
1007 bss_data.autotune_mode = mode.as_u64();
1008 bss_data.autotune_generation = generation;
1009 }
1010
1011 fn apply_runtime_tunables(
1012 &mut self,
1013 tunables: RuntimeTunables,
1014 mode: AutoTuneMode,
1015 generation: u64,
1016 ) {
1017 Self::write_runtime_tunables(&mut self.skel, tunables, mode, generation);
1018 }
1019
1020 fn exited(&self) -> bool {
1021 uei_exited!(&self.skel, uei)
1022 }
1023
1024 fn run(&mut self, shutdown: Arc<AtomicBool>, autotune_enabled: bool) -> Result<UserExitInfo> {
1025 let (res_ch, req_ch) = self.stats_server.channels();
1026 let mut autotuner = autotune_enabled.then(|| AutoTuner::new(self.get_metrics()));
1027 let mut next_tune_at = Instant::now() + Duration::from_secs(1);
1028
1029 while !shutdown.load(Ordering::Relaxed) && !self.exited() {
1030 match req_ch.recv_timeout(Duration::from_millis(250)) {
1031 Ok(()) => res_ch.send(self.get_metrics())?,
1032 Err(RecvTimeoutError::Timeout) => {}
1033 Err(e) => Err(e)?,
1034 }
1035
1036 if let Some(autotuner) = autotuner.as_mut() {
1037 if Instant::now() >= next_tune_at {
1038 let current = self.get_metrics();
1039 if let Some((mode, tunables, generation)) = autotuner.update(¤t) {
1040 self.apply_runtime_tunables(tunables, mode, generation);
1041 info!(
1042 "autotune={} gen={} reserve_cap={}us shared_slice={}us refill_floor={}us preempt_budget={}us preempt_refill={}us debt_min={} urgent_burst_max={} reserved_quota_max={} reserved_lane_max={} local_burst_max={}",
1043 mode.as_str(),
1044 generation,
1045 tunables.reserved_max_ns / 1000,
1046 tunables.shared_slice_ns / 1000,
1047 tunables.interactive_floor_ns / 1000,
1048 tunables.preempt_budget_min_ns / 1000,
1049 tunables.preempt_refill_min_ns / 1000,
1050 tunables.latency_debt_urgent_min,
1051 tunables.urgent_latency_burst_max,
1052 tunables.reserved_quota_burst_max,
1053 tunables.reserved_lane_burst_max,
1054 tunables.local_reserved_burst_max,
1055 );
1056 }
1057 next_tune_at = Instant::now() + Duration::from_secs(1);
1058 }
1059 }
1060 }
1061
1062 let _ = self.struct_ops.take();
1063 uei_report!(&self.skel, uei)
1064 }
1065}
1066
1067fn main() -> Result<()> {
1068 let opts = Opts::parse();
1069
1070 if let Some(shell) = opts.completions {
1071 generate(
1072 shell,
1073 &mut Opts::command(),
1074 SCHEDULER_NAME,
1075 &mut std::io::stdout(),
1076 );
1077 return Ok(());
1078 }
1079
1080 let monitor_only = opts.monitor.is_some();
1081
1082 if opts.version {
1083 println!("{} {}", SCHEDULER_NAME, full_version());
1084 return Ok(());
1085 }
1086
1087 if !monitor_only {
1088 simplelog::SimpleLogger::init(
1089 if opts.debug {
1090 simplelog::LevelFilter::Debug
1091 } else {
1092 simplelog::LevelFilter::Info
1093 },
1094 simplelog::Config::default(),
1095 )?;
1096
1097 info!("{} {}", SCHEDULER_NAME, full_version());
1098 info!("Starting {} scheduler", SCHEDULER_NAME);
1099 }
1100
1101 let shutdown = Arc::new(AtomicBool::new(false));
1102 let shutdown_clone = shutdown.clone();
1103
1104 ctrlc::set_handler(move || {
1105 shutdown_clone.store(true, Ordering::Relaxed);
1106 })?;
1107
1108 if let Some(intv) = opts.monitor.or(opts.stats) {
1109 let monitor_shutdown = shutdown.clone();
1110 let jh = std::thread::spawn(move || {
1111 if let Err(err) = stats::monitor(Duration::from_secs_f64(intv), monitor_shutdown) {
1112 log::warn!("stats monitor thread finished with error: {err}");
1113 }
1114 });
1115
1116 if monitor_only {
1117 let _ = jh.join();
1118 return Ok(());
1119 }
1120 }
1121
1122 let mut open_object = MaybeUninit::<libbpf_rs::OpenObject>::uninit();
1123 let mut sched = Scheduler::init(&opts, &mut open_object)?;
1124 sched.run(shutdown, !opts.no_autotune)?;
1125
1126 info!("Scheduler exited");
1127
1128 Ok(())
1129}