scx_flow/rust/
config.rs

1// SPDX-License-Identifier: GPL-2.0
2//! Validated scheduling constants for the flow scheduler.
3//!
4//! Copyright (c) 2026 Galih Tama <galpt@v.recipes>
5
6//! Holds the validated constants with defaults that match intf.h.
7
8use anyhow::Result;
9use anyhow::bail;
10
11/// Fixed slice in nanos at 1ms. Every insert uses this slice.
12pub const QUANTUM_NS: u64 = 1_000_000;
13/// Dynamic slice floor in nanos at 10us. Fresh waits clamp the
14/// saturated remaining time to this floor with the quantum ceiling.
15pub const SLICE_MIN_NS: u64 = 10_000;
16/// Ultra-low latency boundary in nanos at 50us. Caps the T1 tier,
17/// so audio plus frame pacing bursts pace tightly with no extra hold.
18pub const SLICE_ULL_NS: u64 = 50_000;
19/// Low latency boundary in nanos at 100us. Caps the T2 tier, so
20/// compositor plus UI bursts rotate each tenth quantum with no hold.
21/// Matches the latency slice cap, so urgent wakeups keep one bound.
22pub const SLICE_LL_NS: u64 = 100_000;
23/// Medium-low latency boundary in nanos at 500us. Caps the T3 tier,
24/// so IPC plus microservice bursts hold margin with no storm.
25pub const SLICE_MLL_NS: u64 = 500_000;
26/// Normal latency boundary in nanos at 1ms. Caps the T4 tier at one
27/// quantum, so desktop plus worker bursts rotate each quantum.
28pub const SLICE_NL_NS: u64 = 1_000_000;
29/// Extended normal latency boundary in nanos at 5ms. Caps the T5
30/// tier, so bulk I/O plus build bursts hold room with no starve.
31pub const SLICE_ENL_NS: u64 = 5_000_000;
32/// Dynamic slice ceiling in nanos at 20ms. Caps the T6 tier plus the
33/// multi-tier spectrum, so throughput bursts hold room with an 80ms
34/// period guarding starvation.
35pub const SLICE_MAX_NS: u64 = 20_000_000;
36/// Default period in nanos at 80ms. Holds four 20ms slices, so a
37/// fully used throughput task still leaves room for waits plus
38/// retries inside the period with no starvation under 20ms holds.
39pub const PERIOD_NS: u64 = 80_000_000;
40/// Least predictor value in nanos at 1. Clamps short bursts.
41pub const PRED_MIN_NS: u64 = 1;
42/// Largest predictor value in nanos at 1s. Clamps long bursts.
43pub const PRED_MAX_NS: u64 = 1_000_000_000;
44/// Base capacity in units at 1024. Symmetric hosts share the base.
45pub const CAP_BASE: u32 = 1024;
46/// Base weight with a neutral share.
47pub const WEIGHT_BASE: u32 = 128;
48/// Least weight admitted.
49pub const WEIGHT_MIN: u32 = 1;
50/// Largest weight admitted.
51pub const WEIGHT_MAX: u32 = 16_384;
52/// Max hint rows bound shared with the BPF header.
53pub const HINT_MAX: u64 = 8192;
54/// RED bound in nanos at 256us. Caps the maximum exceeding time with
55/// the fixed bound shared by admit plus reclaim plus credit.
56pub const RED_EMAX_NS: u64 = 256_000;
57/// RED tolerance in nanos at 64us. Holds hard task slack only with the
58/// fixed value shared by admit plus reclaim.
59pub const RED_TOL_NS: u64 = 64_000;
60/// Adaptive grow step in nanos at 64us frozen for wire compat only.
61/// Whitelisted with no use after the proportional law.
62pub const ADAPT_GROW_NS: u64 = 64_000;
63/// Adaptive shrink step in nanos at 128us frozen for wire compat only.
64/// Whitelisted with no use after the proportional law.
65pub const ADAPT_SHRINK_NS: u64 = 128_000;
66/// Proportional adapt cap in nanos at 1ms. Caps one shift step.
67pub const ADAPT_PROP_MAX_NS: u64 = 1_000_000;
68/// Proportional adapt shift at 3. Maps exceed plus slack to one eighth.
69pub const ADAPT_PROP_SHIFT: u32 = 3;
70/// Proportional deadband shift at 5. Holds one thirty-second of the base
71/// as the quiet band with shifts only.
72pub const ADAPT_DEADBAND_SHIFT: u32 = 5;
73/// Proportional deadband floor in nanos at 8us. Keeps the quiet band
74/// above one tick on small slices.
75pub const ADAPT_DEADBAND_MIN_NS: u64 = 8_000;
76/// Half-damp shift at 1. Halves the capped step once, so one correction
77/// moves at most one quarter base.
78pub const ADAPT_DAMP_SHIFT: u32 = 1;
79/// Predictor rise shift at 3. Moves the average one eighth toward a
80/// larger sample, so bursts grow fast.
81pub const PRED_UP_SHIFT: u32 = 3;
82/// Predictor fall shift at 4. Moves the average one sixteenth toward a
83/// smaller sample, so bursts decay slow.
84pub const PRED_DOWN_SHIFT: u32 = 4;
85/// Deviation widen shift at 2. Moves the deviation one quarter toward a
86/// larger error, so ragged bursts gain margin fast.
87pub const PRED_DEV_UP_SHIFT: u32 = 2;
88/// Deviation narrow shift at 3. Moves the deviation one eighth toward a
89/// smaller error, so stable bursts keep margin.
90pub const PRED_DEV_DOWN_SHIFT: u32 = 3;
91/// Deviation margin shift at 1. Adds half deviation to the period, so
92/// the deadline holds average plus one and a half deviation.
93pub const PRED_DEV_MARGIN_SHIFT: u32 = 1;
94/// Latency slice cap in nanos at 100us. Caps the dynamic slice plus
95/// the carryover plus the adapt step for latency work, so urgent
96/// wakeups rotate each tenth quantum with no extra hold. The cap
97/// equals the preempt tail, so a latency head at the cap never passes
98/// the tail gate and floors with no resume. Adapt steps cap at 1ms
99/// then re-clamp here, so one shift never holds past the cap. RED Emax
100/// stays at 256us past the cap, so a capped latency slice never pays
101/// as victim with no swap.
102pub const SLICE_LAT_NS: u64 = 100_000;
103/// Latency sleep floor in nanos at 20us. Vetoes only sleeps below this
104/// floor toward batch with no knob, so the 100us slice cap stays
105/// decoupled with no extra hold. Like fair.c, the floor paces service
106/// with no fixed priority hold. Unknown sleep fails open with no stall.
107pub const LAT_SLEEP_MIN_NS: u64 = 20_000;
108/// Latency deadline bound in nanos at 4ms. Holds one twentieth period.
109pub const D_LAT_NS: u64 = 4_000_000;
110/// Preempt margin in nanos at 100us. Leads the occupant key, so near
111/// ties never bounce with no knob.
112pub const PREEMPT_MARGIN_NS: u64 = 100_000;
113/// Preempt tail in nanos at 100us. Holds the owner leftover, so nearly
114/// done owners finish with no knob.
115pub const PREEMPT_TAIL_NS: u64 = 100_000;
116
117/// Default fixed slice in nanos.
118const DEF_QUANTUM_NS: u64 = QUANTUM_NS;
119
120/// Mirror of flow_adapt_deadband in intf.h for host tests.
121/// Holds the larger of base shift 5 plus the 8us floor.
122#[cfg(test)]
123pub fn adapt_deadband(base: u64) -> u64 {
124    let band = base >> ADAPT_DEADBAND_SHIFT;
125    if band < ADAPT_DEADBAND_MIN_NS {
126        return ADAPT_DEADBAND_MIN_NS;
127    }
128    band
129}
130
131/// Mirror of flow_adapt_prop in intf.h for host tests.
132/// Shifts only without a divide, capped at 1ms plus half of base then
133/// halved once more, clamped 10us to 1ms with proportional deadband.
134/// An exceed shrinks, a slack grows, both zero holds, exceed wins.
135#[cfg(test)]
136pub fn adapt_prop(cur: u32, exceed: u64, slack: u64) -> u32 {
137    let base = if cur == 0 { QUANTUM_NS } else { cur as u64 };
138    let band = adapt_deadband(base);
139    // Clamp helper keeps the same 10us plus 1ms bounds as BPF.
140    let clamp = |v: u64| -> u32 {
141        if v < SLICE_MIN_NS {
142            return SLICE_MIN_NS as u32;
143        }
144        if v > QUANTUM_NS {
145            return QUANTUM_NS as u32;
146        }
147        v as u32
148    };
149    if exceed != 0 {
150        if exceed <= band {
151            return clamp(base);
152        }
153        let mut step = exceed >> ADAPT_PROP_SHIFT;
154        if step > ADAPT_PROP_MAX_NS {
155            step = ADAPT_PROP_MAX_NS;
156        }
157        if step > (base >> 1) {
158            step = base >> 1;
159        }
160        step >>= ADAPT_DAMP_SHIFT;
161        if step == 0 {
162            return clamp(base);
163        }
164        if base <= step {
165            return SLICE_MIN_NS as u32;
166        }
167        return clamp(base - step);
168    }
169    if slack != 0 {
170        if slack <= band {
171            return clamp(base);
172        }
173        let mut step = slack >> ADAPT_PROP_SHIFT;
174        if step > ADAPT_PROP_MAX_NS {
175            step = ADAPT_PROP_MAX_NS;
176        }
177        if step > (base >> 1) {
178            step = base >> 1;
179        }
180        step >>= ADAPT_DAMP_SHIFT;
181        if step == 0 {
182            return clamp(base);
183        }
184        return clamp(base.saturating_add(step));
185    }
186    clamp(base)
187}
188
189/// Mirror of flow_slice_lat_clamp in intf.h for host tests.
190/// Caps the slice at 100us for latency with no knob.
191#[cfg(test)]
192pub fn slice_lat_clamp(slice: u32, is_lat: bool) -> u32 {
193    if is_lat && (slice as u64) > SLICE_LAT_NS {
194        return SLICE_LAT_NS as u32;
195    }
196    slice
197}
198
199/// Mirror of flow_pred_avg in intf.h for host tests.
200/// Rise shift 3 plus fall shift 4 with 1ns to 1s clamp.
201#[cfg(test)]
202pub fn pred_avg(avg: u64, delta: u64) -> u64 {
203    let d = if delta == 0 { PRED_MIN_NS } else { delta }.clamp(PRED_MIN_NS, PRED_MAX_NS);
204    if avg == 0 {
205        return d;
206    }
207    if d > avg {
208        let diff = (d - avg) >> PRED_UP_SHIFT;
209        return avg.saturating_add(diff).clamp(PRED_MIN_NS, PRED_MAX_NS);
210    }
211    let diff = (avg - d) >> PRED_DOWN_SHIFT;
212    if diff > avg {
213        return PRED_MIN_NS;
214    }
215    (avg - diff).clamp(PRED_MIN_NS, PRED_MAX_NS)
216}
217
218/// Mirror of flow_pred_dev in intf.h for host tests.
219/// Widen shift 2 plus narrow shift 3 with first floor at average quarter.
220#[cfg(test)]
221pub fn pred_dev(dev: u64, avg: u64, delta: u64) -> u64 {
222    let d = if delta == 0 { PRED_MIN_NS } else { delta }.clamp(PRED_MIN_NS, PRED_MAX_NS);
223    let a = if avg == 0 { d } else { avg };
224    let mut err = d.abs_diff(a);
225    if err == 0 {
226        err = PRED_MIN_NS;
227    }
228    err = err.clamp(PRED_MIN_NS, PRED_MAX_NS);
229    if dev == 0 {
230        let floor = avg >> 2;
231        if floor > err {
232            return floor.clamp(PRED_MIN_NS, PRED_MAX_NS);
233        }
234        return err;
235    }
236    if err > dev {
237        let diff = (err - dev) >> PRED_DEV_UP_SHIFT;
238        return dev.saturating_add(diff).clamp(PRED_MIN_NS, PRED_MAX_NS);
239    }
240    let diff = (dev - err) >> PRED_DEV_DOWN_SHIFT;
241    if diff > dev {
242        return PRED_MIN_NS;
243    }
244    (dev - diff).clamp(PRED_MIN_NS, PRED_MAX_NS)
245}
246
247/// Mirror of flow_pred_period in intf.h for host tests.
248/// Adds average plus deviation plus half deviation margin.
249#[cfg(test)]
250pub fn pred_period(avg: u64, dev: u64) -> u64 {
251    if avg == 0 {
252        return PERIOD_NS;
253    }
254    let margin = dev >> PRED_DEV_MARGIN_SHIFT;
255    let sum = avg.saturating_add(dev).saturating_add(margin);
256    if sum == u64::MAX {
257        return PRED_MAX_NS;
258    }
259    sum.clamp(PRED_MIN_NS, PRED_MAX_NS)
260}
261
262/// Mirror of flow_classify_latency_tier in intf.h for host tests.
263/// Adds avg plus dev plus half dev margin with saturation, so stable
264/// bursts keep a tight tier while ragged bursts hold margin with no
265/// wrap. Critical forces ULL, then 50us plus 100us plus 500us plus 1ms
266/// plus 5ms bound ULL plus LL plus MLL plus NL plus ENL, else 20ms holds
267/// throughput. Host-test mirror only at 4.8.14 with no BPF slice caller.
268#[cfg(test)]
269pub fn classify_tier(avg: u64, dev: u64, is_crit: bool) -> u32 {
270    let margin = dev >> PRED_DEV_MARGIN_SHIFT;
271    let pred = avg.saturating_add(dev).saturating_add(margin);
272    if is_crit || pred <= SLICE_ULL_NS {
273        return SLICE_ULL_NS as u32;
274    }
275    if pred <= SLICE_LL_NS {
276        return SLICE_LL_NS as u32;
277    }
278    if pred <= SLICE_MLL_NS {
279        return SLICE_MLL_NS as u32;
280    }
281    if pred <= SLICE_NL_NS {
282        return SLICE_NL_NS as u32;
283    }
284    if pred <= SLICE_ENL_NS {
285        return SLICE_ENL_NS as u32;
286    }
287    SLICE_MAX_NS as u32
288}
289
290/// Mirror of flow_red_cost in intf.h for host tests.
291/// Costs burst else slice else quantum clamped to 10us plus 20ms, so the
292/// guarantee holds without shaping shares plus throughput never
293/// underestimates past the quantum.
294#[cfg(test)]
295pub fn red_cost(avg: u64, slice: u32) -> u64 {
296    if avg != 0 {
297        if avg < SLICE_MIN_NS {
298            return SLICE_MIN_NS;
299        }
300        if avg > SLICE_MAX_NS {
301            return SLICE_MAX_NS;
302        }
303        return avg;
304    }
305    if slice != 0 {
306        let s = slice as u64;
307        if s < SLICE_MIN_NS {
308            return SLICE_MIN_NS;
309        }
310        if s > SLICE_MAX_NS {
311            return SLICE_MAX_NS;
312        }
313        return s;
314    }
315    QUANTUM_NS
316}
317
318/// Mirror of flow_lat_deadline in intf.h for host tests.
319/// Takes max 4ms else slice plus margin with saturation, then adds now
320/// with saturation for ns accuracy without a divide.
321#[cfg(test)]
322pub fn lat_deadline(now: u64, slice: u32) -> u64 {
323    let need = (slice as u64).saturating_add(PREEMPT_MARGIN_NS);
324    let bound = if need > D_LAT_NS { need } else { D_LAT_NS };
325    now.saturating_add(bound)
326}
327
328/// Mirror of flow_lat_crit_ext in intf.h for host tests.
329/// Advisory veto toward batch with 20us sleep floor plus quantum-relative
330/// bounds using shifts only, so unknown sleep, clamp, and share fail open.
331/// The sleep floor stays decoupled from the 100us slice cap. Newcomers with
332/// no history stay batch with probation. The burst test adds average plus
333/// deviation plus half deviation margin, so RED plus slice agree.
334#[cfg(test)]
335pub fn lat_crit_ext(
336    avg: u64,
337    dev: u64,
338    hint_us: u32,
339    sleep_ns: u64,
340    uclamp_min: u32,
341    eff_w: u32,
342) -> bool {
343    if avg == 0 {
344        return false;
345    }
346    {
347        let margin = dev >> PRED_DEV_MARGIN_SHIFT;
348        let pred = avg.saturating_add(dev).saturating_add(margin);
349        if pred == u64::MAX {
350            return false;
351        }
352        if pred > QUANTUM_NS {
353            return false;
354        }
355    }
356    if hint_us != 0 {
357        let h = hint_us as u64;
358        if h > 18_446_744_073_709_551 {
359            return false;
360        }
361        if h * 1000 > QUANTUM_NS {
362            return false;
363        }
364    }
365    if sleep_ns != 0 && sleep_ns < LAT_SLEEP_MIN_NS {
366        return false;
367    }
368    if uclamp_min != 0 && uclamp_min < 512 {
369        return false;
370    }
371    if eff_w != 0 && eff_w < WEIGHT_BASE {
372        return false;
373    }
374    true
375}
376
377/// Mirror of flow_lat_crit in intf.h for host tests.
378/// Newcomers with no history stay batch with probation. Tests average
379/// plus deviation plus half deviation margin against the quantum.
380#[cfg(test)]
381pub fn lat_crit(avg: u64, dev: u64) -> bool {
382    if avg == 0 {
383        return false;
384    }
385    let margin = dev >> PRED_DEV_MARGIN_SHIFT;
386    let pred = avg.saturating_add(dev).saturating_add(margin);
387    if pred == u64::MAX {
388        return false;
389    }
390    pred <= QUANTUM_NS
391}
392
393/// Mirror of flow_is_lat in intf.h for host tests.
394/// Unifies base plus ext, so RED and slice agree.
395#[cfg(test)]
396pub fn is_lat(
397    avg: u64,
398    dev: u64,
399    hint_us: u32,
400    sleep_ns: u64,
401    uclamp_min: u32,
402    eff_w: u32,
403) -> bool {
404    if !lat_crit(avg, dev) {
405        return false;
406    }
407    lat_crit_ext(avg, dev, hint_us, sleep_ns, uclamp_min, eff_w)
408}
409
410/// Wrap-safe order for host tests mirroring flow_time_before.
411/// Uses the signed diff, so the u64 wrap keeps order with no branch.
412#[cfg(test)]
413fn time_before(a: u64, b: u64) -> bool {
414    (a.wrapping_sub(b) as i64) < 0
415}
416
417/// Mirror of flow_sleep_ns in intf.h for host tests.
418/// Isolates now minus wait minus burst, so queue plus run never count
419/// as sleep with wrap safety and probation.
420#[cfg(test)]
421pub fn sleep_ns(old_wait: u64, now: u64, avg: u64) -> u64 {
422    if old_wait == 0 {
423        return 0;
424    }
425    if !time_before(old_wait, now) {
426        return 0;
427    }
428    let mut gap = now.wrapping_sub(old_wait);
429    if avg != 0 {
430        if gap > avg {
431            gap -= avg;
432        } else {
433            gap = 0;
434        }
435    }
436    gap
437}
438
439/// Mirror of flow_preempt_ok in preempt.bpf.c for host tests.
440/// Needs lat arrival against batch owner with 100us margin lead plus
441/// 100us tail left on start plus inherit cur, so near ties plus nearly
442/// done owners never bounce with wrap safety.
443#[cfg(test)]
444pub fn preempt_ok(
445    arr_key: u64,
446    occ_key: u64,
447    now: u64,
448    occ_run_at: u64,
449    occ_cur: u32,
450    arr_lat: bool,
451    occ_lat: bool,
452) -> bool {
453    if !arr_lat || occ_lat {
454        return false;
455    }
456    if arr_key == 0 || arr_key == u64::MAX {
457        return false;
458    }
459    if occ_key == 0 || occ_key == u64::MAX {
460        return false;
461    }
462    if !time_before(arr_key, occ_key) {
463        return false;
464    }
465    let margin = arr_key.saturating_add(PREEMPT_MARGIN_NS);
466    if margin == u64::MAX {
467        return false;
468    }
469    if !time_before(margin, occ_key) {
470        return false;
471    }
472    if occ_run_at == 0 || occ_run_at == u64::MAX {
473        return false;
474    }
475    let base = if occ_cur == 0 {
476        QUANTUM_NS
477    } else {
478        occ_cur as u64
479    };
480    let occ_end = occ_run_at.saturating_add(base);
481    if occ_end == u64::MAX {
482        return false;
483    }
484    let tail = now.saturating_add(PREEMPT_TAIL_NS);
485    if tail == u64::MAX {
486        return false;
487    }
488    if !time_before(tail, occ_end) {
489        return false;
490    }
491    true
492}
493
494/// Mirror of flow_slice_resume in intf.h for host tests.
495/// Keeps start plus cur leftover clamped to 10us plus 100us else 1ms
496/// with wrap-safe order, so a wrapped clock never reads as leftover.
497/// Equality fails closed and floors, so a head at the 100us cap never
498/// resumes past the equal tail.
499#[cfg(test)]
500pub fn slice_resume(start: u64, now: u64, is_lat: bool, cur: u32) -> u32 {
501    let base = if cur == 0 { QUANTUM_NS } else { cur as u64 };
502    if start == 0 || start == u64::MAX {
503        return base as u32;
504    }
505    let end = start.saturating_add(base);
506    if end == u64::MAX {
507        return base as u32;
508    }
509    let rem = if !time_before(now, end) && now != end {
510        0
511    } else if time_before(now, end) {
512        end.wrapping_sub(now)
513    } else {
514        0
515    };
516    let cap = if is_lat { SLICE_LAT_NS } else { QUANTUM_NS };
517    if rem < SLICE_MIN_NS {
518        return SLICE_MIN_NS as u32;
519    }
520    if rem > cap {
521        return cap as u32;
522    }
523    rem as u32
524}
525
526/// Validated scheduling constants.
527#[derive(Debug, Clone, PartialEq, Eq)]
528pub struct Config {
529    /// Fixed slice in nanos. Always 1ms with no knob.
530    pub quantum_ns: u64,
531}
532
533impl Default for Config {
534    /// Compile time defaults from the shared header.
535    fn default() -> Self {
536        Self {
537            quantum_ns: DEF_QUANTUM_NS,
538        }
539    }
540}
541
542impl Config {
543    /// Validate the constants against the bounds the BPF side relies on.
544    /// An invalid value is a programming fault, not a runtime state.
545    /// The slice stays fixed at 1ms with base weight 128 in range
546    /// 1 to 16384. The period stays at 80ms with predictor 1ns to 1s.
547    /// Fresh waits clamp remaining time to the 10us floor plus the 1ms
548    /// ceiling with misses holding else flooring only. Tiers pace at
549    /// 50us plus 100us plus 500us plus 1ms plus 5ms with a 20ms
550    /// ceiling plus a 10us floor, so throughput holds room with no starvation.
551    /// Dispatch moves at most one hint threaded move per tier capped
552    /// by remaining slots with visits capped at 8 per pass shared across
553    /// five tiers with steal window 4 to 8 plus BSF
554    /// four disjoint past SSF eight for twelve unique peers on hosts
555    /// with at least twelve CPUs with node-local phases plus drain plus
556    /// minimum plus id tiebreak, and RED admits with residual plus
557    /// exceed plus fixed 64us tolerance plus fixed 256us Emax used only
558    /// for the guarantee with base capacity 1024. Queues hold 1024 local plus
559    /// 16 node plus machine plus reject with ids in the 0x5100 region.
560    /// Hints hold 8192 flat rows with period plus weight and no timer
561    /// wait. Preempt needs 100us margin plus 100us tail strictly with a
562    /// floor at 100us and one kick per wait gated on eligibility. Fairness
563    /// bounds lag at 2ms with vruntime plus virtual deadline pacing queue
564    /// order. Stats hold 17 counters at 136B with preempt kicks plus
565    /// skipped plus RED rejects plus reclaims. Adaptive shrinks by exceed
566    /// right 3 capped 1ms plus half then halved on late else grows by slack
567    /// right 3 capped 1ms plus half then halved on early with deadband hold
568    /// plus clamp to 10us plus 1ms without changing virtual time. Latency
569    /// caps the slice at 100us plus the key at now plus max 4ms else slice
570    /// plus 100us with RED on the original deadline. Latency vetoes sleeps
571    /// below 20us toward batch with unknown open, so the sleep floor stays
572    /// decoupled from the slice cap. The predictor rises fast with shift 3
573    /// plus falls slow with shift 4, widens fast with shift 2 plus narrows
574    /// slow with shift 3, and adds half deviation margin, so the deadline
575    /// holds average plus one and a half deviation. Fixed RED keeps 64us
576    /// tolerance plus 256us Emax with task at 72B.
577    pub fn validate(&self) -> Result<()> {
578        if self.quantum_ns != QUANTUM_NS {
579            bail!("quantum bad {}", self.quantum_ns);
580        }
581        if SLICE_MIN_NS != 10_000 {
582            bail!("slice floor bad");
583        }
584        if SLICE_MIN_NS >= QUANTUM_NS {
585            bail!("slice floor over ceiling bad");
586        }
587        if WEIGHT_MIN != 1 || WEIGHT_BASE != 128 || WEIGHT_MAX != 16_384 {
588            bail!("weight bounds bad");
589        }
590        if PERIOD_NS != 80_000_000 {
591            bail!("period bounds bad");
592        }
593        if PRED_MIN_NS != 1 || PRED_MAX_NS != 1_000_000_000 {
594            bail!("predictor bounds bad");
595        }
596        if CAP_BASE != 1024 {
597            bail!("capacity base bad");
598        }
599        if HINT_MAX != 8192 {
600            bail!("hint bound bad");
601        }
602        if crate::bpf_intf::flow_consts_FLOW_QUANTUM_NS as u64 != QUANTUM_NS {
603            bail!("quantum header bad");
604        }
605        if crate::bpf_intf::flow_consts_FLOW_SLICE_MIN_NS as u64 != SLICE_MIN_NS {
606            bail!("slice floor header bad");
607        }
608        if crate::bpf_intf::flow_consts_FLOW_PERIOD_NS as u64 != PERIOD_NS {
609            bail!("period header bad");
610        }
611        if crate::bpf_intf::flow_consts_FLOW_PRED_MIN_NS as u64 != PRED_MIN_NS {
612            bail!("pred floor bad");
613        }
614        if crate::bpf_intf::flow_consts_FLOW_PRED_MAX_NS as u64 != PRED_MAX_NS {
615            bail!("pred ceiling bad");
616        }
617        if crate::bpf_intf::flow_consts_FLOW_WEIGHT_MIN as u64 != WEIGHT_MIN as u64 {
618            bail!("weight floor bad");
619        }
620        if crate::bpf_intf::flow_consts_FLOW_WEIGHT_BASE as u64 != WEIGHT_BASE as u64 {
621            bail!("weight base bad");
622        }
623        if crate::bpf_intf::flow_consts_FLOW_WEIGHT_MAX as u64 != WEIGHT_MAX as u64 {
624            bail!("weight top bad");
625        }
626        if crate::bpf_intf::flow_consts_FLOW_CAP_BASE as u64 != CAP_BASE as u64 {
627            bail!("cap base bad");
628        }
629        if crate::bpf_intf::flow_consts_FLOW_HINT_MAX as u64 != HINT_MAX {
630            bail!("hint header bad");
631        }
632        if crate::bpf_intf::flow_consts_FLOW_PREEMPT_MARGIN_NS as u64 != PREEMPT_MARGIN_NS {
633            bail!("margin bad");
634        }
635        if PREEMPT_MARGIN_NS != 100_000 {
636            bail!("margin bounds bad");
637        }
638        if crate::bpf_intf::flow_consts_FLOW_PREEMPT_TAIL_NS as u64 != PREEMPT_TAIL_NS {
639            bail!("tail bad");
640        }
641        if PREEMPT_TAIL_NS != 100_000 {
642            bail!("tail bounds bad");
643        }
644        if crate::bpf_intf::flow_consts_FLOW_VLAG_MAX_NS as u64 != 2_000_000 {
645            bail!("lag bound bad");
646        }
647        if crate::bpf_intf::flow_consts_FLOW_MAX_DSQS as u64 != 1042 {
648            bail!("dsq count bad");
649        }
650        if crate::bpf_intf::flow_consts_FLOW_DISPATCH_MAX_VISIT as u64 != 8 {
651            bail!("visit bound bad");
652        }
653        if crate::bpf_intf::flow_consts_FLOW_STEAL_MIN_PEERS as u64 != 4
654            || crate::bpf_intf::flow_consts_FLOW_STEAL_MAX_PEERS as u64 != 8
655        {
656            bail!("steal window bad");
657        }
658        if crate::bpf_intf::flow_consts_FLOW_BSF_MAX_PEERS as u64 != 4 {
659            bail!("bsf bound bad");
660        }
661        if crate::bpf_intf::flow_consts_FLOW_BSF_MAX_PEERS as u64
662            > crate::bpf_intf::flow_consts_FLOW_DISPATCH_MAX_VISIT as u64
663        {
664            bail!("bsf over visit bad");
665        }
666        if crate::bpf_intf::flow_consts_FLOW_OVERFLOW as u64 != 0x5A01 {
667            bail!("overflow id bad");
668        }
669        if crate::bpf_intf::flow_consts_FLOW_RED_EMAX_NS as u64 != RED_EMAX_NS {
670            bail!("red emax bad");
671        }
672        if crate::bpf_intf::flow_consts_FLOW_RED_TOL_NS as u64 != RED_TOL_NS {
673            bail!("red tol bad");
674        }
675        if crate::bpf_intf::flow_consts_FLOW_ADAPT_GROW_NS as u64 != ADAPT_GROW_NS {
676            bail!("adapt grow bad");
677        }
678        if crate::bpf_intf::flow_consts_FLOW_ADAPT_SHRINK_NS as u64 != ADAPT_SHRINK_NS {
679            bail!("adapt shrink bad");
680        }
681        if crate::bpf_intf::flow_consts_FLOW_ADAPT_PROP_MAX_NS as u64 != ADAPT_PROP_MAX_NS {
682            bail!("adapt prop max bad");
683        }
684        if crate::bpf_intf::flow_consts_FLOW_ADAPT_PROP_SHIFT as u64 != ADAPT_PROP_SHIFT as u64 {
685            bail!("adapt prop shift bad");
686        }
687        if ADAPT_PROP_MAX_NS != 1_000_000 {
688            bail!("prop max bounds bad");
689        }
690        if ADAPT_PROP_SHIFT != 3 {
691            bail!("prop shift bounds bad");
692        }
693        if ADAPT_PROP_MAX_NS > QUANTUM_NS {
694            bail!("prop max over ceiling bad");
695        }
696        if SLICE_LAT_NS != 100_000 {
697            bail!("slice lat bad");
698        }
699        if D_LAT_NS != 4_000_000 {
700            bail!("d lat bad");
701        }
702        if SLICE_LAT_NS <= SLICE_MIN_NS {
703            bail!("slice lat floor bad");
704        }
705        if SLICE_LAT_NS >= QUANTUM_NS {
706            bail!("slice lat over ceiling bad");
707        }
708        if D_LAT_NS >= PERIOD_NS {
709            bail!("d lat over period bad");
710        }
711        if SLICE_ULL_NS != 50_000 {
712            bail!("slice ull bad");
713        }
714        if SLICE_LL_NS != 100_000 {
715            bail!("slice ll bad");
716        }
717        if SLICE_MLL_NS != 500_000 {
718            bail!("slice mll bad");
719        }
720        if SLICE_NL_NS != 1_000_000 {
721            bail!("slice nl bad");
722        }
723        if SLICE_ENL_NS != 5_000_000 {
724            bail!("slice enl bad");
725        }
726        if SLICE_MAX_NS != 20_000_000 {
727            bail!("slice max bad");
728        }
729        if !(SLICE_MIN_NS < SLICE_ULL_NS
730            && SLICE_ULL_NS < SLICE_LL_NS
731            && SLICE_LL_NS < SLICE_MLL_NS
732            && SLICE_MLL_NS < SLICE_NL_NS
733            && SLICE_NL_NS < SLICE_ENL_NS
734            && SLICE_ENL_NS < SLICE_MAX_NS)
735        {
736            bail!("tier order bad");
737        }
738        if SLICE_LL_NS != SLICE_LAT_NS {
739            bail!("tier ll lat bad");
740        }
741        if SLICE_NL_NS != QUANTUM_NS {
742            bail!("tier nl quantum bad");
743        }
744        if PERIOD_NS != 4 * SLICE_MAX_NS {
745            bail!("period slice max bad");
746        }
747        if crate::bpf_intf::flow_consts_FLOW_SLICE_ULL_NS as u64 != SLICE_ULL_NS {
748            bail!("slice ull header bad");
749        }
750        if crate::bpf_intf::flow_consts_FLOW_SLICE_LL_NS as u64 != SLICE_LL_NS {
751            bail!("slice ll header bad");
752        }
753        if crate::bpf_intf::flow_consts_FLOW_SLICE_MLL_NS as u64 != SLICE_MLL_NS {
754            bail!("slice mll header bad");
755        }
756        if crate::bpf_intf::flow_consts_FLOW_SLICE_NL_NS as u64 != SLICE_NL_NS {
757            bail!("slice nl header bad");
758        }
759        if crate::bpf_intf::flow_consts_FLOW_SLICE_ENL_NS as u64 != SLICE_ENL_NS {
760            bail!("slice enl header bad");
761        }
762        if crate::bpf_intf::flow_consts_FLOW_SLICE_MAX_NS as u64 != SLICE_MAX_NS {
763            bail!("slice max header bad");
764        }
765        if PRED_UP_SHIFT != 3 {
766            bail!("pred up bad");
767        }
768        if PRED_DOWN_SHIFT != 4 {
769            bail!("pred down bad");
770        }
771        if PRED_DEV_UP_SHIFT != 2 {
772            bail!("pred dev up bad");
773        }
774        if PRED_DEV_DOWN_SHIFT != 3 {
775            bail!("pred dev down bad");
776        }
777        if PRED_DEV_MARGIN_SHIFT != 1 {
778            bail!("pred margin bad");
779        }
780        if ADAPT_DEADBAND_SHIFT != 5 {
781            bail!("deadband shift bad");
782        }
783        if ADAPT_DEADBAND_MIN_NS != 8_000 {
784            bail!("deadband floor bad");
785        }
786        if ADAPT_DAMP_SHIFT != 1 {
787            bail!("damp bad");
788        }
789        if crate::bpf_intf::flow_consts_FLOW_PRED_UP_SHIFT as u64 != PRED_UP_SHIFT as u64 {
790            bail!("pred up header bad");
791        }
792        if crate::bpf_intf::flow_consts_FLOW_PRED_DOWN_SHIFT as u64 != PRED_DOWN_SHIFT as u64 {
793            bail!("pred down header bad");
794        }
795        if crate::bpf_intf::flow_consts_FLOW_PRED_DEV_UP_SHIFT as u64 != PRED_DEV_UP_SHIFT as u64 {
796            bail!("pred dev up header bad");
797        }
798        if crate::bpf_intf::flow_consts_FLOW_PRED_DEV_DOWN_SHIFT as u64
799            != PRED_DEV_DOWN_SHIFT as u64
800        {
801            bail!("pred dev down header bad");
802        }
803        if crate::bpf_intf::flow_consts_FLOW_PRED_DEV_MARGIN_SHIFT as u64
804            != PRED_DEV_MARGIN_SHIFT as u64
805        {
806            bail!("pred margin header bad");
807        }
808        if crate::bpf_intf::flow_consts_FLOW_ADAPT_DEADBAND_SHIFT as u64
809            != ADAPT_DEADBAND_SHIFT as u64
810        {
811            bail!("deadband shift header bad");
812        }
813        if crate::bpf_intf::flow_consts_FLOW_ADAPT_DEADBAND_MIN_NS as u64 != ADAPT_DEADBAND_MIN_NS {
814            bail!("deadband floor header bad");
815        }
816        if crate::bpf_intf::flow_consts_FLOW_ADAPT_DAMP_SHIFT as u64 != ADAPT_DAMP_SHIFT as u64 {
817            bail!("damp header bad");
818        }
819        if crate::bpf_intf::flow_consts_FLOW_SLICE_LAT_NS as u64 != SLICE_LAT_NS {
820            bail!("slice lat header bad");
821        }
822        if LAT_SLEEP_MIN_NS != 20_000 {
823            bail!("lat sleep bad");
824        }
825        if LAT_SLEEP_MIN_NS < PRED_MIN_NS {
826            bail!("lat sleep floor bad");
827        }
828        if LAT_SLEEP_MIN_NS >= SLICE_LAT_NS {
829            bail!("lat sleep over slice bad");
830        }
831        if crate::bpf_intf::flow_consts_FLOW_LAT_SLEEP_MIN_NS as u64 != LAT_SLEEP_MIN_NS {
832            bail!("lat sleep header bad");
833        }
834        if crate::bpf_intf::flow_consts_FLOW_D_LAT_NS as u64 != D_LAT_NS {
835            bail!("d lat header bad");
836        }
837        if RED_TOL_NS != 64_000 {
838            bail!("red tol bad");
839        }
840        if RED_EMAX_NS != 256_000 {
841            bail!("red emax bad");
842        }
843        if std::mem::size_of::<crate::bpf_intf::flow_sched_stats>() != 136 {
844            bail!("stats size bad");
845        }
846        if std::mem::size_of::<crate::bpf_intf::flow_task_ctx>() != 72 {
847            bail!("task size bad");
848        }
849        Ok(())
850    }
851
852    /// One line summary of the constants for the start log.
853    /// Values print in microseconds for brevity.
854    pub fn describe(&self) -> String {
855        format!("quantum={}us", self.quantum_ns / 1000,)
856    }
857}
858
859/// Builder for Config used only by tests.
860/// Production uses Config default directly.
861#[cfg(test)]
862#[derive(Debug, Clone, Default)]
863pub struct ConfigBuilder {
864    quantum_ns: Option<u64>,
865}
866
867#[cfg(test)]
868impl ConfigBuilder {
869    /// Set the fixed slice.
870    pub fn quantum_ns(mut self, v: u64) -> Self {
871        self.quantum_ns = Some(v);
872        self
873    }
874    /// Assemble and validate the result.
875    pub fn build(self) -> Result<Config> {
876        let d = Config::default();
877        let cfg = Config {
878            quantum_ns: self.quantum_ns.unwrap_or(d.quantum_ns),
879        };
880        cfg.validate()?;
881        Ok(cfg)
882    }
883}
884
885#[cfg(test)]
886mod tests {
887    use super::*;
888
889    #[test]
890    fn defaults_are_valid() {
891        Config::default().validate().unwrap();
892    }
893
894    #[test]
895    fn builder_defaults_match_config() {
896        let cfg = ConfigBuilder::default().build().unwrap();
897        assert_eq!(cfg, Config::default());
898    }
899
900    #[test]
901    fn rejects_bad_quantum() {
902        let a = ConfigBuilder::default().quantum_ns(1).build();
903        assert!(a.is_err());
904        let b = ConfigBuilder::default().quantum_ns(2_000_000).build();
905        assert!(b.is_err());
906    }
907
908    #[test]
909    /// Summary holds the fixed slice with no knob.
910    fn describe_is_stable() {
911        let s = Config::default().describe();
912        assert!(s.contains("quantum=1000us"));
913        assert!(!s.contains("batch"));
914    }
915
916    #[test]
917    /// Dynamic slice spans the 10us floor to the 1ms ceiling.
918    fn slice_bounds_match_intf_h() {
919        assert_eq!(SLICE_MIN_NS, 10_000);
920        assert_eq!(
921            crate::bpf_intf::flow_consts_FLOW_SLICE_MIN_NS as u64,
922            SLICE_MIN_NS
923        );
924        const _: () = assert!(SLICE_MIN_NS < QUANTUM_NS);
925    }
926
927    #[test]
928    /// Defaults match the shared header with local plus shared queues.
929    fn defaults_match_intf_h() {
930        assert_eq!(
931            Config::default().quantum_ns,
932            crate::bpf_intf::flow_consts_FLOW_QUANTUM_NS as u64
933        );
934        assert_eq!(QUANTUM_NS, 1_000_000);
935        assert_eq!(PERIOD_NS, 80_000_000);
936        assert_eq!(SLICE_ULL_NS, 50_000);
937        assert_eq!(SLICE_LL_NS, 100_000);
938        assert_eq!(SLICE_MLL_NS, 500_000);
939        assert_eq!(SLICE_NL_NS, 1_000_000);
940        assert_eq!(SLICE_ENL_NS, 5_000_000);
941        assert_eq!(SLICE_MAX_NS, 20_000_000);
942        assert_eq!(PERIOD_NS, 4 * SLICE_MAX_NS);
943        assert_eq!(PRED_MIN_NS, 1);
944        assert_eq!(PRED_MAX_NS, 1_000_000_000);
945        assert_eq!(CAP_BASE, 1024);
946        assert_eq!(WEIGHT_MIN, 1);
947        assert_eq!(WEIGHT_BASE, 128);
948        assert_eq!(WEIGHT_MAX, 16_384);
949        assert_eq!(HINT_MAX, 8192);
950        assert_eq!(RED_EMAX_NS, 256_000);
951        assert_eq!(RED_TOL_NS, 64_000);
952        assert_eq!(ADAPT_GROW_NS, 64_000);
953        assert_eq!(ADAPT_SHRINK_NS, 128_000);
954        assert_eq!(crate::bpf_intf::flow_consts_FLOW_MAX_DSQS as u64, 1042);
955        assert_eq!(
956            crate::bpf_intf::flow_consts_FLOW_VLAG_MAX_NS as u64,
957            2_000_000
958        );
959        assert_eq!(crate::bpf_intf::flow_consts_FLOW_MACHINE as u64, 0x5A00);
960        assert_eq!(crate::bpf_intf::flow_consts_FLOW_OVERFLOW as u64, 0x5A01);
961        assert_eq!(
962            crate::bpf_intf::flow_consts_FLOW_DISPATCH_MAX_VISIT as u64,
963            8
964        );
965        assert_eq!(crate::bpf_intf::flow_consts_FLOW_BSF_MAX_PEERS as u64, 4);
966        assert_eq!(crate::bpf_intf::flow_consts_FLOW_LOCAL_BASE as u64, 0x5100);
967        assert_eq!(crate::bpf_intf::flow_consts_FLOW_NODE_BASE as u64, 0x5900);
968        assert_eq!(crate::bpf_intf::flow_consts_FLOW_HINT_MAX as u64, HINT_MAX);
969        assert_eq!(
970            crate::bpf_intf::flow_consts_FLOW_PRED_MIN_NS as u64,
971            PRED_MIN_NS
972        );
973        assert_eq!(
974            crate::bpf_intf::flow_consts_FLOW_PRED_MAX_NS as u64,
975            PRED_MAX_NS
976        );
977        assert_eq!(
978            crate::bpf_intf::flow_consts_FLOW_CAP_BASE as u64,
979            CAP_BASE as u64
980        );
981        assert_eq!(
982            crate::bpf_intf::flow_consts_FLOW_WEIGHT_BASE as u64,
983            WEIGHT_BASE as u64
984        );
985        assert_eq!(
986            crate::bpf_intf::flow_consts_FLOW_WEIGHT_MAX as u64,
987            WEIGHT_MAX as u64
988        );
989        assert_eq!(
990            crate::bpf_intf::flow_consts_FLOW_PREEMPT_MARGIN_NS as u64,
991            PREEMPT_MARGIN_NS
992        );
993        assert_eq!(PREEMPT_MARGIN_NS, 100_000);
994        assert_eq!(
995            crate::bpf_intf::flow_consts_FLOW_PREEMPT_TAIL_NS as u64,
996            PREEMPT_TAIL_NS
997        );
998        assert_eq!(PREEMPT_TAIL_NS, 100_000);
999        assert_eq!(
1000            crate::bpf_intf::flow_consts_FLOW_RED_EMAX_NS as u64,
1001            RED_EMAX_NS
1002        );
1003        assert_eq!(
1004            crate::bpf_intf::flow_consts_FLOW_RED_TOL_NS as u64,
1005            RED_TOL_NS
1006        );
1007        assert_eq!(
1008            crate::bpf_intf::flow_consts_FLOW_ADAPT_GROW_NS as u64,
1009            ADAPT_GROW_NS
1010        );
1011        assert_eq!(
1012            crate::bpf_intf::flow_consts_FLOW_ADAPT_SHRINK_NS as u64,
1013            ADAPT_SHRINK_NS
1014        );
1015        assert_eq!(
1016            crate::bpf_intf::flow_consts_FLOW_ADAPT_PROP_MAX_NS as u64,
1017            ADAPT_PROP_MAX_NS
1018        );
1019        assert_eq!(
1020            crate::bpf_intf::flow_consts_FLOW_ADAPT_PROP_SHIFT as u64,
1021            ADAPT_PROP_SHIFT as u64
1022        );
1023        assert_eq!(ADAPT_PROP_MAX_NS, 1_000_000);
1024        assert_eq!(ADAPT_PROP_SHIFT, 3);
1025        assert_eq!(ADAPT_DEADBAND_SHIFT, 5);
1026        assert_eq!(ADAPT_DEADBAND_MIN_NS, 8_000);
1027        assert_eq!(ADAPT_DAMP_SHIFT, 1);
1028        assert_eq!(PRED_UP_SHIFT, 3);
1029        assert_eq!(PRED_DOWN_SHIFT, 4);
1030        assert_eq!(PRED_DEV_UP_SHIFT, 2);
1031        assert_eq!(PRED_DEV_DOWN_SHIFT, 3);
1032        assert_eq!(PRED_DEV_MARGIN_SHIFT, 1);
1033        assert_eq!(SLICE_LAT_NS, 100_000);
1034        assert_eq!(LAT_SLEEP_MIN_NS, 20_000);
1035        assert_eq!(D_LAT_NS, 4_000_000);
1036        const _: () = assert!(D_LAT_NS < PERIOD_NS);
1037        assert_eq!(
1038            crate::bpf_intf::flow_consts_FLOW_SLICE_ULL_NS as u64,
1039            SLICE_ULL_NS
1040        );
1041        assert_eq!(
1042            crate::bpf_intf::flow_consts_FLOW_SLICE_LL_NS as u64,
1043            SLICE_LL_NS
1044        );
1045        assert_eq!(
1046            crate::bpf_intf::flow_consts_FLOW_SLICE_MLL_NS as u64,
1047            SLICE_MLL_NS
1048        );
1049        assert_eq!(
1050            crate::bpf_intf::flow_consts_FLOW_SLICE_NL_NS as u64,
1051            SLICE_NL_NS
1052        );
1053        assert_eq!(
1054            crate::bpf_intf::flow_consts_FLOW_SLICE_ENL_NS as u64,
1055            SLICE_ENL_NS
1056        );
1057        assert_eq!(
1058            crate::bpf_intf::flow_consts_FLOW_SLICE_MAX_NS as u64,
1059            SLICE_MAX_NS
1060        );
1061        assert_eq!(
1062            crate::bpf_intf::flow_consts_FLOW_PRED_UP_SHIFT as u64,
1063            PRED_UP_SHIFT as u64
1064        );
1065        assert_eq!(
1066            crate::bpf_intf::flow_consts_FLOW_PRED_DOWN_SHIFT as u64,
1067            PRED_DOWN_SHIFT as u64
1068        );
1069        assert_eq!(
1070            crate::bpf_intf::flow_consts_FLOW_PRED_DEV_UP_SHIFT as u64,
1071            PRED_DEV_UP_SHIFT as u64
1072        );
1073        assert_eq!(
1074            crate::bpf_intf::flow_consts_FLOW_PRED_DEV_DOWN_SHIFT as u64,
1075            PRED_DEV_DOWN_SHIFT as u64
1076        );
1077        assert_eq!(
1078            crate::bpf_intf::flow_consts_FLOW_PRED_DEV_MARGIN_SHIFT as u64,
1079            PRED_DEV_MARGIN_SHIFT as u64
1080        );
1081        assert_eq!(
1082            crate::bpf_intf::flow_consts_FLOW_ADAPT_DEADBAND_SHIFT as u64,
1083            ADAPT_DEADBAND_SHIFT as u64
1084        );
1085        assert_eq!(
1086            crate::bpf_intf::flow_consts_FLOW_ADAPT_DEADBAND_MIN_NS as u64,
1087            ADAPT_DEADBAND_MIN_NS
1088        );
1089        assert_eq!(
1090            crate::bpf_intf::flow_consts_FLOW_ADAPT_DAMP_SHIFT as u64,
1091            ADAPT_DAMP_SHIFT as u64
1092        );
1093        assert_eq!(
1094            crate::bpf_intf::flow_consts_FLOW_SLICE_LAT_NS as u64,
1095            SLICE_LAT_NS
1096        );
1097        assert_eq!(
1098            crate::bpf_intf::flow_consts_FLOW_LAT_SLEEP_MIN_NS as u64,
1099            LAT_SLEEP_MIN_NS
1100        );
1101        assert_eq!(crate::bpf_intf::flow_consts_FLOW_D_LAT_NS as u64, D_LAT_NS);
1102        assert_eq!(
1103            std::mem::size_of::<crate::bpf_intf::flow_sched_stats>(),
1104            136
1105        );
1106        assert_eq!(std::mem::size_of::<crate::bpf_intf::flow_task_ctx>(), 72);
1107    }
1108
1109    #[test]
1110    /// Proportional adapt shrinks on exceed plus grows on slack.
1111    fn prop_shrink_grow_matches_intf_h() {
1112        // Deadband holds tiny exceed plus slack with no hunt.
1113        assert_eq!(adapt_deadband(1_000_000), 31_250);
1114        assert_eq!(adapt_deadband(10_000), 8_000);
1115        assert_eq!(adapt_prop(1_000_000, 8_000, 0), 1_000_000);
1116        assert_eq!(adapt_prop(500_000, 0, 1_600), 500_000);
1117        // Exceed 64us shrinks 4us with shift 3 plus half-damp.
1118        assert_eq!(adapt_prop(1_000_000, 64_000, 0), 996_000);
1119        // Slack 64us grows 4us with shift 3 plus half-damp.
1120        assert_eq!(adapt_prop(500_000, 0, 64_000), 504_000);
1121        // Huge exceed caps at half of base then halved to 750k.
1122        assert_eq!(adapt_prop(1_000_000, 10_000_000, 0), 750_000);
1123        // Mid exceed 2ms shrinks 125us with shift 3 plus half-damp.
1124        assert_eq!(adapt_prop(1_000_000, 2_000_000, 0), 875_000);
1125        // Huge slack caps at 1ms grow then halved clamped to 1ms.
1126        assert_eq!(adapt_prop(900_000, 0, 10_000_000), 1_000_000);
1127        // Small base shrinks to the 10us floor.
1128        assert_eq!(adapt_prop(10_000, 80_000, 0), 10_000);
1129        // On-time hold keeps the clamped slice.
1130        assert_eq!(adapt_prop(500_000, 0, 0), 500_000);
1131        // Zero slice inherits the quantum then holds in deadband.
1132        assert_eq!(adapt_prop(0, 8_000, 0), 1_000_000);
1133        // Tiny exceed below 8ns holds without a divide.
1134        assert_eq!(adapt_prop(500_000, 7, 0), 500_000);
1135        // Early slack inside deadband holds with no hunt.
1136        assert_eq!(adapt_prop(500_000, 0, 200), 500_000);
1137        // Exceed wins when both hold.
1138        assert_eq!(
1139            adapt_prop(500_000, 64_000, 1_600),
1140            adapt_prop(500_000, 64_000, 0)
1141        );
1142    }
1143
1144    #[test]
1145    /// Predictor rises fast plus falls slow with margin.
1146    fn pred_up_down_margin_matches_intf_h() {
1147        // First sample sets the average at once.
1148        assert_eq!(pred_avg(0, 100_000), 100_000);
1149        // Rise 80k over 100k moves 10k with shift 3.
1150        assert_eq!(pred_avg(100_000, 180_000), 110_000);
1151        // Fall 80k under 100k moves 5k with shift 4.
1152        assert_eq!(pred_avg(100_000, 20_000), 95_000);
1153        // Deviation widens fast with shift 2.
1154        assert_eq!(pred_dev(10_000, 110_000, 180_000), 25_000);
1155        // Period adds half deviation margin.
1156        assert_eq!(pred_period(100_000, 10_000), 115_000);
1157        assert_eq!(pred_period(0, 0), PERIOD_NS);
1158        // Headers agree on every shift.
1159        assert_eq!(
1160            crate::bpf_intf::flow_consts_FLOW_PRED_UP_SHIFT as u64,
1161            PRED_UP_SHIFT as u64
1162        );
1163        assert_eq!(
1164            crate::bpf_intf::flow_consts_FLOW_PRED_DOWN_SHIFT as u64,
1165            PRED_DOWN_SHIFT as u64
1166        );
1167        assert_eq!(
1168            crate::bpf_intf::flow_consts_FLOW_ADAPT_DEADBAND_MIN_NS as u64,
1169            ADAPT_DEADBAND_MIN_NS
1170        );
1171    }
1172
1173    #[test]
1174    /// RED costs the pre-clamp slice without a post-clamp underestimate.
1175    fn red_uses_pre_clamp_not_post() {
1176        // Fresh 1ms pre-clamp caps to 100us for latency slice.
1177        let pre = 1_000_000;
1178        let post = slice_lat_clamp(pre, true);
1179        assert_eq!(post, 100_000);
1180        // RED still costs the 1ms pre-clamp without an underestimate.
1181        assert_eq!(red_cost(0, pre), 1_000_000);
1182        assert_eq!(red_cost(0, post), 100_000);
1183        assert!(red_cost(0, pre) > red_cost(0, post));
1184        // Burst wins over slice with the same 10us plus 20ms clamp.
1185        assert_eq!(red_cost(900_000, 1_000_000), 900_000);
1186        assert_eq!(red_cost(0, 0), QUANTUM_NS);
1187        assert_eq!(red_cost(1, 1_000_000), SLICE_MIN_NS);
1188        // Throughput bursts past 1ms keep room to 20ms with no cap.
1189        assert_eq!(red_cost(5_000_000, 0), 5_000_000);
1190        assert_eq!(red_cost(0, 5_000_000), 5_000_000);
1191        assert_eq!(red_cost(30_000_000, 0), SLICE_MAX_NS);
1192        assert_eq!(red_cost(0, 30_000_000), SLICE_MAX_NS);
1193    }
1194
1195    #[test]
1196    /// Latency slice clamps at 100us plus deadline at max 4ms.
1197    fn lat_bounds_match_intf_h() {
1198        const _: () = assert!(SLICE_LAT_NS > SLICE_MIN_NS);
1199        const _: () = assert!(SLICE_LAT_NS < QUANTUM_NS);
1200        const _: () = assert!(LAT_SLEEP_MIN_NS < SLICE_LAT_NS);
1201        const _: () = assert!(LAT_SLEEP_MIN_NS >= PRED_MIN_NS);
1202        assert_eq!(SLICE_LAT_NS, 100_000);
1203        assert_eq!(LAT_SLEEP_MIN_NS, 20_000);
1204        assert_eq!(D_LAT_NS, 4_000_000);
1205        const _: () = assert!(D_LAT_NS < PERIOD_NS);
1206        assert_eq!(PERIOD_NS, 80_000_000);
1207        // Non latency keeps the slice.
1208        assert_eq!(slice_lat_clamp(1_000_000, false), 1_000_000);
1209        // Latency caps at 100us.
1210        assert_eq!(slice_lat_clamp(1_000_000, true), 100_000);
1211        assert_eq!(slice_lat_clamp(100_000, true), 100_000);
1212        // Deadline floors at 4ms plus adds slice plus 100us.
1213        assert_eq!(lat_deadline(0, 100_000), 4_000_000);
1214        assert_eq!(lat_deadline(1_000, 1_000_000), 1_000 + 4_000_000);
1215        assert_eq!(lat_deadline(0, 5_000_000), 5_100_000);
1216        // Saturates on wrap with ns accuracy.
1217        assert_eq!(lat_deadline(u64::MAX - 1_000, 100_000), u64::MAX);
1218    }
1219
1220    #[test]
1221    /// Extended latency vetoes batch hints with probation.
1222    fn lat_ext_veto_matches_intf_h() {
1223        // Newcomers with no history stay batch with probation.
1224        assert!(!lat_crit_ext(0, 0, 0, 0, 0, 0));
1225        assert!(!lat_crit(0, 0));
1226        assert!(!is_lat(0, 0, 0, 0, 0, 0));
1227        // Short burst with neutral signals keeps latency.
1228        assert!(lat_crit_ext(100_000, 10_000, 0, 1_000_000, 0, 128));
1229        assert!(lat_crit(100_000, 10_000));
1230        assert!(is_lat(100_000, 10_000, 0, 1_000_000, 0, 128));
1231        // Unified latency matches base plus ext, so RED agrees.
1232        assert!(!is_lat(900_000, 200_000, 0, 1_000_000, 0, 128));
1233        assert!(!is_lat(100_000, 10_000, 2000, 1_000_000, 0, 128));
1234        assert!(!is_lat(100_000, 10_000, 0, 10_000, 0, 128));
1235        assert!(is_lat(100_000, 10_000, 0, 30_000, 0, 128));
1236        assert!(is_lat(100_000, 10_000, 0, 50_000, 0, 128));
1237        assert!(is_lat(100_000, 10_000, 0, 150_000, 0, 128));
1238        assert!(is_lat(100_000, 10_000, 0, 0, 0, 128));
1239        // Long burst past quantum vetoes.
1240        assert!(!lat_crit_ext(900_000, 200_000, 0, 1_000_000, 0, 128));
1241        // Large hint past quantum vetoes.
1242        assert!(!lat_crit_ext(100_000, 10_000, 2000, 1_000_000, 0, 128));
1243        // Small hint keeps latency.
1244        assert!(lat_crit_ext(100_000, 10_000, 500, 1_000_000, 0, 128));
1245        // Short sleep below 20us vetoes, 30us plus 50us plus 150us keep,
1246        // unknown sleep fails open with no stall.
1247        assert!(!lat_crit_ext(100_000, 10_000, 0, 10_000, 0, 128));
1248        assert!(lat_crit_ext(100_000, 10_000, 0, 30_000, 0, 128));
1249        assert!(lat_crit_ext(100_000, 10_000, 0, 50_000, 0, 128));
1250        assert!(lat_crit_ext(100_000, 10_000, 0, 150_000, 0, 128));
1251        assert!(lat_crit_ext(100_000, 10_000, 0, 0, 0, 128));
1252        // Explicit low clamp vetoes, unknown passes.
1253        assert!(!lat_crit_ext(100_000, 10_000, 0, 1_000_000, 100, 128));
1254        assert!(lat_crit_ext(100_000, 10_000, 0, 1_000_000, 1024, 128));
1255        // Light share vetoes, heavy keeps.
1256        assert!(!lat_crit_ext(100_000, 10_000, 0, 1_000_000, 0, 32));
1257        assert!(lat_crit_ext(100_000, 10_000, 0, 1_000_000, 0, 1024));
1258    }
1259
1260    #[test]
1261    /// Sleep isolates now minus wait minus burst with probation.
1262    fn sleep_isolates_gap_minus_burst() {
1263        // Gap 1ms minus 200us burst leaves 800us sleep.
1264        assert_eq!(sleep_ns(1_000, 1_001_000, 200_000), 800_000);
1265        // Burst-covered gap reads zero with no underflow.
1266        assert_eq!(sleep_ns(1_000, 1_001_000, 2_000_000), 0);
1267        // No history keeps the full gap.
1268        assert_eq!(sleep_ns(1_000, 1_001_000, 0), 1_000_000);
1269        // Unknown wait plus wait at now both read zero.
1270        assert_eq!(sleep_ns(0, 1_000, 100_000), 0);
1271        assert_eq!(sleep_ns(1_000, 1_000, 100_000), 0);
1272        // Wrap keeps order: wait near max to small now reads 1501ns.
1273        assert_eq!(sleep_ns(u64::MAX - 1_000, 500, 0), 1_501);
1274        // Wrapped gap minus burst keeps the remainder with no underflow.
1275        assert_eq!(sleep_ns(u64::MAX - 1_000, 500, 1_000), 501);
1276        assert_eq!(sleep_ns(u64::MAX - 1_000, 500, 5_000), 0);
1277    }
1278
1279    #[test]
1280    /// Resume keeps start plus cur leftover with clamp.
1281    fn resume_uses_cur_not_quantum() {
1282        // Start plus 500us cur with 400us left keeps 400us batch.
1283        assert_eq!(
1284            slice_resume(1_000, 1_000 + 100_000, false, 500_000),
1285            400_000
1286        );
1287        // Latency caps the same leftover at 100us.
1288        assert_eq!(slice_resume(1_000, 1_000 + 100_000, true, 500_000), 100_000);
1289        // Start plus 250us cur keeps 150us rem capped to 100us for latency.
1290        assert_eq!(slice_resume(1_000, 1_000 + 100_000, true, 250_000), 100_000);
1291        // Bad start inherits the stored slice.
1292        assert_eq!(slice_resume(0, 1_000, false, 500_000), 500_000);
1293        assert_eq!(slice_resume(0, 1_000, true, 500_000), 500_000);
1294        // Past end floors to the minimum with no zero slice.
1295        assert_eq!(slice_resume(1_000, 1_000 + 600_000, false, 500_000), 10_000);
1296        // Saturated end near wrap inherits with no wrap to front.
1297        assert_eq!(slice_resume(u64::MAX - 10, 100, false, 500_000), 500_000);
1298    }
1299
1300    #[test]
1301    /// Preempt needs lat arrival against batch owner only.
1302    fn preempt_needs_lat_to_batch_only() {
1303        // Arrival 1ms leads occupant 2ms with margin plus tail held.
1304        assert!(preempt_ok(
1305            1_000_000, 2_000_000, 500_000, 500_000, 1_000_000, true, false
1306        ));
1307        // Lat to lat plus batch to batch plus batch to lat fail closed.
1308        assert!(!preempt_ok(
1309            1_000_000, 2_000_000, 500_000, 500_000, 1_000_000, true, true
1310        ));
1311        assert!(!preempt_ok(
1312            1_000_000, 2_000_000, 500_000, 500_000, 1_000_000, false, false
1313        ));
1314        assert!(!preempt_ok(
1315            1_000_000, 2_000_000, 500_000, 500_000, 1_000_000, false, true
1316        ));
1317        // Near tie within 100us margin never bounces.
1318        assert!(!preempt_ok(
1319            1_950_000, 2_000_000, 500_000, 500_000, 1_000_000, true, false
1320        ));
1321        // Nearly done owner with tail left fails closed.
1322        assert!(!preempt_ok(
1323            1_000_000, 2_000_000, 1_450_000, 500_000, 1_000_000, true, false
1324        ));
1325        // Zero cur inherits quantum, so the same lead still wins.
1326        assert!(preempt_ok(
1327            1_000_000, 2_000_000, 500_000, 500_000, 0, true, false
1328        ));
1329        // Short 50us cur leaves no tail, so the kick fails closed.
1330        assert!(!preempt_ok(
1331            1_000_000, 2_000_000, 500_000, 500_000, 50_000, true, false
1332        ));
1333    }
1334
1335    #[test]
1336    /// Miss plus skip plus period stay fixed without drift.
1337    fn miss_skip_period_are_intentional() {
1338        // Misses stay lifetime: newcomers with no history stay batch.
1339        assert!(!is_lat(0, 0, 0, 0, 0, 0));
1340        // Skips count every hold: lat to lat fails closed.
1341        assert!(!preempt_ok(
1342            1_000_000, 2_000_000, 500_000, 500_000, 1_000_000, true, true
1343        ));
1344        // Period caps in-tier keys at now plus one period.
1345        assert_eq!(PERIOD_NS, 80_000_000);
1346        assert_eq!(PERIOD_NS, 4 * SLICE_MAX_NS);
1347        const _: () = assert!(D_LAT_NS < PERIOD_NS);
1348    }
1349
1350    #[test]
1351    /// Tier ceilings map predicted bursts without a divide.
1352    fn classify_tier_matches_intf_h() {
1353        const _: () = assert!(SLICE_ULL_NS < SLICE_MLL_NS);
1354        const _: () = assert!(SLICE_MLL_NS < SLICE_NL_NS);
1355        const _: () = assert!(SLICE_NL_NS < SLICE_ENL_NS);
1356        const _: () = assert!(SLICE_ENL_NS < SLICE_MAX_NS);
1357        assert_eq!(SLICE_ULL_NS, 50_000);
1358        assert_eq!(SLICE_LL_NS, 100_000);
1359        assert_eq!(SLICE_MLL_NS, 500_000);
1360        assert_eq!(SLICE_NL_NS, 1_000_000);
1361        assert_eq!(SLICE_ENL_NS, 5_000_000);
1362        assert_eq!(SLICE_MAX_NS, 20_000_000);
1363        // Critical forces ULL even past the ceiling.
1364        assert_eq!(classify_tier(20_000_000, 0, true), 50_000);
1365        // Prediction at or below 50us earns ULL with half dev margin.
1366        assert_eq!(classify_tier(30_000, 10_000, false), 50_000);
1367        assert_eq!(classify_tier(50_000, 0, false), 50_000);
1368        // 100us plus 500us plus 1ms plus 5ms bound LL plus MLL plus NL plus ENL.
1369        assert_eq!(classify_tier(60_000, 10_000, false), 100_000);
1370        assert_eq!(classify_tier(300_000, 100_000, false), 500_000);
1371        assert_eq!(classify_tier(600_000, 100_000, false), 1_000_000);
1372        assert_eq!(classify_tier(3_000_000, 1_000_000, false), 5_000_000);
1373        // Past 5ms holds the 20ms ceiling with saturation.
1374        assert_eq!(classify_tier(6_000_000, 0, false), 20_000_000);
1375        assert_eq!(classify_tier(u64::MAX - 10, 20, false), 20_000_000);
1376        // Headers agree on every bound.
1377        assert_eq!(
1378            crate::bpf_intf::flow_consts_FLOW_SLICE_ULL_NS as u64,
1379            SLICE_ULL_NS
1380        );
1381        assert_eq!(
1382            crate::bpf_intf::flow_consts_FLOW_SLICE_MAX_NS as u64,
1383            SLICE_MAX_NS
1384        );
1385        assert_eq!(
1386            crate::bpf_intf::flow_consts_FLOW_PERIOD_NS as u64,
1387            PERIOD_NS
1388        );
1389        // Fresh waits cap the tier at 1ms, so bulk still rotates.
1390        let bulk = classify_tier(6_000_000, 0, false);
1391        assert_eq!(bulk, SLICE_MAX_NS as u32);
1392        assert_eq!(bulk.min(QUANTUM_NS as u32), QUANTUM_NS as u32);
1393        let audio = classify_tier(30_000, 10_000, false);
1394        assert_eq!(audio, SLICE_ULL_NS as u32);
1395        assert!(audio < QUANTUM_NS as u32);
1396    }
1397
1398    #[test]
1399    /// Task stays at 72B with stats at 136B plus fixed RED.
1400    fn abi_sizes_stay_pinned() {
1401        assert_eq!(std::mem::size_of::<crate::bpf_intf::flow_task_ctx>(), 72);
1402        assert_eq!(
1403            std::mem::size_of::<crate::bpf_intf::flow_sched_stats>(),
1404            136
1405        );
1406        assert_eq!(RED_EMAX_NS, 256_000);
1407        assert_eq!(
1408            crate::bpf_intf::flow_consts_FLOW_RED_EMAX_NS as u64,
1409            RED_EMAX_NS
1410        );
1411        assert_eq!(RED_TOL_NS, 64_000);
1412        assert_eq!(
1413            crate::bpf_intf::flow_consts_FLOW_RED_TOL_NS as u64,
1414            RED_TOL_NS
1415        );
1416    }
1417
1418    #[test]
1419    /// Fixed RED keeps 64us tolerance plus 256us Emax.
1420    #[allow(clippy::assertions_on_constants)]
1421    fn fixed_red_matches_intf_h() {
1422        assert_eq!(RED_TOL_NS, 64_000);
1423        assert_eq!(RED_EMAX_NS, 256_000);
1424    }
1425}
scx_flow/rust/config.rs

scx_flow/rust/
config.rs
