hermit/xstate_probe/
irq_counter.rs

1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9//! A per-thread count of the interrupts the probe thread takes, from the core
10//! PMU (<https://github.com/rrnewton/hermit/issues/4243>), condition C11).
11//!
12//! The XINUSE probe's reference trial is valid only if nothing saved and
13//! reloaded the thread's FPU state inside it. Without a context switch, Linux
14//! does that only through `kernel_fpu_begin` in interrupt or softirq context, or
15//! in the task's own syscall, so a trial that took no interrupt, no context
16//! switch and no syscall cannot contain a reload. This counter supplies the
17//! interrupt half of that certificate, and every property it rests on is
18//! checked rather than assumed:
19//!
20//! - the raw event is used only on CPU models whose perf tables document it;
21//! - it is opened pinned, so it is never time-shared, and every read checks
22//!   that its time enabled equals its time running (both are cumulative, so
23//!   equality at a read proves it counted for its whole enabled history);
24//! - a failed read is an error, never a value that could compare equal;
25//! - at startup its count over a spin must bind to the kernel's own count of
26//!   this CPU's interrupts in `/proc/interrupts`.
27
28use std::fs::File;
29use std::io::Read;
30use std::os::fd::AsRawFd;
31use std::os::fd::FromRawFd;
32use std::os::fd::OwnedFd;
33
34/// The CPU's vendor and display family/model, from CPUID leaves 0 and 1.
35#[derive(Clone, Copy, Debug, PartialEq, Eq)]
36pub struct CpuModel {
37    /// CPUID leaf 0 EBX: `Auth` (AMD) or `Genu` (Intel).
38    pub vendor_ebx: u32,
39    pub family: u32,
40    pub model: u32,
41}
42
43const AUTHENTIC_AMD_EBX: u32 = 0x6874_7541;
44const GENUINE_INTEL_EBX: u32 = 0x756e_6547;
45
46impl CpuModel {
47    /// The display family and model from leaf 1 EAX, as the SDM and the AMD APM
48    /// define them: the extended family is added only for family 0xF, and the
49    /// extended model is used only for families 0x6 and 0xF and above.
50    pub fn from_leaves(vendor_ebx: u32, leaf1_eax: u32) -> Self {
51        let mut family = (leaf1_eax >> 8) & 0xf;
52        let mut model = (leaf1_eax >> 4) & 0xf;
53        if family == 0xf {
54            family += (leaf1_eax >> 20) & 0xff;
55        }
56        if family == 0x6 || family >= 0xf {
57            model |= ((leaf1_eax >> 16) & 0xf) << 4;
58        }
59        CpuModel {
60            vendor_ebx,
61            family,
62            model,
63        }
64    }
65
66    /// The host's model, read with the CPUID instruction.
67    pub fn host() -> Self {
68        let leaf0 = std::arch::x86_64::__cpuid(0);
69        let leaf1 = std::arch::x86_64::__cpuid(1);
70        Self::from_leaves(leaf0.ebx, leaf1.eax)
71    }
72}
73
74/// The raw PMU configuration of the interrupts-taken event on `cpu`, or `None`
75/// where perf's event tables do not document one.
76///
77/// - AMD `LsIntTaken` (event 0x2C): Zen 2 and later, that is family 17h from
78///   model 0x30, and families 19h and 1Ah. Zen 1 does not list it.
79/// - Intel `HW_INTERRUPTS.RECEIVED` (event 0xCB, umask 0x01): Skylake client
80///   and the client parts on the same core (Kaby, Coffee and Comet Lake),
81///   Skylake-X, Cascade and Cooper Lake, Sapphire, Emerald and Granite Rapids,
82///   Goldmont, Goldmont Plus, Tremont (Elkhart and Jasper Lake) and Snow Ridge.
83///   Haswell, Broadwell, Ice Lake (including Ice Lake-X), Tiger, Alder, Raptor,
84///   Meteor, Lunar and Arrow Lake and Sierra Forest do not list it, so they are
85///   refused rather than given an event that might count something else.
86pub fn interrupts_taken_event(cpu: CpuModel) -> Option<u64> {
87    const INTEL_MODELS: &[u32] = &[
88        0x4e, 0x5e, 0x8e, 0x9e, 0xa5, 0xa6, // Skylake client and same-core successors
89        0x55, // Skylake-X, Cascade Lake, Cooper Lake
90        0x8f, // Sapphire Rapids
91        0xcf, // Emerald Rapids
92        0xad, 0xae, // Granite Rapids
93        0x5c, 0x5f, // Goldmont
94        0x7a, // Goldmont Plus
95        0x96, 0x9c, // Tremont: Elkhart Lake, Jasper Lake
96        0x86, // Snow Ridge
97    ];
98    match cpu.vendor_ebx {
99        AUTHENTIC_AMD_EBX
100            if (cpu.family == 0x17 && cpu.model >= 0x30)
101                || cpu.family == 0x19
102                || cpu.family == 0x1a =>
103        {
104            Some(0x2c)
105        }
106        GENUINE_INTEL_EBX if cpu.family == 6 && INTEL_MODELS.contains(&cpu.model) => Some(0x01cb),
107        _ => None,
108    }
109}
110
111/// Why the counter cannot certify trials on this host.
112#[derive(Debug, PartialEq, Eq)]
113pub enum CounterRefusal {
114    /// No documented interrupts-taken event for this CPU model.
115    UndocumentedModel(CpuModel),
116    /// `perf_event_open` failed; carries the error and `perf_event_paranoid`.
117    Open { error: String, paranoid: String },
118    /// A read returned fewer bytes than its three values: the pinned event
119    /// could not be scheduled and is in the error state, or the descriptor is
120    /// gone.
121    Read,
122    /// The event was not on the PMU for part of its enabled time.
123    Multiplexed { enabled: u64, running: u64 },
124    /// The startup spin did not bind the counter to the kernel's count.
125    Calibration(String),
126}
127
128impl std::fmt::Display for CounterRefusal {
129    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
130        match self {
131            CounterRefusal::UndocumentedModel(cpu) => write!(
132                f,
133                "no documented interrupts-taken PMU event for CPU family {:#x} model {:#x}",
134                cpu.family, cpu.model
135            ),
136            CounterRefusal::Open { error, paranoid } => write!(
137                f,
138                "perf_event_open of the interrupts-taken event failed: {error} \
139                 (perf_event_paranoid is {paranoid}; counting kernel-mode interrupts \
140                 needs 1 or less, or CAP_PERFMON)"
141            ),
142            CounterRefusal::Read => {
143                write!(f, "reading the interrupts-taken counter failed")
144            }
145            CounterRefusal::Multiplexed { enabled, running } => write!(
146                f,
147                "the interrupts-taken counter was not on the PMU for its whole \
148                 enabled time (enabled {enabled} ns, running {running} ns)"
149            ),
150            CounterRefusal::Calibration(why) => {
151                write!(f, "the interrupts-taken counter failed calibration: {why}")
152            }
153        }
154    }
155}
156
157/// An open, pinned interrupts-taken counter for the calling thread.
158pub struct IrqCounter {
159    fd: OwnedFd,
160}
161
162// `perf_event_attr` field offsets used below (include/uapi/linux/perf_event.h).
163const PERF_TYPE_RAW: u32 = 4;
164const PERF_FORMAT_TOTAL_TIME_ENABLED: u64 = 1 << 0;
165const PERF_FORMAT_TOTAL_TIME_RUNNING: u64 = 1 << 1;
166/// The flag word's bitfields in `perf_event_attr` order: `disabled` (bit 0),
167/// `inherit` (bit 1), `pinned` (bit 2, "must always be on PMU").
168const ATTR_FLAG_DISABLED: u64 = 1 << 0;
169const ATTR_FLAG_INHERIT: u64 = 1 << 1;
170const ATTR_FLAG_PINNED: u64 = 1 << 2;
171/// The counter's flag word: enabled, not inherited by threads this thread
172/// starts (the peer), pinned, and with no mode excluded.
173const COUNTER_FLAGS: u64 = ATTR_FLAG_PINNED & !(ATTR_FLAG_DISABLED | ATTR_FLAG_INHERIT);
174const PERF_ATTR_SIZE_VER0: u32 = 64;
175const PERF_FLAG_FD_CLOEXEC: libc::c_ulong = 1 << 3;
176
177#[repr(C)]
178#[derive(Default)]
179struct PerfEventAttrVer0 {
180    type_: u32,
181    size: u32,
182    config: u64,
183    sample_period: u64,
184    sample_type: u64,
185    read_format: u64,
186    flags: u64,
187    wakeup_events: u32,
188    bp_type: u32,
189    config1: u64,
190}
191
192impl IrqCounter {
193    /// Opens the counter for the calling thread, counting user and kernel mode,
194    /// pinned, with both times in each read.
195    pub fn open(cpu: CpuModel) -> Result<Self, CounterRefusal> {
196        let config = interrupts_taken_event(cpu).ok_or(CounterRefusal::UndocumentedModel(cpu))?;
197        let attr = PerfEventAttrVer0 {
198            type_: PERF_TYPE_RAW,
199            size: PERF_ATTR_SIZE_VER0,
200            config,
201            read_format: PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING,
202            flags: COUNTER_FLAGS,
203            ..Default::default()
204        };
205        // SAFETY: `attr` is a valid PERF_ATTR_SIZE_VER0 structure that outlives
206        // the call; pid 0 and cpu -1 select the calling thread on any CPU.
207        let fd = unsafe {
208            libc::syscall(
209                libc::SYS_perf_event_open,
210                &attr as *const PerfEventAttrVer0,
211                0,
212                -1,
213                -1,
214                PERF_FLAG_FD_CLOEXEC,
215            )
216        };
217        if fd < 0 {
218            let error = std::io::Error::last_os_error().to_string();
219            let paranoid = std::fs::read_to_string("/proc/sys/kernel/perf_event_paranoid")
220                .map(|s| s.trim().to_owned())
221                .unwrap_or_else(|e| format!("unreadable ({e})"));
222            return Err(CounterRefusal::Open { error, paranoid });
223        }
224        // SAFETY: the kernel just returned this descriptor and nothing else owns it.
225        let fd = unsafe { OwnedFd::from_raw_fd(fd as i32) };
226        Ok(IrqCounter { fd })
227    }
228
229    /// The interrupts taken so far, after checking that the read returned all
230    /// three values and that the event was on the PMU for all of its enabled
231    /// time.
232    pub fn read(&self) -> Result<u64, CounterRefusal> {
233        let mut values = [0u64; 3];
234        // SAFETY: `values` is 24 writable bytes that outlive the call.
235        let n = unsafe {
236            libc::read(
237                self.fd.as_raw_fd(),
238                values.as_mut_ptr().cast(),
239                std::mem::size_of_val(&values),
240            )
241        };
242        check_read(n, values)
243    }
244}
245
246/// The checks behind [`IrqCounter::read`], separated so tests can drive them.
247fn check_read(n: isize, [value, enabled, running]: [u64; 3]) -> Result<u64, CounterRefusal> {
248    if n != 24 {
249        return Err(CounterRefusal::Read);
250    }
251    if enabled != running {
252        return Err(CounterRefusal::Multiplexed { enabled, running });
253    }
254    Ok(value)
255}
256
257/// One CPU's local-timer count and its column total in `/proc/interrupts`.
258#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
259pub struct CpuInterrupts {
260    pub local_timer: u64,
261    pub total: u64,
262}
263
264/// Parses `/proc/interrupts` text for `cpu`'s column. Rows whose entry in that
265/// column is not a number (the trailing descriptions of `ERR` and `MIS`) are
266/// skipped; the `LOC` row must be present.
267pub fn parse_proc_interrupts(text: &str, cpu: usize) -> Option<CpuInterrupts> {
268    let mut lines = text.lines();
269    let want = format!("CPU{cpu}");
270    let column = lines.next()?.split_whitespace().position(|t| t == want)?;
271    let mut local_timer = None;
272    let mut total = 0u64;
273    for line in lines {
274        let mut fields = line.split_whitespace();
275        let Some(label) = fields.next() else { continue };
276        if let Some(Ok(count)) = fields.nth(column).map(str::parse::<u64>) {
277            total = total.checked_add(count)?;
278            if label == "LOC:" {
279                local_timer = Some(count);
280            }
281        }
282    }
283    Some(CpuInterrupts {
284        local_timer: local_timer?,
285        total,
286    })
287}
288
289/// Reads `/proc/interrupts` whole. It is about 1.4 MB on a 316-CPU machine;
290/// the caller parses it after taking its other samples, so parsing time does
291/// not widen the interval the counter must cover.
292pub fn snapshot_proc_interrupts() -> Result<String, CounterRefusal> {
293    let mut text = String::with_capacity(4 << 20);
294    File::open("/proc/interrupts")
295        .and_then(|mut f| f.read_to_string(&mut text))
296        .map_err(|e| CounterRefusal::Calibration(format!("cannot read /proc/interrupts: {e}")))?;
297    Ok(text)
298}
299
300/// `cpu`'s interrupt counts from a [`snapshot_proc_interrupts`] text.
301pub fn interrupts_of(text: &str, cpu: usize) -> Result<CpuInterrupts, CounterRefusal> {
302    parse_proc_interrupts(text, cpu).ok_or_else(|| {
303        CounterRefusal::Calibration(format!("/proc/interrupts has no LOC count for CPU{cpu}"))
304    })
305}
306
307/// The fewest local-timer interrupts a calibration must see, summed over its
308/// clean spins. [`CALIBRATION_SLACK`] is absolute, so this floor is what keeps
309/// the binding tight: at 190 interrupts the counter must see at least 180 of
310/// them, about 95%, when the reads around the spins take no time. (A single
311/// 70 ms spin sees about 70, where the slack alone would admit about 86%.) A
312/// tickless (`nohz_full`) CPU sees almost none, and there the binding says
313/// nothing.
314pub const MIN_CALIBRATION_TICKS: u64 = 190;
315/// A fixed allowance on top of the proportional one in [`calibration_verdict`].
316pub const CALIBRATION_SLACK: u64 = 10;
317
318/// Whether a calibration binds the counter to the kernel's count: over its
319/// clean spins, summed, the counter must see at least this CPU's local-timer
320/// interrupts and no more than all of its interrupts.
321///
322/// The `/proc/interrupts` reads around the spin lie outside the counter's
323/// interval, and reading the file takes milliseconds on a large machine, so
324/// timer interrupts during them are in the kernel's count only. `outside` is the
325/// share of the measured interval that the counter did not cover (from the TSC),
326/// and the allowance is that share of the kernel's count, plus
327/// [`CALIBRATION_SLACK`].
328pub fn calibration_verdict(
329    counter: u64,
330    kernel: CpuInterrupts,
331    outside: f64,
332) -> Result<(), String> {
333    let allowance =
334        (kernel.total as f64 * outside.clamp(0.0, 1.0)).ceil() as u64 + CALIBRATION_SLACK;
335    if kernel.local_timer < MIN_CALIBRATION_TICKS {
336        return Err(format!(
337            "only {} local-timer interrupts during the calibration (need \
338             {MIN_CALIBRATION_TICKS}); a tickless CPU cannot bind the counter",
339            kernel.local_timer
340        ));
341    }
342    if counter + allowance < kernel.local_timer {
343        return Err(format!(
344            "the counter saw {counter} interrupts but the CPU took {} local-timer interrupts \
345             (allowing {allowance} for the reads around the spin)",
346            kernel.local_timer
347        ));
348    }
349    if counter > kernel.total + CALIBRATION_SLACK {
350        return Err(format!(
351            "the counter saw {counter} interrupts but the CPU took only {} in all",
352            kernel.total
353        ));
354    }
355    Ok(())
356}
357
358#[cfg(test)]
359mod tests {
360    use super::*;
361
362    fn amd(family: u32, model: u32) -> CpuModel {
363        CpuModel {
364            vendor_ebx: AUTHENTIC_AMD_EBX,
365            family,
366            model,
367        }
368    }
369
370    fn intel(model: u32) -> CpuModel {
371        CpuModel {
372            vendor_ebx: GENUINE_INTEL_EBX,
373            family: 6,
374            model,
375        }
376    }
377
378    #[test]
379    fn display_family_and_model_follow_the_extended_fields() {
380        // EPYC 9D85 (Zen 5): family 0xF + 0x0B = 0x1A, model 0x11.
381        assert_eq!(
382            CpuModel::from_leaves(AUTHENTIC_AMD_EBX, 0x00b1_0f11),
383            amd(0x1a, 0x11)
384        );
385        // Sapphire Rapids: family 6, extended model 8, model 0xF.
386        assert_eq!(
387            CpuModel::from_leaves(GENUINE_INTEL_EBX, 0x0008_06f8),
388            intel(0x8f)
389        );
390    }
391
392    #[test]
393    fn the_event_is_chosen_only_where_perf_documents_it() {
394        assert_eq!(interrupts_taken_event(amd(0x1a, 0x11)), Some(0x2c));
395        assert_eq!(interrupts_taken_event(amd(0x19, 0x01)), Some(0x2c));
396        assert_eq!(interrupts_taken_event(amd(0x17, 0x31)), Some(0x2c));
397        // Zen 1 does not list LsIntTaken.
398        assert_eq!(interrupts_taken_event(amd(0x17, 0x01)), None);
399        assert_eq!(interrupts_taken_event(intel(0x55)), Some(0x01cb));
400        assert_eq!(interrupts_taken_event(intel(0x8f)), Some(0x01cb));
401        // Ice Lake-X has XCR0 0x2e7 but no documented event 0xCB.
402        assert_eq!(interrupts_taken_event(intel(0x6a)), None);
403        // Haswell and Alder Lake do not list it either.
404        assert_eq!(interrupts_taken_event(intel(0x3f)), None);
405        assert_eq!(interrupts_taken_event(intel(0x97)), None);
406        assert_eq!(
407            interrupts_taken_event(CpuModel {
408                vendor_ebx: 0,
409                family: 6,
410                model: 0x55
411            }),
412            None
413        );
414    }
415
416    #[test]
417    fn a_read_must_return_all_three_values_with_equal_times() {
418        assert_eq!(check_read(24, [7, 100, 100]), Ok(7));
419        // A failed read returns no value that could compare equal to another.
420        assert_eq!(check_read(-1, [0, 0, 0]), Err(CounterRefusal::Read));
421        assert_eq!(check_read(0, [0, 0, 0]), Err(CounterRefusal::Read));
422        assert_eq!(
423            check_read(24, [7, 100, 90]),
424            Err(CounterRefusal::Multiplexed {
425                enabled: 100,
426                running: 90
427            })
428        );
429    }
430
431    const INTERRUPTS: &str = "            CPU0       CPU1       CPU2\n\
432           0:         44          0          0  IR-IO-APIC    2-edge      timer\n\
433         LOC:  982944332  217340956  303290468   Local timer interrupts\n\
434         RES:         10         20         30   Rescheduling interrupts\n\
435         ERR:          0\n\
436         MIS:          0\n";
437
438    #[test]
439    fn proc_interrupts_is_read_by_the_cpus_column() {
440        assert_eq!(
441            parse_proc_interrupts(INTERRUPTS, 1),
442            Some(CpuInterrupts {
443                local_timer: 217_340_956,
444                total: 217_340_976
445            })
446        );
447        assert_eq!(parse_proc_interrupts(INTERRUPTS, 3), None);
448        assert_eq!(
449            parse_proc_interrupts("            CPU0\n  0:  1  timer\n", 0),
450            None,
451            "no LOC row"
452        );
453    }
454
455    #[test]
456    fn calibration_binds_the_counter_to_the_kernels_count() {
457        let kernel = CpuInterrupts {
458            local_timer: 196,
459            total: 197,
460        };
461        // Measured on CPU 3 of an EPYC 9D85: counter 194, LOC 196, total 197.
462        assert_eq!(calibration_verdict(194, kernel, 0.0), Ok(()));
463        assert!(
464            calibration_verdict(100, kernel, 0.0).is_err(),
465            "misses ticks"
466        );
467        assert!(
468            calibration_verdict(400, kernel, 0.0).is_err(),
469            "counts something else"
470        );
471        // Measured: counter 198, LOC 224, with the reads outside the counter's
472        // interval taking a tenth of the measured time.
473        let busy = CpuInterrupts {
474            local_timer: 224,
475            total: 230,
476        };
477        assert_eq!(calibration_verdict(198, busy, 0.1), Ok(()));
478        assert!(calibration_verdict(198, busy, 0.0).is_err());
479        assert!(
480            calibration_verdict(150, busy, 0.1).is_err(),
481            "still misses ticks"
482        );
483        assert!(
484            calibration_verdict(
485                3,
486                CpuInterrupts {
487                    local_timer: 3,
488                    total: 3
489                },
490                0.0
491            )
492            .is_err(),
493            "tickless CPU"
494        );
495    }
496
497    /// The counter is pinned and counts only the calling thread: bit 2 of the
498    /// flag word is `pinned` and bit 1 is `inherit` in `perf_event_attr`'s
499    /// bitfield order (`disabled`, `inherit`, `pinned`, `exclusive`, ...).
500    #[test]
501    fn the_counter_is_pinned_enabled_and_not_inherited() {
502        assert_eq!(
503            (ATTR_FLAG_DISABLED, ATTR_FLAG_INHERIT, ATTR_FLAG_PINNED),
504            (1, 2, 4)
505        );
506        assert_eq!(COUNTER_FLAGS & ATTR_FLAG_PINNED, ATTR_FLAG_PINNED);
507        assert_eq!(COUNTER_FLAGS & (ATTR_FLAG_INHERIT | ATTR_FLAG_DISABLED), 0);
508        // Bits 4-9 clear: no exclude_* bit (4-7), so user, kernel, hypervisor
509        // and idle all count, and no mmap or comm records (8-9).
510        assert_eq!(COUNTER_FLAGS & (0x3f << 4), 0);
511    }
512
513    /// The tightest the binding is, pinned: at the 190-interrupt floor with no
514    /// time outside the counter, 180 passes and 179 does not. And one short
515    /// spin cannot pass on its own, however its counts line up: a 70 ms spin
516    /// measured counter 72, LOC 73, total 74 with 4.7% outside, where the slack
517    /// alone would admit a counter of 59 (81%).
518    #[test]
519    fn calibration_needs_about_95_percent_of_the_timer_interrupts() {
520        let floor = CpuInterrupts {
521            local_timer: MIN_CALIBRATION_TICKS,
522            total: MIN_CALIBRATION_TICKS,
523        };
524        assert_eq!(calibration_verdict(180, floor, 0.0), Ok(()));
525        assert!(calibration_verdict(179, floor, 0.0).is_err());
526        let one_short_spin = CpuInterrupts {
527            local_timer: 73,
528            total: 74,
529        };
530        assert!(calibration_verdict(72, one_short_spin, 0.047).is_err());
531        assert!(calibration_verdict(59, one_short_spin, 0.047).is_err());
532    }
533}
hermit/xstate_probe/irq_counter.rs

hermit/xstate_probe/
irq_counter.rs
