detcore/
netlink_route.rs

1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9//! Determinize the statistics counters carried by `NETLINK_ROUTE` link dumps.
10//!
11//! A guest that resolves a hostname or enumerates interfaces reaches
12//! `NETLINK_ROUTE` through glibc, which issues `RTM_GETLINK` and receives one
13//! `RTM_NEWLINK` message per interface. Those messages carry LIVE COUNTERS that
14//! advance continuously on a running host, so two executions of the same guest
15//! milliseconds apart receive byte-different payloads of IDENTICAL LENGTH.
16//!
17//! MEASURED NATIVELY, outside Hermit, on two consecutive `RTM_GETLINK` dumps:
18//! both replies were 3060 bytes, 47 bytes differed, and of the first twelve
19//! 64-bit fields that moved, TWELVE INCREASED AND ZERO DECREASED — monotonic
20//! counters. Every differing byte was attributed to an attribute; none fell
21//! outside one:
22//!
23//! ```text
24//!   AF_SPEC -> AF_INET6 -> IFLA_INET6_STATS       30 bytes
25//!   AF_SPEC -> AF_INET6 -> IFLA_INET6_CACHEINFO     4 bytes (guest payload only)
26//!   IFLA_STATS64                               8 bytes
27//!   IFLA_STATS                                 8 bytes
28//! ```
29//!
30//! THE ATTRIBUTION IS WHY THIS ZEROES THREE ATTRIBUTES RATHER THAN ONE. The
31//! obvious fix — zero `IFLA_STATS64` — addresses 8 of 47 differing bytes and
32//! leaves the guest still diverging on the other 39. The nested IPv6 SNMP block
33//! inside `AF_SPEC` is the LARGEST contributor and is easy to miss because it is
34//! two levels of nesting below the message.
35//!
36//! Nothing else in the dump moved: no timestamps, no sequence numbers, no
37//! addresses, no interface indices. Zeroing exactly these three makes the reply
38//! byte-stable without touching anything a guest legitimately reads for
39//! identity or topology.
40//!
41//! The counters are ZEROED rather than virtualized to advance. Freezing them is
42//! the same choice `crate::sock_diag` already makes for socket inodes, and a
43//! guest under Hermit has no deterministic clock against which advancing
44//! counters would mean anything. A guest that reads interface statistics sees
45//! zeros; that is a deliberate and stated consequence.
46
47/// `nlmsghdr` is 16 bytes: len(u32) type(u16) flags(u16) seq(u32) pid(u32).
48const NLMSG_HDRLEN: usize = 16;
49const NLMSG_ALIGNTO: usize = 4;
50
51/// `rtattr` is 4 bytes: len(u16) type(u16).
52const RTA_HDRLEN: usize = 4;
53const RTA_ALIGNTO: usize = 4;
54
55/// `ifinfomsg`, which follows the `nlmsghdr` in an `RTM_NEWLINK` message.
56const IFINFOMSG_LEN: usize = 16;
57
58/// `RTM_NEWLINK`, the reply type for an `RTM_GETLINK` dump.
59const RTM_NEWLINK: u16 = 16;
60
61/// `RTM_NEWADDR`, the reply type for an `RTM_GETADDR` dump. glibc's
62/// `getifaddrs` issues BOTH dumps, so a guest that enumerates interfaces
63/// receives both message types and both must be determinized.
64const RTM_NEWADDR: u16 = 20;
65
66/// `ifaddrmsg`, which follows the `nlmsghdr` in an `RTM_NEWADDR` message. It is
67/// EIGHT bytes, not the sixteen of `ifinfomsg` -- getting this wrong walks the
68/// attributes from the wrong offset and silently matches nothing.
69const IFADDRMSG_LEN: usize = 8;
70
71/// `ifa_cacheinfo`, carrying `cstamp` and `tstamp` -- timestamps in hundredths
72/// of a second since boot, which advance between runs.
73const IFA_CACHEINFO: u16 = 6;
74
75const IFLA_STATS: u16 = 7;
76const IFLA_STATS64: u16 = 23;
77const IFLA_AF_SPEC: u16 = 26;
78
79const AF_INET6: u16 = 10;
80const IFLA_INET6_STATS: u16 = 3;
81const IFLA_INET6_ICMP6STATS: u16 = 6;
82/// `ifla_cacheinfo`, which carries `tstamp` and `reachable_time`. MEASURED on
83/// the real guest payload: after zeroing the three statistics attributes, FOUR
84/// BYTES still differed between two runs and every one of them was here. The
85/// native host sample did not expose it -- those fields happened to match in
86/// that pair -- which is why the fix had to be tested against the ACTUAL
87/// failing buffer rather than a synthetic dump.
88const IFLA_INET6_CACHEINFO: u16 = 5;
89
90fn align_up(value: usize, align: usize) -> usize {
91    value.div_ceil(align) * align
92}
93
94/// Zero the live counters in a `NETLINK_ROUTE` reply, in place.
95///
96/// Returns whether anything was modified, so the caller can skip writing the
97/// buffer back to guest memory when there was nothing to change. A malformed or
98/// truncated buffer is left ALONE rather than partially rewritten: this is a
99/// determinism sanitizer, and a half-rewritten reply would be a worse guest
100/// observation than an undeterminized one.
101pub fn sanitize_route_link_stats(buf: &mut [u8]) -> bool {
102    let mut modified = false;
103    let mut offset = 0usize;
104
105    while offset + NLMSG_HDRLEN <= buf.len() {
106        let len = u32::from_ne_bytes(match buf[offset..offset + 4].try_into() {
107            Ok(bytes) => bytes,
108            Err(_) => return modified,
109        }) as usize;
110        let msg_type = u16::from_ne_bytes(match buf[offset + 4..offset + 6].try_into() {
111            Ok(bytes) => bytes,
112            Err(_) => return modified,
113        });
114
115        // A length shorter than the header, or one that runs past the buffer,
116        // means this is not a well-formed message stream. Stop rather than guess.
117        if len < NLMSG_HDRLEN || offset + len > buf.len() {
118            return modified;
119        }
120
121        match msg_type {
122            RTM_NEWLINK => {
123                let body = offset + NLMSG_HDRLEN + IFINFOMSG_LEN;
124                let end = offset + len;
125                if body <= end {
126                    zero_link_attrs(buf, body, end, &mut modified);
127                }
128            }
129            RTM_NEWADDR => {
130                let body = offset + NLMSG_HDRLEN + IFADDRMSG_LEN;
131                let end = offset + len;
132                if body <= end {
133                    zero_addr_attrs(buf, body, end, &mut modified);
134                }
135            }
136            _ => {}
137        }
138
139        offset += align_up(len, NLMSG_ALIGNTO);
140    }
141
142    modified
143}
144
145/// Walk the top-level `IFLA_*` attributes of one `RTM_NEWLINK` message.
146fn zero_link_attrs(buf: &mut [u8], mut attr: usize, end: usize, modified: &mut bool) {
147    while attr + RTA_HDRLEN <= end {
148        let (alen, atype) = match attr_header(buf, attr) {
149            Some(header) => header,
150            None => return,
151        };
152        if alen < RTA_HDRLEN || attr + alen > end {
153            return;
154        }
155        let payload = attr + RTA_HDRLEN;
156        let payload_end = attr + alen;
157
158        match atype {
159            IFLA_STATS | IFLA_STATS64 => zero_range(buf, payload, payload_end, modified),
160            IFLA_AF_SPEC => zero_af_spec(buf, payload, payload_end, modified),
161            _ => {}
162        }
163
164        attr += align_up(alen, RTA_ALIGNTO);
165    }
166}
167
168/// Walk the `IFA_*` attributes of one `RTM_NEWADDR` message, zeroing the cache
169/// timestamps. MEASURED on the real guest payload: a 156-byte reply held two
170/// `RTM_NEWADDR` messages and every one of its eight differing bytes was in
171/// `IFA_CACHEINFO`.
172fn zero_addr_attrs(buf: &mut [u8], mut attr: usize, end: usize, modified: &mut bool) {
173    while attr + RTA_HDRLEN <= end {
174        let (alen, atype) = match attr_header(buf, attr) {
175            Some(header) => header,
176            None => return,
177        };
178        if alen < RTA_HDRLEN || attr + alen > end {
179            return;
180        }
181        if atype == IFA_CACHEINFO {
182            zero_range(buf, attr + RTA_HDRLEN, attr + alen, modified);
183        }
184        attr += align_up(alen, RTA_ALIGNTO);
185    }
186}
187
188/// `IFLA_AF_SPEC` nests one block per address family; the IPv6 block carries the
189/// SNMP counter arrays. This is the largest source of drift and the one a
190/// stats-only fix misses.
191fn zero_af_spec(buf: &mut [u8], mut fam: usize, end: usize, modified: &mut bool) {
192    while fam + RTA_HDRLEN <= end {
193        let (flen, family) = match attr_header(buf, fam) {
194            Some(header) => header,
195            None => return,
196        };
197        if flen < RTA_HDRLEN || fam + flen > end {
198            return;
199        }
200
201        if family == AF_INET6 {
202            let mut inner = fam + RTA_HDRLEN;
203            let inner_end = fam + flen;
204            while inner + RTA_HDRLEN <= inner_end {
205                let (ilen, itype) = match attr_header(buf, inner) {
206                    Some(header) => header,
207                    None => return,
208                };
209                if ilen < RTA_HDRLEN || inner + ilen > inner_end {
210                    return;
211                }
212                if matches!(
213                    itype,
214                    IFLA_INET6_STATS | IFLA_INET6_ICMP6STATS | IFLA_INET6_CACHEINFO
215                ) {
216                    zero_range(buf, inner + RTA_HDRLEN, inner + ilen, modified);
217                }
218                inner += align_up(ilen, RTA_ALIGNTO);
219            }
220        }
221
222        fam += align_up(flen, RTA_ALIGNTO);
223    }
224}
225
226fn attr_header(buf: &[u8], at: usize) -> Option<(usize, u16)> {
227    let len = u16::from_ne_bytes(buf.get(at..at + 2)?.try_into().ok()?) as usize;
228    let kind = u16::from_ne_bytes(buf.get(at + 2..at + 4)?.try_into().ok()?);
229    Some((len, kind))
230}
231
232fn zero_range(buf: &mut [u8], from: usize, to: usize, modified: &mut bool) {
233    if from >= to || to > buf.len() {
234        return;
235    }
236    for byte in &mut buf[from..to] {
237        if *byte != 0 {
238            *byte = 0;
239            *modified = true;
240        }
241    }
242}
243
244#[cfg(test)]
245mod test {
246    use super::*;
247
248    fn rtattr(kind: u16, payload: &[u8]) -> Vec<u8> {
249        let len = RTA_HDRLEN + payload.len();
250        let mut out = Vec::new();
251        out.extend_from_slice(&(len as u16).to_ne_bytes());
252        out.extend_from_slice(&kind.to_ne_bytes());
253        out.extend_from_slice(payload);
254        out.resize(align_up(len, RTA_ALIGNTO), 0);
255        out
256    }
257
258    fn newlink(attrs: &[u8]) -> Vec<u8> {
259        let len = NLMSG_HDRLEN + IFINFOMSG_LEN + attrs.len();
260        let mut out = Vec::new();
261        out.extend_from_slice(&(len as u32).to_ne_bytes());
262        out.extend_from_slice(&RTM_NEWLINK.to_ne_bytes());
263        out.extend_from_slice(&0u16.to_ne_bytes());
264        out.extend_from_slice(&0u32.to_ne_bytes());
265        out.extend_from_slice(&0u32.to_ne_bytes());
266        out.extend_from_slice(&[0u8; IFINFOMSG_LEN]);
267        out.extend_from_slice(attrs);
268        out
269    }
270
271    #[test]
272    fn zeroes_link_stats64() {
273        let mut msg = newlink(&rtattr(IFLA_STATS64, &[7u8; 16]));
274        let payload = NLMSG_HDRLEN + IFINFOMSG_LEN + RTA_HDRLEN;
275        assert!(sanitize_route_link_stats(&mut msg));
276        assert!(
277            msg[payload..].iter().all(|b| *b == 0),
278            "IFLA_STATS64 counters were not fully zeroed"
279        );
280    }
281
282    #[test]
283    fn zeroes_legacy_link_stats() {
284        let mut msg = newlink(&rtattr(IFLA_STATS, &[9u8; 12]));
285        assert!(sanitize_route_link_stats(&mut msg));
286        assert!(!msg.contains(&9));
287    }
288
289    /// THE CASE A STATS-ONLY FIX MISSES. IFLA_INET6_STATS lives two levels down,
290    /// inside AF_SPEC's AF_INET6 block, and supplied 30 of the 47 differing
291    /// bytes in the native measurement.
292    #[test]
293    fn zeroes_ipv6_stats_nested_two_levels_inside_af_spec() {
294        let inner = rtattr(IFLA_INET6_STATS, &[5u8; 24]);
295        let family = rtattr(AF_INET6, &inner);
296        let mut msg = newlink(&rtattr(IFLA_AF_SPEC, &family));
297        assert!(sanitize_route_link_stats(&mut msg));
298        assert!(
299            !msg.contains(&5),
300            "nested IPv6 SNMP counters were left undeterminized"
301        );
302    }
303
304    /// Identity and topology attributes must survive. Zeroing an interface name
305    /// or index would be a far worse guest observation than a drifting counter.
306    #[test]
307    fn leaves_non_counter_attributes_alone() {
308        const IFLA_IFNAME: u16 = 3;
309        let mut msg = newlink(&rtattr(IFLA_IFNAME, b"eth0\0"));
310        assert!(!sanitize_route_link_stats(&mut msg));
311        assert!(
312            msg.windows(4).any(|w| w == b"eth0"),
313            "the interface name was modified"
314        );
315    }
316
317    /// A reply carrying no counters must report "unmodified" so the caller can
318    /// skip writing the buffer back into guest memory.
319    #[test]
320    fn reports_unmodified_when_there_is_nothing_to_zero() {
321        let mut msg = newlink(&rtattr(IFLA_STATS64, &[0u8; 16]));
322        assert!(!sanitize_route_link_stats(&mut msg));
323    }
324
325    /// A truncated stream is left ALONE rather than half-rewritten.
326    #[test]
327    fn refuses_to_rewrite_a_truncated_message() {
328        let full = newlink(&rtattr(IFLA_STATS64, &[3u8; 16]));
329        let mut truncated = full[..full.len() - 6].to_vec();
330        let before = truncated.clone();
331        sanitize_route_link_stats(&mut truncated);
332        assert_eq!(
333            truncated, before,
334            "a truncated reply must not be partially rewritten"
335        );
336    }
337
338    /// Only RTM_NEWLINK is touched; an unrelated message type passes through.
339    #[test]
340    fn ignores_message_types_that_are_not_newlink() {
341        let mut msg = newlink(&rtattr(IFLA_STATS64, &[4u8; 16]));
342        msg[4..6].copy_from_slice(&20u16.to_ne_bytes());
343        assert!(!sanitize_route_link_stats(&mut msg));
344        assert!(msg.contains(&4));
345    }
346}
detcore/netlink_route.rs

detcore/
netlink_route.rs
