1use serde::Deserialize;
10use serde::Serialize;
11
12pub type Error = anyhow::Error;
13
14pub use anyhow::Context;
15
16#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, Eq, PartialEq)]
21pub enum FailureKind {
22 #[default]
24 Error,
25 Panic,
33 PolicyRefusal,
46 SkidOvershoot { count: u64 },
51 RunTimeout,
63 BackendRunFailure,
67 TerminalOutcome(detcore::TerminalOutcome),
75}
76
77#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
103pub struct SerializableError {
104 error: String,
106 context: Vec<String>,
108 #[serde(default)]
110 kind: FailureKind,
111 cleanup_stage: Option<crate::HermitCleanupStage>,
114}
115
116impl SerializableError {
117 pub fn kind(&self) -> FailureKind {
119 self.kind
120 }
121
122 pub fn cleanup_stage(&self) -> Option<crate::HermitCleanupStage> {
124 self.cleanup_stage
125 }
126
127 pub fn into_panic(mut self) -> Self {
130 self.kind = FailureKind::Panic;
131 self
132 }
133}
134
135fn is_policy_refusal(err: &Error) -> bool {
155 if detcore::ReplayDepartureError::in_error(err).is_some() {
158 return true;
159 }
160 fn refuses(cause: &(dyn std::error::Error + 'static)) -> bool {
165 cause.is::<detcore::UnsupportedSyscallError>()
166 || cause.is::<detcore::preemptions::RecordAdmissionRefusal>()
167 || cause.is::<detcore::preemptions::RecordFormatRefusal>()
168 || cause.is::<detcore::ChosenBinaryInputRefusal>()
169 || cause.is::<detcore::TerminalReplayRefusal>()
170 || cause.is::<detcore::PriorityHistoryRefusal>()
171 || cause.is::<detcore::ReplayDepartureError>()
172 || cause.is::<reverie_ptrace::UntracedStubSyscall>()
173 || is_chosen_root_id_taken(cause)
174 || is_kvm_host_resources_refusal(cause)
175 || is_kvm_named_refusal(cause)
176 || is_kvm_vfork_child_refusal(cause)
177 }
178 #[cfg(target_arch = "x86_64")]
184 fn is_kvm_host_resources_refusal(cause: &(dyn std::error::Error + 'static)) -> bool {
185 cause
186 .downcast_ref::<reverie_kvm::Error>()
187 .is_some_and(reverie_kvm::Error::is_host_resources_refusal)
188 }
189 #[cfg(not(target_arch = "x86_64"))]
190 fn is_kvm_host_resources_refusal(_cause: &(dyn std::error::Error + 'static)) -> bool {
191 false
192 }
193 #[cfg(target_arch = "x86_64")]
199 fn is_kvm_named_refusal(cause: &(dyn std::error::Error + 'static)) -> bool {
200 cause
201 .downcast_ref::<reverie_kvm::Error>()
202 .is_some_and(reverie_kvm::Error::is_named_refusal)
203 }
204 #[cfg(not(target_arch = "x86_64"))]
205 fn is_kvm_named_refusal(_cause: &(dyn std::error::Error + 'static)) -> bool {
206 false
207 }
208 #[cfg(target_arch = "x86_64")]
217 fn is_kvm_vfork_child_refusal(cause: &(dyn std::error::Error + 'static)) -> bool {
218 cause
219 .downcast_ref::<reverie_kvm::Error>()
220 .is_some_and(|error| {
221 matches!(
222 error.primary(),
223 reverie_kvm::Error::VforkChildCapability { .. }
224 )
225 })
226 }
227 #[cfg(not(target_arch = "x86_64"))]
228 fn is_kvm_vfork_child_refusal(_cause: &(dyn std::error::Error + 'static)) -> bool {
229 false
230 }
231 fn is_chosen_root_id_taken(cause: &(dyn std::error::Error + 'static)) -> bool {
237 cause
238 .downcast_ref::<reverie::process::Error>()
239 .is_some_and(|error| {
240 error.errno() == reverie::syscalls::Errno::EEXIST
241 && error.context() == reverie::process::Context::Clone
242 })
243 }
244 if err.chain().any(refuses) {
245 return true;
246 }
247 err.chain().any(|cause| {
250 cause
251 .downcast_ref::<reverie::Error>()
252 .is_some_and(|reverie_error| match reverie_error {
253 reverie::Error::Tool(inner) => inner.chain().any(refuses),
254 _ => false,
255 })
256 })
257}
258
259pub(crate) fn classify_ptrace_primary(error: &reverie::Error) -> FailureKind {
262 match error {
263 reverie::Error::Tool(inner)
268 if inner.is::<crate::SchedulerEndedRun>()
269 && detcore::ReplayDepartureError::in_error(inner).is_none() =>
270 {
271 FailureKind::Error
272 }
273 reverie::Error::Tool(inner) => classify_unwrapped(inner),
274 _ => FailureKind::Error,
275 }
276}
277
278fn finalization_primary(error: &Error) -> Option<&Error> {
281 if let Some(pair) = error.downcast_ref::<detcore::FinalizationFailure>() {
282 return Some(pair.primary());
283 }
284 error
285 .chain()
286 .find_map(|cause| match cause.downcast_ref::<reverie::Error>() {
287 Some(reverie::Error::Tool(inner)) => finalization_primary(inner),
288 _ => None,
289 })
290}
291
292fn classify_preserving_cleanup(error: &Error) -> (FailureKind, Option<crate::HermitCleanupStage>) {
293 if let Some(primary) = finalization_primary(error) {
294 classify_preserving_cleanup(primary)
295 } else if let Some(error) = error.downcast_ref::<crate::HermitCleanupUnconfirmed>() {
296 (error.primary_kind(), Some(error.stage()))
297 } else if let Some(error) = error.downcast_ref::<crate::HermitPtraceFailure>() {
298 (error.kind(), None)
299 } else {
300 (classify_unwrapped(error), None)
301 }
302}
303
304fn classify_unwrapped(error: &Error) -> FailureKind {
305 if let Some(primary) = finalization_primary(error) {
306 return classify_preserving_cleanup(primary).0;
307 }
308 if let Some(error) = error.downcast_ref::<crate::SkidOvershootError>() {
309 FailureKind::SkidOvershoot {
310 count: error.count(),
311 }
312 } else if error
313 .chain()
314 .any(|cause| cause.is::<detcore_model::virtual_timeout::VirtualTimeoutWithDebugger>())
315 || is_policy_refusal(error)
316 || error
317 .downcast_ref::<crate::LiteinstInGuestRefusal>()
318 .is_some()
319 || error
320 .chain()
321 .any(|cause| cause.is::<crate::cpu_profile_admission::CpuProfileRefusal>())
322 || error
323 .chain()
324 .any(|cause| cause.is::<crate::preemption_admission::UnsupportedPreemption>())
325 || error
326 .chain()
327 .any(|cause| cause.is::<crate::preemption_admission::UnqualifiedAnnotatedReplay>())
328 || error
329 .chain()
330 .any(|cause| cause.is::<crate::kvm_address_space::KvmAddressSpaceRefusal>())
331 {
332 FailureKind::PolicyRefusal
333 } else if error.downcast_ref::<crate::GuestTimedOut>().is_some() {
334 FailureKind::RunTimeout
335 } else if let Some(ended) = error.downcast_ref::<crate::SchedulerEndedRun>() {
336 FailureKind::TerminalOutcome(ended.outcome)
337 } else if error.downcast_ref::<crate::BackendRunFailure>().is_some() {
338 FailureKind::BackendRunFailure
339 } else {
340 FailureKind::Error
341 }
342}
343
344impl From<Error> for SerializableError {
345 fn from(err: Error) -> Self {
346 let error = err.to_string();
347 let context = err.chain().skip(1).map(ToString::to_string).collect();
348 let (kind, cleanup_stage) = classify_preserving_cleanup(&err);
355 Self {
356 error,
357 context,
358 kind,
359 cleanup_stage,
360 }
361 }
362}
363
364impl From<SerializableError> for Error {
365 fn from(mut err: SerializableError) -> Self {
366 if let Some(root_cause) = err.context.pop() {
367 let mut error = Self::msg(root_cause);
368
369 while let Some(context) = err.context.pop() {
370 error = error.context(context);
371 }
372
373 error.context(err.error)
374 } else {
375 Self::msg(err.error)
376 }
377 }
378}
379
380#[cfg(test)]
381pub(crate) async fn terminal_replay_refusal_for_test() -> Error {
382 use detcore::preemptions::PreemptionRecord;
383 use detcore::preemptions::boundary_recording::BoundaryRecording;
384 use detcore::preemptions::boundary_recording::Recording;
385 use detcore_model::pid::DetTid;
386 use detcore_model::schedule::SchedEvent;
387 use detcore_model::schedule::boundary::Record;
388 use detcore_model::schedule::boundary::Schedule;
389 let directory = tempfile::tempdir().unwrap();
390 let path = directory.path().join("unconsumed.json");
391 let recording = Recording::Boundaries(
392 BoundaryRecording::new(
393 PreemptionRecord::default(),
394 Schedule::new(vec![Record::Event(SchedEvent::branches(
395 DetTid::from_raw(1),
396 1,
397 ))])
398 .unwrap(),
399 )
400 .unwrap(),
401 );
402 std::fs::write(&path, recording.encode().unwrap()).unwrap();
403 detcore::GlobalState::init_for_external_scheduler(&detcore::Config {
404 sequentialize_threads: true,
405 strict_schedule_boundaries: true,
406 replay_schedule_from: Some(path),
407 record_preemptions: false,
408 ..detcore::Config::default()
409 })
410 .clean_up(false, &None)
411 .await
412 .unwrap_err()
413}
414
415#[cfg(test)]
416mod tests {
417 use super::*;
418
419 #[test]
420 fn record_format_refusal_keeps_its_kind_through_context_and_reverie() {
421 use detcore::preemptions::RecordFormatRefusal;
422 use detcore::preemptions::boundary_recording::Recording;
423
424 const RECORDS: [&[u8]; 2] = [
425 br#"{"per_thread":{},"global":[],"format_version":4}"#,
426 br#"{"per_thread":{},"global":[],"format_version":3,"time_skips":[]}"#,
427 ];
428 for contextual in [false, true] {
429 for wrapped in [false, true] {
430 let adapt = |mut error: Error| {
431 if contextual {
432 error = error.context("reading schedule before guest startup");
433 }
434 if wrapped {
435 error = Error::new(reverie::Error::Tool(error));
436 }
437 if contextual {
438 error = error.context("outer launch boundary");
439 }
440 error
441 };
442 for bytes in RECORDS {
443 let refusal = Recording::decode(bytes).unwrap_err();
444 assert!(refusal.is::<RecordFormatRefusal>());
445 let prose = adapt(Error::msg(refusal.to_string()));
446 let serialized = SerializableError::from(adapt(refusal));
447 assert_eq!(serialized.kind(), FailureKind::PolicyRefusal);
448 let bytes =
449 bincode::serde::encode_to_vec(&serialized, bincode::config::legacy())
450 .unwrap();
451 let (decoded, used): (SerializableError, usize) =
452 bincode::serde::decode_from_slice(&bytes, bincode::config::legacy())
453 .unwrap();
454 assert_eq!(used, bytes.len());
455 assert_eq!(decoded, serialized);
456 assert_eq!(SerializableError::from(prose).kind(), FailureKind::Error);
457 }
458 for error in [
459 Recording::decode(b"{").unwrap_err(),
460 Error::new(std::io::Error::from(std::io::ErrorKind::NotFound)),
461 ] {
462 assert!(!error.is::<RecordFormatRefusal>());
463 assert_eq!(
464 SerializableError::from(adapt(error)).kind(),
465 FailureKind::Error
466 );
467 }
468 }
469 }
470 }
471
472 #[tokio::test]
473 async fn terminal_refusal_is_typed_direct_and_through_reverie() {
474 for wrapped in [false, true] {
475 let error = terminal_replay_refusal_for_test().await;
476 assert!(error.is::<detcore::TerminalReplayRefusal>());
477 let error = if wrapped {
478 Error::new(reverie::Error::Tool(error))
479 } else {
480 error
481 };
482 let serialized = SerializableError::from(error);
483 assert_eq!(serialized.kind(), FailureKind::PolicyRefusal);
484 assert!(format!("{serialized:?}").contains("Incomplete"));
485 let bytes =
486 bincode::serde::encode_to_vec(&serialized, bincode::config::legacy()).unwrap();
487 let (decoded, used): (SerializableError, usize) =
488 bincode::serde::decode_from_slice(&bytes, bincode::config::legacy()).unwrap();
489 assert_eq!(used, bytes.len());
490 assert_eq!(decoded, serialized);
491 }
492 for error in [
493 Error::new(std::io::Error::other("recording write failed")),
494 Error::msg("terminal replay refusal: slice-boundary protocol refusal"),
495 ] {
496 assert_eq!(SerializableError::from(error).kind(), FailureKind::Error);
497 }
498 }
499
500 #[tokio::test]
501 async fn finalization_secondary_never_overrides_primary_classification() {
502 for (primary, kind) in [
503 (
504 Error::new(std::io::Error::other("ordinary primary")),
505 FailureKind::Error,
506 ),
507 (
508 Error::new(crate::GuestTimedOut {
509 limit: std::time::Duration::from_secs(3),
510 survivors: None,
511 }),
512 FailureKind::RunTimeout,
513 ),
514 (
515 Error::new(crate::BackendRunFailure),
516 FailureKind::BackendRunFailure,
517 ),
518 (
519 Error::new(crate::SchedulerEndedRun::new(
520 detcore::TerminalOutcome::GuestDeadlock,
521 None,
522 )),
523 FailureKind::TerminalOutcome(detcore::TerminalOutcome::GuestDeadlock),
524 ),
525 (
526 Error::new(crate::SchedulerEndedRun::new(
527 detcore::TerminalOutcome::VirtualTimeout { bound_ns: 9 },
528 None,
529 )),
530 FailureKind::TerminalOutcome(detcore::TerminalOutcome::VirtualTimeout {
531 bound_ns: 9,
532 }),
533 ),
534 (
535 Error::new(crate::SkidOvershootError::new(2)),
536 FailureKind::SkidOvershoot { count: 2 },
537 ),
538 ] {
539 let paired = detcore::combine_finalization::<()>(
540 Err(primary),
541 Err(terminal_replay_refusal_for_test().await),
542 )
543 .unwrap_err();
544 let serialized = SerializableError::from(Error::new(reverie::Error::Tool(paired)));
545 assert_eq!(serialized.kind(), kind);
546 assert!(
547 serialized.error.contains("Incomplete"),
548 "secondary must survive serialization"
549 );
550 }
551 let paired = detcore::combine_finalization::<()>(
552 Err(terminal_replay_refusal_for_test().await),
553 Err(Error::new(std::io::Error::other("secondary write failure"))),
554 )
555 .unwrap_err();
556 let serialized = SerializableError::from(paired);
557 assert_eq!(serialized.kind(), FailureKind::PolicyRefusal);
558 assert!(serialized.error.contains("secondary write failure"));
559 }
560
561 #[test]
562 fn unsupported_preemption_keeps_its_policy_class_across_the_container_boundary() {
563 let refusal = crate::preemption_admission::refuse_unsupported_preemption_with_capabilities(
564 &detcore::Config::default(),
565 "dbt",
566 reverie::BackendCapabilities::DBT,
567 true,
568 false,
569 )
570 .unwrap_err();
571 let serialized = SerializableError::from(Error::new(refusal).context("launch refused"));
572 assert_eq!(serialized.kind(), FailureKind::PolicyRefusal);
573 assert!(serialized.context.iter().any(|message| {
574 message.contains("unsupported_preemption") && message.contains("--backend=dbt")
575 }));
576 }
577
578 #[test]
579 fn a_kvm_address_space_refusal_keeps_its_policy_class_across_the_container_boundary() {
580 let refusal = crate::kvm_address_space::KvmAddressSpaceRefusal::for_limit(
581 libc::rlimit {
582 rlim_cur: 16 << 30,
583 rlim_max: 16 << 30,
584 },
585 3 << 30,
586 )
587 .expect("a finite limit is refused");
588 let serialized = SerializableError::from(Error::new(refusal).context("launch refused"));
589 assert_eq!(serialized.kind(), FailureKind::PolicyRefusal);
590 assert!(serialized.context.iter().any(|message| {
591 message.contains("kvm_address_space_limit") && message.contains("--backend=kvm")
592 }));
593 }
594
595 #[test]
596 fn into_serializable_error() {
597 let error = Error::msg("root cause")
598 .context("a")
599 .context("b")
600 .context("c");
601
602 assert_eq!(
603 SerializableError::from(error),
604 SerializableError {
605 error: "c".into(),
606 context: vec!["b".into(), "a".into(), "root cause".into(),],
607 kind: FailureKind::Error,
608 cleanup_stage: None,
609 }
610 );
611 }
612
613 #[test]
614 fn from_serializable_error() {
615 let error = Error::from(SerializableError {
616 error: "c".into(),
617 context: vec!["b".into(), "a".into(), "root cause".into()],
618 kind: FailureKind::Error,
619 cleanup_stage: None,
620 });
621
622 assert_eq!(
623 error
624 .chain()
625 .map(ToString::to_string)
626 .collect::<Vec<String>>(),
627 ["c", "b", "a", "root cause"]
628 .iter()
629 .map(ToString::to_string)
630 .collect::<Vec<String>>(),
631 )
632 }
633
634 #[test]
635 fn skid_overshoot_is_serialized_as_a_policy_refusal() {
636 let refusal = SerializableError::from(Error::new(crate::SkidOvershootError::new(2)));
637 assert_eq!(refusal.kind(), FailureKind::SkidOvershoot { count: 2 });
638 assert!(refusal.error.contains("observed 2 HERMIT_SKID_OVERSHOOT"));
639
640 let contextual = SerializableError::from(
641 Error::msg("guest execution also failed").context(crate::SkidOvershootError::new(1)),
642 );
643 assert_eq!(contextual.kind(), FailureKind::SkidOvershoot { count: 1 });
644 assert!(
645 contextual
646 .error
647 .contains("observed 1 HERMIT_SKID_OVERSHOOT")
648 );
649
650 let ordinary = SerializableError::from(Error::msg("ordinary failure"));
651 assert_eq!(ordinary.kind(), FailureKind::Error);
652 }
653
654 #[test]
657 fn in_guest_liteinst_refusal_is_serialized_as_a_policy_refusal() {
658 let refusal = SerializableError::from(Error::new(crate::LiteinstInGuestRefusal::new(
659 "guest program /static is statically linked",
660 )));
661 assert_eq!(refusal.kind(), FailureKind::PolicyRefusal);
662 assert_eq!(
663 refusal.error,
664 "--backend=liteinst (in-guest LiteInst) refuses this run: guest program /static is \
665 statically linked"
666 );
667 }
668
669 #[test]
674 fn an_untraced_stub_syscall_is_a_policy_refusal() {
675 let refusal = || reverie_ptrace::UntracedStubSyscall {
676 tid: 7,
677 address: 0x7100_0002,
678 };
679 let primary = reverie::Error::Tool(Error::new(refusal()));
680 assert_eq!(
681 classify_ptrace_primary(&primary),
682 FailureKind::PolicyRefusal
683 );
684 let wrapped = Error::new(reverie::Error::Tool(Error::new(refusal())))
685 .context("the ptrace backend failed");
686 assert_eq!(classify_unwrapped(&wrapped), FailureKind::PolicyRefusal);
687 let serialized = SerializableError::from(Error::new(refusal()));
688 assert_eq!(serialized.kind(), FailureKind::PolicyRefusal);
689 assert!(
690 serialized.error.contains("0x71000002"),
691 "{}",
692 serialized.error
693 );
694 }
695
696 #[test]
702 fn a_terminal_outcome_keeps_its_kind_only_once_the_run_finished() {
703 for outcome in [
704 detcore::TerminalOutcome::GuestDeadlock,
705 detcore::TerminalOutcome::VirtualTimeout { bound_ns: 5 },
706 ] {
707 let finished =
708 SerializableError::from(Error::new(crate::SchedulerEndedRun::new(outcome, None)));
709 assert_eq!(finished.kind(), FailureKind::TerminalOutcome(outcome));
710 let primary =
711 reverie::Error::Tool(Error::new(crate::SchedulerEndedRun::new(outcome, None)));
712 assert_eq!(classify_ptrace_primary(&primary), FailureKind::Error);
713 }
714 }
715
716 #[test]
718 fn in_guest_trap_refusal_is_serialized_as_a_policy_refusal() {
719 let refusal =
720 SerializableError::from(Error::new(crate::LiteinstInGuestRefusal::for_backend(
721 crate::Backend::InGuestTrap,
722 "guest program /static is statically linked",
723 )));
724 assert_eq!(refusal.kind(), FailureKind::PolicyRefusal);
725 assert_eq!(
726 refusal.error,
727 "--backend=in-guest-trap (in-guest LiteInst, site patching off) refuses this run: \
728 guest program /static is statically linked"
729 );
730 }
731}