linera_base/
panic_hook.rs

1// Copyright (c) Zefchain Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Process-wide reporting of panics.
5//!
6//! Tokio catches a panic at the task boundary and hands it to whoever joins the task, so a
7//! panicking task neither stops the runtime nor, on its own, produces anything a monitoring
8//! system can act on: the default hook writes to standard error and nothing else. The hook
9//! installed here reports the panic through `tracing` and the metrics registry first, so
10//! that panics are visible wherever the process's other logs and metrics are collected.
11
12use std::{
13    panic::PanicHookInfo,
14    sync::{Once, OnceLock},
15};
16
17#[cfg(with_metrics)]
18pub(crate) mod metrics {
19    use prometheus::IntCounter;
20
21    use crate::prometheus_util::register_int_counter;
22
23    crate::declare_metrics! {
24        /// Panics observed by the hook installed by [`super::init`].
25        ///
26        /// A panic does not stop the process, so this counter is often the only durable signal
27        /// that one happened: whatever the panicking task was responsible for has stopped, and
28        /// the effect on the rest of the process depends entirely on who was joining it. Any
29        /// increase deserves investigation.
30        pub(crate) static PANICS: IntCounter =
31            register_int_counter("panics_total", "Number of panics observed");
32    }
33}
34
35/// A panic hook, in the form [`std::panic::take_hook`] returns it.
36type PanicHook = Box<dyn Fn(&PanicHookInfo<'_>) + Sync + Send>;
37
38/// The hook that was installed before ours, which we delegate to so that the standard
39/// message and the `RUST_BACKTRACE` backtrace are still printed.
40///
41/// A `OnceLock` rather than a lock, so that reading it from inside a panic costs nothing
42/// and cannot be contended.
43static PREVIOUS_HOOK: OnceLock<PanicHook> = OnceLock::new();
44
45static INIT: Once = Once::new();
46
47/// Installs a panic hook that reports panics through `tracing` and the metrics registry
48/// before delegating to the hook that was previously installed.
49///
50/// Calling this more than once has no further effect. It does not change what a panic
51/// *does* — the process still unwinds the panicking task and keeps running — only what is
52/// recorded about it.
53pub fn init() {
54    INIT.call_once(|| {
55        #[cfg(with_metrics)]
56        metrics::init_metrics();
57        PREVIOUS_HOOK
58            .set(std::panic::take_hook())
59            .unwrap_or_else(|_| unreachable!("`call_once` runs this at most once"));
60        std::panic::set_hook(Box::new(report_panic));
61    });
62}
63
64/// The panic message, for the two payload types `panic!` produces.
65///
66/// `PanicHookInfo::payload_as_str` does the same thing, but is not yet stable in the
67/// toolchain the release branches pin, and this code is backported to them.
68fn payload_message<'a>(info: &'a PanicHookInfo<'_>) -> &'a str {
69    let payload = info.payload();
70    payload
71        .downcast_ref::<&str>()
72        .copied()
73        .or_else(|| payload.downcast_ref::<String>().map(String::as_str))
74        .unwrap_or("<non-string payload>")
75}
76
77fn report_panic(info: &PanicHookInfo<'_>) {
78    #[cfg(with_metrics)]
79    metrics::PANICS.inc();
80
81    let thread = std::thread::current();
82    tracing::error!(
83        thread = thread.name().unwrap_or("<unnamed>"),
84        location = info.location().map(tracing::field::display),
85        message = payload_message(info),
86        "Panic",
87    );
88
89    if let Some(previous) = PREVIOUS_HOOK.get() {
90        previous(info);
91    }
92}
93
94#[cfg(test)]
95mod tests {
96    use std::{
97        panic::AssertUnwindSafe,
98        sync::atomic::{AtomicUsize, Ordering},
99    };
100
101    use super::*;
102
103    /// Installing the hook twice must not chain it to itself, which would make every panic
104    /// report grow by one line per call.
105    #[test]
106    fn test_init_is_idempotent() {
107        static DELEGATIONS: AtomicUsize = AtomicUsize::new(0);
108
109        std::panic::set_hook(Box::new(|_| {
110            DELEGATIONS.fetch_add(1, Ordering::SeqCst);
111        }));
112        init();
113        init();
114
115        let panicked = std::panic::catch_unwind(AssertUnwindSafe(|| panic!("boom"))).is_err();
116
117        assert!(panicked);
118        assert_eq!(
119            DELEGATIONS.load(Ordering::SeqCst),
120            1,
121            "the hook installed before `init` ran exactly once",
122        );
123    }
124
125    /// `register_int_counter` applies the `linera` namespace itself, so a name that already
126    /// carries the prefix is exported twice over, as `linera_linera_panics_total`.
127    #[cfg(with_metrics)]
128    #[test]
129    fn the_counter_is_exported_under_a_single_linera_prefix() {
130        crate::init_metrics();
131
132        let names = prometheus::gather()
133            .iter()
134            .map(|family| family.get_name().to_owned())
135            .collect::<Vec<_>>();
136
137        assert!(names.iter().any(|name| name == "linera_panics_total"));
138        assert!(!names.iter().any(|name| name.contains("linera_linera")));
139    }
140}
linera_base/panic_hook.rs

linera_base/
panic_hook.rs
