linera_service/tracing/
opentelemetry.rs

1// Copyright (c) Zefchain Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4//! OpenTelemetry integration for tracing with OTLP export.
5
6use opentelemetry::{global, propagation::TextMapCompositePropagator, trace::TracerProvider};
7use opentelemetry_otlp::{SpanExporter, WithExportConfig};
8#[cfg(with_testing)]
9use opentelemetry_sdk::trace::InMemorySpanExporter;
10use opentelemetry_sdk::{
11    propagation::{BaggagePropagator, TraceContextPropagator},
12    trace::{BatchSpanProcessor, SdkTracerProvider},
13    Resource,
14};
15use tracing_opentelemetry::OpenTelemetryLayer;
16use tracing_subscriber::{
17    filter::{filter_fn, FilterFn},
18    layer::Layer,
19    prelude::__tracing_subscriber_SubscriberExt as _,
20    util::SubscriberInitExt,
21};
22
23/// Creates a filter that excludes spans with the `opentelemetry.skip` field.
24///
25/// Any span that declares an `opentelemetry.skip` field will be excluded from export,
26/// regardless of the field's value. This is a limitation of the tracing metadata API.
27///
28/// Usage examples:
29/// ```ignore
30/// // Always skip this span
31/// #[tracing::instrument(fields(opentelemetry.skip = true))]
32/// fn internal_helper() { }
33///
34/// // Conditionally skip based on a parameter
35/// #[tracing::instrument(fields(opentelemetry.skip = should_skip))]
36/// fn my_function(should_skip: bool) {
37///     // Will be skipped if should_skip is true when called
38///     // Note: The field must be declared in the span, so the span is
39///     // created with knowledge that it might be skipped
40/// }
41/// ```
42fn opentelemetry_skip_filter() -> FilterFn<impl Fn(&tracing::Metadata<'_>) -> bool> {
43    filter_fn(|metadata| {
44        if !metadata.is_span() {
45            return false;
46        }
47        metadata.fields().field("opentelemetry.skip").is_none()
48    })
49}
50
51/// Initializes tracing with a custom OpenTelemetry tracer provider.
52///
53/// This is an internal function used by both production and test code.
54fn init_with_tracer_provider(log_name: &str, tracer_provider: &SdkTracerProvider) {
55    global::set_tracer_provider(tracer_provider.clone());
56    let tracer = tracer_provider.tracer("linera");
57
58    let opentelemetry_layer =
59        OpenTelemetryLayer::new(tracer).with_filter(opentelemetry_skip_filter());
60
61    let config = crate::tracing::get_env_config(log_name);
62    let maybe_log_file_layer = config.maybe_log_file_layer();
63    let stderr_layer = config.stderr_layer();
64
65    tracing_subscriber::registry()
66        .with(opentelemetry_layer)
67        .with(config.env_filter)
68        .with(maybe_log_file_layer)
69        .with(stderr_layer)
70        .init();
71
72    linera_base::panic_hook::init();
73}
74
75/// Builds an OpenTelemetry layer with the opentelemetry.skip filter.
76///
77/// This is used for testing to avoid setting the global subscriber.
78/// Returns the layer, exporter, and tracer provider (which must be kept alive and shutdown).
79#[cfg(with_testing)]
80pub fn build_opentelemetry_layer_with_test_exporter(
81    log_name: &str,
82) -> (
83    impl tracing_subscriber::Layer<tracing_subscriber::Registry>,
84    InMemorySpanExporter,
85    SdkTracerProvider,
86) {
87    let exporter = InMemorySpanExporter::default();
88    let exporter_clone = exporter.clone();
89
90    let resource = Resource::builder()
91        .with_service_name(log_name.to_string())
92        .build();
93
94    let tracer_provider = SdkTracerProvider::builder()
95        .with_resource(resource)
96        .with_simple_exporter(exporter)
97        .with_sampler(opentelemetry_sdk::trace::Sampler::AlwaysOn)
98        .build();
99
100    global::set_tracer_provider(tracer_provider.clone());
101    let tracer = tracer_provider.tracer("linera");
102    let opentelemetry_layer =
103        OpenTelemetryLayer::new(tracer).with_filter(opentelemetry_skip_filter());
104
105    (opentelemetry_layer, exporter_clone, tracer_provider)
106}
107
108/// Sets up the global text map propagator with TraceContext and Baggage support.
109///
110/// This enables:
111/// - W3C TraceContext propagation (traceparent, tracestate headers)
112/// - W3C Baggage propagation (baggage header for traffic_type, etc.)
113fn setup_propagator() {
114    let propagator = TextMapCompositePropagator::new(vec![
115        Box::new(TraceContextPropagator::new()),
116        Box::new(BaggagePropagator::new()),
117    ]);
118    global::set_text_map_propagator(propagator);
119}
120
121/// Initializes tracing with OpenTelemetry OTLP exporter.
122///
123/// Exports traces using the OTLP protocol to any OpenTelemetry-compatible backend.
124/// Requires the `opentelemetry` feature.
125/// Only enables OpenTelemetry if LINERA_OTLP_EXPORTER_ENDPOINT env var is set.
126/// This prevents DNS errors in environments where OpenTelemetry is not deployed.
127pub fn init(log_name: &str, otlp_endpoint: Option<&str>) {
128    // Set up composite propagator for TraceContext and Baggage
129    setup_propagator();
130
131    // Check if OpenTelemetry endpoint is configured via parameter or env var
132    let endpoint = match otlp_endpoint {
133        Some(ep) if !ep.is_empty() => ep.to_string(),
134        _ => match std::env::var("LINERA_OTLP_EXPORTER_ENDPOINT") {
135            Ok(ep) if !ep.is_empty() => ep,
136            _ => {
137                crate::tracing::init(log_name);
138                return;
139            }
140        },
141    };
142
143    let resource = Resource::builder()
144        .with_service_name(log_name.to_string())
145        .build();
146
147    let exporter = SpanExporter::builder()
148        .with_tonic()
149        .with_endpoint(endpoint)
150        .build()
151        .expect("Failed to create OTLP exporter");
152
153    // Configure batch processor for high-throughput scenarios
154    // Larger queue (16k instead of 2k default) to handle benchmark load
155    // Faster export (100ms instead of 5s default) to prevent queue buildup
156    let batch_config = opentelemetry_sdk::trace::BatchConfigBuilder::default()
157        .with_max_queue_size(16384) // 8x default, enough for 8 shards under load
158        .with_max_export_batch_size(2048) // Larger batches for efficiency
159        .with_scheduled_delay(std::time::Duration::from_millis(100)) // Fast export to prevent queue buildup
160        .build();
161
162    let batch_processor = BatchSpanProcessor::new(exporter, batch_config);
163
164    let tracer_provider = SdkTracerProvider::builder()
165        .with_resource(resource)
166        .with_span_processor(batch_processor)
167        .with_sampler(opentelemetry_sdk::trace::Sampler::AlwaysOn)
168        .build();
169
170    init_with_tracer_provider(log_name, &tracer_provider);
171}
linera_service/tracing/opentelemetry.rs

linera_service/tracing/
opentelemetry.rs
