From 8751ad02d46a3f56d10828c8db840ff102df14b6 Mon Sep 17 00:00:00 2001 From: Cosmin Constantin Lazar Date: Fri, 12 Jan 2024 06:15:29 +0100 Subject: [PATCH 01/13] Expose log batch_config configuration --- opentelemetry-otlp/src/logs.rs | 19 +++++++++++++++++-- opentelemetry-sdk/src/logs/log_processor.rs | 5 +++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/opentelemetry-otlp/src/logs.rs b/opentelemetry-otlp/src/logs.rs index 21f8fbb0ed..28424014df 100644 --- a/opentelemetry-otlp/src/logs.rs +++ b/opentelemetry-otlp/src/logs.rs @@ -42,6 +42,7 @@ impl OtlpPipeline { OtlpLogPipeline { log_config: None, exporter_builder: NoExporterConfig(()), + batch_config: None, } } } @@ -124,6 +125,7 @@ impl opentelemetry_sdk::export::logs::LogExporter for LogExporter { pub struct OtlpLogPipeline { exporter_builder: EB, log_config: Option, + batch_config: Option, } impl OtlpLogPipeline { @@ -132,6 +134,12 @@ impl OtlpLogPipeline { self.log_config = Some(log_config); self } + + /// Set the batch span processor configuration, and it will override the env vars. + pub fn with_batch_config(mut self, batch_config: opentelemetry_sdk::logs::BatchConfig) -> Self { + self.batch_config = Some(batch_config); + self + } } impl OtlpLogPipeline { @@ -143,6 +151,7 @@ impl OtlpLogPipeline { OtlpLogPipeline { exporter_builder: pipeline.into(), log_config: self.log_config, + batch_config: self.batch_config, } } } @@ -174,6 +183,7 @@ impl OtlpLogPipeline { self.exporter_builder.build_log_exporter()?, self.log_config, runtime, + self.batch_config, )) } } @@ -202,9 +212,14 @@ fn build_batch_with_exporter( exporter: LogExporter, log_config: Option, runtime: R, + batch_config: Option, ) -> opentelemetry_sdk::logs::Logger { - let mut provider_builder = - opentelemetry_sdk::logs::LoggerProvider::builder().with_batch_exporter(exporter, runtime); + let mut provider_builder = opentelemetry_sdk::logs::LoggerProvider::builder(); + let batch_processor = opentelemetry_sdk::logs::BatchLogProcessor::builder(exporter, runtime) + .with_batch_config(batch_config.unwrap_or_default()) + .build(); + provider_builder = provider_builder.with_log_processor(batch_processor); + if let Some(config) = log_config { provider_builder = provider_builder.with_config(config); } diff --git a/opentelemetry-sdk/src/logs/log_processor.rs b/opentelemetry-sdk/src/logs/log_processor.rs index cc3ffd5e13..6996618883 100644 --- a/opentelemetry-sdk/src/logs/log_processor.rs +++ b/opentelemetry-sdk/src/logs/log_processor.rs @@ -342,6 +342,11 @@ where BatchLogProcessorBuilder { config, ..self } } + /// Set the BatchConfig for [BatchLogProcessorBuilder] + pub fn with_batch_config(self, config: BatchConfig) -> Self { + BatchLogProcessorBuilder { config, ..self } + } + /// Build a batch processor pub fn build(self) -> BatchLogProcessor { BatchLogProcessor::new(Box::new(self.exporter), self.config, self.runtime) From 3c71d08b9b58a123c27c5a8786fb4301c283c58d Mon Sep 17 00:00:00 2001 From: Cosmin Constantin Lazar Date: Fri, 12 Jan 2024 06:49:17 +0100 Subject: [PATCH 02/13] Support configuration from env variables for log BatchConfig --- opentelemetry-sdk/src/logs/log_processor.rs | 72 +++++++++++++++++++-- 1 file changed, 66 insertions(+), 6 deletions(-) diff --git a/opentelemetry-sdk/src/logs/log_processor.rs b/opentelemetry-sdk/src/logs/log_processor.rs index 6996618883..791683fef1 100644 --- a/opentelemetry-sdk/src/logs/log_processor.rs +++ b/opentelemetry-sdk/src/logs/log_processor.rs @@ -13,12 +13,30 @@ use opentelemetry::{ global, logs::{LogError, LogResult}, }; -use std::sync::Mutex; +use std::{env, sync::Mutex}; use std::{ fmt::{self, Debug, Formatter}, + str::FromStr, time::Duration, }; +/// Delay interval between two consecutive exports. +const OTEL_BLRP_SCHEDULE_DELAY: &str = "OTEL_BLRP_SCHEDULE_DELAY"; +/// Default delay interval between two consecutive exports. +const OTEL_BLRP_SCHEDULE_DELAY_DEFAULT: u64 = 1_000; +/// Maximum allowed time to export data. +const OTEL_BLRP_EXPORT_TIMEOUT: &str = "OTEL_BLRP_EXPORT_TIMEOUT"; +/// Default maximum allowed time to export data. +const OTEL_BLRP_EXPORT_TIMEOUT_DEFAULT: u64 = 30_000; +/// Maximum queue size. +const OTEL_BLRP_MAX_QUEUE_SIZE: &str = "OTEL_BLRP_MAX_QUEUE_SIZE"; +/// Default maximum queue size. +const OTEL_BLRP_MAX_QUEUE_SIZE_DEFAULT: usize = 2_048; +/// Maximum batch size, must be less than or equal to OTEL_BLRP_MAX_QUEUE_SIZE. +const OTEL_BLRP_MAX_EXPORT_BATCH_SIZE: &str = "OTEL_BLRP_MAX_EXPORT_BATCH_SIZE"; +/// Default maximum batch size. +const OTEL_BLRP_MAX_EXPORT_BATCH_SIZE_DEFAULT: usize = 512; + /// The interface for plugging into a [`Logger`]. /// /// [`Logger`]: crate::logs::Logger @@ -281,12 +299,54 @@ pub struct BatchConfig { impl Default for BatchConfig { fn default() -> Self { - BatchConfig { - max_queue_size: 2_048, - scheduled_delay: Duration::from_millis(1_000), - max_export_batch_size: 512, - max_export_timeout: Duration::from_millis(30_000), + let mut config = BatchConfig { + max_queue_size: OTEL_BLRP_MAX_QUEUE_SIZE_DEFAULT, + scheduled_delay: Duration::from_millis(OTEL_BLRP_SCHEDULE_DELAY_DEFAULT), + max_export_batch_size: OTEL_BLRP_MAX_EXPORT_BATCH_SIZE_DEFAULT, + max_export_timeout: Duration::from_millis(OTEL_BLRP_EXPORT_TIMEOUT_DEFAULT), + }; + + if let Some(max_queue_size) = env::var(OTEL_BLRP_MAX_QUEUE_SIZE) + .ok() + .and_then(|queue_size| usize::from_str(&queue_size).ok()) + { + config.max_queue_size = max_queue_size; + } + + if let Some(max_export_batch_size) = env::var(OTEL_BLRP_MAX_EXPORT_BATCH_SIZE) + .ok() + .and_then(|batch_size| usize::from_str(&batch_size).ok()) + { + if max_export_batch_size > config.max_queue_size { + config.max_export_batch_size = config.max_queue_size; + } else { + config.max_export_batch_size = max_export_batch_size; + } + } + + // max export batch size must be less or equal to max queue size. + // we set max export batch size to max queue size if it's larger than max queue size. + if config.max_export_batch_size > config.max_queue_size { + config.max_export_batch_size = config.max_queue_size; + } + + if let Some(scheduled_delay) = env::var(OTEL_BLRP_SCHEDULE_DELAY) + .ok() + .or_else(|| env::var("OTEL_BLRP_SCHEDULE_DELAY_MILLIS").ok()) + .and_then(|delay| u64::from_str(&delay).ok()) + { + config.scheduled_delay = Duration::from_millis(scheduled_delay); } + + if let Some(max_export_timeout) = env::var(OTEL_BLRP_EXPORT_TIMEOUT) + .ok() + .or_else(|| env::var("OTEL_BLRP_EXPORT_TIMEOUT_MILLIS").ok()) + .and_then(|s| u64::from_str(&s).ok()) + { + config.max_export_timeout = Duration::from_millis(max_export_timeout); + } + + config } } From a170fc81a2e9db81c3105d3f96763d401bf4f875 Mon Sep 17 00:00:00 2001 From: Cosmin Constantin Lazar Date: Fri, 12 Jan 2024 07:14:39 +0100 Subject: [PATCH 03/13] Test default BatchConfig --- opentelemetry-sdk/src/logs/log_processor.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/opentelemetry-sdk/src/logs/log_processor.rs b/opentelemetry-sdk/src/logs/log_processor.rs index 791683fef1..e57b8a81e2 100644 --- a/opentelemetry-sdk/src/logs/log_processor.rs +++ b/opentelemetry-sdk/src/logs/log_processor.rs @@ -425,3 +425,18 @@ enum BatchMessage { /// Shut down the worker thread, push all logs in buffer to the backend. Shutdown(oneshot::Sender), } + +#[cfg(all(test, feature="testing", feature="logs"))] +mod tests{ + use std::time::Duration; + + #[test] + fn test_default_batch_config_adheres_to_specification(){ + let config = super::BatchConfig::default(); + + assert_eq!(config.scheduled_delay, Duration::from_millis(1000)); + assert_eq!(config.max_export_timeout, Duration::from_millis(30000)); + assert_eq!(config.max_queue_size, 2048); + assert_eq!(config.max_export_batch_size, 512); + } +} \ No newline at end of file From d8d34e86ea0e7e04a396a355764691cec29b505b Mon Sep 17 00:00:00 2001 From: Cosmin Constantin Lazar Date: Fri, 12 Jan 2024 21:14:42 +0100 Subject: [PATCH 04/13] Add tests for BatchConfig default --- opentelemetry-sdk/src/logs/log_processor.rs | 77 +++++++++++++++++++-- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/opentelemetry-sdk/src/logs/log_processor.rs b/opentelemetry-sdk/src/logs/log_processor.rs index e57b8a81e2..0e3626436f 100644 --- a/opentelemetry-sdk/src/logs/log_processor.rs +++ b/opentelemetry-sdk/src/logs/log_processor.rs @@ -426,12 +426,17 @@ enum BatchMessage { Shutdown(oneshot::Sender), } -#[cfg(all(test, feature="testing", feature="logs"))] -mod tests{ +#[cfg(all(test, feature = "testing", feature = "logs"))] +mod tests { use std::time::Duration; + use super::{ + OTEL_BLRP_EXPORT_TIMEOUT, OTEL_BLRP_MAX_EXPORT_BATCH_SIZE, OTEL_BLRP_MAX_QUEUE_SIZE, + OTEL_BLRP_SCHEDULE_DELAY, + }; + #[test] - fn test_default_batch_config_adheres_to_specification(){ + fn test_default_batch_config_adheres_to_specification() { let config = super::BatchConfig::default(); assert_eq!(config.scheduled_delay, Duration::from_millis(1000)); @@ -439,4 +444,68 @@ mod tests{ assert_eq!(config.max_queue_size, 2048); assert_eq!(config.max_export_batch_size, 512); } -} \ No newline at end of file + + #[test] + fn test_batch_config_configurable_by_env_vars() { + let env_vars = vec![ + (OTEL_BLRP_SCHEDULE_DELAY, Some("2000")), + (OTEL_BLRP_EXPORT_TIMEOUT, Some("60000")), + (OTEL_BLRP_MAX_QUEUE_SIZE, Some("4096")), + (OTEL_BLRP_MAX_EXPORT_BATCH_SIZE, Some("1024")), + ]; + + let config = temp_env::with_vars(env_vars, || super::BatchConfig::default()); + + assert_eq!(config.scheduled_delay, Duration::from_millis(2000)); + assert_eq!(config.max_export_timeout, Duration::from_millis(60000)); + assert_eq!(config.max_queue_size, 4096); + assert_eq!(config.max_export_batch_size, 1024); + } + + #[test] + fn test_batch_config_configurable_by_env_vars_millis() { + let env_vars = vec![ + ("OTEL_BLRP_SCHEDULE_DELAY_MILLIS", Some("3000")), + ("OTEL_BLRP_EXPORT_TIMEOUT_MILLIS", Some("70000")), + ]; + + let config = temp_env::with_vars(env_vars, || super::BatchConfig::default()); + + assert_eq!(config.scheduled_delay, Duration::from_millis(3000)); + assert_eq!(config.max_export_timeout, Duration::from_millis(70000)); + assert_eq!(config.max_queue_size, 2048); + assert_eq!(config.max_export_batch_size, 512); + } + + #[test] + fn test_batch_config_configurable_by_env_vars_precedence() { + let env_vars = vec![ + (OTEL_BLRP_SCHEDULE_DELAY, Some("2000")), + ("OTEL_BLRP_SCHEDULE_DELAY_MILLIS", Some("3000")), + (OTEL_BLRP_EXPORT_TIMEOUT, Some("60000")), + ("OTEL_BLRP_EXPORT_TIMEOUT_MILLIS", Some("70000")), + ]; + + let config = temp_env::with_vars(env_vars, || super::BatchConfig::default()); + + assert_eq!(config.scheduled_delay, Duration::from_millis(2000)); + assert_eq!(config.max_export_timeout, Duration::from_millis(60000)); + assert_eq!(config.max_queue_size, 2048); + assert_eq!(config.max_export_batch_size, 512); + } + + #[test] + fn test_batch_config_max_export_batch_size_validation(){ + let env_vars = vec![ + (OTEL_BLRP_MAX_QUEUE_SIZE, Some("256")), + (OTEL_BLRP_MAX_EXPORT_BATCH_SIZE, Some("1024")), + ]; + + let config = temp_env::with_vars(env_vars, || super::BatchConfig::default()); + + assert_eq!(config.max_queue_size, 256); + assert_eq!(config.max_export_batch_size, 256); + assert_eq!(config.scheduled_delay, Duration::from_millis(1000)); + assert_eq!(config.max_export_timeout, Duration::from_millis(30000)); + } +} From d12295f33d5be45a8256462506836e5b5c2d2051 Mon Sep 17 00:00:00 2001 From: Cosmin Constantin Lazar Date: Fri, 12 Jan 2024 21:46:14 +0100 Subject: [PATCH 05/13] Test BatchConfig field setters --- opentelemetry-sdk/src/logs/log_processor.rs | 67 ++++++++++++++++++--- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/opentelemetry-sdk/src/logs/log_processor.rs b/opentelemetry-sdk/src/logs/log_processor.rs index 0e3626436f..f49d7b8f45 100644 --- a/opentelemetry-sdk/src/logs/log_processor.rs +++ b/opentelemetry-sdk/src/logs/log_processor.rs @@ -350,6 +350,43 @@ impl Default for BatchConfig { } } +impl BatchConfig { + /// Set max_queue_size for [`BatchConfig`]. + /// It's the maximum queue size to buffer logs for delayed processing. + /// If the queue gets full it will drop the logs. + /// The default value of is 2048. + pub fn with_max_queue_size(mut self, max_queue_size: usize) -> Self { + self.max_queue_size = max_queue_size; + self + } + + /// Set scheduled_delay for [`BatchConfig`]. + /// It's the delay interval in milliseconds between two consecutive processing of batches. + /// The default value is 1000 milliseconds. + pub fn with_scheduled_delay(mut self, scheduled_delay: Duration) -> Self { + self.scheduled_delay = scheduled_delay; + self + } + + /// Set max_export_timeout for [`BatchConfig`]. + /// It's the maximum duration to export a batch of data. + /// The default value is 30000 milliseconds. + pub fn with_max_export_timeout(mut self, max_export_timeout: Duration) -> Self { + self.max_export_timeout = max_export_timeout; + self + } + + /// Set max_export_batch_size for [`BatchConfig`]. + /// It's the maximum number of logs to process in a single batch. If there are + /// more than one batch worth of logs then it processes multiple batches + /// of logs one batch after the other without any delay. + /// The default value is 512. + pub fn with_max_export_batch_size(mut self, max_export_batch_size: usize) -> Self { + self.max_export_batch_size = max_export_batch_size; + self + } +} + /// A builder for creating [`BatchLogProcessor`] instances. /// #[derive(Debug)] @@ -402,7 +439,7 @@ where BatchLogProcessorBuilder { config, ..self } } - /// Set the BatchConfig for [BatchLogProcessorBuilder] + /// Set the BatchConfig for [`BatchLogProcessorBuilder`] pub fn with_batch_config(self, config: BatchConfig) -> Self { BatchLogProcessorBuilder { config, ..self } } @@ -429,7 +466,7 @@ enum BatchMessage { #[cfg(all(test, feature = "testing", feature = "logs"))] mod tests { use std::time::Duration; - + use crate::logs::BatchConfig; use super::{ OTEL_BLRP_EXPORT_TIMEOUT, OTEL_BLRP_MAX_EXPORT_BATCH_SIZE, OTEL_BLRP_MAX_QUEUE_SIZE, OTEL_BLRP_SCHEDULE_DELAY, @@ -437,7 +474,7 @@ mod tests { #[test] fn test_default_batch_config_adheres_to_specification() { - let config = super::BatchConfig::default(); + let config = BatchConfig::default(); assert_eq!(config.scheduled_delay, Duration::from_millis(1000)); assert_eq!(config.max_export_timeout, Duration::from_millis(30000)); @@ -454,7 +491,7 @@ mod tests { (OTEL_BLRP_MAX_EXPORT_BATCH_SIZE, Some("1024")), ]; - let config = temp_env::with_vars(env_vars, || super::BatchConfig::default()); + let config = temp_env::with_vars(env_vars, || BatchConfig::default()); assert_eq!(config.scheduled_delay, Duration::from_millis(2000)); assert_eq!(config.max_export_timeout, Duration::from_millis(60000)); @@ -469,7 +506,7 @@ mod tests { ("OTEL_BLRP_EXPORT_TIMEOUT_MILLIS", Some("70000")), ]; - let config = temp_env::with_vars(env_vars, || super::BatchConfig::default()); + let config = temp_env::with_vars(env_vars, || BatchConfig::default()); assert_eq!(config.scheduled_delay, Duration::from_millis(3000)); assert_eq!(config.max_export_timeout, Duration::from_millis(70000)); @@ -486,7 +523,7 @@ mod tests { ("OTEL_BLRP_EXPORT_TIMEOUT_MILLIS", Some("70000")), ]; - let config = temp_env::with_vars(env_vars, || super::BatchConfig::default()); + let config = temp_env::with_vars(env_vars, || BatchConfig::default()); assert_eq!(config.scheduled_delay, Duration::from_millis(2000)); assert_eq!(config.max_export_timeout, Duration::from_millis(60000)); @@ -495,17 +532,31 @@ mod tests { } #[test] - fn test_batch_config_max_export_batch_size_validation(){ + fn test_batch_config_max_export_batch_size_validation() { let env_vars = vec![ (OTEL_BLRP_MAX_QUEUE_SIZE, Some("256")), (OTEL_BLRP_MAX_EXPORT_BATCH_SIZE, Some("1024")), ]; - let config = temp_env::with_vars(env_vars, || super::BatchConfig::default()); + let config = temp_env::with_vars(env_vars, || BatchConfig::default()); assert_eq!(config.max_queue_size, 256); assert_eq!(config.max_export_batch_size, 256); assert_eq!(config.scheduled_delay, Duration::from_millis(1000)); assert_eq!(config.max_export_timeout, Duration::from_millis(30000)); } + + #[test] + fn test_batch_config_with_fields() { + let batch = BatchConfig::default() + .with_max_export_batch_size(1) + .with_scheduled_delay(Duration::from_millis(2)) + .with_max_export_timeout(Duration::from_millis(3)) + .with_max_queue_size(4); + + assert_eq!(batch.max_export_batch_size, 1); + assert_eq!(batch.scheduled_delay, Duration::from_millis(2)); + assert_eq!(batch.max_export_timeout, Duration::from_millis(3)); + assert_eq!(batch.max_queue_size, 4); + } } From 055466ddd6a053648646016d65e33e3e8510512a Mon Sep 17 00:00:00 2001 From: Cosmin Constantin Lazar Date: Sun, 14 Jan 2024 20:26:28 +0100 Subject: [PATCH 06/13] Test constant names and values and use in test --- opentelemetry-sdk/src/logs/log_processor.rs | 49 ++++++++++++++------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/opentelemetry-sdk/src/logs/log_processor.rs b/opentelemetry-sdk/src/logs/log_processor.rs index f49d7b8f45..98d4cfe067 100644 --- a/opentelemetry-sdk/src/logs/log_processor.rs +++ b/opentelemetry-sdk/src/logs/log_processor.rs @@ -379,7 +379,7 @@ impl BatchConfig { /// Set max_export_batch_size for [`BatchConfig`]. /// It's the maximum number of logs to process in a single batch. If there are /// more than one batch worth of logs then it processes multiple batches - /// of logs one batch after the other without any delay. + /// of logs one batch after the other without any delay. /// The default value is 512. pub fn with_max_export_batch_size(mut self, max_export_batch_size: usize) -> Self { self.max_export_batch_size = max_export_batch_size; @@ -465,21 +465,40 @@ enum BatchMessage { #[cfg(all(test, feature = "testing", feature = "logs"))] mod tests { - use std::time::Duration; - use crate::logs::BatchConfig; use super::{ - OTEL_BLRP_EXPORT_TIMEOUT, OTEL_BLRP_MAX_EXPORT_BATCH_SIZE, OTEL_BLRP_MAX_QUEUE_SIZE, - OTEL_BLRP_SCHEDULE_DELAY, + BatchLogProcessor, OTEL_BLRP_EXPORT_TIMEOUT, OTEL_BLRP_MAX_EXPORT_BATCH_SIZE, + OTEL_BLRP_MAX_QUEUE_SIZE, OTEL_BLRP_SCHEDULE_DELAY, + }; + use crate::{ + logs::{BatchConfig, log_processor::{OTEL_BLRP_MAX_QUEUE_SIZE_DEFAULT, OTEL_BLRP_SCHEDULE_DELAY_DEFAULT, OTEL_BLRP_MAX_EXPORT_BATCH_SIZE_DEFAULT, OTEL_BLRP_EXPORT_TIMEOUT_DEFAULT}}, + runtime, + testing::logs::InMemoryLogsExporter, }; + use std::time::Duration; + + #[test] + fn test_default_const_values() { + assert_eq!(OTEL_BLRP_SCHEDULE_DELAY, "OTEL_BLRP_SCHEDULE_DELAY"); + assert_eq!(OTEL_BLRP_SCHEDULE_DELAY_DEFAULT, 1_000); + assert_eq!(OTEL_BLRP_EXPORT_TIMEOUT, "OTEL_BLRP_EXPORT_TIMEOUT"); + assert_eq!(OTEL_BLRP_EXPORT_TIMEOUT_DEFAULT, 30_000); + assert_eq!(OTEL_BLRP_MAX_QUEUE_SIZE, "OTEL_BLRP_MAX_QUEUE_SIZE"); + assert_eq!(OTEL_BLRP_MAX_QUEUE_SIZE_DEFAULT, 2_048); + assert_eq!( + OTEL_BLRP_MAX_EXPORT_BATCH_SIZE, + "OTEL_BLRP_MAX_EXPORT_BATCH_SIZE" + ); + assert_eq!(OTEL_BLRP_MAX_EXPORT_BATCH_SIZE_DEFAULT, 512); + } #[test] fn test_default_batch_config_adheres_to_specification() { let config = BatchConfig::default(); - assert_eq!(config.scheduled_delay, Duration::from_millis(1000)); - assert_eq!(config.max_export_timeout, Duration::from_millis(30000)); - assert_eq!(config.max_queue_size, 2048); - assert_eq!(config.max_export_batch_size, 512); + assert_eq!(config.scheduled_delay, Duration::from_millis(OTEL_BLRP_SCHEDULE_DELAY_DEFAULT)); + assert_eq!(config.max_export_timeout, Duration::from_millis(OTEL_BLRP_EXPORT_TIMEOUT_DEFAULT)); + assert_eq!(config.max_queue_size, OTEL_BLRP_MAX_QUEUE_SIZE_DEFAULT); + assert_eq!(config.max_export_batch_size, OTEL_BLRP_MAX_EXPORT_BATCH_SIZE_DEFAULT); } #[test] @@ -510,8 +529,8 @@ mod tests { assert_eq!(config.scheduled_delay, Duration::from_millis(3000)); assert_eq!(config.max_export_timeout, Duration::from_millis(70000)); - assert_eq!(config.max_queue_size, 2048); - assert_eq!(config.max_export_batch_size, 512); + assert_eq!(config.max_queue_size, OTEL_BLRP_MAX_QUEUE_SIZE_DEFAULT); + assert_eq!(config.max_export_batch_size, OTEL_BLRP_MAX_EXPORT_BATCH_SIZE_DEFAULT); } #[test] @@ -527,8 +546,8 @@ mod tests { assert_eq!(config.scheduled_delay, Duration::from_millis(2000)); assert_eq!(config.max_export_timeout, Duration::from_millis(60000)); - assert_eq!(config.max_queue_size, 2048); - assert_eq!(config.max_export_batch_size, 512); + assert_eq!(config.max_queue_size, OTEL_BLRP_MAX_QUEUE_SIZE_DEFAULT); + assert_eq!(config.max_export_batch_size, OTEL_BLRP_MAX_EXPORT_BATCH_SIZE_DEFAULT); } #[test] @@ -542,8 +561,8 @@ mod tests { assert_eq!(config.max_queue_size, 256); assert_eq!(config.max_export_batch_size, 256); - assert_eq!(config.scheduled_delay, Duration::from_millis(1000)); - assert_eq!(config.max_export_timeout, Duration::from_millis(30000)); + assert_eq!(config.scheduled_delay, Duration::from_millis(OTEL_BLRP_SCHEDULE_DELAY_DEFAULT)); + assert_eq!(config.max_export_timeout, Duration::from_millis(OTEL_BLRP_EXPORT_TIMEOUT_DEFAULT)); } #[test] From 0dba432c54f38dea6efda623b3116002131af354 Mon Sep 17 00:00:00 2001 From: Cosmin Constantin Lazar Date: Sun, 14 Jan 2024 20:26:56 +0100 Subject: [PATCH 07/13] Test batch log processor --- opentelemetry-sdk/src/logs/log_processor.rs | 34 +++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/opentelemetry-sdk/src/logs/log_processor.rs b/opentelemetry-sdk/src/logs/log_processor.rs index 98d4cfe067..6c970b3710 100644 --- a/opentelemetry-sdk/src/logs/log_processor.rs +++ b/opentelemetry-sdk/src/logs/log_processor.rs @@ -578,4 +578,38 @@ mod tests { assert_eq!(batch.max_export_timeout, Duration::from_millis(3)); assert_eq!(batch.max_queue_size, 4); } + + #[test] + fn test_build_batch_log_processor_builder() { + let mut env_vars = vec![ + (OTEL_BLRP_MAX_EXPORT_BATCH_SIZE, Some("500")), + (OTEL_BLRP_SCHEDULE_DELAY, Some("I am not number")), + (OTEL_BLRP_EXPORT_TIMEOUT, Some("2046")), + ]; + temp_env::with_vars(env_vars.clone(), || { + let builder = BatchLogProcessor::builder(InMemoryLogsExporter::default(), runtime::Tokio); + + assert_eq!(builder.config.max_export_batch_size, 500); + assert_eq!( + builder.config.scheduled_delay, + Duration::from_millis(OTEL_BLRP_SCHEDULE_DELAY_DEFAULT) + ); + assert_eq!( + builder.config.max_queue_size, + OTEL_BLRP_MAX_QUEUE_SIZE_DEFAULT + ); + assert_eq!( + builder.config.max_export_timeout, + Duration::from_millis(2046) + ); + }); + + env_vars.push((OTEL_BLRP_MAX_QUEUE_SIZE, Some("120"))); + + temp_env::with_vars(env_vars, || { + let builder = BatchLogProcessor::builder(InMemoryLogsExporter::default(), runtime::Tokio); + assert_eq!(builder.config.max_export_batch_size, 120); + assert_eq!(builder.config.max_queue_size, 120); + }); + } } From 4afc4ab715403abba3923d2bb92bc3ae2173eadb Mon Sep 17 00:00:00 2001 From: Cosmin Constantin Lazar Date: Sun, 14 Jan 2024 20:37:11 +0100 Subject: [PATCH 08/13] Remove duplicate logic --- opentelemetry-sdk/src/logs/log_processor.rs | 57 +++++++++++++++------ 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/opentelemetry-sdk/src/logs/log_processor.rs b/opentelemetry-sdk/src/logs/log_processor.rs index 6c970b3710..6d9c56c166 100644 --- a/opentelemetry-sdk/src/logs/log_processor.rs +++ b/opentelemetry-sdk/src/logs/log_processor.rs @@ -317,11 +317,7 @@ impl Default for BatchConfig { .ok() .and_then(|batch_size| usize::from_str(&batch_size).ok()) { - if max_export_batch_size > config.max_queue_size { - config.max_export_batch_size = config.max_queue_size; - } else { - config.max_export_batch_size = max_export_batch_size; - } + config.max_export_batch_size = max_export_batch_size; } // max export batch size must be less or equal to max queue size. @@ -470,7 +466,13 @@ mod tests { OTEL_BLRP_MAX_QUEUE_SIZE, OTEL_BLRP_SCHEDULE_DELAY, }; use crate::{ - logs::{BatchConfig, log_processor::{OTEL_BLRP_MAX_QUEUE_SIZE_DEFAULT, OTEL_BLRP_SCHEDULE_DELAY_DEFAULT, OTEL_BLRP_MAX_EXPORT_BATCH_SIZE_DEFAULT, OTEL_BLRP_EXPORT_TIMEOUT_DEFAULT}}, + logs::{ + log_processor::{ + OTEL_BLRP_EXPORT_TIMEOUT_DEFAULT, OTEL_BLRP_MAX_EXPORT_BATCH_SIZE_DEFAULT, + OTEL_BLRP_MAX_QUEUE_SIZE_DEFAULT, OTEL_BLRP_SCHEDULE_DELAY_DEFAULT, + }, + BatchConfig, + }, runtime, testing::logs::InMemoryLogsExporter, }; @@ -495,10 +497,19 @@ mod tests { fn test_default_batch_config_adheres_to_specification() { let config = BatchConfig::default(); - assert_eq!(config.scheduled_delay, Duration::from_millis(OTEL_BLRP_SCHEDULE_DELAY_DEFAULT)); - assert_eq!(config.max_export_timeout, Duration::from_millis(OTEL_BLRP_EXPORT_TIMEOUT_DEFAULT)); + assert_eq!( + config.scheduled_delay, + Duration::from_millis(OTEL_BLRP_SCHEDULE_DELAY_DEFAULT) + ); + assert_eq!( + config.max_export_timeout, + Duration::from_millis(OTEL_BLRP_EXPORT_TIMEOUT_DEFAULT) + ); assert_eq!(config.max_queue_size, OTEL_BLRP_MAX_QUEUE_SIZE_DEFAULT); - assert_eq!(config.max_export_batch_size, OTEL_BLRP_MAX_EXPORT_BATCH_SIZE_DEFAULT); + assert_eq!( + config.max_export_batch_size, + OTEL_BLRP_MAX_EXPORT_BATCH_SIZE_DEFAULT + ); } #[test] @@ -530,7 +541,10 @@ mod tests { assert_eq!(config.scheduled_delay, Duration::from_millis(3000)); assert_eq!(config.max_export_timeout, Duration::from_millis(70000)); assert_eq!(config.max_queue_size, OTEL_BLRP_MAX_QUEUE_SIZE_DEFAULT); - assert_eq!(config.max_export_batch_size, OTEL_BLRP_MAX_EXPORT_BATCH_SIZE_DEFAULT); + assert_eq!( + config.max_export_batch_size, + OTEL_BLRP_MAX_EXPORT_BATCH_SIZE_DEFAULT + ); } #[test] @@ -547,7 +561,10 @@ mod tests { assert_eq!(config.scheduled_delay, Duration::from_millis(2000)); assert_eq!(config.max_export_timeout, Duration::from_millis(60000)); assert_eq!(config.max_queue_size, OTEL_BLRP_MAX_QUEUE_SIZE_DEFAULT); - assert_eq!(config.max_export_batch_size, OTEL_BLRP_MAX_EXPORT_BATCH_SIZE_DEFAULT); + assert_eq!( + config.max_export_batch_size, + OTEL_BLRP_MAX_EXPORT_BATCH_SIZE_DEFAULT + ); } #[test] @@ -561,8 +578,14 @@ mod tests { assert_eq!(config.max_queue_size, 256); assert_eq!(config.max_export_batch_size, 256); - assert_eq!(config.scheduled_delay, Duration::from_millis(OTEL_BLRP_SCHEDULE_DELAY_DEFAULT)); - assert_eq!(config.max_export_timeout, Duration::from_millis(OTEL_BLRP_EXPORT_TIMEOUT_DEFAULT)); + assert_eq!( + config.scheduled_delay, + Duration::from_millis(OTEL_BLRP_SCHEDULE_DELAY_DEFAULT) + ); + assert_eq!( + config.max_export_timeout, + Duration::from_millis(OTEL_BLRP_EXPORT_TIMEOUT_DEFAULT) + ); } #[test] @@ -587,8 +610,9 @@ mod tests { (OTEL_BLRP_EXPORT_TIMEOUT, Some("2046")), ]; temp_env::with_vars(env_vars.clone(), || { - let builder = BatchLogProcessor::builder(InMemoryLogsExporter::default(), runtime::Tokio); - + let builder = + BatchLogProcessor::builder(InMemoryLogsExporter::default(), runtime::Tokio); + assert_eq!(builder.config.max_export_batch_size, 500); assert_eq!( builder.config.scheduled_delay, @@ -607,7 +631,8 @@ mod tests { env_vars.push((OTEL_BLRP_MAX_QUEUE_SIZE, Some("120"))); temp_env::with_vars(env_vars, || { - let builder = BatchLogProcessor::builder(InMemoryLogsExporter::default(), runtime::Tokio); + let builder = + BatchLogProcessor::builder(InMemoryLogsExporter::default(), runtime::Tokio); assert_eq!(builder.config.max_export_batch_size, 120); assert_eq!(builder.config.max_queue_size, 120); }); From fa7d4b5bea03edabbadc723c8fca175be31ecc45 Mon Sep 17 00:00:00 2001 From: Cosmin Constantin Lazar Date: Sun, 14 Jan 2024 21:09:17 +0100 Subject: [PATCH 09/13] Fix lint errors --- opentelemetry-sdk/src/logs/log_processor.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/opentelemetry-sdk/src/logs/log_processor.rs b/opentelemetry-sdk/src/logs/log_processor.rs index 6d9c56c166..35ff31a58d 100644 --- a/opentelemetry-sdk/src/logs/log_processor.rs +++ b/opentelemetry-sdk/src/logs/log_processor.rs @@ -521,7 +521,7 @@ mod tests { (OTEL_BLRP_MAX_EXPORT_BATCH_SIZE, Some("1024")), ]; - let config = temp_env::with_vars(env_vars, || BatchConfig::default()); + let config = temp_env::with_vars(env_vars, BatchConfig::default); assert_eq!(config.scheduled_delay, Duration::from_millis(2000)); assert_eq!(config.max_export_timeout, Duration::from_millis(60000)); @@ -536,7 +536,7 @@ mod tests { ("OTEL_BLRP_EXPORT_TIMEOUT_MILLIS", Some("70000")), ]; - let config = temp_env::with_vars(env_vars, || BatchConfig::default()); + let config = temp_env::with_vars(env_vars, BatchConfig::default); assert_eq!(config.scheduled_delay, Duration::from_millis(3000)); assert_eq!(config.max_export_timeout, Duration::from_millis(70000)); @@ -556,7 +556,7 @@ mod tests { ("OTEL_BLRP_EXPORT_TIMEOUT_MILLIS", Some("70000")), ]; - let config = temp_env::with_vars(env_vars, || BatchConfig::default()); + let config = temp_env::with_vars(env_vars, BatchConfig::default); assert_eq!(config.scheduled_delay, Duration::from_millis(2000)); assert_eq!(config.max_export_timeout, Duration::from_millis(60000)); @@ -574,7 +574,7 @@ mod tests { (OTEL_BLRP_MAX_EXPORT_BATCH_SIZE, Some("1024")), ]; - let config = temp_env::with_vars(env_vars, || BatchConfig::default()); + let config = temp_env::with_vars(env_vars, BatchConfig::default); assert_eq!(config.max_queue_size, 256); assert_eq!(config.max_export_batch_size, 256); From a28bf82018ab3b4c09119f78d00a0cbc16bcfee5 Mon Sep 17 00:00:00 2001 From: Cosmin Constantin Lazar Date: Tue, 16 Jan 2024 20:25:05 +0100 Subject: [PATCH 10/13] Test custom batch configuration --- opentelemetry-sdk/src/logs/log_processor.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/opentelemetry-sdk/src/logs/log_processor.rs b/opentelemetry-sdk/src/logs/log_processor.rs index 35ff31a58d..44018675b3 100644 --- a/opentelemetry-sdk/src/logs/log_processor.rs +++ b/opentelemetry-sdk/src/logs/log_processor.rs @@ -637,4 +637,23 @@ mod tests { assert_eq!(builder.config.max_queue_size, 120); }); } + + #[test] + fn test_build_batch_log_processor_builder_with_custom_config() { + let expected = BatchConfig::default() + .with_max_export_batch_size(1) + .with_scheduled_delay(Duration::from_millis(2)) + .with_max_export_timeout(Duration::from_millis(3)) + .with_max_queue_size(4); + + let builder = BatchLogProcessor::builder(InMemoryLogsExporter::default(), runtime::Tokio) + .with_batch_config(expected); + + let actual = &builder.config; + assert_eq!(actual.max_export_batch_size, 1); + assert_eq!(actual.scheduled_delay, Duration::from_millis(2)); + assert_eq!(actual.max_export_timeout, Duration::from_millis(3)); + assert_eq!(actual.max_queue_size, 4); + } + } From d37b369fbb9fe54e41cd46306030f085335948cd Mon Sep 17 00:00:00 2001 From: Cosmin Constantin Lazar Date: Tue, 16 Jan 2024 21:05:58 +0100 Subject: [PATCH 11/13] Fix spell error :) Fixing copy paste leftovers --- opentelemetry-otlp/src/logs.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opentelemetry-otlp/src/logs.rs b/opentelemetry-otlp/src/logs.rs index 28424014df..bb69b06238 100644 --- a/opentelemetry-otlp/src/logs.rs +++ b/opentelemetry-otlp/src/logs.rs @@ -135,7 +135,7 @@ impl OtlpLogPipeline { self } - /// Set the batch span processor configuration, and it will override the env vars. + /// Set the batch log processor configuration, and it will override the env vars. pub fn with_batch_config(mut self, batch_config: opentelemetry_sdk::logs::BatchConfig) -> Self { self.batch_config = Some(batch_config); self @@ -169,7 +169,7 @@ impl OtlpLogPipeline { )) } - /// Install the configured log exporter and a batch span processor using the + /// Install the configured log exporter and a batch log processor using the /// specified runtime. /// /// Returns a [`Logger`] with the name `opentelemetry-otlp` and the current crate version. From 1f0e6a2b90a4291d0f242cdd4b656d43d6f7fe75 Mon Sep 17 00:00:00 2001 From: Cosmin Constantin Lazar Date: Tue, 16 Jan 2024 21:15:37 +0100 Subject: [PATCH 12/13] Fix lint errors --- opentelemetry-sdk/src/logs/log_processor.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/opentelemetry-sdk/src/logs/log_processor.rs b/opentelemetry-sdk/src/logs/log_processor.rs index 44018675b3..f62e68413b 100644 --- a/opentelemetry-sdk/src/logs/log_processor.rs +++ b/opentelemetry-sdk/src/logs/log_processor.rs @@ -655,5 +655,4 @@ mod tests { assert_eq!(actual.max_export_timeout, Duration::from_millis(3)); assert_eq!(actual.max_queue_size, 4); } - } From 380d9e482a906fd1ecd3021c78ad270dda3ca920 Mon Sep 17 00:00:00 2001 From: Cosmin Constantin Lazar Date: Fri, 19 Jan 2024 19:48:03 +0100 Subject: [PATCH 13/13] Add changelog --- opentelemetry-sdk/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/opentelemetry-sdk/CHANGELOG.md b/opentelemetry-sdk/CHANGELOG.md index 363e71e5d7..a68a812d9c 100644 --- a/opentelemetry-sdk/CHANGELOG.md +++ b/opentelemetry-sdk/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added - [#1410](https://github.com/open-telemetry/opentelemetry-rust/pull/1410) Add experimental synchronous gauge +- [#1471](https://github.com/open-telemetry/opentelemetry-rust/pull/1471) Configure batch log record processor via [`OTEL_BLRP_*`](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/sdk-environment-variables.md#batch-logrecord-processor) environment variables and via `OtlpLogPipeline::with_batch_config` ### Changed