From 088273f009208f0198214c89887ccf9defef63aa Mon Sep 17 00:00:00 2001 From: Thomas Eizinger Date: Mon, 13 Jan 2025 19:26:25 +0100 Subject: [PATCH] feat(clients): reduce memory usage of background logger thread (#7748) In order to not block the main thread, `connlib` uses a background thread to write log files to disk. By default, the channel with this background thread can hold 128_000 items (https://docs.rs/tracing-appender/latest/tracing_appender/non_blocking/constant.DEFAULT_BUFFERED_LINES_LIMIT.html). This results in a significant chunk of memory being allocated that we don't necessarily need. --- rust/logging/src/file.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/rust/logging/src/file.rs b/rust/logging/src/file.rs index 3f7ce156b..7dea564b1 100644 --- a/rust/logging/src/file.rs +++ b/rust/logging/src/file.rs @@ -27,6 +27,17 @@ use tracing_subscriber::Layer; const LOG_FILE_BASE_NAME: &str = "connlib"; pub const TIME_FORMAT: &str = "[year]-[month]-[day]-[hour]-[minute]-[second]"; +/// How many lines we will at most buffer in the channel with the background thread that writes to disk. +/// +/// We don't need this number to be very high because: +/// a. `connlib` doesn't actually log a lot +/// b. The background continuously reads from the channel and writes to disk. +/// +/// This buffer only needs to be able to handle bursts. +/// +/// As per docs on [`tracing_appender::non_blocking::DEFAULT_BUFFERED_LINES_LIMIT`], this is a power of 2. +const MAX_BUFFERED_LINES: usize = 1024; + /// Create a new file logger layer. pub fn layer(log_dir: &Path) -> (Box + Send + Sync + 'static>, Handle) where @@ -62,7 +73,9 @@ fn new_appender(directory: PathBuf, file_extension: &'static str) -> (NonBlockin file_extension, }; - let (non_blocking, guard) = tracing_appender::non_blocking(appender); + let (non_blocking, guard) = tracing_appender::non_blocking::NonBlockingBuilder::default() + .buffered_lines_limit(MAX_BUFFERED_LINES) + .finish(appender); (non_blocking, guard) }