From e414b8467a729e0bf9b167d8b6cebe9e7c5d4c65 Mon Sep 17 00:00:00 2001 From: xfocus3 Date: Sun, 31 May 2026 19:34:18 +0100 Subject: [PATCH] fix(gcp_cloud_storage sink): apply timezone to key_prefix strftime The `gcp_cloud_storage` sink built its key partitioner from the raw `key_prefix` template without applying the configured `timezone` offset, so strftime specifiers in `key_prefix` (e.g. `%Y%m%d/`) were always rendered in UTC. The `aws_s3` sink already applies the offset via `Template::with_tz_offset`; this change mirrors that behavior so date-based object partitioning honors the `timezone` (or global `timezone`) setting. Adds a regression test covering positive/negative/none offsets around the UTC day boundary. Closes #25090 --- .../25090_gcs_key_prefix_timezone.fix.md | 3 + src/sinks/gcp/cloud_storage.rs | 61 +++++++++++++++++-- 2 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 changelog.d/25090_gcs_key_prefix_timezone.fix.md diff --git a/changelog.d/25090_gcs_key_prefix_timezone.fix.md b/changelog.d/25090_gcs_key_prefix_timezone.fix.md new file mode 100644 index 0000000000000..97730fffdf442 --- /dev/null +++ b/changelog.d/25090_gcs_key_prefix_timezone.fix.md @@ -0,0 +1,3 @@ +The `gcp_cloud_storage` sink now applies the configured `timezone` option (or the global `timezone`) to `strftime` specifiers in `key_prefix`, matching the behavior of the `aws_s3` sink. Previously, date-based partitioning in `key_prefix` (for example `key_prefix = "%Y%m%d/"`) was always rendered in UTC and ignored the `timezone` setting, so object paths could land in the wrong dated directory around the UTC day boundary. + +authors: xfocus3 diff --git a/src/sinks/gcp/cloud_storage.rs b/src/sinks/gcp/cloud_storage.rs index c63abdb6689f9..d37357dd346fd 100644 --- a/src/sinks/gcp/cloud_storage.rs +++ b/src/sinks/gcp/cloud_storage.rs @@ -302,7 +302,12 @@ impl GcsSinkConfig { let batch_settings = self.batch.into_batcher_settings()?; - let partitioner = self.key_partitioner()?; + let offset = self + .timezone + .or(cx.globals.timezone) + .and_then(timezone_to_offset); + + let partitioner = self.key_partitioner(offset)?; let protocol = get_http_scheme_from_uri(&base_url.parse::().unwrap()); @@ -317,10 +322,11 @@ impl GcsSinkConfig { Ok(VectorSink::from_event_streamsink(sink)) } - fn key_partitioner(&self) -> crate::Result { + fn key_partitioner(&self, offset: Option) -> crate::Result { Ok(KeyPartitioner::new( Template::try_from(self.key_prefix.as_deref().unwrap_or("date=%F/")) - .context(KeyPrefixTemplateSnafu)?, + .context(KeyPrefixTemplateSnafu)? + .with_tz_offset(offset), None, )) } @@ -554,7 +560,7 @@ mod tests { ..default_config((None::, TextSerializerConfig::default()).into()) }; let key = sink_config - .key_partitioner() + .key_partitioner(None) .unwrap() .partition(&Event::Log(event)) .expect("key wasn't provided"); @@ -562,6 +568,51 @@ mod tests { assert_eq!(key, "key: value"); } + #[test] + fn gcs_key_prefix_honors_timezone_offset() { + // Regression test for #25090: the `timezone` option must be applied to + // strftime specifiers in `key_prefix`, matching the `aws_s3` sink. + use chrono::{FixedOffset, TimeZone, Utc}; + + let mut event = LogEvent::from("message"); + // 2026-04-01T00:30:00Z is still 2026-03-31 in a UTC-08:00 zone and + // already 2026-04-01 in a UTC+08:00 zone. + let timestamp = Utc.with_ymd_and_hms(2026, 4, 1, 0, 30, 0).unwrap(); + event.insert("timestamp", timestamp); + + let sink_config = GcsSinkConfig { + key_prefix: Some("%Y%m%d/".into()), + ..default_config((None::, TextSerializerConfig::default()).into()) + }; + + // Positive offset (e.g. Asia/Taipei, UTC+08:00) rolls the date forward. + let plus_eight = FixedOffset::east_opt(8 * 3600); + let key = sink_config + .key_partitioner(plus_eight) + .unwrap() + .partition(&Event::Log(event.clone())) + .expect("key wasn't provided"); + assert_eq!(key, "20260401/"); + + // Negative offset (e.g. America/Los_Angeles, UTC-08:00) keeps the + // previous calendar day. + let minus_eight = FixedOffset::west_opt(8 * 3600); + let key = sink_config + .key_partitioner(minus_eight) + .unwrap() + .partition(&Event::Log(event.clone())) + .expect("key wasn't provided"); + assert_eq!(key, "20260331/"); + + // No timezone configured falls back to UTC. + let key = sink_config + .key_partitioner(None) + .unwrap() + .partition(&Event::Log(event)) + .expect("key wasn't provided"); + assert_eq!(key, "20260401/"); + } + fn request_settings(sink_config: &GcsSinkConfig, context: SinkContext) -> RequestSettings { RequestSettings::new(sink_config, context).expect("Could not create request settings") } @@ -584,7 +635,7 @@ mod tests { }; let log = LogEvent::default().into(); let key = sink_config - .key_partitioner() + .key_partitioner(None) .unwrap() .partition(&log) .expect("key wasn't provided");