Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions changelog.d/25090_gcs_key_prefix_timezone.fix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
The `gcp_cloud_storage` sink now applies the configured `timezone` option (or the global `timezone`) to `strftime` specifiers in `key_prefix`, matching the behavior of the `aws_s3` sink. Previously, date-based partitioning in `key_prefix` (for example `key_prefix = "%Y%m%d/"`) was always rendered in UTC and ignored the `timezone` setting, so object paths could land in the wrong dated directory around the UTC day boundary.

authors: xfocus3
61 changes: 56 additions & 5 deletions src/sinks/gcp/cloud_storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,12 @@ impl GcsSinkConfig {

let batch_settings = self.batch.into_batcher_settings()?;

let partitioner = self.key_partitioner()?;
let offset = self
.timezone
.or(cx.globals.timezone)
.and_then(timezone_to_offset);
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve DST-aware timezone rules

When timezone is a named zone with daylight saving time, this converts it once with timezone_to_offset, which freezes the offset based on Utc::now() at sink build time before Template formats each event timestamp. In a sink configured with America/Los_Angeles, if Vector starts during PDT (-07) but renders a winter event such as 2026-12-01T07:30:00Z (still Nov 30 in PST -08), the key prefix is formatted using -07 and lands under Dec 1 instead of Nov 30. The template needs the named timezone, or an offset computed from the event timestamp, rather than a build-time fixed offset.

Useful? React with 👍 / 👎.


let partitioner = self.key_partitioner(offset)?;

let protocol = get_http_scheme_from_uri(&base_url.parse::<Uri>().unwrap());

Expand All @@ -317,10 +322,11 @@ impl GcsSinkConfig {
Ok(VectorSink::from_event_streamsink(sink))
}

fn key_partitioner(&self) -> crate::Result<KeyPartitioner> {
fn key_partitioner(&self, offset: Option<FixedOffset>) -> crate::Result<KeyPartitioner> {
Ok(KeyPartitioner::new(
Template::try_from(self.key_prefix.as_deref().unwrap_or("date=%F/"))
.context(KeyPrefixTemplateSnafu)?,
.context(KeyPrefixTemplateSnafu)?
.with_tz_offset(offset),
None,
))
}
Expand Down Expand Up @@ -554,14 +560,59 @@ mod tests {
..default_config((None::<FramingConfig>, TextSerializerConfig::default()).into())
};
let key = sink_config
.key_partitioner()
.key_partitioner(None)
.unwrap()
.partition(&Event::Log(event))
.expect("key wasn't provided");

assert_eq!(key, "key: value");
}

#[test]
fn gcs_key_prefix_honors_timezone_offset() {
// Regression test for #25090: the `timezone` option must be applied to
// strftime specifiers in `key_prefix`, matching the `aws_s3` sink.
use chrono::{FixedOffset, TimeZone, Utc};

let mut event = LogEvent::from("message");
// 2026-04-01T00:30:00Z is still 2026-03-31 in a UTC-08:00 zone and
// already 2026-04-01 in a UTC+08:00 zone.
let timestamp = Utc.with_ymd_and_hms(2026, 4, 1, 0, 30, 0).unwrap();
event.insert("timestamp", timestamp);

let sink_config = GcsSinkConfig {
key_prefix: Some("%Y%m%d/".into()),
..default_config((None::<FramingConfig>, TextSerializerConfig::default()).into())
};

// Positive offset (e.g. Asia/Taipei, UTC+08:00) rolls the date forward.
let plus_eight = FixedOffset::east_opt(8 * 3600);
let key = sink_config
.key_partitioner(plus_eight)
.unwrap()
.partition(&Event::Log(event.clone()))
.expect("key wasn't provided");
assert_eq!(key, "20260401/");

// Negative offset (e.g. America/Los_Angeles, UTC-08:00) keeps the
// previous calendar day.
let minus_eight = FixedOffset::west_opt(8 * 3600);
let key = sink_config
.key_partitioner(minus_eight)
.unwrap()
.partition(&Event::Log(event.clone()))
.expect("key wasn't provided");
assert_eq!(key, "20260331/");

// No timezone configured falls back to UTC.
let key = sink_config
.key_partitioner(None)
.unwrap()
.partition(&Event::Log(event))
.expect("key wasn't provided");
assert_eq!(key, "20260401/");
}

fn request_settings(sink_config: &GcsSinkConfig, context: SinkContext) -> RequestSettings {
RequestSettings::new(sink_config, context).expect("Could not create request settings")
}
Expand All @@ -584,7 +635,7 @@ mod tests {
};
let log = LogEvent::default().into();
let key = sink_config
.key_partitioner()
.key_partitioner(None)
.unwrap()
.partition(&log)
.expect("key wasn't provided");
Expand Down
Loading