Add compression ratio calculation and per-column compression stats (#18184)

pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java

@@ -116,6 +116,18 @@ public enum ControllerGauge implements AbstractMetrics.Gauge {
   // Percentage of segments we failed to get size for
   TABLE_STORAGE_EST_MISSING_SEGMENT_PERCENT("TableStorageEstMissingSegmentPercent", false),
 
+  // Forward index compression ratio scaled by 100 (e.g., 4.5x ratio → 450). Divide by 100 to get actual ratio.
+  TABLE_COMPRESSION_RATIO_PERCENT("TableCompressionRatioPercent", false),
+
+  // Raw (uncompressed) forward index size per replica
+  TABLE_RAW_FORWARD_INDEX_SIZE_PER_REPLICA("TableRawForwardIndexSizePerReplica", false),
+
+  // Compressed forward index size per replica
+  TABLE_COMPRESSED_FORWARD_INDEX_SIZE_PER_REPLICA("TableCompressedForwardIndexSizePerReplica", false),
+
+  // Size per replica broken down by storage tier
+  TABLE_TIERED_STORAGE_SIZE("TableTieredStorageSize", false),
+
   // Number of scheduled Cron jobs
   CRON_SCHEDULER_JOB_SCHEDULED("cronSchedulerJobScheduled", false),
 

pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ColumnCompressionStatsInfo.java

@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.restlet.resources;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+
+/// Per-column forward index compression statistics, as reported by each server for a given segment.
+///
+/// For raw (non-dictionary) columns, both `rawIngestSizeInBytes` and `onDiskSizeInBytes` are populated
+/// and `codec` reflects the compression algorithm. For dictionary-encoded columns, `codec` is
+/// {@link #CODEC_DICT_ENCODED} and only `onDiskSizeInBytes` reflects the on-disk forward index size. Columns without a
+/// forward index (forward-index-disabled) are excluded entirely.
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ColumnCompressionStatsInfo {
+  /// Sentinel codec value for dictionary-encoded columns.
+  public static final String CODEC_DICT_ENCODED = "DICT_ENCODED";
+
+  private final String _column;
+
+  /// Total uncompressed byte size of values written to the forward index during segment creation.
+  /// `-1` (sentinel) when unavailable — e.g. for dictionary-encoded columns or old segments built before
+  /// stats tracking was enabled.
+  private final long _rawIngestSizeInBytes;
+
+  /// On-disk byte size of the forward index file for this column in this segment.
+  private final long _onDiskSizeInBytes;
+
+  /// Compression ratio (`rawIngestSizeInBytes / onDiskSizeInBytes`). `0` when unavailable.
+  private final double _compressionRatio;
+
+  /// Compression codec name (e.g. `"ZSTANDARD"`, `"LZ4"`, `"SNAPPY"`, `"PASS_THROUGH"`),
+  /// {@link #CODEC_DICT_ENCODED} for dictionary-encoded columns, or `"MIXED"` when segments in the same table use
+  /// different codecs for this column.
+  private final String _codec;
+
+  /// Names of all indexes present on this column in this segment (e.g. `["forward_index", "inverted_index"]`).
+  private final List<String> _indexes;
+
+  /// Per-codec breakdown. Null unless codec is `"MIXED"` — in that case maps codec name to sizes and segment count.
+  private final Map<String, CodecBreakdownEntry> _codecBreakdown;
+
+  @JsonCreator
+  public ColumnCompressionStatsInfo(
+      @JsonProperty("column") String column,
+      @JsonProperty("rawIngestSizeInBytes") long rawIngestSizeInBytes,
+      @JsonProperty("onDiskSizeInBytes") long onDiskSizeInBytes,
+      @JsonProperty("compressionRatio") double compressionRatio,
+      @JsonProperty("codec") @Nullable String codec,
+      @JsonProperty("indexes") @Nullable List<String> indexes,
+      @JsonProperty("codecBreakdown") @Nullable Map<String, CodecBreakdownEntry> codecBreakdown) {
+    _column = column;
+    _rawIngestSizeInBytes = rawIngestSizeInBytes;
+    _onDiskSizeInBytes = onDiskSizeInBytes;
+    _compressionRatio = compressionRatio;
+    _codec = codec;
+    _indexes = indexes;
+    _codecBreakdown = codecBreakdown;
+  }
+
+  public String getColumn() {
+    return _column;
+  }
+
+  public long getRawIngestSizeInBytes() {
+    return _rawIngestSizeInBytes;
+  }
+
+  public long getOnDiskSizeInBytes() {
+    return _onDiskSizeInBytes;
+  }
+
+  public double getCompressionRatio() {
+    return _compressionRatio;
+  }
+
+  @Nullable
+  public String getCodec() {
+    return _codec;
+  }
+
+  @Nullable
+  @JsonInclude(JsonInclude.Include.NON_NULL)
+  public List<String> getIndexes() {
+    return _indexes;
+  }
+
+  @Nullable
+  @JsonInclude(JsonInclude.Include.NON_NULL)
+  public Map<String, CodecBreakdownEntry> getCodecBreakdown() {
+    return _codecBreakdown;
+  }
+
+  /// Per-codec breakdown entry in the {@code codecBreakdown} map. Only present when {@code codec="MIXED"}.
+  @JsonIgnoreProperties(ignoreUnknown = true)
+  public static class CodecBreakdownEntry {
+    private final int _segments;
+    private final long _rawIngestSizeInBytes;
+    private final long _onDiskSizeInBytes;
+
+    @JsonCreator
+    public CodecBreakdownEntry(
+        @JsonProperty("segments") int segments,
+        @JsonProperty("rawIngestSizeInBytes") long rawIngestSizeInBytes,
+        @JsonProperty("onDiskSizeInBytes") long onDiskSizeInBytes) {
+      _segments = segments;
+      _rawIngestSizeInBytes = rawIngestSizeInBytes;
+      _onDiskSizeInBytes = onDiskSizeInBytes;
+    }
+
+    public int getSegments() {
+      return _segments;
+    }
+
+    public long getRawIngestSizeInBytes() {
+      return _rawIngestSizeInBytes;
+    }
+
+    public long getOnDiskSizeInBytes() {
+      return _onDiskSizeInBytes;
+    }
+  }
+}

pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/CompressionStatsSummary.java

@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.restlet.resources;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+
+/// Table-level compression statistics summary, aggregated across all servers for a sub-table type
+/// (offline or realtime). Reported under the `compressionStats` key in the
+/// `GET /tables/{tableName}/size` response.
+///
+/// Sizes are "per replica" — each segment's contribution is the maximum reported across its replicas,
+/// so the total reflects a single logical copy of the data rather than the physical replication factor.
+///
+/// `isPartialCoverage` is true when one or more segments lack stats (e.g. built before
+/// `compressionStatsEnabled` was set), meaning the ratio is computed from a subset of segments only.
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class CompressionStatsSummary {
+  /// Sum of per-replica max raw ingest forward index sizes across all segments that have stats.
+  private final long _rawIngestSizePerReplicaInBytes;
+
+  /// Sum of per-replica max on-disk forward index sizes across all segments that have stats.
+  private final long _onDiskSizePerReplicaInBytes;
+
+  /// Overall ratio of raw to compressed size (`rawSize / compressedSize`). `0` when no segments have stats.
+  private final double _compressionRatio;
+
+  /// Number of segments that have compression stats (built with `compressionStatsEnabled=true`).
+  private final int _segmentsWithStats;
+
+  /// Total number of segments in the sub-table.
+  private final int _totalSegments;
+
+  /// True when `segmentsWithStats < totalSegments`, meaning the ratio covers only a subset of segments.
+  private final boolean _isPartialCoverage;
+
+  @JsonCreator
+  public CompressionStatsSummary(
+      @JsonProperty("rawIngestSizePerReplicaInBytes") long rawIngestSizePerReplicaInBytes,
+      @JsonProperty("onDiskSizePerReplicaInBytes") long onDiskSizePerReplicaInBytes,
+      @JsonProperty("compressionRatio") double compressionRatio,
+      @JsonProperty("segmentsWithStats") int segmentsWithStats,
+      @JsonProperty("totalSegments") int totalSegments,
+      @JsonProperty("isPartialCoverage") boolean isPartialCoverage) {
+    _rawIngestSizePerReplicaInBytes = rawIngestSizePerReplicaInBytes;
+    _onDiskSizePerReplicaInBytes = onDiskSizePerReplicaInBytes;
+    _compressionRatio = compressionRatio;
+    _segmentsWithStats = segmentsWithStats;
+    _totalSegments = totalSegments;
+    _isPartialCoverage = isPartialCoverage;
+  }
+
+  public long getRawIngestSizePerReplicaInBytes() {
+    return _rawIngestSizePerReplicaInBytes;
+  }
+
+  public long getOnDiskSizePerReplicaInBytes() {
+    return _onDiskSizePerReplicaInBytes;
+  }
+
+  public double getCompressionRatio() {
+    return _compressionRatio;
+  }
+
+  public int getSegmentsWithStats() {
+    return _segmentsWithStats;
+  }
+
+  public int getTotalSegments() {
+    return _totalSegments;
+  }
+
+  @JsonProperty("isPartialCoverage")
+  public boolean isPartialCoverage() {
+    return _isPartialCoverage;
+  }
+}

pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/SegmentSizeInfo.java

@@ -21,18 +21,44 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Map;
+import javax.annotation.Nullable;
 
 
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class SegmentSizeInfo {
   private final String _segmentName;
   private final long _diskSizeInBytes;
+  /// Segment-level aggregate raw ingest size across all columns that have stats. `-1` when unavailable.
+  private final long _rawIngestSizeBytes;
+  /// Segment-level aggregate on-disk size across all columns that have stats. `-1` when unavailable.
+  private final long _onDiskSizeBytes;
+  private final String _tier;
+  private final Map<String, ColumnCompressionStatsInfo> _columnCompressionStats;
+
+  public SegmentSizeInfo(String segmentName, long sizeBytes) {
+    this(segmentName, sizeBytes, -1, -1, null, null);
+  }
+
+  public SegmentSizeInfo(String segmentName, long sizeBytes, long rawIngestSizeBytes,
+      long onDiskSizeBytes, @Nullable String tier) {
+    this(segmentName, sizeBytes, rawIngestSizeBytes, onDiskSizeBytes, tier, null);
+  }
 
   @JsonCreator
   public SegmentSizeInfo(@JsonProperty("segmentName") String segmentName,
-      @JsonProperty("diskSizeInBytes") long sizeBytes) {
+      @JsonProperty("diskSizeInBytes") long sizeBytes,
+      @JsonProperty("rawIngestSizeBytes") long rawIngestSizeBytes,
+      @JsonProperty("onDiskSizeBytes") long onDiskSizeBytes,
+      @JsonProperty("tier") @Nullable String tier,
+      @JsonProperty("columnCompressionStats") @Nullable Map<String, ColumnCompressionStatsInfo>
+          columnCompressionStats) {
     _segmentName = segmentName;
     _diskSizeInBytes = sizeBytes;
+    _rawIngestSizeBytes = rawIngestSizeBytes;
+    _onDiskSizeBytes = onDiskSizeBytes;
+    _tier = tier;
+    _columnCompressionStats = columnCompressionStats;
   }
 
   public String getSegmentName() {
@@ -43,6 +69,24 @@ public long getDiskSizeInBytes() {
     return _diskSizeInBytes;
   }
 
+  public long getRawIngestSizeBytes() {
+    return _rawIngestSizeBytes;
+  }
+
+  public long getOnDiskSizeBytes() {
+    return _onDiskSizeBytes;
+  }
+
+  @Nullable
+  public String getTier() {
+    return _tier;
+  }
+
+  @Nullable
+  public Map<String, ColumnCompressionStatsInfo> getColumnCompressionStats() {
+    return _columnCompressionStats;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {

pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/StorageBreakdownInfo.java

@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.restlet.resources;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Map;
+
+
+/// Storage breakdown by tier, reported under the `storageBreakdown` key in the
+/// `GET /tables/{tableName}/size` and `GET /tables/{tableName}/metadata` responses.
+/// Maps tier name (e.g. `"default"`, `"hotTier"`) to segment count and per-replica size.
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class StorageBreakdownInfo {
+
+  private final Map<String, TierInfo> _tiers;
+
+  @JsonCreator
+  public StorageBreakdownInfo(@JsonProperty("tiers") Map<String, TierInfo> tiers) {
+    _tiers = tiers;
+  }
+
+  public Map<String, TierInfo> getTiers() {
+    return _tiers;
+  }
+
+  /// Segment count and per-replica on-disk size for a single storage tier.
+  @JsonIgnoreProperties(ignoreUnknown = true)
+  public static class TierInfo {
+    private final int _count;
+    private final long _sizePerReplicaInBytes;
+
+    @JsonCreator
+    public TierInfo(@JsonProperty("count") int count,
+        @JsonProperty("sizePerReplicaInBytes") long sizePerReplicaInBytes) {
+      _count = count;
+      _sizePerReplicaInBytes = sizePerReplicaInBytes;
+    }
+
+    public int getCount() {
+      return _count;
+    }
+
+    public long getSizePerReplicaInBytes() {
+      return _sizePerReplicaInBytes;
+    }
+  }
+}