Skip to content

Commit f538ace

Browse files
committed
wip: compression with orderby and segmentby attributes
1 parent 4c7b549 commit f538ace

23 files changed

Lines changed: 2398 additions & 12 deletions

src/Eftdb.Design/Scaffolding/HypertableAnnotationApplier.cs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,18 @@ public void ApplyAnnotations(DatabaseTable table, object featureInfo)
2727
table[HypertableAnnotations.ChunkSkipColumns] = string.Join(",", info.ChunkSkipColumns);
2828
}
2929

30+
// Apply SegmentBy annotation if present
31+
if (info.CompressionSegmentBy.Count > 0)
32+
{
33+
table[HypertableAnnotations.CompressionSegmentBy] = string.Join(", ", info.CompressionSegmentBy);
34+
}
35+
36+
// Apply OrderBy annotation if present
37+
if (info.CompressionOrderBy.Count > 0)
38+
{
39+
table[HypertableAnnotations.CompressionOrderBy] = string.Join(", ", info.CompressionOrderBy);
40+
}
41+
3042
if (info.AdditionalDimensions.Count > 0)
3143
{
3244
table[HypertableAnnotations.AdditionalDimensions] = JsonSerializer.Serialize(info.AdditionalDimensions);

src/Eftdb.Design/Scaffolding/HypertableScaffoldingExtractor.cs

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ public sealed record HypertableInfo(
1313
string TimeColumnName,
1414
string ChunkTimeInterval,
1515
bool CompressionEnabled,
16+
List<string> CompressionSegmentBy,
17+
List<string> CompressionOrderBy,
1618
List<string> ChunkSkipColumns,
1719
List<Dimension> AdditionalDimensions
1820
);
@@ -32,6 +34,7 @@ List<Dimension> AdditionalDimensions
3234

3335
GetHypertableSettings(connection, hypertables, compressionSettings);
3436
GetChunkSkipColumns(connection, hypertables);
37+
GetCompressionConfiguration(connection, hypertables);
3538

3639
// Convert to object dictionary to match interface
3740
return hypertables.ToDictionary(
@@ -99,6 +102,8 @@ FROM timescaledb_information.dimensions
99102
TimeColumnName: columnName,
100103
ChunkTimeInterval: chunkInterval.ToString(),
101104
CompressionEnabled: compressionEnabled,
105+
CompressionSegmentBy: [],
106+
CompressionOrderBy: [],
102107
ChunkSkipColumns: [],
103108
AdditionalDimensions: []
104109
);
@@ -159,5 +164,58 @@ FROM _timescaledb_catalog.chunk_column_stats AS ccs
159164
}
160165
}
161166
}
167+
168+
private static void GetCompressionConfiguration(DbConnection connection, Dictionary<(string, string), HypertableInfo> hypertables)
169+
{
170+
using DbCommand command = connection.CreateCommand();
171+
172+
// This view provides the column-level details for compression.
173+
// segmentby_column_index is not null for segment columns.
174+
// orderby_column_index is not null for order columns.
175+
command.CommandText = @"
176+
SELECT
177+
hypertable_schema,
178+
hypertable_name,
179+
attname,
180+
segmentby_column_index,
181+
orderby_column_index,
182+
orderby_asc,
183+
orderby_nullsfirst
184+
FROM timescaledb_information.compression_settings
185+
ORDER BY hypertable_schema, hypertable_name, segmentby_column_index, orderby_column_index;";
186+
187+
using DbDataReader reader = command.ExecuteReader();
188+
while (reader.Read())
189+
{
190+
string schema = reader.GetString(0);
191+
string name = reader.GetString(1);
192+
string columnName = reader.GetString(2);
193+
194+
// Find the corresponding hypertable info
195+
if (!hypertables.TryGetValue((schema, name), out HypertableInfo? info))
196+
{
197+
continue;
198+
}
199+
200+
// Handle SegmentBy
201+
if (!reader.IsDBNull(3)) // segmentby_column_index
202+
{
203+
info.CompressionSegmentBy.Add(columnName);
204+
}
205+
206+
// Handle OrderBy
207+
if (!reader.IsDBNull(4)) // orderby_column_index
208+
{
209+
bool isAscending = reader.GetBoolean(5);
210+
bool isNullsFirst = reader.GetBoolean(6);
211+
212+
string direction = isAscending ? "ASC" : "DESC";
213+
string nulls = isNullsFirst ? "NULLS FIRST" : "NULLS LAST";
214+
215+
// Reconstruct the full string format: "colName DESC NULLS LAST"
216+
info.CompressionOrderBy.Add($"{columnName} {direction} {nulls}");
217+
}
218+
}
219+
}
162220
}
163221
}

src/Eftdb/Abstractions/OrderBy.cs

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
using System.Linq.Expressions;
2+
3+
namespace CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions
4+
{
5+
/// <summary>
6+
/// Represents an ordering specification for a column.
7+
/// </summary>
8+
/// <param name="columnName">The name of the column to order by.</param>
9+
/// <param name="isAscending">
10+
/// If true, orders Ascending (ASC).
11+
/// If false, orders Descending (DESC).
12+
/// If null, uses database default (ASC).
13+
/// </param>
14+
/// <param name="nullsFirst">
15+
/// If true, forces NULLS FIRST.
16+
/// If false, forces NULLS LAST.
17+
/// If null, uses database default (NULLS LAST for ASC, NULLS FIRST for DESC).
18+
/// </param>
19+
public class OrderBy(string columnName, bool? isAscending = null, bool? nullsFirst = null)
20+
{
21+
public string ColumnName { get; } = columnName;
22+
public bool? IsAscending { get; } = isAscending;
23+
public bool? NullsFirst { get; } = nullsFirst;
24+
25+
public string ToSql()
26+
{
27+
var sb = new System.Text.StringBuilder(ColumnName);
28+
29+
// Only append direction if explicitly set
30+
if (IsAscending.HasValue)
31+
{
32+
sb.Append(IsAscending.Value ? " ASC" : " DESC");
33+
}
34+
35+
// Only append NULLS clause if explicitly set
36+
if (NullsFirst.HasValue)
37+
{
38+
sb.Append(NullsFirst.Value ? " NULLS FIRST" : " NULLS LAST");
39+
}
40+
41+
return sb.ToString();
42+
}
43+
}
44+
45+
/// <summary>
46+
/// Fluent builder for creating OrderBy instances.
47+
/// </summary>
48+
public static class OrderByBuilder
49+
{
50+
public static OrderByConfiguration<TEntity> For<TEntity>(Expression<Func<TEntity, object>> expression) => new(expression);
51+
}
52+
53+
/// <summary>
54+
/// Fluent configuration for creating OrderBy instances.
55+
/// </summary>
56+
public class OrderByConfiguration<TEntity>(Expression<Func<TEntity, object>> expression)
57+
{
58+
private readonly string _propertyName = GetPropertyName(expression);
59+
60+
public OrderBy Default(bool? nullsFirst = null) => new(_propertyName, null, nullsFirst);
61+
public OrderBy Ascending(bool? nullsFirst = null) => new(_propertyName, true, nullsFirst);
62+
public OrderBy Descending(bool? nullsFirst = null) => new(_propertyName, false, nullsFirst);
63+
64+
// Helper to extract the string name from the expression
65+
private static string GetPropertyName(Expression<Func<TEntity, object>> expression)
66+
{
67+
if (expression.Body is MemberExpression member) return member.Member.Name;
68+
if (expression.Body is UnaryExpression unary && unary.Operand is MemberExpression m) return m.Member.Name;
69+
throw new ArgumentException("Invalid expression. Please use a simple property access expression.");
70+
}
71+
}
72+
73+
/// <summary>
74+
/// Fluent builder for creating OrderBy instances using lambda expressions.
75+
/// </summary>
76+
/// <typeparam name="TEntity"></typeparam>
77+
public class OrderBySelector<TEntity>
78+
{
79+
public OrderBy By(Expression<Func<TEntity, object>> expression, bool? nullsFirst = null)
80+
=> new(GetPropertyName(expression), null, nullsFirst);
81+
82+
public OrderBy ByAscending(Expression<Func<TEntity, object>> expression, bool? nullsFirst = null)
83+
=> new(GetPropertyName(expression), true, nullsFirst);
84+
85+
public OrderBy ByDescending(Expression<Func<TEntity, object>> expression, bool? nullsFirst = null)
86+
=> new(GetPropertyName(expression), false, nullsFirst);
87+
88+
// Internal helper to get property names from expressions
89+
private static string GetPropertyName(Expression<Func<TEntity, object>> expression)
90+
{
91+
if (expression.Body is MemberExpression m) return m.Member.Name;
92+
if (expression.Body is UnaryExpression u && u.Operand is MemberExpression m2) return m2.Member.Name;
93+
throw new ArgumentException("Expression must be a property access.");
94+
}
95+
}
96+
97+
/// <summary>
98+
/// Extension methods for creating OrderBy instances.
99+
/// </summary>
100+
public static class OrderByExtensions
101+
{
102+
/// <summary>
103+
/// Creates an ascending OrderBy instance.
104+
/// </summary>
105+
/// <param name="columnName">The name of the column to order by.</param>
106+
/// <param name="nullsFirst">Whether nulls should appear first.</param>
107+
public static OrderBy Ascending(this string columnName, bool nullsFirst = false)
108+
{
109+
return new OrderBy(columnName, true, nullsFirst);
110+
}
111+
112+
/// <summary>
113+
/// Creates a descending OrderBy instance.
114+
/// </summary>
115+
/// <param name="columnName">The name of the column to order by.</param>
116+
/// <param name="nullsFirst">Whether nulls should appear first.</param>
117+
public static OrderBy Descending(this string columnName, bool nullsFirst = false)
118+
{
119+
return new OrderBy(columnName, false, nullsFirst);
120+
}
121+
}
122+
}

src/Eftdb/Configuration/Hypertable/HypertableAnnotations.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ public static class HypertableAnnotations
88
public const string IsHypertable = "TimescaleDB:IsHypertable";
99
public const string HypertableTimeColumn = "TimescaleDB:TimeColumnName";
1010
public const string EnableCompression = "TimescaleDB:EnableCompression";
11+
public const string CompressionSegmentBy = "TimescaleDB:CompressionSegmentBy";
12+
public const string CompressionOrderBy = "TimescaleDB:CompressionOrderBy";
1113
public const string MigrateData = "TimescaleDB:MigrateData";
1214
public const string ChunkTimeInterval = "TimescaleDB:ChunkTimeInterval";
1315
public const string ChunkSkipColumns = "TimescaleDB:ChunkSkipColumns";

src/Eftdb/Configuration/Hypertable/HypertableAttribute.cs

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,25 @@ public sealed class HypertableAttribute : Attribute
1414
/// </summary>
1515
public bool EnableCompression { get; set; } = false;
1616

17+
/// <summary>
18+
/// Specifies the columns to group by when compressing the hypertable.
19+
/// Maps to <c>timescaledb.compress_segmentby</c>.
20+
/// </summary>
21+
/// <example>
22+
/// <code>[Hypertable("time", CompressionSegmentBy = ["device_id", "tenant_id"])]</code>
23+
/// </example>
24+
public string[]? CompressionSegmentBy { get; set; } = null;
25+
26+
/// <summary>
27+
/// Specifies the columns to order by within each compressed segment.
28+
/// Maps to <c>timescaledb.compress_orderby</c>.
29+
/// Since attributes cannot use Expressions, you must specify the full SQL syntax if direction is needed.
30+
/// </summary>
31+
/// <example>
32+
/// <code>[Hypertable("time", CompressionOrderBy = ["time DESC", "value ASC NULLS LAST"])]</code>
33+
/// </example>
34+
public string[]? CompressionOrderBy { get; set; } = null;
35+
1736
/// <summary>
1837
/// Specifies whether existing data should be migrated when converting a table to a hypertable.
1938
/// </summary>

src/Eftdb/Configuration/Hypertable/HypertableConvention.cs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,20 @@ public void ProcessEntityTypeAdded(IConventionEntityTypeBuilder entityTypeBuilde
4848
entityTypeBuilder.HasAnnotation(HypertableAnnotations.EnableCompression, true);
4949
entityTypeBuilder.HasAnnotation(HypertableAnnotations.ChunkSkipColumns, string.Join(",", attribute.ChunkSkipColumns));
5050
}
51+
52+
if (attribute.CompressionSegmentBy != null && attribute.CompressionSegmentBy.Length > 0)
53+
{
54+
/// SegmentBy requires compression to be enabled
55+
entityTypeBuilder.HasAnnotation(HypertableAnnotations.EnableCompression, true);
56+
entityTypeBuilder.HasAnnotation(HypertableAnnotations.CompressionSegmentBy, string.Join(", ", attribute.CompressionSegmentBy));
57+
}
58+
59+
if (attribute.CompressionOrderBy != null && attribute.CompressionOrderBy.Length > 0)
60+
{
61+
/// OrderBy requires compression to be enabled
62+
entityTypeBuilder.HasAnnotation(HypertableAnnotations.EnableCompression, true);
63+
entityTypeBuilder.HasAnnotation(HypertableAnnotations.CompressionOrderBy, string.Join(", ", attribute.CompressionOrderBy));
64+
}
5165
}
5266
}
5367
}

src/Eftdb/Configuration/Hypertable/HypertableTypeBuilder.cs

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,62 @@ public static EntityTypeBuilder<TEntity> EnableCompression<TEntity>(
128128
return entityTypeBuilder;
129129
}
130130

131+
/// <summary>
132+
/// Specifies the columns to group by when compressing the hypertable (SegmentBy).
133+
/// </summary>
134+
/// <remarks>
135+
/// Valid settings for <c>timescaledb.compress_segmentby</c>.
136+
/// Columns used for segmenting are not compressed themselves but are used as keys to group rows.
137+
/// Good candidates are columns with low cardinality (e.g., "device_id", "tenant_id").
138+
/// </remarks>
139+
public static EntityTypeBuilder<TEntity> WithCompressionSegmentBy<TEntity>(
140+
this EntityTypeBuilder<TEntity> entityTypeBuilder,
141+
params Expression<Func<TEntity, object>>[] segmentByColumns) where TEntity : class
142+
{
143+
string[] columnNames = [.. segmentByColumns.Select(GetPropertyName)];
144+
145+
entityTypeBuilder.HasAnnotation(HypertableAnnotations.CompressionSegmentBy, string.Join(", ", columnNames));
146+
entityTypeBuilder.HasAnnotation(HypertableAnnotations.EnableCompression, true);
147+
148+
return entityTypeBuilder;
149+
}
150+
151+
/// <summary>
152+
/// Specifies the columns to order by within each compressed segment using explicit OrderBy definitions.
153+
/// </summary>
154+
/// <remarks>
155+
/// Uses the <see cref="OrderByBuilder"/> to define direction and null handling.
156+
/// Example: <c>.WithCompressionOrderBy(OrderByBuilder.For&lt;T&gt;(x => x.Time).Descending())</c>
157+
/// </remarks>
158+
public static EntityTypeBuilder<TEntity> WithCompressionOrderBy<TEntity>(
159+
this EntityTypeBuilder<TEntity> entityTypeBuilder,
160+
params OrderBy[] orderByRules) where TEntity : class
161+
{
162+
string annotationValue = string.Join(", ", orderByRules.Select(r => r.ToSql()));
163+
164+
entityTypeBuilder.HasAnnotation(HypertableAnnotations.CompressionOrderBy, annotationValue);
165+
entityTypeBuilder.HasAnnotation(HypertableAnnotations.EnableCompression, true);
166+
167+
return entityTypeBuilder;
168+
}
169+
170+
/// <summary>
171+
/// Specifies the columns to order by within each compressed segment using the OrderBySelector.
172+
/// </summary>
173+
/// <remarks>
174+
/// Provides a simplified syntax for defining order.
175+
/// Example: <c>.WithCompressionOrderBy(s => [s.ByDescending(x => x.Time), s.By(x => x.Value)])</c>
176+
/// </remarks>
177+
public static EntityTypeBuilder<TEntity> WithCompressionOrderBy<TEntity>(
178+
this EntityTypeBuilder<TEntity> entityTypeBuilder,
179+
Func<OrderBySelector<TEntity>, IEnumerable<OrderBy>> orderSelector) where TEntity : class
180+
{
181+
var selector = new OrderBySelector<TEntity>();
182+
var rules = orderSelector(selector);
183+
184+
return entityTypeBuilder.WithCompressionOrderBy(rules.ToArray());
185+
}
186+
131187
/// <summary>
132188
/// Specifies whether existing data should be migrated when converting a table to a hypertable.
133189
/// </summary>

0 commit comments

Comments
 (0)