Skip to content

Commit 5cc415e

Browse files
sebastian-ederersebastian-ederer
committed
feat: Add support for retention policies
Note that scaffolding the context couldn't be tested because of an issue in efcore itself (see README). This will be tested as soon as a fix is available.
1 parent 931960c commit 5cc415e

25 files changed

Lines changed: 992 additions & 7 deletions

‎samples/Eftdb.Samples.DatabaseFirst/README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@
22

33
This project demonstrates how to use the **Database-First** approach with [TimescaleDB](https://www.timescale.com/) using the `CmdScale.EntityFrameworkCore.TimescaleDB` package.
44

5+
6+
> [!WARNING]
7+
> Currently the `dotnet ef dbcontext scaffold` command can't be tested because of an issue in `efcore` (see https://github.com/dotnet/efcore/issues/37201).
8+
> <br />TODO: Test as soon as there is a fix available
9+
510
---
611

712
## Required NuGet Packages

‎samples/Eftdb.Samples.Shared/Configurations/ApiRequestAggregateConfiguration.cs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions;
2+
using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ContinuousAggregate;
3+
using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ContinuousAggregatePolicy;
4+
using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.RetentionPolicy;
5+
using CmdScale.EntityFrameworkCore.TimescaleDB.Samples.Shared.Models;
6+
using Microsoft.EntityFrameworkCore;
7+
using Microsoft.EntityFrameworkCore.Metadata.Builders;
8+
9+
namespace CmdScale.EntityFrameworkCore.TimescaleDB.Samples.Shared.Configurations
10+
{
11+
public class ApiRequestAggregateConfiguration : IEntityTypeConfiguration<ApiRequestAggregate>
12+
{
13+
public void Configure(EntityTypeBuilder<ApiRequestAggregate> builder)
14+
{
15+
builder.HasNoKey();
16+
builder.IsContinuousAggregate<ApiRequestAggregate, ApiRequestLog>("api_request_hourly_stats", "1 hour", x => x.Time, true)
17+
.AddAggregateFunction(x => x.AverageDurationMs, x => x.DurationMs, EAggregateFunction.Avg)
18+
.AddAggregateFunction(x => x.MaxDurationMs, x => x.DurationMs, EAggregateFunction.Max)
19+
.AddAggregateFunction(x => x.MinDurationMs, x => x.DurationMs, EAggregateFunction.Min)
20+
.AddGroupByColumn(x => x.ServiceName)
21+
.WithRefreshPolicy(startOffset: "2 days", endOffset: "1 hour", scheduleInterval: "1 hour");
22+
builder.WithRetentionPolicy(
23+
dropAfter: "90 days",
24+
scheduleInterval: "1 day",
25+
maxRetries: 3,
26+
retryPeriod: "15 minutes");
27+
}
28+
}
29+
}

‎samples/Eftdb.Samples.Shared/Configurations/ApiRequestLogConfiguration.cs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable;
2+
using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.RetentionPolicy;
3+
using CmdScale.EntityFrameworkCore.TimescaleDB.Samples.Shared.Models;
4+
using Microsoft.EntityFrameworkCore;
5+
using Microsoft.EntityFrameworkCore.Metadata.Builders;
6+
7+
namespace CmdScale.EntityFrameworkCore.TimescaleDB.Samples.Shared.Configurations
8+
{
9+
public class ApiRequestLogConfiguration : IEntityTypeConfiguration<ApiRequestLog>
10+
{
11+
public void Configure(EntityTypeBuilder<ApiRequestLog> builder)
12+
{
13+
builder.ToTable("ApiRequestLogs");
14+
builder.HasNoKey()
15+
.IsHypertable(x => x.Time)
16+
.WithChunkTimeInterval("1 day");
17+
builder.WithRetentionPolicy(
18+
dropCreatedBefore: "30 days",
19+
scheduleInterval: "1 day",
20+
maxRetries: 5,
21+
retryPeriod: "10 minutes");
22+
}
23+
}
24+
}

‎samples/Eftdb.Samples.Shared/Models/ApiRequestAggregate.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
namespace CmdScale.EntityFrameworkCore.TimescaleDB.Samples.Shared.Models
2+
{
3+
public class ApiRequestAggregate
4+
{
5+
public DateTime TimeBucket { get; set; }
6+
public string ServiceName { get; set; } = string.Empty;
7+
public double AverageDurationMs { get; set; }
8+
public double MaxDurationMs { get; set; }
9+
public double MinDurationMs { get; set; }
10+
}
11+
}

‎samples/Eftdb.Samples.Shared/Models/ApiRequestLog.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
namespace CmdScale.EntityFrameworkCore.TimescaleDB.Samples.Shared.Models
2+
{
3+
public class ApiRequestLog
4+
{
5+
public DateTime Time { get; set; }
6+
public string Method { get; set; } = string.Empty;
7+
public string Path { get; set; } = string.Empty;
8+
public int StatusCode { get; set; }
9+
public double DurationMs { get; set; }
10+
public string ServiceName { get; set; } = string.Empty;
11+
}
12+
}

‎samples/Eftdb.Samples.Shared/Models/ApplicationLog.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable;
2+
using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.RetentionPolicy;
3+
using Microsoft.EntityFrameworkCore;
4+
5+
namespace CmdScale.EntityFrameworkCore.TimescaleDB.Samples.Shared.Models
6+
{
7+
[Hypertable(nameof(Time), ChunkTimeInterval = "1 day")]
8+
[PrimaryKey(nameof(Id), nameof(Time))]
9+
[RetentionPolicy("30 days",
10+
InitialStart = "2025-10-01T03:00:00Z",
11+
ScheduleInterval = "1 day",
12+
MaxRetries = 3,
13+
RetryPeriod = "5 minutes")]
14+
public class ApplicationLog
15+
{
16+
public Guid Id { get; set; }
17+
public DateTime Time { get; set; }
18+
public string ServiceName { get; set; } = string.Empty;
19+
public string Level { get; set; } = string.Empty;
20+
public string Message { get; set; } = string.Empty;
21+
public string? ExceptionDetails { get; set; }
22+
}
23+
}

‎samples/Eftdb.Samples.Shared/TimescaleContext.cs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
using CmdScale.EntityFrameworkCore.TimescaleDB.Samples.Shared.Models;
1+
using CmdScale.EntityFrameworkCore.TimescaleDB.Samples.Shared.Models;
22
using Microsoft.EntityFrameworkCore;
33

44
namespace CmdScale.EntityFrameworkCore.TimescaleDB.Samples.Shared
@@ -13,6 +13,9 @@ public class TimescaleContext(DbContextOptions<TimescaleContext> options) : DbCo
1313
public DbSet<TradeWithId> TradesWithId { get; set; }
1414
public DbSet<TradeAggregate> TradeAggregates { get; set; }
1515
public DbSet<WeatherAggregate> WeatherAggregates { get; set; }
16+
public DbSet<ApplicationLog> ApplicationLogs { get; set; }
17+
public DbSet<ApiRequestLog> ApiRequestLogs { get; set; }
18+
public DbSet<ApiRequestAggregate> ApiRequestAggregates { get; set; }
1619

1720
protected override void OnModelCreating(ModelBuilder modelBuilder)
1821
{
@@ -21,4 +24,4 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
2124
modelBuilder.HasDefaultSchema("custom_schema");
2225
}
2326
}
24-
}
27+
}

‎src/Eftdb.Design/Scaffolding/RetentionPolicyAnnotationApplier.cs‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.RetentionPolicy;
2+
using Microsoft.EntityFrameworkCore.Scaffolding.Metadata;
3+
using static CmdScale.EntityFrameworkCore.TimescaleDB.Design.Scaffolding.RetentionPolicyScaffoldingExtractor;
4+
5+
namespace CmdScale.EntityFrameworkCore.TimescaleDB.Design.Scaffolding
6+
{
7+
/// <summary>
8+
/// Applies retention policy annotations to scaffolded database tables.
9+
/// </summary>
10+
public sealed class RetentionPolicyAnnotationApplier : IAnnotationApplier
11+
{
12+
public void ApplyAnnotations(DatabaseTable table, object featureInfo)
13+
{
14+
if (featureInfo is not RetentionPolicyInfo policyInfo)
15+
{
16+
throw new ArgumentException($"Expected {nameof(RetentionPolicyInfo)}, got {featureInfo.GetType().Name}", nameof(featureInfo));
17+
}
18+
19+
table[RetentionPolicyAnnotations.HasRetentionPolicy] = true;
20+
21+
if (!string.IsNullOrWhiteSpace(policyInfo.DropAfter))
22+
{
23+
table[RetentionPolicyAnnotations.DropAfter] = policyInfo.DropAfter;
24+
}
25+
26+
if (!string.IsNullOrWhiteSpace(policyInfo.DropCreatedBefore))
27+
{
28+
table[RetentionPolicyAnnotations.DropCreatedBefore] = policyInfo.DropCreatedBefore;
29+
}
30+
31+
if (policyInfo.InitialStart.HasValue)
32+
{
33+
table[RetentionPolicyAnnotations.InitialStart] = policyInfo.InitialStart.Value;
34+
}
35+
36+
// Set annotations only if they differ from TimescaleDB defaults
37+
if (policyInfo.ScheduleInterval != DefaultValues.RetentionPolicyScheduleInterval)
38+
{
39+
table[RetentionPolicyAnnotations.ScheduleInterval] = policyInfo.ScheduleInterval;
40+
}
41+
42+
if (policyInfo.MaxRuntime != DefaultValues.RetentionPolicyMaxRuntime)
43+
{
44+
table[RetentionPolicyAnnotations.MaxRuntime] = policyInfo.MaxRuntime;
45+
}
46+
47+
if (policyInfo.MaxRetries != DefaultValues.RetentionPolicyMaxRetries)
48+
{
49+
table[RetentionPolicyAnnotations.MaxRetries] = policyInfo.MaxRetries;
50+
}
51+
52+
if (policyInfo.RetryPeriod != DefaultValues.RetentionPolicyScheduleInterval)
53+
{
54+
table[RetentionPolicyAnnotations.RetryPeriod] = policyInfo.RetryPeriod;
55+
}
56+
}
57+
}
58+
}

‎src/Eftdb.Design/Scaffolding/RetentionPolicyScaffoldingExtractor.cs‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
using System.Data;
2+
using System.Data.Common;
3+
using System.Text.Json;
4+
5+
namespace CmdScale.EntityFrameworkCore.TimescaleDB.Design.Scaffolding
6+
{
7+
/// <summary>
8+
/// Extracts retention policy metadata from a TimescaleDB database for scaffolding.
9+
/// </summary>
10+
public sealed class RetentionPolicyScaffoldingExtractor : ITimescaleFeatureExtractor
11+
{
12+
public sealed record RetentionPolicyInfo(
13+
string? DropAfter,
14+
string? DropCreatedBefore,
15+
DateTime? InitialStart,
16+
string? ScheduleInterval,
17+
string? MaxRuntime,
18+
int? MaxRetries,
19+
string? RetryPeriod
20+
);
21+
22+
public Dictionary<(string Schema, string TableName), object> Extract(DbConnection connection)
23+
{
24+
bool wasOpen = connection.State == ConnectionState.Open;
25+
if (!wasOpen)
26+
{
27+
connection.Open();
28+
}
29+
30+
try
31+
{
32+
Dictionary<(string, string), RetentionPolicyInfo> retentionPolicies = [];
33+
34+
using (DbCommand command = connection.CreateCommand())
35+
{
36+
command.CommandText = @"
37+
SELECT
38+
j.hypertable_schema,
39+
j.hypertable_name,
40+
j.config,
41+
j.initial_start,
42+
j.schedule_interval::text,
43+
j.max_runtime::text,
44+
j.max_retries,
45+
j.retry_period::text
46+
FROM timescaledb_information.jobs AS j
47+
WHERE j.proc_name = 'policy_retention';";
48+
49+
using DbDataReader reader = command.ExecuteReader();
50+
while (reader.Read())
51+
{
52+
string schema = reader.GetString(0);
53+
string name = reader.GetString(1);
54+
string? configJson = reader.IsDBNull(2) ? null : reader.GetString(2);
55+
DateTime? initialStart = reader.IsDBNull(3) ? null : reader.GetDateTime(3);
56+
string? scheduleInterval = reader.IsDBNull(4) ? null : reader.GetString(4);
57+
string? maxRuntime = reader.IsDBNull(5) ? null : reader.GetString(5);
58+
int? maxRetries = reader.IsDBNull(6) ? null : reader.GetInt32(6);
59+
string? retryPeriod = reader.IsDBNull(7) ? null : reader.GetString(7);
60+
61+
// Parse the JSONB config to extract drop_after or drop_created_before
62+
string? dropAfter = null;
63+
string? dropCreatedBefore = null;
64+
65+
if (!string.IsNullOrWhiteSpace(configJson))
66+
{
67+
using JsonDocument doc = JsonDocument.Parse(configJson);
68+
JsonElement root = doc.RootElement;
69+
70+
if (root.TryGetProperty("drop_after", out JsonElement dropAfterElement))
71+
{
72+
dropAfter = IntervalParsingHelper.ParseIntervalOrInteger(dropAfterElement);
73+
}
74+
75+
if (root.TryGetProperty("drop_created_before", out JsonElement dropCreatedBeforeElement))
76+
{
77+
dropCreatedBefore = IntervalParsingHelper.ParseIntervalOrInteger(dropCreatedBeforeElement);
78+
}
79+
}
80+
81+
// A retention policy must have either drop_after or drop_created_before
82+
if (string.IsNullOrWhiteSpace(dropAfter) && string.IsNullOrWhiteSpace(dropCreatedBefore))
83+
{
84+
continue;
85+
}
86+
87+
retentionPolicies[(schema, name)] = new RetentionPolicyInfo(
88+
DropAfter: dropAfter,
89+
DropCreatedBefore: dropCreatedBefore,
90+
InitialStart: initialStart,
91+
ScheduleInterval: scheduleInterval,
92+
MaxRuntime: maxRuntime,
93+
MaxRetries: maxRetries,
94+
RetryPeriod: retryPeriod
95+
);
96+
}
97+
}
98+
99+
// Convert to object dictionary to match interface
100+
return retentionPolicies.ToDictionary(
101+
kvp => kvp.Key,
102+
kvp => (object)kvp.Value
103+
);
104+
}
105+
finally
106+
{
107+
if (!wasOpen)
108+
{
109+
connection.Close();
110+
}
111+
}
112+
}
113+
}
114+
}

‎src/Eftdb.Design/TimescaleCSharpMigrationOperationGenerator.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ protected override void Generate(MigrationOperation operation, IndentedStringBui
1515

1616
HypertableOperationGenerator? hypertableOperationGenerator = null;
1717
ReorderPolicyOperationGenerator? reorderPolicyOperationGenerator = null;
18+
RetentionPolicyOperationGenerator? retentionPolicyOperationGenerator = null;
1819
ContinuousAggregateOperationGenerator? continuousAggregateOperationGenerator = null;
1920
ContinuousAggregatePolicyOperationGenerator? continuousAggregatePolicyOperationGenerator = null;
2021

@@ -45,6 +46,19 @@ protected override void Generate(MigrationOperation operation, IndentedStringBui
4546
statements = reorderPolicyOperationGenerator.Generate(dropReorder);
4647
break;
4748

49+
case AddRetentionPolicyOperation addRetention:
50+
retentionPolicyOperationGenerator ??= new(isDesignTime: true);
51+
statements = retentionPolicyOperationGenerator.Generate(addRetention);
52+
break;
53+
case AlterRetentionPolicyOperation alterRetention:
54+
retentionPolicyOperationGenerator ??= new(isDesignTime: true);
55+
statements = retentionPolicyOperationGenerator.Generate(alterRetention);
56+
break;
57+
case DropRetentionPolicyOperation dropRetention:
58+
retentionPolicyOperationGenerator ??= new(isDesignTime: true);
59+
statements = retentionPolicyOperationGenerator.Generate(dropRetention);
60+
break;
61+
4862
case CreateContinuousAggregateOperation createContinuousAggregate:
4963
continuousAggregateOperationGenerator ??= new(isDesignTime: true);
5064
statements = continuousAggregateOperationGenerator.Generate(createContinuousAggregate);

0 commit comments

Comments
 (0)