Add “in” filter (#64)

* Add “in” filter

* Fix linter warnings

Author: winsmith

Sources/DataTransferObjects/Query/CustomQuery+CompileDown.swift

@@ -162,6 +162,8 @@ public extension CustomQuery {
             return filter
         case .null:
             return filter
+        case .in:
+            return filter
         case .and(let filterExpression):
             return Filter.and(.init(fields: filterExpression.fields.map { compileRelativeFilterInterval(filter: $0) }))
         case .or(let filterExpression):

Sources/DataTransferObjects/Query/Filter.swift

@@ -193,22 +193,34 @@ extension FilterEquals.MatchValue: Codable {
         var container = encoder.singleValueContainer()
 
         switch self {
-        case .string(let value):
+        case let .string(value):
             try container.encode(value)
-        case .int(let value):
+        case let .int(value):
             try container.encode(value)
-        case .double(let value):
+        case let .double(value):
             try container.encode(value)
-        case .arrayString(let value):
+        case let .arrayString(value):
             try container.encode(value)
-        case .arrayInt(let value):
+        case let .arrayInt(value):
             try container.encode(value)
-        case .arrayDouble(let value):
+        case let .arrayDouble(value):
             try container.encode(value)
         }
     }
 }
 
+/// The in filter can match input rows against a set of values, where a match occurs if the value is contained in the set.
+public struct FilterIn: Codable, Hashable, Equatable, Sendable {
+    public init(dimension: String, values: [String]) {
+        self.dimension = dimension
+        self.values = values
+    }
+
+    public let dimension: String
+    public let values: [String]
+    // extractionFn not supported atm
+}
+
 /// The null filter matches rows where a column value is null.
 public struct FilterNull: Codable, Hashable, Equatable, Sendable {
     public init(column: String) {
@@ -223,7 +235,7 @@ public struct FilterNull: Codable, Hashable, Equatable, Sendable {
 public indirect enum Filter: Codable, Hashable, Equatable, Sendable {
     /// The selector filter will match a specific dimension with a specific value.
     /// Selector filters can be used as the base filters for more complex Boolean
-    /// expressions of filters.
+    /// expressions of filters (deprecated -- use equals instead)
     case selector(FilterSelector)
 
     /// The column comparison filter is similar to the selector filter, but instead
@@ -252,6 +264,10 @@ public indirect enum Filter: Codable, Hashable, Equatable, Sendable {
     /// The null filter matches rows where a column value is null.
     case null(FilterNull)
 
+    /// The in filter can match input rows against a set of values, where a match occurs
+    /// if the value is contained in the set.
+    case `in`(FilterIn)
+
     // logical expression filters
     case and(FilterExpression)
     case or(FilterExpression)
@@ -280,6 +296,8 @@ public indirect enum Filter: Codable, Hashable, Equatable, Sendable {
             self = try .equals(FilterEquals(from: decoder))
         case "null":
             self = try .null(FilterNull(from: decoder))
+        case "in":
+            self = try .in(FilterIn(from: decoder))
         case "and":
             self = try .and(FilterExpression(from: decoder))
         case "or":
@@ -321,6 +339,9 @@ public indirect enum Filter: Codable, Hashable, Equatable, Sendable {
         case let .null(null):
             try container.encode("null", forKey: .type)
             try null.encode(to: encoder)
+        case let .in(inFilter):
+            try container.encode("in", forKey: .type)
+            try inFilter.self.encode(to: encoder)
         case let .and(and):
             try container.encode("and", forKey: .type)
             try and.encode(to: encoder)

Sources/DataTransferObjects/QueryGeneration/CustomQuery+Retention.swift

@@ -4,58 +4,58 @@ import DateOperations
 extension CustomQuery {
     func precompiledRetentionQuery() throws -> CustomQuery {
         var query = self
-        
+
         // Get the query intervals - we need at least one interval
         guard let queryIntervals = intervals ?? relativeIntervals?.map({ QueryTimeInterval.from(relativeTimeInterval: $0) }),
               let firstInterval = queryIntervals.first else {
             throw QueryGenerationError.keyMissing(reason: "Missing intervals for retention query")
         }
-        
+
         let beginDate = firstInterval.beginningDate
         let endDate = firstInterval.endDate
-        
+
         // Use the query's granularity to determine retention period, defaulting to month if not specified
         let retentionGranularity = query.granularity ?? .month
-        
+
         // Validate minimum interval based on granularity
         try validateMinimumInterval(from: beginDate, to: endDate, granularity: retentionGranularity)
-        
+
         // Split into intervals based on the specified granularity
         let retentionIntervals = try splitIntoIntervals(from: beginDate, to: endDate, granularity: retentionGranularity)
-        
+
         // Generate Aggregators
         var aggregators = [Aggregator]()
         for interval in retentionIntervals {
             aggregators.append(aggregator(for: interval))
         }
-        
+
         // Generate Post-Aggregators
         var postAggregators = [PostAggregator]()
         for row in retentionIntervals {
             for column in retentionIntervals where column >= row {
                 postAggregators.append(postAggregatorBetween(interval1: row, interval2: column))
             }
         }
-        
+
         // Set the query properties
         query.queryType = .groupBy
         query.granularity = .all
         query.aggregations = uniqued(aggregators)
         query.postAggregations = uniqued(postAggregators)
-        
+
         return query
     }
-    
+
     private func uniqued<T: Hashable>(_ array: [T]) -> [T] {
         var set = Set<T>()
         return array.filter { set.insert($0).inserted }
     }
-    
+
     // MARK: - Helper Methods
-    
+
     private func validateMinimumInterval(from beginDate: Date, to endDate: Date, granularity: QueryGranularity) throws {
         let calendar = Calendar.current
-        
+
         switch granularity {
         case .day:
             let components = calendar.dateComponents([.day], from: beginDate, to: endDate)
@@ -86,11 +86,11 @@ extension CustomQuery {
             throw QueryGenerationError.notImplemented(reason: "Retention queries support day, week, month, quarter, or year granularity")
         }
     }
-    
+
     private func splitIntoIntervals(from fromDate: Date, to toDate: Date, granularity: QueryGranularity) throws -> [DateInterval] {
         let calendar = Calendar.current
         var intervals = [DateInterval]()
-        
+
         switch granularity {
         case .day:
             let numberOfDays = numberOfUnitsBetween(beginDate: fromDate, endDate: toDate, component: .day)
@@ -100,7 +100,7 @@ extension CustomQuery {
                 let endOfDay = startOfDay.end(of: .day) ?? startOfDay
                 intervals.append(DateInterval(start: startOfDay, end: endOfDay))
             }
-            
+
         case .week:
             let numberOfWeeks = numberOfUnitsBetween(beginDate: fromDate, endDate: toDate, component: .weekOfYear)
             for week in 0...numberOfWeeks {
@@ -109,7 +109,7 @@ extension CustomQuery {
                 let endOfWeek = startOfWeek.end(of: .weekOfYear) ?? startOfWeek
                 intervals.append(DateInterval(start: startOfWeek, end: endOfWeek))
             }
-            
+
         case .month:
             let numberOfMonths = numberOfUnitsBetween(beginDate: fromDate, endDate: toDate, component: .month)
             for month in 0...numberOfMonths {
@@ -118,7 +118,7 @@ extension CustomQuery {
                 let endOfMonth = startOfMonth.end(of: .month) ?? startOfMonth
                 intervals.append(DateInterval(start: startOfMonth, end: endOfMonth))
             }
-            
+
         case .quarter:
             let numberOfQuarters = numberOfUnitsBetween(beginDate: fromDate, endDate: toDate, component: .quarter)
             for quarter in 0...numberOfQuarters {
@@ -127,7 +127,7 @@ extension CustomQuery {
                 let endOfQuarter = startOfQuarter.end(of: .quarter) ?? startOfQuarter
                 intervals.append(DateInterval(start: startOfQuarter, end: endOfQuarter))
             }
-            
+
         case .year:
             let numberOfYears = numberOfUnitsBetween(beginDate: fromDate, endDate: toDate, component: .year)
             for year in 0...numberOfYears {
@@ -136,18 +136,18 @@ extension CustomQuery {
                 let endOfYear = startOfYear.end(of: .year) ?? startOfYear
                 intervals.append(DateInterval(start: startOfYear, end: endOfYear))
             }
-            
+
         default:
             throw QueryGenerationError.notImplemented(reason: "Retention queries support day, week, month, quarter, or year granularity")
         }
-        
+
         return intervals
     }
-    
+
     private func numberOfUnitsBetween(beginDate: Date, endDate: Date, component: Calendar.Component) -> Int {
         let calendar = Calendar.current
         let components = calendar.dateComponents([component], from: beginDate, to: endDate)
-        
+
         switch component {
         case .day:
             return components.day ?? 0
@@ -163,13 +163,13 @@ extension CustomQuery {
             return 0
         }
     }
-    
+
     private func title(for interval: DateInterval) -> String {
         let formatter = ISO8601DateFormatter()
         formatter.formatOptions = [.withFullDate]
         return "\(formatter.string(from: interval.start))_\(formatter.string(from: interval.end))"
     }
-    
+
     private func aggregator(for interval: DateInterval) -> Aggregator {
         .filtered(.init(
             filter: .interval(.init(
@@ -182,7 +182,7 @@ extension CustomQuery {
             ))
         ))
     }
-    
+
     private func postAggregatorBetween(interval1: DateInterval, interval2: DateInterval) -> PostAggregator {
         .thetaSketchEstimate(.init(
             name: "retention_\(title(for: interval1))_\(title(for: interval2))",
@@ -195,4 +195,4 @@ extension CustomQuery {
             ))
         ))
     }
-}
+}

Tests/QueryGenerationTests/RetentionQueryGenerationTests.swift

@@ -129,57 +129,57 @@ final class RetentionQueryGenerationTests: XCTestCase {
             granularity: .month
         )
         XCTAssertThrowsError(try monthQuery1.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false))
-        
+
         let monthQuery2 = CustomQuery(
             queryType: .retention,
             dataSource: "com.telemetrydeck.all",
             intervals: [QueryTimeInterval(beginningDate: begin_august, endDate: end_august)],
             granularity: .month
         )
         XCTAssertThrowsError(try monthQuery2.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false))
-        
+
         let monthQuery3 = CustomQuery(
             queryType: .retention,
             dataSource: "com.telemetrydeck.all",
             intervals: [QueryTimeInterval(beginningDate: begin_august, endDate: end_september)],
             granularity: .month
         )
         XCTAssertNoThrow(try monthQuery3.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false))
-        
+
         // Test daily retention
         let startDate = Date(iso8601String: "2022-08-01T00:00:00.000Z")!
         let sameDay = Date(iso8601String: "2022-08-01T12:00:00.000Z")!
         let nextDay = Date(iso8601String: "2022-08-02T00:00:00.000Z")!
-        
+
         let dayQuery1 = CustomQuery(
             queryType: .retention,
             dataSource: "com.telemetrydeck.all",
             intervals: [QueryTimeInterval(beginningDate: startDate, endDate: sameDay)],
             granularity: .day
         )
         XCTAssertThrowsError(try dayQuery1.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false))
-        
+
         let dayQuery2 = CustomQuery(
             queryType: .retention,
             dataSource: "com.telemetrydeck.all",
             intervals: [QueryTimeInterval(beginningDate: startDate, endDate: nextDay)],
             granularity: .day
         )
         XCTAssertNoThrow(try dayQuery2.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false))
-        
+
         // Test weekly retention
         let weekStart = Date(iso8601String: "2022-08-01T00:00:00.000Z")!
         let weekMid = Date(iso8601String: "2022-08-05T00:00:00.000Z")!
         let weekEnd = Date(iso8601String: "2022-08-08T00:00:00.000Z")!
-        
+
         let weekQuery1 = CustomQuery(
             queryType: .retention,
             dataSource: "com.telemetrydeck.all",
             intervals: [QueryTimeInterval(beginningDate: weekStart, endDate: weekMid)],
             granularity: .week
         )
         XCTAssertThrowsError(try weekQuery1.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false))
-        
+
         let weekQuery2 = CustomQuery(
             queryType: .retention,
             dataSource: "com.telemetrydeck.all",
@@ -204,22 +204,22 @@ final class RetentionQueryGenerationTests: XCTestCase {
             )],
             granularity: .month  // Explicitly set to month
         )
-        
+
         let compiledQuery = try query.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [appID], isSuperOrg: true)
-        
+
         // Verify the compiled query has the expected structure
         XCTAssertEqual(compiledQuery.queryType, .groupBy)
         XCTAssertEqual(compiledQuery.granularity, .all)
         XCTAssertNotNil(compiledQuery.aggregations)
         XCTAssertNotNil(compiledQuery.postAggregations)
-        
+
         // The generated query should match the expected structure from tinyQuery
         // (though the exact aggregator names might differ due to date formatting)
     }
-    
+
     func testRetentionWithDifferentGranularities() throws {
         let appID = UUID(uuidString: "79167A27-EBBF-4012-9974-160624E5D07B")!
-        
+
         // Test daily retention - 7 days should generate 8 intervals (0-7 inclusive)
         let dailyQuery = CustomQuery(
             queryType: .retention,
@@ -233,12 +233,12 @@ final class RetentionQueryGenerationTests: XCTestCase {
             )],
             granularity: .day
         )
-        
+
         let compiledDailyQuery = try dailyQuery.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [appID], isSuperOrg: true)
         XCTAssertEqual(compiledDailyQuery.aggregations?.count, 7) // 7 days
         // Post-aggregations should be n*(n+1)/2 for n intervals
         XCTAssertEqual(compiledDailyQuery.postAggregations?.count, 28) // 7*8/2 = 28
-        
+
         // Test weekly retention - 4 weeks
         let weeklyQuery = CustomQuery(
             queryType: .retention,
@@ -252,11 +252,11 @@ final class RetentionQueryGenerationTests: XCTestCase {
             )],
             granularity: .week
         )
-        
+
         let compiledWeeklyQuery = try weeklyQuery.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [appID], isSuperOrg: true)
         XCTAssertEqual(compiledWeeklyQuery.aggregations?.count, 5) // 5 weeks (spans into 5th week)
         XCTAssertEqual(compiledWeeklyQuery.postAggregations?.count, 15) // 5*6/2 = 15
-        
+
         // Test monthly retention - 3 months
         let monthlyQuery = CustomQuery(
             queryType: .retention,
@@ -270,7 +270,7 @@ final class RetentionQueryGenerationTests: XCTestCase {
             )],
             granularity: .month
         )
-        
+
         let compiledMonthlyQuery = try monthlyQuery.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [appID], isSuperOrg: true)
         XCTAssertEqual(compiledMonthlyQuery.aggregations?.count, 3) // 3 months
         XCTAssertEqual(compiledMonthlyQuery.postAggregations?.count, 6) // 3*4/2 = 6