diff --git a/OpenASO.xcodeproj/project.pbxproj b/OpenASO.xcodeproj/project.pbxproj index 1e2c7ac..75bd6dd 100644 --- a/OpenASO.xcodeproj/project.pbxproj +++ b/OpenASO.xcodeproj/project.pbxproj @@ -89,6 +89,9 @@ A0D71B4C0000000000000017 /* KeywordRankingChartView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A0D71B4C0000000000000018 /* KeywordRankingChartView.swift */; }; A0D71B4C0000000000000019 /* AppDetailPreviewFixtures.swift in Sources */ = {isa = PBXBuildFile; fileRef = A0D71B4C000000000000001A /* AppDetailPreviewFixtures.swift */; }; A0D71B4C000000000000001B /* AppDetailCSVTransfer.swift in Sources */ = {isa = PBXBuildFile; fileRef = A0D71B4C000000000000001C /* AppDetailCSVTransfer.swift */; }; + A19000000000000000000001 /* SearchRanking/AppStoreWebRankingProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = A19000000000000000000011 /* SearchRanking/AppStoreWebRankingProvider.swift */; }; + A19000000000000000000002 /* SearchRanking/PrimaryFallbackSearchRankingProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = A19000000000000000000012 /* SearchRanking/PrimaryFallbackSearchRankingProvider.swift */; }; + A19000000000000000000003 /* AppStoreWebRankingProviderTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A19000000000000000000013 /* AppStoreWebRankingProviderTests.swift */; }; A10000000000000000000002 /* OpenASOSchemaV1.swift in Sources */ = {isa = PBXBuildFile; fileRef = A10000000000000000000001 /* OpenASOSchemaV1.swift */; }; D18000000000000000000003 /* OpenASOSchemaV2.swift in Sources */ = {isa = PBXBuildFile; fileRef = D18000000000000000000001 /* OpenASOSchemaV2.swift */; }; D18000000000000000000004 /* TrackedAppKeywordRefreshAttempt.swift in Sources */ = {isa = PBXBuildFile; fileRef = D18000000000000000000002 /* TrackedAppKeywordRefreshAttempt.swift */; }; @@ -108,6 +111,27 @@ D20B00000000000000000007 /* Persistence/EstimatedKeywordDifficultyStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20B00000000000000000014 /* Persistence/EstimatedKeywordDifficultyStore.swift */; }; D20B00000000000000000008 /* Persistence/EstimatedKeywordDifficultyStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20B00000000000000000014 /* Persistence/EstimatedKeywordDifficultyStore.swift */; }; D20B00000000000000000009 /* EstimatedKeywordDifficultyStoreTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20B00000000000000000015 /* EstimatedKeywordDifficultyStoreTests.swift */; }; + D20C00000000000000000001 /* SearchRanking/EstimatedKeywordDifficultyPersistenceAdapter.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000011 /* SearchRanking/EstimatedKeywordDifficultyPersistenceAdapter.swift */; }; + D20C00000000000000000032 /* SearchRanking/EstimatedKeywordDifficultyPersistenceAdapter.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000011 /* SearchRanking/EstimatedKeywordDifficultyPersistenceAdapter.swift */; }; + D20C00000000000000000002 /* Persistence/EstimatedKeywordDifficultyPresentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000012 /* Persistence/EstimatedKeywordDifficultyPresentation.swift */; }; + D20C00000000000000000003 /* Persistence/EstimatedKeywordDifficultyPresentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000012 /* Persistence/EstimatedKeywordDifficultyPresentation.swift */; }; + D20C00000000000000000004 /* EstimatedKeywordDifficultySummaryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000013 /* EstimatedKeywordDifficultySummaryTests.swift */; }; + D20C00000000000000000005 /* EstimatedKeywordDifficultyPersistenceAdapterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000014 /* EstimatedKeywordDifficultyPersistenceAdapterTests.swift */; }; + D20C00000000000000000006 /* Persistence/TrackedKeywordDeletionService.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000015 /* Persistence/TrackedKeywordDeletionService.swift */; }; + D20C00000000000000000007 /* TrackedKeywordDeletionServiceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000016 /* TrackedKeywordDeletionServiceTests.swift */; }; + D20C00000000000000000008 /* EstimatedKeywordDifficultyCSVExportTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000017 /* EstimatedKeywordDifficultyCSVExportTests.swift */; }; + D20C00000000000000000020 /* EstimatedKeywordDifficultyCell.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000040 /* EstimatedKeywordDifficultyCell.swift */; }; + D20C00000000000000000021 /* EstimatedKeywordDifficultyDetailContent.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000041 /* EstimatedKeywordDifficultyDetailContent.swift */; }; + D20C00000000000000000022 /* EstimatedKeywordDifficultyDetailDataSource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000042 /* EstimatedKeywordDifficultyDetailDataSource.swift */; }; + D20C00000000000000000023 /* EstimatedKeywordDifficultyDetailModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000043 /* EstimatedKeywordDifficultyDetailModel.swift */; }; + D20C00000000000000000024 /* EstimatedKeywordDifficultyDetailSheet.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000044 /* EstimatedKeywordDifficultyDetailSheet.swift */; }; + D20C00000000000000000025 /* EstimatedKeywordDifficultyDisplayNames.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000045 /* EstimatedKeywordDifficultyDisplayNames.swift */; }; + D20C00000000000000000026 /* EstimatedKeywordDifficultyEstimateSection.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000046 /* EstimatedKeywordDifficultyEstimateSection.swift */; }; + D20C00000000000000000027 /* EstimatedKeywordDifficultyEvidenceSection.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000047 /* EstimatedKeywordDifficultyEvidenceSection.swift */; }; + D20C00000000000000000028 /* EstimatedKeywordDifficultyProvenanceSection.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000048 /* EstimatedKeywordDifficultyProvenanceSection.swift */; }; + D20C00000000000000000029 /* EstimatedKeywordDifficultyRowPresentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000049 /* EstimatedKeywordDifficultyRowPresentation.swift */; }; + D20C00000000000000000030 /* EstimatedKeywordDifficultyDetailModelTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000050 /* EstimatedKeywordDifficultyDetailModelTests.swift */; }; + D20C00000000000000000031 /* EstimatedKeywordDifficultyWorkspaceUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20C00000000000000000051 /* EstimatedKeywordDifficultyWorkspaceUITests.swift */; }; A20000000000000000000001 /* KeywordTableCells.swift in Sources */ = {isa = PBXBuildFile; fileRef = A20000000000000000000011 /* KeywordTableCells.swift */; }; A20000000000000000000002 /* KeywordNotesSheet.swift in Sources */ = {isa = PBXBuildFile; fileRef = A20000000000000000000012 /* KeywordNotesSheet.swift */; }; A20000000000000000000003 /* KeywordRankingListSheet.swift in Sources */ = {isa = PBXBuildFile; fileRef = A20000000000000000000013 /* KeywordRankingListSheet.swift */; }; @@ -229,6 +253,9 @@ C5A400000000000000000003 /* CSVFormatsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C5A400000000000000000004 /* CSVFormatsTests.swift */; }; C70260E8797CC353D9AD1B3A /* SearchRanking/SearchModels.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1C1EE58C790084CF7A83CA8A /* SearchRanking/SearchModels.swift */; }; CB1733C5A8778748B70BA4FD /* AppDetailView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 378F7AF5B5856F0183D027B4 /* AppDetailView.swift */; }; + D20A00000000000000000001 /* SearchRanking/KeywordDifficultyEstimator.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20A00000000000000000011 /* SearchRanking/KeywordDifficultyEstimator.swift */; }; + D20A00000000000000000002 /* SearchRanking/KeywordDifficultyEstimator.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20A00000000000000000011 /* SearchRanking/KeywordDifficultyEstimator.swift */; }; + D20A00000000000000000003 /* KeywordDifficultyEstimatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D20A00000000000000000012 /* KeywordDifficultyEstimatorTests.swift */; }; D30000000000000000000002 /* AppNamespace.swift in Sources */ = {isa = PBXBuildFile; fileRef = D30000000000000000000001 /* AppNamespace.swift */; }; DA7A00000000000000000002 /* PersistentStoreRecoveryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA7A00000000000000000001 /* PersistentStoreRecoveryView.swift */; }; D6807507E1AF0AB52CB2C7E2 /* SearchRanking/DefaultAppResolver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 465129A02F247ED683558D8B /* SearchRanking/DefaultAppResolver.swift */; }; @@ -318,6 +345,9 @@ A0BEEF000000000000000012 /* Storefront/ScreenshotDownloadService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Storefront/ScreenshotDownloadService.swift; sourceTree = ""; }; A0BEEF000000000000000013 /* Storefront/ScreenshotDownloadProgressStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Storefront/ScreenshotDownloadProgressStore.swift; sourceTree = ""; }; A0BEEF000000000000000015 /* ScreenshotDownloadServiceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScreenshotDownloadServiceTests.swift; sourceTree = ""; }; + A19000000000000000000011 /* SearchRanking/AppStoreWebRankingProvider.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchRanking/AppStoreWebRankingProvider.swift; sourceTree = ""; }; + A19000000000000000000012 /* SearchRanking/PrimaryFallbackSearchRankingProvider.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchRanking/PrimaryFallbackSearchRankingProvider.swift; sourceTree = ""; }; + A19000000000000000000013 /* AppStoreWebRankingProviderTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppStoreWebRankingProviderTests.swift; sourceTree = ""; }; A0D71B4C0000000000000001 /* AppDetailToolbarViews.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDetailToolbarViews.swift; sourceTree = ""; }; A0D71B4C0000000000000003 /* AppRatingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppRatingsView.swift; sourceTree = ""; }; A0D71B4C0000000000000007 /* AppDetailSupportTypes.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDetailSupportTypes.swift; sourceTree = ""; }; @@ -341,6 +371,25 @@ D20B00000000000000000013 /* EstimatedKeywordDifficultyResultEvidenceRecord.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultyResultEvidenceRecord.swift; sourceTree = ""; }; D20B00000000000000000014 /* Persistence/EstimatedKeywordDifficultyStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Persistence/EstimatedKeywordDifficultyStore.swift; sourceTree = ""; }; D20B00000000000000000015 /* EstimatedKeywordDifficultyStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultyStoreTests.swift; sourceTree = ""; }; + D20C00000000000000000011 /* SearchRanking/EstimatedKeywordDifficultyPersistenceAdapter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchRanking/EstimatedKeywordDifficultyPersistenceAdapter.swift; sourceTree = ""; }; + D20C00000000000000000012 /* Persistence/EstimatedKeywordDifficultyPresentation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Persistence/EstimatedKeywordDifficultyPresentation.swift; sourceTree = ""; }; + D20C00000000000000000013 /* EstimatedKeywordDifficultySummaryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultySummaryTests.swift; sourceTree = ""; }; + D20C00000000000000000014 /* EstimatedKeywordDifficultyPersistenceAdapterTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultyPersistenceAdapterTests.swift; sourceTree = ""; }; + D20C00000000000000000015 /* Persistence/TrackedKeywordDeletionService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Persistence/TrackedKeywordDeletionService.swift; sourceTree = ""; }; + D20C00000000000000000016 /* TrackedKeywordDeletionServiceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TrackedKeywordDeletionServiceTests.swift; sourceTree = ""; }; + D20C00000000000000000017 /* EstimatedKeywordDifficultyCSVExportTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultyCSVExportTests.swift; sourceTree = ""; }; + D20C00000000000000000040 /* EstimatedKeywordDifficultyCell.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultyCell.swift; sourceTree = ""; }; + D20C00000000000000000041 /* EstimatedKeywordDifficultyDetailContent.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultyDetailContent.swift; sourceTree = ""; }; + D20C00000000000000000042 /* EstimatedKeywordDifficultyDetailDataSource.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultyDetailDataSource.swift; sourceTree = ""; }; + D20C00000000000000000043 /* EstimatedKeywordDifficultyDetailModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultyDetailModel.swift; sourceTree = ""; }; + D20C00000000000000000044 /* EstimatedKeywordDifficultyDetailSheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultyDetailSheet.swift; sourceTree = ""; }; + D20C00000000000000000045 /* EstimatedKeywordDifficultyDisplayNames.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultyDisplayNames.swift; sourceTree = ""; }; + D20C00000000000000000046 /* EstimatedKeywordDifficultyEstimateSection.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultyEstimateSection.swift; sourceTree = ""; }; + D20C00000000000000000047 /* EstimatedKeywordDifficultyEvidenceSection.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultyEvidenceSection.swift; sourceTree = ""; }; + D20C00000000000000000048 /* EstimatedKeywordDifficultyProvenanceSection.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultyProvenanceSection.swift; sourceTree = ""; }; + D20C00000000000000000049 /* EstimatedKeywordDifficultyRowPresentation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultyRowPresentation.swift; sourceTree = ""; }; + D20C00000000000000000050 /* EstimatedKeywordDifficultyDetailModelTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultyDetailModelTests.swift; sourceTree = ""; }; + D20C00000000000000000051 /* EstimatedKeywordDifficultyWorkspaceUITests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedKeywordDifficultyWorkspaceUITests.swift; sourceTree = ""; }; A20000000000000000000011 /* KeywordTableCells.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeywordTableCells.swift; sourceTree = ""; }; A20000000000000000000012 /* KeywordNotesSheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeywordNotesSheet.swift; sourceTree = ""; }; A20000000000000000000013 /* KeywordRankingListSheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeywordRankingListSheet.swift; sourceTree = ""; }; @@ -426,6 +475,8 @@ C15A00000000000000000002 /* OpenASOMCP */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = OpenASOMCP; sourceTree = BUILT_PRODUCTS_DIR; }; C5A400000000000000000002 /* CSVFormats.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CSVFormats.swift; sourceTree = ""; }; C5A400000000000000000004 /* CSVFormatsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CSVFormatsTests.swift; sourceTree = ""; }; + D20A00000000000000000011 /* SearchRanking/KeywordDifficultyEstimator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchRanking/KeywordDifficultyEstimator.swift; sourceTree = ""; }; + D20A00000000000000000012 /* KeywordDifficultyEstimatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeywordDifficultyEstimatorTests.swift; sourceTree = ""; }; D30000000000000000000001 /* AppNamespace.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppNamespace.swift; sourceTree = ""; }; D30000000000000000000005 /* OpenASO.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = OpenASO.entitlements; sourceTree = ""; }; DA7A00000000000000000001 /* PersistentStoreRecoveryView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PersistentStoreRecoveryView.swift; sourceTree = ""; }; @@ -483,13 +534,21 @@ D02000000000000000000007 /* ReleasedV1StoreFixture.swift */, D09D00000000000000000007 /* TrackedKeywordRefreshStatusTests.swift */, D20B00000000000000000015 /* EstimatedKeywordDifficultyStoreTests.swift */, + D20C00000000000000000013 /* EstimatedKeywordDifficultySummaryTests.swift */, + D20C00000000000000000014 /* EstimatedKeywordDifficultyPersistenceAdapterTests.swift */, + D20C00000000000000000016 /* TrackedKeywordDeletionServiceTests.swift */, + D20C00000000000000000017 /* EstimatedKeywordDifficultyCSVExportTests.swift */, + D20C00000000000000000050 /* EstimatedKeywordDifficultyDetailModelTests.swift */, + D20C00000000000000000051 /* EstimatedKeywordDifficultyWorkspaceUITests.swift */, D02000000000000000000004 /* v0.3.2-v1 */, 0A5C00000000000000000006 /* AppStoreConnectReviewServiceTests.swift */, 0A5C00000000000000000008 /* AppStoreWebMetadataProviderTests.swift */, + A19000000000000000000013 /* AppStoreWebRankingProviderTests.swift */, 0A1B2C3D4E5F67890123450E /* AppStorefrontRatingServiceTests.swift */, C5A400000000000000000004 /* CSVFormatsTests.swift */, B16D1F1F736DDAB911B87E77 /* DefaultAppResolverTests.swift */, 8C6BDA000FFE3FC055867E78 /* ITunesSearchFallbackProviderTests.swift */, + D20A00000000000000000012 /* KeywordDifficultyEstimatorTests.swift */, 04E8C4E18F1A4D519C0A1128 /* KeywordMetricsServiceTests.swift */, A2000000000000000000001B /* KeywordRankingHistoryTests.swift */, B7000000000000000000001A /* AddKeywordsStorefrontProjectionTests.swift */, @@ -560,6 +619,8 @@ 0A1B2C3D4E5F678901234514 /* RatingsReviews/AppStorefrontReviewService.swift */, A30000000000000000000002 /* Persistence/BackgroundModelStore.swift */, D20B00000000000000000014 /* Persistence/EstimatedKeywordDifficultyStore.swift */, + D20C00000000000000000012 /* Persistence/EstimatedKeywordDifficultyPresentation.swift */, + D20C00000000000000000015 /* Persistence/TrackedKeywordDeletionService.swift */, A0D71B4C0000000000000014 /* Persistence/DailyRefreshScheduler.swift */, 465129A02F247ED683558D8B /* SearchRanking/DefaultAppResolver.swift */, 04E8C4E18F1A4D519C0A1132 /* AI/FoundationModelsAIService.swift */, @@ -568,6 +629,10 @@ F008A0000000000000000011 /* Networking/ProviderRequestGate.swift */, F008B0000000000000000011 /* Networking/ProviderHTTPClientPipeline.swift */, DEB53B31A2B107CEE141F744 /* SearchRanking/ITunesSearchFallbackProvider.swift */, + D20A00000000000000000011 /* SearchRanking/KeywordDifficultyEstimator.swift */, + D20C00000000000000000011 /* SearchRanking/EstimatedKeywordDifficultyPersistenceAdapter.swift */, + A19000000000000000000011 /* SearchRanking/AppStoreWebRankingProvider.swift */, + A19000000000000000000012 /* SearchRanking/PrimaryFallbackSearchRankingProvider.swift */, 5C5530D41B3742698F02134C /* Credentials/KeychainService.swift */, C0DEFACE0000000000000101 /* MCP/OpenASOMCPDTOs.swift */, C0DEFACE0000000000000103 /* MCP/OpenASOMCPService.swift */, @@ -616,6 +681,16 @@ isa = PBXGroup; children = ( 7E564C613BC84E13B31CB08D /* KeywordTableView.swift */, + D20C00000000000000000040 /* EstimatedKeywordDifficultyCell.swift */, + D20C00000000000000000041 /* EstimatedKeywordDifficultyDetailContent.swift */, + D20C00000000000000000042 /* EstimatedKeywordDifficultyDetailDataSource.swift */, + D20C00000000000000000043 /* EstimatedKeywordDifficultyDetailModel.swift */, + D20C00000000000000000044 /* EstimatedKeywordDifficultyDetailSheet.swift */, + D20C00000000000000000045 /* EstimatedKeywordDifficultyDisplayNames.swift */, + D20C00000000000000000046 /* EstimatedKeywordDifficultyEstimateSection.swift */, + D20C00000000000000000047 /* EstimatedKeywordDifficultyEvidenceSection.swift */, + D20C00000000000000000048 /* EstimatedKeywordDifficultyProvenanceSection.swift */, + D20C00000000000000000049 /* EstimatedKeywordDifficultyRowPresentation.swift */, A20000000000000000000011 /* KeywordTableCells.swift */, A20000000000000000000012 /* KeywordNotesSheet.swift */, A20000000000000000000017 /* KeywordRankingHistorySheet.swift */, @@ -996,6 +1071,18 @@ 0A1B2C3D4E5F678901234513 /* RatingsReviews/AppStorefrontReviewService.swift in Sources */, A30000000000000000000001 /* Persistence/BackgroundModelStore.swift in Sources */, D20B00000000000000000007 /* Persistence/EstimatedKeywordDifficultyStore.swift in Sources */, + D20C00000000000000000002 /* Persistence/EstimatedKeywordDifficultyPresentation.swift in Sources */, + D20C00000000000000000006 /* Persistence/TrackedKeywordDeletionService.swift in Sources */, + D20C00000000000000000020 /* EstimatedKeywordDifficultyCell.swift in Sources */, + D20C00000000000000000021 /* EstimatedKeywordDifficultyDetailContent.swift in Sources */, + D20C00000000000000000022 /* EstimatedKeywordDifficultyDetailDataSource.swift in Sources */, + D20C00000000000000000023 /* EstimatedKeywordDifficultyDetailModel.swift in Sources */, + D20C00000000000000000024 /* EstimatedKeywordDifficultyDetailSheet.swift in Sources */, + D20C00000000000000000025 /* EstimatedKeywordDifficultyDisplayNames.swift in Sources */, + D20C00000000000000000026 /* EstimatedKeywordDifficultyEstimateSection.swift in Sources */, + D20C00000000000000000027 /* EstimatedKeywordDifficultyEvidenceSection.swift in Sources */, + D20C00000000000000000028 /* EstimatedKeywordDifficultyProvenanceSection.swift in Sources */, + D20C00000000000000000029 /* EstimatedKeywordDifficultyRowPresentation.swift in Sources */, 04E8C4E18F1A4D519C0A1136 /* CodableAppStorage.swift in Sources */, C5A400000000000000000001 /* CSVFormats.swift in Sources */, A0D71B4C0000000000000013 /* Persistence/DailyRefreshScheduler.swift in Sources */, @@ -1005,6 +1092,8 @@ F008A0000000000000000001 /* Networking/ProviderRequestGate.swift in Sources */, F008B0000000000000000001 /* Networking/ProviderHTTPClientPipeline.swift in Sources */, AB171C45C65C542EFCD3CEBD /* SearchRanking/ITunesSearchFallbackProvider.swift in Sources */, + A19000000000000000000001 /* SearchRanking/AppStoreWebRankingProvider.swift in Sources */, + A19000000000000000000002 /* SearchRanking/PrimaryFallbackSearchRankingProvider.swift in Sources */, 6C612150F69149F0A3F988EE /* Credentials/KeychainService.swift in Sources */, A0D71B4C0000000000000011 /* SearchRanking/KeywordInsightsService.swift in Sources */, A0D71B4C000000000000000F /* KeywordInsightsTypes.swift in Sources */, @@ -1066,6 +1155,8 @@ C0DEC0DE0000000000000004 /* PinIconButton.swift in Sources */, 5FAADABB0DA1EEDB3699E66C /* SearchRanking/RankingMatcher.swift in Sources */, 33B2BE83B4CFC8970D7A3482 /* SearchRanking/RankingRefreshCoordinator.swift in Sources */, + D20A00000000000000000001 /* SearchRanking/KeywordDifficultyEstimator.swift in Sources */, + D20C00000000000000000001 /* SearchRanking/EstimatedKeywordDifficultyPersistenceAdapter.swift in Sources */, 9A6937276AF0E3CAEDA90C36 /* RankingSource.swift in Sources */, A40000000000000000000003 /* RatingsDetail.swift in Sources */, A40000000000000000000001 /* RatingsDashboardSupport.swift in Sources */, @@ -1111,12 +1202,20 @@ D02000000000000000000006 /* ReleasedV1StoreFixture.swift in Sources */, D09D00000000000000000008 /* TrackedKeywordRefreshStatusTests.swift in Sources */, D20B00000000000000000009 /* EstimatedKeywordDifficultyStoreTests.swift in Sources */, + D20C00000000000000000004 /* EstimatedKeywordDifficultySummaryTests.swift in Sources */, + D20C00000000000000000005 /* EstimatedKeywordDifficultyPersistenceAdapterTests.swift in Sources */, + D20C00000000000000000007 /* TrackedKeywordDeletionServiceTests.swift in Sources */, + D20C00000000000000000008 /* EstimatedKeywordDifficultyCSVExportTests.swift in Sources */, + D20C00000000000000000030 /* EstimatedKeywordDifficultyDetailModelTests.swift in Sources */, + D20C00000000000000000031 /* EstimatedKeywordDifficultyWorkspaceUITests.swift in Sources */, 0A5C00000000000000000005 /* AppStoreConnectReviewServiceTests.swift in Sources */, 0A5C00000000000000000007 /* AppStoreWebMetadataProviderTests.swift in Sources */, + A19000000000000000000003 /* AppStoreWebRankingProviderTests.swift in Sources */, 0A1B2C3D4E5F67890123450D /* AppStorefrontRatingServiceTests.swift in Sources */, C5A400000000000000000003 /* CSVFormatsTests.swift in Sources */, 3A3B766B1E13C1E131179C31 /* DefaultAppResolverTests.swift in Sources */, A301EA7838159D6EE3E53FD0 /* ITunesSearchFallbackProviderTests.swift in Sources */, + D20A00000000000000000003 /* KeywordDifficultyEstimatorTests.swift in Sources */, 04E8C4E18F1A4D519C0A1127 /* KeywordMetricsServiceTests.swift in Sources */, A2000000000000000000000B /* KeywordRankingHistoryTests.swift in Sources */, B7000000000000000000000A /* AddKeywordsStorefrontProjectionTests.swift in Sources */, @@ -1156,6 +1255,7 @@ D20B00000000000000000006 /* EstimatedKeywordDifficultyResultEvidenceRecord.swift in Sources */, C15A00000000000000000108 /* Persistence/BackgroundModelStore.swift in Sources */, D20B00000000000000000008 /* Persistence/EstimatedKeywordDifficultyStore.swift in Sources */, + D20C00000000000000000003 /* Persistence/EstimatedKeywordDifficultyPresentation.swift in Sources */, C15A00000000000000000109 /* Persistence/ModelContainerFactory.swift in Sources */, C15A0000000000000000010A /* OpenASOError.swift in Sources */, C15A0000000000000000010B /* AppPlatform.swift in Sources */, @@ -1182,6 +1282,8 @@ C15A0000000000000000011E /* Networking/HTTPClient.swift in Sources */, C15A0000000000000000011F /* SearchRanking/AppResolver.swift in Sources */, C15A00000000000000000120 /* SearchRanking/SearchModels.swift in Sources */, + D20A00000000000000000002 /* SearchRanking/KeywordDifficultyEstimator.swift in Sources */, + D20C00000000000000000032 /* SearchRanking/EstimatedKeywordDifficultyPersistenceAdapter.swift in Sources */, C15A00000000000000000121 /* SearchRanking/DefaultAppResolver.swift in Sources */, C15A00000000000000000122 /* Storefront/AppCatalogService.swift in Sources */, C15A00000000000000000123 /* Storefront/AppStoreWebMetadataProvider.swift in Sources */, diff --git a/OpenASO/App/AppServices.swift b/OpenASO/App/AppServices.swift index 1beb873..f41843a 100644 --- a/OpenASO/App/AppServices.swift +++ b/OpenASO/App/AppServices.swift @@ -113,7 +113,7 @@ final class AppServices { webSessionStore: appleAdsWebSessionStore ) let keywordInsightsService = KeywordInsightsService() - let rankingProvider = ITunesSearchFallbackProvider(httpClient: httpClient) + let rankingProvider = SearchRankingProviderFactory.makeProduction(httpClient: httpClient) let mcpRankingRefreshScheduler = OpenASOMCPRankingRefreshScheduler() let refreshProgressStore = AppRefreshProgressStore() let storefrontCatalog = StorefrontCatalog() diff --git a/OpenASO/App/RootSidebarView.swift b/OpenASO/App/RootSidebarView.swift index e89c947..4cf2a83 100644 --- a/OpenASO/App/RootSidebarView.swift +++ b/OpenASO/App/RootSidebarView.swift @@ -610,50 +610,49 @@ struct RootSidebarView: View { } private func deleteApp(_ trackedApp: TrackedApp) { - let trackedKeywords: [TrackedAppKeyword] - do { - trackedKeywords = try fetchTrackedKeywords(for: trackedApp) - } catch { + guard let backgroundModelStore = services.backgroundModelStore else { currentAlert = SidebarAlertContext( title: "Delete Failed", - message: OpenASOError.map(error).localizedDescription + message: OpenASOError.providerUnavailable( + "OpenASO’s data store is not ready." + ).localizedDescription ) return } - let keywordCount = trackedKeywords.count - if selectedApp?.persistentModelID == trackedApp.persistentModelID { + let request = TrackedAppDeletionRequest(app: trackedApp) + let wasSelected = selectedApp?.persistentModelID == trackedApp.persistentModelID + if wasSelected { selectedApp = trackedApps.first { $0.persistentModelID != trackedApp.persistentModelID } } + appPendingDeletion = nil - do { - try TrackedKeywordRefreshStatusStore.deleteStatuses( - for: trackedKeywords.map(\.identityKey), - in: modelContext - ) - for track in trackedKeywords { - modelContext.delete(track) + Task { @MainActor in + do { + let result = try await TrackedKeywordDeletionService.deleteApp( + request, + using: backgroundModelStore + ) + if result.deletedApp { + services.analyticsService.capture( + .trackedAppRemoved(keywordCount: result.deletedTrackCount) + ) + } else if wasSelected { + selectedApp = trackedApps.first { $0.appStoreID == request.appStoreID } + ?? selectedApp + } + } catch { + if wasSelected { + selectedApp = trackedApps.first { $0.appStoreID == request.appStoreID } + ?? selectedApp + } + currentAlert = SidebarAlertContext( + title: "Delete Failed", + message: OpenASOError.map(error).localizedDescription + ) } - modelContext.delete(trackedApp) - try modelContext.save() - services.analyticsService.capture(.trackedAppRemoved(keywordCount: keywordCount)) - } catch { - currentAlert = SidebarAlertContext( - title: "Delete Failed", - message: OpenASOError.map(error).localizedDescription - ) } } - - private func fetchTrackedKeywords(for trackedApp: TrackedApp) throws -> [TrackedAppKeyword] { - let appStoreID = trackedApp.appStoreID - let descriptor = FetchDescriptor( - predicate: #Predicate { track in - track.appStoreID == appStoreID - } - ) - return try modelContext.fetch(descriptor) - } } private struct SidebarUtilityRow: View { diff --git a/OpenASO/Features/AppDetail/AppDetailCSVTransfer.swift b/OpenASO/Features/AppDetail/AppDetailCSVTransfer.swift index fb59437..9e51360 100644 --- a/OpenASO/Features/AppDetail/AppDetailCSVTransfer.swift +++ b/OpenASO/Features/AppDetail/AppDetailCSVTransfer.swift @@ -286,3 +286,116 @@ struct AppDetailRatingsCSVExporter { value.formatted(.number.precision(.fractionLength(0...4))) } } + +@MainActor +struct AppDetailEstimatedDifficultyCSVExporter { + static func makeDocument( + appStoreID: Int64, + appName: String, + selectedStorefrontFilter: StorefrontFilter, + selectedPlatformFilter: PlatformFilter, + difficultyFilterRange: ClosedRange, + searchText: String, + backgroundModelStore: BackgroundModelStore?, + asOf date: Date = .now + ) async throws -> CSVDocument { + guard let backgroundModelStore else { + throw OpenASOError.providerUnavailable( + "Estimated-difficulty evidence export is unavailable until the model store is ready." + ) + } + + let selectedStorefront: String? + switch selectedStorefrontFilter { + case .all: + selectedStorefront = nil + case .storefront(let code, _): + selectedStorefront = code + .trimmingCharacters(in: .whitespacesAndNewlines) + .lowercased() + } + + let selectedPlatformRaw: String? + switch selectedPlatformFilter { + case .all: + selectedPlatformRaw = nil + case .platform(let platform): + selectedPlatformRaw = platform.rawValue + } + + let activeDifficultyRange = MetricFilterRange.difficulty.isDefault( + difficultyFilterRange + ) ? nil : difficultyFilterRange + let normalizedSearch = searchText.trimmingCharacters(in: .whitespacesAndNewlines) + let items = try await backgroundModelStore.read { modelContext in + try fetchItems( + appStoreID: appStoreID, + appName: appName, + selectedStorefront: selectedStorefront, + selectedPlatformRaw: selectedPlatformRaw, + activeDifficultyRange: activeDifficultyRange, + normalizedSearch: normalizedSearch, + in: modelContext + ) + } + return CSVDocument( + text: EstimatedKeywordDifficultyCSVFormat.encode(items: items, asOf: date) + ) + } + + nonisolated private static func fetchItems( + appStoreID: Int64, + appName: String, + selectedStorefront: String?, + selectedPlatformRaw: String?, + activeDifficultyRange: ClosedRange?, + normalizedSearch: String, + in modelContext: ModelContext + ) throws -> [EstimatedKeywordDifficultyCSVItem] { + let targetAppStoreID = appStoreID + let descriptor = FetchDescriptor( + predicate: #Predicate { track in + track.appStoreID == targetAppStoreID + }, + sortBy: [ + SortDescriptor(\TrackedAppKeyword.term, order: .forward), + SortDescriptor(\TrackedAppKeyword.storefront, order: .forward), + SortDescriptor(\TrackedAppKeyword.platformRaw, order: .forward) + ] + ) + let scopedTracks = try modelContext.fetch(descriptor).filter { track in + guard selectedStorefront == nil || track.storefront == selectedStorefront else { + return false + } + guard selectedPlatformRaw == nil || track.platformRaw == selectedPlatformRaw else { + return false + } + return normalizedSearch.isEmpty || track.term.localizedStandardContains(normalizedSearch) + } + let snapshots = try EstimatedKeywordDifficultyStore.snapshots( + queryKeys: scopedTracks.map(\.queryKey), + in: modelContext + ) + let tracks = scopedTracks.filter { track in + guard let activeDifficultyRange else { + return true + } + guard let score = snapshots[track.queryKey]?.score else { + return false + } + return activeDifficultyRange.contains(Double(score)) + } + + return tracks.map { track in + EstimatedKeywordDifficultyCSVItem( + appName: appName, + appStoreID: appStoreID, + keyword: track.term, + queryKey: track.queryKey, + storefront: track.storefront, + platformRaw: track.platformRaw, + snapshot: snapshots[track.queryKey] + ) + } + } +} diff --git a/OpenASO/Features/AppDetail/AppDetailSupportTypes.swift b/OpenASO/Features/AppDetail/AppDetailSupportTypes.swift index 83474cd..fac8661 100644 --- a/OpenASO/Features/AppDetail/AppDetailSupportTypes.swift +++ b/OpenASO/Features/AppDetail/AppDetailSupportTypes.swift @@ -255,7 +255,7 @@ enum MetricFilterRange { case .popularity: return "Popularity" case .difficulty: - return "Difficulty" + return "Estimated Difficulty" case .position: return "Position" case .change: diff --git a/OpenASO/Features/AppDetail/AppDetailToolbarViews.swift b/OpenASO/Features/AppDetail/AppDetailToolbarViews.swift index 2656874..c4d1a27 100644 --- a/OpenASO/Features/AppDetail/AppDetailToolbarViews.swift +++ b/OpenASO/Features/AppDetail/AppDetailToolbarViews.swift @@ -150,6 +150,7 @@ struct AppDetailToolbarSearchField: View { struct AppDetailImportExportToolbarMenu: View { let exportAction: () -> Void let exportHistoryAction: () -> Void + let exportEstimatedDifficultyEvidenceAction: () -> Void let importAction: () -> Void let isImportDisabled: Bool @@ -163,6 +164,13 @@ struct AppDetailImportExportToolbarMenu: View { Label("Export Historical Rankings CSV", systemImage: "clock.arrow.circlepath") } + Button(action: exportEstimatedDifficultyEvidenceAction) { + Label( + "Export Estimated Difficulty Evidence CSV", + systemImage: "chart.bar.doc.horizontal" + ) + } + Button(action: importAction) { Label("Import CSV", systemImage: "square.and.arrow.down") } diff --git a/OpenASO/Features/AppDetail/AppDetailView.swift b/OpenASO/Features/AppDetail/AppDetailView.swift index d23e9b0..f0b35f8 100644 --- a/OpenASO/Features/AppDetail/AppDetailView.swift +++ b/OpenASO/Features/AppDetail/AppDetailView.swift @@ -120,6 +120,7 @@ struct AppDetailView: View { AppDetailImportExportToolbarMenu( exportAction: prepareCSVExport, exportHistoryAction: prepareKeywordHistoryCSVExport, + exportEstimatedDifficultyEvidenceAction: prepareEstimatedDifficultyCSVExport, importAction: showCSVImporter, isImportDisabled: isProcessingCSVImport || isImportingCSV ) @@ -601,6 +602,17 @@ struct AppDetailView: View { return sanitizedName.isEmpty ? "ratings.csv" : "\(sanitizedName)-ratings.csv" } + private var estimatedDifficultyExportFilename: String { + let sanitizedName = appName + .components(separatedBy: CharacterSet.alphanumerics.inverted) + .filter { !$0.isEmpty } + .joined(separator: "-") + .lowercased() + let baseName = sanitizedName.isEmpty ? "keywords" : sanitizedName + + return "\(baseName)-estimated-difficulty-evidence.csv" + } + private func makeExportDocument() throws -> CSVDocument { let tracks = try fetchTrackedKeywords() let metricsByQueryKey = try services.keywordMetricsService.metricsMap( @@ -634,24 +646,16 @@ struct AppDetailView: View { private func makeKeywordHistoryExportDocument() throws -> CSVDocument { let tracks = try fetchTrackedKeywords() - let metricsByQueryKey = try services.keywordMetricsService.metricsMap( - for: tracks.map(\.queryKey), - in: modelContext + return try KeywordHistoryCSVExporter.makeDocument( + tracks: tracks, + appName: appName, + appStoreID: appStoreID, + selectedStorefrontFilter: selectedStorefrontFilter, + workspaceState: keywordWorkspaceState, + searchText: searchText, + modelContext: modelContext, + storefrontTitle: storefrontTitle(for:) ) - let rows = tracks - .filter { matchesExportStorefront($0) } - .filter { matchesExportPlatform($0) } - .filter { matchesExportSearch($0) } - .filter { matchesExportMetrics($0, metrics: metricsByQueryKey[$0.queryKey]) } - .filter(matchesExportChangedOnly) - .flatMap { track in - historicalRankingRows( - for: track, - metrics: metricsByQueryKey[track.queryKey] - ) - } - - return CSVDocument(text: KeywordRankingHistoryCSVFormat.encode(rows: rows)) } private func makeRatingsExportDocument() async throws -> CSVDocument { @@ -665,6 +669,18 @@ struct AppDetailView: View { ) } + private func makeEstimatedDifficultyExportDocument() async throws -> CSVDocument { + try await AppDetailEstimatedDifficultyCSVExporter.makeDocument( + appStoreID: appStoreID, + appName: appName, + selectedStorefrontFilter: selectedStorefrontFilter, + selectedPlatformFilter: keywordWorkspaceState.selectedPlatformFilter, + difficultyFilterRange: keywordWorkspaceState.difficultyFilterRange, + searchText: searchText, + backgroundModelStore: services.backgroundModelStore + ) + } + private func prepareCSVExport() { do { exportDocument = try makeExportDocument() @@ -700,6 +716,24 @@ struct AppDetailView: View { } } + private func prepareEstimatedDifficultyCSVExport() { + Task { + do { + exportDocument = try await makeEstimatedDifficultyExportDocument() + exportDefaultFilename = estimatedDifficultyExportFilename + isExportingCSV = true + services.analyticsService.capture( + .csvExported(type: "estimated_difficulty_evidence") + ) + } catch { + transferAlert = TrackedKeywordTransferAlert( + title: "Export Failed", + message: error.localizedDescription + ) + } + } + } + private func showCSVImporter() { guard !isProcessingCSVImport, !isImportingCSV else { return @@ -718,135 +752,6 @@ struct AppDetailView: View { return String(previousRank - currentRank) } - private func historicalRankingRows( - for track: TrackedAppKeyword, - metrics: KeywordDailyMetric? - ) -> [KeywordRankingHistoryCSVRow] { - let snapshots = filteredSnapshots(for: track).sorted { $0.searchedAt < $1.searchedAt } - guard - snapshots.count > 1, - let periodStartRank = snapshots.first?.rank, - let periodEndRank = snapshots.last?.rank, - matchesExportValue(periodStartRank - periodEndRank, in: keywordWorkspaceState.changeFilterRange, configuration: .change) - else { - return [] - } - - var previousRank: Int? - return snapshots.map { snapshot in - let snapshotChange = changeText(currentRank: snapshot.rank, previousRank: previousRank) - let periodChange = changeText(currentRank: snapshot.rank, previousRank: periodStartRank) - previousRank = snapshot.rank - - return KeywordRankingHistoryCSVRow( - appName: appName, - appID: String(appStoreID), - platform: track.platform.rawValue, - keyword: track.term, - storeDomain: track.storefront, - store: storefrontTitle(for: track.storefront), - observedAt: KeywordRankingHistoryCSVFormat.string(from: snapshot.searchedAt), - ranking: snapshot.rank.map(String.init) ?? "1000", - change: snapshotChange, - periodChange: periodChange, - popularity: metrics?.popularityScore.map(String.init) ?? "", - difficulty: metrics?.difficultyScore.map(String.init) ?? "", - appsInRanking: String(snapshot.resultCount), - source: snapshot.sourceRaw, - error: snapshot.errorMessage ?? "" - ) - } - } - - private func filteredSnapshots(for track: TrackedAppKeyword) -> [TrackedKeywordDailyRanking] { - track.sortedSnapshots.filter { snapshot in - guard let cutoffDate = keywordWorkspaceState.selectedDateRange.cutoffDate else { - return true - } - - return snapshot.searchedAt >= cutoffDate - } - } - - private func matchesExportStorefront(_ track: TrackedAppKeyword) -> Bool { - switch selectedStorefrontFilter { - case .all: - return true - case .storefront(let code, _): - return track.storefront == code - } - } - - private func matchesExportPlatform(_ track: TrackedAppKeyword) -> Bool { - keywordWorkspaceState.selectedPlatformFilter.matches(track.platform) - } - - private func matchesExportSearch(_ track: TrackedAppKeyword) -> Bool { - let trimmedSearch = searchText.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmedSearch.isEmpty else { - return true - } - - return track.term.localizedCaseInsensitiveContains(trimmedSearch) - } - - private func matchesExportMetrics(_ track: TrackedAppKeyword, metrics: KeywordDailyMetric?) -> Bool { - guard matchesExportValue(metrics?.popularityScore, in: keywordWorkspaceState.popularityFilterRange, configuration: .popularity), - matchesExportValue(metrics?.difficultyScore, in: keywordWorkspaceState.difficultyFilterRange, configuration: .difficulty) - else { - return false - } - - let snapshots = filteredSnapshots(for: track) - guard let latestRank = snapshots.last?.rank else { - return MetricFilterRange.position.isDefault(keywordWorkspaceState.positionFilterRange) - } - - return matchesExportValue(latestRank, in: keywordWorkspaceState.positionFilterRange, configuration: .position) - } - - private func matchesExportChangedOnly(_ track: TrackedAppKeyword) -> Bool { - guard keywordWorkspaceState.showsOnlyChangedKeywords else { - return true - } - - let rankedSnapshots = filteredSnapshots(for: track).compactMap(\.rank) - guard let firstRank = rankedSnapshots.first, let latestRank = rankedSnapshots.last else { - return false - } - - return firstRank != latestRank - } - - private func matchesExportValue(_ value: Int?, in range: ClosedRange, configuration: MetricFilterRange) -> Bool { - if configuration.isDefault(range) { - return true - } - - guard let value else { - return false - } - - return range.contains(Double(value)) - } - - private func changeText(currentRank: Int?, previousRank: Int?) -> String { - guard let currentRank, let previousRank else { - return "0" - } - - let change = previousRank - currentRank - if change > 0 { - return "+\(change)" - } - - if change < 0 { - return String(change) - } - - return "0" - } - private func importCSV(from result: Result<[URL], Error>) { guard !isProcessingCSVImport else { return @@ -1315,3 +1220,277 @@ struct AppDetailView: View { return "\(storefront.flagEmoji) \(storefront.name)" } } + +extension AppDetailView { + enum KeywordHistoryCSVExporter { + static func makeDocument( + tracks: [TrackedAppKeyword], + appName: String, + appStoreID: Int64, + selectedStorefrontFilter: StorefrontFilter, + workspaceState: KeywordWorkspaceState, + searchText: String, + modelContext: ModelContext, + storefrontTitle: (String) -> String + ) throws -> CSVDocument { + let queryKeys = tracks.map(\.queryKey) + let metricsByQueryKey = try metricsMap( + queryKeys: queryKeys, + in: modelContext + ) + let estimatedDifficultyByQueryKey = try EstimatedKeywordDifficultyStore.summaries( + queryKeys: queryKeys, + in: modelContext + ) + let rows = tracks + .filter { + matchesStorefront( + $0, + selectedStorefrontFilter: selectedStorefrontFilter + ) + } + .filter { + workspaceState.selectedPlatformFilter.matches($0.platform) + } + .filter { + matchesSearch($0, searchText: searchText) + } + .filter { track in + matchesMetrics( + track, + metrics: metricsByQueryKey[track.queryKey], + estimatedDifficulty: estimatedDifficultyByQueryKey[track.queryKey], + workspaceState: workspaceState + ) + } + .filter { + matchesChangedOnly($0, workspaceState: workspaceState) + } + .flatMap { track in + historicalRows( + for: track, + metrics: metricsByQueryKey[track.queryKey], + appName: appName, + appStoreID: appStoreID, + workspaceState: workspaceState, + storefrontTitle: storefrontTitle + ) + } + + return CSVDocument(text: KeywordRankingHistoryCSVFormat.encode(rows: rows)) + } + + private static func metricsMap( + queryKeys: [String], + in modelContext: ModelContext + ) throws -> [String: KeywordDailyMetric] { + let uniqueQueryKeys = Array(Set(queryKeys)) + guard !uniqueQueryKeys.isEmpty else { return [:] } + + let targetQueryKeys = uniqueQueryKeys + let descriptor = FetchDescriptor( + predicate: #Predicate { metric in + targetQueryKeys.contains(metric.queryKey) + } + ) + return Dictionary( + uniqueKeysWithValues: try modelContext.fetch(descriptor).map { + ($0.queryKey, $0) + } + ) + } + + private static func historicalRows( + for track: TrackedAppKeyword, + metrics: KeywordDailyMetric?, + appName: String, + appStoreID: Int64, + workspaceState: KeywordWorkspaceState, + storefrontTitle: (String) -> String + ) -> [KeywordRankingHistoryCSVRow] { + let snapshots = filteredSnapshots( + for: track, + selectedDateRange: workspaceState.selectedDateRange + ) + guard + snapshots.count > 1, + let periodStartRank = snapshots.first?.rank, + let periodEndRank = snapshots.last?.rank, + matchesValue( + periodStartRank - periodEndRank, + in: workspaceState.changeFilterRange, + configuration: .change + ) + else { + return [] + } + + var previousRank: Int? + return snapshots.map { snapshot in + let snapshotChange = changeText( + currentRank: snapshot.rank, + previousRank: previousRank + ) + let periodChange = changeText( + currentRank: snapshot.rank, + previousRank: periodStartRank + ) + previousRank = snapshot.rank + + return KeywordRankingHistoryCSVRow( + appName: appName, + appID: String(appStoreID), + platform: track.platform.rawValue, + keyword: track.term, + storeDomain: track.storefront, + store: storefrontTitle(track.storefront), + observedAt: KeywordRankingHistoryCSVFormat.string(from: snapshot.searchedAt), + ranking: snapshot.rank.map(String.init) ?? "1000", + change: snapshotChange, + periodChange: periodChange, + popularity: metrics?.popularityScore.map(String.init) ?? "", + difficulty: metrics?.difficultyScore.map(String.init) ?? "", + appsInRanking: String(snapshot.resultCount), + source: snapshot.sourceRaw, + error: snapshot.errorMessage ?? "" + ) + } + } + + private static func filteredSnapshots( + for track: TrackedAppKeyword, + selectedDateRange: TrendDateRange + ) -> [TrackedKeywordDailyRanking] { + track.sortedSnapshots.filter { snapshot in + guard let cutoffDate = selectedDateRange.cutoffDate else { + return true + } + + return snapshot.searchedAt >= cutoffDate + } + } + + private static func matchesStorefront( + _ track: TrackedAppKeyword, + selectedStorefrontFilter: StorefrontFilter + ) -> Bool { + switch selectedStorefrontFilter { + case .all: + return true + case .storefront(let code, _): + return track.storefront == code + } + } + + private static func matchesSearch( + _ track: TrackedAppKeyword, + searchText: String + ) -> Bool { + let trimmedSearch = searchText.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmedSearch.isEmpty else { + return true + } + + return track.term.localizedCaseInsensitiveContains(trimmedSearch) + } + + private static func matchesMetrics( + _ track: TrackedAppKeyword, + metrics: KeywordDailyMetric?, + estimatedDifficulty: EstimatedKeywordDifficultySummary?, + workspaceState: KeywordWorkspaceState + ) -> Bool { + guard matchesValue( + metrics?.popularityScore, + in: workspaceState.popularityFilterRange, + configuration: .popularity + ), matchesValue( + estimatedDifficultyScore(from: estimatedDifficulty), + in: workspaceState.difficultyFilterRange, + configuration: .difficulty + ) else { + return false + } + + let snapshots = filteredSnapshots( + for: track, + selectedDateRange: workspaceState.selectedDateRange + ) + guard let latestRank = snapshots.last?.rank else { + return MetricFilterRange.position.isDefault( + workspaceState.positionFilterRange + ) + } + + return matchesValue( + latestRank, + in: workspaceState.positionFilterRange, + configuration: .position + ) + } + + private static func estimatedDifficultyScore( + from summary: EstimatedKeywordDifficultySummary? + ) -> Int? { + guard let summary, summary.state == .estimated else { return nil } + return summary.score + } + + private static func matchesChangedOnly( + _ track: TrackedAppKeyword, + workspaceState: KeywordWorkspaceState + ) -> Bool { + guard workspaceState.showsOnlyChangedKeywords else { + return true + } + + let rankedSnapshots = filteredSnapshots( + for: track, + selectedDateRange: workspaceState.selectedDateRange + ).compactMap(\.rank) + guard let firstRank = rankedSnapshots.first, + let latestRank = rankedSnapshots.last + else { + return false + } + + return firstRank != latestRank + } + + private static func matchesValue( + _ value: Int?, + in range: ClosedRange, + configuration: MetricFilterRange + ) -> Bool { + if configuration.isDefault(range) { + return true + } + + guard let value else { + return false + } + + return range.contains(Double(value)) + } + + private static func changeText( + currentRank: Int?, + previousRank: Int? + ) -> String { + guard let currentRank, let previousRank else { + return "0" + } + + let change = previousRank - currentRank + if change > 0 { + return "+\(change)" + } + + if change < 0 { + return String(change) + } + + return "0" + } + } +} diff --git a/OpenASO/Features/AppDetail/Keywords/AppKeywordsView.swift b/OpenASO/Features/AppDetail/Keywords/AppKeywordsView.swift index 6ee4372..570c80a 100644 --- a/OpenASO/Features/AppDetail/Keywords/AppKeywordsView.swift +++ b/OpenASO/Features/AppDetail/Keywords/AppKeywordsView.swift @@ -164,12 +164,18 @@ struct AppKeywordsView: View { let topResultsByCrawlKey: [String: [KeywordRankingAppSummary]] } + private struct MetricSnapshotBundle: Sendable { + let legacyByQueryKey: [String: KeywordMetricsSnapshot] + let estimatedDifficultyByQueryKey: [String: EstimatedKeywordDifficultySummary] + } + private static let rankingFetchChunkSize = 500 private func makeRows( from tracks: [TrackedAppKeyword], snapshotBuckets: SnapshotBuckets, metricsByQueryKey: [String: KeywordMetricsSnapshot], + estimatedDifficultyByQueryKey: [String: EstimatedKeywordDifficultySummary], refreshStatusesByIdentityKey: [String: KeywordRefreshStatusSnapshot] ) -> [KeywordWorkspaceRow] { let storefrontLookup = storefrontLookup @@ -182,6 +188,7 @@ struct AppKeywordsView: View { snapshotBuckets: snapshotBuckets, storefrontLookup: storefrontLookup, metricsByQueryKey: metricsByQueryKey, + estimatedDifficultyByQueryKey: estimatedDifficultyByQueryKey, refreshStatusesByIdentityKey: refreshStatusesByIdentityKey ) else { continue @@ -241,13 +248,14 @@ struct AppKeywordsView: View { ) { try Task.checkCancellation() let platformTracks = tracks.filter(matchesPlatform) - let loadedMetrics = try await loadMetricsSnapshots(for: platformTracks.map(\.queryKey)) + let loadedMetrics = try await loadMetricSnapshots(for: platformTracks.map(\.queryKey)) try Task.checkCancellation() let snapshotBuckets = try fetchSnapshotBuckets(for: platformTracks) let loadedRows = makeRows( from: platformTracks, snapshotBuckets: snapshotBuckets, - metricsByQueryKey: loadedMetrics, + metricsByQueryKey: loadedMetrics.legacyByQueryKey, + estimatedDifficultyByQueryKey: loadedMetrics.estimatedDifficultyByQueryKey, refreshStatusesByIdentityKey: TrackedKeywordRefreshStatusStore.snapshots( from: refreshStatuses ) @@ -270,18 +278,39 @@ struct AppKeywordsView: View { } } - private func loadMetricsSnapshots(for queryKeys: [String]) async throws -> [String: KeywordMetricsSnapshot] { + private func loadMetricSnapshots(for queryKeys: [String]) async throws -> MetricSnapshotBundle { guard !queryKeys.isEmpty else { - return [:] + return MetricSnapshotBundle( + legacyByQueryKey: [:], + estimatedDifficultyByQueryKey: [:] + ) } if let backgroundModelStore = services.backgroundModelStore { return try await backgroundModelStore.read { modelContext in - try KeywordMetricsSnapshot.map(for: queryKeys, in: modelContext) + try MetricSnapshotBundle( + legacyByQueryKey: KeywordMetricsSnapshot.map( + for: queryKeys, + in: modelContext + ), + estimatedDifficultyByQueryKey: EstimatedKeywordDifficultyStore.summaries( + queryKeys: queryKeys, + in: modelContext + ) + ) } } - return try KeywordMetricsSnapshot.map(for: queryKeys, in: modelContext) + return try MetricSnapshotBundle( + legacyByQueryKey: KeywordMetricsSnapshot.map( + for: queryKeys, + in: modelContext + ), + estimatedDifficultyByQueryKey: EstimatedKeywordDifficultyStore.summaries( + queryKeys: queryKeys, + in: modelContext + ) + ) } private func matchesPlatform(for track: TrackedAppKeyword) -> Bool { @@ -307,6 +336,7 @@ struct AppKeywordsView: View { snapshotBuckets: SnapshotBuckets, storefrontLookup: [String: StorefrontDefinition], metricsByQueryKey: [String: KeywordMetricsSnapshot], + estimatedDifficultyByQueryKey: [String: EstimatedKeywordDifficultySummary], refreshStatusesByIdentityKey: [String: KeywordRefreshStatusSnapshot] ) -> KeywordWorkspaceRow? { let latestSnapshot = snapshotBuckets.latestByTrackKey[track.identityKey] @@ -317,6 +347,7 @@ struct AppKeywordsView: View { track: track, storefront: storefrontLookup[track.storefront], metrics: metricsByQueryKey[track.queryKey], + estimatedDifficulty: estimatedDifficultyByQueryKey[track.queryKey], refreshStatus: TrackedKeywordRefreshStatusStore.snapshot( for: track, persisted: refreshStatusesByIdentityKey[track.identityKey] diff --git a/OpenASO/Features/AppDetail/Keywords/KeywordWorkspaceProjection.swift b/OpenASO/Features/AppDetail/Keywords/KeywordWorkspaceProjection.swift index 01bf3b2..4a8e88c 100644 --- a/OpenASO/Features/AppDetail/Keywords/KeywordWorkspaceProjection.swift +++ b/OpenASO/Features/AppDetail/Keywords/KeywordWorkspaceProjection.swift @@ -65,7 +65,7 @@ enum KeywordWorkspaceProjection { return rows.filter { row in matchesSearch(row, searchText: searchText) && matches(row.metrics?.popularityScore, in: filters.popularityRange, configuration: .popularity) - && matches(row.metrics?.difficultyScore, in: filters.difficultyRange, configuration: .difficulty) + && matches(row.estimatedDifficultyScore, in: filters.difficultyRange, configuration: .difficulty) && matches(row.currentRank, in: filters.positionRange, configuration: .position) && matches(row.trendDelta, in: filters.changeRange, configuration: .change) && (!filters.showsOnlyChangedKeywords || row.trendDelta.map { $0 != 0 } == true) diff --git a/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyCell.swift b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyCell.swift new file mode 100644 index 0000000..b58c152 --- /dev/null +++ b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyCell.swift @@ -0,0 +1,68 @@ +import SwiftUI + +struct EstimatedKeywordDifficultyCell: View { + let keyword: String + let summary: EstimatedKeywordDifficultySummary? + let asOf: Date + let showDetails: () -> Void + + private var presentation: EstimatedKeywordDifficultyRowPresentation { + EstimatedKeywordDifficultyRowPresentation(summary: summary, asOf: asOf) + } + + var body: some View { + Button(action: showDetails) { + HStack(spacing: 6) { + switch presentation.state { + case .missing: + Label("Not estimated", systemImage: "minus.circle") + .foregroundStyle(.secondary) + case .estimated(let score, _, let confidence, let isStale): + MetricBarView( + value: score, + maxValue: 100, + colorScale: .lowGreenHighRed, + placeholder: "-" + ) + + Text(confidence.displayName) + .font(.caption) + .foregroundStyle(.secondary) + + if isStale { + Image(systemName: "clock.badge.exclamationmark") + .foregroundStyle(.orange) + .accessibilityHidden(true) + } + case .unavailable(_, let isStale): + Label("Unavailable", systemImage: "exclamationmark.circle") + .foregroundStyle(.secondary) + + if isStale { + Image(systemName: "clock.badge.exclamationmark") + .foregroundStyle(.orange) + .accessibilityHidden(true) + } + case .unsupported(let isStale): + Label("Unavailable", systemImage: "questionmark.circle") + .foregroundStyle(.secondary) + + if isStale { + Image(systemName: "clock.badge.exclamationmark") + .foregroundStyle(.orange) + .accessibilityHidden(true) + } + } + } + .font(.subheadline) + .frame(maxWidth: .infinity, alignment: .leading) + .contentShape(.rect) + } + .buttonStyle(.plain) + .help("Show estimated difficulty details") + .accessibilityElement(children: .ignore) + .accessibilityLabel("Estimated difficulty for \(keyword)") + .accessibilityValue(presentation.accessibilityValue) + .accessibilityHint("Opens the local estimate, provenance, and ranking evidence.") + } +} diff --git a/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyDetailContent.swift b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyDetailContent.swift new file mode 100644 index 0000000..8776b8d --- /dev/null +++ b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyDetailContent.swift @@ -0,0 +1,27 @@ +import SwiftUI + +struct EstimatedKeywordDifficultyDetailContent: View { + let snapshot: EstimatedKeywordDifficultySnapshot + + var body: some View { + ScrollView { + VStack(alignment: .leading, spacing: 16) { + GroupBox { + Label { + Text("This is a local heuristic derived from public App Store ranking evidence. It is not Apple Ads difficulty, Search Popularity, or an Apple-provided metric.") + .fixedSize(horizontal: false, vertical: true) + } icon: { + Image(systemName: "info.circle.fill") + .foregroundStyle(.blue) + } + .frame(maxWidth: .infinity, alignment: .leading) + } + + EstimatedKeywordDifficultyEstimateSection(snapshot: snapshot) + EstimatedKeywordDifficultyEvidenceSection(snapshot: snapshot) + EstimatedKeywordDifficultyProvenanceSection(snapshot: snapshot) + } + .padding(20) + } + } +} diff --git a/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyDetailDataSource.swift b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyDetailDataSource.swift new file mode 100644 index 0000000..26eb4d7 --- /dev/null +++ b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyDetailDataSource.swift @@ -0,0 +1,45 @@ +import SwiftData + +@MainActor +struct EstimatedKeywordDifficultyDetailDataSource { + typealias LoadOperation = @MainActor ( + _ queryKey: String + ) async throws -> EstimatedKeywordDifficultySnapshot? + + private let loadOperation: LoadOperation + + init(load: @escaping LoadOperation) { + self.loadOperation = load + } + + func load(queryKey: String) async throws -> EstimatedKeywordDifficultySnapshot? { + try await loadOperation(queryKey) + } + + static func production( + backgroundModelStore: BackgroundModelStore?, + fallbackModelContext: ModelContext + ) -> EstimatedKeywordDifficultyDetailDataSource { + EstimatedKeywordDifficultyDetailDataSource { queryKey in + try Task.checkCancellation() + + if let backgroundModelStore { + let snapshot = try await backgroundModelStore.read { modelContext in + try EstimatedKeywordDifficultyStore.snapshot( + queryKey: queryKey, + in: modelContext + ) + } + try Task.checkCancellation() + return snapshot + } + + let snapshot = try EstimatedKeywordDifficultyStore.snapshot( + queryKey: queryKey, + in: fallbackModelContext + ) + try Task.checkCancellation() + return snapshot + } + } +} diff --git a/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyDetailModel.swift b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyDetailModel.swift new file mode 100644 index 0000000..4d8ab4f --- /dev/null +++ b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyDetailModel.swift @@ -0,0 +1,60 @@ +import Observation + +@Observable +@MainActor +final class EstimatedKeywordDifficultyDetailModel { + enum State: Equatable, Sendable { + case idle + case loading + case missing + case loaded(EstimatedKeywordDifficultySnapshot) + case failed(String) + } + + private(set) var state: State = .idle + @ObservationIgnored private var requestGeneration = 0 + + var snapshot: EstimatedKeywordDifficultySnapshot? { + guard case let .loaded(snapshot) = state else { return nil } + return snapshot + } + + var errorMessage: String? { + guard case let .failed(message) = state else { return nil } + return message + } + + func load( + queryKey: String, + using dataSource: EstimatedKeywordDifficultyDetailDataSource + ) async { + guard !Task.isCancelled else { return } + + requestGeneration &+= 1 + let generation = requestGeneration + state = .loading + + do { + let snapshot = try await dataSource.load(queryKey: queryKey) + try Task.checkCancellation() + guard generation == requestGeneration else { return } + if let snapshot { + state = .loaded(snapshot) + } else { + state = .missing + } + } catch is CancellationError { + guard generation == requestGeneration else { return } + state = .idle + } catch { + guard generation == requestGeneration else { return } + guard !Task.isCancelled else { + state = .idle + return + } + state = .failed( + "OpenASO couldn’t load the saved estimated-difficulty details. Try again." + ) + } + } +} diff --git a/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyDetailSheet.swift b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyDetailSheet.swift new file mode 100644 index 0000000..4b77afd --- /dev/null +++ b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyDetailSheet.swift @@ -0,0 +1,86 @@ +import SwiftData +import SwiftUI + +struct EstimatedKeywordDifficultyDetailSheet: View { + @Environment(AppServices.self) private var services + @Environment(\.dismiss) private var dismiss + + let keyword: String + let queryKey: String + let summary: EstimatedKeywordDifficultySummary? + let modelContext: ModelContext + + @State private var detailModel = EstimatedKeywordDifficultyDetailModel() + @State private var retryToken = 0 + + var body: some View { + VStack(spacing: 0) { + HStack(alignment: .firstTextBaseline) { + VStack(alignment: .leading, spacing: 4) { + Text("Estimated Difficulty") + .font(.title2.bold()) + Text(keyword) + .font(.headline) + Text(scopeDescription) + .foregroundStyle(.secondary) + } + + Spacer() + + Button("Done", action: dismiss.callAsFunction) + .keyboardShortcut(.defaultAction) + } + .padding(20) + + Divider() + + switch detailModel.state { + case .idle, .loading: + ProgressView("Loading estimated difficulty…") + .frame(maxWidth: .infinity, maxHeight: .infinity) + case .missing: + ContentUnavailableView { + Label("No Estimated Difficulty Yet", systemImage: "gauge.with.dots.needle.0percent") + } description: { + Text("Refresh this keyword to calculate a local estimate from its App Store ranking evidence.") + } actions: { + Button("Try Again", action: retryLoading) + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + case .loaded(let snapshot): + EstimatedKeywordDifficultyDetailContent(snapshot: snapshot) + case .failed(let message): + ContentUnavailableView { + Label("Unable to Load Estimated Difficulty", systemImage: "exclamationmark.triangle") + } description: { + Text(message) + } actions: { + Button("Try Again", action: retryLoading) + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + } + } + .task(id: retryToken) { + let dataSource = EstimatedKeywordDifficultyDetailDataSource.production( + backgroundModelStore: services.backgroundModelStore, + fallbackModelContext: modelContext + ) + await detailModel.load(queryKey: queryKey, using: dataSource) + } + .frame(minWidth: 720, idealWidth: 820, minHeight: 620, idealHeight: 780) + } + + private var scopeDescription: String { + let storefront = detailModel.snapshot?.storefront.uppercased() + ?? summary?.storefront.uppercased() + ?? "Unknown country" + let platform = detailModel.snapshot?.platform?.displayName + ?? summary?.platform?.displayName + ?? "Unknown platform" + return "\(storefront) · \(platform)" + } + + private func retryLoading() { + retryToken &+= 1 + } +} diff --git a/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyDisplayNames.swift b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyDisplayNames.swift new file mode 100644 index 0000000..df00312 --- /dev/null +++ b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyDisplayNames.swift @@ -0,0 +1,97 @@ +import Foundation + +extension EstimatedKeywordDifficultyConfidence { + var displayName: String { + switch self { + case .low: + "Low" + case .medium: + "Medium" + case .high: + "High" + } + } +} + +extension EstimatedKeywordDifficultyUnavailableReason { + var displayName: String { + switch self { + case .emptyKeyword: + "Empty keyword" + case .insufficientResults: + "Not enough ranking results" + case .insufficientRatingEvidence: + "Not enough rating evidence" + } + } + + var guidance: String { + switch self { + case .emptyKeyword: + "Enter a keyword before requesting a ranking refresh." + case .insufficientResults: + "Refresh again when the App Store returns at least three unique ranking results." + case .insufficientRatingEvidence: + "The sampled ranking needs rating counts for at least three results before OpenASO can estimate difficulty." + } + } +} + +extension EstimatedKeywordDifficultySource { + var displayName: String { + switch self { + case .topResultsHeuristic: + "Top-results heuristic" + } + } +} + +extension EstimatedKeywordDifficultyFallbackCategory { + var displayName: String { + switch self { + case .transport: + "Connection" + case .httpStatus: + "HTTP response" + case .response: + "Response validation" + case .provider: + "Provider" + case .other: + "Other" + } + } +} + +extension EstimatedKeywordDifficultyFallbackResponseFailure { + var displayName: String { + switch self { + case .serializedServerDataMissing: + "Server data was missing" + case .decodingFailed: + "Response could not be decoded" + case .nonHTTPResponse: + "Response was not HTTP" + case .requestIntentMissing: + "Search request could not be identified" + case .requestIntentAmbiguous: + "Search request was ambiguous" + case .pageShapeChanged: + "Search page format changed" + case .authoritativeShelfMissing: + "Primary result shelf was missing" + case .authoritativeShelfAmbiguous: + "Primary result shelf was ambiguous" + case .malformedSearchResult: + "A search result was malformed" + case .truncatedResults: + "Search results were incomplete" + } + } +} + +extension EstimatedKeywordDifficultySnapshot { + var fallbackProvider: RankingSource? { + fallbackProviderRaw.flatMap(RankingSource.init(rawValue:)) + } +} diff --git a/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyEstimateSection.swift b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyEstimateSection.swift new file mode 100644 index 0000000..e214f20 --- /dev/null +++ b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyEstimateSection.swift @@ -0,0 +1,72 @@ +import SwiftUI + +struct EstimatedKeywordDifficultyEstimateSection: View { + let snapshot: EstimatedKeywordDifficultySnapshot + + var body: some View { + TimelineView(.periodic(from: .now, by: 60)) { timeline in + content(isStale: snapshot.isStale(asOf: timeline.date)) + } + } + + private func content(isStale: Bool) -> some View { + GroupBox("Estimate") { + VStack(alignment: .leading, spacing: 10) { + switch snapshot.state { + case .estimated: + if let score = snapshot.score, + let confidenceScore = snapshot.confidenceScore, + let confidence = snapshot.confidence { + LabeledContent("Estimated difficulty") { + Text(score, format: .number) + .monospacedDigit() + Text(" / 100") + .foregroundStyle(.secondary) + } + + LabeledContent("Confidence") { + Text(confidence.displayName) + Text("(\(confidenceScore) / 100)") + .foregroundStyle(.secondary) + .monospacedDigit() + } + } else { + LabeledContent("Status", value: "Stored estimate is unavailable") + } + case .unavailable: + if let unavailableReason = snapshot.unavailableReason { + LabeledContent("Status", value: "Unavailable") + LabeledContent("Reason", value: unavailableReason.displayName) + Text(unavailableReason.guidance) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } else { + LabeledContent("Status", value: "Stored estimate is unavailable") + } + case nil: + LabeledContent("Status", value: "Stored estimate is unavailable") + } + + Divider() + + LabeledContent("Evidence freshness") { + Label( + isStale ? "Stale" : "Current", + systemImage: isStale + ? "clock.badge.exclamationmark" + : "checkmark.circle.fill" + ) + .foregroundStyle(isStale ? .orange : .green) + } + + if isStale { + Text("Refresh this keyword to calculate a new estimate from current ranking evidence.") + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.top, 4) + } + } +} diff --git a/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyEvidenceSection.swift b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyEvidenceSection.swift new file mode 100644 index 0000000..8e4d57c --- /dev/null +++ b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyEvidenceSection.swift @@ -0,0 +1,149 @@ +import SwiftUI + +struct EstimatedKeywordDifficultyEvidenceSection: View { + let snapshot: EstimatedKeywordDifficultySnapshot + + var body: some View { + GroupBox("Evidence") { + VStack(alignment: .leading, spacing: 14) { + Grid(alignment: .leading, horizontalSpacing: 24, verticalSpacing: 8) { + GridRow { + LabeledContent("Results considered", value: snapshot.consideredResultCount.formatted()) + LabeledContent("Results with ratings", value: snapshot.ratedResultCount.formatted()) + } + GridRow { + LabeledContent( + "Weighted rating coverage", + value: "\(snapshot.weightedRatingCoveragePercentage)%" + ) + LabeledContent( + "Rating authority", + value: scoreText(snapshot.ratingAuthorityScore) + ) + } + GridRow { + LabeledContent( + "Metadata saturation", + value: scoreText(snapshot.metadataSaturationScore) + ) + LabeledContent( + "Maximum rating count", + value: countText(snapshot.maximumRatingCount) + ) + } + GridRow { + LabeledContent( + "Median rating count", + value: countText(snapshot.medianRatingCount) + ) + LabeledContent( + "Exact phrase matches", + value: "\(snapshot.exactTitlePhraseMatchCount) title, \(snapshot.exactSubtitlePhraseMatchCount) subtitle" + ) + } + } + + Divider() + + if snapshot.resultEvidence.isEmpty { + Text("No individual ranking results were available for this calculation.") + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } else { + Text("Ranking results used") + .font(.headline) + + ForEach(snapshot.resultEvidence.indices, id: \.self) { index in + let evidence = snapshot.resultEvidence[index] + EstimatedKeywordDifficultyEvidenceRow(evidence: evidence) + + if index < snapshot.resultEvidence.count - 1 { + Divider() + } + } + } + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.top, 4) + } + } + + private func scoreText(_ value: Int?) -> String { + value.map { "\($0) / 100" } ?? "Not available" + } + + private func countText(_ value: Int?) -> String { + value?.formatted(.number) ?? "Not available" + } +} + +private struct EstimatedKeywordDifficultyEvidenceRow: View { + let evidence: EstimatedKeywordDifficultyResultEvidence + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + HStack(alignment: .firstTextBaseline) { + Text("#\(evidence.position)") + .font(.headline.monospacedDigit()) + .accessibilityLabel("Rank \(evidence.position)") + + VStack(alignment: .leading, spacing: 2) { + Text(evidence.title) + .font(.headline) + if let subtitle = evidence.subtitle, !subtitle.isEmpty { + Text(subtitle) + .foregroundStyle(.secondary) + } + } + + Spacer() + + Text("App ID \(evidence.appStoreID)") + .font(.caption.monospacedDigit()) + .foregroundStyle(.secondary) + } + + ViewThatFits(in: .horizontal) { + HStack(spacing: 16) { + evidenceLabels + } + + VStack(alignment: .leading, spacing: 4) { + evidenceLabels + } + } + .font(.caption) + .foregroundStyle(.secondary) + } + .accessibilityElement(children: .combine) + } + + @ViewBuilder + private var evidenceLabels: some View { + Text("Ratings: \(countText(evidence.ratingCount))") + Text("Authority: \(scoreText(evidence.ratingAuthorityScore))") + Text("Title coverage: \(percentageText(evidence.titleTokenCoveragePercentage))") + Text("Combined coverage: \(percentageText(evidence.combinedTokenCoveragePercentage))") + Text("Metadata match: \(scoreText(evidence.metadataMatchScore))") + if evidence.exactTitlePhraseMatch { + Label("Exact title phrase", systemImage: "checkmark.circle.fill") + .foregroundStyle(.green) + } + if evidence.exactSubtitlePhraseMatch { + Label("Exact subtitle phrase", systemImage: "checkmark.circle.fill") + .foregroundStyle(.green) + } + } + + private func countText(_ value: Int?) -> String { + value?.formatted(.number) ?? "Not available" + } + + private func scoreText(_ value: Int?) -> String { + value.map { "\($0) / 100" } ?? "Not available" + } + + private func percentageText(_ value: Int?) -> String { + value.map { "\($0)%" } ?? "Not available" + } +} diff --git a/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyProvenanceSection.swift b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyProvenanceSection.swift new file mode 100644 index 0000000..5a2249d --- /dev/null +++ b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyProvenanceSection.swift @@ -0,0 +1,66 @@ +import SwiftUI + +struct EstimatedKeywordDifficultyProvenanceSection: View { + let snapshot: EstimatedKeywordDifficultySnapshot + + private var staleAt: Date { + snapshot.rankingFetchedAt.addingTimeInterval( + EstimatedKeywordDifficultyFreshness.maximumAge + ) + } + + var body: some View { + GroupBox("Provenance") { + VStack(alignment: .leading, spacing: 10) { + LabeledContent( + "Estimation source", + value: snapshot.estimationSource?.displayName ?? "Unsupported source" + ) + LabeledContent( + "Ranking source", + value: snapshot.rankingSource?.displayName ?? "Unsupported source" + ) + LabeledContent("Ranking fetched") { + Text(snapshot.rankingFetchedAt, format: .dateTime.year().month().day().hour().minute().second()) + } + LabeledContent("Computed") { + Text(snapshot.computedAt, format: .dateTime.year().month().day().hour().minute().second()) + } + LabeledContent("Becomes stale") { + Text(staleAt, format: .dateTime.year().month().day().hour().minute().second()) + } + + Divider() + + LabeledContent("Algorithm", value: snapshot.algorithmIdentifier) + LabeledContent("Algorithm version", value: snapshot.algorithmVersion.formatted()) + LabeledContent("Requested result limit", value: snapshot.requestedResultLimit.formatted()) + LabeledContent("Provider result count", value: snapshot.providerResultCount.formatted()) + + if snapshot.fallbackProvider != nil || snapshot.fallbackCategory != nil { + Divider() + Text("Fallback") + .font(.headline) + + if let fallbackProvider = snapshot.fallbackProvider { + LabeledContent("Provider", value: fallbackProvider.displayName) + } + if let fallbackCategory = snapshot.fallbackCategory { + LabeledContent("Reason category", value: fallbackCategory.displayName) + } + if let fallbackTransportCode = snapshot.fallbackTransportCode { + LabeledContent("Transport code", value: fallbackTransportCode.formatted()) + } + if let fallbackHTTPStatus = snapshot.fallbackHTTPStatus { + LabeledContent("HTTP status", value: fallbackHTTPStatus.formatted()) + } + if let fallbackResponseFailure = snapshot.fallbackResponseFailure { + LabeledContent("Response classification", value: fallbackResponseFailure.displayName) + } + } + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.top, 4) + } + } +} diff --git a/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyRowPresentation.swift b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyRowPresentation.swift new file mode 100644 index 0000000..e7ddc4f --- /dev/null +++ b/OpenASO/Features/AppDetail/Keywords/Table/EstimatedKeywordDifficultyRowPresentation.swift @@ -0,0 +1,66 @@ +import Foundation + +struct EstimatedKeywordDifficultyRowPresentation: Equatable, Sendable { + enum State: Equatable, Sendable { + case missing + case estimated( + score: Int, + confidenceScore: Int, + confidence: EstimatedKeywordDifficultyConfidence, + isStale: Bool + ) + case unavailable( + reason: EstimatedKeywordDifficultyUnavailableReason, + isStale: Bool + ) + case unsupported(isStale: Bool) + } + + let state: State + + init(summary: EstimatedKeywordDifficultySummary?, asOf date: Date) { + let presentation = EstimatedKeywordDifficultyPresentation( + summary: summary, + asOf: date + ) + + switch presentation.value { + case .missing: + state = .missing + case .estimated(let score, let confidenceScore, let confidence): + state = .estimated( + score: score, + confidenceScore: confidenceScore, + confidence: confidence, + isStale: presentation.isStale + ) + case .unavailable(let reason): + state = .unavailable( + reason: reason, + isStale: presentation.isStale + ) + case .unsupported: + state = .unsupported(isStale: presentation.isStale) + } + } + + var accessibilityValue: String { + switch state { + case .missing: + "Not estimated." + case .estimated(let score, let confidenceScore, let confidence, false): + "\(score) out of 100. \(confidence.displayName) confidence, \(confidenceScore) out of 100. Current." + case .estimated(let score, let confidenceScore, let confidence, true): + "\(score) out of 100. \(confidence.displayName) confidence, \(confidenceScore) out of 100. Stale." + case .unavailable(let reason, let isStale): + "Unavailable: \(reason.displayName)." + staleAccessibilitySuffix(isStale) + case .unsupported(let isStale): + "Unavailable because the stored estimate is not supported." + + staleAccessibilitySuffix(isStale) + } + } + + private func staleAccessibilitySuffix(_ isStale: Bool) -> String { + isStale ? " The ranking evidence is stale." : "" + } +} diff --git a/OpenASO/Features/AppDetail/Keywords/Table/KeywordTableSupport.swift b/OpenASO/Features/AppDetail/Keywords/Table/KeywordTableSupport.swift index 2246a48..af10852 100644 --- a/OpenASO/Features/AppDetail/Keywords/Table/KeywordTableSupport.swift +++ b/OpenASO/Features/AppDetail/Keywords/Table/KeywordTableSupport.swift @@ -256,25 +256,20 @@ struct KeywordMetricsSnapshot: Sendable { for queryKeys: [String], in modelContext: ModelContext ) throws -> [String: KeywordMetricsSnapshot] { - guard !queryKeys.isEmpty else { + let selectedQueryKeys = Array(Set(queryKeys)) + guard !selectedQueryKeys.isEmpty else { return [:] } - var snapshotsByQueryKey: [String: KeywordMetricsSnapshot] = [:] - snapshotsByQueryKey.reserveCapacity(queryKeys.count) - - for queryKey in Set(queryKeys) { - let targetQueryKey = queryKey - var descriptor = FetchDescriptor( - predicate: #Predicate { metrics in - metrics.queryKey == targetQueryKey - } - ) - descriptor.fetchLimit = 1 - - if let metrics = try modelContext.fetch(descriptor).first { - snapshotsByQueryKey[metrics.queryKey] = KeywordMetricsSnapshot(metrics) + let descriptor = FetchDescriptor( + predicate: #Predicate { metrics in + selectedQueryKeys.contains(metrics.queryKey) } + ) + var snapshotsByQueryKey: [String: KeywordMetricsSnapshot] = [:] + snapshotsByQueryKey.reserveCapacity(selectedQueryKeys.count) + for metrics in try modelContext.fetch(descriptor) { + snapshotsByQueryKey[metrics.queryKey] = KeywordMetricsSnapshot(metrics) } return snapshotsByQueryKey @@ -287,6 +282,7 @@ struct KeywordWorkspaceRow: Identifiable { let track: TrackedAppKeyword let storefront: StorefrontDefinition? let metrics: KeywordMetricsSnapshot? + let estimatedDifficulty: EstimatedKeywordDifficultySummary? let refreshStatus: KeywordRefreshStatusSnapshot let latestSnapshot: KeywordRankingCrawlSummary? let trendSnapshots: [KeywordRankingCrawlSummary] @@ -300,6 +296,7 @@ struct KeywordWorkspaceRow: Identifiable { track: TrackedAppKeyword, storefront: StorefrontDefinition?, metrics: KeywordMetricsSnapshot?, + estimatedDifficulty: EstimatedKeywordDifficultySummary? = nil, refreshStatus: KeywordRefreshStatusSnapshot = .empty, latestSnapshot: KeywordRankingCrawlSummary?, trendSnapshots: [KeywordRankingCrawlSummary], @@ -309,6 +306,7 @@ struct KeywordWorkspaceRow: Identifiable { self.track = track self.storefront = storefront self.metrics = metrics + self.estimatedDifficulty = estimatedDifficulty self.refreshStatus = refreshStatus self.latestSnapshot = latestSnapshot self.currentRank = latestSnapshot?.rank @@ -334,6 +332,7 @@ struct KeywordWorkspaceRow: Identifiable { track: TrackedAppKeyword, storefront: StorefrontDefinition?, metrics: KeywordDailyMetric?, + estimatedDifficulty: EstimatedKeywordDifficultySummary? = nil, refreshStatus: KeywordRefreshStatusSnapshot = .empty, latestSnapshot: KeywordRankingCrawlSummary?, trendSnapshots: [KeywordRankingCrawlSummary], @@ -344,6 +343,7 @@ struct KeywordWorkspaceRow: Identifiable { track: track, storefront: storefront, metrics: metrics.map(KeywordMetricsSnapshot.init), + estimatedDifficulty: estimatedDifficulty, refreshStatus: refreshStatus, latestSnapshot: latestSnapshot, trendSnapshots: trendSnapshots, @@ -356,6 +356,7 @@ struct KeywordWorkspaceRow: Identifiable { track: TrackedAppKeyword, storefront: StorefrontDefinition?, metrics: KeywordDailyMetric?, + estimatedDifficulty: EstimatedKeywordDifficultySummary? = nil, refreshStatus: KeywordRefreshStatusSnapshot = .empty, latestSnapshot: TrackedKeywordDailyRanking?, trendSnapshots: [TrackedKeywordDailyRanking], @@ -365,6 +366,7 @@ struct KeywordWorkspaceRow: Identifiable { track: track, storefront: storefront, metrics: metrics.map(KeywordMetricsSnapshot.init), + estimatedDifficulty: estimatedDifficulty, refreshStatus: refreshStatus, latestSnapshot: latestSnapshot.map(KeywordRankingCrawlSummary.init), trendSnapshots: trendSnapshots.map(KeywordRankingCrawlSummary.init), @@ -385,6 +387,15 @@ struct KeywordWorkspaceRow: Identifiable { var popularitySortValue: Int { metrics?.popularityScore ?? -1 } + var estimatedDifficultyScore: Int? { + guard estimatedDifficulty?.state == .estimated else { return nil } + return estimatedDifficulty?.score + } + + var estimatedDifficultySortValue: Int { + estimatedDifficultyScore ?? Int.max + } + var positionSortValue: Int { currentRank ?? Int.max } var trendSortValue: Int { trendDelta ?? Int.min } diff --git a/OpenASO/Features/AppDetail/Keywords/Table/KeywordTableView.swift b/OpenASO/Features/AppDetail/Keywords/Table/KeywordTableView.swift index ead83b2..3f8d902 100644 --- a/OpenASO/Features/AppDetail/Keywords/Table/KeywordTableView.swift +++ b/OpenASO/Features/AppDetail/Keywords/Table/KeywordTableView.swift @@ -27,6 +27,7 @@ struct KeywordTableView: View { @State private var selection = Set() @State private var presentedRankingRow: KeywordWorkspaceRow? @State private var presentedRankingHistoryRow: KeywordWorkspaceRow? + @State private var presentedEstimatedDifficultyRow: KeywordWorkspaceRow? @State private var presentedNotesRow: KeywordWorkspaceRow? @State private var actionErrorMessage: String? @State private var rowsPendingDeletion: [KeywordWorkspaceRow] = [] @@ -134,94 +135,29 @@ struct KeywordTableView: View { ) .frame(maxWidth: .infinity, maxHeight: .infinity) } else { - Table(sortedTableRows, selection: $selection, sortOrder: $sortOrder) { - TableColumn("Keyword", value: \.keywordSortValue) { tableRow in - KeywordCell(row: tableRow.row) - } - .width(min: 160, ideal: 230) - - TableColumn("Last updated", value: \.lastUpdatedSortValue) { tableRow in - KeywordLastUpdatedCell(row: tableRow.row) - } - .width(min: 92, ideal: 112, max: 132) - - TableColumn("Country", value: \.storefrontSortValue) { tableRow in - KeywordStoreCell(row: tableRow.row) - } - .width(min: 100, ideal: 148) - - if showsPlatformColumn { - TableColumn("Platform", value: \.platformSortValue) { tableRow in - KeywordPlatformCell(platform: tableRow.row.track.platform) - } - .width(min: 92, ideal: 104, max: 116) - } - - TableColumn("Popularity", value: \.popularitySortValue) { tableRow in - KeywordPopularityCell( - row: tableRow.row, - requiresAppleAdsReconnect: services.appleAdsWebSessionStore.requiresReconnect - ) { - openAppleAdsSettings() - } - } - .width(min: 112, ideal: 124, max: 136) - - TableColumn("Position", value: \.positionSortValue) { tableRow in - KeywordPositionCell(row: tableRow.row) - } - .width(min: 76, ideal: 88, max: 100) - - TableColumn("Trend", value: \.trendSortValue) { tableRow in - Button { - presentedRankingHistoryRow = tableRow.row - } label: { - KeywordTrendCell(row: tableRow.row) - .frame(maxWidth: .infinity, alignment: .leading) - .contentShape(.rect) - } - .buttonStyle(.plain) - .help("Show ranking history") - .accessibilityLabel("Ranking trend for \(tableRow.row.track.term)") - .accessibilityValue( - "\(tableRow.row.trendAccessibilityText) " - + "\(tableRow.row.storefront?.name ?? tableRow.row.track.storefront.uppercased()), " - + "\(tableRow.row.track.platform.displayName)." - ) - .accessibilityHint("Opens ranking history.") - } - .width(min: 104, ideal: 120, max: 132) - - TableColumn("Apps in Ranking") { tableRow in - AppsInRankingButton( - row: tableRow.row, - trackedAppStoreID: trackedAppStoreID, - modelContext: modelContext, - appCatalogService: appCatalogService, - appIconStore: appIconStore, - presentRanking: presentRanking - ) - } - .width(min: 132, ideal: 220) - - TableColumn("Notes") { tableRow in - KeywordNotesCell(row: tableRow.row) { - presentedNotesRow = tableRow.row - } - } - .width(min: 120, ideal: 180) - - TableColumn("Chart", value: \.chartSelectionSortValue) { tableRow in - ChartSelectionButton( - isSelected: tableRow.isSelectedForChart, - setSelection: { isSelected in - setChartSelection(isSelected, for: tableRow.row) - } - ) - .frame(maxWidth: .infinity, alignment: .center) - } - .width(min: 56, ideal: 60, max: 68) - } + KeywordRowsTable( + rows: sortedTableRows, + selection: $selection, + sortOrder: $sortOrder, + showsPlatformColumn: showsPlatformColumn, + requiresAppleAdsReconnect: services.appleAdsWebSessionStore.requiresReconnect, + trackedAppStoreID: trackedAppStoreID, + modelContext: modelContext, + appCatalogService: appCatalogService, + appIconStore: appIconStore, + openAppleAdsSettings: openAppleAdsSettings, + presentEstimatedDifficulty: { row in + presentedEstimatedDifficultyRow = row + }, + presentRankingHistory: { row in + presentedRankingHistoryRow = row + }, + presentRanking: presentRanking, + presentNotes: { row in + presentedNotesRow = row + }, + setChartSelection: setChartSelection + ) .tint(.accentColor) .contextMenu(forSelectionType: PersistentIdentifier.self) { selectedIDs in let contextRows = selectedRows(for: selectedIDs) @@ -270,6 +206,14 @@ struct KeywordTableView: View { modelContext: modelContext ) } + .sheet(item: $presentedEstimatedDifficultyRow) { row in + EstimatedKeywordDifficultyDetailSheet( + keyword: row.track.term, + queryKey: row.track.queryKey, + summary: row.estimatedDifficulty, + modelContext: modelContext + ) + } .sheet(item: $presentedNotesRow) { row in KeywordNotesSheet(track: row.track) } @@ -348,24 +292,39 @@ struct KeywordTableView: View { private func deleteRows(_ rows: [KeywordWorkspaceRow]) { guard !rows.isEmpty else { return } + guard let backgroundModelStore = services.backgroundModelStore else { + actionErrorMessage = OpenASOError.providerUnavailable( + "OpenASO’s data store is not ready." + ).localizedDescription + return + } - var chartKeys = selectedChartKeywordKeys + let requests = rows.map { TrackedKeywordDeletionRequest(track: $0.track) } + let rowIDsByIdentityKey = Dictionary( + uniqueKeysWithValues: rows.map { ($0.track.identityKey, $0.id) } + ) - do { - try TrackedKeywordRefreshStatusStore.deleteStatuses( - for: rows.map(\.track.identityKey), - in: modelContext - ) - rows.forEach { row in - chartKeys.remove(row.track.identityKey) - modelContext.delete(row.track) + Task { @MainActor in + do { + let result = try await TrackedKeywordDeletionService.deleteTracks( + requests, + using: backgroundModelStore + ) + let deletedIdentityKeys = Set(result.deletedIdentityKeys) + guard !deletedIdentityKeys.isEmpty else { return } + + var chartKeys = selectedChartKeywordKeys + chartKeys.subtract(deletedIdentityKeys) + saveChartSelection(chartKeys) + selection.subtract( + deletedIdentityKeys.compactMap { rowIDsByIdentityKey[$0] } + ) + services.analyticsService.capture( + .keywordDeleted(deleteCount: result.deletedTrackCount) + ) + } catch { + actionErrorMessage = OpenASOError.map(error).localizedDescription } - try modelContext.save() - saveChartSelection(chartKeys) - selection.subtract(rows.map(\.id)) - services.analyticsService.capture(.keywordDeleted(deleteCount: rows.count)) - } catch { - actionErrorMessage = OpenASOError.map(error).localizedDescription } } @@ -509,6 +468,160 @@ struct KeywordTableView: View { } } +private struct KeywordRowsTable: View { + let rows: [KeywordTablePresentationRow] + @Binding var selection: Set + @Binding var sortOrder: [KeyPathComparator] + + let showsPlatformColumn: Bool + let requiresAppleAdsReconnect: Bool + let trackedAppStoreID: Int64 + let modelContext: ModelContext + let appCatalogService: AppCatalogService + let appIconStore: AppIconStore + let openAppleAdsSettings: () -> Void + let presentEstimatedDifficulty: (KeywordWorkspaceRow) -> Void + let presentRankingHistory: (KeywordWorkspaceRow) -> Void + let presentRanking: (KeywordWorkspaceRow) -> Void + let presentNotes: (KeywordWorkspaceRow) -> Void + let setChartSelection: (Bool, KeywordWorkspaceRow) -> Void + + var body: some View { + TimelineView(.periodic(from: .now, by: 60)) { timeline in + Table(rows, selection: $selection, sortOrder: $sortOrder) { + identityColumns + metricColumns(asOf: timeline.date) + actionColumns + } + } + } + + @TableColumnBuilder< + KeywordTablePresentationRow, + KeyPathComparator + > + private var identityColumns: some TableColumnContent< + KeywordTablePresentationRow, + KeyPathComparator + > { + TableColumn("Keyword", value: \.keywordSortValue) { tableRow in + KeywordCell(row: tableRow.row) + } + .width(min: 160, ideal: 230) + + TableColumn("Last updated", value: \.lastUpdatedSortValue) { tableRow in + KeywordLastUpdatedCell(row: tableRow.row) + } + .width(min: 92, ideal: 112, max: 132) + + TableColumn("Country", value: \.storefrontSortValue) { tableRow in + KeywordStoreCell(row: tableRow.row) + } + .width(min: 100, ideal: 148) + } + + @TableColumnBuilder< + KeywordTablePresentationRow, + KeyPathComparator + > + private func metricColumns(asOf date: Date) -> some TableColumnContent< + KeywordTablePresentationRow, + KeyPathComparator + > { + if showsPlatformColumn { + TableColumn("Platform", value: \.platformSortValue) { tableRow in + KeywordPlatformCell(platform: tableRow.row.track.platform) + } + .width(min: 92, ideal: 104, max: 116) + } + + TableColumn("Popularity", value: \.popularitySortValue) { tableRow in + KeywordPopularityCell( + row: tableRow.row, + requiresAppleAdsReconnect: requiresAppleAdsReconnect, + openAppleAdsSettings: openAppleAdsSettings + ) + } + .width(min: 112, ideal: 124, max: 136) + + TableColumn("Estimated Difficulty", value: \.estimatedDifficultySortValue) { tableRow in + EstimatedKeywordDifficultyCell( + keyword: tableRow.row.track.term, + summary: tableRow.row.estimatedDifficulty, + asOf: date, + showDetails: { + presentEstimatedDifficulty(tableRow.row) + } + ) + } + .width(min: 150, ideal: 166, max: 190) + + TableColumn("Position", value: \.positionSortValue) { tableRow in + KeywordPositionCell(row: tableRow.row) + } + .width(min: 76, ideal: 88, max: 100) + } + + @TableColumnBuilder< + KeywordTablePresentationRow, + KeyPathComparator + > + private var actionColumns: some TableColumnContent< + KeywordTablePresentationRow, + KeyPathComparator + > { + TableColumn("Trend", value: \.trendSortValue) { tableRow in + Button { + presentRankingHistory(tableRow.row) + } label: { + KeywordTrendCell(row: tableRow.row) + .frame(maxWidth: .infinity, alignment: .leading) + .contentShape(.rect) + } + .buttonStyle(.plain) + .help("Show ranking history") + .accessibilityLabel("Ranking trend for \(tableRow.row.track.term)") + .accessibilityValue( + "\(tableRow.row.trendAccessibilityText) " + + "\(tableRow.row.storefront?.name ?? tableRow.row.track.storefront.uppercased()), " + + "\(tableRow.row.track.platform.displayName)." + ) + .accessibilityHint("Opens ranking history.") + } + .width(min: 104, ideal: 120, max: 132) + + TableColumn("Apps in Ranking") { tableRow in + AppsInRankingButton( + row: tableRow.row, + trackedAppStoreID: trackedAppStoreID, + modelContext: modelContext, + appCatalogService: appCatalogService, + appIconStore: appIconStore, + presentRanking: presentRanking + ) + } + .width(min: 132, ideal: 220) + + TableColumn("Notes") { tableRow in + KeywordNotesCell(row: tableRow.row) { + presentNotes(tableRow.row) + } + } + .width(min: 120, ideal: 180) + + TableColumn("Chart", value: \.chartSelectionSortValue) { tableRow in + ChartSelectionButton( + isSelected: tableRow.isSelectedForChart, + setSelection: { isSelected in + setChartSelection(isSelected, tableRow.row) + } + ) + .frame(maxWidth: .infinity, alignment: .center) + } + .width(min: 56, ideal: 60, max: 68) + } +} + private struct KeywordTablePresentationRow: Identifiable { let row: KeywordWorkspaceRow let isSelectedForChart: Bool @@ -519,6 +632,7 @@ private struct KeywordTablePresentationRow: Identifiable { var storefrontSortValue: String { row.storefrontSortValue } var platformSortValue: Int { row.track.platform.tableSortValue } var popularitySortValue: Int { row.popularitySortValue } + var estimatedDifficultySortValue: Int { row.estimatedDifficultySortValue } var positionSortValue: Int { row.positionSortValue } var trendSortValue: Int { row.trendSortValue } var chartSelectionSortValue: Int { isSelectedForChart ? 0 : 1 } diff --git a/OpenASO/Models/AppStorefrontMetadata.swift b/OpenASO/Models/AppStorefrontMetadata.swift index e95e914..996d381 100644 --- a/OpenASO/Models/AppStorefrontMetadata.swift +++ b/OpenASO/Models/AppStorefrontMetadata.swift @@ -113,4 +113,5 @@ enum AppStorefrontMetadataSource: String, Codable, Sendable { case iTunesSearch case iTunesLookup case appStoreWeb + case appStoreWebSearch } diff --git a/OpenASO/Models/KeywordRankingCrawl.swift b/OpenASO/Models/KeywordRankingCrawl.swift index 41f5ac9..e27a11c 100644 --- a/OpenASO/Models/KeywordRankingCrawl.swift +++ b/OpenASO/Models/KeywordRankingCrawl.swift @@ -86,7 +86,7 @@ final class KeywordRankingCrawl { } var source: RankingSource { - get { RankingSource(rawValue: sourceRaw) ?? .appStoreWeb } + get { RankingSource(rawValue: sourceRaw) ?? .iTunesFallback } set { sourceRaw = newValue.rawValue } } diff --git a/OpenASO/Models/TrackedKeywordDailyRanking.swift b/OpenASO/Models/TrackedKeywordDailyRanking.swift index 1ea844c..af532fe 100644 --- a/OpenASO/Models/TrackedKeywordDailyRanking.swift +++ b/OpenASO/Models/TrackedKeywordDailyRanking.swift @@ -48,7 +48,7 @@ final class TrackedKeywordDailyRanking { } var source: RankingSource { - get { RankingSource(rawValue: sourceRaw) ?? .appStoreWeb } + get { RankingSource(rawValue: sourceRaw) ?? .iTunesFallback } set { sourceRaw = newValue.rawValue } } diff --git a/OpenASO/Services/AppDetail/AppDetailRefreshService.swift b/OpenASO/Services/AppDetail/AppDetailRefreshService.swift index 61f5fa8..8b2e226 100644 --- a/OpenASO/Services/AppDetail/AppDetailRefreshService.swift +++ b/OpenASO/Services/AppDetail/AppDetailRefreshService.swift @@ -448,6 +448,7 @@ final class AppDetailRefreshService: Sendable { outcomes.append(contentsOf: batchOutcome.outcomes) statsRebuildRequests.formUnion(batchOutcome.statsRebuildRequests) failureCount += batchOutcome.failureCount + await persistEstimatedDifficulty(for: batchOutcome.successfulPageResults) for pageResult in batchOutcome.successfulPageResults { refreshCoordinator.scheduleTopRankingMetadataEnrichment(for: pageResult) } @@ -492,6 +493,7 @@ final class AppDetailRefreshService: Sendable { try? await backgroundModelStore.write { modelContext in _ = try refreshCoordinator.recordRefreshFailure( identityKey: targetRequest.identityKey, + trackCreatedAt: targetRequest.trackCreatedAt, error: error, in: modelContext, saveChanges: false @@ -554,35 +556,21 @@ final class AppDetailRefreshService: Sendable { var successfulPageResults: [RankingRefreshPageResult] = [] for pageResult in pageResults { - do { - _ = try refreshCoordinator.persistRankingPage( - pageResult, - in: modelContext, - rebuildDerivedStats: false, - saveChanges: false, - scheduleMetadataEnrichment: false - ) - if let statsRebuildRequest = RankingStatsRebuildRequest(pageRequest: pageResult.request) { - statsRebuildRequests.insert(statsRebuildRequest) - } - successfulPageResults.append(pageResult) - outcomes.append(KeywordBackgroundRefreshOutcome( - trackIdentityKey: pageResult.request.identityKey, - error: nil - )) - } catch { - let mappedError = OpenASOError.map(error) - _ = try? refreshCoordinator.recordRefreshFailure( - identityKey: pageResult.request.identityKey, - error: mappedError, - in: modelContext, - saveChanges: false - ) - outcomes.append(KeywordBackgroundRefreshOutcome( - trackIdentityKey: pageResult.request.identityKey, - error: mappedError - )) + _ = try refreshCoordinator.persistRankingPage( + pageResult, + in: modelContext, + rebuildDerivedStats: false, + saveChanges: false, + scheduleMetadataEnrichment: false + ) + if let statsRebuildRequest = RankingStatsRebuildRequest(pageRequest: pageResult.request) { + statsRebuildRequests.insert(statsRebuildRequest) } + successfulPageResults.append(pageResult) + outcomes.append(KeywordBackgroundRefreshOutcome( + trackIdentityKey: pageResult.request.identityKey, + error: nil + )) } return RankingPersistenceBatchOutcome( @@ -592,24 +580,94 @@ final class AppDetailRefreshService: Sendable { ) } } catch { - let mappedError = OpenASOError.map(error) - try? await backgroundModelStore.write { modelContext in - for pageResult in pageResults { - _ = try? refreshCoordinator.recordRefreshFailure( + return await persistRankingPagesIndividually(pageResults) + } + } + + private func persistRankingPagesIndividually( + _ pageResults: [RankingRefreshPageResult] + ) async -> RankingPersistenceBatchOutcome { + var outcomes: [KeywordBackgroundRefreshOutcome] = [] + var statsRebuildRequests = Set() + var successfulPageResults: [RankingRefreshPageResult] = [] + + for pageResult in pageResults { + do { + try await backgroundModelStore.write { modelContext in + _ = try refreshCoordinator.persistRankingPage( + pageResult, + in: modelContext, + rebuildDerivedStats: false, + saveChanges: false, + scheduleMetadataEnrichment: false + ) + } + if let statsRebuildRequest = RankingStatsRebuildRequest(pageRequest: pageResult.request) { + statsRebuildRequests.insert(statsRebuildRequest) + } + successfulPageResults.append(pageResult) + outcomes.append(KeywordBackgroundRefreshOutcome( + trackIdentityKey: pageResult.request.identityKey, + error: nil + )) + } catch { + let mappedError = OpenASOError.map(error) + try? await backgroundModelStore.write { modelContext in + _ = try refreshCoordinator.recordRefreshFailure( identityKey: pageResult.request.identityKey, + trackCreatedAt: pageResult.request.trackCreatedAt, error: mappedError, in: modelContext, saveChanges: false ) } + outcomes.append(KeywordBackgroundRefreshOutcome( + trackIdentityKey: pageResult.request.identityKey, + error: mappedError + )) + } + } + + return RankingPersistenceBatchOutcome( + outcomes: outcomes, + statsRebuildRequests: statsRebuildRequests, + successfulPageResults: successfulPageResults + ) + } + + /// Difficulty is optional derived data, so it is committed only after the + /// corresponding rankings. A failed batch is retried payload-by-payload to + /// preserve valid peers without changing any ranking outcome or status. + private func persistEstimatedDifficulty( + for pageResults: [RankingRefreshPageResult] + ) async { + let pageResults = pageResults.filter { $0.estimatedDifficultyPayload != nil } + guard !pageResults.isEmpty else { return } + + do { + try await backgroundModelStore.write { modelContext in + for pageResult in pageResults { + try EstimatedKeywordDifficultyPersistenceAdapter.upsertIfCurrent( + for: pageResult, + in: modelContext + ) + } + } + } catch { + for pageResult in pageResults { + do { + _ = try await backgroundModelStore.write { modelContext in + try EstimatedKeywordDifficultyPersistenceAdapter.upsertIfCurrent( + for: pageResult, + in: modelContext + ) + } + } catch { + OpenASOLog.refresh.error( + "Skipped optional estimated difficulty persistence: \(String(reflecting: error), privacy: .private(mask: .hash))" + ) + } } - return RankingPersistenceBatchOutcome( - outcomes: pageResults.map { - KeywordBackgroundRefreshOutcome(trackIdentityKey: $0.request.identityKey, error: mappedError) - }, - statsRebuildRequests: [], - successfulPageResults: [] - ) } } diff --git a/OpenASO/Services/MCP/OpenASOMCPDTOs.swift b/OpenASO/Services/MCP/OpenASOMCPDTOs.swift index 0297dd2..233f83c 100644 --- a/OpenASO/Services/MCP/OpenASOMCPDTOs.swift +++ b/OpenASO/Services/MCP/OpenASOMCPDTOs.swift @@ -201,6 +201,21 @@ struct OpenASOMCPKeywordOverviewSummary: Codable, Sendable { let latestRefreshAt: Date? } +struct OpenASOMCPEstimatedKeywordDifficultySummary: Codable, Equatable, Sendable { + let state: String + let score: Int? + let confidenceScore: Int? + let confidence: String? + let unavailableReason: String? + let estimationSource: String + let algorithmIdentifier: String + let algorithmVersion: Int + let rankingSource: String + let rankingFetchedAt: Date + let computedAt: Date + let isStale: Bool +} + struct OpenASOMCPKeywordSummary: Codable, Identifiable, Sendable { let id: String let trackIdentityKey: String @@ -215,14 +230,81 @@ struct OpenASOMCPKeywordSummary: Codable, Identifiable, Sendable { let resultCount: Int? let popularityScore: Int? let difficultyScore: Int? + let estimatedDifficulty: OpenASOMCPEstimatedKeywordDifficultySummary? let notes: String let rankingStatusMessage: String? let popularityStatusMessage: String? let statusMessage: String? let lastRefreshAt: Date? + let latestRankingSource: String? + let latestRankingObservedAt: Date? let createdAt: Date } +struct OpenASOMCPEstimatedKeywordDifficultyFallback: Codable, Equatable, Sendable { + let provider: String + let category: String + let transportCode: Int? + let httpStatus: Int? + let responseFailure: String? +} + +struct OpenASOMCPEstimatedKeywordDifficultyEvidence: Codable, Equatable, Identifiable, Sendable { + var id: String { "\(position)::\(appStoreID)" } + + let position: Int + let appStoreID: String + let title: String + let subtitle: String? + let ratingCount: Int? + let ratingAuthorityScore: Int? + let titleTokenCoveragePercentage: Int? + let combinedTokenCoveragePercentage: Int? + let metadataMatchScore: Int? + let exactTitlePhraseMatch: Bool + let exactSubtitlePhraseMatch: Bool +} + +/// A read-only, app-scoped view of the stored local heuristic and its bounded +/// evidence. `state == "missing"` is distinct from an unavailable estimate. +struct OpenASOMCPEstimatedKeywordDifficulty: Codable, Equatable, Sendable { + let appStoreID: String + let keyword: String + let queryKey: String + let storefront: String + let platform: String + let state: String + let calculationID: String? + let score: Int? + let confidenceScore: Int? + let confidence: String? + let unavailableReason: String? + let estimationSource: String? + let algorithmIdentifier: String? + let algorithmVersion: Int? + let requestedResultLimit: Int? + let providerResultCount: Int? + let consideredResultCount: Int? + let ratedResultCount: Int? + let weightedRatingCoveragePercentage: Int? + let maximumRatingCount: Int? + let medianRatingCount: Int? + let ratingAuthorityScore: Int? + let metadataSaturationScore: Int? + let exactTitlePhraseMatchCount: Int? + let exactSubtitlePhraseMatchCount: Int? + let rankingSource: String? + let rankingFetchedAt: Date? + let computedAt: Date? + let isStale: Bool? + let fallback: OpenASOMCPEstimatedKeywordDifficultyFallback? + let notes: [String] + let availableEvidenceCount: Int + let returnedEvidenceCount: Int + let evidenceTruncated: Bool + let evidence: [OpenASOMCPEstimatedKeywordDifficultyEvidence] +} + struct OpenASOMCPStoredRankedApp: Codable, Identifiable, Sendable { let id: String let position: Int @@ -406,6 +488,23 @@ struct OpenASOMCPCompetitorKeywordEvidence: Codable, Sendable, Hashable { let bestRank: Int let latestRank: Int let latestObservedAt: Date + let source: String +} + +struct OpenASOMCPRankingFallbackContext: Codable, Sendable, Equatable { + let provider: String + let category: String + let transportCode: Int? + let httpStatus: Int? + let responseFailure: String? +} + +struct OpenASOMCPRankingProvenance: Codable, Sendable, Equatable { + let source: String + let storefront: String + let platform: String + let fetchedAt: Date + let fallbackContext: OpenASOMCPRankingFallbackContext? } struct OpenASOMCPRankedApp: Codable, Identifiable, Sendable { @@ -430,6 +529,7 @@ struct OpenASOMCPKeywordRankingEvidence: Codable, Sendable { let resultCount: Int let source: String let observedAt: Date + let fallbackContext: OpenASOMCPRankingFallbackContext? let targetRank: Int? let topRatedAppCount: Int let maximumRatingCount: Int? @@ -451,6 +551,7 @@ struct OpenASOMCPKeywordCandidate: Codable, Identifiable, Sendable { let topRatedAppCount: Int let maximumRatingCount: Int? let topApps: [OpenASOMCPRankedApp] + let rankingProvenance: OpenASOMCPRankingProvenance? } struct OpenASOMCPKeywordVerificationError: Codable, Sendable, Equatable { @@ -502,6 +603,7 @@ struct OpenASOMCPReviewDownloadResult: Codable, Sendable { struct OpenASOMCPKeywordRefreshOutcome: Codable, Sendable { let track: OpenASOMCPKeywordSummary + let rankingProvenance: OpenASOMCPRankingProvenance? let error: OpenASOMCPErrorDTO? } diff --git a/OpenASO/Services/MCP/OpenASOMCPRuntime.swift b/OpenASO/Services/MCP/OpenASOMCPRuntime.swift index bd62b02..a8358a2 100644 --- a/OpenASO/Services/MCP/OpenASOMCPRuntime.swift +++ b/OpenASO/Services/MCP/OpenASOMCPRuntime.swift @@ -7,10 +7,17 @@ enum OpenASOMCPRuntime { ) async throws -> Server { let modelContainer = try ModelContainerFactory.makeModelContainer(isStoredInMemoryOnly: false) let backgroundModelStore = BackgroundModelStore(modelContainer: modelContainer) - let httpClient = URLSessionHTTPClient() + let refreshObservationClock = RefreshObservationClock.live + let refreshMetricsRecorder = RefreshMetricsRecorder(clock: refreshObservationClock) + let httpClient = ProviderHTTPClientPipeline.make( + transport: URLSessionHTTPClient(), + mode: .production(defaults: .standard), + refreshMetricsRecorder: refreshMetricsRecorder, + refreshObservationClock: refreshObservationClock + ) let appResolver = DefaultAppResolver(httpClient: httpClient) let appCatalogService = AppCatalogService(appResolver: appResolver) - let rankingProvider = ITunesSearchFallbackProvider(httpClient: httpClient) + let rankingProvider = SearchRankingProviderFactory.makeProduction(httpClient: httpClient) let rankingRefreshCoordinator = await RankingRefreshCoordinator( rankingProvider: rankingProvider, appCatalogService: appCatalogService diff --git a/OpenASO/Services/MCP/OpenASOMCPServer.swift b/OpenASO/Services/MCP/OpenASOMCPServer.swift index 7aecd45..8d2614c 100644 --- a/OpenASO/Services/MCP/OpenASOMCPServer.swift +++ b/OpenASO/Services/MCP/OpenASOMCPServer.swift @@ -194,6 +194,16 @@ struct OpenASOMCPServerFactory: Sendable { ) return try Self.toolResult(result) + case "get_estimated_keyword_difficulty": + let result = try await service.getEstimatedKeywordDifficulty( + appStoreID: try arguments.requiredInt64("appStoreID"), + keyword: try arguments.requiredString("keyword"), + storefront: try arguments.requiredString("storefront"), + platform: arguments.string("platform"), + evidenceLimit: arguments.int("evidence_limit") + ) + return try Self.toolResult(result) + case "score_keywords": let result = try await service.scoreKeywords( appStoreID: try arguments.requiredInt64("appStoreID"), @@ -486,6 +496,16 @@ private extension OpenASOMCPServerFactory { required: ["appStoreID"], optional: commonAppFilters.merging(["limit": .integer, "cursor": .string]) { current, _ in current } ), readOnly: true), + tool("get_estimated_keyword_difficulty", "Read one stored local estimated-difficulty heuristic with bounded evidence and provenance. This does not call Apple or Apple Ads.", schema( + required: ["appStoreID", "keyword", "storefront"], + optional: [ + "appStoreID": .integer, + "keyword": .string, + "storefront": .string, + "platform": .string, + "evidence_limit": .integer + ] + ), readOnly: true), tool("score_keywords", "Classify tracked keywords into defend, attack, long-tail, brand, experimental, or noisy buckets using rank, popularity, and phrase-quality heuristics.", schema( required: ["appStoreID"], optional: commonAppFilters diff --git a/OpenASO/Services/MCP/OpenASOMCPService.swift b/OpenASO/Services/MCP/OpenASOMCPService.swift index e69f56d..b68aeb8 100644 --- a/OpenASO/Services/MCP/OpenASOMCPService.swift +++ b/OpenASO/Services/MCP/OpenASOMCPService.swift @@ -1,5 +1,6 @@ import CryptoKit import Foundation +import OSLog import SwiftData private enum OpenASOMCPHistoryKind: String, Codable, Equatable, Sendable { @@ -344,11 +345,20 @@ actor OpenASOMCPRankingRefreshScheduler { private func schedulingRecency(for candidate: Candidate) -> ( date: Date, sequence: UInt64?, identityKey: String )? { - let attempt = attemptsByIdentityKey[candidate.request.identityKey] + let generation = candidate.request.trackCreatedAt + func belongsToCurrentGeneration(_ date: Date) -> Bool { + generation.map { date >= $0 } ?? true + } + let attempt = attemptsByIdentityKey[candidate.request.identityKey].flatMap { attempt in + belongsToCurrentGeneration(attempt.attemptedAt) ? attempt : nil + } let persistedRecency = [ candidate.lastSuccessfulRefreshAt, candidate.lastDurableAttemptAt, - ].compactMap { $0 }.max() + ].compactMap { (date: Date?) -> Date? in + guard let date, belongsToCurrentGeneration(date) else { return nil } + return date + }.max() switch (persistedRecency, attempt) { case (nil, nil): @@ -402,7 +412,13 @@ final class OpenASOMCPService: Sendable { _ requests: [RankingRefreshRequest], _ appStoreID: Int64, _ attemptedAt: Date - ) async throws -> Void + ) async throws -> [RankingRefreshRequest] + typealias EstimatedDifficultyPayloadBuilder = @Sendable ( + _ request: RankingRefreshRequest, + _ page: SearchRankingPage, + _ requestedResultLimit: Int, + _ rankingFetchedAt: Date + ) -> EstimatedKeywordDifficultyPersistencePayload? private enum ResponseLimits { static let overviewStorefrontMetadata = 8 @@ -457,6 +473,8 @@ final class OpenASOMCPService: Sendable { private let rankingRefreshCoordinator: RankingRefreshCoordinator? private let rankingRefreshScheduler: OpenASOMCPRankingRefreshScheduler private let persistRankingRefreshAttempts: RankingRefreshAttemptsPersistence + private let estimatedDifficultyPayloadBuilder: EstimatedDifficultyPayloadBuilder + private let rankingPersistenceDidCommit: (@Sendable () async -> Void)? private let reviewService: AppStorefrontReviewService? private let keywordMetricsService: KeywordMetricsService? private let popularityContextAppStoreIDProvider: @MainActor @Sendable () -> Int64? @@ -473,6 +491,8 @@ final class OpenASOMCPService: Sendable { rankingRefreshCoordinator: RankingRefreshCoordinator? = nil, rankingRefreshScheduler: OpenASOMCPRankingRefreshScheduler = OpenASOMCPRankingRefreshScheduler(), persistRankingRefreshAttempts: RankingRefreshAttemptsPersistence? = nil, + estimatedDifficultyPayloadBuilder: EstimatedDifficultyPayloadBuilder? = nil, + rankingPersistenceDidCommit: (@Sendable () async -> Void)? = nil, reviewService: AppStorefrontReviewService? = nil, keywordMetricsService: KeywordMetricsService? = nil, popularityContextAppStoreIDProvider: @escaping @MainActor @Sendable () -> Int64? = { nil }, @@ -490,16 +510,28 @@ final class OpenASOMCPService: Sendable { self.persistRankingRefreshAttempts = persistRankingRefreshAttempts ?? { [backgroundModelStore] requests, appStoreID, attemptedAt in try await backgroundModelStore.write { modelContext in - for request in requests { - try Self.upsertRankingRefreshAttempt( - request: request, - appStoreID: appStoreID, - attemptedAt: attemptedAt, - in: modelContext - ) - } + try Self.acceptRankingRefreshAttempts( + requests, + appStoreID: appStoreID, + attemptedAt: attemptedAt, + in: modelContext + ) } } + self.estimatedDifficultyPayloadBuilder = estimatedDifficultyPayloadBuilder ?? { + request, + page, + requestedResultLimit, + rankingFetchedAt in + EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: request, + page: page, + requestedResultLimit: requestedResultLimit, + rankingFetchedAt: rankingFetchedAt, + now: now + ) + } + self.rankingPersistenceDidCommit = rankingPersistenceDidCommit self.reviewService = reviewService self.keywordMetricsService = keywordMetricsService self.popularityContextAppStoreIDProvider = popularityContextAppStoreIDProvider @@ -884,6 +916,7 @@ final class OpenASOMCPService: Sendable { let appStoreID = try OpenASOMCPValidation.appStoreID(appStoreID) let storefronts = Set(try OpenASOMCPValidation.storefronts(storefronts)) let platform = try platform.map(OpenASOMCPValidation.platform) + let asOf = now() return try await backgroundModelStore.read { modelContext in let descriptor = FetchDescriptor( predicate: #Predicate { track in @@ -903,6 +936,10 @@ final class OpenASOMCPService: Sendable { queryKeys: Array(Set(tracks.map(\.queryKey))), in: modelContext ) + let estimatedDifficultyByQueryKey = try EstimatedKeywordDifficultyStore.summaries( + queryKeys: Array(Set(tracks.map(\.queryKey))), + in: modelContext + ) let refreshStatuses = try TrackedKeywordRefreshStatusStore.snapshots( for: tracks.map(\.identityKey), in: modelContext @@ -912,7 +949,9 @@ final class OpenASOMCPService: Sendable { Self.keywordSummary( track: $0, metrics: metricsByQueryKey[$0.queryKey], - refreshStatus: refreshStatuses[$0.identityKey] + estimatedDifficulty: estimatedDifficultyByQueryKey[$0.queryKey], + refreshStatus: refreshStatuses[$0.identityKey], + asOf: asOf ) } return OpenASOMCPPage( @@ -928,6 +967,69 @@ final class OpenASOMCPService: Sendable { } } + func getEstimatedKeywordDifficulty( + appStoreID: Int64, + keyword: String, + storefront: String, + platform: String? = nil, + evidenceLimit: Int? = nil + ) async throws -> OpenASOMCPEstimatedKeywordDifficulty { + let appStoreID = try OpenASOMCPValidation.appStoreID(appStoreID) + let keyword = try OpenASOMCPValidation.nonEmpty( + keyword, + fieldName: "keyword", + maximumLength: OpenASOMCPValidation.maximumHistoryKeywordLength + ) + let storefront = try OpenASOMCPValidation.storefront(storefront) + let platform = try OpenASOMCPValidation.platform(platform) + let evidenceLimit = OpenASOMCPValidation.cappedLimit( + evidenceLimit, + default: EstimatedKeywordDifficultyStore.maximumEvidenceResultCount, + maximum: EstimatedKeywordDifficultyStore.maximumEvidenceResultCount + ) + let queryKey = KeywordQuery.makeQueryKey( + term: keyword, + storefront: storefront, + platform: platform + ) + let identityKey = TrackedAppKeyword.makeIdentityKey( + appStoreID: appStoreID, + term: keyword, + storefront: storefront, + platform: platform + ) + let asOf = now() + + return try await backgroundModelStore.read { modelContext in + var trackDescriptor = FetchDescriptor( + predicate: #Predicate { track in + track.identityKey == identityKey + } + ) + trackDescriptor.fetchLimit = 1 + guard try modelContext.fetch(trackDescriptor).first != nil, + let snapshot = try EstimatedKeywordDifficultyStore.snapshot( + queryKey: queryKey, + in: modelContext + ) + else { + return Self.missingEstimatedKeywordDifficulty( + appStoreID: appStoreID, + keyword: keyword, + queryKey: queryKey, + storefront: storefront, + platform: platform + ) + } + return Self.estimatedKeywordDifficulty( + appStoreID: appStoreID, + snapshot: snapshot, + evidenceLimit: evidenceLimit, + asOf: asOf + ) + } + } + func listKeywordRankingHistory( appStoreID: Int64, storefronts: [String]? = nil, @@ -1321,6 +1423,7 @@ final class OpenASOMCPService: Sendable { guard !storefronts.isEmpty else { throw OpenASOError.providerUnavailable("Select at least one storefront.") } + let createdAt = now() return try await backgroundModelStore.write { modelContext in guard let trackedApp = try Self.fetchTrackedApp(appStoreID: appStoreID, in: modelContext) @@ -1363,7 +1466,8 @@ final class OpenASOMCPService: Sendable { storefront: storefront, platform: platform, trackedApp: trackedApp, - query: query + query: query, + createdAt: createdAt ) trackedApp.keywordTracks.append(track) modelContext.insert(track) @@ -1426,11 +1530,17 @@ final class OpenASOMCPService: Sendable { for: track, in: modelContext ) + let estimatedDifficulty = try EstimatedKeywordDifficultyStore.summary( + queryKey: track.queryKey, + in: modelContext + ) return OpenASOMCPKeywordNotesResult( track: Self.keywordSummary( track: track, metrics: metrics[track.queryKey], - refreshStatus: refreshStatus + estimatedDifficulty: estimatedDifficulty, + refreshStatus: refreshStatus, + asOf: now() ), summary: OpenASOMCPMutationSummary( inserted: 0, @@ -1797,18 +1907,20 @@ final class OpenASOMCPService: Sendable { limit: trackLimit ) + let acceptedRequests: [RankingRefreshRequest] if !reservation.requests.isEmpty { do { try Task.checkCancellation() let attemptedAt = now() - // Commit the whole reservation before the first external call so a - // failed attempt-state transaction leaves the provider untouched. - try await persistRankingRefreshAttempts( + // Validate and commit the reservation before the first external call + // so stale generations and failed attempt-state transactions leave the + // provider untouched. + acceptedRequests = try await persistRankingRefreshAttempts( reservation.requests, appStoreID, attemptedAt ) - for request in reservation.requests { + for request in acceptedRequests { try await rankingRefreshScheduler.markAttemptStarted( request, in: reservation, @@ -1819,11 +1931,18 @@ final class OpenASOMCPService: Sendable { await rankingRefreshScheduler.release(reservation) throw error } + } else { + acceptedRequests = [] } - var fetched: [(RankingRefreshRequest, SearchRankingPage?, OpenASOError?)] = [] + var fetched: [( + request: RankingRefreshRequest, + page: SearchRankingPage?, + fetchedAt: Date?, + error: OpenASOError? + )] = [] do { - for request in reservation.requests { + for request in acceptedRequests { try Task.checkCancellation() do { let page = try await Self.withRankingSearchTimeout { @@ -1834,44 +1953,68 @@ final class OpenASOMCPService: Sendable { limit: resultLimit ) } - fetched.append((request, page, nil)) + fetched.append((request, page, now(), nil)) } catch is CancellationError { throw CancellationError() } catch { try Task.checkCancellation() - fetched.append((request, nil, OpenASOError.map(error))) + fetched.append((request, nil, nil, OpenASOError.map(error))) } } try Task.checkCancellation() let fetchedResults = fetched - let result = try await backgroundModelStore.write { modelContext in + let persistenceResult = try await backgroundModelStore.write { modelContext in var outcomes: [OpenASOMCPKeywordRefreshOutcome] = [] + var enrichmentPageResults: [RankingRefreshPageResult] = [] + var difficultyPageResults: [RankingRefreshPageResult] = [] + var staleGenerationCount = 0 for item in fetchedResults { - if let page = item.1 { - let observedAt = now() + guard + let currentTrack = try Self.fetchTrackedKeyword( + identityKey: item.request.identityKey, + in: modelContext + ), + currentTrack.appStoreID == appStoreID, + item.request.matchesGeneration(of: currentTrack) + else { + staleGenerationCount += 1 + continue + } + + if let page = item.page, let observedAt = item.fetchedAt { + let difficultyPayload = estimatedDifficultyPayloadBuilder( + item.request, + page, + resultLimit, + observedAt + ) + let pageResult = RankingRefreshPageResult( + request: item.request, + page: page, + searchedAt: observedAt, + observedHour: nil, + submissionCount: 0, + winningCount: 0, + confidence: nil, + requestedResultLimit: resultLimit, + estimatedDifficultyPayload: difficultyPayload + ) + difficultyPageResults.append(pageResult) let track: TrackedAppKeyword if let rankingRefreshCoordinator { - let pageResult = RankingRefreshPageResult( - request: item.0, - page: page, - searchedAt: observedAt, - observedHour: nil, - submissionCount: 0, - winningCount: 0, - confidence: nil - ) track = try rankingRefreshCoordinator.persistRankingPage( pageResult, in: modelContext, rebuildDerivedStats: true, saveChanges: false, - scheduleMetadataEnrichment: true + scheduleMetadataEnrichment: false ).keywordTrack + enrichmentPageResults.append(pageResult) } else { track = try Self.persistRankingPage( page, - request: item.0, + request: item.request, appStoreID: appStoreID, observedAt: observedAt, appCatalogService: appCatalogService, @@ -1883,19 +2026,28 @@ final class OpenASOMCPService: Sendable { for: track, in: modelContext ) + let estimatedDifficulty = try EstimatedKeywordDifficultyStore.summary( + queryKey: track.queryKey, + in: modelContext + ) outcomes.append( OpenASOMCPKeywordRefreshOutcome( track: Self.keywordSummary( track: track, metrics: metrics[track.queryKey], - refreshStatus: refreshStatus + estimatedDifficulty: estimatedDifficulty, + refreshStatus: refreshStatus, + asOf: now() + ), + rankingProvenance: Self.rankingProvenance( + request: item.request, + page: page, + fetchedAt: observedAt ), error: nil )) - } else if let error = item.2, - let track = try Self.fetchTrackedKeyword( - identityKey: item.0.identityKey, in: modelContext) - { + } else if let error = item.error { + let track = currentTrack try TrackedKeywordRefreshStatusStore.set( "Ranking failed to refresh. \(error.localizedDescription)", domain: .ranking, @@ -1908,41 +2060,168 @@ final class OpenASOMCPService: Sendable { for: track, in: modelContext ) + let estimatedDifficulty = try EstimatedKeywordDifficultyStore.summary( + queryKey: track.queryKey, + in: modelContext + ) outcomes.append( OpenASOMCPKeywordRefreshOutcome( track: Self.keywordSummary( track: track, metrics: metrics[track.queryKey], - refreshStatus: refreshStatus + estimatedDifficulty: estimatedDifficulty, + refreshStatus: refreshStatus, + asOf: now() ), + rankingProvenance: nil, error: OpenASOMCPErrorDTO(error) )) } } let failures = outcomes.filter { $0.error != nil }.count - return OpenASOMCPKeywordRefreshResult( - summary: OpenASOMCPMutationSummary( - inserted: 0, - updated: 0, - skipped: max(0, candidateBatch.totalCount - reservation.requests.count), - refreshed: outcomes.count - failures, - failed: failures + return ( + result: OpenASOMCPKeywordRefreshResult( + summary: OpenASOMCPMutationSummary( + inserted: 0, + updated: 0, + skipped: max(0, candidateBatch.totalCount - acceptedRequests.count) + + staleGenerationCount, + refreshed: outcomes.count - failures, + failed: failures + ), + outcomes: outcomes, + notes: Self.keywordRankingRefreshNotes( + requestedLimit: requestedLimit, + appliedLimit: trackLimit, + staleGenerationCount: staleGenerationCount + ) ), - outcomes: outcomes, - notes: Self.keywordRankingRefreshNotes( - requestedLimit: requestedLimit, - appliedLimit: trackLimit - ) + enrichmentPageResults: enrichmentPageResults, + difficultyPageResults: difficultyPageResults ) } + await rankingPersistenceDidCommit?() + await persistEstimatedDifficulty( + for: persistenceResult.difficultyPageResults + ) + let refreshedOutcomes = await refreshedKeywordOutcomes( + persistenceResult.result.outcomes + ) + for pageResult in persistenceResult.enrichmentPageResults { + rankingRefreshCoordinator?.scheduleTopRankingMetadataEnrichment(for: pageResult) + } await rankingRefreshScheduler.release(reservation) - return result + return OpenASOMCPKeywordRefreshResult( + summary: persistenceResult.result.summary, + outcomes: refreshedOutcomes, + notes: persistenceResult.result.notes, + batchSummary: persistenceResult.result.batchSummary + ) } catch { await rankingRefreshScheduler.release(reservation) throw error } } + /// Difficulty is optional derived data. Persist it only after ranking writes + /// commit, and retry a failed batch one payload at a time so a rejected + /// estimate cannot roll back valid peer estimates or ranking observations. + private func persistEstimatedDifficulty( + for pageResults: [RankingRefreshPageResult] + ) async { + let pageResults = pageResults.filter { $0.estimatedDifficultyPayload != nil } + guard !pageResults.isEmpty else { return } + + do { + try await backgroundModelStore.write { modelContext in + for pageResult in pageResults { + try EstimatedKeywordDifficultyPersistenceAdapter.upsertIfCurrent( + for: pageResult, + in: modelContext + ) + } + } + } catch { + for pageResult in pageResults { + do { + _ = try await backgroundModelStore.write { modelContext in + try EstimatedKeywordDifficultyPersistenceAdapter.upsertIfCurrent( + for: pageResult, + in: modelContext + ) + } + } catch { + OpenASOLog.refresh.error( + "Skipped optional estimated difficulty persistence: \(String(reflecting: error), privacy: .private(mask: .hash))" + ) + } + } + } + } + + private func refreshedKeywordOutcomes( + _ outcomes: [OpenASOMCPKeywordRefreshOutcome] + ) async -> [OpenASOMCPKeywordRefreshOutcome] { + let identityKeys = Array(Set(outcomes.map(\.track.trackIdentityKey))) + guard !identityKeys.isEmpty else { return outcomes } + let asOf = now() + + do { + return try await backgroundModelStore.read { modelContext in + let targetIdentityKeys = identityKeys + let descriptor = FetchDescriptor( + predicate: #Predicate { track in + targetIdentityKeys.contains(track.identityKey) + } + ) + let tracks = try modelContext.fetch(descriptor) + let queryKeys = Array(Set(tracks.map(\.queryKey))) + let metrics = try Self.metricsByQueryKey( + queryKeys: queryKeys, + in: modelContext + ) + let difficulties = try EstimatedKeywordDifficultyStore.summaries( + queryKeys: queryKeys, + in: modelContext + ) + let statuses = try TrackedKeywordRefreshStatusStore.snapshots( + for: tracks.map(\.identityKey), + in: modelContext + ) + let summaries = Dictionary(uniqueKeysWithValues: tracks.map { track in + ( + track.identityKey, + ( + createdAt: track.createdAt, + summary: Self.keywordSummary( + track: track, + metrics: metrics[track.queryKey], + estimatedDifficulty: difficulties[track.queryKey], + refreshStatus: statuses[track.identityKey], + asOf: asOf + ) + ) + ) + }) + + return outcomes.map { outcome in + guard let current = summaries[outcome.track.trackIdentityKey], + current.createdAt == outcome.track.createdAt + else { + return outcome + } + return OpenASOMCPKeywordRefreshOutcome( + track: current.summary, + rankingProvenance: outcome.rankingProvenance, + error: outcome.error + ) + } + } + } catch { + return outcomes + } + } + func refreshKeywordMetrics( appStoreID: Int64, storefronts: [String]? = nil, @@ -2030,6 +2309,10 @@ final class OpenASOMCPService: Sendable { try Self.fetchTrackedKeyword(identityKey: $0, in: modelContext) } let metrics = try Self.metricsByQueryKey(queryKeys: tracks.map(\.queryKey), in: modelContext) + let estimatedDifficulty = try EstimatedKeywordDifficultyStore.summaries( + queryKeys: tracks.map(\.queryKey), + in: modelContext + ) let refreshStatuses = try TrackedKeywordRefreshStatusStore.snapshots( for: tracks.map(\.identityKey), in: modelContext @@ -2048,8 +2331,11 @@ final class OpenASOMCPService: Sendable { track: Self.keywordSummary( track: track, metrics: metric, - refreshStatus: refreshStatus + estimatedDifficulty: estimatedDifficulty[track.queryKey], + refreshStatus: refreshStatus, + asOf: now() ), + rankingProvenance: nil, error: error.map { OpenASOMCPErrorDTO( code: $0.localizedCaseInsensitiveContains("Connect an Apple Ads") @@ -2692,6 +2978,38 @@ final class OpenASOMCPService: Sendable { return rankingProvider } + static func acceptRankingRefreshAttempts( + _ requests: [RankingRefreshRequest], + appStoreID: Int64, + attemptedAt: Date, + in modelContext: ModelContext + ) throws -> [RankingRefreshRequest] { + var acceptedRequests: [RankingRefreshRequest] = [] + acceptedRequests.reserveCapacity(requests.count) + + for request in requests { + guard + let track = try fetchTrackedKeyword( + identityKey: request.identityKey, + in: modelContext + ), + track.appStoreID == appStoreID, + request.matchesGeneration(of: track) + else { + continue + } + try upsertRankingRefreshAttempt( + request: request, + appStoreID: appStoreID, + attemptedAt: attemptedAt, + in: modelContext + ) + acceptedRequests.append(request) + } + + return acceptedRequests + } + private static func upsertRankingRefreshAttempt( request: RankingRefreshRequest, appStoreID: Int64, @@ -2724,13 +3042,19 @@ final class OpenASOMCPService: Sendable { private static func keywordRankingRefreshNotes( requestedLimit: Int?, - appliedLimit: Int + appliedLimit: Int, + staleGenerationCount: Int ) -> [String] { var notes = [ "Candidates with neither a successful nor attempted refresh are selected first, then by least-recent activity using the latest successful, durable-attempt, and process-attempt times, with stable identity-key ties.", "Attempt recency persists across OpenASO relaunches, while in-flight reservations are shared across MCP connections for the current process; lastRefreshAt remains the time of the latest successful refresh.", "Separate OpenASO processes do not share in-flight reservations, so simultaneous stdio processes can make duplicate attempts even though later calls observe the durable attempt recency.", ] + if staleGenerationCount > 0 { + notes.append( + "Skipped \(staleGenerationCount) ranking result(s) because the tracked keyword changed while its refresh was in flight." + ) + } if let requestedLimit, requestedLimit != appliedLimit { notes.append( "The requested track limit of \(requestedLimit) was clamped to \(appliedLimit); the supported range is 1...\(ResponseLimits.maximumKeywordRefreshTrackLimit)." @@ -3017,40 +3341,57 @@ extension OpenASOMCPService { storefront: storefront, platform: platform ) - try await backgroundModelStore.write { modelContext in + let persistedPageResult = try await backgroundModelStore.write { modelContext in let observedAt = now() + let track = try Self.ensureTrackedKeyword( + request: request, + appStoreID: appStoreID, + in: modelContext + ) + let persistedRequest = RankingRefreshRequest(track: track) + let difficultyPayload = estimatedDifficultyPayloadBuilder( + persistedRequest, + page, + ResponseLimits.defaultRankingAppLimit, + observedAt + ) + let pageResult = RankingRefreshPageResult( + request: persistedRequest, + page: page, + searchedAt: observedAt, + observedHour: nil, + submissionCount: 0, + winningCount: 0, + confidence: nil, + requestedResultLimit: ResponseLimits.defaultRankingAppLimit, + estimatedDifficultyPayload: difficultyPayload + ) if let rankingRefreshCoordinator { - _ = try Self.ensureTrackedKeyword( - request: request, - appStoreID: appStoreID, - in: modelContext - ) - let pageResult = RankingRefreshPageResult( - request: request, - page: page, - searchedAt: observedAt, - observedHour: nil, - submissionCount: 0, - winningCount: 0, - confidence: nil - ) _ = try rankingRefreshCoordinator.persistRankingPage( pageResult, in: modelContext, rebuildDerivedStats: true, saveChanges: false, - scheduleMetadataEnrichment: true + scheduleMetadataEnrichment: false ) + return pageResult } else { _ = try Self.persistRankingPage( page, - request: request, + request: persistedRequest, appStoreID: appStoreID, observedAt: observedAt, appCatalogService: appCatalogService, in: modelContext ) } + return pageResult + } + await persistEstimatedDifficulty(for: [persistedPageResult]) + if rankingRefreshCoordinator != nil { + rankingRefreshCoordinator?.scheduleTopRankingMetadataEnrichment( + for: persistedPageResult + ) } isTracked = true } @@ -3069,7 +3410,14 @@ extension OpenASOMCPService { resultCount: evidence.resultCount, topRatedAppCount: evidence.topRatedAppCount, maximumRatingCount: evidence.maximumRatingCount, - topApps: evidence.topApps + topApps: evidence.topApps, + rankingProvenance: OpenASOMCPRankingProvenance( + source: evidence.source, + storefront: evidence.storefront, + platform: evidence.platform, + fetchedAt: evidence.observedAt, + fallbackContext: evidence.fallbackContext + ) )) } } @@ -3192,6 +3540,7 @@ extension OpenASOMCPService { resultCount: page.resultCount, source: page.source.rawValue, observedAt: observedAt, + fallbackContext: page.fallbackContext.map(Self.rankingFallbackContext), targetRank: targetAppStoreID.flatMap { id in page.items.first { $0.appStoreID == id }?.position }, @@ -3203,6 +3552,63 @@ extension OpenASOMCPService { ) } + fileprivate static func rankingProvenance( + request: RankingRefreshRequest, + page: SearchRankingPage, + fetchedAt: Date + ) -> OpenASOMCPRankingProvenance { + OpenASOMCPRankingProvenance( + source: page.source.rawValue, + storefront: request.storefront, + platform: request.platform.rawValue, + fetchedAt: fetchedAt, + fallbackContext: page.fallbackContext.map(rankingFallbackContext) + ) + } + + fileprivate static func rankingFallbackContext( + _ context: SearchRankingFailureContext + ) -> OpenASOMCPRankingFallbackContext { + let category: String + let transportCode: Int? + let httpStatus: Int? + let responseFailure: String? + switch context.category { + case .transport(let code): + category = "transport" + transportCode = code + httpStatus = nil + responseFailure = nil + case .httpStatus(let status): + category = "httpStatus" + transportCode = nil + httpStatus = status + responseFailure = nil + case .response(let failure): + category = "response" + transportCode = nil + httpStatus = nil + responseFailure = failure.rawValue + case .provider: + category = "provider" + transportCode = nil + httpStatus = nil + responseFailure = nil + case .other: + category = "other" + transportCode = nil + httpStatus = nil + responseFailure = nil + } + return OpenASOMCPRankingFallbackContext( + provider: context.provider.rawValue, + category: category, + transportCode: transportCode, + httpStatus: httpStatus, + responseFailure: responseFailure + ) + } + fileprivate static func rankedApp(_ item: SearchRankingItem) -> OpenASOMCPRankedApp { OpenASOMCPRankedApp( id: String(item.appStoreID), @@ -3901,7 +4307,9 @@ extension OpenASOMCPService { fileprivate static func keywordSummary( track: TrackedAppKeyword, metrics: KeywordDailyMetric?, - refreshStatus: KeywordRefreshStatusSnapshot? = nil + estimatedDifficulty: EstimatedKeywordDifficultySummary? = nil, + refreshStatus: KeywordRefreshStatusSnapshot? = nil, + asOf: Date = Date() ) -> OpenASOMCPKeywordSummary { @@ -3932,15 +4340,154 @@ extension OpenASOMCPService { resultCount: latest?.resultCount ?? track.rankingAppCount, popularityScore: metrics?.popularityScore, difficultyScore: metrics?.difficultyScore, + estimatedDifficulty: estimatedDifficulty.map { + OpenASOMCPEstimatedKeywordDifficultySummary( + state: $0.stateRaw, + score: $0.score, + confidenceScore: $0.confidenceScore, + confidence: $0.confidenceRaw, + unavailableReason: $0.unavailableReasonRaw, + estimationSource: $0.estimationSourceRaw, + algorithmIdentifier: $0.algorithmIdentifier, + algorithmVersion: $0.algorithmVersion, + rankingSource: $0.rankingSourceRaw, + rankingFetchedAt: $0.rankingFetchedAt, + computedAt: $0.computedAt, + isStale: $0.isStale(asOf: asOf) + ) + }, notes: track.notes, rankingStatusMessage: resolvedStatus.rankingMessage, popularityStatusMessage: resolvedStatus.popularityMessage, statusMessage: resolvedStatus.preferredMessage, lastRefreshAt: track.lastRefreshAt, + latestRankingSource: latest?.source.rawValue, + latestRankingObservedAt: latest?.searchedAt, createdAt: track.createdAt ) } + private static func missingEstimatedKeywordDifficulty( + appStoreID: Int64, + keyword: String, + queryKey: String, + storefront: String, + platform: AppPlatform + ) -> OpenASOMCPEstimatedKeywordDifficulty { + OpenASOMCPEstimatedKeywordDifficulty( + appStoreID: String(appStoreID), + keyword: keyword, + queryKey: queryKey, + storefront: storefront, + platform: platform.rawValue, + state: "missing", + calculationID: nil, + score: nil, + confidenceScore: nil, + confidence: nil, + unavailableReason: nil, + estimationSource: nil, + algorithmIdentifier: nil, + algorithmVersion: nil, + requestedResultLimit: nil, + providerResultCount: nil, + consideredResultCount: nil, + ratedResultCount: nil, + weightedRatingCoveragePercentage: nil, + maximumRatingCount: nil, + medianRatingCount: nil, + ratingAuthorityScore: nil, + metadataSaturationScore: nil, + exactTitlePhraseMatchCount: nil, + exactSubtitlePhraseMatchCount: nil, + rankingSource: nil, + rankingFetchedAt: nil, + computedAt: nil, + isStale: nil, + fallback: nil, + notes: [], + availableEvidenceCount: 0, + returnedEvidenceCount: 0, + evidenceTruncated: false, + evidence: [] + ) + } + + private static func estimatedKeywordDifficulty( + appStoreID: Int64, + snapshot: EstimatedKeywordDifficultySnapshot, + evidenceLimit: Int, + asOf: Date + ) -> OpenASOMCPEstimatedKeywordDifficulty { + let evidence = snapshot.resultEvidence.prefix(evidenceLimit).map { + OpenASOMCPEstimatedKeywordDifficultyEvidence( + position: $0.position, + appStoreID: String($0.appStoreID), + title: $0.title, + subtitle: $0.subtitle, + ratingCount: $0.ratingCount, + ratingAuthorityScore: $0.ratingAuthorityScore, + titleTokenCoveragePercentage: $0.titleTokenCoveragePercentage, + combinedTokenCoveragePercentage: $0.combinedTokenCoveragePercentage, + metadataMatchScore: $0.metadataMatchScore, + exactTitlePhraseMatch: $0.exactTitlePhraseMatch, + exactSubtitlePhraseMatch: $0.exactSubtitlePhraseMatch + ) + } + let fallback: OpenASOMCPEstimatedKeywordDifficultyFallback? + if let provider = snapshot.fallbackProviderRaw, + let category = snapshot.fallbackCategoryRaw + { + fallback = OpenASOMCPEstimatedKeywordDifficultyFallback( + provider: provider, + category: category, + transportCode: snapshot.fallbackTransportCode, + httpStatus: snapshot.fallbackHTTPStatus, + responseFailure: snapshot.fallbackResponseFailureRaw + ) + } else { + fallback = nil + } + + return OpenASOMCPEstimatedKeywordDifficulty( + appStoreID: String(appStoreID), + keyword: snapshot.keyword, + queryKey: snapshot.queryKey, + storefront: snapshot.storefront, + platform: snapshot.platformRaw, + state: snapshot.stateRaw, + calculationID: snapshot.calculationID.uuidString.lowercased(), + score: snapshot.score, + confidenceScore: snapshot.confidenceScore, + confidence: snapshot.confidenceRaw, + unavailableReason: snapshot.unavailableReasonRaw, + estimationSource: snapshot.estimationSourceRaw, + algorithmIdentifier: snapshot.algorithmIdentifier, + algorithmVersion: snapshot.algorithmVersion, + requestedResultLimit: snapshot.requestedResultLimit, + providerResultCount: snapshot.providerResultCount, + consideredResultCount: snapshot.consideredResultCount, + ratedResultCount: snapshot.ratedResultCount, + weightedRatingCoveragePercentage: snapshot.weightedRatingCoveragePercentage, + maximumRatingCount: snapshot.maximumRatingCount, + medianRatingCount: snapshot.medianRatingCount, + ratingAuthorityScore: snapshot.ratingAuthorityScore, + metadataSaturationScore: snapshot.metadataSaturationScore, + exactTitlePhraseMatchCount: snapshot.exactTitlePhraseMatchCount, + exactSubtitlePhraseMatchCount: snapshot.exactSubtitlePhraseMatchCount, + rankingSource: snapshot.rankingSourceRaw, + rankingFetchedAt: snapshot.rankingFetchedAt, + computedAt: snapshot.computedAt, + isStale: snapshot.isStale(asOf: asOf), + fallback: fallback, + notes: snapshot.notes, + availableEvidenceCount: snapshot.resultEvidence.count, + returnedEvidenceCount: evidence.count, + evidenceTruncated: evidence.count < snapshot.resultEvidence.count, + evidence: evidence + ) + } + fileprivate static func keywordScore(_ keyword: OpenASOMCPKeywordSummary) -> OpenASOMCPKeywordScore { @@ -4325,6 +4872,11 @@ extension OpenASOMCPService { } let track = try ensureTrackedKeyword( request: request, appStoreID: trackedApp.appStoreID, in: modelContext) + guard request.matchesGeneration(of: track) else { + throw OpenASOError.providerUnavailable( + "The keyword changed while its ranking refresh was in flight. Refresh it again." + ) + } let query = track.query var catalogCache = try appCatalogService.makeSearchRankingPageCache( @@ -4336,6 +4888,9 @@ extension OpenASOMCPService { _ = try appCatalogService.upsertStoreApp( from: item, storefrontCode: request.storefront, + rankingSource: page.source, + fetchedAt: observedAt, + requestedPlatform: request.platform, in: modelContext, cache: &catalogCache ) @@ -4649,16 +5204,20 @@ private struct CompetitorAccumulator { } let existing = evidenceByQueryKey[row.queryKey] + let keepsExistingLatest = (existing?.latestObservedAt ?? .distantPast) > row.observedAt let evidence = OpenASOMCPCompetitorKeywordEvidence( queryKey: row.queryKey, keyword: keyword, storefront: row.storefront, platform: row.platformRaw, bestRank: min(existing?.bestRank ?? row.position, row.position), - latestRank: (existing?.latestObservedAt ?? Date.distantPast) > row.observedAt + latestRank: keepsExistingLatest ? existing?.latestRank ?? row.position : row.position, - latestObservedAt: max(existing?.latestObservedAt ?? row.observedAt, row.observedAt) + latestObservedAt: max(existing?.latestObservedAt ?? row.observedAt, row.observedAt), + source: keepsExistingLatest + ? existing?.source ?? row.observation.source.rawValue + : row.observation.source.rawValue ) evidenceByQueryKey[row.queryKey] = evidence } diff --git a/OpenASO/Services/Networking/RefreshObservability.swift b/OpenASO/Services/Networking/RefreshObservability.swift index 6585c42..589a26e 100644 --- a/OpenASO/Services/Networking/RefreshObservability.swift +++ b/OpenASO/Services/Networking/RefreshObservability.swift @@ -416,7 +416,15 @@ struct RefreshRequestClassification: Sendable { } case "apps.apple.com": provider = .appStoreWeb - endpoint = .ratingsPage + let pathComponents = path.split(separator: "/") + let rankingPlatforms: Set = ["iphone", "ipad", "mac"] + if pathComponents.count == 3, + rankingPlatforms.contains(pathComponents[1]), + pathComponents[2] == "search" { + endpoint = .rankingSearch + } else { + endpoint = .ratingsPage + } case "api.appstoreconnect.apple.com": provider = .appStoreConnect endpoint = path.contains("customerreviews") ? .appStoreConnectReviews : .other diff --git a/OpenASO/Services/Persistence/EstimatedKeywordDifficultyPresentation.swift b/OpenASO/Services/Persistence/EstimatedKeywordDifficultyPresentation.swift new file mode 100644 index 0000000..d9840a8 --- /dev/null +++ b/OpenASO/Services/Persistence/EstimatedKeywordDifficultyPresentation.swift @@ -0,0 +1,159 @@ +import Foundation + +/// Lightweight estimated-difficulty data for list and filter surfaces. +/// +/// The persisted result evidence is deliberately omitted. Callers that need to +/// explain a calculation should load `EstimatedKeywordDifficultySnapshot` +/// explicitly instead of hydrating every row's evidence. +struct EstimatedKeywordDifficultySummary: Equatable, Sendable { + let queryKey: String + let calculationID: UUID + let keyword: String + let storefront: String + let platformRaw: String + let stateRaw: String + let score: Int? + let confidenceScore: Int? + let confidenceRaw: String? + let unavailableReasonRaw: String? + let estimationSourceRaw: String + let algorithmIdentifier: String + let algorithmVersion: Int + let requestedResultLimit: Int + let providerResultCount: Int + let consideredResultCount: Int + let ratedResultCount: Int + let weightedRatingCoveragePercentage: Int + let maximumRatingCount: Int? + let medianRatingCount: Int? + let ratingAuthorityScore: Int? + let metadataSaturationScore: Int? + let exactTitlePhraseMatchCount: Int + let exactSubtitlePhraseMatchCount: Int + let rankingSourceRaw: String + let rankingFetchedAt: Date + let computedAt: Date + let fallbackProviderRaw: String? + let fallbackCategoryRaw: String? + let fallbackTransportCode: Int? + let fallbackHTTPStatus: Int? + let fallbackResponseFailureRaw: String? + let notes: [String] + + var platform: AppPlatform? { + AppPlatform(rawValue: platformRaw) + } + + var state: EstimatedKeywordDifficultyState? { + EstimatedKeywordDifficultyState(rawValue: stateRaw) + } + + var confidence: EstimatedKeywordDifficultyConfidence? { + confidenceRaw.flatMap(EstimatedKeywordDifficultyConfidence.init(rawValue:)) + } + + var unavailableReason: EstimatedKeywordDifficultyUnavailableReason? { + unavailableReasonRaw.flatMap(EstimatedKeywordDifficultyUnavailableReason.init(rawValue:)) + } + + var estimationSource: EstimatedKeywordDifficultySource? { + EstimatedKeywordDifficultySource(rawValue: estimationSourceRaw) + } + + var rankingSource: RankingSource? { + RankingSource(rawValue: rankingSourceRaw) + } + + var fallbackProvider: RankingSource? { + fallbackProviderRaw.flatMap(RankingSource.init(rawValue:)) + } + + var fallbackCategory: EstimatedKeywordDifficultyFallbackCategory? { + fallbackCategoryRaw.flatMap(EstimatedKeywordDifficultyFallbackCategory.init(rawValue:)) + } + + var fallbackResponseFailure: EstimatedKeywordDifficultyFallbackResponseFailure? { + fallbackResponseFailureRaw.flatMap( + EstimatedKeywordDifficultyFallbackResponseFailure.init(rawValue:) + ) + } + + func isStale(asOf date: Date) -> Bool { + EstimatedKeywordDifficultyFreshness.isStale( + rankingFetchedAt: rankingFetchedAt, + asOf: date + ) + } +} + +/// One freshness definition shared by row, detail, export, and MCP surfaces. +/// Freshness follows the ranking evidence rather than a later recomputation. +enum EstimatedKeywordDifficultyFreshness { + static let maximumAge: TimeInterval = 24 * 60 * 60 + + static func isStale(rankingFetchedAt: Date, asOf date: Date) -> Bool { + date.timeIntervalSince(rankingFetchedAt) >= maximumAge + } +} + +/// A presentation-safe interpretation that keeps missing and persisted +/// unavailable values distinct while treating staleness as an orthogonal flag. +struct EstimatedKeywordDifficultyPresentation: Equatable, Sendable { + enum Value: Equatable, Sendable { + case missing + case estimated( + score: Int, + confidenceScore: Int, + confidence: EstimatedKeywordDifficultyConfidence + ) + case unavailable(reason: EstimatedKeywordDifficultyUnavailableReason) + case unsupported + } + + let value: Value + let isStale: Bool + let rankingFetchedAt: Date? + + init(summary: EstimatedKeywordDifficultySummary?, asOf date: Date) { + guard let summary else { + value = .missing + isStale = false + rankingFetchedAt = nil + return + } + + rankingFetchedAt = summary.rankingFetchedAt + isStale = summary.isStale(asOf: date) + switch summary.state { + case .estimated: + if let score = summary.score, + let confidenceScore = summary.confidenceScore, + let confidence = summary.confidence { + value = .estimated( + score: score, + confidenceScore: confidenceScore, + confidence: confidence + ) + } else { + value = .unsupported + } + case .unavailable: + if let reason = summary.unavailableReason { + value = .unavailable(reason: reason) + } else { + value = .unsupported + } + case nil: + value = .unsupported + } + } +} + +extension EstimatedKeywordDifficultySnapshot { + func isStale(asOf date: Date) -> Bool { + EstimatedKeywordDifficultyFreshness.isStale( + rankingFetchedAt: rankingFetchedAt, + asOf: date + ) + } +} diff --git a/OpenASO/Services/Persistence/EstimatedKeywordDifficultyStore.swift b/OpenASO/Services/Persistence/EstimatedKeywordDifficultyStore.swift index 15296c9..70f3849 100644 --- a/OpenASO/Services/Persistence/EstimatedKeywordDifficultyStore.swift +++ b/OpenASO/Services/Persistence/EstimatedKeywordDifficultyStore.swift @@ -285,6 +285,11 @@ enum EstimatedKeywordDifficultyStoreError: Error, Equatable { } enum EstimatedKeywordDifficultyStore { + private struct DifficultyRevisionKey: Hashable { + let queryKey: String + let calculationID: UUID + } + static let maximumEvidenceResultCount = 10 static let maximumRequestedResultCount = 200 private static let maximumTextLength = 1_000 @@ -374,26 +379,69 @@ enum EstimatedKeywordDifficultyStore { return try snapshot(for: metric, in: modelContext) } + static func summary( + queryKey: String, + in modelContext: ModelContext + ) throws -> EstimatedKeywordDifficultySummary? { + try summaries(queryKeys: [queryKey], in: modelContext)[queryKey] + } + + /// Loads scalar values only. Evidence rows are intentionally not fetched, + /// making this suitable for keyword lists and filter projections. + static func summaries( + queryKeys: [String], + in modelContext: ModelContext + ) throws -> [String: EstimatedKeywordDifficultySummary] { + let selectedMetrics = try latestMetrics(queryKeys: queryKeys, in: modelContext) + var result: [String: EstimatedKeywordDifficultySummary] = [:] + result.reserveCapacity(selectedMetrics.count) + for (queryKey, metric) in selectedMetrics { + result[queryKey] = summary(from: metric) + } + return result + } + static func snapshots( queryKeys: [String], in modelContext: ModelContext ) throws -> [String: EstimatedKeywordDifficultySnapshot] { - let targetQueryKeys = Array(Set(queryKeys)) - guard !targetQueryKeys.isEmpty else { return [:] } + let selectedMetrics = try latestMetrics(queryKeys: queryKeys, in: modelContext) + guard !selectedMetrics.isEmpty else { return [:] } - let descriptor = FetchDescriptor( - predicate: #Predicate { metric in - targetQueryKeys.contains(metric.queryKey) + let selectedQueryKeys = Array(selectedMetrics.keys) + let evidenceDescriptor = FetchDescriptor( + predicate: #Predicate { evidence in + selectedQueryKeys.contains(evidence.queryKey) } ) - let groupedMetrics = Dictionary( - grouping: try modelContext.fetch(descriptor), - by: \.queryKey - ) + var evidenceByRevision: [DifficultyRevisionKey: [EstimatedKeywordDifficultyResultEvidence]] = [:] + for record in try modelContext.fetch(evidenceDescriptor) { + guard selectedMetrics[record.queryKey]?.calculationID == record.calculationID else { + continue + } + let revisionKey = DifficultyRevisionKey( + queryKey: record.queryKey, + calculationID: record.calculationID + ) + evidenceByRevision[revisionKey, default: []].append(evidence(from: record)) + } + var result: [String: EstimatedKeywordDifficultySnapshot] = [:] - for (queryKey, metrics) in groupedMetrics { - guard let metric = latestMetric(in: metrics) else { continue } - result[queryKey] = try snapshot(for: metric, in: modelContext) + result.reserveCapacity(selectedMetrics.count) + for (queryKey, metric) in selectedMetrics { + let revisionKey = DifficultyRevisionKey( + queryKey: queryKey, + calculationID: metric.calculationID + ) + let evidence = (evidenceByRevision[revisionKey] ?? []) + .sorted { left, right in + if left.position != right.position { + return left.position < right.position + } + return left.appStoreID < right.appStoreID + } + .prefix(maximumEvidenceResultCount) + result[queryKey] = snapshot(from: metric, resultEvidence: Array(evidence)) } return result } @@ -615,9 +663,7 @@ enum EstimatedKeywordDifficultyStore { rankingSource: RankingSource ) throws { guard let fallback else { - guard rankingSource != .iTunesFallback else { - throw EstimatedKeywordDifficultyStoreError.invalidFallback - } + // A direct iTunes page has no fallback transition to describe. return } guard rankingSource == .iTunesFallback, @@ -694,6 +740,31 @@ enum EstimatedKeywordDifficultyStore { } } + private static func latestMetrics( + queryKeys: [String], + in modelContext: ModelContext + ) throws -> [String: EstimatedKeywordDifficultyMetric] { + let targetQueryKeys = Array(Set(queryKeys)) + guard !targetQueryKeys.isEmpty else { return [:] } + + let descriptor = FetchDescriptor( + predicate: #Predicate { metric in + targetQueryKeys.contains(metric.queryKey) + } + ) + let groupedMetrics = Dictionary( + grouping: try modelContext.fetch(descriptor), + by: \.queryKey + ) + var result: [String: EstimatedKeywordDifficultyMetric] = [:] + result.reserveCapacity(groupedMetrics.count) + for (queryKey, metrics) in groupedMetrics { + guard let metric = latestMetric(in: metrics) else { continue } + result[queryKey] = metric + } + return result + } + private static func compareRevision( _ left: EstimatedKeywordDifficultyMetric, to right: EstimatedKeywordDifficultyMetric @@ -916,6 +987,46 @@ enum EstimatedKeywordDifficultyStore { ) } + private static func summary( + from metric: EstimatedKeywordDifficultyMetric + ) -> EstimatedKeywordDifficultySummary { + EstimatedKeywordDifficultySummary( + queryKey: metric.queryKey, + calculationID: metric.calculationID, + keyword: metric.keyword, + storefront: metric.storefront, + platformRaw: metric.platformRaw, + stateRaw: metric.stateRaw, + score: metric.score, + confidenceScore: metric.confidenceScore, + confidenceRaw: metric.confidenceRaw, + unavailableReasonRaw: metric.unavailableReasonRaw, + estimationSourceRaw: metric.estimationSourceRaw, + algorithmIdentifier: metric.algorithmIdentifier, + algorithmVersion: metric.algorithmVersion, + requestedResultLimit: metric.requestedResultLimit, + providerResultCount: metric.providerResultCount, + consideredResultCount: metric.consideredResultCount, + ratedResultCount: metric.ratedResultCount, + weightedRatingCoveragePercentage: metric.weightedRatingCoveragePercentage, + maximumRatingCount: metric.maximumRatingCount, + medianRatingCount: metric.medianRatingCount, + ratingAuthorityScore: metric.ratingAuthorityScore, + metadataSaturationScore: metric.metadataSaturationScore, + exactTitlePhraseMatchCount: metric.exactTitlePhraseMatchCount, + exactSubtitlePhraseMatchCount: metric.exactSubtitlePhraseMatchCount, + rankingSourceRaw: metric.rankingSourceRaw, + rankingFetchedAt: metric.rankingFetchedAt, + computedAt: metric.computedAt, + fallbackProviderRaw: metric.fallbackProviderRaw, + fallbackCategoryRaw: metric.fallbackCategoryRaw, + fallbackTransportCode: metric.fallbackTransportCode, + fallbackHTTPStatus: metric.fallbackHTTPStatus, + fallbackResponseFailureRaw: metric.fallbackResponseFailureRaw, + notes: metric.notes + ) + } + private static func snapshot( from payload: EstimatedKeywordDifficultyPersistencePayload ) -> EstimatedKeywordDifficultySnapshot { diff --git a/OpenASO/Services/Persistence/TrackedKeywordDeletionService.swift b/OpenASO/Services/Persistence/TrackedKeywordDeletionService.swift new file mode 100644 index 0000000..a7d8761 --- /dev/null +++ b/OpenASO/Services/Persistence/TrackedKeywordDeletionService.swift @@ -0,0 +1,243 @@ +import Foundation +import SwiftData + +struct TrackedKeywordDeletionRequest: Hashable, Sendable { + let identityKey: String + let trackCreatedAt: Date + + init(identityKey: String, trackCreatedAt: Date) { + self.identityKey = identityKey + self.trackCreatedAt = trackCreatedAt + } + + init(track: TrackedAppKeyword) { + self.init( + identityKey: track.identityKey, + trackCreatedAt: track.createdAt + ) + } +} + +struct TrackedAppDeletionRequest: Hashable, Sendable { + let appStoreID: Int64 + let appCreatedAt: Date + + init(appStoreID: Int64, appCreatedAt: Date) { + self.appStoreID = appStoreID + self.appCreatedAt = appCreatedAt + } + + init(app: TrackedApp) { + self.init( + appStoreID: app.appStoreID, + appCreatedAt: app.createdAt + ) + } +} + +struct TrackedKeywordDeletionResult: Equatable, Sendable { + let deletedIdentityKeys: [String] + let deletedQueryKeys: [String] + + var deletedTrackCount: Int { + deletedIdentityKeys.count + } +} + +struct TrackedAppDeletionResult: Equatable, Sendable { + let deletedApp: Bool + let keywordDeletion: TrackedKeywordDeletionResult + + var deletedTrackCount: Int { + keywordDeletion.deletedTrackCount + } +} + +/// Coordinates destructive keyword changes through the process-wide model-store +/// actor and verifies model generations again inside the write transaction. +/// Provider work that finishes later is rejected by the matching generation +/// checks in the ranking persistence path. +enum TrackedKeywordDeletionService { + @discardableResult + static func deleteTracks( + _ requests: [TrackedKeywordDeletionRequest], + using store: BackgroundModelStore + ) async throws -> TrackedKeywordDeletionResult { + try await store.write { modelContext in + try deleteTracks(requests, in: modelContext) + } + } + + @discardableResult + static func deleteApp( + _ request: TrackedAppDeletionRequest, + using store: BackgroundModelStore + ) async throws -> TrackedAppDeletionResult { + try await store.write { modelContext in + try deleteApp(request, in: modelContext) + } + } + + @discardableResult + static func deleteTracks( + _ requests: [TrackedKeywordDeletionRequest], + in modelContext: ModelContext + ) throws -> TrackedKeywordDeletionResult { + let requestSet = Set(requests) + let identityKeys = Array(Set(requests.map(\.identityKey))) + guard !identityKeys.isEmpty else { + return TrackedKeywordDeletionResult( + deletedIdentityKeys: [], + deletedQueryKeys: [] + ) + } + + let trackDescriptor = FetchDescriptor( + predicate: #Predicate { track in + identityKeys.contains(track.identityKey) + } + ) + let tracks = try modelContext.fetch(trackDescriptor).filter { track in + requestSet.contains(TrackedKeywordDeletionRequest(track: track)) + } + guard !tracks.isEmpty else { + return TrackedKeywordDeletionResult( + deletedIdentityKeys: [], + deletedQueryKeys: [] + ) + } + + let deletedIdentityKeys = Set(tracks.map(\.identityKey)) + let affectedQueryKeys = Set(tracks.map(\.queryKey)) + let allTracks = try modelContext.fetch(FetchDescriptor()) + let remainingQueryKeys = Set( + allTracks.lazy + .filter { !deletedIdentityKeys.contains($0.identityKey) } + .map(\.queryKey) + ) + let orphanedQueryKeys = affectedQueryKeys.subtracting(remainingQueryKeys) + + try deleteRefreshState( + trackIdentityKeys: Array(deletedIdentityKeys), + in: modelContext + ) + try deleteTrackRankings( + trackIdentityKeys: Array(deletedIdentityKeys), + in: modelContext + ) + for track in tracks { + modelContext.delete(track) + } + + if !orphanedQueryKeys.isEmpty { + try deleteOrphanedEstimatedDifficulty( + queryKeys: Array(orphanedQueryKeys), + in: modelContext + ) + } + + return TrackedKeywordDeletionResult( + deletedIdentityKeys: deletedIdentityKeys.sorted(), + deletedQueryKeys: orphanedQueryKeys.sorted() + ) + } + + @discardableResult + static func deleteApp( + _ request: TrackedAppDeletionRequest, + in modelContext: ModelContext + ) throws -> TrackedAppDeletionResult { + let appStoreID = request.appStoreID + var appDescriptor = FetchDescriptor( + predicate: #Predicate { app in + app.appStoreID == appStoreID + } + ) + appDescriptor.fetchLimit = 1 + guard let app = try modelContext.fetch(appDescriptor).first, + app.createdAt == request.appCreatedAt + else { + return TrackedAppDeletionResult( + deletedApp: false, + keywordDeletion: TrackedKeywordDeletionResult( + deletedIdentityKeys: [], + deletedQueryKeys: [] + ) + ) + } + + let trackDescriptor = FetchDescriptor( + predicate: #Predicate { track in + track.appStoreID == appStoreID + } + ) + let keywordDeletion = try deleteTracks( + try modelContext.fetch(trackDescriptor).map(TrackedKeywordDeletionRequest.init(track:)), + in: modelContext + ) + modelContext.delete(app) + + return TrackedAppDeletionResult( + deletedApp: true, + keywordDeletion: keywordDeletion + ) + } + + private static func deleteRefreshState( + trackIdentityKeys: [String], + in modelContext: ModelContext + ) throws { + try TrackedKeywordRefreshStatusStore.deleteStatuses( + for: trackIdentityKeys, + in: modelContext + ) + + let attemptDescriptor = FetchDescriptor( + predicate: #Predicate { attempt in + trackIdentityKeys.contains(attempt.trackIdentityKey) + } + ) + for attempt in try modelContext.fetch(attemptDescriptor) { + modelContext.delete(attempt) + } + } + + private static func deleteTrackRankings( + trackIdentityKeys: [String], + in modelContext: ModelContext + ) throws { + let snapshotDescriptor = FetchDescriptor( + predicate: #Predicate { snapshot in + trackIdentityKeys.contains(snapshot.trackIdentityKey) + } + ) + let snapshots = try modelContext.fetch(snapshotDescriptor) + let snapshotKeys = snapshots.map(\.snapshotKey) + if !snapshotKeys.isEmpty { + let resultDescriptor = FetchDescriptor( + predicate: #Predicate { result in + snapshotKeys.contains(result.snapshotKey) + } + ) + for result in try modelContext.fetch(resultDescriptor) { + modelContext.delete(result) + } + } + for snapshot in snapshots { + modelContext.delete(snapshot) + } + } + + /// Ranking history, popularity metrics, and the normalized query remain + /// available for audit/history views. Only the current estimated heuristic + /// is invalid once no active track owns its generation. + private static func deleteOrphanedEstimatedDifficulty( + queryKeys: [String], + in modelContext: ModelContext + ) throws { + try EstimatedKeywordDifficultyStore.delete( + queryKeys: queryKeys, + in: modelContext + ) + } +} diff --git a/OpenASO/Services/SearchRanking/AppStoreWebRankingProvider.swift b/OpenASO/Services/SearchRanking/AppStoreWebRankingProvider.swift new file mode 100644 index 0000000..d1bb9f7 --- /dev/null +++ b/OpenASO/Services/SearchRanking/AppStoreWebRankingProvider.swift @@ -0,0 +1,450 @@ +import Foundation + +final class AppStoreWebRankingProvider: SearchRankingProvider { + private let httpClient: any HTTPClient + + init(httpClient: any HTTPClient) { + self.httpClient = httpClient + } + + func search( + keyword: String, + storefrontCode: String, + platform: AppPlatform, + limit: Int + ) async throws -> SearchRankingPage { + let validatedRequest = try SearchRankingRequestValidator.validate( + keyword: keyword, + storefrontCode: storefrontCode, + platform: platform, + limit: limit + ) + let request = try Self.urlRequest(for: validatedRequest) + + try Task.checkCancellation() + let (data, response) = try await httpClient.data(for: request) + try Task.checkCancellation() + + guard let httpResponse = response as? HTTPURLResponse else { + throw SearchRankingProviderError.responseFailure(.nonHTTPResponse) + } + guard (200 ..< 300).contains(httpResponse.statusCode) else { + throw SearchRankingProviderError.httpStatus(httpResponse.statusCode) + } + + return try Self.parse(data, request: validatedRequest) + } + + private static func urlRequest( + for request: ValidatedSearchRankingRequest + ) throws -> URLRequest { + var components = URLComponents() + components.scheme = "https" + components.host = "apps.apple.com" + components.path = "/\(request.storefrontCode)/\(pathComponent(for: request.platform))/search" + components.queryItems = [URLQueryItem(name: "term", value: request.keyword)] + + guard let url = components.url else { + throw SearchRankingProviderError.invalidStorefront + } + + var urlRequest = URLRequest(url: url, timeoutInterval: 20) + urlRequest.setValue( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + + "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15", + forHTTPHeaderField: "User-Agent" + ) + urlRequest.setValue( + "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + forHTTPHeaderField: "Accept" + ) + return urlRequest + } + + private static func pathComponent(for platform: AppPlatform) -> String { + switch platform { + case .iphone: + "iphone" + case .ipad: + "ipad" + case .mac: + "mac" + } + } + + private static func parse( + _ data: Data, + request: ValidatedSearchRankingRequest + ) throws -> SearchRankingPage { + guard let html = String(data: data, encoding: .utf8) else { + throw SearchRankingProviderError.responseFailure(.decodingFailed) + } + + let serializedMatches = serializedServerDataMatches(in: html) + guard !serializedMatches.isEmpty else { + throw SearchRankingProviderError.responseFailure(.serializedServerDataMissing) + } + guard serializedMatches.count == 1 else { + throw SearchRankingProviderError.responseFailure(.pageShapeChanged) + } + + let jsonData = Data(htmlDecoded(serializedMatches[0]).utf8) + let envelope: SearchResultsEnvelope + do { + envelope = try JSONDecoder().decode(SearchResultsEnvelope.self, from: jsonData) + } catch { + throw SearchRankingProviderError.responseFailure(.decodingFailed) + } + + let expectedPlatform = pathComponent(for: request.platform) + let matchingEntries = envelope.data.filter { entry in + entry.intent?.kind == "SearchResultsPageIntent" + && entry.intent?.term == request.keyword + && entry.intent?.storefront.lowercased() == request.storefrontCode + && entry.intent?.platform == expectedPlatform + } + guard !matchingEntries.isEmpty else { + throw SearchRankingProviderError.responseFailure(.requestIntentMissing) + } + guard matchingEntries.count == 1 else { + throw SearchRankingProviderError.responseFailure(.requestIntentAmbiguous) + } + + let page = matchingEntries[0].page + guard page.kind == "SearchResultsPage", let shelves = page.shelves else { + throw SearchRankingProviderError.responseFailure(.pageShapeChanged) + } + + let authoritativeShelves = shelves.filter { shelf in + shelf.id == "SearchResults.shelfId" + && shelf.kind == "Shelf" + && shelf.contentType == "searchResult" + } + guard !authoritativeShelves.isEmpty else { + throw SearchRankingProviderError.responseFailure(.authoritativeShelfMissing) + } + guard authoritativeShelves.count == 1 else { + throw SearchRankingProviderError.responseFailure(.authoritativeShelfAmbiguous) + } + guard let results = authoritativeShelves[0].items else { + throw SearchRankingProviderError.responseFailure(.pageShapeChanged) + } + + var seenAppStoreIDs = Set() + var items: [SearchRankingItem] = [] + for result in results { + switch result.kind { + case "AppSearchResult": + guard result.resultType == "content" else { + if result.resultType == "ad" { continue } + throw SearchRankingProviderError.responseFailure(.malformedSearchResult) + } + fallthrough + case "AppEventSearchResult": + if result.kind == "AppEventSearchResult", result.resultType != "appEvent" { + throw SearchRankingProviderError.responseFailure(.malformedSearchResult) + } + guard let lockup = result.lockup, + let appStoreID = lockup.appStoreID, + appStoreID > 0 else { + throw SearchRankingProviderError.responseFailure(.malformedSearchResult) + } + guard seenAppStoreIDs.insert(appStoreID).inserted else { continue } + guard let name = nonEmpty(lockup.title) else { + throw SearchRankingProviderError.responseFailure(.malformedSearchResult) + } + + let screenshots = screenshotURLs(from: lockup.screenshots) + items.append(SearchRankingItem( + position: items.count + 1, + appStoreID: appStoreID, + bundleID: nonEmpty(lockup.bundleID), + name: name, + subtitle: nonEmpty(lockup.subtitle), + sellerName: nonEmpty(lockup.developerName), + iconURLString: artworkURLString(from: lockup.icon, width: 100, height: 100), + screenshotURLs: screenshots.iphone, + ipadScreenshotURLs: screenshots.ipad, + ratingCount: ratingCount(from: lockup.ratingCount), + averageRating: lockup.rating, + platform: request.platform + )) + case "EditorialSearchResult", + "DeveloperSearchResult", + "AppBundleSearchResult", + "InAppSearchResult", + "GroupingSearchResult": + continue + default: + throw SearchRankingProviderError.responseFailure(.malformedSearchResult) + } + } + + if items.count < request.limit { + // Apple server-renders only an initial shelf. A non-null nextPage means + // returning this slice as a complete ranking would produce false misses. + guard page.hasNextPageField else { + throw SearchRankingProviderError.responseFailure(.pageShapeChanged) + } + guard page.nextPage == nil else { + throw SearchRankingProviderError.responseFailure(.truncatedResults) + } + } + + return SearchRankingPage( + items: Array(items.prefix(request.limit)), + source: .appStoreWeb + ) + } + + private static func serializedServerDataMatches(in html: String) -> [String] { + let pattern = #"]*id=["']serialized-server-data["'][^>]*>(.*?)"# + guard let regex = try? NSRegularExpression(pattern: pattern, options: [.dotMatchesLineSeparators]) else { + return [] + } + + let fullRange = NSRange(html.startIndex ..< html.endIndex, in: html) + return regex.matches(in: html, range: fullRange).compactMap { match in + guard match.numberOfRanges > 1, + let captureRange = Range(match.range(at: 1), in: html) else { + return nil + } + return String(html[captureRange]).trimmingCharacters(in: .whitespacesAndNewlines) + } + } + + private static func htmlDecoded(_ value: String) -> String { + value + .replacingOccurrences(of: """, with: "\"") + .replacingOccurrences(of: """, with: "\"") + .replacingOccurrences(of: "'", with: "'") + .replacingOccurrences(of: "<", with: "<") + .replacingOccurrences(of: ">", with: ">") + .replacingOccurrences(of: "&", with: "&") + } + + private static func screenshotURLs( + from groups: [SearchResultsLockup.Screenshots]? + ) -> (iphone: [String], ipad: [String]) { + var iphone: [String] = [] + var ipad: [String] = [] + + for group in groups ?? [] { + let urls = deduplicated(group.artwork.compactMap { + artworkURLString(from: $0, width: $0.width, height: $0.height) + }) + switch group.mediaPlatform?.appPlatform { + case "phone": + iphone.append(contentsOf: urls) + case "pad": + ipad.append(contentsOf: urls) + default: + continue + } + } + + return (deduplicated(iphone), deduplicated(ipad)) + } + + private static func artworkURLString( + from artwork: SearchResultsArtwork?, + width: Int?, + height: Int? + ) -> String? { + guard let artwork else { return nil } + if let url = nonEmpty(artwork.url) { + return url + } + guard var template = nonEmpty(artwork.template) else { return nil } + + template = template.replacingOccurrences(of: "{w}", with: String(width ?? artwork.width ?? 100)) + template = template.replacingOccurrences(of: "{h}", with: String(height ?? artwork.height ?? 100)) + template = template.replacingOccurrences(of: "{c}", with: "bb") + template = template.replacingOccurrences(of: "{f}", with: "jpg") + return template + } + + private static func ratingCount(from value: SearchResultsScalar?) -> Int? { + guard let value else { return nil } + switch value { + case .integer(let count): + return count >= 0 ? count : nil + case .double(let count): + guard count.isFinite, count >= 0, count <= Double(Int.max) else { return nil } + return Int(count.rounded()) + case .string(let rawValue): + let normalized = rawValue + .trimmingCharacters(in: .whitespacesAndNewlines) + .replacingOccurrences(of: ",", with: "") + guard !normalized.isEmpty, + normalized.allSatisfy(\.isNumber), + let count = Int(normalized), + count >= 0 else { + return nil + } + return count + } + } + + private static func nonEmpty(_ value: String?) -> String? { + guard let value else { return nil } + let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? nil : trimmed + } + + private static func deduplicated(_ values: [String]) -> [String] { + var seen = Set() + return values.filter { seen.insert($0).inserted } + } +} + +private struct SearchResultsEnvelope: Decodable { + let data: [SearchResultsEntry] +} + +private struct SearchResultsEntry: Decodable { + let intent: SearchResultsIntent? + let page: SearchResultsPagePayload + + private enum CodingKeys: String, CodingKey { + case intent + case page = "data" + } +} + +private struct SearchResultsIntent: Decodable { + let term: String + let storefront: String + let platform: String + let kind: String + + private enum CodingKeys: String, CodingKey { + case term + case storefront + case platform + case kind = "$kind" + } +} + +private struct SearchResultsPagePayload: Decodable { + let kind: String? + let shelves: [SearchResultsShelf]? + let nextPage: SearchResultsNextPage? + let hasNextPageField: Bool + + private enum CodingKeys: String, CodingKey { + case kind = "$kind" + case shelves + case nextPage + } + + init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + kind = try container.decodeIfPresent(String.self, forKey: .kind) + shelves = try container.decodeIfPresent([SearchResultsShelf].self, forKey: .shelves) + nextPage = try container.decodeIfPresent(SearchResultsNextPage.self, forKey: .nextPage) + hasNextPageField = container.contains(.nextPage) + } +} + +private struct SearchResultsNextPage: Decodable {} + +private struct SearchResultsShelf: Decodable { + let id: String? + let kind: String? + let contentType: String? + let items: [SearchResultsItem]? + + private enum CodingKeys: String, CodingKey { + case id + case kind = "$kind" + case contentType + case items + } +} + +private struct SearchResultsItem: Decodable { + let kind: String? + let resultType: String? + let lockup: SearchResultsLockup? + + private enum CodingKeys: String, CodingKey { + case kind = "$kind" + case resultType + case lockup + } +} + +private struct SearchResultsLockup: Decodable { + struct Screenshots: Decodable { + struct MediaPlatform: Decodable { + let appPlatform: String? + } + + let artwork: [SearchResultsArtwork] + let mediaPlatform: MediaPlatform? + } + + let appStoreID: Int64? + let bundleID: String? + let title: String? + let subtitle: String? + let developerName: String? + let icon: SearchResultsArtwork? + let screenshots: [Screenshots]? + let rating: Double? + let ratingCount: SearchResultsScalar? + + private enum CodingKeys: String, CodingKey { + case appStoreID = "adamId" + case bundleID = "bundleId" + case title + case subtitle + case developerName + case icon + case screenshots + case rating + case ratingCount + } + + init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + if let stringID = try? container.decode(String.self, forKey: .appStoreID) { + appStoreID = Int64(stringID) + } else { + appStoreID = try? container.decode(Int64.self, forKey: .appStoreID) + } + bundleID = try? container.decode(String.self, forKey: .bundleID) + title = try? container.decode(String.self, forKey: .title) + subtitle = try? container.decode(String.self, forKey: .subtitle) + developerName = try? container.decode(String.self, forKey: .developerName) + icon = try? container.decode(SearchResultsArtwork.self, forKey: .icon) + screenshots = try? container.decode([Screenshots].self, forKey: .screenshots) + rating = try? container.decode(Double.self, forKey: .rating) + ratingCount = try? container.decode(SearchResultsScalar.self, forKey: .ratingCount) + } +} + +private struct SearchResultsArtwork: Decodable { + let url: String? + let template: String? + let width: Int? + let height: Int? +} + +private enum SearchResultsScalar: Decodable { + case integer(Int) + case double(Double) + case string(String) + + init(from decoder: any Decoder) throws { + let container = try decoder.singleValueContainer() + if let value = try? container.decode(Int.self) { + self = .integer(value) + } else if let value = try? container.decode(Double.self) { + self = .double(value) + } else { + self = .string(try container.decode(String.self)) + } + } +} diff --git a/OpenASO/Services/SearchRanking/EstimatedKeywordDifficultyPersistenceAdapter.swift b/OpenASO/Services/SearchRanking/EstimatedKeywordDifficultyPersistenceAdapter.swift new file mode 100644 index 0000000..c79a949 --- /dev/null +++ b/OpenASO/Services/SearchRanking/EstimatedKeywordDifficultyPersistenceAdapter.swift @@ -0,0 +1,243 @@ +import Foundation +import SwiftData + +/// Converts the pure estimator output and redacted ranking provenance into the +/// append-only V4 persistence contract. Invalid or incomplete provider +/// provenance returns `nil` so an otherwise valid ranking refresh still lands. +enum EstimatedKeywordDifficultyPersistenceAdapter { + /// Revalidates the track generation inside the estimate transaction. This + /// prevents a provider result from restoring orphaned query-level data + /// after its originating track was deleted or replaced. + @discardableResult + static func upsertIfCurrent( + for pageResult: RankingRefreshPageResult, + in modelContext: ModelContext + ) throws -> EstimatedKeywordDifficultyUpsertOutcome? { + guard let payload = pageResult.estimatedDifficultyPayload else { return nil } + let identityKey = pageResult.request.identityKey + var descriptor = FetchDescriptor( + predicate: #Predicate { track in + track.identityKey == identityKey + } + ) + descriptor.fetchLimit = 1 + guard let track = try modelContext.fetch(descriptor).first, + pageResult.request.matchesGeneration(of: track), + payload.queryKey == track.queryKey + else { + return nil + } + return try EstimatedKeywordDifficultyStore.upsert(payload, in: modelContext) + } + + static func payload( + request: RankingRefreshRequest, + page: SearchRankingPage, + requestedResultLimit: Int, + rankingFetchedAt: Date, + calculationID: UUID = UUID(), + now: () -> Date = { Date() } + ) -> EstimatedKeywordDifficultyPersistencePayload? { + guard (1 ... EstimatedKeywordDifficultyStore.maximumRequestedResultCount) + .contains(requestedResultLimit), + page.resultCount <= requestedResultLimit, + request.queryKey == KeywordQuery.makeQueryKey( + term: request.term, + storefront: request.storefront, + platform: request.platform + ), + let fallback = fallbackProvenance(for: page) + else { + return nil + } + + let estimation = KeywordDifficultyEstimator.estimate( + keyword: request.term, + searchResults: page.items + ) + // Provenance records when calculation finished, not when it began. + // Clamp non-monotonic injected clocks so the persistence invariant + // still holds relative to the ranking evidence timestamp. + let computedAt = max(now(), rankingFetchedAt) + let fields = persistenceFields(for: estimation) + + return EstimatedKeywordDifficultyPersistencePayload( + queryKey: request.queryKey, + calculationID: calculationID, + keyword: request.term, + storefront: request.storefront, + platform: request.platform, + result: fields.result, + algorithmIdentifier: fields.algorithmIdentifier, + algorithmVersion: fields.algorithmVersion, + requestedResultLimit: requestedResultLimit, + providerResultCount: page.resultCount, + evidence: evidence(fields.evidence), + rankingSource: page.source, + rankingFetchedAt: rankingFetchedAt, + computedAt: computedAt, + fallback: fallback.value, + notes: fields.notes + ) + } + + private static func persistenceFields( + for estimation: KeywordDifficultyEstimation + ) -> ( + result: EstimatedKeywordDifficultyPersistenceResult, + algorithmIdentifier: String, + algorithmVersion: Int, + evidence: KeywordDifficultyEvidence, + notes: [String] + ) { + switch estimation { + case .estimated(let estimate): + return ( + .estimated( + score: estimate.score, + confidenceScore: estimate.confidenceScore, + confidence: confidence(estimate.confidence) + ), + estimate.algorithmIdentifier, + estimate.algorithmVersion, + estimate.evidence, + estimate.notes + ) + case .unavailable(let unavailable): + return ( + .unavailable(reason: unavailableReason(unavailable.reason)), + unavailable.algorithmIdentifier, + unavailable.algorithmVersion, + unavailable.evidence, + unavailable.notes + ) + } + } + + private static func evidence( + _ evidence: KeywordDifficultyEvidence + ) -> EstimatedKeywordDifficultyEvidence { + EstimatedKeywordDifficultyEvidence( + consideredResultCount: evidence.consideredResultCount, + ratedResultCount: evidence.ratedResultCount, + weightedRatingCoveragePercentage: evidence.weightedRatingCoveragePercentage, + maximumRatingCount: evidence.maximumRatingCount, + medianRatingCount: evidence.medianRatingCount, + ratingAuthorityScore: evidence.ratingAuthorityScore, + metadataSaturationScore: evidence.metadataSaturationScore, + resultEvidence: evidence.resultEvidence.map { result in + EstimatedKeywordDifficultyResultEvidence( + position: result.position, + appStoreID: result.appStoreID, + title: result.title, + subtitle: result.subtitle, + ratingCount: result.ratingCount, + ratingAuthorityScore: result.ratingAuthorityScore, + titleTokenCoveragePercentage: result.titleTokenCoveragePercentage, + combinedTokenCoveragePercentage: result.combinedTokenCoveragePercentage, + metadataMatchScore: result.metadataMatchScore, + exactTitlePhraseMatch: result.exactTitlePhraseMatch, + exactSubtitlePhraseMatch: result.exactSubtitlePhraseMatch + ) + } + ) + } + + private static func confidence( + _ confidence: EstimatedKeywordDifficulty.Confidence + ) -> EstimatedKeywordDifficultyConfidence { + switch confidence { + case .low: .low + case .medium: .medium + case .high: .high + } + } + + private static func unavailableReason( + _ reason: KeywordDifficultyUnavailable.Reason + ) -> EstimatedKeywordDifficultyUnavailableReason { + switch reason { + case .emptyKeyword: .emptyKeyword + case .insufficientResults: .insufficientResults + case .insufficientRatingEvidence: .insufficientRatingEvidence + } + } + + private struct OptionalFallback { + let value: EstimatedKeywordDifficultyFallbackProvenance? + } + + /// A wrapper distinguishes valid primary provenance (`nil`) from invalid + /// fallback provenance (outer `nil`). + private static func fallbackProvenance( + for page: SearchRankingPage + ) -> OptionalFallback? { + switch page.source { + case .appStoreWeb: + guard page.fallbackContext == nil else { return nil } + return OptionalFallback(value: nil) + case .iTunesFallback: + guard let context = page.fallbackContext else { + return OptionalFallback(value: nil) + } + guard context.provider == .appStoreWeb else { return nil } + return OptionalFallback(value: EstimatedKeywordDifficultyFallbackProvenance( + provider: .appStoreWeb, + category: fallbackCategory(context.category), + transportCode: transportCode(context.category), + httpStatus: httpStatus(context.category), + responseFailure: responseFailure(context.category) + )) + } + } + + private static func fallbackCategory( + _ category: SearchRankingFailureCategory + ) -> EstimatedKeywordDifficultyFallbackCategory { + switch category { + case .transport: .transport + case .httpStatus: .httpStatus + case .response: .response + case .provider: .provider + case .other: .other + } + } + + private static func transportCode(_ category: SearchRankingFailureCategory) -> Int? { + guard case .transport(let code) = category else { return nil } + return code + } + + private static func httpStatus(_ category: SearchRankingFailureCategory) -> Int? { + guard case .httpStatus(let status) = category else { return nil } + return status + } + + private static func responseFailure( + _ category: SearchRankingFailureCategory + ) -> EstimatedKeywordDifficultyFallbackResponseFailure? { + guard case .response(let failure) = category else { return nil } + switch failure { + case .serializedServerDataMissing: + return .serializedServerDataMissing + case .decodingFailed: + return .decodingFailed + case .nonHTTPResponse: + return .nonHTTPResponse + case .requestIntentMissing: + return .requestIntentMissing + case .requestIntentAmbiguous: + return .requestIntentAmbiguous + case .pageShapeChanged: + return .pageShapeChanged + case .authoritativeShelfMissing: + return .authoritativeShelfMissing + case .authoritativeShelfAmbiguous: + return .authoritativeShelfAmbiguous + case .malformedSearchResult: + return .malformedSearchResult + case .truncatedResults: + return .truncatedResults + } + } +} diff --git a/OpenASO/Services/SearchRanking/ITunesSearchFallbackProvider.swift b/OpenASO/Services/SearchRanking/ITunesSearchFallbackProvider.swift index 407fc5e..5f2ac84 100644 --- a/OpenASO/Services/SearchRanking/ITunesSearchFallbackProvider.swift +++ b/OpenASO/Services/SearchRanking/ITunesSearchFallbackProvider.swift @@ -8,18 +8,19 @@ final class ITunesSearchFallbackProvider: SearchRankingProvider { } func search(keyword: String, storefrontCode: String, platform: AppPlatform, limit: Int) async throws -> SearchRankingPage { - let trimmedKeyword = keyword.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmedKeyword.isEmpty else { - throw OpenASOError.emptyQuery - } - let cappedLimit = min(max(1, limit), SearchRankingCrawl.fullKeywordRankingLimit) + let validatedRequest = try SearchRankingRequestValidator.validate( + keyword: keyword, + storefrontCode: storefrontCode, + platform: platform, + limit: limit + ) var components = URLComponents(string: "https://itunes.apple.com/search")! components.queryItems = [ - URLQueryItem(name: "term", value: trimmedKeyword), - URLQueryItem(name: "entity", value: "software"), - URLQueryItem(name: "country", value: storefrontCode.lowercased()), - URLQueryItem(name: "limit", value: String(cappedLimit)) + URLQueryItem(name: "term", value: validatedRequest.keyword), + URLQueryItem(name: "entity", value: Self.entity(for: validatedRequest.platform)), + URLQueryItem(name: "country", value: validatedRequest.storefrontCode), + URLQueryItem(name: "limit", value: String(validatedRequest.limit)) ] var request = URLRequest(url: components.url!) @@ -52,13 +53,24 @@ final class ITunesSearchFallbackProvider: SearchRankingProvider { appletvScreenshotURLs: payload.appletvScreenshotUrls ?? [], ratingCount: payload.userRatingCount, averageRating: payload.averageUserRating, - platform: platform + platform: validatedRequest.platform ) } return SearchRankingPage(items: items, source: .iTunesFallback) } + private static func entity(for platform: AppPlatform) -> String { + switch platform { + case .iphone: + "software" + case .ipad: + "iPadSoftware" + case .mac: + "macSoftware" + } + } + private static let decoder: JSONDecoder = { let decoder = JSONDecoder() decoder.dateDecodingStrategy = .iso8601 diff --git a/OpenASO/Services/SearchRanking/KeywordDifficultyEstimator.swift b/OpenASO/Services/SearchRanking/KeywordDifficultyEstimator.swift new file mode 100644 index 0000000..e480fb8 --- /dev/null +++ b/OpenASO/Services/SearchRanking/KeywordDifficultyEstimator.swift @@ -0,0 +1,421 @@ +import Foundation + +struct KeywordDifficultyRankedResult: Equatable, Sendable { + let position: Int + let appStoreID: Int64 + let title: String + let subtitle: String? + let ratingCount: Int? + + init( + position: Int, + appStoreID: Int64, + title: String, + subtitle: String? = nil, + ratingCount: Int? + ) { + self.position = position + self.appStoreID = appStoreID + self.title = title + self.subtitle = subtitle + self.ratingCount = ratingCount + } + + init(searchResult: SearchRankingItem) { + self.init( + position: searchResult.position, + appStoreID: searchResult.appStoreID, + title: searchResult.name, + subtitle: searchResult.subtitle, + ratingCount: searchResult.ratingCount + ) + } +} + +enum KeywordDifficultyEstimation: Equatable, Sendable { + case estimated(EstimatedKeywordDifficulty) + case unavailable(KeywordDifficultyUnavailable) +} + +struct EstimatedKeywordDifficulty: Equatable, Sendable { + enum Confidence: String, Equatable, Sendable { + case low + case medium + case high + } + + let score: Int + let confidenceScore: Int + let confidence: Confidence + let algorithmIdentifier: String + let algorithmVersion: Int + let evidence: KeywordDifficultyEvidence + let notes: [String] +} + +struct KeywordDifficultyUnavailable: Equatable, Sendable { + enum Reason: String, Equatable, Sendable { + case emptyKeyword + case insufficientResults + case insufficientRatingEvidence + } + + let reason: Reason + let algorithmIdentifier: String + let algorithmVersion: Int + let evidence: KeywordDifficultyEvidence + let notes: [String] +} + +struct KeywordDifficultyResultEvidence: Equatable, Sendable { + let position: Int + let appStoreID: Int64 + let title: String + let subtitle: String? + let ratingCount: Int? + let ratingAuthorityScore: Int? + let titleTokenCoveragePercentage: Int? + let combinedTokenCoveragePercentage: Int? + let metadataMatchScore: Int? + let exactTitlePhraseMatch: Bool + let exactSubtitlePhraseMatch: Bool +} + +struct KeywordDifficultyEvidence: Equatable, Sendable { + let consideredResultCount: Int + let ratedResultCount: Int + let weightedRatingCoveragePercentage: Int + let maximumRatingCount: Int? + let medianRatingCount: Int? + let ratingAuthorityScore: Int? + let metadataSaturationScore: Int? + let resultEvidence: [KeywordDifficultyResultEvidence] + + var ratingCoveragePercentage: Int { + guard consideredResultCount > 0 else { return 0 } + return Int((Double(ratedResultCount) / Double(consideredResultCount) * 100).rounded()) + } + + var exactTitlePhraseMatchCount: Int { + resultEvidence.count(where: \.exactTitlePhraseMatch) + } + + var exactSubtitlePhraseMatchCount: Int { + resultEvidence.count(where: \.exactSubtitlePhraseMatch) + } +} + +enum KeywordDifficultyEstimator { + static let algorithmIdentifier = "top10-authority-saturation" + static let algorithmVersion = 1 + static let maximumResultCount = 10 + static let minimumResultCount = 3 + static let minimumRatedResultCount = 3 + + static func estimate( + keyword: String, + searchResults: [SearchRankingItem] + ) -> KeywordDifficultyEstimation { + estimate( + keyword: keyword, + rankedResults: searchResults.map(KeywordDifficultyRankedResult.init(searchResult:)) + ) + } + + static func estimate( + keyword: String, + rankedResults: [KeywordDifficultyRankedResult] + ) -> KeywordDifficultyEstimation { + let consideredResults = normalizedTopResults(rankedResults) + let keywordTokens = tokens(keyword) + let resultEvidence = makeResultEvidence( + consideredResults, + keywordTokens: keywordTokens + ) + let evidence = makeEvidence(resultEvidence) + + guard !keywordTokens.isEmpty else { + return unavailable( + reason: .emptyKeyword, + evidence: evidence, + note: "Estimated competition is unavailable because the keyword is empty." + ) + } + guard consideredResults.count >= minimumResultCount else { + return unavailable( + reason: .insufficientResults, + evidence: evidence, + note: "Estimated competition is unavailable until at least \(minimumResultCount) unique top-ten ranking results are available." + ) + } + guard evidence.ratedResultCount >= minimumRatedResultCount else { + return unavailable( + reason: .insufficientRatingEvidence, + evidence: evidence, + note: "Estimated competition is unavailable until at least \(minimumRatedResultCount) top-ten results include rating counts." + ) + } + + let ratingAuthority = Double(evidence.ratingAuthorityScore ?? 0) + let metadataSaturation = Double(evidence.metadataSaturationScore ?? 0) + let score = boundedScore(ratingAuthority * 0.70 + metadataSaturation * 0.30) + let confidenceScore = confidenceScore( + consideredResultCount: evidence.consideredResultCount, + weightedRatingCoveragePercentage: evidence.weightedRatingCoveragePercentage + ) + let confidence = confidence(for: confidenceScore) + + return .estimated(EstimatedKeywordDifficulty( + score: score, + confidenceScore: confidenceScore, + confidence: confidence, + algorithmIdentifier: algorithmIdentifier, + algorithmVersion: algorithmVersion, + evidence: evidence, + notes: [ + "Heuristic estimated competition based on the current top App Store search results.", + "Uses 70% rating-count authority and 30% title/subtitle keyword saturation.", + "This is not Apple Ads difficulty, Search Popularity, or an Apple-provided metric.", + "Evidence confidence \(confidence.rawValue) (\(confidenceScore)/100): \(evidence.ratedResultCount) of \(evidence.consideredResultCount) sampled results included rating counts." + ] + )) + } + + private static func normalizedTopResults( + _ results: [KeywordDifficultyRankedResult] + ) -> [KeywordDifficultyRankedResult] { + // Provider positions are not a unique key. Keep different apps that share + // a position ordered by app ID, then deduplicate only repeated app IDs. + let sortedResults = results + .filter { (1...maximumResultCount).contains($0.position) } + .sorted { + if $0.position != $1.position { return $0.position < $1.position } + if $0.appStoreID != $1.appStoreID { return $0.appStoreID < $1.appStoreID } + let leftHasRatingEvidence = $0.ratingCount.map { $0 >= 0 } ?? false + let rightHasRatingEvidence = $1.ratingCount.map { $0 >= 0 } ?? false + if leftHasRatingEvidence != rightHasRatingEvidence { + return leftHasRatingEvidence + } + if leftHasRatingEvidence, + let leftRatingCount = $0.ratingCount, + let rightRatingCount = $1.ratingCount, + leftRatingCount != rightRatingCount + { + return leftRatingCount < rightRatingCount + } + if $0.title != $1.title { return $0.title < $1.title } + let leftSubtitle = $0.subtitle ?? "" + let rightSubtitle = $1.subtitle ?? "" + if leftSubtitle != rightSubtitle { return leftSubtitle < rightSubtitle } + if ($0.ratingCount == nil) != ($1.ratingCount == nil) { + return $0.ratingCount == nil + } + if let leftRatingCount = $0.ratingCount, + let rightRatingCount = $1.ratingCount, + leftRatingCount != rightRatingCount + { + return leftRatingCount < rightRatingCount + } + return false + } + + var seenAppStoreIDs = Set() + return Array(sortedResults.filter { + seenAppStoreIDs.insert($0.appStoreID).inserted + }.prefix(maximumResultCount)) + } + + private static func makeResultEvidence( + _ results: [KeywordDifficultyRankedResult], + keywordTokens: [String] + ) -> [KeywordDifficultyResultEvidence] { + let uniqueKeywordTokens = Set(keywordTokens) + return results.map { result in + let validRatingCount = result.ratingCount.flatMap { $0 >= 0 ? $0 : nil } + let titleTokens = tokens(result.title) + let subtitleTokens = tokens(result.subtitle ?? "") + let titleTokenSet = Set(titleTokens) + let combinedTokenSet = titleTokenSet.union(subtitleTokens) + let exactTitlePhraseMatch = containsPhrase(keywordTokens, in: titleTokens) + let exactSubtitlePhraseMatch = containsPhrase(keywordTokens, in: subtitleTokens) + + let titleCoverage = tokenCoverage( + keywordTokens: uniqueKeywordTokens, + candidateTokens: titleTokenSet + ) + let combinedCoverage = tokenCoverage( + keywordTokens: uniqueKeywordTokens, + candidateTokens: combinedTokenSet + ) + let metadataMatchScore: Double? + if keywordTokens.isEmpty { + metadataMatchScore = nil + } else { + metadataMatchScore = max( + titleCoverage * 100, + combinedCoverage * 75 + ) + } + + return KeywordDifficultyResultEvidence( + position: result.position, + appStoreID: result.appStoreID, + title: result.title, + subtitle: result.subtitle, + ratingCount: validRatingCount, + ratingAuthorityScore: validRatingCount.map(ratingAuthorityScore), + titleTokenCoveragePercentage: keywordTokens.isEmpty ? nil : boundedScore(titleCoverage * 100), + combinedTokenCoveragePercentage: keywordTokens.isEmpty ? nil : boundedScore(combinedCoverage * 100), + metadataMatchScore: metadataMatchScore.map(boundedScore), + exactTitlePhraseMatch: exactTitlePhraseMatch, + exactSubtitlePhraseMatch: exactSubtitlePhraseMatch + ) + } + } + + private static func makeEvidence( + _ resultEvidence: [KeywordDifficultyResultEvidence] + ) -> KeywordDifficultyEvidence { + let ratedEvidence = resultEvidence.filter { $0.ratingCount != nil } + let ratingCounts = ratedEvidence.compactMap(\.ratingCount).sorted() + let totalWeight = resultEvidence.reduce(0.0) { partialResult, evidence in + partialResult + positionWeight(evidence.position) + } + let ratedWeight = ratedEvidence.reduce(0.0) { partialResult, evidence in + partialResult + positionWeight(evidence.position) + } + let weightedRatingCoverage = totalWeight > 0 ? ratedWeight / totalWeight * 100 : 0 + + return KeywordDifficultyEvidence( + consideredResultCount: resultEvidence.count, + ratedResultCount: ratedEvidence.count, + weightedRatingCoveragePercentage: boundedScore(weightedRatingCoverage), + maximumRatingCount: ratingCounts.last, + medianRatingCount: medianRatingCount(ratingCounts), + ratingAuthorityScore: weightedAverage( + ratedEvidence, + value: { $0.ratingAuthorityScore } + ), + metadataSaturationScore: weightedAverage( + resultEvidence, + value: { $0.metadataMatchScore } + ), + resultEvidence: resultEvidence + ) + } + + private static func weightedAverage( + _ evidence: [KeywordDifficultyResultEvidence], + value: (KeywordDifficultyResultEvidence) -> Int? + ) -> Int? { + let availableEvidence = evidence.compactMap { item -> (KeywordDifficultyResultEvidence, Int)? in + value(item).map { (item, $0) } + } + guard !availableEvidence.isEmpty else { return nil } + + let totalWeight = availableEvidence.reduce(0.0) { partialResult, pair in + partialResult + positionWeight(pair.0.position) + } + guard totalWeight > 0 else { return nil } + let weightedValue = availableEvidence.reduce(0.0) { partialResult, pair in + partialResult + Double(pair.1) * positionWeight(pair.0.position) + } + return boundedScore(weightedValue / totalWeight) + } + + private static func ratingAuthorityScore(_ ratingCount: Int) -> Int { + boundedScore(min(1, log10(Double(ratingCount) + 1) / 6) * 100) + } + + private static func medianRatingCount(_ sortedRatingCounts: [Int]) -> Int? { + guard !sortedRatingCounts.isEmpty else { return nil } + let middleIndex = sortedRatingCounts.count / 2 + guard sortedRatingCounts.count.isMultiple(of: 2) else { + return sortedRatingCounts[middleIndex] + } + + let lower = sortedRatingCounts[middleIndex - 1] + let upper = sortedRatingCounts[middleIndex] + return lower + (upper - lower) / 2 + } + + private static func tokenCoverage( + keywordTokens: Set, + candidateTokens: Set + ) -> Double { + guard !keywordTokens.isEmpty else { return 0 } + return Double(keywordTokens.intersection(candidateTokens).count) / Double(keywordTokens.count) + } + + private static func containsPhrase(_ phrase: [String], in candidate: [String]) -> Bool { + guard !phrase.isEmpty, phrase.count <= candidate.count else { return false } + if phrase.count == 1 { + return candidate.contains(phrase[0]) + } + + for startIndex in 0...(candidate.count - phrase.count) { + if candidate[startIndex..<(startIndex + phrase.count)].elementsEqual(phrase) { + return true + } + } + return false + } + + private static func tokens(_ value: String) -> [String] { + value + .folding( + options: [.caseInsensitive, .diacriticInsensitive], + locale: Locale(identifier: "en_US_POSIX") + ) + .lowercased() + .split { !$0.isLetter && !$0.isNumber } + .map(String.init) + } + + private static func positionWeight(_ position: Int) -> Double { + Double(maximumResultCount - position + 1) + } + + private static func confidenceScore( + consideredResultCount: Int, + weightedRatingCoveragePercentage: Int + ) -> Int { + let sampleCoverage = Double(consideredResultCount) / Double(maximumResultCount) + let ratingCoverage = Double(weightedRatingCoveragePercentage) / 100 + return boundedScore(sqrt(sampleCoverage * ratingCoverage) * 100) + } + + private static func confidence( + for score: Int + ) -> EstimatedKeywordDifficulty.Confidence { + switch score { + case 85...: + return .high + case 65...: + return .medium + default: + return .low + } + } + + private static func unavailable( + reason: KeywordDifficultyUnavailable.Reason, + evidence: KeywordDifficultyEvidence, + note: String + ) -> KeywordDifficultyEstimation { + .unavailable(KeywordDifficultyUnavailable( + reason: reason, + algorithmIdentifier: algorithmIdentifier, + algorithmVersion: algorithmVersion, + evidence: evidence, + notes: [ + "This would be a heuristic estimate, not Apple Ads difficulty or an Apple-provided metric.", + note + ] + )) + } + + private static func boundedScore(_ value: Double) -> Int { + Int(min(100, max(0, value)).rounded()) + } +} diff --git a/OpenASO/Services/SearchRanking/KeywordSuggestionService.swift b/OpenASO/Services/SearchRanking/KeywordSuggestionService.swift index 87dbe32..24b9137 100644 --- a/OpenASO/Services/SearchRanking/KeywordSuggestionService.swift +++ b/OpenASO/Services/SearchRanking/KeywordSuggestionService.swift @@ -409,6 +409,7 @@ private struct KeywordSuggestionEvidence { private(set) var bestObservedRank = Int.max private(set) var currentObservedRank = Int.max private(set) var latestObservedAt = Date.distantPast + private(set) var source: RankingSource = .iTunesFallback private(set) var observationCount = 0 private(set) var observedDays: Set = [] private var collapsedEvidenceKeys: Set = [] @@ -429,6 +430,7 @@ private struct KeywordSuggestionEvidence { if observedAt > latestObservedAt || (observedAt == latestObservedAt && item.position < currentObservedRank) { latestObservedAt = observedAt currentObservedRank = item.position + source = item.observation.source } return } @@ -437,6 +439,7 @@ private struct KeywordSuggestionEvidence { if observedAt > latestObservedAt || (observedAt == latestObservedAt && item.position < currentObservedRank) { latestObservedAt = observedAt currentObservedRank = item.position + source = item.observation.source } observationCount += 1 observedDays.insert(day) @@ -508,7 +511,7 @@ private struct ScoredKeywordSuggestion { bestObservedRank: evidence.bestObservedRank, currentObservedRank: evidence.currentObservedRank, latestObservedAt: evidence.latestObservedAt, - source: .appStoreWeb + source: evidence.source ) } diff --git a/OpenASO/Services/SearchRanking/PrimaryFallbackSearchRankingProvider.swift b/OpenASO/Services/SearchRanking/PrimaryFallbackSearchRankingProvider.swift new file mode 100644 index 0000000..81e0658 --- /dev/null +++ b/OpenASO/Services/SearchRanking/PrimaryFallbackSearchRankingProvider.swift @@ -0,0 +1,190 @@ +import Foundation + +final class PrimaryFallbackSearchRankingProvider: SearchRankingProvider { + typealias FallbackPlatformSupport = @Sendable (AppPlatform) -> Bool + + private let primary: any SearchRankingProvider + private let fallback: any SearchRankingProvider + private let fallbackSupportsPlatform: FallbackPlatformSupport + + init( + primary: any SearchRankingProvider, + fallback: any SearchRankingProvider + ) { + self.primary = primary + self.fallback = fallback + self.fallbackSupportsPlatform = { platform in + switch platform { + case .iphone, .ipad, .mac: + true + } + } + } + + init( + primary: any SearchRankingProvider, + fallback: any SearchRankingProvider, + fallbackSupportsPlatform: @escaping FallbackPlatformSupport + ) { + self.primary = primary + self.fallback = fallback + self.fallbackSupportsPlatform = fallbackSupportsPlatform + } + + func search( + keyword: String, + storefrontCode: String, + platform: AppPlatform, + limit: Int + ) async throws -> SearchRankingPage { + let request = try SearchRankingRequestValidator.validate( + keyword: keyword, + storefrontCode: storefrontCode, + platform: platform, + limit: limit + ) + + do { + try Task.checkCancellation() + return try await primary.search( + keyword: request.keyword, + storefrontCode: request.storefrontCode, + platform: request.platform, + limit: request.limit + ) + } catch { + try Task.checkCancellation() + if Self.mustPropagate(error) { + throw error + } + guard let primaryContext = Self.fallbackContext( + for: error, + provider: .appStoreWeb, + requiresRecognizedFallbackFailure: true + ) else { + throw error + } + guard fallbackSupportsPlatform(request.platform) else { + throw SearchRankingProviderError.fallbackUnavailable( + platform: request.platform, + primary: primaryContext + ) + } + + do { + try Task.checkCancellation() + let page = try await fallback.search( + keyword: request.keyword, + storefrontCode: request.storefrontCode, + platform: request.platform, + limit: request.limit + ) + try Task.checkCancellation() + return SearchRankingPage( + items: page.items, + source: page.source, + fallbackContext: primaryContext + ) + } catch { + try Task.checkCancellation() + if Self.mustPropagate(error) { + throw error + } + let fallbackContext = Self.fallbackContext( + for: error, + provider: .iTunesFallback, + requiresRecognizedFallbackFailure: false + ) ?? SearchRankingFailureContext( + provider: .iTunesFallback, + category: .other + ) + throw SearchRankingProviderError.allProvidersFailed( + primary: primaryContext, + fallback: fallbackContext + ) + } + } + } + + private static func mustPropagate(_ error: any Error) -> Bool { + if error is CancellationError { + return true + } + if let urlError = error as? URLError { + return urlError.code == .cancelled || urlError.code == .timedOut + } + if let error = error as? OpenASOError { + switch error { + case .emptyQuery, .invalidAppStoreID: + return true + case .appNotFound, + .networkUnavailable, + .rateLimited, + .decodingFailed, + .unexpectedResponse, + .primaryProviderUnavailable, + .providerUnavailable: + return false + } + } + if let error = error as? SearchRankingProviderError { + switch error { + case .invalidStorefront, + .invalidLimit, + .unsupportedPlatform, + .fallbackUnavailable, + .allProvidersFailed: + return true + case .httpStatus, .responseFailure: + return false + } + } + return false + } + + private static func fallbackContext( + for error: any Error, + provider: SearchRankingFailureContext.Provider, + requiresRecognizedFallbackFailure: Bool + ) -> SearchRankingFailureContext? { + let category: SearchRankingFailureCategory + if let error = error as? URLError { + category = .transport(code: error.code.rawValue) + } else if let error = error as? SearchRankingProviderError { + switch error { + case .httpStatus(let statusCode): + category = .httpStatus(statusCode) + case .responseFailure(let failure): + category = .response(failure) + case .invalidStorefront, + .invalidLimit, + .unsupportedPlatform, + .fallbackUnavailable, + .allProvidersFailed: + return nil + } + } else if let error = error as? OpenASOError { + switch error { + case .appNotFound: + category = .httpStatus(404) + case .networkUnavailable: + category = .transport(code: nil) + case .rateLimited: + category = .httpStatus(429) + case .decodingFailed: + category = .response(.decodingFailed) + case .unexpectedResponse: + category = .response(.nonHTTPResponse) + case .primaryProviderUnavailable, .providerUnavailable: + category = .provider + case .emptyQuery, .invalidAppStoreID: + return nil + } + } else { + guard !requiresRecognizedFallbackFailure else { return nil } + category = .other + } + + return SearchRankingFailureContext(provider: provider, category: category) + } +} diff --git a/OpenASO/Services/SearchRanking/RankingRefreshCoordinator.swift b/OpenASO/Services/SearchRanking/RankingRefreshCoordinator.swift index d5b3a51..a266247 100644 --- a/OpenASO/Services/SearchRanking/RankingRefreshCoordinator.swift +++ b/OpenASO/Services/SearchRanking/RankingRefreshCoordinator.swift @@ -8,19 +8,22 @@ struct RankingRefreshRequest: Sendable { let term: String let storefront: String let platform: AppPlatform + let trackCreatedAt: Date? init( identityKey: String, queryKey: String, term: String, storefront: String, - platform: AppPlatform + platform: AppPlatform, + trackCreatedAt: Date? = nil ) { self.identityKey = identityKey self.queryKey = queryKey self.term = term self.storefront = storefront self.platform = platform + self.trackCreatedAt = trackCreatedAt } init(track: TrackedAppKeyword) { @@ -29,6 +32,12 @@ struct RankingRefreshRequest: Sendable { self.term = track.term self.storefront = track.storefront self.platform = track.platform + self.trackCreatedAt = track.createdAt + } + + func matchesGeneration(of track: TrackedAppKeyword) -> Bool { + identityKey == track.identityKey + && (trackCreatedAt.map { $0 == track.createdAt } ?? true) } } @@ -78,7 +87,8 @@ struct RankingRequestGroup: Sendable { ), term: normalizedTerm, storefront: normalizedStorefront, - platform: request.platform + platform: request.platform, + trackCreatedAt: nil ) groupIndexByKey[key] = groups.count groups.append(Self( @@ -99,7 +109,9 @@ struct RankingRequestGroup: Sendable { observedHour: pageResult.observedHour, submissionCount: pageResult.submissionCount, winningCount: pageResult.winningCount, - confidence: pageResult.confidence + confidence: pageResult.confidence, + requestedResultLimit: pageResult.requestedResultLimit, + estimatedDifficultyPayload: pageResult.estimatedDifficultyPayload ) } } @@ -113,6 +125,30 @@ struct RankingRefreshPageResult: Sendable { let submissionCount: Int let winningCount: Int let confidence: String? + let requestedResultLimit: Int + let estimatedDifficultyPayload: EstimatedKeywordDifficultyPersistencePayload? + + init( + request: RankingRefreshRequest, + page: SearchRankingPage, + searchedAt: Date, + observedHour: Int?, + submissionCount: Int, + winningCount: Int, + confidence: String?, + requestedResultLimit: Int = SearchRankingCrawl.fullKeywordRankingLimit, + estimatedDifficultyPayload: EstimatedKeywordDifficultyPersistencePayload? = nil + ) { + self.request = request + self.page = page + self.searchedAt = searchedAt + self.observedHour = observedHour + self.submissionCount = submissionCount + self.winningCount = winningCount + self.confidence = confidence + self.requestedResultLimit = requestedResultLimit + self.estimatedDifficultyPayload = estimatedDifficultyPayload + } } struct RankingMetadataEnrichmentRequest: Hashable, Sendable { @@ -151,11 +187,19 @@ struct RankingStatsRebuildRequest: Hashable, Sendable { } final class RankingRefreshCoordinator: Sendable { + typealias EstimatedDifficultyPayloadBuilder = @Sendable ( + _ request: RankingRefreshRequest, + _ page: SearchRankingPage, + _ requestedResultLimit: Int, + _ rankingFetchedAt: Date + ) -> EstimatedKeywordDifficultyPersistencePayload? + private let rankingProvider: any SearchRankingProvider private let appCatalogService: AppCatalogService private let analyticsService: AnalyticsService private let refreshTriggerRecorder: (@Sendable (Date) async -> Void)? private let metadataEnrichmentHandler: (@Sendable ([RankingMetadataEnrichmentRequest]) async -> Void)? + private let estimatedDifficultyPayloadBuilder: EstimatedDifficultyPayloadBuilder @MainActor init( @@ -163,7 +207,19 @@ final class RankingRefreshCoordinator: Sendable { appCatalogService: AppCatalogService, analyticsService: AnalyticsService? = nil, refreshTriggerRecorder: (@Sendable (Date) async -> Void)? = nil, - metadataEnrichmentHandler: (@Sendable ([RankingMetadataEnrichmentRequest]) async -> Void)? = nil + metadataEnrichmentHandler: (@Sendable ([RankingMetadataEnrichmentRequest]) async -> Void)? = nil, + estimatedDifficultyPayloadBuilder: @escaping EstimatedDifficultyPayloadBuilder = { + request, + page, + requestedResultLimit, + rankingFetchedAt in + EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: request, + page: page, + requestedResultLimit: requestedResultLimit, + rankingFetchedAt: rankingFetchedAt + ) + } ) { self.rankingProvider = rankingProvider self.appCatalogService = appCatalogService @@ -173,6 +229,7 @@ final class RankingRefreshCoordinator: Sendable { ) self.refreshTriggerRecorder = refreshTriggerRecorder self.metadataEnrichmentHandler = metadataEnrichmentHandler + self.estimatedDifficultyPayloadBuilder = estimatedDifficultyPayloadBuilder } @MainActor @@ -232,14 +289,22 @@ final class RankingRefreshCoordinator: Sendable { platform: request.platform, limit: limit ) + let fetchedAt = Date() return .success(RankingRefreshPageResult( request: request, page: page, - searchedAt: .now, + searchedAt: fetchedAt, observedHour: nil, submissionCount: 1, winningCount: 1, - confidence: "single_source" + confidence: "single_source", + requestedResultLimit: limit, + estimatedDifficultyPayload: estimatedDifficultyPayloadBuilder( + request, + page, + limit, + fetchedAt + ) )) } catch { return .failure(OpenASOError.map(error)) @@ -251,6 +316,7 @@ final class RankingRefreshCoordinator: Sendable { limit: Int = SearchRankingCrawl.fullKeywordRankingLimit ) -> @Sendable (RankingRefreshRequest) async -> Result { let rankingProvider = rankingProvider + let estimatedDifficultyPayloadBuilder = estimatedDifficultyPayloadBuilder return { request in do { let page = try await rankingProvider.search( @@ -259,14 +325,22 @@ final class RankingRefreshCoordinator: Sendable { platform: request.platform, limit: limit ) + let fetchedAt = Date() return .success(RankingRefreshPageResult( request: request, page: page, - searchedAt: .now, + searchedAt: fetchedAt, observedHour: nil, submissionCount: 1, winningCount: 1, - confidence: "single_source" + confidence: "single_source", + requestedResultLimit: limit, + estimatedDifficultyPayload: estimatedDifficultyPayloadBuilder( + request, + page, + limit, + fetchedAt + ) )) } catch { return .failure(OpenASOError.map(error)) @@ -285,26 +359,68 @@ final class RankingRefreshCoordinator: Sendable { guard let track = try fetchTrackedAppKeyword(identityKey: pageResult.request.identityKey, in: modelContext) else { throw OpenASOError.appNotFound } + guard pageResult.request.matchesGeneration(of: track) else { + throw OpenASOError.providerUnavailable( + "The keyword changed while its ranking refresh was in flight. Refresh it again." + ) + } - return try persistRankingPage( - pageResult.page, - searchedAt: pageResult.searchedAt, - observedHour: pageResult.observedHour, - submissionCount: pageResult.submissionCount, - winningCount: pageResult.winningCount, - confidence: pageResult.confidence, - track: track, - trackedApp: track.trackedApp, - in: modelContext, - rebuildDerivedStats: rebuildDerivedStats, - saveChanges: saveChanges, - scheduleMetadataEnrichment: scheduleMetadataEnrichment - ) + let snapshot: TrackedKeywordDailyRanking + do { + snapshot = try persistRankingPage( + pageResult.page, + searchedAt: pageResult.searchedAt, + observedHour: pageResult.observedHour, + submissionCount: pageResult.submissionCount, + winningCount: pageResult.winningCount, + confidence: pageResult.confidence, + track: track, + trackedApp: track.trackedApp, + in: modelContext, + rebuildDerivedStats: rebuildDerivedStats, + saveChanges: false, + scheduleMetadataEnrichment: false + ) + + if saveChanges { + try modelContext.save() + } + } catch { + if saveChanges { + modelContext.rollback() + } + throw error + } + + // Ranking observations are authoritative; the local difficulty metric + // is optional derived data. Commit the ranking first, then isolate a + // rejected estimate so it cannot turn a valid provider response into + // a ranking failure. Batched callers pass `saveChanges: false` and + // persist their estimates in a separate background-store transaction. + if saveChanges, let payload = pageResult.estimatedDifficultyPayload, + payload.queryKey == track.queryKey + { + do { + _ = try EstimatedKeywordDifficultyStore.upsert(payload, in: modelContext) + try modelContext.save() + } catch { + modelContext.rollback() + OpenASOLog.refresh.error( + "Skipped optional estimated difficulty persistence: \(String(reflecting: error), privacy: .private(mask: .hash))" + ) + } + } + + if scheduleMetadataEnrichment { + scheduleTopRankingMetadataEnrichment(for: pageResult) + } + return snapshot } @discardableResult func recordRefreshFailure( identityKey: String, + trackCreatedAt: Date? = nil, error: OpenASOError, in modelContext: ModelContext, saveChanges: Bool = true @@ -312,6 +428,9 @@ final class RankingRefreshCoordinator: Sendable { guard let track = try fetchTrackedAppKeyword(identityKey: identityKey, in: modelContext) else { return nil } + guard trackCreatedAt.map({ $0 == track.createdAt }) ?? true else { + return nil + } try TrackedKeywordRefreshStatusStore.set( "Ranking failed to refresh. \(error.localizedDescription)", @@ -429,9 +548,12 @@ final class RankingRefreshCoordinator: Sendable { ) for item in page.items { - _ = try appCatalogService.upsertStoreApp( + let storeApp = try appCatalogService.upsertStoreApp( from: item, storefrontCode: track.storefront, + rankingSource: page.source, + fetchedAt: searchedAt, + requestedPlatform: track.platform, in: modelContext, cache: &catalogCache ) @@ -439,6 +561,8 @@ final class RankingRefreshCoordinator: Sendable { from: item, storefront: track.storefront, observedAt: snapshot.searchedAt, + source: storefrontRatingSource(for: page.source), + storeApp: storeApp, in: modelContext, cache: &ratingCache ) @@ -747,6 +871,8 @@ final class RankingRefreshCoordinator: Sendable { from item: SearchRankingItem, storefront: String, observedAt: Date, + source: AppStorefrontSource, + storeApp: StoreApp, in modelContext: ModelContext, cache: inout RatingPageCache ) { @@ -771,7 +897,8 @@ final class RankingRefreshCoordinator: Sendable { submissionCount: 1, winningCount: 1, confidence: "single_source", - source: .iTunesSearch + source: source, + storeApp: storeApp ) if snapshot.modelContext != nil, observedAt < snapshot.observedAt { return @@ -780,6 +907,9 @@ final class RankingRefreshCoordinator: Sendable { modelContext.insert(snapshot) cache.snapshotsByIdentityKey[snapshotKey] = snapshot } + if snapshot.storeApp !== storeApp { + snapshot.storeApp = storeApp + } let snapshotChanged = snapshot.modelContext == nil || snapshot.ratingCount != item.ratingCount || snapshot.averageRating != item.averageRating @@ -787,7 +917,8 @@ final class RankingRefreshCoordinator: Sendable { || snapshot.submissionCount != 1 || snapshot.winningCount != 1 || snapshot.confidenceRaw != "single_source" - || snapshot.source != .iTunesSearch + || snapshot.observedAt != observedAt + || snapshot.source != source if snapshotChanged { snapshot.ratingCount = item.ratingCount snapshot.averageRating = item.averageRating @@ -796,7 +927,7 @@ final class RankingRefreshCoordinator: Sendable { snapshot.submissionCount = 1 snapshot.winningCount = 1 snapshot.confidenceRaw = "single_source" - snapshot.source = .iTunesSearch + snapshot.source = source } let latestKey = LatestAppRating.makeIdentityKey( @@ -813,7 +944,8 @@ final class RankingRefreshCoordinator: Sendable { submissionCount: 1, winningCount: 1, confidence: "single_source", - source: .iTunesSearch + source: source, + storeApp: storeApp ) if latest.modelContext != nil, observedAt < latest.observedAt { return @@ -822,6 +954,9 @@ final class RankingRefreshCoordinator: Sendable { modelContext.insert(latest) cache.latestByIdentityKey[latestKey] = latest } + if latest.storeApp !== storeApp { + latest.storeApp = storeApp + } let latestChanged = latest.modelContext == nil || latest.ratingCount != item.ratingCount || latest.averageRating != item.averageRating @@ -829,7 +964,8 @@ final class RankingRefreshCoordinator: Sendable { || latest.submissionCount != 1 || latest.winningCount != 1 || latest.confidenceRaw != "single_source" - || latest.source != .iTunesSearch + || latest.observedAt != observedAt + || latest.source != source if latestChanged { latest.ratingCount = item.ratingCount latest.averageRating = item.averageRating @@ -838,7 +974,16 @@ final class RankingRefreshCoordinator: Sendable { latest.submissionCount = 1 latest.winningCount = 1 latest.confidenceRaw = "single_source" - latest.source = .iTunesSearch + latest.source = source + } + } + + private func storefrontRatingSource(for rankingSource: RankingSource) -> AppStorefrontSource { + switch rankingSource { + case .appStoreWeb: + return .appStorePage + case .iTunesFallback: + return .iTunesSearch } } @@ -951,14 +1096,14 @@ final class RankingRefreshCoordinator: Sendable { } catch { let mappedError = OpenASOError.map(error) do { - try TrackedKeywordRefreshStatusStore.set( - "Ranking failed to refresh. \(mappedError.localizedDescription)", - domain: .ranking, - for: track, + _ = try recordRefreshFailure( + identityKey: targetPageResult.request.identityKey, + trackCreatedAt: targetPageResult.request.trackCreatedAt, + error: mappedError, in: modelContext ) - try modelContext.save() } catch { + modelContext.rollback() OpenASOLog.refresh.error( "Failed to persist ranking refresh status: \(String(reflecting: error), privacy: .private(mask: .hash))" ) @@ -985,14 +1130,14 @@ final class RankingRefreshCoordinator: Sendable { } do { - try TrackedKeywordRefreshStatusStore.set( - "Ranking failed to refresh. \(error.localizedDescription)", - domain: .ranking, - for: track, + _ = try recordRefreshFailure( + identityKey: targetRequest.identityKey, + trackCreatedAt: targetRequest.trackCreatedAt, + error: error, in: modelContext ) - try modelContext.save() } catch { + modelContext.rollback() OpenASOLog.refresh.error( "Failed to persist ranking refresh status: \(String(reflecting: error), privacy: .private(mask: .hash))" ) diff --git a/OpenASO/Services/SearchRanking/SearchModels.swift b/OpenASO/Services/SearchRanking/SearchModels.swift index 5ef0ba5..85a9cfe 100644 --- a/OpenASO/Services/SearchRanking/SearchModels.swift +++ b/OpenASO/Services/SearchRanking/SearchModels.swift @@ -1,5 +1,9 @@ import Foundation +enum SearchRankingCrawl { + static let fullKeywordRankingLimit = 200 +} + struct ResolvedApp: Identifiable, Hashable, Sendable { let appStoreID: Int64 let bundleID: String? @@ -138,8 +142,135 @@ struct SearchRankingItem: Identifiable, Hashable, Sendable { struct SearchRankingPage: Sendable { let items: [SearchRankingItem] let source: RankingSource + let fallbackContext: SearchRankingFailureContext? + + init( + items: [SearchRankingItem], + source: RankingSource, + fallbackContext: SearchRankingFailureContext? = nil + ) { + self.items = items + self.source = source + self.fallbackContext = fallbackContext + } var resultCount: Int { items.count } } + +enum SearchRankingResponseFailure: String, Equatable, Sendable { + case serializedServerDataMissing + case decodingFailed + case nonHTTPResponse + case requestIntentMissing + case requestIntentAmbiguous + case pageShapeChanged + case authoritativeShelfMissing + case authoritativeShelfAmbiguous + case malformedSearchResult + case truncatedResults +} + +enum SearchRankingFailureCategory: Equatable, Sendable { + case transport(code: Int?) + case httpStatus(Int) + case response(SearchRankingResponseFailure) + case provider + case other +} + +struct SearchRankingFailureContext: Equatable, Sendable { + enum Provider: String, Equatable, Sendable { + case appStoreWeb + case iTunesFallback + } + + let provider: Provider + let category: SearchRankingFailureCategory +} + +enum SearchRankingProviderError: LocalizedError, Sendable { + case invalidStorefront + case invalidLimit + case unsupportedPlatform(AppPlatform) + case httpStatus(Int) + case responseFailure(SearchRankingResponseFailure) + case fallbackUnavailable(platform: AppPlatform, primary: SearchRankingFailureContext) + case allProvidersFailed( + primary: SearchRankingFailureContext, + fallback: SearchRankingFailureContext + ) + + var errorDescription: String? { + switch self { + case .invalidStorefront: + return "Enter a valid two-letter App Store storefront code." + case .invalidLimit: + return "The ranking result limit must be between 1 and \(SearchRankingCrawl.fullKeywordRankingLimit)." + case .unsupportedPlatform(let platform): + return "This ranking provider does not support \(platform.displayName)." + case .httpStatus(let statusCode): + return "The ranking provider returned HTTP \(statusCode)." + case .responseFailure: + return "The ranking provider response format changed and could not be decoded safely." + case .fallbackUnavailable(let platform, let primary): + return "The primary ranking provider failed (\(Self.summary(primary.category))), and no fallback supports \(platform.displayName)." + case .allProvidersFailed(let primary, let fallback): + return "The primary ranking provider failed (\(Self.summary(primary.category))), and the fallback also failed (\(Self.summary(fallback.category)))." + } + } + + private static func summary(_ category: SearchRankingFailureCategory) -> String { + switch category { + case .transport(let code): + return code.map { "transport \($0)" } ?? "transport" + case .httpStatus(let statusCode): + return "HTTP \(statusCode)" + case .response(let failure): + return "response \(failure.rawValue)" + case .provider: + return "provider" + case .other: + return "other" + } + } +} + +struct ValidatedSearchRankingRequest: Sendable { + let keyword: String + let storefrontCode: String + let platform: AppPlatform + let limit: Int +} + +enum SearchRankingRequestValidator { + static func validate( + keyword: String, + storefrontCode: String, + platform: AppPlatform, + limit: Int + ) throws -> ValidatedSearchRankingRequest { + let normalizedKeyword = keyword.trimmingCharacters(in: .whitespacesAndNewlines) + guard !normalizedKeyword.isEmpty else { + throw OpenASOError.emptyQuery + } + + let normalizedStorefront = StorefrontCatalog.normalizedStorefrontCode(storefrontCode) + guard normalizedStorefront.utf8.count == 2, + normalizedStorefront.utf8.allSatisfy({ (97 ... 122).contains($0) }) else { + throw SearchRankingProviderError.invalidStorefront + } + + guard (1 ... SearchRankingCrawl.fullKeywordRankingLimit).contains(limit) else { + throw SearchRankingProviderError.invalidLimit + } + + return ValidatedSearchRankingRequest( + keyword: normalizedKeyword, + storefrontCode: normalizedStorefront, + platform: platform, + limit: limit + ) + } +} diff --git a/OpenASO/Services/SearchRanking/SearchRankingProvider.swift b/OpenASO/Services/SearchRanking/SearchRankingProvider.swift index 07f7eae..5e13c60 100644 --- a/OpenASO/Services/SearchRanking/SearchRankingProvider.swift +++ b/OpenASO/Services/SearchRanking/SearchRankingProvider.swift @@ -4,6 +4,11 @@ protocol SearchRankingProvider: Sendable { func search(keyword: String, storefrontCode: String, platform: AppPlatform, limit: Int) async throws -> SearchRankingPage } -enum SearchRankingCrawl { - static let fullKeywordRankingLimit = 200 +enum SearchRankingProviderFactory { + static func makeProduction(httpClient: any HTTPClient) -> any SearchRankingProvider { + PrimaryFallbackSearchRankingProvider( + primary: AppStoreWebRankingProvider(httpClient: httpClient), + fallback: ITunesSearchFallbackProvider(httpClient: httpClient) + ) + } } diff --git a/OpenASO/Services/Storefront/AppCatalogService.swift b/OpenASO/Services/Storefront/AppCatalogService.swift index 489b76a..6f25151 100644 --- a/OpenASO/Services/Storefront/AppCatalogService.swift +++ b/OpenASO/Services/Storefront/AppCatalogService.swift @@ -85,6 +85,9 @@ final class AppCatalogService: Sendable { func upsertStoreApp( from item: SearchRankingItem, storefrontCode: String? = nil, + rankingSource: RankingSource, + fetchedAt: Date, + requestedPlatform: AppPlatform, in modelContext: ModelContext ) throws -> StoreApp { var cache = try makeSearchRankingPageCache( @@ -95,6 +98,9 @@ final class AppCatalogService: Sendable { return try upsertStoreApp( from: item, storefrontCode: storefrontCode, + rankingSource: rankingSource, + fetchedAt: fetchedAt, + requestedPlatform: requestedPlatform, in: modelContext, cache: &cache ) @@ -124,14 +130,20 @@ final class AppCatalogService: Sendable { func upsertStoreApp( from item: SearchRankingItem, storefrontCode: String? = nil, + rankingSource: RankingSource, + fetchedAt: Date, + requestedPlatform: AppPlatform, in modelContext: ModelContext, cache: inout SearchRankingPageCache ) throws -> StoreApp { let normalizedStorefront = normalizedStorefrontCode(storefrontCode) + let metadataSource = metadataSource(for: rankingSource) let storeApp: StoreApp + let isExisting: Bool if let existing = try cache.storeAppsByID[item.appStoreID] ?? fetchStoreApp(appStoreID: item.appStoreID, in: modelContext) { storeApp = existing cache.storeAppsByID[item.appStoreID] = existing + isExisting = true } else { storeApp = StoreApp( appStoreID: item.appStoreID, @@ -142,39 +154,48 @@ final class AppCatalogService: Sendable { iconURLString: item.iconURLString, defaultStorefront: normalizedStorefront, supportedLanguageCodes: normalizedLanguageCodes(item.supportedLanguageCodes), - supportedLanguageCodesSource: .iTunesSearch, - supportedLanguageCodesFetchedAt: .now, + supportedLanguageCodesSource: metadataSource, + supportedLanguageCodesFetchedAt: item.supportedLanguageCodes.isEmpty ? nil : fetchedAt, releaseDate: item.releaseDate, currentVersionReleaseDate: item.currentVersionReleaseDate, version: item.version, primaryGenreID: item.primaryGenreID, primaryGenreName: item.primaryGenreName, - defaultPlatform: item.platform + defaultPlatform: requestedPlatform, + lastMetadataRefreshAt: fetchedAt ) modelContext.insert(storeApp) cache.storeAppsByID[item.appStoreID] = storeApp + isExisting = false } - update( - storeApp, - storefront: normalizedStorefront, - bundleID: item.bundleID, - name: item.name, - subtitle: item.subtitle, - sellerName: item.sellerName, - iconURLString: item.iconURLString, - supportedLanguageCodes: item.supportedLanguageCodes, - supportedLanguageCodesSource: .iTunesSearch, - releaseDate: item.releaseDate, - currentVersionReleaseDate: item.currentVersionReleaseDate, - version: item.version, - primaryGenreID: item.primaryGenreID, - primaryGenreName: item.primaryGenreName, - defaultPlatform: item.platform - ) + if !isExisting || fetchedAt >= storeApp.lastMetadataRefreshAt { + update( + storeApp, + storefront: normalizedStorefront, + bundleID: item.bundleID, + name: item.name, + subtitle: item.subtitle, + sellerName: item.sellerName, + iconURLString: item.iconURLString, + supportedLanguageCodes: item.supportedLanguageCodes, + supportedLanguageCodesSource: metadataSource, + supportedLanguageCodesFetchedAt: fetchedAt, + releaseDate: item.releaseDate, + currentVersionReleaseDate: item.currentVersionReleaseDate, + version: item.version, + primaryGenreID: item.primaryGenreID, + primaryGenreName: item.primaryGenreName, + defaultPlatform: isExisting ? nil : requestedPlatform, + fetchedAt: fetchedAt + ) + } try upsertStorefrontMetadata( from: item, storefront: normalizedStorefront, + source: metadataSource, + fetchedAt: fetchedAt, + requestedPlatform: requestedPlatform, storeApp: storeApp, in: modelContext, cache: &cache @@ -189,9 +210,21 @@ final class AppCatalogService: Sendable { } } - func upsertStoreApps(from items: [SearchRankingItem], in modelContext: ModelContext) throws { + func upsertStoreApps( + from items: [SearchRankingItem], + rankingSource: RankingSource, + fetchedAt: Date, + requestedPlatform: AppPlatform, + in modelContext: ModelContext + ) throws { for item in items { - _ = try upsertStoreApp(from: item, in: modelContext) + _ = try upsertStoreApp( + from: item, + rankingSource: rankingSource, + fetchedAt: fetchedAt, + requestedPlatform: requestedPlatform, + in: modelContext + ) } } @@ -312,7 +345,14 @@ final class AppCatalogService: Sendable { } let metadataIsStale = metadata.lastFetchedAt < freshnessCutoff - let hasFreshWebEnrichment = metadata.source == .appStoreWeb && metadata.lastFetchedAt >= freshnessCutoff + let isSearchOnlyMetadata = metadata.source == .appStoreWebSearch + let hasFreshWebEnrichment = metadata.source == .appStoreWeb + && metadata.lastFetchedAt >= freshnessCutoff + // A successful App Store page fetch is also the bounded attempt marker + // for fields that the page does not expose, such as language codes. Keep + // those fields visibly absent, but do not refetch both providers on every + // ranking refresh until the detail-page freshness window expires. + let languageDataNeedsEnrichment = languageDataIsStale && !hasFreshWebEnrichment let desiredPlatform = platform.rawValue let hasDesiredPlatformScreenshots = metadata.screenshots.contains { $0.platformRaw == desiredPlatform @@ -321,7 +361,10 @@ final class AppCatalogService: Sendable { let isMissingMetadata = nonEmpty(metadata.name) == nil || nonEmpty(metadata.subtitle) == nil let metadataHasGaps = isMissingMetadata || !hasAnyScreenshots || !hasDesiredPlatformScreenshots - return languageDataIsStale || metadataIsStale || (metadataHasGaps && !hasFreshWebEnrichment) + return languageDataNeedsEnrichment + || metadataIsStale + || isSearchOnlyMetadata + || (metadataHasGaps && !hasFreshWebEnrichment) } private func fetchStoreApp(appStoreID: Int64, in modelContext: ModelContext) throws -> StoreApp? { @@ -367,12 +410,14 @@ final class AppCatalogService: Sendable { iconURLString: String?, supportedLanguageCodes: [String] = [], supportedLanguageCodesSource: AppStorefrontMetadataSource? = nil, + supportedLanguageCodesFetchedAt: Date? = nil, releaseDate: Date?, currentVersionReleaseDate: Date?, version: String?, primaryGenreID: Int?, primaryGenreName: String?, - defaultPlatform: AppPlatform + defaultPlatform: AppPlatform?, + fetchedAt: Date = .now ) { var changed = false if let bundleID, !bundleID.isEmpty { @@ -400,10 +445,22 @@ final class AppCatalogService: Sendable { let normalizedLanguages = normalizedLanguageCodes(supportedLanguageCodes) if !normalizedLanguages.isEmpty { - let languagesChanged = assignIfChanged(storeApp, \.supportedLanguageCodes, normalizedLanguages) - || assignIfChanged(storeApp, \.supportedLanguageCodesSourceRaw, supportedLanguageCodesSource?.rawValue) - if languagesChanged { - storeApp.supportedLanguageCodesFetchedAt = .now + let languagesChanged = assignIfChanged( + storeApp, + \.supportedLanguageCodes, + normalizedLanguages + ) + let languageSourceChanged = assignIfChanged( + storeApp, + \.supportedLanguageCodesSourceRaw, + supportedLanguageCodesSource?.rawValue + ) + let languageEvidenceDate = supportedLanguageCodesFetchedAt ?? fetchedAt + if languagesChanged + || languageSourceChanged + || storeApp.supportedLanguageCodesFetchedAt != languageEvidenceDate + { + storeApp.supportedLanguageCodesFetchedAt = languageEvidenceDate changed = true } } @@ -428,39 +485,20 @@ final class AppCatalogService: Sendable { changed = assignIfChanged(storeApp, \.primaryGenreName, primaryGenreName) || changed } - changed = assignIfChanged(storeApp, \.defaultPlatformRaw, defaultPlatform.rawValue) || changed - if changed { - storeApp.lastMetadataRefreshAt = .now + if let defaultPlatform { + changed = assignIfChanged(storeApp, \.defaultPlatformRaw, defaultPlatform.rawValue) || changed + } + if changed || storeApp.lastMetadataRefreshAt != fetchedAt { + storeApp.lastMetadataRefreshAt = fetchedAt } } private func upsertStorefrontMetadata( from item: SearchRankingItem, storefront: String, - storeApp: StoreApp, - in modelContext: ModelContext - ) throws { - let identityKey = AppStorefrontMetadata.makeIdentityKey( - appStoreID: item.appStoreID, - storefront: storefront - ) - var cache = SearchRankingPageCache( - storeAppsByID: [storeApp.appStoreID: storeApp], - storefrontMetadataByIdentityKey: try fetchStorefrontMetadata(identityKey: identityKey, in: modelContext) - .map { [identityKey: $0] } ?? [:] - ) - try upsertStorefrontMetadata( - from: item, - storefront: storefront, - storeApp: storeApp, - in: modelContext, - cache: &cache - ) - } - - private func upsertStorefrontMetadata( - from item: SearchRankingItem, - storefront: String, + source: AppStorefrontMetadataSource, + fetchedAt: Date, + requestedPlatform: AppPlatform, storeApp: StoreApp, in modelContext: ModelContext, cache: inout SearchRankingPageCache @@ -470,15 +508,17 @@ final class AppCatalogService: Sendable { storefront: storefront ) let metadata: AppStorefrontMetadata + let isNewMetadata: Bool if let existing = try cache.storefrontMetadataByIdentityKey[identityKey] ?? fetchStorefrontMetadata(identityKey: identityKey, in: modelContext) { metadata = existing cache.storefrontMetadataByIdentityKey[identityKey] = existing + isNewMetadata = false } else { metadata = AppStorefrontMetadata( appStoreID: item.appStoreID, storefront: storefront, - defaultPlatform: item.platform, + defaultPlatform: requestedPlatform, name: item.name, subtitle: item.subtitle, sellerName: item.sellerName, @@ -491,51 +531,66 @@ final class AppCatalogService: Sendable { primaryGenreID: item.primaryGenreID, primaryGenreName: item.primaryGenreName, isAvailable: true, - source: .iTunesSearch, + source: source, + lastFetchedAt: fetchedAt, storeApp: storeApp ) modelContext.insert(metadata) storeApp.storefrontMetadata.append(metadata) cache.storefrontMetadataByIdentityKey[identityKey] = metadata + isNewMetadata = true } + guard fetchedAt >= metadata.lastFetchedAt else { return } + + // App Store Web search rows are intentionally sparse. Once a detail + // record exists, keep its aggregate fields and freshness independent of + // ranking evidence. Search-only records can still evolve normally. + let preservesExistingDetailMetadata = !isNewMetadata + && source == .appStoreWebSearch + && metadata.source != .appStoreWebSearch + var changed = false - changed = assignIfChanged(metadata, \.appStoreID, item.appStoreID) || changed - changed = assignIfChanged(metadata, \.storefront, storefront) || changed - changed = assignIfChanged(metadata, \.defaultPlatformRaw, item.platform.rawValue) || changed - changed = assignIfChanged(metadata, \.name, item.name) || changed - if let subtitle = nonEmpty(item.subtitle) { - changed = assignIfChanged(metadata, \.subtitle, subtitle) || changed - } - if let sellerName = nonEmpty(item.sellerName) { - changed = assignIfChanged(metadata, \.sellerName, sellerName) || changed - } - if let descriptionText = nonEmpty(item.descriptionText) { - changed = assignIfChanged(metadata, \.descriptionText, descriptionText) || changed - } - if let releaseNotes = nonEmpty(item.releaseNotes) { - changed = assignIfChanged(metadata, \.releaseNotes, releaseNotes) || changed - } - if let iconURLString = nonEmpty(item.iconURLString) { - changed = assignIfChanged(metadata, \.iconURLString, iconURLString) || changed - } - if let version = nonEmpty(item.version) { - changed = assignIfChanged(metadata, \.version, version) || changed - } - if let releaseDate = item.releaseDate { - changed = assignIfChanged(metadata, \.releaseDate, releaseDate) || changed - } - if let currentVersionReleaseDate = item.currentVersionReleaseDate { - changed = assignIfChanged(metadata, \.currentVersionReleaseDate, currentVersionReleaseDate) || changed - } - if let primaryGenreID = item.primaryGenreID { - changed = assignIfChanged(metadata, \.primaryGenreID, primaryGenreID) || changed + if !preservesExistingDetailMetadata { + changed = assignIfChanged(metadata, \.appStoreID, item.appStoreID) || changed + changed = assignIfChanged(metadata, \.storefront, storefront) || changed + changed = assignIfChanged(metadata, \.name, item.name) || changed + if let subtitle = nonEmpty(item.subtitle) { + changed = assignIfChanged(metadata, \.subtitle, subtitle) || changed + } + if let sellerName = nonEmpty(item.sellerName) { + changed = assignIfChanged(metadata, \.sellerName, sellerName) || changed + } + if let descriptionText = nonEmpty(item.descriptionText) { + changed = assignIfChanged(metadata, \.descriptionText, descriptionText) || changed + } + if let releaseNotes = nonEmpty(item.releaseNotes) { + changed = assignIfChanged(metadata, \.releaseNotes, releaseNotes) || changed + } + if let iconURLString = nonEmpty(item.iconURLString) { + changed = assignIfChanged(metadata, \.iconURLString, iconURLString) || changed + } + if let version = nonEmpty(item.version) { + changed = assignIfChanged(metadata, \.version, version) || changed + } + if let releaseDate = item.releaseDate { + changed = assignIfChanged(metadata, \.releaseDate, releaseDate) || changed + } + if let currentVersionReleaseDate = item.currentVersionReleaseDate { + changed = assignIfChanged(metadata, \.currentVersionReleaseDate, currentVersionReleaseDate) || changed + } + if let primaryGenreID = item.primaryGenreID { + changed = assignIfChanged(metadata, \.primaryGenreID, primaryGenreID) || changed + } + if let primaryGenreName = nonEmpty(item.primaryGenreName) { + changed = assignIfChanged(metadata, \.primaryGenreName, primaryGenreName) || changed + } + changed = assignIfChanged(metadata, \.isAvailable, true) || changed } - if let primaryGenreName = nonEmpty(item.primaryGenreName) { - changed = assignIfChanged(metadata, \.primaryGenreName, primaryGenreName) || changed + let updatesAggregateFreshness = !preservesExistingDetailMetadata + if updatesAggregateFreshness { + changed = assignIfChanged(metadata, \.sourceRaw, source.rawValue) || changed } - changed = assignIfChanged(metadata, \.isAvailable, true) || changed - changed = assignIfChanged(metadata, \.sourceRaw, AppStorefrontMetadataSource.iTunesSearch.rawValue) || changed if metadata.storeApp !== storeApp { metadata.storeApp = storeApp changed = true @@ -545,10 +600,14 @@ final class AppCatalogService: Sendable { for: metadata, item: item, storefront: storefront, + source: source, + fetchedAt: fetchedAt, in: modelContext ) - if changed || screenshotsChanged { - metadata.lastFetchedAt = .now + if updatesAggregateFreshness + && (changed || screenshotsChanged || metadata.lastFetchedAt != fetchedAt) + { + metadata.lastFetchedAt = fetchedAt } } @@ -665,6 +724,8 @@ final class AppCatalogService: Sendable { for metadata: AppStorefrontMetadata, item: SearchRankingItem, storefront: String, + source: AppStorefrontMetadataSource, + fetchedAt: Date, in modelContext: ModelContext ) -> Bool { replaceScreenshots( @@ -676,6 +737,9 @@ final class AppCatalogService: Sendable { ("ipad", item.ipadScreenshotURLs), ("tv", item.appletvScreenshotURLs) ], + source: source, + fetchedAt: fetchedAt, + preservesUnspecifiedPlatforms: true, in: modelContext ) } @@ -735,6 +799,9 @@ final class AppCatalogService: Sendable { appStoreID: Int64, storefront: String, groups: [(platform: String, urls: [String])], + source: AppStorefrontMetadataSource? = nil, + fetchedAt: Date = .now, + preservesUnspecifiedPlatforms: Bool = false, in modelContext: ModelContext ) -> Bool { let replacements = groups.flatMap { group in @@ -755,6 +822,9 @@ final class AppCatalogService: Sendable { appStoreID: appStoreID, storefront: storefront, replacements: replacements, + source: source, + fetchedAt: fetchedAt, + preservesUnspecifiedPlatforms: preservesUnspecifiedPlatforms, in: modelContext ) } @@ -765,20 +835,65 @@ final class AppCatalogService: Sendable { appStoreID: Int64, storefront: String, replacements: [ScreenshotReplacementItem], + source: AppStorefrontMetadataSource? = nil, + fetchedAt: Date = .now, + preservesUnspecifiedPlatforms: Bool = false, in modelContext: ModelContext ) -> Bool { - let replacements = replacements.sortedForComparison + var replacements = replacements.sortedForComparison guard !replacements.isEmpty else { return false } + let screenshotSource = source ?? metadata.source let existing = metadata.screenshots - if existing.screenshotReplacementItems == replacements { - return false + if preservesUnspecifiedPlatforms { + let incomingByPlatform = Dictionary(grouping: replacements, by: \.platformRaw) + let acceptedPlatforms = Set(incomingByPlatform.compactMap { platform, incoming -> String? in + let existingForPlatform = existing.filter { $0.platformRaw == platform } + if let newestExistingDate = existingForPlatform.map(\.lastFetchedAt).max(), + fetchedAt < newestExistingDate { + return nil + } + + let incomingIsSparse = incoming.allSatisfy { + $0.displayTypeRaw == "default" && $0.width == nil && $0.height == nil + } + let existingIsRicher = existingForPlatform.contains { + $0.displayTypeRaw != "default" || $0.width != nil || $0.height != nil + } + if screenshotSource == .appStoreWebSearch && incomingIsSparse && existingIsRicher { + return nil + } + return platform + }) + replacements.removeAll { !acceptedPlatforms.contains($0.platformRaw) } + guard !replacements.isEmpty else { return false } + } + + let replacementPlatforms = Set(replacements.map(\.platformRaw)) + let existingToReplace = preservesUnspecifiedPlatforms + ? existing.filter { replacementPlatforms.contains($0.platformRaw) } + : existing + if existingToReplace.screenshotReplacementItems == replacements { + var changed = false + for screenshot in existingToReplace { + if screenshot.source != screenshotSource { + screenshot.source = screenshotSource + changed = true + } + if screenshot.lastFetchedAt != fetchedAt { + screenshot.lastFetchedAt = fetchedAt + changed = true + } + } + return changed } - for screenshot in existing { + for screenshot in existingToReplace { modelContext.delete(screenshot) } - metadata.screenshots.removeAll() + metadata.screenshots.removeAll { screenshot in + existingToReplace.contains { $0 === screenshot } + } for replacement in replacements { let screenshot = AppStoreScreenshot( @@ -790,7 +905,8 @@ final class AppCatalogService: Sendable { urlString: replacement.urlString, width: replacement.width, height: replacement.height, - source: metadata.source, + source: screenshotSource, + lastFetchedAt: fetchedAt, metadata: metadata ) metadata.screenshots.append(screenshot) @@ -808,6 +924,15 @@ final class AppCatalogService: Sendable { return normalized.isEmpty ? "default" : normalized } + private func metadataSource(for rankingSource: RankingSource) -> AppStorefrontMetadataSource { + switch rankingSource { + case .appStoreWeb: + return .appStoreWebSearch + case .iTunesFallback: + return .iTunesSearch + } + } + private func fetchStorefrontMetadata(identityKey: String, in modelContext: ModelContext) throws -> AppStorefrontMetadata? { let targetIdentityKey = identityKey var descriptor = FetchDescriptor( diff --git a/OpenASO/Utilities/CSVFormats.swift b/OpenASO/Utilities/CSVFormats.swift index 7b8357e..e80915b 100644 --- a/OpenASO/Utilities/CSVFormats.swift +++ b/OpenASO/Utilities/CSVFormats.swift @@ -519,3 +519,229 @@ enum KeywordRankingHistoryCSVFormat { ] } } + +/// One tracked-keyword scope paired with its latest independently persisted +/// estimated-difficulty calculation. A missing snapshot is intentionally kept +/// so exports can distinguish "not calculated" from a stored unavailable result. +struct EstimatedKeywordDifficultyCSVItem: Equatable, Sendable { + let appName: String + let appStoreID: Int64 + let keyword: String + let queryKey: String + let storefront: String + let platformRaw: String + let snapshot: EstimatedKeywordDifficultySnapshot? +} + +/// Dedicated export-only format for estimated-difficulty provenance. This is +/// deliberately separate from `TrackedKeywordCSVFormat`; it has no decoder and +/// cannot import heuristic values into the legacy `Difficulty` field. +enum EstimatedKeywordDifficultyCSVFormat { + static let evidenceExportType = "estimated_keyword_difficulty_evidence" + static let summaryExportType = "estimated_keyword_difficulty_summary" + + static let headers = [ + "OpenASO Export Type", + "App Name", + "App Id", + "Keyword", + "Query Key", + "Storefront", + "Platform", + "State", + "Estimated Difficulty", + "Confidence Score", + "Confidence", + "Unavailable Reason", + "Estimation Source", + "Algorithm Identifier", + "Algorithm Version", + "Requested Result Limit", + "Provider Result Count", + "Considered Result Count", + "Rated Result Count", + "Weighted Rating Coverage Percentage", + "Maximum Rating Count", + "Median Rating Count", + "Rating Authority Score", + "Metadata Saturation Score", + "Exact Title Phrase Match Count", + "Exact Subtitle Phrase Match Count", + "Ranking Source", + "Ranking Fetched At", + "Computed At", + "Stale At", + "Is Stale", + "Fallback Provider", + "Fallback Category", + "Fallback Transport Code", + "Fallback HTTP Status", + "Fallback Response Failure", + "Notes JSON", + "Evidence Position", + "Evidence App Store Id", + "Evidence Title", + "Evidence Subtitle", + "Evidence Rating Count", + "Evidence Rating Authority Score", + "Evidence Title Token Coverage Percentage", + "Evidence Combined Token Coverage Percentage", + "Evidence Metadata Match Score", + "Evidence Exact Title Phrase Match", + "Evidence Exact Subtitle Phrase Match" + ] + + static func encode(items: [EstimatedKeywordDifficultyCSVItem], asOf date: Date) -> String { + let rows = items + .sorted(by: itemPrecedes) + .flatMap { item in + fields(for: item, asOf: date) + } + return CSVTable.encode(headers: headers, rows: rows) + } + + private static func fields( + for item: EstimatedKeywordDifficultyCSVItem, + asOf date: Date + ) -> [[String]] { + guard let snapshot = item.snapshot else { + return [fields(for: item, snapshot: nil, evidence: nil, asOf: date)] + } + + let evidence = snapshot.resultEvidence.sorted(by: evidencePrecedes) + guard !evidence.isEmpty else { + return [fields(for: item, snapshot: snapshot, evidence: nil, asOf: date)] + } + + return evidence.map { + fields(for: item, snapshot: snapshot, evidence: $0, asOf: date) + } + } + + private static func fields( + for item: EstimatedKeywordDifficultyCSVItem, + snapshot: EstimatedKeywordDifficultySnapshot?, + evidence: EstimatedKeywordDifficultyResultEvidence?, + asOf date: Date + ) -> [String] { + let rankingFetchedAt = snapshot?.rankingFetchedAt + let isStale = rankingFetchedAt.map { + EstimatedKeywordDifficultyFreshness.isStale( + rankingFetchedAt: $0, + asOf: date + ) + } + let staleAt = rankingFetchedAt?.addingTimeInterval( + EstimatedKeywordDifficultyFreshness.maximumAge + ) + + return [ + evidence == nil ? summaryExportType : evidenceExportType, + spreadsheetSafeExternalText(item.appName), + String(item.appStoreID), + spreadsheetSafeExternalText(snapshot?.keyword ?? item.keyword), + spreadsheetSafeExternalText(snapshot?.queryKey ?? item.queryKey), + spreadsheetSafeExternalText(snapshot?.storefront ?? item.storefront), + snapshot?.platformRaw ?? item.platformRaw, + snapshot?.stateRaw ?? "missing", + string(snapshot?.score), + string(snapshot?.confidenceScore), + snapshot?.confidenceRaw ?? "", + snapshot?.unavailableReasonRaw ?? "", + snapshot?.estimationSourceRaw ?? "", + snapshot?.algorithmIdentifier ?? "", + string(snapshot?.algorithmVersion), + string(snapshot?.requestedResultLimit), + string(snapshot?.providerResultCount), + string(snapshot?.consideredResultCount), + string(snapshot?.ratedResultCount), + string(snapshot?.weightedRatingCoveragePercentage), + string(snapshot?.maximumRatingCount), + string(snapshot?.medianRatingCount), + string(snapshot?.ratingAuthorityScore), + string(snapshot?.metadataSaturationScore), + string(snapshot?.exactTitlePhraseMatchCount), + string(snapshot?.exactSubtitlePhraseMatchCount), + snapshot?.rankingSourceRaw ?? "", + CSVTable.string(from: rankingFetchedAt), + CSVTable.string(from: snapshot?.computedAt), + CSVTable.string(from: staleAt), + isStale.map { String($0) } ?? "", + snapshot?.fallbackProviderRaw ?? "", + snapshot?.fallbackCategoryRaw ?? "", + string(snapshot?.fallbackTransportCode), + string(snapshot?.fallbackHTTPStatus), + snapshot?.fallbackResponseFailureRaw ?? "", + notesJSON(snapshot?.notes ?? []), + string(evidence?.position), + string(evidence?.appStoreID), + spreadsheetSafeExternalText(evidence?.title ?? ""), + spreadsheetSafeExternalText(evidence?.subtitle ?? ""), + string(evidence?.ratingCount), + string(evidence?.ratingAuthorityScore), + string(evidence?.titleTokenCoveragePercentage), + string(evidence?.combinedTokenCoveragePercentage), + string(evidence?.metadataMatchScore), + evidence.map { String($0.exactTitlePhraseMatch) } ?? "", + evidence.map { String($0.exactSubtitlePhraseMatch) } ?? "" + ] + } + + private static func itemPrecedes( + _ left: EstimatedKeywordDifficultyCSVItem, + _ right: EstimatedKeywordDifficultyCSVItem + ) -> Bool { + let leftFields = [ + left.keyword.lowercased(), + left.storefront.lowercased(), + left.platformRaw, + left.queryKey + ] + let rightFields = [ + right.keyword.lowercased(), + right.storefront.lowercased(), + right.platformRaw, + right.queryKey + ] + return leftFields.lexicographicallyPrecedes(rightFields) + } + + private static func evidencePrecedes( + _ left: EstimatedKeywordDifficultyResultEvidence, + _ right: EstimatedKeywordDifficultyResultEvidence + ) -> Bool { + if left.position != right.position { + return left.position < right.position + } + if left.appStoreID != right.appStoreID { + return left.appStoreID < right.appStoreID + } + if left.title != right.title { + return left.title < right.title + } + return (left.subtitle ?? "") < (right.subtitle ?? "") + } + + private static func string(_ value: T?) -> String where T: LosslessStringConvertible { + value.map { String($0) } ?? "" + } + + private static func notesJSON(_ notes: [String]) -> String { + guard let data = try? JSONEncoder().encode(notes) else { + return "[]" + } + return String(decoding: data, as: UTF8.self) + } + + /// CSV quoting does not stop spreadsheet applications from interpreting + /// externally controlled cells as formulas. Prefix only public/user text + /// in this dedicated export so legacy import-compatible CSV stays intact. + private static func spreadsheetSafeExternalText(_ value: String) -> String { + guard let first = value.unicodeScalars.first, + ["=", "+", "-", "@", "\t", "\n", "\r"].contains(String(first)) + else { + return value + } + return "'\(value)" + } +} diff --git a/OpenASOTests/AppServicesDependencyTests.swift b/OpenASOTests/AppServicesDependencyTests.swift index cc08f78..ec52834 100644 --- a/OpenASOTests/AppServicesDependencyTests.swift +++ b/OpenASOTests/AppServicesDependencyTests.swift @@ -220,6 +220,77 @@ struct AppServicesDependencyTests { #expect(provider.resultCounts[.success] == 1) } + @Test + func appServicesUsesWebRankingFirstAndFallsBackWithObservedContext() async throws { + let defaults = Self.makeDefaults() + let observationClock = RefreshObservationClock(nowNanoseconds: { 1_000 }) + let recorder = RefreshMetricsRecorder(clock: observationClock) + var requestedURLs: [String] = [] + let transport = MockHTTPClient { request in + let url = try #require(request.url) + requestedURLs.append(url.absoluteString) + + switch url.host() { + case "apps.apple.com": + #expect(url.absoluteString == "https://apps.apple.com/us/iphone/search?term=notes") + return ( + Data(), + makeHTTPURLResponse(url: url, statusCode: 503) + ) + case "itunes.apple.com": + #expect(url.absoluteString == "https://itunes.apple.com/search?term=notes&entity=software&country=us&limit=10") + return ( + Data(#"{"results":[{"trackId":321,"bundleId":"com.example.fallback","trackName":"Fallback App","sellerName":"Example"}]}"#.utf8), + makeHTTPURLResponse(url: url, statusCode: 200) + ) + default: + throw OpenASOError.providerUnavailable("Unexpected ranking URL.") + } + } + let services = AppServices( + httpClient: transport, + defaults: defaults, + keychain: InMemoryKeychainService(), + loadsEnvironmentCredentials: false, + allowsIconNetworkFetches: false, + refreshObservationClock: observationClock, + refreshMetricsRecorder: recorder, + providerRequestGateMode: .disabled + ) + let runID = await recorder.begin( + trigger: .manual, + workspace: .keywords, + requestedTrackCount: 1, + requestedStorefrontCount: 1 + ) + + let page = try await RefreshObservationScope.$runID.withValue(runID) { + try await services.rankingProvider.search( + keyword: "notes", + storefrontCode: "US", + platform: .iphone, + limit: 10 + ) + } + let summary = try #require(await recorder.finish(runID: runID)) + let webProvider = try #require(summary.providers[.appStoreWeb]) + let fallbackProvider = try #require(summary.providers[.iTunesStore]) + + #expect(requestedURLs.count == 2) + #expect(page.items.map(\.appStoreID) == [321]) + #expect(page.source == .iTunesFallback) + #expect(page.fallbackContext == SearchRankingFailureContext( + provider: .appStoreWeb, + category: .httpStatus(503) + )) + #expect(webProvider.requestCount == 1) + #expect(webProvider.endpointCounts[.rankingSearch] == 1) + #expect(webProvider.resultCounts[.serverFailure] == 1) + #expect(fallbackProvider.requestCount == 1) + #expect(fallbackProvider.endpointCounts[.rankingSearch] == 1) + #expect(fallbackProvider.resultCounts[.success] == 1) + } + @Test func savedKeychainItemsLoadWhenPresenceFlagsExist() throws { let defaults = Self.makeDefaults() diff --git a/OpenASOTests/AppStoreWebRankingProviderTests.swift b/OpenASOTests/AppStoreWebRankingProviderTests.swift new file mode 100644 index 0000000..53a56d5 --- /dev/null +++ b/OpenASOTests/AppStoreWebRankingProviderTests.swift @@ -0,0 +1,496 @@ +import Foundation +import Testing +@testable import OpenASO + +@MainActor +struct AppStoreWebRankingProviderTests { + @Test + func formsExactRequestAndParsesOnlyTheOrderedSearchShelf() async throws { + let client = MockHTTPClient { request in + let url = try #require(request.url) + let components = try #require(URLComponents(url: url, resolvingAgainstBaseURL: false)) + #expect(components.scheme == "https") + #expect(components.host == "apps.apple.com") + #expect(components.path == "/gb/iphone/search") + #expect(components.queryItems == [URLQueryItem(name: "term", value: "budget planner")]) + #expect(request.value(forHTTPHeaderField: "User-Agent")?.contains("Safari") == true) + #expect(request.value(forHTTPHeaderField: "Accept")?.contains("text/html") == true) + #expect(request.timeoutInterval == 20) + + return ( + Self.orderedFixture, + makeHTTPURLResponse(url: url, statusCode: 200) + ) + } + + let provider = AppStoreWebRankingProvider(httpClient: client) + let page = try await provider.search( + keyword: " budget planner\n", + storefrontCode: " GB ", + platform: .iphone, + limit: 2 + ) + + #expect(page.source == .appStoreWeb) + #expect(page.fallbackContext == nil) + #expect(page.items.map(\.appStoreID) == [10, 20]) + #expect(page.items.map(\.position) == [1, 2]) + #expect(page.items[0].name == "First App") + #expect(page.items[0].bundleID == "com.example.first") + #expect(page.items[0].sellerName == "Example Ltd") + #expect(page.items[0].iconURLString == "https://example.com/icon/100x100bb.jpg") + #expect(page.items[0].screenshotURLs == ["https://example.com/phone/1200x2400bb.jpg"]) + #expect(page.items[0].ipadScreenshotURLs == ["https://example.com/pad/2048x2732bb.jpg"]) + #expect(page.items[0].ratingCount == 105_000) + #expect(page.items[0].averageRating == 4.8) + #expect(page.items[1].name == "Event App") + #expect(page.items[1].ratingCount == nil) + #expect(page.items.allSatisfy { $0.platform == .iphone }) + } + + @Test + func formsPlatformSpecificSearchPaths() async throws { + var requestedPaths: [String] = [] + var returnedPlatforms: [AppPlatform] = [] + let client = MockHTTPClient { request in + let url = try #require(request.url) + requestedPaths.append(url.path()) + let platform = try #require(url.pathComponents.dropLast().last) + return ( + Self.singleResultFixture(platform: platform), + makeHTTPURLResponse(url: url, statusCode: 200) + ) + } + let provider = AppStoreWebRankingProvider(httpClient: client) + + returnedPlatforms.append(try await provider.search( + keyword: "notes", storefrontCode: "us", platform: .iphone, limit: 1 + ).items[0].platform) + returnedPlatforms.append(try await provider.search( + keyword: "notes", storefrontCode: "us", platform: .ipad, limit: 1 + ).items[0].platform) + returnedPlatforms.append(try await provider.search( + keyword: "notes", storefrontCode: "us", platform: .mac, limit: 1 + ).items[0].platform) + + #expect(requestedPaths == [ + "/us/iphone/search", + "/us/ipad/search", + "/us/mac/search", + ]) + #expect(returnedPlatforms == [.iphone, .ipad, .mac]) + } + + @Test + func validEmptySearchDoesNotInvokeFallback() async throws { + let client = MockHTTPClient { request in + let url = try #require(request.url) + return ( + Self.emptyFixture(platform: "iphone"), + makeHTTPURLResponse(url: url, statusCode: 200) + ) + } + let primary = AppStoreWebRankingProvider(httpClient: client) + let fallback = ScriptedRankingProvider(behavior: .page(Self.fallbackPage)) + let provider = PrimaryFallbackSearchRankingProvider(primary: primary, fallback: fallback) + + let page = try await provider.search( + keyword: "notes", + storefrontCode: "us", + platform: .iphone, + limit: 10 + ) + + #expect(page.items.isEmpty) + #expect(page.source == .appStoreWeb) + #expect(await fallback.callCount == 0) + } + + @Test + func rejectsDecoyShelvesAndChangedAuthoritativeShapes() async throws { + let decoyClient = MockHTTPClient { request in + let url = try #require(request.url) + return ( + Self.decoyOnlyFixture, + makeHTTPURLResponse(url: url, statusCode: 200) + ) + } + let changedItemClient = MockHTTPClient { request in + let url = try #require(request.url) + return ( + Self.changedItemFixture, + makeHTTPURLResponse(url: url, statusCode: 200) + ) + } + let truncatedClient = MockHTTPClient { request in + let url = try #require(request.url) + return ( + Data(#""#.utf8), + makeHTTPURLResponse(url: url, statusCode: 200) + ) + } + let incompleteShelfClient = MockHTTPClient { request in + let url = try #require(request.url) + return ( + Self.incompleteShelfFixture, + makeHTTPURLResponse(url: url, statusCode: 200) + ) + } + + await Self.expectResponseFailure(.authoritativeShelfMissing) { + _ = try await AppStoreWebRankingProvider(httpClient: decoyClient).search( + keyword: "notes", + storefrontCode: "us", + platform: .iphone, + limit: 10 + ) + } + await Self.expectResponseFailure(.malformedSearchResult) { + _ = try await AppStoreWebRankingProvider(httpClient: changedItemClient).search( + keyword: "notes", + storefrontCode: "us", + platform: .iphone, + limit: 10 + ) + } + await Self.expectResponseFailure(.decodingFailed) { + _ = try await AppStoreWebRankingProvider(httpClient: truncatedClient).search( + keyword: "notes", + storefrontCode: "us", + platform: .iphone, + limit: 10 + ) + } + await Self.expectResponseFailure(.truncatedResults) { + _ = try await AppStoreWebRankingProvider(httpClient: incompleteShelfClient).search( + keyword: "notes", + storefrontCode: "us", + platform: .iphone, + limit: 2 + ) + } + } + + @Test + func fallsBackOnlyForRecognizedPrimaryFailures() async throws { + let recognizedFailures: [ScriptedRankingProvider.Behavior] = [ + .urlError(.cannotConnectToHost), + .httpStatus(503), + .response(.decodingFailed), + .response(.authoritativeShelfMissing), + .response(.truncatedResults), + ] + + for failure in recognizedFailures { + let primary = ScriptedRankingProvider(behavior: failure) + let fallback = ScriptedRankingProvider(behavior: .page(Self.fallbackPage)) + let provider = PrimaryFallbackSearchRankingProvider(primary: primary, fallback: fallback) + + let page = try await provider.search( + keyword: "notes", + storefrontCode: "us", + platform: .iphone, + limit: 10 + ) + + #expect(page.source == .iTunesFallback) + #expect(page.fallbackContext?.provider == .appStoreWeb) + #expect(await fallback.callCount == 1) + } + + let unknownPrimary = ScriptedRankingProvider(behavior: .unknown) + let unusedFallback = ScriptedRankingProvider(behavior: .page(Self.fallbackPage)) + let conservativeProvider = PrimaryFallbackSearchRankingProvider( + primary: unknownPrimary, + fallback: unusedFallback + ) + do { + _ = try await conservativeProvider.search( + keyword: "notes", + storefrontCode: "us", + platform: .iphone, + limit: 10 + ) + Issue.record("Expected the unknown primary failure to propagate.") + } catch is UnknownRankingProviderError { + // Expected: unknown errors are not assumed to be transport failures. + } + #expect(await unusedFallback.callCount == 0) + } + + @Test + func cancellationDeadlineAndValidationNeverInvokeFallback() async throws { + for behavior in [ + ScriptedRankingProvider.Behavior.cancellation, + .urlError(.timedOut), + ] { + let primary = ScriptedRankingProvider(behavior: behavior) + let fallback = ScriptedRankingProvider(behavior: .page(Self.fallbackPage)) + let provider = PrimaryFallbackSearchRankingProvider(primary: primary, fallback: fallback) + + do { + _ = try await provider.search( + keyword: "notes", + storefrontCode: "us", + platform: .iphone, + limit: 10 + ) + Issue.record("Expected cancellation or deadline to propagate.") + } catch { + if case .urlError(.timedOut) = behavior { + #expect((error as? URLError)?.code == .timedOut) + } else { + #expect(error is CancellationError) + } + } + #expect(await fallback.callCount == 0) + } + + let neverPrimary = ScriptedRankingProvider(behavior: .page(Self.fallbackPage)) + let neverFallback = ScriptedRankingProvider(behavior: .page(Self.fallbackPage)) + let provider = PrimaryFallbackSearchRankingProvider(primary: neverPrimary, fallback: neverFallback) + let invalidInputs: [(String, String, Int)] = [ + (" ", "us", 10), + ("notes", "not-a-storefront", 10), + ("notes", "us", 0), + ("notes", "us", SearchRankingCrawl.fullKeywordRankingLimit + 1), + ] + for (keyword, storefront, limit) in invalidInputs { + do { + _ = try await provider.search( + keyword: keyword, + storefrontCode: storefront, + platform: .iphone, + limit: limit + ) + Issue.record("Expected request validation to fail.") + } catch { + #expect(error is OpenASOError || error is SearchRankingProviderError) + } + } + #expect(await neverPrimary.callCount == 0) + #expect(await neverFallback.callCount == 0) + } + + @Test + func macPrimaryFailureUsesTheMacSoftwareFallback() async throws { + let primary = ScriptedRankingProvider(behavior: .httpStatus(503)) + let fallbackClient = MockHTTPClient { request in + let url = try #require(request.url) + #expect(url.absoluteString == "https://itunes.apple.com/search?term=notes&entity=macSoftware&country=us&limit=10") + return ( + Data(#"{"results":[]}"#.utf8), + makeHTTPURLResponse(url: url, statusCode: 200) + ) + } + let fallback = ITunesSearchFallbackProvider(httpClient: fallbackClient) + let provider = PrimaryFallbackSearchRankingProvider(primary: primary, fallback: fallback) + + let page = try await provider.search( + keyword: "notes", + storefrontCode: "us", + platform: .mac, + limit: 10 + ) + + #expect(page.source == .iTunesFallback) + #expect(page.items.isEmpty) + #expect(page.fallbackContext == SearchRankingFailureContext( + provider: .appStoreWeb, + category: .httpStatus(503) + )) + } + + @Test + func explicitlyUnsupportedFallbackSkipsProviderAndKeepsRedactedPrimaryContext() async throws { + let primary = ScriptedRankingProvider(behavior: .urlError(.cannotConnectToHost)) + let fallback = ScriptedRankingProvider(behavior: .page(Self.fallbackPage)) + let provider = PrimaryFallbackSearchRankingProvider( + primary: primary, + fallback: fallback, + fallbackSupportsPlatform: { _ in false } + ) + + do { + _ = try await provider.search( + keyword: "sensitive customer query", + storefrontCode: "us", + platform: .mac, + limit: 10 + ) + Issue.record("Expected the explicitly unsupported fallback to be skipped.") + } catch let error as SearchRankingProviderError { + guard case .fallbackUnavailable(let platform, let primaryContext) = error else { + Issue.record("Expected fallbackUnavailable, received \(error).") + return + } + #expect(platform == .mac) + #expect(primaryContext == SearchRankingFailureContext( + provider: .appStoreWeb, + category: .transport(code: URLError.Code.cannotConnectToHost.rawValue) + )) + #expect(!error.localizedDescription.contains("sensitive customer query")) + } + + #expect(await fallback.callCount == 0) + } + + @Test + func doubleFailurePreservesOnlyRedactedContexts() async throws { + let primary = ScriptedRankingProvider(behavior: .urlError(.cannotConnectToHost)) + let fallback = ScriptedRankingProvider( + behavior: .openASO(.providerUnavailable("secret-token=do-not-report")) + ) + let provider = PrimaryFallbackSearchRankingProvider(primary: primary, fallback: fallback) + + do { + _ = try await provider.search( + keyword: "sensitive customer query", + storefrontCode: "us", + platform: .iphone, + limit: 10 + ) + Issue.record("Expected both providers to fail.") + } catch let error as SearchRankingProviderError { + guard case .allProvidersFailed(let primaryContext, let fallbackContext) = error else { + Issue.record("Expected allProvidersFailed, received \(error).") + return + } + #expect(primaryContext == SearchRankingFailureContext( + provider: .appStoreWeb, + category: .transport(code: URLError.Code.cannotConnectToHost.rawValue) + )) + #expect(fallbackContext == SearchRankingFailureContext( + provider: .iTunesFallback, + category: .provider + )) + #expect(!error.localizedDescription.contains("sensitive customer query")) + #expect(!error.localizedDescription.contains("secret-token")) + } + } + + private static func expectResponseFailure( + _ expected: SearchRankingResponseFailure, + operation: () async throws -> Void + ) async { + do { + try await operation() + Issue.record("Expected response failure \(expected.rawValue).") + } catch let error as SearchRankingProviderError { + guard case .responseFailure(let actual) = error else { + Issue.record("Expected responseFailure, received \(error).") + return + } + #expect(actual == expected) + } catch { + Issue.record("Expected SearchRankingProviderError, received \(error).") + } + } + + private static func emptyFixture(platform: String) -> Data { + Data(""" + + """.utf8) + } + + private static func singleResultFixture(platform: String) -> Data { + Data(""" + + """.utf8) + } + + private static let orderedFixture = Data(""" + + """.utf8) + + private static let decoyOnlyFixture = Data(""" + + """.utf8) + + private static let changedItemFixture = Data(""" + + """.utf8) + + private static let incompleteShelfFixture = Data(""" + + """.utf8) + + private static let fallbackPage = SearchRankingPage( + items: [SearchRankingItem( + position: 1, + appStoreID: 42, + bundleID: "com.example.fallback", + name: "Fallback App", + sellerName: "Example", + platform: .iphone + )], + source: .iTunesFallback + ) +} + +private actor ScriptedRankingProvider: SearchRankingProvider { + enum Behavior: Sendable { + case page(SearchRankingPage) + case urlError(URLError.Code) + case httpStatus(Int) + case response(SearchRankingResponseFailure) + case openASO(OpenASOError) + case cancellation + case unknown + } + + private let behavior: Behavior + private(set) var callCount = 0 + + init(behavior: Behavior) { + self.behavior = behavior + } + + func search( + keyword: String, + storefrontCode: String, + platform: AppPlatform, + limit: Int + ) async throws -> SearchRankingPage { + callCount += 1 + switch behavior { + case .page(let page): + return page + case .urlError(let code): + throw URLError(code) + case .httpStatus(let statusCode): + throw SearchRankingProviderError.httpStatus(statusCode) + case .response(let failure): + throw SearchRankingProviderError.responseFailure(failure) + case .openASO(let error): + throw error + case .cancellation: + throw CancellationError() + case .unknown: + throw UnknownRankingProviderError() + } + } +} + +private struct UnknownRankingProviderError: Error {} diff --git a/OpenASOTests/EstimatedKeywordDifficultyCSVExportTests.swift b/OpenASOTests/EstimatedKeywordDifficultyCSVExportTests.swift new file mode 100644 index 0000000..7c075c2 --- /dev/null +++ b/OpenASOTests/EstimatedKeywordDifficultyCSVExportTests.swift @@ -0,0 +1,676 @@ +import Foundation +import SwiftData +import Testing +@testable import OpenASO + +@MainActor +struct EstimatedKeywordDifficultyCSVExportTests { + @Test + func formatEmitsExactEvidenceContractAndSummaryRowsDeterministically() throws { + let rankingFetchedAt = Date(timeIntervalSince1970: 1_700_000_000) + let asOf = rankingFetchedAt.addingTimeInterval( + EstimatedKeywordDifficultyFreshness.maximumAge + ) + let estimated = makeSnapshot( + keyword: "focus timer", + stateRaw: EstimatedKeywordDifficultyState.estimated.rawValue, + score: 67, + confidenceScore: 83, + confidenceRaw: EstimatedKeywordDifficultyConfidence.medium.rawValue, + unavailableReasonRaw: nil, + rankingFetchedAt: rankingFetchedAt, + fallbackProviderRaw: RankingSource.appStoreWeb.rawValue, + fallbackCategoryRaw: EstimatedKeywordDifficultyFallbackCategory.response.rawValue, + fallbackResponseFailureRaw: EstimatedKeywordDifficultyFallbackResponseFailure.pageShapeChanged.rawValue, + notes: ["Heuristic, evidence", "Uses \"quotes\""], + resultEvidence: [ + makeEvidence(position: 2, appStoreID: 222, title: "Second"), + makeEvidence(position: 1, appStoreID: 111, title: "First, Result") + ] + ) + let unavailable = makeSnapshot( + keyword: "zebra planner", + stateRaw: EstimatedKeywordDifficultyState.unavailable.rawValue, + score: nil, + confidenceScore: nil, + confidenceRaw: nil, + unavailableReasonRaw: EstimatedKeywordDifficultyUnavailableReason.insufficientResults.rawValue, + rankingFetchedAt: rankingFetchedAt.addingTimeInterval(60), + fallbackProviderRaw: nil, + fallbackCategoryRaw: nil, + fallbackResponseFailureRaw: nil, + notes: [], + resultEvidence: [] + ) + let items = [ + makeItem(keyword: unavailable.keyword, snapshot: unavailable), + makeItem(keyword: estimated.keyword, snapshot: estimated) + ] + + let firstCSV = EstimatedKeywordDifficultyCSVFormat.encode(items: items, asOf: asOf) + let secondCSV = EstimatedKeywordDifficultyCSVFormat.encode( + items: Array(items.reversed()), + asOf: asOf + ) + #expect(firstCSV == secondCSV) + + let table = CSVTable.parse(firstCSV) + let headers = try #require(table.first) + #expect(headers == expectedHeaders) + #expect(headers.count == 48) + #expect(table.count == 4) + #expect(table.dropFirst().allSatisfy { $0.count == 48 }) + + let firstEvidence = rowDictionary(headers: headers, row: table[1]) + #expect(firstEvidence["OpenASO Export Type"] == EstimatedKeywordDifficultyCSVFormat.evidenceExportType) + #expect(firstEvidence["App Name"] == "Writer, Pro") + #expect(firstEvidence["State"] == "estimated") + #expect(firstEvidence["Estimated Difficulty"] == "67") + #expect(firstEvidence["Confidence Score"] == "83") + #expect(firstEvidence["Evidence Position"] == "1") + #expect(firstEvidence["Evidence App Store Id"] == "111") + #expect(firstEvidence["Evidence Title"] == "First, Result") + #expect(firstEvidence["Fallback Provider"] == RankingSource.appStoreWeb.rawValue) + #expect(firstEvidence["Fallback Category"] == EstimatedKeywordDifficultyFallbackCategory.response.rawValue) + #expect(firstEvidence["Fallback Response Failure"] == EstimatedKeywordDifficultyFallbackResponseFailure.pageShapeChanged.rawValue) + #expect(firstEvidence["Stale At"] == CSVTable.string(from: asOf)) + #expect(firstEvidence["Is Stale"] == "true") + #expect(firstEvidence["Notes JSON"] == "[\"Heuristic, evidence\",\"Uses \\\"quotes\\\"\"]") + + let secondEvidence = rowDictionary(headers: headers, row: table[2]) + #expect(secondEvidence["Evidence Position"] == "2") + #expect(secondEvidence["Evidence App Store Id"] == "222") + + let summary = rowDictionary(headers: headers, row: table[3]) + #expect(summary["OpenASO Export Type"] == EstimatedKeywordDifficultyCSVFormat.summaryExportType) + #expect(summary["State"] == "unavailable") + #expect(summary["Unavailable Reason"] == "insufficientResults") + #expect(summary["Estimated Difficulty"] == "") + #expect(summary["Evidence Position"] == "") + #expect(summary["Notes JSON"] == "[]") + } + + @Test + func missingSnapshotExportsOneNonImportableSummaryWithoutFreshnessOrEvidence() throws { + let item = makeItem(keyword: "uncalculated keyword", snapshot: nil) + let csv = EstimatedKeywordDifficultyCSVFormat.encode( + items: [item], + asOf: Date(timeIntervalSince1970: 1_800_000_000) + ) + let table = CSVTable.parse(csv) + let headers = try #require(table.first) + #expect(table.count == 2) + + let row = rowDictionary(headers: headers, row: table[1]) + #expect(row["OpenASO Export Type"] == EstimatedKeywordDifficultyCSVFormat.summaryExportType) + #expect(row["State"] == "missing") + #expect(row["Estimated Difficulty"] == "") + #expect(row["Ranking Fetched At"] == "") + #expect(row["Computed At"] == "") + #expect(row["Stale At"] == "") + #expect(row["Is Stale"] == "") + #expect(row["Evidence Position"] == "") + #expect(row["Evidence App Store Id"] == "") + #expect(row["Notes JSON"] == "[]") + + #expect(throws: CSVError.missingColumn("Store Domain")) { + _ = try TrackedKeywordCSVFormat.decode(csv) + } + } + + @Test + func dedicatedExportNeutralizesSpreadsheetFormulaPrefixesInExternalText() throws { + let fetchedAt = Date(timeIntervalSince1970: 1_700_000_000) + let keyword = "+KEYWORD" + let snapshot = makeSnapshot( + keyword: keyword, + storefront: "=STORE", + stateRaw: EstimatedKeywordDifficultyState.estimated.rawValue, + score: 55, + confidenceScore: 70, + confidenceRaw: EstimatedKeywordDifficultyConfidence.medium.rawValue, + unavailableReasonRaw: nil, + rankingFetchedAt: fetchedAt, + fallbackProviderRaw: nil, + fallbackCategoryRaw: nil, + fallbackResponseFailureRaw: nil, + notes: [], + resultEvidence: [ + makeEvidence( + position: 1, + appStoreID: 111, + title: "-TITLE", + subtitle: "@SUBTITLE" + ), + makeEvidence( + position: 2, + appStoreID: 222, + title: "\tTITLE", + subtitle: "\rSUBTITLE" + ), + makeEvidence( + position: 3, + appStoreID: 333, + title: "\nTITLE", + subtitle: "+SUBTITLE" + ), + ] + ) + let item = EstimatedKeywordDifficultyCSVItem( + appName: "=APP", + appStoreID: 123, + keyword: keyword, + queryKey: snapshot.queryKey, + storefront: "gb", + platformRaw: AppPlatform.iphone.rawValue, + snapshot: snapshot + ) + + let table = CSVTable.parse( + EstimatedKeywordDifficultyCSVFormat.encode(items: [item], asOf: fetchedAt) + ) + let headers = try #require(table.first) + let first = rowDictionary(headers: headers, row: table[1]) + let second = rowDictionary(headers: headers, row: table[2]) + let third = rowDictionary(headers: headers, row: table[3]) + + #expect(first["App Name"] == "'=APP") + #expect(first["Keyword"] == "'+KEYWORD") + #expect(first["Query Key"] == "'\(snapshot.queryKey)") + #expect(first["Storefront"] == "'=STORE") + #expect(first["Evidence Title"] == "'-TITLE") + #expect(first["Evidence Subtitle"] == "'@SUBTITLE") + #expect(second["Evidence Title"] == "'\tTITLE") + #expect(second["Evidence Subtitle"] == "'\rSUBTITLE") + #expect(third["Evidence Title"] == "'\nTITLE") + #expect(third["Evidence Subtitle"] == "'+SUBTITLE") + } + + @Test + func exporterUsesBackgroundSnapshotsAndRespectsAppStorefrontPlatformAndSearch() async throws { + let container = try ModelContainerFactory.makeModelContainer(isStoredInMemoryOnly: true) + let context = ModelContext(container) + let selectedApp = TrackedApp( + appStoreID: 123, + bundleID: "com.example.selected", + name: "Selected", + sellerName: "Example", + defaultPlatform: .ipad + ) + let otherApp = TrackedApp( + appStoreID: 456, + bundleID: "com.example.other", + name: "Other", + sellerName: "Example", + defaultPlatform: .ipad + ) + context.insert(selectedApp) + context.insert(otherApp) + + let matchingTrack = try insertTrack( + term: "Focus Timer", + storefront: "gb", + platform: .ipad, + app: selectedApp, + in: context + ) + _ = try insertTrack( + term: "Focus Timer US", + storefront: "us", + platform: .ipad, + app: selectedApp, + in: context + ) + _ = try insertTrack( + term: "Focus Timer Mac", + storefront: "gb", + platform: .mac, + app: selectedApp, + in: context + ) + _ = try insertTrack( + term: "Sleep Sounds", + storefront: "gb", + platform: .ipad, + app: selectedApp, + in: context + ) + _ = try insertTrack( + term: "Focus Timer Other App", + storefront: "gb", + platform: .ipad, + app: otherApp, + in: context + ) + context.insert(KeywordDailyMetric( + queryKey: matchingTrack.queryKey, + keyword: matchingTrack.term, + storefront: matchingTrack.storefront, + platform: matchingTrack.platform, + popularityScore: 80, + difficultyScore: 99, + source: .appleAdsPopularity, + updatedAt: Date(timeIntervalSince1970: 1_700_000_000) + )) + _ = try EstimatedKeywordDifficultyStore.upsert( + makePayload(for: matchingTrack, score: 25), + in: context + ) + try context.save() + + let document = try await AppDetailEstimatedDifficultyCSVExporter.makeDocument( + appStoreID: selectedApp.appStoreID, + appName: selectedApp.name, + selectedStorefrontFilter: .storefront(code: "GB", title: "🇬🇧 United Kingdom"), + selectedPlatformFilter: .platform(.ipad), + difficultyFilterRange: 20 ... 30, + searchText: "focus timer", + backgroundModelStore: BackgroundModelStore(modelContainer: container), + asOf: Date(timeIntervalSince1970: 1_700_086_400) + ) + + let table = CSVTable.parse(document.text) + let headers = try #require(table.first) + #expect(table.count == 4) + #expect(!headers.contains("Difficulty")) + let row = rowDictionary(headers: headers, row: table[1]) + #expect(row["App Id"] == "123") + #expect(row["Keyword"] == "Focus Timer") + #expect(row["Storefront"] == "gb") + #expect(row["Platform"] == "ipad") + #expect(row["State"] == "estimated") + #expect(row["Estimated Difficulty"] == "25") + #expect(row["Estimated Difficulty"] != "99") + + let legacyOnlyRangeDocument = try await AppDetailEstimatedDifficultyCSVExporter.makeDocument( + appStoreID: selectedApp.appStoreID, + appName: selectedApp.name, + selectedStorefrontFilter: .storefront(code: "GB", title: "🇬🇧 United Kingdom"), + selectedPlatformFilter: .platform(.ipad), + difficultyFilterRange: 90 ... 100, + searchText: "focus timer", + backgroundModelStore: BackgroundModelStore(modelContainer: container), + asOf: Date(timeIntervalSince1970: 1_700_086_400) + ) + #expect(CSVTable.parse(legacyOnlyRangeDocument.text).count == 1) + } + + @Test + func historyExporterFiltersOnEstimatedDifficultyAndKeepsLegacyDifficultyOutput() throws { + let container = try ModelContainerFactory.makeModelContainer(isStoredInMemoryOnly: true) + let context = ModelContext(container) + let app = TrackedApp( + appStoreID: 123, + bundleID: "com.example.selected", + name: "Selected", + sellerName: "Example", + defaultPlatform: .iphone + ) + context.insert(app) + + let estimatedTrack = try insertTrack( + term: "Focus Timer", + storefront: "gb", + platform: .iphone, + app: app, + in: context + ) + let missingTrack = try insertTrack( + term: "Missing Estimate", + storefront: "gb", + platform: .iphone, + app: app, + in: context + ) + let unavailableTrack = try insertTrack( + term: "Unavailable Estimate", + storefront: "gb", + platform: .iphone, + app: app, + in: context + ) + addHistory(to: estimatedTrack, in: context) + addHistory(to: missingTrack, in: context) + addHistory(to: unavailableTrack, in: context) + + context.insert(KeywordDailyMetric( + queryKey: estimatedTrack.queryKey, + keyword: estimatedTrack.term, + storefront: estimatedTrack.storefront, + platform: estimatedTrack.platform, + popularityScore: 80, + difficultyScore: 99, + source: .appleAdsPopularity, + updatedAt: Date(timeIntervalSince1970: 1_700_000_000) + )) + _ = try EstimatedKeywordDifficultyStore.upsert( + makePayload(for: estimatedTrack, score: 25), + in: context + ) + _ = try EstimatedKeywordDifficultyStore.upsert( + makeUnavailablePayload(for: unavailableTrack), + in: context + ) + try context.save() + + var workspaceState = KeywordWorkspaceState() + workspaceState.selectedDateRange = .allTime + workspaceState.difficultyFilterRange = 20 ... 30 + let narrowedDocument = try AppDetailView.KeywordHistoryCSVExporter.makeDocument( + tracks: [estimatedTrack, missingTrack, unavailableTrack], + appName: app.name, + appStoreID: app.appStoreID, + selectedStorefrontFilter: .all, + workspaceState: workspaceState, + searchText: "", + modelContext: context, + storefrontTitle: { $0.uppercased() } + ) + + let narrowedTable = CSVTable.parse(narrowedDocument.text) + let headers = try #require(narrowedTable.first) + #expect(narrowedTable.count == 3) + #expect(headers.contains("Difficulty")) + #expect(!headers.contains("Estimated Difficulty")) + for row in narrowedTable.dropFirst() { + let values = rowDictionary(headers: headers, row: row) + #expect(values["Keyword"] == estimatedTrack.term) + #expect(values["Difficulty"] == "99") + } + + workspaceState.difficultyFilterRange = MetricFilterRange.difficulty.defaultRange + let defaultDocument = try AppDetailView.KeywordHistoryCSVExporter.makeDocument( + tracks: [estimatedTrack, missingTrack, unavailableTrack], + appName: app.name, + appStoreID: app.appStoreID, + selectedStorefrontFilter: .all, + workspaceState: workspaceState, + searchText: "", + modelContext: context, + storefrontTitle: { $0.uppercased() } + ) + let defaultTable = CSVTable.parse(defaultDocument.text) + #expect(defaultTable.count == 7) + #expect(Set(defaultTable.dropFirst().compactMap { row in + rowDictionary(headers: headers, row: row)["Keyword"] + }) == Set([estimatedTrack.term, missingTrack.term, unavailableTrack.term])) + } + + private var expectedHeaders: [String] { + [ + "OpenASO Export Type", + "App Name", + "App Id", + "Keyword", + "Query Key", + "Storefront", + "Platform", + "State", + "Estimated Difficulty", + "Confidence Score", + "Confidence", + "Unavailable Reason", + "Estimation Source", + "Algorithm Identifier", + "Algorithm Version", + "Requested Result Limit", + "Provider Result Count", + "Considered Result Count", + "Rated Result Count", + "Weighted Rating Coverage Percentage", + "Maximum Rating Count", + "Median Rating Count", + "Rating Authority Score", + "Metadata Saturation Score", + "Exact Title Phrase Match Count", + "Exact Subtitle Phrase Match Count", + "Ranking Source", + "Ranking Fetched At", + "Computed At", + "Stale At", + "Is Stale", + "Fallback Provider", + "Fallback Category", + "Fallback Transport Code", + "Fallback HTTP Status", + "Fallback Response Failure", + "Notes JSON", + "Evidence Position", + "Evidence App Store Id", + "Evidence Title", + "Evidence Subtitle", + "Evidence Rating Count", + "Evidence Rating Authority Score", + "Evidence Title Token Coverage Percentage", + "Evidence Combined Token Coverage Percentage", + "Evidence Metadata Match Score", + "Evidence Exact Title Phrase Match", + "Evidence Exact Subtitle Phrase Match" + ] + } + + private func makeItem( + keyword: String, + snapshot: EstimatedKeywordDifficultySnapshot? + ) -> EstimatedKeywordDifficultyCSVItem { + EstimatedKeywordDifficultyCSVItem( + appName: "Writer, Pro", + appStoreID: 123, + keyword: keyword, + queryKey: KeywordQuery.makeQueryKey( + term: keyword, + storefront: "gb", + platform: .iphone + ), + storefront: "gb", + platformRaw: AppPlatform.iphone.rawValue, + snapshot: snapshot + ) + } + + private func makeSnapshot( + keyword: String, + storefront: String = "gb", + stateRaw: String, + score: Int?, + confidenceScore: Int?, + confidenceRaw: String?, + unavailableReasonRaw: String?, + rankingFetchedAt: Date, + fallbackProviderRaw: String?, + fallbackCategoryRaw: String?, + fallbackResponseFailureRaw: String?, + notes: [String], + resultEvidence: [EstimatedKeywordDifficultyResultEvidence] + ) -> EstimatedKeywordDifficultySnapshot { + EstimatedKeywordDifficultySnapshot( + queryKey: KeywordQuery.makeQueryKey( + term: keyword, + storefront: storefront, + platform: .iphone + ), + calculationID: UUID(uuid: (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32)), + keyword: keyword, + storefront: storefront, + platformRaw: AppPlatform.iphone.rawValue, + stateRaw: stateRaw, + score: score, + confidenceScore: confidenceScore, + confidenceRaw: confidenceRaw, + unavailableReasonRaw: unavailableReasonRaw, + estimationSourceRaw: EstimatedKeywordDifficultySource.topResultsHeuristic.rawValue, + algorithmIdentifier: "top10-authority-saturation", + algorithmVersion: 1, + requestedResultLimit: 10, + providerResultCount: 8, + consideredResultCount: resultEvidence.count, + ratedResultCount: resultEvidence.count, + weightedRatingCoveragePercentage: resultEvidence.isEmpty ? 0 : 100, + maximumRatingCount: resultEvidence.compactMap(\.ratingCount).max(), + medianRatingCount: resultEvidence.isEmpty ? nil : 150, + ratingAuthorityScore: resultEvidence.isEmpty ? nil : 62, + metadataSaturationScore: resultEvidence.isEmpty ? nil : 71, + exactTitlePhraseMatchCount: resultEvidence.count(where: \.exactTitlePhraseMatch), + exactSubtitlePhraseMatchCount: resultEvidence.count(where: \.exactSubtitlePhraseMatch), + rankingSourceRaw: RankingSource.iTunesFallback.rawValue, + rankingFetchedAt: rankingFetchedAt, + computedAt: rankingFetchedAt.addingTimeInterval(30), + fallbackProviderRaw: fallbackProviderRaw, + fallbackCategoryRaw: fallbackCategoryRaw, + fallbackTransportCode: nil, + fallbackHTTPStatus: nil, + fallbackResponseFailureRaw: fallbackResponseFailureRaw, + notes: notes, + resultEvidence: resultEvidence + ) + } + + private func makeEvidence( + position: Int, + appStoreID: Int64, + title: String, + subtitle: String? = nil + ) -> EstimatedKeywordDifficultyResultEvidence { + EstimatedKeywordDifficultyResultEvidence( + position: position, + appStoreID: appStoreID, + title: title, + subtitle: subtitle ?? "Subtitle \(position)", + ratingCount: position * 100, + ratingAuthorityScore: 40 + position, + titleTokenCoveragePercentage: 80 + position, + combinedTokenCoveragePercentage: 85 + position, + metadataMatchScore: 70 + position, + exactTitlePhraseMatch: position == 1, + exactSubtitlePhraseMatch: position == 2 + ) + } + + private func makePayload( + for track: TrackedAppKeyword, + score: Int + ) -> EstimatedKeywordDifficultyPersistencePayload { + let evidence: [EstimatedKeywordDifficultyResultEvidence] = [ + makeEvidence(position: 1, appStoreID: 10_001, title: "Focus Result 1"), + makeEvidence(position: 2, appStoreID: 10_002, title: "Focus Result 2"), + makeEvidence(position: 3, appStoreID: 10_003, title: "Focus Result 3") + ] + let rankingFetchedAt = Date(timeIntervalSince1970: 1_700_000_000) + return EstimatedKeywordDifficultyPersistencePayload( + queryKey: track.queryKey, + calculationID: UUID(uuid: (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33)), + keyword: track.term, + storefront: track.storefront, + platform: track.platform, + result: .estimated( + score: score, + confidenceScore: 80, + confidence: .medium + ), + algorithmIdentifier: "top10-authority-saturation", + algorithmVersion: 1, + requestedResultLimit: 10, + providerResultCount: 3, + evidence: EstimatedKeywordDifficultyEvidence( + consideredResultCount: 3, + ratedResultCount: 3, + weightedRatingCoveragePercentage: 100, + maximumRatingCount: 300, + medianRatingCount: 200, + ratingAuthorityScore: 62, + metadataSaturationScore: 71, + resultEvidence: evidence + ), + rankingSource: .appStoreWeb, + rankingFetchedAt: rankingFetchedAt, + computedAt: rankingFetchedAt.addingTimeInterval(30), + notes: ["Estimated from current ranking evidence."] + ) + } + + private func makeUnavailablePayload( + for track: TrackedAppKeyword + ) -> EstimatedKeywordDifficultyPersistencePayload { + let rankingFetchedAt = Date(timeIntervalSince1970: 1_700_000_000) + return EstimatedKeywordDifficultyPersistencePayload( + queryKey: track.queryKey, + calculationID: UUID(uuid: (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34)), + keyword: track.term, + storefront: track.storefront, + platform: track.platform, + result: .unavailable(reason: .insufficientResults), + algorithmIdentifier: "top10-authority-saturation", + algorithmVersion: 1, + requestedResultLimit: 10, + providerResultCount: 0, + evidence: EstimatedKeywordDifficultyEvidence( + consideredResultCount: 0, + ratedResultCount: 0, + weightedRatingCoveragePercentage: 0, + maximumRatingCount: nil, + medianRatingCount: nil, + ratingAuthorityScore: nil, + metadataSaturationScore: nil, + resultEvidence: [] + ), + rankingSource: .appStoreWeb, + rankingFetchedAt: rankingFetchedAt, + computedAt: rankingFetchedAt.addingTimeInterval(30), + notes: ["Estimate unavailable from current ranking evidence."] + ) + } + + private func addHistory( + to track: TrackedAppKeyword, + in context: ModelContext + ) { + let older = TrackedKeywordDailyRanking( + rank: 8, + searchedAt: Date(timeIntervalSince1970: 1_700_000_000), + source: .appStoreWeb, + resultCount: 10, + keywordTrack: track + ) + let newer = TrackedKeywordDailyRanking( + rank: 5, + searchedAt: Date(timeIntervalSince1970: 1_700_086_400), + source: .appStoreWeb, + resultCount: 10, + keywordTrack: track + ) + track.snapshots.append(contentsOf: [older, newer]) + context.insert(older) + context.insert(newer) + } + + private func insertTrack( + term: String, + storefront: String, + platform: AppPlatform, + app: TrackedApp, + in modelContext: ModelContext + ) throws -> TrackedAppKeyword { + let query = try KeywordQuery.fetchOrInsert( + term: term, + storefront: storefront, + platform: platform, + in: modelContext + ) + let track = TrackedAppKeyword( + term: term, + storefront: storefront, + platform: platform, + trackedApp: app, + query: query + ) + app.keywordTracks.append(track) + modelContext.insert(track) + return track + } + + private func rowDictionary(headers: [String], row: [String]) -> [String: String] { + Dictionary(uniqueKeysWithValues: headers.enumerated().map { index, header in + (header, row.indices.contains(index) ? row[index] : "") + }) + } +} diff --git a/OpenASOTests/EstimatedKeywordDifficultyDetailModelTests.swift b/OpenASOTests/EstimatedKeywordDifficultyDetailModelTests.swift new file mode 100644 index 0000000..df21611 --- /dev/null +++ b/OpenASOTests/EstimatedKeywordDifficultyDetailModelTests.swift @@ -0,0 +1,160 @@ +import Foundation +import Testing +@testable import OpenASO + +@MainActor +struct EstimatedKeywordDifficultyDetailModelTests { + @Test + func publishesLoadedMissingAndFailedStatesDistinctly() async { + let loadedModel = EstimatedKeywordDifficultyDetailModel() + let expected = snapshot(queryKey: "loaded", score: 64) + await loadedModel.load( + queryKey: "loaded", + using: EstimatedKeywordDifficultyDetailDataSource { _ in expected } + ) + #expect(loadedModel.state == .loaded(expected)) + + let missingModel = EstimatedKeywordDifficultyDetailModel() + await missingModel.load( + queryKey: "missing", + using: EstimatedKeywordDifficultyDetailDataSource { _ in nil } + ) + #expect(missingModel.state == .missing) + + let failedModel = EstimatedKeywordDifficultyDetailModel() + await failedModel.load( + queryKey: "failed", + using: EstimatedKeywordDifficultyDetailDataSource { _ in + throw DetailLoaderError.failed + } + ) + guard case let .failed(message) = failedModel.state else { + Issue.record("Expected a distinct failed state") + return + } + #expect(!message.isEmpty) + } + + @Test + func supersededCompletionCannotReplaceLatestSnapshot() async { + let model = EstimatedKeywordDifficultyDetailModel() + let loader = ControlledEstimatedKeywordDifficultyLoader() + let dataSource = EstimatedKeywordDifficultyDetailDataSource(load: loader.load) + + let staleTask = Task { @MainActor in + await model.load(queryKey: "stale", using: dataSource) + } + await loader.waitForRequestCount(1) + + let freshTask = Task { @MainActor in + await model.load(queryKey: "fresh", using: dataSource) + } + await loader.waitForRequestCount(2) + + let freshSnapshot = snapshot(queryKey: "fresh", score: 88) + loader.succeedRequest(at: 1, with: freshSnapshot) + await freshTask.value + #expect(model.snapshot == freshSnapshot) + + loader.succeedRequest(at: 0, with: snapshot(queryKey: "stale", score: 12)) + await staleTask.value + #expect(model.snapshot == freshSnapshot) + } + + @Test + func cancellationRestoresIdleStateWithoutPublishingResult() async { + let model = EstimatedKeywordDifficultyDetailModel() + let loader = ControlledEstimatedKeywordDifficultyLoader() + let dataSource = EstimatedKeywordDifficultyDetailDataSource(load: loader.load) + + let task = Task { @MainActor in + await model.load(queryKey: "cancelled", using: dataSource) + } + await loader.waitForRequestCount(1) + task.cancel() + loader.succeedRequest(at: 0, with: snapshot(queryKey: "cancelled", score: 55)) + await task.value + + #expect(model.state == .idle) + } + + private func snapshot( + queryKey: String, + score: Int + ) -> EstimatedKeywordDifficultySnapshot { + let fetchedAt = Date(timeIntervalSinceReferenceDate: 800_000_000) + return EstimatedKeywordDifficultySnapshot( + queryKey: queryKey, + calculationID: UUID(), + keyword: queryKey, + storefront: "us", + platformRaw: AppPlatform.iphone.rawValue, + stateRaw: EstimatedKeywordDifficultyState.estimated.rawValue, + score: score, + confidenceScore: 80, + confidenceRaw: EstimatedKeywordDifficultyConfidence.high.rawValue, + unavailableReasonRaw: nil, + estimationSourceRaw: EstimatedKeywordDifficultySource.topResultsHeuristic.rawValue, + algorithmIdentifier: "top10-authority-saturation", + algorithmVersion: 1, + requestedResultLimit: 10, + providerResultCount: 10, + consideredResultCount: 10, + ratedResultCount: 8, + weightedRatingCoveragePercentage: 80, + maximumRatingCount: 10_000, + medianRatingCount: 500, + ratingAuthorityScore: 70, + metadataSaturationScore: 60, + exactTitlePhraseMatchCount: 2, + exactSubtitlePhraseMatchCount: 1, + rankingSourceRaw: RankingSource.appStoreWeb.rawValue, + rankingFetchedAt: fetchedAt, + computedAt: fetchedAt.addingTimeInterval(1), + fallbackProviderRaw: nil, + fallbackCategoryRaw: nil, + fallbackTransportCode: nil, + fallbackHTTPStatus: nil, + fallbackResponseFailureRaw: nil, + notes: [], + resultEvidence: [] + ) + } +} + +private enum DetailLoaderError: Error { + case failed +} + +@MainActor +private final class ControlledEstimatedKeywordDifficultyLoader { + private struct Request { + let queryKey: String + let continuation: CheckedContinuation< + Result, + Never + > + } + + private var requests: [Request] = [] + + func load(queryKey: String) async throws -> EstimatedKeywordDifficultySnapshot? { + let result = await withCheckedContinuation { continuation in + requests.append(Request(queryKey: queryKey, continuation: continuation)) + } + return try result.get() + } + + func waitForRequestCount(_ expectedCount: Int) async { + while requests.count < expectedCount { + await Task.yield() + } + } + + func succeedRequest( + at index: Int, + with snapshot: EstimatedKeywordDifficultySnapshot? + ) { + requests[index].continuation.resume(returning: .success(snapshot)) + } +} diff --git a/OpenASOTests/EstimatedKeywordDifficultyPersistenceAdapterTests.swift b/OpenASOTests/EstimatedKeywordDifficultyPersistenceAdapterTests.swift new file mode 100644 index 0000000..9d1198b --- /dev/null +++ b/OpenASOTests/EstimatedKeywordDifficultyPersistenceAdapterTests.swift @@ -0,0 +1,538 @@ +import Foundation +import SwiftData +import Testing +@testable import OpenASO + +@MainActor +struct EstimatedKeywordDifficultyPersistenceAdapterTests { + @Test + func primaryPageMapsEstimatorOutputAndRetainsExactMetadata() throws { + let request = makeRequest() + let page = SearchRankingPage(items: estimatedItems(), source: .appStoreWeb) + let calculationID = UUID(uuidString: "00000000-0000-0000-0000-000000002001")! + let rankingFetchedAt = Date(timeIntervalSince1970: 1_720_000_000) + let computedAt = rankingFetchedAt.addingTimeInterval(2) + + let payload = try #require(EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: request, + page: page, + requestedResultLimit: 10, + rankingFetchedAt: rankingFetchedAt, + calculationID: calculationID, + now: { computedAt } + )) + let estimate = try requireEstimate( + KeywordDifficultyEstimator.estimate( + keyword: request.term, + searchResults: page.items + ) + ) + + #expect(payload.queryKey == request.queryKey) + #expect(payload.calculationID == calculationID) + #expect(payload.keyword == "focus timer") + #expect(payload.storefront == "us") + #expect(payload.platform == .iphone) + #expect(payload.result == .estimated( + score: estimate.score, + confidenceScore: estimate.confidenceScore, + confidence: persistenceConfidence(estimate.confidence) + )) + #expect(payload.estimationSource == .topResultsHeuristic) + #expect(payload.algorithmIdentifier == estimate.algorithmIdentifier) + #expect(payload.algorithmVersion == estimate.algorithmVersion) + #expect(payload.requestedResultLimit == 10) + #expect(payload.providerResultCount == page.resultCount) + #expect(payload.evidence == persistenceEvidence(estimate.evidence)) + #expect(payload.rankingSource == .appStoreWeb) + #expect(payload.rankingFetchedAt == rankingFetchedAt) + #expect(payload.computedAt == computedAt) + #expect(payload.fallback == nil) + #expect(payload.notes == estimate.notes) + + let firstEvidence = try #require(payload.evidence.resultEvidence.first) + #expect(firstEvidence.title == " Focus Timer++: Deep Work! ") + #expect(firstEvidence.subtitle == " Stay focused — privately! ") + } + + @Test + func fallbackPageMapsStructuredProvenance() throws { + let provenanceCases: [( + input: SearchRankingFailureCategory, + expected: EstimatedKeywordDifficultyFallbackProvenance + )] = [ + ( + .transport(code: -1009), + EstimatedKeywordDifficultyFallbackProvenance( + provider: .appStoreWeb, + category: .transport, + transportCode: -1009 + ) + ), + ( + .httpStatus(503), + EstimatedKeywordDifficultyFallbackProvenance( + provider: .appStoreWeb, + category: .httpStatus, + httpStatus: 503 + ) + ), + ( + .provider, + EstimatedKeywordDifficultyFallbackProvenance( + provider: .appStoreWeb, + category: .provider + ) + ), + ( + .other, + EstimatedKeywordDifficultyFallbackProvenance( + provider: .appStoreWeb, + category: .other + ) + ) + ] + + for provenanceCase in provenanceCases { + let page = SearchRankingPage( + items: estimatedItems(), + source: .iTunesFallback, + fallbackContext: SearchRankingFailureContext( + provider: .appStoreWeb, + category: provenanceCase.input + ) + ) + let payload = try #require(makePayload(page: page)) + #expect(payload.rankingSource == .iTunesFallback) + #expect(payload.fallback == provenanceCase.expected) + } + + let responseCases: [( + input: SearchRankingResponseFailure, + expected: EstimatedKeywordDifficultyFallbackResponseFailure + )] = [ + (.serializedServerDataMissing, .serializedServerDataMissing), + (.decodingFailed, .decodingFailed), + (.nonHTTPResponse, .nonHTTPResponse), + (.requestIntentMissing, .requestIntentMissing), + (.requestIntentAmbiguous, .requestIntentAmbiguous), + (.pageShapeChanged, .pageShapeChanged), + (.authoritativeShelfMissing, .authoritativeShelfMissing), + (.authoritativeShelfAmbiguous, .authoritativeShelfAmbiguous), + (.malformedSearchResult, .malformedSearchResult), + (.truncatedResults, .truncatedResults) + ] + + for responseCase in responseCases { + let page = SearchRankingPage( + items: estimatedItems(), + source: .iTunesFallback, + fallbackContext: SearchRankingFailureContext( + provider: .appStoreWeb, + category: .response(responseCase.input) + ) + ) + let payload = try #require(makePayload(page: page)) + #expect(payload.fallback == EstimatedKeywordDifficultyFallbackProvenance( + provider: .appStoreWeb, + category: .response, + responseFailure: responseCase.expected + )) + } + } + + @Test + func directITunesPageKeepsFallbackProvenanceNil() throws { + let page = SearchRankingPage( + items: estimatedItems(), + source: .iTunesFallback, + fallbackContext: nil + ) + + let payload = try #require(makePayload(page: page)) + + #expect(payload.rankingSource == .iTunesFallback) + #expect(payload.fallback == nil) + } + + @Test + func unavailableEstimatorStatesMapWithoutInventingAnEstimate() throws { + let cases: [( + term: String, + items: [SearchRankingItem], + reason: EstimatedKeywordDifficultyUnavailableReason + )] = [ + ("", estimatedItems(), .emptyKeyword), + ("focus timer", Array(estimatedItems().prefix(2)), .insufficientResults), + ( + "focus timer", + estimatedItems().map { item in + rankingItem( + position: item.position, + appStoreID: item.appStoreID, + title: item.name, + subtitle: item.subtitle, + ratingCount: nil + ) + }, + .insufficientRatingEvidence + ) + ] + + for unavailableCase in cases { + let request = makeRequest(term: unavailableCase.term) + let page = SearchRankingPage( + items: unavailableCase.items, + source: .appStoreWeb + ) + let payload = try #require(makePayload(request: request, page: page)) + let unavailable = try requireUnavailable( + KeywordDifficultyEstimator.estimate( + keyword: request.term, + searchResults: page.items + ) + ) + + #expect(payload.result == .unavailable(reason: unavailableCase.reason)) + #expect(payload.algorithmIdentifier == unavailable.algorithmIdentifier) + #expect(payload.algorithmVersion == unavailable.algorithmVersion) + #expect(payload.evidence == persistenceEvidence(unavailable.evidence)) + #expect(payload.notes == unavailable.notes) + } + } + + @Test + func explicitCalculationIdentifierProducesOneStablePayload() throws { + let request = makeRequest() + let page = SearchRankingPage(items: estimatedItems(), source: .appStoreWeb) + let calculationID = UUID(uuidString: "00000000-0000-0000-0000-000000002002")! + let rankingFetchedAt = Date(timeIntervalSince1970: 1_720_000_100) + let computedAt = rankingFetchedAt.addingTimeInterval(1) + + let first = try #require(EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: request, + page: page, + requestedResultLimit: 10, + rankingFetchedAt: rankingFetchedAt, + calculationID: calculationID, + now: { computedAt } + )) + let second = try #require(EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: request, + page: page, + requestedResultLimit: 10, + rankingFetchedAt: rankingFetchedAt, + calculationID: calculationID, + now: { computedAt } + )) + + #expect(first.calculationID == calculationID) + #expect(second.calculationID == calculationID) + #expect(first == second) + } + + @Test + func invalidProvenanceQueryKeysAndLimitsAreRejectedAndClockIsClamped() throws { + let request = makeRequest() + let fetchedAt = Date(timeIntervalSince1970: 1_720_000_200) + let validPage = SearchRankingPage( + items: estimatedItems(), + source: .appStoreWeb + ) + let primaryWithFallback = SearchRankingPage( + items: estimatedItems(), + source: .appStoreWeb, + fallbackContext: SearchRankingFailureContext( + provider: .appStoreWeb, + category: .provider + ) + ) + let fallbackAttributingItself = SearchRankingPage( + items: estimatedItems(), + source: .iTunesFallback, + fallbackContext: SearchRankingFailureContext( + provider: .iTunesFallback, + category: .httpStatus(500) + ) + ) + + #expect(makePayload(request: request, page: primaryWithFallback) == nil) + #expect(makePayload(request: request, page: fallbackAttributingItself) == nil) + var clockCallCount = 0 + let clampedPayload = try #require(EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: request, + page: validPage, + requestedResultLimit: 10, + rankingFetchedAt: fetchedAt, + now: { + clockCallCount += 1 + return fetchedAt.addingTimeInterval(-1) + } + )) + #expect(clockCallCount == 1) + #expect(clampedPayload.computedAt == fetchedAt) + #expect(EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: RankingRefreshRequest( + identityKey: request.identityKey, + queryKey: "different::us::iphone", + term: request.term, + storefront: request.storefront, + platform: request.platform + ), + page: validPage, + requestedResultLimit: 10, + rankingFetchedAt: fetchedAt + ) == nil) + #expect(EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: request, + page: validPage, + requestedResultLimit: 0, + rankingFetchedAt: fetchedAt + ) == nil) + #expect(EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: request, + page: validPage, + requestedResultLimit: EstimatedKeywordDifficultyStore.maximumRequestedResultCount + 1, + rankingFetchedAt: fetchedAt + ) == nil) + #expect(EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: request, + page: validPage, + requestedResultLimit: validPage.resultCount - 1, + rankingFetchedAt: fetchedAt + ) == nil) + } + + @Test + func optionalUpsertRejectsReplacedTrackGeneration() throws { + let container = try ModelContainerFactory.makeModelContainer( + isStoredInMemoryOnly: true + ) + let modelContext = ModelContext(container) + let trackedApp = TrackedApp( + appStoreID: 123, + bundleID: "example.generation", + name: "Generation App", + sellerName: "Example", + defaultPlatform: .iphone + ) + let query = try KeywordQuery.fetchOrInsert( + term: "focus timer", + storefront: "us", + platform: .iphone, + in: modelContext + ) + let original = TrackedAppKeyword( + term: "focus timer", + storefront: "us", + platform: .iphone, + trackedApp: trackedApp, + query: query, + createdAt: Date(timeIntervalSince1970: 100) + ) + trackedApp.keywordTracks.append(original) + modelContext.insert(trackedApp) + modelContext.insert(original) + try modelContext.save() + + let page = SearchRankingPage(items: estimatedItems(), source: .appStoreWeb) + let fetchedAt = Date(timeIntervalSince1970: 200) + let originalRequest = RankingRefreshRequest(track: original) + let payload = try #require(EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: originalRequest, + page: page, + requestedResultLimit: 10, + rankingFetchedAt: fetchedAt, + now: { fetchedAt } + )) + let originalPageResult = RankingRefreshPageResult( + request: originalRequest, + page: page, + searchedAt: fetchedAt, + observedHour: nil, + submissionCount: 1, + winningCount: 1, + confidence: "single_source", + requestedResultLimit: 10, + estimatedDifficultyPayload: payload + ) + + _ = try TrackedKeywordDeletionService.deleteTracks( + [TrackedKeywordDeletionRequest(track: original)], + in: modelContext + ) + let replacement = TrackedAppKeyword( + term: "focus timer", + storefront: "us", + platform: .iphone, + trackedApp: trackedApp, + query: query, + createdAt: Date(timeIntervalSince1970: 300) + ) + trackedApp.keywordTracks.append(replacement) + modelContext.insert(replacement) + try modelContext.save() + + #expect(try EstimatedKeywordDifficultyPersistenceAdapter.upsertIfCurrent( + for: originalPageResult, + in: modelContext + ) == nil) + #expect(try EstimatedKeywordDifficultyStore.snapshot( + queryKey: replacement.queryKey, + in: modelContext + ) == nil) + } +} + +@MainActor +private func makePayload( + request: RankingRefreshRequest = makeRequest(), + page: SearchRankingPage +) -> EstimatedKeywordDifficultyPersistencePayload? { + let rankingFetchedAt = Date(timeIntervalSince1970: 1_720_000_000) + return EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: request, + page: page, + requestedResultLimit: 10, + rankingFetchedAt: rankingFetchedAt, + calculationID: UUID(uuidString: "00000000-0000-0000-0000-000000002000")!, + now: { rankingFetchedAt.addingTimeInterval(1) } + ) +} + +@MainActor +private func makeRequest( + term: String = "focus timer", + storefront: String = "us", + platform: AppPlatform = .iphone +) -> RankingRefreshRequest { + let queryKey = KeywordQuery.makeQueryKey( + term: term, + storefront: storefront, + platform: platform + ) + return RankingRefreshRequest( + identityKey: "123::\(queryKey)", + queryKey: queryKey, + term: term, + storefront: storefront, + platform: platform + ) +} + +@MainActor +private func estimatedItems() -> [SearchRankingItem] { + [ + rankingItem( + position: 1, + appStoreID: 101, + title: " Focus Timer++: Deep Work! ", + subtitle: " Stay focused — privately! ", + ratingCount: 12_000 + ), + rankingItem( + position: 2, + appStoreID: 102, + title: "Deep Focus Clock", + subtitle: "Timer and study sessions", + ratingCount: 8_000 + ), + rankingItem( + position: 3, + appStoreID: 103, + title: "Study Sessions", + subtitle: "Focus timer for work", + ratingCount: 4_000 + ), + rankingItem( + position: 4, + appStoreID: 104, + title: "Quiet Productivity", + subtitle: nil, + ratingCount: 1_000 + ) + ] +} + +@MainActor +private func rankingItem( + position: Int, + appStoreID: Int64, + title: String, + subtitle: String?, + ratingCount: Int? +) -> SearchRankingItem { + SearchRankingItem( + position: position, + appStoreID: appStoreID, + bundleID: "com.example.\(appStoreID)", + name: title, + subtitle: subtitle, + sellerName: "Example Seller", + ratingCount: ratingCount, + platform: .iphone + ) +} + +private func persistenceConfidence( + _ confidence: EstimatedKeywordDifficulty.Confidence +) -> EstimatedKeywordDifficultyConfidence { + switch confidence { + case .low: .low + case .medium: .medium + case .high: .high + } +} + +private func persistenceEvidence( + _ evidence: KeywordDifficultyEvidence +) -> EstimatedKeywordDifficultyEvidence { + EstimatedKeywordDifficultyEvidence( + consideredResultCount: evidence.consideredResultCount, + ratedResultCount: evidence.ratedResultCount, + weightedRatingCoveragePercentage: evidence.weightedRatingCoveragePercentage, + maximumRatingCount: evidence.maximumRatingCount, + medianRatingCount: evidence.medianRatingCount, + ratingAuthorityScore: evidence.ratingAuthorityScore, + metadataSaturationScore: evidence.metadataSaturationScore, + resultEvidence: evidence.resultEvidence.map { result in + EstimatedKeywordDifficultyResultEvidence( + position: result.position, + appStoreID: result.appStoreID, + title: result.title, + subtitle: result.subtitle, + ratingCount: result.ratingCount, + ratingAuthorityScore: result.ratingAuthorityScore, + titleTokenCoveragePercentage: result.titleTokenCoveragePercentage, + combinedTokenCoveragePercentage: result.combinedTokenCoveragePercentage, + metadataMatchScore: result.metadataMatchScore, + exactTitlePhraseMatch: result.exactTitlePhraseMatch, + exactSubtitlePhraseMatch: result.exactSubtitlePhraseMatch + ) + } + ) +} + +private func requireEstimate( + _ estimation: KeywordDifficultyEstimation +) throws -> EstimatedKeywordDifficulty { + guard case .estimated(let estimate) = estimation else { + Issue.record("Expected an estimated keyword difficulty result") + throw AdapterTestError.unexpectedEstimation + } + return estimate +} + +private func requireUnavailable( + _ estimation: KeywordDifficultyEstimation +) throws -> KeywordDifficultyUnavailable { + guard case .unavailable(let unavailable) = estimation else { + Issue.record("Expected an unavailable keyword difficulty result") + throw AdapterTestError.unexpectedEstimation + } + return unavailable +} + +private enum AdapterTestError: Error { + case unexpectedEstimation +} diff --git a/OpenASOTests/EstimatedKeywordDifficultySummaryTests.swift b/OpenASOTests/EstimatedKeywordDifficultySummaryTests.swift new file mode 100644 index 0000000..0515636 --- /dev/null +++ b/OpenASOTests/EstimatedKeywordDifficultySummaryTests.swift @@ -0,0 +1,320 @@ +import Foundation +import SwiftData +import Testing +@testable import OpenASO + +@MainActor +struct EstimatedKeywordDifficultySummaryTests { + @Test + func bulkReadersChooseTheSameLatestRevisionWithoutLeakingOlderEvidence() throws { + let context = try makeContext() + let earlier = makePayload( + calculationID: UUID(uuidString: "00000000-0000-0000-0000-000000000010")!, + score: 41 + ) + let laterTieBreaker = makePayload( + calculationID: UUID(uuidString: "00000000-0000-0000-0000-000000000020")!, + score: 81 + ) + context.insert(makeMetric(from: earlier)) + context.insert(makeMetric(from: laterTieBreaker)) + for result in earlier.evidence.resultEvidence { + context.insert(makeEvidenceRecord(result, payload: earlier)) + } + for result in laterTieBreaker.evidence.resultEvidence { + context.insert(makeEvidenceRecord(result, payload: laterTieBreaker)) + } + try context.save() + + let missingQueryKey = KeywordQuery.makeQueryKey( + term: "not persisted", + storefront: "gb", + platform: .iphone + ) + let summaries = try EstimatedKeywordDifficultyStore.summaries( + queryKeys: [earlier.queryKey, missingQueryKey, earlier.queryKey], + in: context + ) + let snapshots = try EstimatedKeywordDifficultyStore.snapshots( + queryKeys: [earlier.queryKey, missingQueryKey, earlier.queryKey], + in: context + ) + + let summary = try #require(summaries[earlier.queryKey]) + let snapshot = try #require(snapshots[earlier.queryKey]) + #expect(summaries.count == 1) + #expect(snapshots.count == 1) + #expect(summary.calculationID == laterTieBreaker.calculationID) + #expect(snapshot.calculationID == summary.calculationID) + #expect(summary.score == 81) + #expect(snapshot.resultEvidence.map(\.appStoreID) == [20_001, 20_002, 20_003]) + #expect(try EstimatedKeywordDifficultyStore.summary( + queryKey: missingQueryKey, + in: context + ) == nil) + } + + @Test + func presentationKeepsMissingUnavailableAndEstimatedDistinct() throws { + let context = try makeContext() + let fetchedAt = Date(timeIntervalSinceReferenceDate: 800_000_000) + let estimated = makePayload( + calculationID: UUID(uuidString: "00000000-0000-0000-0000-000000000101")!, + rankingFetchedAt: fetchedAt, + computedAt: fetchedAt.addingTimeInterval(60) + ) + let unavailable = makePayload( + calculationID: UUID(uuidString: "00000000-0000-0000-0000-000000000102")!, + keyword: "rare phrase", + result: .unavailable(reason: .insufficientResults), + evidenceCount: 0, + providerResultCount: 0, + rankingFetchedAt: fetchedAt, + computedAt: fetchedAt.addingTimeInterval(86_399) + ) + _ = try EstimatedKeywordDifficultyStore.upsert(estimated, in: context) + _ = try EstimatedKeywordDifficultyStore.upsert(unavailable, in: context) + + let summaries = try EstimatedKeywordDifficultyStore.summaries( + queryKeys: [estimated.queryKey, unavailable.queryKey], + in: context + ) + let beforeBoundary = EstimatedKeywordDifficultyPresentation( + summary: summaries[estimated.queryKey], + asOf: fetchedAt.addingTimeInterval( + EstimatedKeywordDifficultyFreshness.maximumAge - 0.001 + ) + ) + let atBoundary = EstimatedKeywordDifficultyPresentation( + summary: summaries[estimated.queryKey], + asOf: fetchedAt.addingTimeInterval( + EstimatedKeywordDifficultyFreshness.maximumAge + ) + ) + let unavailablePresentation = EstimatedKeywordDifficultyPresentation( + summary: summaries[unavailable.queryKey], + asOf: unavailable.computedAt + ) + let missingPresentation = EstimatedKeywordDifficultyPresentation( + summary: nil, + asOf: fetchedAt + ) + + #expect(beforeBoundary.value == .estimated( + score: 67, + confidenceScore: 83, + confidence: .medium + )) + #expect(!beforeBoundary.isStale) + #expect(atBoundary.isStale) + #expect(unavailablePresentation.value == .unavailable(reason: .insufficientResults)) + #expect(!unavailablePresentation.isStale) + #expect(missingPresentation.value == .missing) + #expect(!missingPresentation.isStale) + #expect(missingPresentation.rankingFetchedAt == nil) + } + + @Test + func directITunesResultRoundTripsWithoutFabricatedFallbackContext() throws { + let context = try makeContext() + let payload = makePayload( + calculationID: UUID(uuidString: "00000000-0000-0000-0000-000000000201")!, + rankingSource: .iTunesFallback, + fallback: nil + ) + + #expect(try EstimatedKeywordDifficultyStore.upsert(payload, in: context) == .inserted) + try context.save() + + let summary = try #require(try EstimatedKeywordDifficultyStore.summary( + queryKey: payload.queryKey, + in: context + )) + let snapshot = try #require(try EstimatedKeywordDifficultyStore.snapshot( + queryKey: payload.queryKey, + in: context + )) + #expect(summary.rankingSource == .iTunesFallback) + #expect(summary.fallbackProvider == nil) + #expect(summary.fallbackCategory == nil) + #expect(snapshot.rankingSource == .iTunesFallback) + #expect(snapshot.fallbackProviderRaw == nil) + #expect(snapshot.fallbackCategoryRaw == nil) + } + + private func makeContext() throws -> ModelContext { + let container = try ModelContainerFactory.makeModelContainer(isStoredInMemoryOnly: true) + return ModelContext(container) + } + + private func makePayload( + calculationID: UUID, + keyword: String = "focus timer", + score: Int = 67, + result: EstimatedKeywordDifficultyPersistenceResult? = nil, + evidenceCount: Int = 3, + providerResultCount: Int = 3, + rankingSource: RankingSource = .appStoreWeb, + rankingFetchedAt: Date = Date(timeIntervalSinceReferenceDate: 800_000_000), + computedAt: Date = Date(timeIntervalSinceReferenceDate: 800_000_060), + fallback: EstimatedKeywordDifficultyFallbackProvenance? = nil + ) -> EstimatedKeywordDifficultyPersistencePayload { + let storefront = "gb" + let platform = AppPlatform.iphone + return EstimatedKeywordDifficultyPersistencePayload( + queryKey: KeywordQuery.makeQueryKey( + term: keyword, + storefront: storefront, + platform: platform + ), + calculationID: calculationID, + keyword: keyword, + storefront: storefront, + platform: platform, + result: result ?? .estimated( + score: score, + confidenceScore: 83, + confidence: .medium + ), + algorithmIdentifier: "top10-authority-saturation", + algorithmVersion: 1, + requestedResultLimit: 10, + providerResultCount: providerResultCount, + evidence: makeEvidence( + count: evidenceCount, + keyword: keyword, + appStoreIDBase: calculationID.uuidString.hasSuffix("10") ? 10_000 : 20_000 + ), + rankingSource: rankingSource, + rankingFetchedAt: rankingFetchedAt, + computedAt: computedAt, + fallback: fallback, + notes: ["Estimated locally from public ranking evidence."] + ) + } + + private func makeEvidence( + count: Int, + keyword: String, + appStoreIDBase: Int64 + ) -> EstimatedKeywordDifficultyEvidence { + let positions = count > 0 ? Array(1 ... count) : [] + let results = positions.map { position in + EstimatedKeywordDifficultyResultEvidence( + position: position, + appStoreID: appStoreIDBase + Int64(position), + title: "\(keyword) result \(position)", + subtitle: "Subtitle \(position)", + ratingCount: position * 100, + ratingAuthorityScore: 40 + position, + titleTokenCoveragePercentage: 80 + position, + combinedTokenCoveragePercentage: 85 + position, + metadataMatchScore: 70 + position, + exactTitlePhraseMatch: position == 1, + exactSubtitlePhraseMatch: position == 2 + ) + } + let ratings = results.compactMap(\.ratingCount).sorted() + return EstimatedKeywordDifficultyEvidence( + consideredResultCount: results.count, + ratedResultCount: ratings.count, + weightedRatingCoveragePercentage: results.isEmpty ? 0 : 100, + maximumRatingCount: ratings.last, + medianRatingCount: median(ratings), + ratingAuthorityScore: results.isEmpty ? nil : 62, + metadataSaturationScore: results.isEmpty ? nil : 71, + resultEvidence: results + ) + } + + private func median(_ values: [Int]) -> Int? { + guard !values.isEmpty else { return nil } + let middle = values.count / 2 + guard values.count.isMultiple(of: 2) else { return values[middle] } + return values[middle - 1] + (values[middle] - values[middle - 1]) / 2 + } + + private func makeMetric( + from payload: EstimatedKeywordDifficultyPersistencePayload + ) -> EstimatedKeywordDifficultyMetric { + let stateRaw: String + let score: Int? + let confidenceScore: Int? + let confidenceRaw: String? + let unavailableReasonRaw: String? + switch payload.result { + case .estimated(let value, let confidenceValue, let confidence): + stateRaw = EstimatedKeywordDifficultyState.estimated.rawValue + score = value + confidenceScore = confidenceValue + confidenceRaw = confidence.rawValue + unavailableReasonRaw = nil + case .unavailable(let reason): + stateRaw = EstimatedKeywordDifficultyState.unavailable.rawValue + score = nil + confidenceScore = nil + confidenceRaw = nil + unavailableReasonRaw = reason.rawValue + } + return EstimatedKeywordDifficultyMetric( + queryKey: payload.queryKey, + calculationID: payload.calculationID, + keyword: payload.keyword, + storefront: payload.storefront, + platformRaw: payload.platform.rawValue, + stateRaw: stateRaw, + score: score, + confidenceScore: confidenceScore, + confidenceRaw: confidenceRaw, + unavailableReasonRaw: unavailableReasonRaw, + estimationSourceRaw: payload.estimationSource.rawValue, + algorithmIdentifier: payload.algorithmIdentifier, + algorithmVersion: payload.algorithmVersion, + requestedResultLimit: payload.requestedResultLimit, + providerResultCount: payload.providerResultCount, + consideredResultCount: payload.evidence.consideredResultCount, + ratedResultCount: payload.evidence.ratedResultCount, + weightedRatingCoveragePercentage: payload.evidence.weightedRatingCoveragePercentage, + maximumRatingCount: payload.evidence.maximumRatingCount, + medianRatingCount: payload.evidence.medianRatingCount, + ratingAuthorityScore: payload.evidence.ratingAuthorityScore, + metadataSaturationScore: payload.evidence.metadataSaturationScore, + exactTitlePhraseMatchCount: payload.evidence.resultEvidence.count( + where: \.exactTitlePhraseMatch + ), + exactSubtitlePhraseMatchCount: payload.evidence.resultEvidence.count( + where: \.exactSubtitlePhraseMatch + ), + rankingSourceRaw: payload.rankingSource.rawValue, + rankingFetchedAt: payload.rankingFetchedAt, + computedAt: payload.computedAt, + fallbackProviderRaw: payload.fallback?.provider.rawValue, + fallbackCategoryRaw: payload.fallback?.category.rawValue, + fallbackTransportCode: payload.fallback?.transportCode, + fallbackHTTPStatus: payload.fallback?.httpStatus, + fallbackResponseFailureRaw: payload.fallback?.responseFailure?.rawValue, + notes: payload.notes + ) + } + + private func makeEvidenceRecord( + _ result: EstimatedKeywordDifficultyResultEvidence, + payload: EstimatedKeywordDifficultyPersistencePayload + ) -> EstimatedKeywordDifficultyResultEvidenceRecord { + EstimatedKeywordDifficultyResultEvidenceRecord( + queryKey: payload.queryKey, + calculationID: payload.calculationID, + position: result.position, + appStoreID: result.appStoreID, + title: result.title, + subtitle: result.subtitle, + ratingCount: result.ratingCount, + ratingAuthorityScore: result.ratingAuthorityScore, + titleTokenCoveragePercentage: result.titleTokenCoveragePercentage, + combinedTokenCoveragePercentage: result.combinedTokenCoveragePercentage, + metadataMatchScore: result.metadataMatchScore, + exactTitlePhraseMatch: result.exactTitlePhraseMatch, + exactSubtitlePhraseMatch: result.exactSubtitlePhraseMatch + ) + } +} diff --git a/OpenASOTests/EstimatedKeywordDifficultyWorkspaceUITests.swift b/OpenASOTests/EstimatedKeywordDifficultyWorkspaceUITests.swift new file mode 100644 index 0000000..2c3d075 --- /dev/null +++ b/OpenASOTests/EstimatedKeywordDifficultyWorkspaceUITests.swift @@ -0,0 +1,262 @@ +import Foundation +import SwiftData +import Testing +@testable import OpenASO + +@MainActor +struct EstimatedKeywordDifficultyWorkspaceUITests { + @Test + func defaultFilterPreservesEveryStateAndNarrowedFilterUsesOnlyEstimatedScore() { + let fetchedAt = Date(timeIntervalSinceReferenceDate: 800_000_000) + let missing = makeRow( + term: "missing", + appStoreID: 1, + legacyDifficulty: 60, + estimatedDifficulty: nil + ) + let unavailable = makeRow( + term: "unavailable", + appStoreID: 2, + legacyDifficulty: 60, + estimatedDifficulty: summary( + keyword: "unavailable", + state: .unavailable, + score: nil, + confidenceScore: nil, + confidence: nil, + unavailableReason: .insufficientResults, + rankingFetchedAt: fetchedAt + ) + ) + let estimated = makeRow( + term: "estimated", + appStoreID: 3, + legacyDifficulty: 5, + estimatedDifficulty: summary( + keyword: "estimated", + state: .estimated, + score: 60, + confidenceScore: 80, + confidence: .high, + unavailableReason: nil, + rankingFetchedAt: fetchedAt + ) + ) + + let defaultRows = KeywordWorkspaceProjection.filteredRows( + [missing, unavailable, estimated], + filters: filters(difficultyRange: MetricFilterRange.difficulty.defaultRange) + ) + #expect(defaultRows.map { $0.track.term } == ["missing", "unavailable", "estimated"]) + + let narrowedRows = KeywordWorkspaceProjection.filteredRows( + [missing, unavailable, estimated], + filters: filters(difficultyRange: 50 ... 70) + ) + #expect(narrowedRows.map { $0.track.term } == ["estimated"]) + #expect(estimated.estimatedDifficultySortValue == 60) + #expect(missing.estimatedDifficultySortValue == Int.max) + } + + @Test + func rowPresentationDistinguishesMissingFreshStaleAndUnavailable() { + let fetchedAt = Date(timeIntervalSinceReferenceDate: 800_000_000) + let estimated = summary( + keyword: "focus", + state: .estimated, + score: 74, + confidenceScore: 88, + confidence: .high, + unavailableReason: nil, + rankingFetchedAt: fetchedAt + ) + let unavailable = summary( + keyword: "rare", + state: .unavailable, + score: nil, + confidenceScore: nil, + confidence: nil, + unavailableReason: .insufficientRatingEvidence, + rankingFetchedAt: fetchedAt + ) + + #expect(EstimatedKeywordDifficultyRowPresentation( + summary: nil, + asOf: fetchedAt + ).state == .missing) + #expect(EstimatedKeywordDifficultyRowPresentation( + summary: estimated, + asOf: fetchedAt.addingTimeInterval( + EstimatedKeywordDifficultyFreshness.maximumAge - 1 + ) + ).state == .estimated( + score: 74, + confidenceScore: 88, + confidence: .high, + isStale: false + )) + #expect(EstimatedKeywordDifficultyRowPresentation( + summary: estimated, + asOf: fetchedAt.addingTimeInterval( + EstimatedKeywordDifficultyFreshness.maximumAge + ) + ).state == .estimated( + score: 74, + confidenceScore: 88, + confidence: .high, + isStale: true + )) + #expect(EstimatedKeywordDifficultyRowPresentation( + summary: unavailable, + asOf: fetchedAt + ).state == .unavailable( + reason: .insufficientRatingEvidence, + isStale: false + )) + } + + @Test + func legacyMetricsSnapshotMapLoadsMultipleQueryKeys() throws { + let container = try ModelContainerFactory.makeModelContainer(isStoredInMemoryOnly: true) + let context = ModelContext(container) + let firstKey = KeywordQuery.makeQueryKey( + term: "first", + storefront: "us", + platform: .iphone + ) + let secondKey = KeywordQuery.makeQueryKey( + term: "second", + storefront: "gb", + platform: .iphone + ) + context.insert(KeywordDailyMetric( + queryKey: firstKey, + keyword: "first", + storefront: "us", + platform: .iphone, + popularityScore: 11, + difficultyScore: 91, + source: .appleAdsPopularity + )) + context.insert(KeywordDailyMetric( + queryKey: secondKey, + keyword: "second", + storefront: "gb", + platform: .iphone, + popularityScore: 22, + difficultyScore: 82, + source: .appleAdsPopularity + )) + try context.save() + + let snapshots = try KeywordMetricsSnapshot.map( + for: [firstKey, secondKey, firstKey, "missing"], + in: context + ) + + #expect(snapshots.count == 2) + #expect(snapshots[firstKey]?.popularityScore == 11) + #expect(snapshots[secondKey]?.popularityScore == 22) + } + + private func filters( + difficultyRange: ClosedRange + ) -> KeywordWorkspaceProjection.Filters { + KeywordWorkspaceProjection.Filters( + searchText: "", + popularityRange: MetricFilterRange.popularity.defaultRange, + difficultyRange: difficultyRange, + positionRange: MetricFilterRange.position.defaultRange, + changeRange: MetricFilterRange.change.defaultRange, + showsOnlyChangedKeywords: false + ) + } + + private func makeRow( + term: String, + appStoreID: Int64, + legacyDifficulty: Int?, + estimatedDifficulty: EstimatedKeywordDifficultySummary? + ) -> KeywordWorkspaceRow { + let trackedApp = TrackedApp( + appStoreID: appStoreID, + bundleID: "com.example.\(appStoreID)", + name: "App \(appStoreID)", + sellerName: "Example", + defaultPlatform: .iphone + ) + let query = KeywordQuery(term: term, storefront: "us", platform: .iphone) + let track = TrackedAppKeyword( + term: term, + storefront: "us", + platform: .iphone, + trackedApp: trackedApp, + query: query + ) + return KeywordWorkspaceRow( + track: track, + storefront: nil, + metrics: KeywordMetricsSnapshot( + popularityScore: nil, + difficultyScore: legacyDifficulty, + updatedAt: .now, + notes: nil + ), + estimatedDifficulty: estimatedDifficulty, + latestSnapshot: nil, + trendSnapshots: [], + rankingApps: [] + ) + } + + private func summary( + keyword: String, + state: EstimatedKeywordDifficultyState, + score: Int?, + confidenceScore: Int?, + confidence: EstimatedKeywordDifficultyConfidence?, + unavailableReason: EstimatedKeywordDifficultyUnavailableReason?, + rankingFetchedAt: Date + ) -> EstimatedKeywordDifficultySummary { + let queryKey = KeywordQuery.makeQueryKey( + term: keyword, + storefront: "us", + platform: .iphone + ) + return EstimatedKeywordDifficultySummary( + queryKey: queryKey, + calculationID: UUID(), + keyword: keyword, + storefront: "us", + platformRaw: AppPlatform.iphone.rawValue, + stateRaw: state.rawValue, + score: score, + confidenceScore: confidenceScore, + confidenceRaw: confidence?.rawValue, + unavailableReasonRaw: unavailableReason?.rawValue, + estimationSourceRaw: EstimatedKeywordDifficultySource.topResultsHeuristic.rawValue, + algorithmIdentifier: "top10-authority-saturation", + algorithmVersion: 1, + requestedResultLimit: 10, + providerResultCount: 10, + consideredResultCount: 10, + ratedResultCount: 8, + weightedRatingCoveragePercentage: 80, + maximumRatingCount: 10_000, + medianRatingCount: 500, + ratingAuthorityScore: 70, + metadataSaturationScore: 60, + exactTitlePhraseMatchCount: 2, + exactSubtitlePhraseMatchCount: 1, + rankingSourceRaw: RankingSource.appStoreWeb.rawValue, + rankingFetchedAt: rankingFetchedAt, + computedAt: rankingFetchedAt, + fallbackProviderRaw: nil, + fallbackCategoryRaw: nil, + fallbackTransportCode: nil, + fallbackHTTPStatus: nil, + fallbackResponseFailureRaw: nil, + notes: [] + ) + } +} diff --git a/OpenASOTests/ITunesSearchFallbackProviderTests.swift b/OpenASOTests/ITunesSearchFallbackProviderTests.swift index 2db3b31..0cbe8d8 100644 --- a/OpenASOTests/ITunesSearchFallbackProviderTests.swift +++ b/OpenASOTests/ITunesSearchFallbackProviderTests.swift @@ -8,6 +8,10 @@ struct ITunesSearchFallbackProviderTests { func returnsOrderedResultsFromITunesSearchAPI() async throws { let client = MockHTTPClient { request in #expect(request.url?.absoluteString.contains("itunes.apple.com/search") == true) + #expect(URLComponents( + url: try #require(request.url), + resolvingAgainstBaseURL: false + )?.queryItems?.first(where: { $0.name == "entity" })?.value == "software") let payload = """ { @@ -61,6 +65,49 @@ struct ITunesSearchFallbackProviderTests { #expect(page.items[0].ipadScreenshotURLs == ["https://example.com/pages-ipad-1.png"]) } + @Test + func usesTheDocumentedEntityForEachRequestedPlatform() async throws { + let scenarios: [(platform: AppPlatform, entity: String)] = [ + (.iphone, "software"), + (.ipad, "iPadSoftware"), + (.mac, "macSoftware"), + ] + + for scenario in scenarios { + let client = MockHTTPClient { request in + let url = try #require(request.url) + #expect(url.absoluteString == "https://itunes.apple.com/search?term=notes&entity=\(scenario.entity)&country=gb&limit=7") + let payload = """ + { + "results": [ + { + "trackId": 123, + "bundleId": "com.example.notes", + "trackName": "Notes", + "sellerName": "Example" + } + ] + } + """ + return ( + Data(payload.utf8), + makeHTTPURLResponse(url: url, statusCode: 200) + ) + } + let provider = ITunesSearchFallbackProvider(httpClient: client) + + let page = try await provider.search( + keyword: " notes ", + storefrontCode: " GB ", + platform: scenario.platform, + limit: 7 + ) + + #expect(page.source == .iTunesFallback) + #expect(page.items.map(\.platform) == [scenario.platform]) + } + } + @Test func appIconStoreRequestsMZStaticArtworkAtRenderedPixelSize() { let apiURL = "https://is1-ssl.mzstatic.com/image/thumb/Purple211/v4/3f/85/8b/icon.png/100x100bb.jpg" diff --git a/OpenASOTests/KeywordDifficultyEstimatorTests.swift b/OpenASOTests/KeywordDifficultyEstimatorTests.swift new file mode 100644 index 0000000..e805413 --- /dev/null +++ b/OpenASOTests/KeywordDifficultyEstimatorTests.swift @@ -0,0 +1,593 @@ +import Testing +@testable import OpenASO + +struct KeywordDifficultyEstimatorTests { + @Test + func rejectsEmptyKeywordWithoutInventingMetadataEvidence() throws { + let unavailable = try requireUnavailable(KeywordDifficultyEstimator.estimate( + keyword: " -- ", + rankedResults: makeResults(count: 10, ratingCount: 1_000) + )) + + #expect(unavailable.reason == .emptyKeyword) + #expect(unavailable.evidence.consideredResultCount == 10) + #expect(unavailable.evidence.ratedResultCount == 10) + #expect(unavailable.evidence.metadataSaturationScore == nil) + #expect(unavailable.evidence.resultEvidence.allSatisfy { $0.metadataMatchScore == nil }) + #expect(unavailable.notes.first?.contains("not Apple Ads") == true) + } + + @Test + func requiresThreeUniqueTopTenResults() throws { + let unavailable = try requireUnavailable(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: [ + makeResult(position: 0, appStoreID: 1, ratingCount: 1_000), + makeResult(position: 1, appStoreID: 2, ratingCount: 1_000), + makeResult(position: 2, appStoreID: 2, ratingCount: 1_000), + makeResult(position: 3, appStoreID: 3, ratingCount: 1_000), + makeResult(position: 11, appStoreID: 4, ratingCount: 1_000) + ] + )) + + #expect(unavailable.reason == .insufficientResults) + #expect(unavailable.evidence.consideredResultCount == 2) + #expect(unavailable.evidence.resultEvidence.map(\.appStoreID) == [2, 3]) + } + + @Test + func distinguishesMissingAndInvalidRatingsFromRealZeroes() throws { + let unavailable = try requireUnavailable(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: [ + makeResult(position: 1, appStoreID: 1, ratingCount: 0), + makeResult(position: 2, appStoreID: 2, ratingCount: 0), + makeResult(position: 3, appStoreID: 3, ratingCount: nil), + makeResult(position: 4, appStoreID: 4, ratingCount: -1), + makeResult(position: 5, appStoreID: 5, ratingCount: nil) + ] + )) + + #expect(unavailable.reason == .insufficientRatingEvidence) + #expect(unavailable.evidence.ratedResultCount == 2) + #expect(unavailable.evidence.ratingCoveragePercentage == 40) + #expect(unavailable.evidence.maximumRatingCount == 0) + #expect(unavailable.evidence.resultEvidence[2].ratingCount == nil) + #expect(unavailable.evidence.resultEvidence[3].ratingCount == nil) + + let estimate = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: makeResults(count: 3, ratingCount: 0) + )) + #expect(estimate.evidence.ratedResultCount == 3) + #expect(estimate.evidence.ratingAuthorityScore == 0) + } + + @Test + func goldenUnmatchedFixtureUsesSeventyPercentRatingAuthority() throws { + let estimate = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: makeResults(count: 10, ratingCount: 999) + )) + + #expect(estimate.score == 35) + #expect(estimate.evidence.ratingAuthorityScore == 50) + #expect(estimate.evidence.metadataSaturationScore == 0) + #expect(estimate.confidenceScore == 100) + #expect(estimate.confidence == .high) + } + + @Test + func goldenExactTitleFixtureAddsThirtyPercentMetadataSaturation() throws { + let estimate = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: makeResults( + count: 10, + ratingCount: 999, + title: "Focus Timer" + ) + )) + + #expect(estimate.score == 65) + #expect(estimate.evidence.ratingAuthorityScore == 50) + #expect(estimate.evidence.metadataSaturationScore == 100) + #expect(estimate.evidence.exactTitlePhraseMatchCount == 10) + #expect(estimate.evidence.exactSubtitlePhraseMatchCount == 0) + } + + @Test + func strongestEvidenceIsBoundedAtOneHundredWithoutOverflow() throws { + let estimate = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: makeResults( + count: 10, + ratingCount: Int.max, + title: "Focus Timer" + ) + )) + + #expect(estimate.score == 100) + #expect(estimate.evidence.ratingAuthorityScore == 100) + #expect(estimate.evidence.metadataSaturationScore == 100) + #expect(estimate.evidence.maximumRatingCount == Int.max) + } + + @Test + func higherRankedAuthorityHasMoreWeight() throws { + var topHeavy = makeResults(count: 10, ratingCount: 0) + topHeavy[0] = makeResult(position: 1, appStoreID: 1, ratingCount: 99_999) + + var bottomHeavy = makeResults(count: 10, ratingCount: 0) + bottomHeavy[9] = makeResult(position: 10, appStoreID: 10, ratingCount: 99_999) + + let topEstimate = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: topHeavy + )) + let bottomEstimate = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: bottomHeavy + )) + + #expect((topEstimate.evidence.ratingAuthorityScore ?? 0) > (bottomEstimate.evidence.ratingAuthorityScore ?? 0)) + #expect(topEstimate.score > bottomEstimate.score) + } + + @Test + func mixedRankGoldenWeightsPositionExactlyOnce() throws { + let estimate = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: [ + makeResult(position: 1, appStoreID: 1, ratingCount: 999_999), + makeResult(position: 2, appStoreID: 2, ratingCount: 0), + makeResult(position: 3, appStoreID: 3, ratingCount: 0) + ] + )) + + #expect(estimate.evidence.ratingAuthorityScore == 37) + #expect(estimate.evidence.metadataSaturationScore == 0) + #expect(estimate.score == 26) + } + + @Test + func ratingAuthorityAndMetadataCoverageAreMonotonic() throws { + let noAuthority = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: makeResults(count: 3, ratingCount: 0) + )) + let mediumAuthority = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: makeResults(count: 3, ratingCount: 999) + )) + let highAuthority = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: makeResults(count: 3, ratingCount: 999_999) + )) + let partialMetadata = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: makeResults( + count: 3, + ratingCount: 0, + title: "Focus Work" + ) + )) + let completeMetadata = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: makeResults( + count: 3, + ratingCount: 0, + title: "Focus Timer" + ) + )) + + #expect((noAuthority.evidence.ratingAuthorityScore ?? 0) < (mediumAuthority.evidence.ratingAuthorityScore ?? 0)) + #expect((mediumAuthority.evidence.ratingAuthorityScore ?? 0) < (highAuthority.evidence.ratingAuthorityScore ?? 0)) + #expect((partialMetadata.evidence.metadataSaturationScore ?? 0) < (completeMetadata.evidence.metadataSaturationScore ?? 0)) + #expect(partialMetadata.score < completeMetadata.score) + } + + @Test + func confidenceTracksSampleAndRatingCompleteness() throws { + let low = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: makeResults(count: 3, ratingCount: 999) + )) + let medium = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: makeResults(count: 5, ratingCount: 999) + )) + let high = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: makeResults(count: 8, ratingCount: 999) + )) + + #expect(low.confidenceScore == 55) + #expect(low.confidence == .low) + #expect(medium.confidenceScore == 71) + #expect(medium.confidence == .medium) + #expect(high.confidenceScore == 89) + #expect(high.confidence == .high) + } + + @Test + func missingTopRankedRatingsLowersConfidenceMoreThanMissingBottomRatings() throws { + let missingTop = (1...10).map { position in + makeResult( + position: position, + appStoreID: Int64(position), + ratingCount: position <= 2 ? nil : 999 + ) + } + let missingBottom = (1...10).map { position in + makeResult( + position: position, + appStoreID: Int64(position), + ratingCount: position >= 9 ? nil : 999 + ) + } + + let topEstimate = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: missingTop + )) + let bottomEstimate = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: missingBottom + )) + + #expect(topEstimate.evidence.ratingCoveragePercentage == 80) + #expect(bottomEstimate.evidence.ratingCoveragePercentage == 80) + #expect(topEstimate.evidence.weightedRatingCoveragePercentage < bottomEstimate.evidence.weightedRatingCoveragePercentage) + #expect(topEstimate.confidenceScore < bottomEstimate.confidenceScore) + #expect(topEstimate.confidence == .medium) + #expect(bottomEstimate.confidence == .high) + } + + @Test + func exactSubtitlePhraseContributesCombinedMetadataCoverage() throws { + let estimate = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: makeResults( + count: 3, + ratingCount: 0, + title: "Generic", + subtitle: "A Focus Timer" + ) + )) + + #expect(estimate.evidence.metadataSaturationScore == 75) + #expect(estimate.evidence.exactTitlePhraseMatchCount == 0) + #expect(estimate.evidence.exactSubtitlePhraseMatchCount == 3) + #expect(estimate.score == 23) + } + + @Test + func dispersedTokensReceivePartialMetadataCredit() throws { + let estimate = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: makeResults( + count: 3, + ratingCount: 0, + title: "Focus Work", + subtitle: "A simple timer" + ) + )) + + #expect(estimate.evidence.metadataSaturationScore == 75) + #expect(estimate.evidence.exactTitlePhraseMatchCount == 0) + #expect(estimate.evidence.exactSubtitlePhraseMatchCount == 0) + #expect(estimate.score == 23) + } + + @Test + func matchingIsCaseDiacriticAndPunctuationInsensitiveButNotSubstringBased() throws { + let matching = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "FOCUS timer", + rankedResults: makeResults( + count: 3, + ratingCount: 0, + title: "Generic", + subtitle: "Fócus—Timer for Work" + ) + )) + let substring = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "car", + rankedResults: makeResults( + count: 3, + ratingCount: 0, + title: "Cart Planner" + ) + )) + + #expect(matching.evidence.metadataSaturationScore == 75) + #expect(matching.evidence.exactSubtitlePhraseMatchCount == 3) + #expect(substring.evidence.metadataSaturationScore == 0) + #expect(substring.evidence.exactTitlePhraseMatchCount == 0) + } + + @Test + func orderingDeduplicationAndTopTenBoundaryAreDeterministic() throws { + var results = makeResults(count: 10, ratingCount: 0) + results.append(makeResult( + position: 2, + appStoreID: 2, + title: "Focus Timer", + ratingCount: 99_999 + )) + results.append(makeResult( + position: 10, + appStoreID: 99, + title: "Focus Timer", + ratingCount: 99_999 + )) + results.append(makeResult( + position: 11, + appStoreID: 11, + title: "Focus Timer", + ratingCount: 99_999 + )) + + let forward = KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: results + ) + let reversed = KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: Array(results.reversed()) + ) + let estimate = try requireEstimate(forward) + + #expect(forward == reversed) + #expect(estimate.evidence.consideredResultCount == 10) + #expect(estimate.evidence.resultEvidence.map(\.appStoreID).contains(11) == false) + #expect(estimate.evidence.resultEvidence.filter { $0.appStoreID == 2 }.count == 1) + #expect(estimate.evidence.resultEvidence.last?.appStoreID == 10) + } + + @Test + func differentAppsSharingProviderPositionsRemainOrderedByPositionThenAppID() throws { + let results = [ + makeResult( + position: 2, + appStoreID: 40, + title: "Position Two App Forty", + subtitle: nil, + ratingCount: 0 + ), + makeResult( + position: 1, + appStoreID: 20, + title: "Position One App Twenty", + subtitle: "Twenty subtitle", + ratingCount: 0 + ), + makeResult( + position: 3, + appStoreID: 50, + title: "Position Three App Fifty", + subtitle: nil, + ratingCount: 0 + ), + makeResult( + position: 1, + appStoreID: 10, + title: "Position One App Ten", + subtitle: "Ten subtitle", + ratingCount: 0 + ), + makeResult( + position: 2, + appStoreID: 30, + title: "Position Two App Thirty", + subtitle: "Thirty subtitle", + ratingCount: 0 + ) + ] + + let forward = KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: results + ) + let reversed = KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: Array(results.reversed()) + ) + let estimate = try requireEstimate(forward) + + #expect(forward == reversed) + #expect(estimate.evidence.resultEvidence.map(\.position) == [1, 1, 2, 2, 3]) + #expect(estimate.evidence.resultEvidence.map(\.appStoreID) == [10, 20, 30, 40, 50]) + #expect(estimate.evidence.resultEvidence.map(\.title) == [ + "Position One App Ten", + "Position One App Twenty", + "Position Two App Thirty", + "Position Two App Forty", + "Position Three App Fifty" + ]) + #expect(estimate.evidence.resultEvidence.map(\.subtitle) == [ + "Ten subtitle", + "Twenty subtitle", + "Thirty subtitle", + nil, + nil + ]) + } + + @Test + func duplicateConflictPrefersValidRatingEvidenceDeterministically() throws { + let duplicates = [ + makeResult( + position: 1, + appStoreID: 1, + title: "A Missing Result", + subtitle: nil, + ratingCount: nil + ), + makeResult( + position: 1, + appStoreID: 1, + title: " Z Valid Result ™ ", + subtitle: " Different—subtitle ", + ratingCount: 99_999 + ), + makeResult( + position: 2, + appStoreID: 1, + title: "Later Duplicate Rank", + subtitle: "Must not be selected", + ratingCount: 999_999 + ), + makeResult(position: 2, appStoreID: 2, ratingCount: 0), + makeResult(position: 3, appStoreID: 3, ratingCount: 0) + ] + + let forward = KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: duplicates + ) + let reversed = KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: Array(duplicates.reversed()) + ) + + #expect(forward == reversed) + let estimate = try requireEstimate(forward) + #expect(estimate.evidence.ratedResultCount == 3) + #expect(estimate.evidence.resultEvidence.map(\.appStoreID) == [1, 2, 3]) + #expect(estimate.evidence.resultEvidence.first?.position == 1) + #expect(estimate.evidence.resultEvidence.first?.title == " Z Valid Result ™ ") + #expect(estimate.evidence.resultEvidence.first?.subtitle == " Different—subtitle ") + #expect(estimate.evidence.resultEvidence.first?.ratingCount == 99_999) + } + + @Test + func reportsConventionalOddAndEvenRatingMedians() throws { + let odd = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: [ + makeResult(position: 1, appStoreID: 1, ratingCount: 0), + makeResult(position: 2, appStoreID: 2, ratingCount: 50_000), + makeResult(position: 3, appStoreID: 3, ratingCount: 100_000) + ] + )) + let even = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: [ + makeResult(position: 1, appStoreID: 1, ratingCount: 0), + makeResult(position: 2, appStoreID: 2, ratingCount: 0), + makeResult(position: 3, appStoreID: 3, ratingCount: 100_000), + makeResult(position: 4, appStoreID: 4, ratingCount: 100_000) + ] + )) + + #expect(odd.evidence.medianRatingCount == 50_000) + #expect(even.evidence.medianRatingCount == 50_000) + } + + @Test + func searchResultAdapterIgnoresExcludedRankingFields() { + let rankedResults = makeResults( + count: 3, + ratingCount: 999, + title: "Focus Timer" + ) + let searchResults = rankedResults.map { result in + SearchRankingItem( + position: result.position, + appStoreID: result.appStoreID, + bundleID: "com.example.\(result.appStoreID)", + name: result.title, + subtitle: result.subtitle, + sellerName: "Example", + descriptionText: "Excluded evidence", + screenshotURLs: ["https://example.com/screenshot.png"], + ratingCount: result.ratingCount, + averageRating: Double(result.position), + platform: .ipad + ) + } + + #expect( + KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: rankedResults + ) == KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + searchResults: searchResults + ) + ) + } + + @Test + func exposesStableVersionPerResultEvidenceAndHonestNotes() throws { + let estimate = try requireEstimate(KeywordDifficultyEstimator.estimate( + keyword: "focus timer", + rankedResults: makeResults(count: 10, ratingCount: 999) + )) + + #expect(estimate.algorithmIdentifier == "top10-authority-saturation") + #expect(estimate.algorithmVersion == 1) + #expect(estimate.evidence.resultEvidence.count == 10) + #expect(estimate.notes.contains { $0.contains("Heuristic estimated competition") }) + #expect(estimate.notes.contains { $0.contains("70%") && $0.contains("30%") }) + #expect(estimate.notes.contains { $0.contains("not Apple Ads difficulty") }) + #expect(estimate.notes.contains { $0.contains("confidence high") }) + } +} + +private enum KeywordDifficultyEstimatorTestError: Error { + case expectedEstimate + case expectedUnavailable +} + +private func requireEstimate( + _ result: KeywordDifficultyEstimation +) throws -> EstimatedKeywordDifficulty { + guard case .estimated(let estimate) = result else { + throw KeywordDifficultyEstimatorTestError.expectedEstimate + } + return estimate +} + +private func requireUnavailable( + _ result: KeywordDifficultyEstimation +) throws -> KeywordDifficultyUnavailable { + guard case .unavailable(let unavailable) = result else { + throw KeywordDifficultyEstimatorTestError.expectedUnavailable + } + return unavailable +} + +private func makeResults( + count: Int, + ratingCount: Int?, + title: String = "Generic App", + subtitle: String? = nil +) -> [KeywordDifficultyRankedResult] { + (0.. KeywordDifficultyRankedResult { + KeywordDifficultyRankedResult( + position: position, + appStoreID: appStoreID, + title: title, + subtitle: subtitle, + ratingCount: ratingCount + ) +} diff --git a/OpenASOTests/KeywordRankingHistoryTests.swift b/OpenASOTests/KeywordRankingHistoryTests.swift index 92651f7..b0c3ed1 100644 --- a/OpenASOTests/KeywordRankingHistoryTests.swift +++ b/OpenASOTests/KeywordRankingHistoryTests.swift @@ -424,6 +424,47 @@ struct KeywordRankingHistoryTests { #expect(makeWorkspaceRow(ranks: [5, 5]).trendAccessibilityText == "Ranking unchanged.") } + @Test + func unknownLegacyRankingSourcesDefaultToConservativeFallback() { + let trackedApp = TrackedApp( + appStoreID: 100, + bundleID: "com.example.app", + name: "Example", + sellerName: "Example", + defaultPlatform: .iphone + ) + let query = KeywordQuery(term: "weather", storefront: "us", platform: .iphone) + let track = TrackedAppKeyword( + term: query.term, + storefront: query.storefront, + platform: query.platform, + trackedApp: trackedApp, + query: query + ) + let snapshot = TrackedKeywordDailyRanking( + rank: 3, + searchedAt: .now, + source: .appStoreWeb, + resultCount: 10, + keywordTrack: track + ) + let crawl = KeywordRankingCrawl( + keyword: query.term, + storefront: query.storefront, + platform: query.platform, + observedAt: .now, + source: .appStoreWeb, + resultCount: 10, + query: query + ) + + snapshot.sourceRaw = "legacy-unknown-source" + crawl.sourceRaw = "legacy-unknown-source" + + #expect(snapshot.source == .iTunesFallback) + #expect(crawl.source == .iTunesFallback) + } + private func makeDataSource( controlledBy controlledDataSource: ControlledHistoryDataSource ) -> KeywordRankingHistoryDataSource { diff --git a/OpenASOTests/OpenASOMCPServerTests.swift b/OpenASOTests/OpenASOMCPServerTests.swift index 64d65f5..6579a94 100644 --- a/OpenASOTests/OpenASOMCPServerTests.swift +++ b/OpenASOTests/OpenASOMCPServerTests.swift @@ -11,6 +11,12 @@ struct OpenASOMCPServerTests { func serverExposesToolsResourcesPromptsAndReturnsJSONToolContent() async throws { let context = try ServerTestContext() try context.insertTrackedApp(appStoreID: 123, name: "Focus Timer") + _ = try await context.service.addKeywords( + appStoreID: 123, + keywords: ["focus timer"], + storefronts: ["us"], + platform: "iphone" + ) let server = await OpenASOMCPServerFactory(service: context.service).makeServer() let client = Client(name: "OpenASO MCP Test Client", version: "1.0") let transports = await InMemoryTransport.createConnectedPair() @@ -35,6 +41,11 @@ struct OpenASOMCPServerTests { #expect(tools.map(\.name).contains("list_competitors")) #expect(tools.map(\.name).contains("score_keywords")) #expect(tools.map(\.name).contains("get_localization_research_context")) + let difficultyTool = try #require( + tools.first { $0.name == "get_estimated_keyword_difficulty" } + ) + #expect(difficultyTool.annotations.readOnlyHint == true) + #expect(difficultyTool.description?.contains("local") == true) let rankingRefreshTool = try #require(tools.first { $0.name == "refresh_keyword_rankings" }) let rankingRefreshDescription = try #require(rankingRefreshTool.description) #expect(rankingRefreshDescription.contains("defaults to 20")) @@ -52,6 +63,24 @@ struct OpenASOMCPServerTests { let appPage = try JSONDecoder.openASOMCP.decode(OpenASOMCPPage.self, from: Data(toolJSON.utf8)) #expect(appPage.items.map(\.appStoreID) == ["123"]) + let difficultyResult = try await client.callTool( + name: "get_estimated_keyword_difficulty", + arguments: [ + "appStoreID": 123, + "keyword": "focus timer", + "storefront": "us", + "evidence_limit": 1, + ] + ) + #expect(difficultyResult.isError == nil) + let difficultyJSON = try #require(difficultyResult.content.first?.textValue) + let difficulty = try JSONDecoder.openASOMCP.decode( + OpenASOMCPEstimatedKeywordDifficulty.self, + from: Data(difficultyJSON.utf8) + ) + #expect(difficulty.state == "missing") + #expect(difficulty.evidence.isEmpty) + let resources = try await client.listResources().resources #expect(resources.map(\.uri).contains("openaso://workspace/summary")) let resourceContent = try await client.readResource(uri: "openaso://apps/123") diff --git a/OpenASOTests/OpenASOMCPServiceTests.swift b/OpenASOTests/OpenASOMCPServiceTests.swift index 84dc103..5836260 100644 --- a/OpenASOTests/OpenASOMCPServiceTests.swift +++ b/OpenASOTests/OpenASOMCPServiceTests.swift @@ -82,6 +82,113 @@ struct OpenASOMCPServiceTests { #expect(notes.track.storefront == "us") } + @Test + func keywordSummariesAndDetailedToolExposeEstimatedDifficultyCompatibly() async throws { + let asOf = isoDate("2026-05-07T12:00:00Z") + let context = try MCPTestContext(now: { asOf }) + let storeApp = try context.insertTrackedApp( + appStoreID: 123, + name: "Focus Timer" + ).storeApp + let track = try context.insertKeyword( + "focus timer", + trackedApp: storeApp, + storefront: "us" + ) + let legacyMetric = KeywordDailyMetric( + queryKey: track.queryKey, + keyword: track.term, + storefront: track.storefront, + platform: track.platform, + popularityScore: 61, + difficultyScore: 42, + source: .appleAdsPopularity, + updatedAt: asOf + ) + context.modelContext.insert(legacyMetric) + + let page = SearchRankingPage( + items: [ + makeRankingItem(position: 1, appStoreID: 201, name: "Focus Keeper", ratingCount: 80_000), + makeRankingItem(position: 2, appStoreID: 202, name: "Deep Work", ratingCount: 15_000), + makeRankingItem(position: 3, appStoreID: 203, name: "Pomodoro Timer", ratingCount: 2_000), + ], + source: .iTunesFallback + ) + let fetchedAt = asOf.addingTimeInterval(-(23 * 60 * 60)) + let payload = try #require(EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: RankingRefreshRequest(track: track), + page: page, + requestedResultLimit: 10, + rankingFetchedAt: fetchedAt, + calculationID: UUID(uuidString: "00000000-0000-0000-0000-000000002020")!, + now: { fetchedAt.addingTimeInterval(5) } + )) + _ = try EstimatedKeywordDifficultyStore.upsert( + payload, + in: context.modelContext + ) + try context.modelContext.save() + + let pageResult = try await context.service.listKeywords( + appStoreID: 123, + storefronts: ["us"], + platform: "iphone" + ) + let summary = try #require(pageResult.items.first) + let expectedScore: Int + switch payload.result { + case .estimated(let score, _, _): + expectedScore = score + case .unavailable: + Issue.record("The three rated results should produce an estimate.") + return + } + #expect(summary.difficultyScore == 42) + #expect(summary.estimatedDifficulty?.state == "estimated") + #expect(summary.estimatedDifficulty?.score == expectedScore) + #expect(summary.estimatedDifficulty?.isStale == false) + #expect(summary.estimatedDifficulty?.rankingFetchedAt == fetchedAt) + + let detail = try await context.service.getEstimatedKeywordDifficulty( + appStoreID: 123, + keyword: " focus timer ", + storefront: "US", + platform: "iphone", + evidenceLimit: 1 + ) + #expect(detail.state == "estimated") + #expect(detail.calculationID == payload.calculationID.uuidString.lowercased()) + #expect(detail.isStale == false) + #expect(detail.rankingFetchedAt == fetchedAt) + #expect(detail.returnedEvidenceCount == 1) + #expect(detail.availableEvidenceCount == 3) + #expect(detail.evidenceTruncated) + #expect(detail.evidence.first?.appStoreID == "201") + + let missing = try await context.service.getEstimatedKeywordDifficulty( + appStoreID: 456, + keyword: "focus timer", + storefront: "us" + ) + #expect(missing.state == "missing") + #expect(missing.isStale == nil) + #expect(missing.evidence.isEmpty) + + let encoded = try JSONEncoder().encode(summary) + var legacyObject = try #require( + JSONSerialization.jsonObject(with: encoded) as? [String: Any] + ) + legacyObject.removeValue(forKey: "estimatedDifficulty") + let legacyData = try JSONSerialization.data(withJSONObject: legacyObject) + let decodedLegacy = try JSONDecoder().decode( + OpenASOMCPKeywordSummary.self, + from: legacyData + ) + #expect(decodedLegacy.difficultyScore == 42) + #expect(decodedLegacy.estimatedDifficulty == nil) + } + @Test func listReviewsFiltersAndPaginatesNewestFirst() async throws { let context = try MCPTestContext() @@ -641,6 +748,7 @@ struct OpenASOMCPServiceTests { #expect(competitors.first?.occurrenceCount == 2) #expect(competitors.first?.bestRank == 1) #expect(competitors.first?.evidence.map(\.keyword) == ["focus timer", "pomodoro"]) + #expect(competitors.first?.evidence.allSatisfy { $0.source == RankingSource.iTunesFallback.rawValue } == true) #expect(competitors.allSatisfy { $0.appStoreID != "123" }) let capped = try await service.listCompetitors( @@ -660,11 +768,17 @@ struct OpenASOMCPServiceTests { "calorie tracker::us::iphone": SearchRankingPage(items: [ makeRankingItem(position: 1, appStoreID: 456, name: "MyFitnessPal", ratingCount: 3_000_000), makeRankingItem(position: 2, appStoreID: 123, name: "Cal AI", ratingCount: 50_000) - ], source: .iTunesFallback), + ], source: .iTunesFallback, fallbackContext: SearchRankingFailureContext( + provider: .appStoreWeb, + category: .httpStatus(503) + )), "ai calorie tracker::us::iphone": SearchRankingPage(items: [ makeRankingItem(position: 1, appStoreID: 123, name: "Cal AI", ratingCount: 50_000), makeRankingItem(position: 2, appStoreID: 789, name: "Macro AI", ratingCount: 25_000) - ], source: .iTunesFallback) + ], source: .iTunesFallback, fallbackContext: SearchRankingFailureContext( + provider: .appStoreWeb, + category: .httpStatus(503) + )) ]) let context = try MCPTestContext(rankingProvider: rankingProvider, useRankingRefreshCoordinator: true) let service = context.service @@ -680,6 +794,8 @@ struct OpenASOMCPServiceTests { #expect(result.app.isTracked == false) #expect(result.errors.isEmpty) #expect(result.candidates.contains { $0.keyword == "calorie tracker" && $0.targetRank == 2 }) + #expect(result.candidates.first { $0.keyword == "calorie tracker" }?.rankingProvenance?.source == RankingSource.iTunesFallback.rawValue) + #expect(result.candidates.first { $0.keyword == "calorie tracker" }?.rankingProvenance?.fallbackContext?.httpStatus == 503) #expect(result.candidates.contains { $0.keyword == "ai calorie tracker" && $0.targetRank == 1 }) #expect(result.candidates.first?.topRatedAppCount ?? 0 > 0) } @@ -746,7 +862,10 @@ struct OpenASOMCPServiceTests { "calorie tracker::us::iphone": SearchRankingPage(items: [ makeRankingItem(position: 1, appStoreID: 456, name: "MyFitnessPal", ratingCount: 3_000_000), makeRankingItem(position: 2, appStoreID: 123, name: "Cal AI", ratingCount: 50_000) - ], source: .iTunesFallback) + ], source: .iTunesFallback, fallbackContext: SearchRankingFailureContext( + provider: .appStoreWeb, + category: .response(.truncatedResults) + )) ]) let context = try MCPTestContext(rankingProvider: rankingProvider) @@ -761,6 +880,14 @@ struct OpenASOMCPServiceTests { #expect(result.topRatedAppCount == 2) #expect(result.maximumRatingCount == 3_000_000) #expect(result.topApps.map(\.name) == ["MyFitnessPal", "Cal AI"]) + #expect(result.source == RankingSource.iTunesFallback.rawValue) + #expect(result.fallbackContext == OpenASOMCPRankingFallbackContext( + provider: SearchRankingFailureContext.Provider.appStoreWeb.rawValue, + category: "response", + transportCode: nil, + httpStatus: nil, + responseFailure: SearchRankingResponseFailure.truncatedResults.rawValue + )) } @Test @@ -841,7 +968,10 @@ struct OpenASOMCPServiceTests { "calorie tracker::us::iphone": SearchRankingPage(items: [ makeRankingItem(position: 1, appStoreID: 456, name: "MyFitnessPal", ratingCount: 3_000_000), makeRankingItem(position: 2, appStoreID: 123, name: "Cal AI", ratingCount: 50_000) - ], source: .iTunesFallback) + ], source: .iTunesFallback, fallbackContext: SearchRankingFailureContext( + provider: .appStoreWeb, + category: .httpStatus(503) + )) ]) let context = try MCPTestContext(rankingProvider: rankingProvider) try context.insertTrackedApp(appStoreID: 123, name: "Cal AI") @@ -860,6 +990,38 @@ struct OpenASOMCPServiceTests { #expect(rankingRefresh.summary.refreshed == 1) #expect(rankingRefresh.outcomes.first?.track.latestRank == 2) + #expect(rankingRefresh.outcomes.first?.rankingProvenance?.source == RankingSource.iTunesFallback.rawValue) + #expect(rankingRefresh.outcomes.first?.rankingProvenance?.storefront == "us") + #expect(rankingRefresh.outcomes.first?.rankingProvenance?.platform == AppPlatform.iphone.rawValue) + #expect(rankingRefresh.outcomes.first?.rankingProvenance?.fallbackContext?.httpStatus == 503) + #expect(rankingRefresh.outcomes.first?.track.latestRankingSource == RankingSource.iTunesFallback.rawValue) + #expect(rankingRefresh.outcomes.first?.track.latestRankingObservedAt == rankingRefresh.outcomes.first?.rankingProvenance?.fetchedAt) + + let listedKeywords = try await context.service.listKeywords( + appStoreID: 123, + storefronts: ["us"], + platform: "iphone" + ) + #expect(listedKeywords.items.first?.latestRankingSource == RankingSource.iTunesFallback.rawValue) + #expect(listedKeywords.items.first?.latestRankingObservedAt == rankingRefresh.outcomes.first?.rankingProvenance?.fetchedAt) + #expect(listedKeywords.items.first?.estimatedDifficulty?.state == "unavailable") + #expect( + listedKeywords.items.first?.estimatedDifficulty?.unavailableReason + == EstimatedKeywordDifficultyUnavailableReason.insufficientResults.rawValue + ) + + let estimatedDifficulty = try await context.service.getEstimatedKeywordDifficulty( + appStoreID: 123, + keyword: "calorie tracker", + storefront: "us", + platform: "iphone" + ) + #expect(estimatedDifficulty.state == "unavailable") + #expect(estimatedDifficulty.rankingSource == RankingSource.iTunesFallback.rawValue) + #expect(estimatedDifficulty.fallback?.provider == RankingSource.appStoreWeb.rawValue) + #expect(estimatedDifficulty.fallback?.category == "httpStatus") + #expect(estimatedDifficulty.fallback?.httpStatus == 503) + #expect(estimatedDifficulty.availableEvidenceCount == 2) let competitors = try await context.service.listCompetitors( appStoreID: 123, @@ -876,10 +1038,243 @@ struct OpenASOMCPServiceTests { ) #expect(metricsRefresh.summary.failed == 1) + #expect(metricsRefresh.outcomes.first?.rankingProvenance == nil) #expect(metricsRefresh.outcomes.first?.error?.code == "apple_ads_not_configured") #expect(metricsRefresh.outcomes.first?.track.statusMessage?.contains("Connect an Apple Ads") == true) } + @Test + func refreshKeywordRankingsKeepsValidPeersWhenOptionalDifficultyConflicts() async throws { + let asOf = isoDate("2026-05-07T12:00:00Z") + let conflictingCalculationID = UUID( + uuidString: "00000000-0000-0000-0000-000000002027" + )! + let successfulCalculationID = UUID( + uuidString: "00000000-0000-0000-0000-000000002028" + )! + let conflictingPage = SearchRankingPage(items: [ + makeRankingItem(position: 1, appStoreID: 123, name: "Target", ratingCount: 50_000), + makeRankingItem(position: 2, appStoreID: 201, name: "Peer", ratingCount: 10_000), + makeRankingItem(position: 3, appStoreID: 202, name: "Peer Two", ratingCount: 5_000), + ], source: .iTunesFallback) + let successfulPage = SearchRankingPage(items: [ + makeRankingItem(position: 1, appStoreID: 301, name: "Other", ratingCount: 40_000), + makeRankingItem(position: 2, appStoreID: 123, name: "Target", ratingCount: 20_000), + makeRankingItem(position: 3, appStoreID: 302, name: "Other Two", ratingCount: 2_000), + ], source: .iTunesFallback) + let rankingProvider = StubMCPRankingProvider(pages: [ + "conflict term::us::iphone": conflictingPage, + "valid term::us::iphone": successfulPage, + ]) + let context = try MCPTestContext( + rankingProvider: rankingProvider, + estimatedDifficultyPayloadBuilder: { request, page, limit, fetchedAt in + EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: request, + page: page, + requestedResultLimit: limit, + rankingFetchedAt: fetchedAt, + calculationID: request.term == "conflict term" + ? conflictingCalculationID + : successfulCalculationID, + now: { fetchedAt } + ) + }, + now: { asOf } + ) + try context.insertTrackedApp(appStoreID: 123, name: "Target") + _ = try await context.service.addKeywords( + appStoreID: 123, + keywords: ["conflict term", "valid term"], + storefronts: ["us"], + platform: "iphone" + ) + + let seedContext = ModelContext(context.container) + let conflictIdentityKey = TrackedAppKeyword.makeIdentityKey( + appStoreID: 123, + term: "conflict term", + storefront: "us", + platform: .iphone + ) + let conflictTrack = try #require(try seedContext.fetch( + FetchDescriptor( + predicate: #Predicate { track in + track.identityKey == conflictIdentityKey + } + ) + ).first) + let priorFetchedAt = asOf.addingTimeInterval(-60) + let priorPayload = try #require( + EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: RankingRefreshRequest(track: conflictTrack), + page: SearchRankingPage(items: [ + makeRankingItem(position: 1, appStoreID: 401, name: "Prior", ratingCount: 90_000), + makeRankingItem(position: 2, appStoreID: 402, name: "Prior Two", ratingCount: 30_000), + makeRankingItem(position: 3, appStoreID: 403, name: "Prior Three", ratingCount: 3_000), + ], source: .iTunesFallback), + requestedResultLimit: 25, + rankingFetchedAt: priorFetchedAt, + calculationID: conflictingCalculationID, + now: { priorFetchedAt } + ) + ) + _ = try EstimatedKeywordDifficultyStore.upsert(priorPayload, in: seedContext) + try seedContext.save() + + let result = try await context.service.refreshKeywordRankings( + appStoreID: 123, + storefronts: ["us"], + platform: "iphone" + ) + + #expect(result.summary.refreshed == 2) + #expect(result.summary.failed == 0) + #expect(result.outcomes.count == 2) + #expect(result.outcomes.allSatisfy { $0.error == nil }) + #expect(result.outcomes.first { + $0.track.keyword == "conflict term" + }?.track.estimatedDifficulty?.rankingFetchedAt == priorFetchedAt) + #expect(result.outcomes.first { + $0.track.keyword == "valid term" + }?.track.estimatedDifficulty?.rankingFetchedAt == asOf) + + let verificationContext = ModelContext(context.container) + let rankings = try verificationContext.fetch( + FetchDescriptor() + ) + #expect(rankings.count == 2) + let validQueryKey = KeywordQuery.makeQueryKey( + term: "valid term", + storefront: "us", + platform: .iphone + ) + #expect( + try EstimatedKeywordDifficultyStore.snapshot( + queryKey: conflictTrack.queryKey, + in: verificationContext + )?.rankingFetchedAt == priorFetchedAt + ) + #expect( + try EstimatedKeywordDifficultyStore.snapshot( + queryKey: validQueryKey, + in: verificationContext + )?.calculationID == successfulCalculationID + ) + } + + @Test + func postCommitReplacementRejectsOldDifficultyAndKeepsOutcomeGeneration() async throws { + let asOf = isoDate("2026-05-07T12:00:00Z") + let replacementCreatedAt = asOf.addingTimeInterval(1) + let appStoreID: Int64 = 123 + let term = "generation term" + let identityKey = TrackedAppKeyword.makeIdentityKey( + appStoreID: appStoreID, + term: term, + storefront: "us", + platform: .iphone + ) + let container = try ModelContainerFactory.makeModelContainer( + isStoredInMemoryOnly: true + ) + let backgroundModelStore = BackgroundModelStore(modelContainer: container) + let rankingProvider = StubMCPRankingProvider(pages: [ + "generation term::us::iphone": SearchRankingPage(items: [ + makeRankingItem(position: 1, appStoreID: appStoreID, name: "Target", ratingCount: 50_000), + makeRankingItem(position: 2, appStoreID: 201, name: "Peer", ratingCount: 10_000), + makeRankingItem(position: 3, appStoreID: 202, name: "Peer Two", ratingCount: 5_000), + ], source: .iTunesFallback), + ]) + let context = try MCPTestContext( + modelContainer: container, + backgroundModelStore: backgroundModelStore, + rankingProvider: rankingProvider, + rankingPersistenceDidCommit: { + do { + _ = try await TrackedKeywordDeletionService.deleteTracks( + [TrackedKeywordDeletionRequest( + identityKey: identityKey, + trackCreatedAt: asOf + )], + using: backgroundModelStore + ) + try await backgroundModelStore.write { modelContext in + let targetAppStoreID = appStoreID + var descriptor = FetchDescriptor( + predicate: #Predicate { app in + app.appStoreID == targetAppStoreID + } + ) + descriptor.fetchLimit = 1 + guard let trackedApp = try modelContext.fetch(descriptor).first else { + throw OpenASOError.appNotFound + } + let query = try KeywordQuery.fetchOrInsert( + term: term, + storefront: "us", + platform: .iphone, + in: modelContext + ) + let replacement = TrackedAppKeyword( + term: term, + storefront: "us", + platform: .iphone, + trackedApp: trackedApp, + query: query, + createdAt: replacementCreatedAt + ) + replacement.notes = "replacement generation" + trackedApp.keywordTracks.append(replacement) + modelContext.insert(replacement) + } + } catch { + // State assertions below make hook failures visible. + } + }, + now: { asOf } + ) + try context.insertTrackedApp(appStoreID: appStoreID, name: "Target") + _ = try await context.service.addKeywords( + appStoreID: appStoreID, + keywords: [term], + storefronts: ["us"], + platform: "iphone" + ) + + let result = try await context.service.refreshKeywordRankings( + appStoreID: appStoreID, + storefronts: ["us"], + platform: "iphone" + ) + + #expect(result.summary.refreshed == 1) + #expect(result.summary.failed == 0) + let outcome = try #require(result.outcomes.first) + #expect(outcome.track.createdAt == asOf) + #expect(outcome.track.lastRefreshAt == asOf) + #expect(outcome.track.notes.isEmpty) + + let verificationContext = ModelContext(container) + let replacement = try #require(try verificationContext.fetch( + FetchDescriptor( + predicate: #Predicate { track in + track.identityKey == identityKey + } + ) + ).first) + #expect(replacement.createdAt == replacementCreatedAt) + #expect(replacement.lastRefreshAt == nil) + #expect(replacement.notes == "replacement generation") + #expect(try verificationContext.fetchCount( + FetchDescriptor() + ) == 0) + #expect(try EstimatedKeywordDifficultyStore.snapshot( + queryKey: replacement.queryKey, + in: verificationContext + ) == nil) + } + @Test func refreshKeywordRankingsLimitCapsKeywordTracksRefreshed() async throws { let rankingProvider = StubMCPRankingProvider(pages: [ @@ -924,6 +1319,9 @@ struct OpenASOMCPServiceTests { ) let tracks = try context.modelContext.fetch(FetchDescriptor()) + for track in tracks { + track.createdAt = isoDate("2025-01-01T00:00:00Z") + } let tracksByTerm = Dictionary(uniqueKeysWithValues: tracks.map { ($0.term, $0) }) tracksByTerm["alpha"]?.lastRefreshAt = isoDate("2026-01-01T00:00:00Z") tracksByTerm["beta"]?.lastRefreshAt = isoDate("2026-01-01T00:00:00Z") @@ -1119,6 +1517,118 @@ struct OpenASOMCPServiceTests { #expect(attempts.isEmpty) } + @Test + func rankingAttemptPersistenceRejectsDeletedAndReaddedGenerationBeforeProviderCall() async throws { + let container = try ModelContainerFactory.makeModelContainer(isStoredInMemoryOnly: true) + let backgroundStore = BackgroundModelStore(modelContainer: container) + let rankingProvider = StubMCPRankingProvider(pages: [:]) + let scheduler = OpenASOMCPRankingRefreshScheduler() + let context = try MCPTestContext( + modelContainer: container, + backgroundModelStore: backgroundStore, + rankingProvider: rankingProvider, + rankingRefreshScheduler: scheduler, + persistRankingRefreshAttempts: { requests, appStoreID, attemptedAt in + guard let request = requests.first, + requests.count == 1, + let trackCreatedAt = request.trackCreatedAt + else { + throw InjectedRankingAttemptInterleavingError.invalidReservation + } + try await replaceTrackedKeywordGeneration( + identityKey: request.identityKey, + expectedCreatedAt: trackCreatedAt, + using: backgroundStore + ) + return try await backgroundStore.write { modelContext in + try OpenASOMCPService.acceptRankingRefreshAttempts( + requests, + appStoreID: appStoreID, + attemptedAt: attemptedAt, + in: modelContext + ) + } + } + ) + try context.insertTrackedApp(appStoreID: 123, name: "Cal AI") + _ = try await context.service.addKeywords( + appStoreID: 123, + keywords: ["alpha"], + storefronts: ["us"], + platform: "iphone" + ) + + let result = try await context.service.refreshKeywordRankings( + appStoreID: 123, + storefronts: ["us"], + platform: "iphone", + limit: 1 + ) + + #expect(result.summary.skipped == 1) + #expect(result.summary.refreshed == 0) + #expect(result.summary.failed == 0) + #expect(result.outcomes.isEmpty) + #expect(await rankingProvider.searchedKeysSnapshot().isEmpty) + #expect(await scheduler.attemptCount() == 0) + let attempts = try ModelContext(container).fetch( + FetchDescriptor() + ) + #expect(attempts.isEmpty) + } + + @Test + func rankingResultPersistenceSkipsReaddedGenerationWithoutRollingBackValidPeer() async throws { + let container = try ModelContainerFactory.makeModelContainer(isStoredInMemoryOnly: true) + let backgroundStore = BackgroundModelStore(modelContainer: container) + let rankingProvider = ReaddingMCPRankingProvider( + backgroundStore: backgroundStore, + identityKeyToReplace: "123::alpha::us::iphone" + ) + let context = try MCPTestContext( + modelContainer: container, + backgroundModelStore: backgroundStore, + rankingProvider: rankingProvider + ) + try context.insertTrackedApp(appStoreID: 123, name: "Cal AI") + _ = try await context.service.addKeywords( + appStoreID: 123, + keywords: ["alpha", "beta"], + storefronts: ["us"], + platform: "iphone" + ) + + let result = try await context.service.refreshKeywordRankings( + appStoreID: 123, + storefronts: ["us"], + platform: "iphone", + limit: 2 + ) + + #expect(await rankingProvider.searchedKeysSnapshot() == [ + rankingQueryKey("alpha"), + rankingQueryKey("beta"), + ]) + #expect(result.summary.skipped == 1) + #expect(result.summary.refreshed == 1) + #expect(result.summary.failed == 0) + #expect(result.outcomes.map(\.track.keyword) == ["beta"]) + #expect(result.notes.contains { $0.contains("Skipped 1 ranking result") }) + #expect( + result.summary.skipped + result.summary.refreshed + result.summary.failed == 2 + ) + + let verificationContext = ModelContext(container) + let tracks = try verificationContext.fetch(FetchDescriptor()) + let tracksByTerm = Dictionary(uniqueKeysWithValues: tracks.map { ($0.term, $0) }) + #expect(tracksByTerm["alpha"]?.lastRefreshAt == nil) + #expect(tracksByTerm["beta"]?.lastRefreshAt != nil) + let attempts = try verificationContext.fetch( + FetchDescriptor() + ) + #expect(attempts.map(\.trackIdentityKey) == ["123::beta::us::iphone"]) + } + @Test func backgroundStoreRollsBackFailedAttemptMutation() async throws { let container = try ModelContainerFactory.makeModelContainer(isStoredInMemoryOnly: true) @@ -2901,11 +3411,14 @@ private struct MCPTestContext { @MainActor init( modelContainer: ModelContainer? = nil, + backgroundModelStore: BackgroundModelStore? = nil, resolver: StubMCPAppResolver = StubMCPAppResolver(), rankingProvider: (any SearchRankingProvider)? = nil, useRankingRefreshCoordinator: Bool = false, rankingRefreshScheduler: OpenASOMCPRankingRefreshScheduler = OpenASOMCPRankingRefreshScheduler(), persistRankingRefreshAttempts: OpenASOMCPService.RankingRefreshAttemptsPersistence? = nil, + estimatedDifficultyPayloadBuilder: OpenASOMCPService.EstimatedDifficultyPayloadBuilder? = nil, + rankingPersistenceDidCommit: (@Sendable () async -> Void)? = nil, includeReviewService: Bool = false, includeKeywordMetricsService: Bool = false, popularityContextAppStoreIDProvider: @escaping @MainActor @Sendable () -> Int64? = { nil }, @@ -2919,7 +3432,8 @@ private struct MCPTestContext { self.container = try modelContainer ?? ModelContainerFactory.makeModelContainer(isStoredInMemoryOnly: true) self.modelContext = ModelContext(container) - self.backgroundModelStore = BackgroundModelStore(modelContainer: container) + self.backgroundModelStore = backgroundModelStore + ?? BackgroundModelStore(modelContainer: container) self.resolver = resolver let httpClient = MockHTTPClient(handler: httpHandler) let appCatalogService = AppCatalogService(appResolver: resolver) @@ -2936,7 +3450,7 @@ private struct MCPTestContext { : nil } self.service = OpenASOMCPService( - backgroundModelStore: backgroundModelStore, + backgroundModelStore: self.backgroundModelStore, appResolver: resolver, appCatalogService: appCatalogService, httpClient: httpClient, @@ -2945,6 +3459,8 @@ private struct MCPTestContext { rankingRefreshCoordinator: rankingRefreshCoordinator, rankingRefreshScheduler: rankingRefreshScheduler, persistRankingRefreshAttempts: persistRankingRefreshAttempts, + estimatedDifficultyPayloadBuilder: estimatedDifficultyPayloadBuilder, + rankingPersistenceDidCommit: rankingPersistenceDidCommit, reviewService: includeReviewService ? AppStorefrontReviewService(httpClient: httpClient) : nil, keywordMetricsService: includeKeywordMetricsService ? KeywordMetricsService( @@ -3419,7 +3935,59 @@ private actor StubMCPRankingProvider: SearchRankingProvider { guard let page = pages[key] else { return SearchRankingPage(items: [], source: .iTunesFallback) } - return SearchRankingPage(items: Array(page.items.prefix(limit)), source: page.source) + return SearchRankingPage( + items: Array(page.items.prefix(limit)), + source: page.source, + fallbackContext: page.fallbackContext + ) + } +} + +private actor ReaddingMCPRankingProvider: SearchRankingProvider { + private let backgroundStore: BackgroundModelStore + private let identityKeyToReplace: String + private var didReplaceGeneration = false + private var searchedKeys: [String] = [] + + init( + backgroundStore: BackgroundModelStore, + identityKeyToReplace: String + ) { + self.backgroundStore = backgroundStore + self.identityKeyToReplace = identityKeyToReplace + } + + func searchedKeysSnapshot() -> [String] { + searchedKeys + } + + func search( + keyword: String, + storefrontCode: String, + platform: AppPlatform, + limit _: Int + ) async throws -> SearchRankingPage { + let queryKey = TrackedAppKeyword.makeQueryKey( + term: keyword, + storefront: storefrontCode, + platform: platform + ) + searchedKeys.append(queryKey) + let identityKey = TrackedAppKeyword.makeIdentityKey( + appStoreID: 123, + term: keyword, + storefront: storefrontCode, + platform: platform + ) + if identityKey == identityKeyToReplace, !didReplaceGeneration { + didReplaceGeneration = true + try await replaceTrackedKeywordGeneration( + identityKey: identityKey, + expectedCreatedAt: nil, + using: backgroundStore + ) + } + return SearchRankingPage(items: [], source: .iTunesFallback) } } @@ -3507,6 +4075,85 @@ private actor CancellationAwareMCPRankingProvider: SearchRankingProvider { } } +private struct TrackedKeywordReplacementSeed: Sendable { + let appStoreID: Int64 + let term: String + let storefront: String + let platform: AppPlatform + let replacementCreatedAt: Date +} + +private func replaceTrackedKeywordGeneration( + identityKey: String, + expectedCreatedAt: Date?, + using backgroundStore: BackgroundModelStore +) async throws { + let seed = try await backgroundStore.write { modelContext in + let targetIdentityKey = identityKey + var descriptor = FetchDescriptor( + predicate: #Predicate { track in + track.identityKey == targetIdentityKey + } + ) + descriptor.fetchLimit = 1 + guard let track = try modelContext.fetch(descriptor).first, + expectedCreatedAt.map({ $0 == track.createdAt }) ?? true + else { + throw InjectedRankingAttemptInterleavingError.missingExpectedGeneration + } + let seed = TrackedKeywordReplacementSeed( + appStoreID: track.appStoreID, + term: track.term, + storefront: track.storefront, + platform: track.platform, + replacementCreatedAt: track.createdAt.addingTimeInterval(1) + ) + let deletion = try TrackedKeywordDeletionService.deleteTracks( + [TrackedKeywordDeletionRequest(track: track)], + in: modelContext + ) + guard deletion.deletedTrackCount == 1 else { + throw InjectedRankingAttemptInterleavingError.missingExpectedGeneration + } + return seed + } + + try await backgroundStore.write { modelContext in + let appStoreID = seed.appStoreID + var appDescriptor = FetchDescriptor( + predicate: #Predicate { app in + app.appStoreID == appStoreID + } + ) + appDescriptor.fetchLimit = 1 + guard let trackedApp = try modelContext.fetch(appDescriptor).first else { + throw InjectedRankingAttemptInterleavingError.missingTrackedApp + } + let query = try KeywordQuery.fetchOrInsert( + term: seed.term, + storefront: seed.storefront, + platform: seed.platform, + in: modelContext + ) + let replacement = TrackedAppKeyword( + term: seed.term, + storefront: seed.storefront, + platform: seed.platform, + trackedApp: trackedApp, + query: query, + createdAt: seed.replacementCreatedAt + ) + trackedApp.keywordTracks.append(replacement) + modelContext.insert(replacement) + } +} + +private enum InjectedRankingAttemptInterleavingError: Error { + case invalidReservation + case missingExpectedGeneration + case missingTrackedApp +} + private enum InjectedRankingAttemptPersistenceError: Error { case expected } @@ -3518,7 +4165,7 @@ private actor FailingRankingAttemptPersistenceProbe { _ requests: [RankingRefreshRequest], appStoreID _: Int64, attemptedAt _: Date - ) throws { + ) throws -> [RankingRefreshRequest] { batches.append(requests.map(\.identityKey)) throw InjectedRankingAttemptPersistenceError.expected } diff --git a/OpenASOTests/RankingRefreshCoordinatorTests.swift b/OpenASOTests/RankingRefreshCoordinatorTests.swift index 4a56ea6..7243ab3 100644 --- a/OpenASOTests/RankingRefreshCoordinatorTests.swift +++ b/OpenASOTests/RankingRefreshCoordinatorTests.swift @@ -55,14 +55,30 @@ struct RankingRefreshCoordinatorTests { #expect(groups[2].providerRequest.term == "Numbers") let searchedAt = Date(timeIntervalSince1970: 1_234) + let page = SearchRankingPage(items: [], source: .iTunesFallback) + let calculationID = UUID( + uuidString: "00000000-0000-0000-0000-000000002021" + )! + let difficultyPayload = try #require( + EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: pagesGroup.providerRequest, + page: page, + requestedResultLimit: 10, + rankingFetchedAt: searchedAt, + calculationID: calculationID, + now: { searchedAt } + ) + ) let providerResult = RankingRefreshPageResult( request: pagesGroup.providerRequest, - page: SearchRankingPage(items: [], source: .iTunesFallback), + page: page, searchedAt: searchedAt, observedHour: 42, submissionCount: 3, winningCount: 2, - confidence: "fixture-confidence" + confidence: "fixture-confidence", + requestedResultLimit: 10, + estimatedDifficultyPayload: difficultyPayload ) let fannedResults = pagesGroup.pageResults(fanningOut: providerResult) @@ -73,6 +89,12 @@ struct RankingRefreshCoordinatorTests { #expect(fannedResults.allSatisfy { $0.winningCount == 2 }) #expect(fannedResults.allSatisfy { $0.confidence == "fixture-confidence" }) #expect(fannedResults.allSatisfy { $0.page.source == .iTunesFallback }) + #expect(fannedResults.allSatisfy { + $0.estimatedDifficultyPayload?.calculationID == calculationID + }) + #expect(Set(fannedResults.compactMap { + $0.estimatedDifficultyPayload?.calculationID + }).count == 1) } @Test @@ -144,6 +166,22 @@ struct RankingRefreshCoordinatorTests { #expect(crawls.allSatisfy { $0.confidenceRaw == "single_source" }) #expect(crawls.first(where: { $0.queryKey == fixture.firstTrack.queryKey })?.submissionCount == 1) #expect(crawls.first(where: { $0.queryKey == fixture.firstTrack.queryKey })?.winningCount == 1) + let difficultyMetrics = try modelContext.fetch( + FetchDescriptor() + ) + #expect(difficultyMetrics.count == 2) + let sharedDifficulty = try #require( + try EstimatedKeywordDifficultyStore.snapshot( + queryKey: fixture.firstTrack.queryKey, + in: modelContext + ) + ) + #expect(sharedDifficulty.state == .unavailable) + #expect(sharedDifficulty.unavailableReason == .insufficientRatingEvidence) + #expect(sharedDifficulty.resultEvidence.count == 3) + #expect(try modelContext.fetchCount( + FetchDescriptor() + ) == 5) #expect(await progressRecorder.values() == [ RankingRefreshProgressValue(completed: 0, total: 3, failureCount: 0), RankingRefreshProgressValue(completed: 1, total: 3, failureCount: 0), @@ -158,6 +196,32 @@ struct RankingRefreshCoordinatorTests { let modelContext = ModelContext(container) let fixture = try makeDeduplicatedRefreshFixture(in: modelContext) fixture.firstTrack.term = " Pages " + let priorCalculationID = UUID( + uuidString: "00000000-0000-0000-0000-000000002022" + )! + let priorFetchedAt = Date(timeIntervalSince1970: 100) + let priorPage = SearchRankingPage( + items: [ + rankingItem(position: 1, appStoreID: 991, platform: .iphone), + rankingItem(position: 2, appStoreID: 992, platform: .iphone), + rankingItem(position: 3, appStoreID: 993, platform: .iphone), + ], + source: .appStoreWeb + ) + let priorPayload = try #require( + EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: RankingRefreshRequest(track: fixture.firstTrack), + page: priorPage, + requestedResultLimit: 10, + rankingFetchedAt: priorFetchedAt, + calculationID: priorCalculationID, + now: { priorFetchedAt } + ) + ) + _ = try EstimatedKeywordDifficultyStore.upsert( + priorPayload, + in: modelContext + ) try modelContext.save() let provider = QueryRankingProvider(responses: [ @@ -216,6 +280,14 @@ struct RankingRefreshCoordinatorTests { #expect(crawls.count == 1) #expect(crawls.first?.queryKey == fixture.thirdTrack.queryKey) #expect(crawls.first?.confidenceRaw == "single_source") + let preservedDifficulty = try #require( + try EstimatedKeywordDifficultyStore.snapshot( + queryKey: fixture.firstTrack.queryKey, + in: modelContext + ) + ) + #expect(preservedDifficulty.calculationID == priorCalculationID) + #expect(preservedDifficulty.rankingFetchedAt == priorFetchedAt) #expect(await progressRecorder.values() == [ RankingRefreshProgressValue(completed: 0, total: 3, failureCount: 0), RankingRefreshProgressValue(completed: 1, total: 3, failureCount: 1), @@ -224,6 +296,316 @@ struct RankingRefreshCoordinatorTests { ]) } + @Test + func rankingPersistenceKeepsValidPageWhenOptionalDifficultyConflicts() async throws { + let container = try makeInMemoryContainer() + let modelContext = ModelContext(container) + let fixture = try makeDeduplicatedRefreshFixture(in: modelContext) + let priorCalculationID = UUID( + uuidString: "00000000-0000-0000-0000-000000002023" + )! + let successfulCalculationID = UUID( + uuidString: "00000000-0000-0000-0000-000000002024" + )! + let priorDate = Date(timeIntervalSince1970: 100) + let priorPayload = try #require( + EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: RankingRefreshRequest(track: fixture.firstTrack), + page: SearchRankingPage( + items: [ + rankingItem(position: 1, appStoreID: 901, platform: .iphone), + rankingItem(position: 2, appStoreID: 902, platform: .iphone), + rankingItem(position: 3, appStoreID: 903, platform: .iphone), + ], + source: .appStoreWeb + ), + requestedResultLimit: 10, + rankingFetchedAt: priorDate, + calculationID: priorCalculationID, + now: { priorDate } + ) + ) + _ = try EstimatedKeywordDifficultyStore.upsert(priorPayload, in: modelContext) + try modelContext.save() + + let successfulPage = SearchRankingPage( + items: [ + rankingItem( + position: 1, + appStoreID: fixture.thirdApp.appStoreID, + platform: .iphone + ), + ], + source: .iTunesFallback + ) + let conflictingPage = SearchRankingPage( + items: [ + rankingItem( + position: 1, + appStoreID: fixture.firstApp.appStoreID, + platform: .iphone + ), + rankingItem(position: 2, appStoreID: 999, platform: .iphone), + ], + source: .iTunesFallback + ) + let provider = QueryRankingProvider(responses: [ + QueryRankingProvider.key( + term: fixture.thirdTrack.term, + storefront: fixture.thirdTrack.storefront, + platform: fixture.thirdTrack.platform + ): .page(successfulPage), + QueryRankingProvider.key( + term: fixture.firstTrack.term, + storefront: fixture.firstTrack.storefront, + platform: fixture.firstTrack.platform + ): .page(conflictingPage), + ]) + let failingQueryKey = fixture.firstTrack.queryKey + let coordinator = RankingRefreshCoordinator( + rankingProvider: provider, + appCatalogService: AppCatalogService(appResolver: StubAppResolver()), + estimatedDifficultyPayloadBuilder: { + request, + page, + requestedResultLimit, + rankingFetchedAt in + EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: request, + page: page, + requestedResultLimit: requestedResultLimit, + rankingFetchedAt: rankingFetchedAt, + calculationID: request.queryKey == failingQueryKey + ? priorCalculationID + : successfulCalculationID + ) + } + ) + + let outcomes = await coordinator.refresh( + tracks: [fixture.thirdTrack, fixture.firstTrack], + in: modelContext, + limit: 10 + ) + + #expect(outcomes.count == 2) + #expect( + outcomes.first(where: { + $0.trackID == fixture.thirdTrack.persistentModelID + })?.error == nil + ) + #expect( + outcomes.first(where: { + $0.trackID == fixture.firstTrack.persistentModelID + })?.error == nil + ) + + let verificationContext = ModelContext(container) + let snapshots = try verificationContext.fetch( + FetchDescriptor() + ) + #expect(Set(snapshots.map(\.trackIdentityKey)) == Set([ + fixture.firstTrack.identityKey, + fixture.thirdTrack.identityKey, + ])) + let crawls = try verificationContext.fetch(FetchDescriptor()) + #expect(Set(crawls.map(\.queryKey)) == Set([ + fixture.firstTrack.queryKey, + fixture.thirdTrack.queryKey, + ])) + let rankedResults = try verificationContext.fetch( + FetchDescriptor() + ) + #expect( + rankedResults.count + == successfulPage.resultCount + conflictingPage.resultCount + ) + let observationItems = try verificationContext.fetch( + FetchDescriptor() + ) + #expect(Set(observationItems.map(\.queryKey)) == Set([ + fixture.firstTrack.queryKey, + fixture.thirdTrack.queryKey, + ])) + let storeApps = try verificationContext.fetch(FetchDescriptor()) + #expect(storeApps.contains { $0.appStoreID == 999 }) + + let preservedDifficulty = try #require( + try EstimatedKeywordDifficultyStore.snapshot( + queryKey: fixture.firstTrack.queryKey, + in: verificationContext + ) + ) + #expect(preservedDifficulty.calculationID == priorCalculationID) + #expect(preservedDifficulty.rankingFetchedAt == priorDate) + #expect( + try EstimatedKeywordDifficultyStore.snapshot( + queryKey: fixture.thirdTrack.queryKey, + in: verificationContext + )?.calculationID == successfulCalculationID + ) + let statuses = try TrackedKeywordRefreshStatusStore.snapshots( + for: [fixture.firstTrack.identityKey, fixture.thirdTrack.identityKey], + in: verificationContext + ) + #expect(statuses[fixture.firstTrack.identityKey]?.rankingMessage == nil) + #expect(statuses[fixture.thirdTrack.identityKey]?.rankingMessage == nil) + } + + @Test + func delayedProviderFailureDoesNotWriteStatusToReplacementTrackGeneration() async throws { + let container = try makeInMemoryContainer() + let modelContext = ModelContext(container) + let trackedApp = TrackedApp( + appStoreID: 404, + bundleID: "example.replacement", + name: "Replacement App", + sellerName: "Example", + defaultPlatform: .iphone + ) + let query = try KeywordQuery.fetchOrInsert( + term: "pages", + storefront: "us", + platform: .iphone, + in: modelContext + ) + let originalTrack = TrackedAppKeyword( + term: "pages", + storefront: "us", + platform: .iphone, + trackedApp: trackedApp, + query: query, + createdAt: Date(timeIntervalSince1970: 100) + ) + trackedApp.keywordTracks.append(originalTrack) + modelContext.insert(trackedApp) + modelContext.insert(originalTrack) + try modelContext.save() + + let provider = ControlledRankingProvider() + let coordinator = RankingRefreshCoordinator( + rankingProvider: provider, + appCatalogService: AppCatalogService(appResolver: StubAppResolver()) + ) + let refreshTask = Task { @MainActor in + await coordinator.refresh(tracks: [originalTrack], in: modelContext) + } + await provider.waitUntilRequested() + + trackedApp.keywordTracks.removeAll { + $0.identityKey == originalTrack.identityKey + } + modelContext.delete(originalTrack) + try modelContext.save() + let replacementTrack = TrackedAppKeyword( + term: "pages", + storefront: "us", + platform: .iphone, + trackedApp: trackedApp, + query: query, + createdAt: Date(timeIntervalSince1970: 200) + ) + trackedApp.keywordTracks.append(replacementTrack) + modelContext.insert(replacementTrack) + try modelContext.save() + + await provider.fail(with: .networkUnavailable) + let outcomes = await refreshTask.value + + #expect(outcomes.count == 1) + #expect(outcomes.first?.error == .networkUnavailable) + let statuses = try TrackedKeywordRefreshStatusStore.snapshots( + for: [replacementTrack.identityKey], + in: modelContext + ) + #expect(statuses[replacementTrack.identityKey]?.rankingMessage == nil) + #expect(try modelContext.fetchCount( + FetchDescriptor() + ) == 0) + #expect(replacementTrack.createdAt == Date(timeIntervalSince1970: 200)) + } + + @Test + func delayedRankingPageDoesNotWriteFailureStatusToReplacementTrackGeneration() async throws { + let container = try makeInMemoryContainer() + let modelContext = ModelContext(container) + let trackedApp = TrackedApp( + appStoreID: 405, + bundleID: "example.replacement-page", + name: "Replacement Page App", + sellerName: "Example", + defaultPlatform: .iphone + ) + let query = try KeywordQuery.fetchOrInsert( + term: "pages", + storefront: "us", + platform: .iphone, + in: modelContext + ) + let originalTrack = TrackedAppKeyword( + term: "pages", + storefront: "us", + platform: .iphone, + trackedApp: trackedApp, + query: query, + createdAt: Date(timeIntervalSince1970: 100) + ) + trackedApp.keywordTracks.append(originalTrack) + modelContext.insert(trackedApp) + modelContext.insert(originalTrack) + try modelContext.save() + + let provider = ControlledRankingProvider() + let coordinator = RankingRefreshCoordinator( + rankingProvider: provider, + appCatalogService: AppCatalogService(appResolver: StubAppResolver()) + ) + let refreshTask = Task { @MainActor in + await coordinator.refresh(tracks: [originalTrack], in: modelContext) + } + await provider.waitUntilRequested() + + trackedApp.keywordTracks.removeAll { + $0.identityKey == originalTrack.identityKey + } + modelContext.delete(originalTrack) + try modelContext.save() + let replacementTrack = TrackedAppKeyword( + term: "pages", + storefront: "us", + platform: .iphone, + trackedApp: trackedApp, + query: query, + createdAt: Date(timeIntervalSince1970: 200) + ) + trackedApp.keywordTracks.append(replacementTrack) + modelContext.insert(replacementTrack) + try modelContext.save() + + await provider.succeed(with: SearchRankingPage( + items: [ + rankingItem(position: 1, appStoreID: 999, platform: .iphone), + ], + source: .iTunesFallback + )) + let outcomes = await refreshTask.value + + #expect(outcomes.count == 1) + #expect(outcomes.first?.error != nil) + let statuses = try TrackedKeywordRefreshStatusStore.snapshots( + for: [replacementTrack.identityKey], + in: modelContext + ) + #expect(statuses[replacementTrack.identityKey]?.rankingMessage == nil) + #expect(try modelContext.fetchCount( + FetchDescriptor() + ) == 0) + #expect(try modelContext.fetchCount( + FetchDescriptor() + ) == 0) + #expect(try modelContext.fetchCount(FetchDescriptor()) == 0) + } + @Test func topRankingEnrichmentRequestsDeduplicateAndLimitToTopTwenty() { let items = (1...25).map { position in @@ -396,6 +778,9 @@ struct RankingRefreshCoordinatorTests { let storeApp = try catalogService.upsertStoreApp( from: rankingItem, storefrontCode: "gb", + rankingSource: .iTunesFallback, + fetchedAt: .now, + requestedPlatform: .iphone, in: modelContext ) @@ -442,6 +827,651 @@ struct RankingRefreshCoordinatorTests { #expect(screenshots.first?.urlString == "https://example.com/flighty-iphone.png") } + @Test + func rankingPersistenceKeepsProviderProvenanceTimestampsAndCanonicalPlatform() throws { + let container = try makeInMemoryContainer() + let modelContext = ModelContext(container) + let trackedApp = TrackedApp( + appStoreID: 100, + bundleID: "com.example.target", + name: "Target", + sellerName: "Example", + defaultPlatform: .iphone + ) + let macTrack = try makeTrackedAppKeyword( + term: "notes", + platform: .mac, + trackedApp: trackedApp, + in: modelContext + ) + let ipadTrack = try makeTrackedAppKeyword( + term: "notepad", + platform: .ipad, + trackedApp: trackedApp, + in: modelContext + ) + trackedApp.keywordTracks.append(contentsOf: [macTrack, ipadTrack]) + modelContext.insert(trackedApp) + modelContext.insert(macTrack) + modelContext.insert(ipadTrack) + try modelContext.save() + + let catalogService = AppCatalogService(appResolver: StubAppResolver()) + let coordinator = RankingRefreshCoordinator( + rankingProvider: StubRankingProvider( + page: SearchRankingPage(items: [], source: .iTunesFallback) + ), + appCatalogService: catalogService + ) + let webDate = Date(timeIntervalSince1970: 1_800_000_000) + let fallbackDate = webDate.addingTimeInterval(60) + let item = SearchRankingItem( + position: 1, + appStoreID: 200, + bundleID: "com.example.result", + name: "Result", + sellerName: "Example", + supportedLanguageCodes: ["EN"], + screenshotURLs: ["https://example.com/result.png"], + ipadScreenshotURLs: ["https://example.com/result-ipad.png"], + ratingCount: 42, + averageRating: 4.5, + platform: .mac + ) + + _ = try coordinator.persistRankingPage( + RankingRefreshPageResult( + request: RankingRefreshRequest(track: macTrack), + page: SearchRankingPage(items: [item], source: .appStoreWeb), + searchedAt: webDate, + observedHour: nil, + submissionCount: 1, + winningCount: 1, + confidence: "single_source" + ), + in: modelContext, + scheduleMetadataEnrichment: false + ) + + var storeApp = try #require(modelContext.fetch(FetchDescriptor()).first { + $0.appStoreID == 200 + }) + var metadata = try #require(modelContext.fetch(FetchDescriptor()).first { + $0.appStoreID == 200 + }) + var screenshot = try #require(modelContext.fetch(FetchDescriptor()).first { + $0.appStoreID == 200 + }) + var latestRating = try #require(modelContext.fetch(FetchDescriptor()).first { + $0.appStoreID == 200 + }) + var dailyRating = try #require(modelContext.fetch(FetchDescriptor()).first { + $0.appStoreID == 200 + }) + + #expect(storeApp.defaultPlatform == .mac) + #expect(storeApp.supportedLanguageCodesSource == .appStoreWebSearch) + #expect(storeApp.supportedLanguageCodesFetchedAt == webDate) + #expect(storeApp.lastMetadataRefreshAt == webDate) + #expect(metadata.defaultPlatform == .mac) + #expect(metadata.source == .appStoreWebSearch) + #expect(metadata.lastFetchedAt == webDate) + #expect(screenshot.source == .appStoreWebSearch) + #expect(screenshot.lastFetchedAt == webDate) + #expect(latestRating.source == .appStorePage) + #expect(latestRating.observedAt == webDate) + #expect(dailyRating.source == .appStorePage) + #expect(dailyRating.observedAt == webDate) + #expect(storeApp.storefrontLatest.contains { $0 === latestRating }) + #expect(storeApp.ratingSnapshots.contains { $0 === dailyRating }) + + let fallbackItem = SearchRankingItem( + position: 1, + appStoreID: 200, + bundleID: "com.example.result", + name: "Result", + sellerName: "Example", + supportedLanguageCodes: ["FR"], + screenshotURLs: ["https://example.com/result.png"], + ratingCount: 43, + averageRating: 4.6, + platform: .ipad + ) + + _ = try coordinator.persistRankingPage( + RankingRefreshPageResult( + request: RankingRefreshRequest(track: ipadTrack), + page: SearchRankingPage(items: [fallbackItem], source: .iTunesFallback), + searchedAt: fallbackDate, + observedHour: nil, + submissionCount: 1, + winningCount: 1, + confidence: "single_source" + ), + in: modelContext, + scheduleMetadataEnrichment: false + ) + + storeApp = try #require(modelContext.fetch(FetchDescriptor()).first { $0.appStoreID == 200 }) + metadata = try #require(modelContext.fetch(FetchDescriptor()).first { $0.appStoreID == 200 }) + let screenshots = try modelContext.fetch(FetchDescriptor()).filter { + $0.appStoreID == 200 + } + screenshot = try #require(screenshots.first { $0.platformRaw == "iphone" }) + let retainedIPadScreenshot = try #require(screenshots.first { $0.platformRaw == "ipad" }) + latestRating = try #require(modelContext.fetch(FetchDescriptor()).first { $0.appStoreID == 200 }) + dailyRating = try #require(modelContext.fetch(FetchDescriptor()).first { $0.appStoreID == 200 }) + + #expect(storeApp.defaultPlatform == .mac) + #expect(storeApp.supportedLanguageCodes == ["FR"]) + #expect(storeApp.supportedLanguageCodesSource == .iTunesSearch) + #expect(storeApp.supportedLanguageCodesFetchedAt == fallbackDate) + #expect(metadata.defaultPlatform == .mac) + #expect(metadata.source == .iTunesSearch) + #expect(metadata.lastFetchedAt == fallbackDate) + #expect(screenshot.source == .iTunesSearch) + #expect(screenshot.lastFetchedAt == fallbackDate) + #expect(retainedIPadScreenshot.source == .appStoreWebSearch) + #expect(retainedIPadScreenshot.lastFetchedAt == webDate) + #expect(latestRating.source == .iTunesSearch) + #expect(latestRating.observedAt == fallbackDate) + #expect(latestRating.ratingCount == 43) + #expect(dailyRating.source == .iTunesSearch) + #expect(dailyRating.observedAt == fallbackDate) + #expect(dailyRating.ratingCount == 43) + #expect(storeApp.storefrontLatest.contains { $0 === latestRating }) + #expect(storeApp.ratingSnapshots.contains { $0 === dailyRating }) + + let olderWebItem = SearchRankingItem( + position: 1, + appStoreID: 200, + bundleID: "com.example.result", + name: "Stale Result", + sellerName: "Example", + supportedLanguageCodes: ["DE"], + screenshotURLs: ["https://example.com/stale.png"], + ratingCount: 99, + averageRating: 1, + platform: .mac + ) + _ = try coordinator.persistRankingPage( + RankingRefreshPageResult( + request: RankingRefreshRequest(track: macTrack), + page: SearchRankingPage(items: [olderWebItem], source: .appStoreWeb), + searchedAt: webDate.addingTimeInterval(30), + observedHour: nil, + submissionCount: 1, + winningCount: 1, + confidence: "single_source" + ), + in: modelContext, + scheduleMetadataEnrichment: false + ) + + storeApp = try #require(modelContext.fetch(FetchDescriptor()).first { $0.appStoreID == 200 }) + metadata = try #require(modelContext.fetch(FetchDescriptor()).first { $0.appStoreID == 200 }) + latestRating = try #require(modelContext.fetch(FetchDescriptor()).first { $0.appStoreID == 200 }) + dailyRating = try #require(modelContext.fetch(FetchDescriptor()).first { $0.appStoreID == 200 }) + #expect(storeApp.name == "Result") + #expect(storeApp.supportedLanguageCodes == ["FR"]) + #expect(storeApp.supportedLanguageCodesSource == .iTunesSearch) + #expect(storeApp.supportedLanguageCodesFetchedAt == fallbackDate) + #expect(metadata.source == .iTunesSearch) + #expect(metadata.lastFetchedAt == fallbackDate) + #expect(latestRating.ratingCount == 43) + #expect(latestRating.source == .iTunesSearch) + #expect(latestRating.observedAt == fallbackDate) + #expect(dailyRating.ratingCount == 43) + #expect(dailyRating.source == .iTunesSearch) + #expect(dailyRating.observedAt == fallbackDate) + } + + @Test + func rankingCatalogRejectsOlderMetadataEvidence() throws { + let container = try makeInMemoryContainer() + let modelContext = ModelContext(container) + let catalogService = AppCatalogService(appResolver: StubAppResolver()) + let newerDate = Date(timeIntervalSince1970: 1_800_000_000) + let olderDate = newerDate.addingTimeInterval(-60) + let newerItem = SearchRankingItem( + position: 1, + appStoreID: 300, + bundleID: "com.example.new", + name: "New Evidence", + sellerName: "Example", + screenshotURLs: ["https://example.com/new.png"], + platform: .iphone + ) + let olderItem = SearchRankingItem( + position: 1, + appStoreID: 300, + bundleID: "com.example.old", + name: "Old Evidence", + sellerName: "Old Example", + screenshotURLs: ["https://example.com/old.png"], + platform: .ipad + ) + + _ = try catalogService.upsertStoreApp( + from: newerItem, + storefrontCode: "us", + rankingSource: .appStoreWeb, + fetchedAt: newerDate, + requestedPlatform: .iphone, + in: modelContext + ) + _ = try catalogService.upsertStoreApp( + from: olderItem, + storefrontCode: "us", + rankingSource: .iTunesFallback, + fetchedAt: olderDate, + requestedPlatform: .ipad, + in: modelContext + ) + try modelContext.save() + + let storeApp = try #require(modelContext.fetch(FetchDescriptor()).first { $0.appStoreID == 300 }) + let metadata = try #require(modelContext.fetch(FetchDescriptor()).first { $0.appStoreID == 300 }) + let screenshots = try modelContext.fetch(FetchDescriptor()).filter { $0.appStoreID == 300 } + + #expect(storeApp.name == "New Evidence") + #expect(storeApp.defaultPlatform == .iphone) + #expect(storeApp.lastMetadataRefreshAt == newerDate) + #expect(metadata.name == "New Evidence") + #expect(metadata.source == .appStoreWebSearch) + #expect(metadata.defaultPlatform == .iphone) + #expect(metadata.lastFetchedAt == newerDate) + #expect(screenshots.map(\.urlString) == ["https://example.com/new.png"]) + #expect(screenshots.first?.source == .appStoreWebSearch) + #expect(screenshots.first?.lastFetchedAt == newerDate) + } + + @Test + func shallowWebRankingEvidenceDoesNotSuppressDetailEnrichment() throws { + let container = try makeInMemoryContainer() + let modelContext = ModelContext(container) + let catalogService = AppCatalogService(appResolver: StubAppResolver()) + let detailDate = Date.now.addingTimeInterval(-60) + let richFallbackItem = SearchRankingItem( + position: 1, + appStoreID: 350, + bundleID: "com.example.rich", + name: "Rich Result", + subtitle: "Useful subtitle", + sellerName: "Example", + version: "1.0", + descriptionText: "A complete retained description.", + supportedLanguageCodes: ["EN"], + screenshotURLs: ["https://example.com/rich-phone.png"], + platform: .iphone + ) + _ = try catalogService.upsertStoreApp( + from: richFallbackItem, + storefrontCode: "us", + rankingSource: .iTunesFallback, + fetchedAt: detailDate, + requestedPlatform: .iphone, + in: modelContext + ) + let shallowWebItem = SearchRankingItem( + position: 1, + appStoreID: 350, + bundleID: "com.example.rich", + name: "Rich Result", + sellerName: "Example", + screenshotURLs: ["https://example.com/rich-phone.png"], + platform: .ipad + ) + _ = try catalogService.upsertStoreApp( + from: shallowWebItem, + storefrontCode: "us", + rankingSource: .appStoreWeb, + fetchedAt: .now, + requestedPlatform: .ipad, + in: modelContext + ) + try modelContext.save() + + var metadata = try #require(modelContext.fetch(FetchDescriptor()).first { + $0.appStoreID == 350 + }) + #expect(metadata.source == .iTunesSearch) + #expect(metadata.lastFetchedAt == detailDate) + #expect(try catalogService.shouldEnrichStorefrontMetadata( + appStoreID: 350, + storefrontCode: "us", + platform: .ipad, + freshnessInterval: 5 * 24 * 60 * 60, + in: modelContext + )) + + _ = try catalogService.upsertStoreApp( + from: AppStoreWebMetadata( + appStoreID: 350, + storefront: "us", + name: "Rich Result", + subtitle: "Useful subtitle", + sellerName: "Example", + averageRating: nil, + ratingCount: nil, + screenshotGroups: [ + AppStoreWebScreenshotGroup( + platformRaw: "ipad", + displayTypeRaw: "tablet", + screenshots: [ + AppStoreWebScreenshot( + urlString: "https://example.com/rich-ipad.png", + width: 2_048, + height: 2_732 + ) + ] + ) + ] + ), + storefrontCode: "us", + in: modelContext + ) + try modelContext.save() + + metadata = try #require(modelContext.fetch(FetchDescriptor()).first { + $0.appStoreID == 350 + }) + #expect(metadata.source == .appStoreWeb) + #expect(!(try catalogService.shouldEnrichStorefrontMetadata( + appStoreID: 350, + storefrontCode: "us", + platform: .ipad, + freshnessInterval: 5 * 24 * 60 * 60, + in: modelContext + ))) + } + + @Test + func pureWebDetailBoundsRetriesWhenOptionalFieldsAreUnavailable() throws { + let container = try makeInMemoryContainer() + let modelContext = ModelContext(container) + let catalogService = AppCatalogService(appResolver: StubAppResolver()) + + _ = try catalogService.upsertStoreApp( + from: AppStoreWebMetadata( + appStoreID: 351, + storefront: "us", + name: "Web-only Result", + subtitle: nil, + sellerName: "Example", + averageRating: nil, + ratingCount: nil, + screenshotGroups: [] + ), + storefrontCode: "us", + in: modelContext + ) + try modelContext.save() + + let storeApp = try #require(modelContext.fetch(FetchDescriptor()).first { + $0.appStoreID == 351 + }) + let metadata = try #require(modelContext.fetch(FetchDescriptor()).first { + $0.appStoreID == 351 + }) + #expect(storeApp.supportedLanguageCodes.isEmpty) + #expect(storeApp.supportedLanguageCodesFetchedAt == nil) + #expect(metadata.subtitle == nil) + #expect(metadata.screenshots.isEmpty) + #expect(metadata.source == .appStoreWeb) + #expect(!(try catalogService.shouldEnrichStorefrontMetadata( + appStoreID: 351, + storefrontCode: "us", + platform: .ipad, + freshnessInterval: 5 * 24 * 60 * 60, + in: modelContext + ))) + } + + @Test + func sparseWebRankingDoesNotDowngradeRichSamePlatformScreenshots() throws { + let container = try makeInMemoryContainer() + let modelContext = ModelContext(container) + let catalogService = AppCatalogService(appResolver: StubAppResolver()) + + _ = try catalogService.upsertStoreApp( + from: AppStoreWebMetadata( + appStoreID: 352, + storefront: "us", + name: "Detailed Result", + subtitle: "Detailed subtitle", + sellerName: "Example", + averageRating: nil, + ratingCount: nil, + screenshotGroups: [ + AppStoreWebScreenshotGroup( + platformRaw: "iphone", + displayTypeRaw: "phone_6_9", + screenshots: [ + AppStoreWebScreenshot( + urlString: "https://example.com/detail-phone.png", + width: 1_320, + height: 2_868 + ) + ] + ) + ] + ), + storefrontCode: "us", + in: modelContext + ) + try modelContext.save() + + var metadata = try #require(modelContext.fetch(FetchDescriptor()).first { + $0.appStoreID == 352 + }) + let detailDate = metadata.lastFetchedAt + let sparseDate = detailDate.addingTimeInterval(60) + _ = try catalogService.upsertStoreApp( + from: SearchRankingItem( + position: 1, + appStoreID: 352, + bundleID: "com.example.detail", + name: "Sparse Search Result", + sellerName: "Example", + screenshotURLs: ["https://example.com/sparse-phone.png"], + platform: .iphone + ), + storefrontCode: "us", + rankingSource: .appStoreWeb, + fetchedAt: sparseDate, + requestedPlatform: .iphone, + in: modelContext + ) + try modelContext.save() + + metadata = try #require(modelContext.fetch(FetchDescriptor()).first { + $0.appStoreID == 352 + }) + let screenshot = try #require(modelContext.fetch(FetchDescriptor()).first { + $0.appStoreID == 352 && $0.platformRaw == "iphone" + }) + #expect(metadata.name == "Detailed Result") + #expect(metadata.subtitle == "Detailed subtitle") + #expect(metadata.source == .appStoreWeb) + #expect(metadata.lastFetchedAt == detailDate) + #expect(screenshot.urlString == "https://example.com/detail-phone.png") + #expect(screenshot.displayTypeRaw == "phone_6_9") + #expect(screenshot.width == 1_320) + #expect(screenshot.height == 2_868) + #expect(screenshot.source == .appStoreWeb) + #expect(screenshot.lastFetchedAt <= sparseDate) + } + + @Test + func newerSparsePlatformScreenshotsRejectDelayedFallbackEvidence() throws { + let container = try makeInMemoryContainer() + let modelContext = ModelContext(container) + let catalogService = AppCatalogService(appResolver: StubAppResolver()) + + _ = try catalogService.upsertStoreApp( + from: AppStoreWebMetadata( + appStoreID: 353, + storefront: "us", + name: "Detailed Result", + subtitle: "Detailed subtitle", + sellerName: "Example", + averageRating: nil, + ratingCount: nil, + screenshotGroups: [ + AppStoreWebScreenshotGroup( + platformRaw: "iphone", + displayTypeRaw: "phone_6_9", + screenshots: [ + AppStoreWebScreenshot( + urlString: "https://example.com/detail-phone.png", + width: 1_320, + height: 2_868 + ) + ] + ) + ] + ), + storefrontCode: "us", + in: modelContext + ) + try modelContext.save() + + let detailMetadata = try #require(modelContext.fetch(FetchDescriptor()).first { + $0.appStoreID == 353 + }) + let sparseDate = detailMetadata.lastFetchedAt.addingTimeInterval(120) + let delayedFallbackDate = detailMetadata.lastFetchedAt.addingTimeInterval(60) + _ = try catalogService.upsertStoreApp( + from: SearchRankingItem( + position: 1, + appStoreID: 353, + bundleID: "com.example.detail", + name: "Sparse Search Result", + sellerName: "Example", + ipadScreenshotURLs: ["https://example.com/new-ipad.png"], + platform: .ipad + ), + storefrontCode: "us", + rankingSource: .appStoreWeb, + fetchedAt: sparseDate, + requestedPlatform: .ipad, + in: modelContext + ) + _ = try catalogService.upsertStoreApp( + from: SearchRankingItem( + position: 1, + appStoreID: 353, + bundleID: "com.example.detail", + name: "Delayed Fallback Result", + sellerName: "Example", + ipadScreenshotURLs: ["https://example.com/old-ipad.png"], + platform: .ipad + ), + storefrontCode: "us", + rankingSource: .iTunesFallback, + fetchedAt: delayedFallbackDate, + requestedPlatform: .ipad, + in: modelContext + ) + try modelContext.save() + + let screenshots = try modelContext.fetch(FetchDescriptor()).filter { + $0.appStoreID == 353 + } + let ipadScreenshot = try #require(screenshots.first { $0.platformRaw == "ipad" }) + let iphoneScreenshot = try #require(screenshots.first { $0.platformRaw == "iphone" }) + #expect(ipadScreenshot.urlString == "https://example.com/new-ipad.png") + #expect(ipadScreenshot.source == .appStoreWebSearch) + #expect(ipadScreenshot.lastFetchedAt == sparseDate) + #expect(iphoneScreenshot.urlString == "https://example.com/detail-phone.png") + #expect(iphoneScreenshot.width == 1_320) + #expect(!screenshots.contains { $0.urlString == "https://example.com/old-ipad.png" }) + } + + @Test + func keywordSuggestionsUseTheNewestRankingEvidenceSource() throws { + let container = try makeInMemoryContainer() + let modelContext = ModelContext(container) + let now = Date(timeIntervalSince1970: 1_800_000_000) + let trackedApp = TrackedApp( + appStoreID: 400, + bundleID: "com.example.target", + name: "Target", + sellerName: "Example", + defaultPlatform: .iphone + ) + modelContext.insert(trackedApp) + + func insertEvidence( + keyword: String, + source: RankingSource, + observedAt: Date, + position: Int + ) throws { + let query = try KeywordQuery.fetchOrInsert( + term: keyword, + storefront: "us", + platform: .iphone, + in: modelContext + ) + let observation = KeywordRankingCrawl( + keyword: keyword, + storefront: "us", + platform: .iphone, + observedAt: observedAt, + source: source, + resultCount: 10, + query: query + ) + let item = KeywordAppRanking( + position: position, + appStoreID: trackedApp.appStoreID, + bundleID: trackedApp.bundleID, + name: trackedApp.name, + sellerName: trackedApp.sellerName, + observation: observation + ) + observation.items.append(item) + query.observations.append(observation) + modelContext.insert(observation) + modelContext.insert(item) + } + + try insertEvidence( + keyword: "calendar", + source: .iTunesFallback, + observedAt: now.addingTimeInterval(-86_400), + position: 3 + ) + try insertEvidence( + keyword: "planner", + source: .iTunesFallback, + observedAt: now.addingTimeInterval(-2 * 86_400), + position: 2 + ) + try insertEvidence( + keyword: "planner", + source: .appStoreWeb, + observedAt: now.addingTimeInterval(-86_400), + position: 4 + ) + try modelContext.save() + + let suggestions = try KeywordSuggestionService(now: { now }).suggestions( + for: trackedApp, + in: modelContext + ) + let byKeyword = Dictionary(uniqueKeysWithValues: suggestions.keywordSuggestions.map { + ($0.keyword, $0) + }) + + #expect(byKeyword["calendar"]?.source == .iTunesFallback) + #expect(byKeyword["planner"]?.source == .appStoreWeb) + #expect(byKeyword["planner"]?.currentObservedRank == 4) + #expect(byKeyword["planner"]?.bestObservedRank == 2) + } + @Test func refreshUpdatesExistingDailyRankingPeriod() async throws { let container = try makeInMemoryContainer() @@ -974,6 +2004,224 @@ struct AppDetailRefreshServiceQueueTests { #expect(secondResult.ratingOutcomes.map { $0.storefront } == ["us"]) } + @Test + func keywordBatchKeepsValidPagesWhenOptionalDifficultyConflicts() async throws { + let container = try makeInMemoryContainer() + let modelContext = ModelContext(container) + let trackedApp = TrackedApp( + appStoreID: 505, + bundleID: "example.batch", + name: "Batch App", + sellerName: "Example", + defaultPlatform: .iphone + ) + let failingTrack = try makeTrackedAppKeyword( + term: "pages", + storefront: "us", + trackedApp: trackedApp, + in: modelContext + ) + let successfulTrack = try makeTrackedAppKeyword( + term: "pages", + storefront: "gb", + trackedApp: trackedApp, + in: modelContext + ) + trackedApp.keywordTracks.append(contentsOf: [failingTrack, successfulTrack]) + modelContext.insert(trackedApp) + modelContext.insert(failingTrack) + modelContext.insert(successfulTrack) + try modelContext.save() + + let priorCalculationID = UUID( + uuidString: "00000000-0000-0000-0000-000000002025" + )! + let successfulCalculationID = UUID( + uuidString: "00000000-0000-0000-0000-000000002026" + )! + let priorDate = Date(timeIntervalSince1970: 100) + let priorPayload = try #require( + EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: RankingRefreshRequest(track: failingTrack), + page: SearchRankingPage( + items: [ + rankingItem(position: 1, appStoreID: 801, platform: .iphone), + rankingItem(position: 2, appStoreID: 802, platform: .iphone), + rankingItem(position: 3, appStoreID: 803, platform: .iphone), + ], + source: .appStoreWeb + ), + requestedResultLimit: 10, + rankingFetchedAt: priorDate, + calculationID: priorCalculationID, + now: { priorDate } + ) + ) + _ = try EstimatedKeywordDifficultyStore.upsert(priorPayload, in: modelContext) + try modelContext.save() + + let failingResultAppStoreID: Int64 = 911 + let successfulResultAppStoreID: Int64 = 922 + let provider = QueryRankingProvider(responses: [ + QueryRankingProvider.key( + term: failingTrack.term, + storefront: failingTrack.storefront, + platform: failingTrack.platform + ): .page(SearchRankingPage( + items: [ + rankingItem( + position: 1, + appStoreID: failingResultAppStoreID, + platform: .iphone + ), + ], + source: .iTunesFallback + )), + QueryRankingProvider.key( + term: successfulTrack.term, + storefront: successfulTrack.storefront, + platform: successfulTrack.platform + ): .page(SearchRankingPage( + items: [ + rankingItem( + position: 1, + appStoreID: successfulResultAppStoreID, + platform: .iphone + ), + ], + source: .iTunesFallback + )), + ]) + let failingQueryKey = failingTrack.queryKey + let coordinator = RankingRefreshCoordinator( + rankingProvider: provider, + appCatalogService: AppCatalogService(appResolver: StubAppResolver()), + estimatedDifficultyPayloadBuilder: { + request, + page, + requestedResultLimit, + rankingFetchedAt in + EstimatedKeywordDifficultyPersistenceAdapter.payload( + request: request, + page: page, + requestedResultLimit: requestedResultLimit, + rankingFetchedAt: rankingFetchedAt, + calculationID: request.queryKey == failingQueryKey + ? priorCalculationID + : successfulCalculationID + ) + } + ) + let httpClient = ControlledRatingsHTTPClient() + let service = AppDetailRefreshService( + backgroundModelStore: BackgroundModelStore(modelContainer: container), + refreshCoordinator: coordinator, + keywordMetricsService: KeywordMetricsService( + httpClient: httpClient, + credentialStore: AppleAdsCredentialStore( + defaults: makeDefaults(), + keychain: InMemoryKeychainService(), + loadsEnvironmentCredentials: false + ), + settingsStore: AppSettingsStore(defaults: makeDefaults()), + webSessionStore: AppleAdsWebSessionStore( + defaults: makeDefaults(), + keychain: InMemoryKeychainService() + ) + ), + appStorefrontRatingService: AppStorefrontRatingService(httpClient: httpClient), + appStorefrontReviewService: AppStorefrontReviewService(httpClient: httpClient), + appStoreConnectReviewService: AppStoreConnectReviewService( + httpClient: httpClient, + credentialStore: AppStoreConnectCredentialStore( + defaults: makeDefaults(), + keychain: InMemoryKeychainService() + ) + ) + ) + + let result = await service.refresh(AppDetailRefreshRequest( + app: AppDetailRefreshAppSnapshot( + appStoreID: trackedApp.appStoreID, + bundleID: trackedApp.bundleID, + name: trackedApp.name, + subtitle: trackedApp.subtitle, + sellerName: trackedApp.sellerName, + defaultPlatform: trackedApp.defaultPlatform + ), + workspace: .keywords, + storefrontSelection: .all(codes: ["us", "gb"]), + trackIdentityKeys: [failingTrack.identityKey, successfulTrack.identityKey], + trigger: "manual_refresh", + refreshKeywords: true, + refreshMetrics: false, + refreshRatings: false, + refreshReviews: false, + recordsRatingsReviewsRefresh: false, + popularityContextAppStoreID: nil, + appleAdsWebSession: nil, + appStoreConnectCredentials: AppStoreConnectCredentials( + issuerID: "", + keyID: "", + privateKey: "" + ) + )) + + let outcomesByIdentityKey = Dictionary( + uniqueKeysWithValues: result.keywordOutcomes.map { + ($0.trackIdentityKey, $0) + } + ) + #expect(outcomesByIdentityKey[failingTrack.identityKey]?.error == nil) + #expect(outcomesByIdentityKey[successfulTrack.identityKey]?.error == nil) + + let verificationContext = ModelContext(container) + let snapshots = try verificationContext.fetch( + FetchDescriptor() + ) + #expect(Set(snapshots.map(\.trackIdentityKey)) == Set([ + failingTrack.identityKey, + successfulTrack.identityKey, + ])) + let crawls = try verificationContext.fetch(FetchDescriptor()) + #expect(Set(crawls.map(\.queryKey)) == Set([ + failingTrack.queryKey, + successfulTrack.queryKey, + ])) + let observationItems = try verificationContext.fetch( + FetchDescriptor() + ) + #expect(observationItems.count == 2) + #expect(Set(observationItems.map(\.queryKey)) == Set([ + failingTrack.queryKey, + successfulTrack.queryKey, + ])) + let storeApps = try verificationContext.fetch(FetchDescriptor()) + #expect(storeApps.contains { $0.appStoreID == failingResultAppStoreID }) + #expect(storeApps.contains { $0.appStoreID == successfulResultAppStoreID }) + + let preservedDifficulty = try #require( + try EstimatedKeywordDifficultyStore.snapshot( + queryKey: failingTrack.queryKey, + in: verificationContext + ) + ) + #expect(preservedDifficulty.calculationID == priorCalculationID) + #expect(preservedDifficulty.rankingFetchedAt == priorDate) + #expect( + try EstimatedKeywordDifficultyStore.snapshot( + queryKey: successfulTrack.queryKey, + in: verificationContext + )?.calculationID == successfulCalculationID + ) + let statuses = try TrackedKeywordRefreshStatusStore.snapshots( + for: [failingTrack.identityKey, successfulTrack.identityKey], + in: verificationContext + ) + #expect(statuses[failingTrack.identityKey]?.rankingMessage == nil) + #expect(statuses[successfulTrack.identityKey]?.rankingMessage == nil) + } + private static func request(appStoreID: Int64, appName: String) -> AppDetailRefreshRequest { AppDetailRefreshRequest( app: AppDetailRefreshAppSnapshot( @@ -1159,6 +2407,46 @@ private actor QueryRankingProvider: SearchRankingProvider { } } +private actor ControlledRankingProvider: SearchRankingProvider { + private var didReceiveRequest = false + private var requestWaiters: [CheckedContinuation] = [] + private var resultContinuation: CheckedContinuation? + + func search( + keyword: String, + storefrontCode: String, + platform: AppPlatform, + limit: Int + ) async throws -> SearchRankingPage { + didReceiveRequest = true + let waiters = requestWaiters + requestWaiters.removeAll() + for waiter in waiters { + waiter.resume() + } + return try await withCheckedThrowingContinuation { continuation in + resultContinuation = continuation + } + } + + func waitUntilRequested() async { + guard !didReceiveRequest else { return } + await withCheckedContinuation { continuation in + requestWaiters.append(continuation) + } + } + + func fail(with error: OpenASOError) { + resultContinuation?.resume(throwing: error) + resultContinuation = nil + } + + func succeed(with page: SearchRankingPage) { + resultContinuation?.resume(returning: page) + resultContinuation = nil + } +} + @MainActor private final class StubRankingProvider: SearchRankingProvider { var page: SearchRankingPage @@ -1269,6 +2557,8 @@ private func makeInMemoryContainer() throws -> ModelContainer { TrackedKeywordRefreshStatus.self, TrackedKeywordDailyRanking.self, TrackedKeywordRankedResult.self, + EstimatedKeywordDifficultyMetric.self, + EstimatedKeywordDifficultyResultEvidenceRecord.self, Storefront.self ]) let configuration = ModelConfiguration(schema: schema, isStoredInMemoryOnly: true) diff --git a/OpenASOTests/RefreshObservabilityTests.swift b/OpenASOTests/RefreshObservabilityTests.swift index 64a3a26..48afeba 100644 --- a/OpenASOTests/RefreshObservabilityTests.swift +++ b/OpenASOTests/RefreshObservabilityTests.swift @@ -6,6 +6,29 @@ import Testing @MainActor struct RefreshObservabilityTests { + @Test + func appStoreWebSearchPathsAreRankingEndpointsWhileAppPagesRemainRatingsPages() throws { + for urlString in [ + "https://apps.apple.com/us/iphone/search?term=notes", + "https://apps.apple.com/gb/ipad/search?term=notes", + "https://apps.apple.com/jp/mac/search?term=notes", + ] { + let classification = RefreshRequestClassification(URL(string: urlString)!) + #expect(classification.provider == .appStoreWeb) + #expect(classification.endpoint == .rankingSearch) + } + + for urlString in [ + "https://apps.apple.com/us/app/example/id123", + "https://apps.apple.com/us/iphone/app/example/id123", + "https://apps.apple.com/us/search?term=notes", + ] { + let classification = RefreshRequestClassification(URL(string: urlString)!) + #expect(classification.provider == .appStoreWeb) + #expect(classification.endpoint == .ratingsPage) + } + } + @Test func requestsOutsideRefreshScopeAreNotRecorded() async throws { let recorder = RefreshMetricsRecorder(clock: .constant) @@ -428,7 +451,9 @@ struct RefreshObservabilityTests { #expect(secondResult.keywordOutcomes.count == keywordCount + 1) #expect(secondResult.keywordOutcomes.filter { $0.error != nil }.count == 1) #expect(summaries.count == 2) - #expect(await base.requestCount() == keywordCount * 2) + // The fixture intentionally returns iTunes JSON for every URL. Web + // parsing therefore fails safely and each query exercises fallback. + #expect(await base.requestCount() == keywordCount * 4) #expect(persistedTrackCount == keywordCount) #expect(persistedQueryCount == keywordCount) @@ -437,7 +462,8 @@ struct RefreshObservabilityTests { let metrics = try #require(summary.stages[.keywordMetrics]) let ratings = try #require(summary.stages[.ratings]) let reviews = try #require(summary.stages[.reviews]) - let provider = try #require(summary.providers[.iTunesStore]) + let webProvider = try #require(summary.providers[.appStoreWeb]) + let fallbackProvider = try #require(summary.providers[.iTunesStore]) #expect(summary.requestedTrackCount == keywordCount + 1) #expect(summary.requestedStorefrontCount == 2) @@ -451,9 +477,12 @@ struct RefreshObservabilityTests { #expect(metrics.isSkipped) #expect(ratings.isSkipped) #expect(reviews.isSkipped) - #expect(provider.requestCount == keywordCount) - #expect(provider.endpointCounts[.rankingSearch] == keywordCount) - #expect(provider.resultCounts[.success] == keywordCount) + #expect(webProvider.requestCount == keywordCount) + #expect(webProvider.endpointCounts[.rankingSearch] == keywordCount) + #expect(webProvider.resultCounts[.success] == keywordCount) + #expect(fallbackProvider.requestCount == keywordCount) + #expect(fallbackProvider.endpointCounts[.rankingSearch] == keywordCount) + #expect(fallbackProvider.resultCounts[.success] == keywordCount) #expect(summary.result == .partialFailure) } } @@ -486,6 +515,8 @@ struct RefreshObservabilityTests { let summary = try #require(await fixture.recorder.completedSummaries().only) let rankings = try #require(summary.stages[.rankings]) let metrics = try #require(summary.stages[.keywordMetrics]) + let webProvider = try #require(summary.providers[.appStoreWeb]) + let fallbackProvider = try #require(summary.providers[.iTunesStore]) #expect(result.keywordOutcomes.count == 3) #expect(result.keywordOutcomes.allSatisfy { $0.error == nil }) @@ -495,7 +526,9 @@ struct RefreshObservabilityTests { #expect(metrics.attemptedCount == 3) #expect(metrics.failureCount == 1) #expect(summary.result == .partialFailure) - #expect(await httpClient.rankingRequestCount() == 3) + #expect(webProvider.requestCount == 3) + #expect(fallbackProvider.requestCount == 3) + #expect(await httpClient.rankingRequestCount() == 6) #expect(await httpClient.popularityRequestCount() == 1) } @@ -536,7 +569,8 @@ struct RefreshObservabilityTests { let result = await fixture.service.refresh(fixture.request) let summary = try #require(await fixture.recorder.completedSummaries().only) let rankings = try #require(summary.stages[.rankings]) - let provider = try #require(summary.providers[.iTunesStore]) + let webProvider = try #require(summary.providers[.appStoreWeb]) + let fallbackProvider = try #require(summary.providers[.iTunesStore]) let persisted = try await fixture.backgroundModelStore.read { modelContext in let tracks = try modelContext.fetch(FetchDescriptor()) let snapshots = try modelContext.fetch(FetchDescriptor()) @@ -557,13 +591,15 @@ struct RefreshObservabilityTests { #expect(result.keywordOutcomes.count == 4) #expect(result.keywordOutcomes.allSatisfy { $0.error == nil }) - #expect(await base.requestCount() == 3) + #expect(await base.requestCount() == 6) #expect(summary.resolvedRankingCount == 4) #expect(summary.uniqueRankingQueryCount == 3) #expect(rankings.attemptedCount == 4) #expect(rankings.successCount == 4) - #expect(provider.requestCount == 3) - #expect(provider.endpointCounts[.rankingSearch] == 3) + #expect(webProvider.requestCount == 3) + #expect(webProvider.endpointCounts[.rankingSearch] == 3) + #expect(fallbackProvider.requestCount == 3) + #expect(fallbackProvider.endpointCounts[.rankingSearch] == 3) #expect(persisted.trackCount == 4) #expect(persisted.refreshedTrackCount == 4) #expect(persisted.snapshotIdentityKeys == Set(result.keywordOutcomes.map(\.trackIdentityKey))) @@ -599,7 +635,7 @@ struct RefreshObservabilityTests { let result = await fixture.service.refresh(fixture.request) let summary = try #require(await fixture.recorder.completedSummaries().only) let rankings = try #require(summary.stages[.rankings]) - let provider = try #require(summary.providers[.iTunesStore]) + let provider = try #require(summary.providers[.appStoreWeb]) let persisted = try await fixture.backgroundModelStore.read { modelContext in let tracks = try modelContext.fetch(FetchDescriptor()) let snapshots = try modelContext.fetch(FetchDescriptor()) @@ -642,7 +678,7 @@ struct RefreshObservabilityTests { let result = await fixture.service.refresh(fixture.request) let summary = try #require(await fixture.recorder.completedSummaries().only) let rankings = try #require(summary.stages[.rankings]) - let provider = try #require(summary.providers[.iTunesStore]) + let provider = try #require(summary.providers[.appStoreWeb]) let persistedFailureCount = try await fixture.backgroundModelStore.read { modelContext in let tracks = try modelContext.fetch(FetchDescriptor()) let statuses = try TrackedKeywordRefreshStatusStore.snapshots( diff --git a/OpenASOTests/TrackedKeywordDeletionServiceTests.swift b/OpenASOTests/TrackedKeywordDeletionServiceTests.swift new file mode 100644 index 0000000..7a6951c --- /dev/null +++ b/OpenASOTests/TrackedKeywordDeletionServiceTests.swift @@ -0,0 +1,525 @@ +import Foundation +import SwiftData +import Testing +@testable import OpenASO + +@MainActor +struct TrackedKeywordDeletionServiceTests { + @Test + func appDeletionRemovesOwnedStateAndPreservesSharedQueryUntilLastTrack() async throws { + let container = try ModelContainerFactory.makeModelContainer( + isStoredInMemoryOnly: true + ) + let fixture = try makeSharedQueryFixture(in: container) + let store = BackgroundModelStore(modelContainer: container) + + let appDeletion = try await TrackedKeywordDeletionService.deleteApp( + fixture.firstAppRequest, + using: store + ) + + #expect(appDeletion.deletedApp) + #expect(appDeletion.deletedTrackCount == 1) + #expect(appDeletion.keywordDeletion.deletedQueryKeys.isEmpty) + + var verificationContext = ModelContext(container) + #expect(try verificationContext.fetch(FetchDescriptor()).map(\.appStoreID) == [202]) + #expect(try verificationContext.fetch(FetchDescriptor()).map(\.identityKey) == [ + fixture.secondTrackRequest.identityKey + ]) + #expect(try verificationContext.fetch( + FetchDescriptor() + ).map(\.trackIdentityKey) == [fixture.secondTrackRequest.identityKey]) + #expect(try verificationContext.fetch( + FetchDescriptor() + ).map(\.trackIdentityKey) == [fixture.secondTrackRequest.identityKey]) + #expect(try verificationContext.fetch( + FetchDescriptor() + ).map(\.trackIdentityKey) == [fixture.secondTrackRequest.identityKey]) + #expect(try verificationContext.fetchCount( + FetchDescriptor() + ) == 1) + + #expect(try verificationContext.fetchCount(FetchDescriptor()) == 1) + #expect(try verificationContext.fetchCount(FetchDescriptor()) == 1) + #expect(try verificationContext.fetchCount(FetchDescriptor()) == 1) + #expect(try verificationContext.fetchCount(FetchDescriptor()) == 1) + #expect(try verificationContext.fetchCount(FetchDescriptor()) == 1) + #expect(try verificationContext.fetchCount( + FetchDescriptor() + ) == 1) + #expect(try verificationContext.fetchCount( + FetchDescriptor() + ) == 3) + + let finalDeletion = try await TrackedKeywordDeletionService.deleteTracks( + [fixture.secondTrackRequest], + using: store + ) + + #expect(finalDeletion.deletedTrackCount == 1) + #expect(finalDeletion.deletedQueryKeys == [fixture.queryKey]) + + verificationContext = ModelContext(container) + #expect(try verificationContext.fetchCount(FetchDescriptor()) == 1) + #expect(try verificationContext.fetchCount(FetchDescriptor()) == 0) + #expect(try verificationContext.fetchCount( + FetchDescriptor() + ) == 0) + #expect(try verificationContext.fetchCount( + FetchDescriptor() + ) == 0) + #expect(try verificationContext.fetchCount( + FetchDescriptor() + ) == 0) + #expect(try verificationContext.fetchCount( + FetchDescriptor() + ) == 0) + // Historical query/ranking/popularity data remains available for audit + // views. The current heuristic is the generation-bound state removed + // when its final active track goes away. + #expect(try verificationContext.fetchCount(FetchDescriptor()) == 1) + #expect(try verificationContext.fetchCount(FetchDescriptor()) == 1) + #expect(try verificationContext.fetchCount(FetchDescriptor()) == 1) + #expect(try verificationContext.fetchCount(FetchDescriptor()) == 1) + #expect(try verificationContext.fetchCount(FetchDescriptor()) == 1) + #expect(try verificationContext.fetchCount( + FetchDescriptor() + ) == 0) + #expect(try verificationContext.fetchCount( + FetchDescriptor() + ) == 0) + } + + @Test + func delayedOldGenerationCannotPersistIntoReaddedTrack() async throws { + let container = try ModelContainerFactory.makeModelContainer( + isStoredInMemoryOnly: true + ) + let fixture = try makeOldGenerationFixture(in: container) + let store = BackgroundModelStore(modelContainer: container) + + let deletion = try await TrackedKeywordDeletionService.deleteTracks( + [fixture.deletionRequest], + using: store + ) + #expect(deletion.deletedTrackCount == 1) + + let readdedAt = fixture.oldTrackCreatedAt.addingTimeInterval(60) + let readdContext = ModelContext(container) + let appStoreID = fixture.appStoreID + let trackedApp = try #require(readdContext.fetch(FetchDescriptor( + predicate: #Predicate { app in + app.appStoreID == appStoreID + } + )).first) + let query = try KeywordQuery.fetchOrInsert( + term: "focus timer", + storefront: "us", + platform: .iphone, + in: readdContext + ) + let readdedTrack = TrackedAppKeyword( + term: "focus timer", + storefront: "us", + platform: .iphone, + trackedApp: trackedApp, + query: query, + createdAt: readdedAt + ) + trackedApp.keywordTracks.append(readdedTrack) + readdContext.insert(readdedTrack) + try readdContext.save() + + let delayedPage = SearchRankingPage( + items: rankingItems, + source: .appStoreWeb + ) + let delayedResult = RankingRefreshPageResult( + request: fixture.rankingRequest, + page: delayedPage, + searchedAt: Date(timeIntervalSince1970: 5_000), + observedHour: nil, + submissionCount: 1, + winningCount: 1, + confidence: "single_source", + requestedResultLimit: 10, + estimatedDifficultyPayload: difficultyPayload( + queryKey: fixture.queryKey, + rankingFetchedAt: Date(timeIntervalSince1970: 5_000) + ) + ) + let coordinator = RankingRefreshCoordinator( + rankingProvider: DeletionTestRankingProvider(), + appCatalogService: AppCatalogService( + appResolver: DeletionTestAppResolver() + ) + ) + let delayedContext = ModelContext(container) + + #expect(throws: OpenASOError.providerUnavailable( + "The keyword changed while its ranking refresh was in flight. Refresh it again." + )) { + try coordinator.persistRankingPage( + delayedResult, + in: delayedContext + ) + } + let failureResult = try coordinator.recordRefreshFailure( + identityKey: fixture.rankingRequest.identityKey, + trackCreatedAt: fixture.rankingRequest.trackCreatedAt, + error: .networkUnavailable, + in: delayedContext + ) + #expect(failureResult == nil) + try delayedContext.save() + + let verificationContext = ModelContext(container) + let persistedTracks = try verificationContext.fetch( + FetchDescriptor() + ) + #expect(persistedTracks.count == 1) + #expect(persistedTracks.first?.createdAt == readdedAt) + #expect(try verificationContext.fetchCount( + FetchDescriptor() + ) == 0) + #expect(try verificationContext.fetchCount(FetchDescriptor()) == 0) + #expect(try verificationContext.fetchCount( + FetchDescriptor() + ) == 0) + #expect(try verificationContext.fetchCount( + FetchDescriptor() + ) == 0) + #expect(try verificationContext.fetchCount( + FetchDescriptor() + ) == 0) + } + + private func makeSharedQueryFixture( + in container: ModelContainer + ) throws -> SharedQueryFixture { + let context = ModelContext(container) + let firstApp = TrackedApp( + appStoreID: 101, + bundleID: "com.example.first", + name: "First", + sellerName: "Example", + defaultPlatform: .iphone, + createdAt: Date(timeIntervalSince1970: 100) + ) + let secondApp = TrackedApp( + appStoreID: 202, + bundleID: "com.example.second", + name: "Second", + sellerName: "Example", + defaultPlatform: .iphone, + createdAt: Date(timeIntervalSince1970: 200) + ) + context.insert(firstApp) + context.insert(secondApp) + + let query = try KeywordQuery.fetchOrInsert( + term: "focus timer", + storefront: "us", + platform: .iphone, + in: context + ) + let firstTrack = makeTrack( + app: firstApp, + query: query, + createdAt: Date(timeIntervalSince1970: 300), + in: context + ) + let secondTrack = makeTrack( + app: secondApp, + query: query, + createdAt: Date(timeIntervalSince1970: 400), + in: context + ) + try addTrackRanking( + to: firstTrack, + searchedAt: Date(timeIntervalSince1970: 1_000), + rankedAppStoreID: 901, + in: context + ) + try addTrackRanking( + to: secondTrack, + searchedAt: Date(timeIntervalSince1970: 2_000), + rankedAppStoreID: 902, + in: context + ) + + let crawl = KeywordRankingCrawl( + keyword: "focus timer", + storefront: "us", + platform: .iphone, + observedAt: Date(timeIntervalSince1970: 2_000), + source: .appStoreWeb, + resultCount: 1, + query: query + ) + context.insert(crawl) + let ranking = KeywordAppRanking( + position: 1, + appStoreID: 902, + bundleID: "com.example.ranking", + name: "Ranking App", + sellerName: "Example", + observation: crawl + ) + crawl.items.append(ranking) + context.insert(ranking) + context.insert(KeywordDailyMetric( + queryKey: query.queryKey, + keyword: query.term, + storefront: query.storefront, + platform: query.platform, + popularityScore: 55, + difficultyScore: nil, + source: .appleAdsPopularity, + updatedAt: Date(timeIntervalSince1970: 2_000) + )) + context.insert(AppKeywordStats( + appStoreID: ranking.appStoreID, + queryKey: query.queryKey, + keyword: query.term, + storefront: query.storefront, + platform: query.platform, + rank: ranking.position, + observedAt: crawl.observedAt + )) + try TrackedKeywordRefreshStatusStore.set( + "First status", + domain: .ranking, + for: firstTrack, + in: context + ) + try TrackedKeywordRefreshStatusStore.set( + "Second status", + domain: .ranking, + for: secondTrack, + in: context + ) + context.insert(TrackedAppKeywordRefreshAttempt( + trackIdentityKey: firstTrack.identityKey, + appStoreID: firstTrack.appStoreID, + lastRankingRefreshAttemptAt: Date(timeIntervalSince1970: 2_100) + )) + context.insert(TrackedAppKeywordRefreshAttempt( + trackIdentityKey: secondTrack.identityKey, + appStoreID: secondTrack.appStoreID, + lastRankingRefreshAttemptAt: Date(timeIntervalSince1970: 2_200) + )) + _ = try EstimatedKeywordDifficultyStore.upsert( + difficultyPayload( + queryKey: query.queryKey, + rankingFetchedAt: crawl.observedAt + ), + in: context + ) + try context.save() + + return SharedQueryFixture( + firstAppRequest: TrackedAppDeletionRequest(app: firstApp), + secondTrackRequest: TrackedKeywordDeletionRequest(track: secondTrack), + queryKey: query.queryKey + ) + } + + private func makeOldGenerationFixture( + in container: ModelContainer + ) throws -> OldGenerationFixture { + let context = ModelContext(container) + let app = TrackedApp( + appStoreID: 303, + bundleID: "com.example.generation", + name: "Generation", + sellerName: "Example", + defaultPlatform: .iphone, + createdAt: Date(timeIntervalSince1970: 100) + ) + context.insert(app) + let query = try KeywordQuery.fetchOrInsert( + term: "focus timer", + storefront: "us", + platform: .iphone, + in: context + ) + let createdAt = Date(timeIntervalSince1970: 300) + let track = makeTrack( + app: app, + query: query, + createdAt: createdAt, + in: context + ) + try context.save() + + return OldGenerationFixture( + appStoreID: app.appStoreID, + queryKey: query.queryKey, + oldTrackCreatedAt: createdAt, + deletionRequest: TrackedKeywordDeletionRequest(track: track), + rankingRequest: RankingRefreshRequest(track: track) + ) + } + + private func makeTrack( + app: TrackedApp, + query: KeywordQuery, + createdAt: Date, + in context: ModelContext + ) -> TrackedAppKeyword { + let track = TrackedAppKeyword( + term: query.term, + storefront: query.storefront, + platform: query.platform, + trackedApp: app, + query: query, + createdAt: createdAt + ) + app.keywordTracks.append(track) + context.insert(track) + return track + } + + private func addTrackRanking( + to track: TrackedAppKeyword, + searchedAt: Date, + rankedAppStoreID: Int64, + in context: ModelContext + ) throws { + let snapshot = TrackedKeywordDailyRanking( + rank: 1, + searchedAt: searchedAt, + source: .appStoreWeb, + resultCount: 1, + keywordTrack: track + ) + track.snapshots.append(snapshot) + context.insert(snapshot) + let result = TrackedKeywordRankedResult( + position: 1, + appStoreID: rankedAppStoreID, + bundleID: "com.example.\(rankedAppStoreID)", + name: "Ranked \(rankedAppStoreID)", + sellerName: "Example", + snapshot: snapshot + ) + snapshot.topResults.append(result) + context.insert(result) + } + + private var rankingItems: [SearchRankingItem] { + (1 ... 3).map { position in + SearchRankingItem( + position: position, + appStoreID: Int64(1_000 + position), + bundleID: "com.example.\(position)", + name: "Focus Timer \(position)", + subtitle: "Focus Timer", + sellerName: "Example", + ratingCount: position * 100, + averageRating: 4.5 + ) + } + } + + private func difficultyPayload( + queryKey: String, + rankingFetchedAt: Date + ) -> EstimatedKeywordDifficultyPersistencePayload { + let evidenceItems = rankingItems.map { item in + EstimatedKeywordDifficultyResultEvidence( + position: item.position, + appStoreID: item.appStoreID, + title: item.name, + subtitle: item.subtitle, + ratingCount: item.ratingCount, + ratingAuthorityScore: 50 + item.position, + titleTokenCoveragePercentage: 100, + combinedTokenCoveragePercentage: 100, + metadataMatchScore: 100, + exactTitlePhraseMatch: true, + exactSubtitlePhraseMatch: true + ) + } + return EstimatedKeywordDifficultyPersistencePayload( + queryKey: queryKey, + calculationID: UUID(uuidString: "00000000-0000-0000-0000-0000000000de")!, + keyword: "focus timer", + storefront: "us", + platform: .iphone, + result: .estimated( + score: 65, + confidenceScore: 80, + confidence: .medium + ), + algorithmIdentifier: "top10-authority-saturation", + algorithmVersion: 1, + requestedResultLimit: 10, + providerResultCount: evidenceItems.count, + evidence: EstimatedKeywordDifficultyEvidence( + consideredResultCount: evidenceItems.count, + ratedResultCount: evidenceItems.count, + weightedRatingCoveragePercentage: 100, + maximumRatingCount: 300, + medianRatingCount: 200, + ratingAuthorityScore: 65, + metadataSaturationScore: 100, + resultEvidence: evidenceItems + ), + rankingSource: .appStoreWeb, + rankingFetchedAt: rankingFetchedAt, + computedAt: rankingFetchedAt.addingTimeInterval(1), + notes: ["Test evidence"] + ) + } +} + +private struct SharedQueryFixture { + let firstAppRequest: TrackedAppDeletionRequest + let secondTrackRequest: TrackedKeywordDeletionRequest + let queryKey: String +} + +private struct OldGenerationFixture { + let appStoreID: Int64 + let queryKey: String + let oldTrackCreatedAt: Date + let deletionRequest: TrackedKeywordDeletionRequest + let rankingRequest: RankingRefreshRequest +} + +private actor DeletionTestRankingProvider: SearchRankingProvider { + func search( + keyword: String, + storefrontCode: String, + platform: AppPlatform, + limit: Int + ) async throws -> SearchRankingPage { + SearchRankingPage(items: [], source: .appStoreWeb) + } +} + +private actor DeletionTestAppResolver: AppResolver { + func resolve( + appStoreID: Int64, + storefrontCode: String + ) async throws -> ResolvedApp { + ResolvedApp( + appStoreID: appStoreID, + bundleID: "com.example.\(appStoreID)", + name: "Example", + sellerName: "Example", + defaultPlatform: .iphone + ) + } + + func searchApps( + named query: String, + storefrontCode: String, + limit: Int + ) async throws -> [ResolvedApp] { + [] + } +}