chore(tinybird): update tb pipes to get repos from unified ds [CM-912] (#3788)

mbani01 · web-flow · commit 54ce49698a14 · 2026-01-28T16:39:31.000+01:00
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_0.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_0.pipe
@@ -36,12 +36,15 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND channel
-                IN (SELECT arrayJoin(i.repositories) FROM insightsProjects i where isNull (i.deletedAt))
-                AND activityRelations_bucket_MV_ds_0.segmentId IN (
-                    SELECT segmentId
+                AND (channel, activityRelations_bucket_MV_ds_0.segmentId) IN (
+                    SELECT r.url, r.segmentId
                     FROM repositories r FINAL
-                    WHERE (r.excluded IS NULL OR r.excluded = false)
+                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
+                    WHERE
+                        isNull (r.deletedAt)
+                        AND isNull (i.deletedAt)
+                        AND r.excluded = false
+                        AND r.enabled = true
                 )
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_1.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_1.pipe
@@ -36,12 +36,15 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND channel
-                IN (SELECT arrayJoin(i.repositories) FROM insightsProjects i where isNull (i.deletedAt))
-                AND activityRelations_bucket_MV_ds_1.segmentId IN (
-                    SELECT segmentId
+                AND (channel, activityRelations_bucket_MV_ds_1.segmentId) IN (
+                    SELECT r.url, r.segmentId
                     FROM repositories r FINAL
-                    WHERE (r.excluded IS NULL OR r.excluded = false)
+                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
+                    WHERE
+                        isNull (r.deletedAt)
+                        AND isNull (i.deletedAt)
+                        AND r.excluded = false
+                        AND r.enabled = true
                 )
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_2.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_2.pipe
@@ -36,12 +36,15 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND channel
-                IN (SELECT arrayJoin(i.repositories) FROM insightsProjects i where isNull (i.deletedAt))
-                AND activityRelations_bucket_MV_ds_2.segmentId IN (
-                    SELECT segmentId
+                AND (channel, activityRelations_bucket_MV_ds_2.segmentId) IN (
+                    SELECT r.url, r.segmentId
                     FROM repositories r FINAL
-                    WHERE (r.excluded IS NULL OR r.excluded = false)
+                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
+                    WHERE
+                        isNull (r.deletedAt)
+                        AND isNull (i.deletedAt)
+                        AND r.excluded = false
+                        AND r.enabled = true
                 )
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_3.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_3.pipe
@@ -36,12 +36,15 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND channel
-                IN (SELECT arrayJoin(i.repositories) FROM insightsProjects i where isNull (i.deletedAt))
-                AND activityRelations_bucket_MV_ds_3.segmentId IN (
-                    SELECT segmentId
+                AND (channel, activityRelations_bucket_MV_ds_3.segmentId) IN (
+                    SELECT r.url, r.segmentId
                     FROM repositories r FINAL
-                    WHERE (r.excluded IS NULL OR r.excluded = false)
+                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
+                    WHERE
+                        isNull (r.deletedAt)
+                        AND isNull (i.deletedAt)
+                        AND r.excluded = false
+                        AND r.enabled = true
                 )
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_4.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_4.pipe
@@ -36,12 +36,15 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND channel
-                IN (SELECT arrayJoin(i.repositories) FROM insightsProjects i where isNull (i.deletedAt))
-                AND activityRelations_bucket_MV_ds_4.segmentId IN (
-                    SELECT segmentId
+                AND (channel, activityRelations_bucket_MV_ds_4.segmentId) IN (
+                    SELECT r.url, r.segmentId
                     FROM repositories r FINAL
-                    WHERE (r.excluded IS NULL OR r.excluded = false)
+                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
+                    WHERE
+                        isNull (r.deletedAt)
+                        AND isNull (i.deletedAt)
+                        AND r.excluded = false
+                        AND r.enabled = true
                 )
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_5.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_5.pipe
@@ -36,12 +36,15 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND channel
-                IN (SELECT arrayJoin(i.repositories) FROM insightsProjects i where isNull (i.deletedAt))
-                AND activityRelations_bucket_MV_ds_5.segmentId IN (
-                    SELECT segmentId
+                AND (channel, activityRelations_bucket_MV_ds_5.segmentId) IN (
+                    SELECT r.url, r.segmentId
                     FROM repositories r FINAL
-                    WHERE (r.excluded IS NULL OR r.excluded = false)
+                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
+                    WHERE
+                        isNull (r.deletedAt)
+                        AND isNull (i.deletedAt)
+                        AND r.excluded = false
+                        AND r.enabled = true
                 )
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_6.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_6.pipe
@@ -36,12 +36,15 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND channel
-                IN (SELECT arrayJoin(i.repositories) FROM insightsProjects i where isNull (i.deletedAt))
-                AND activityRelations_bucket_MV_ds_6.segmentId IN (
-                    SELECT segmentId
+                AND (channel, activityRelations_bucket_MV_ds_6.segmentId) IN (
+                    SELECT r.url, r.segmentId
                     FROM repositories r FINAL
-                    WHERE (r.excluded IS NULL OR r.excluded = false)
+                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
+                    WHERE
+                        isNull (r.deletedAt)
+                        AND isNull (i.deletedAt)
+                        AND r.excluded = false
+                        AND r.enabled = true
                 )
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_7.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_7.pipe
@@ -36,12 +36,15 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND channel
-                IN (SELECT arrayJoin(i.repositories) FROM insightsProjects i where isNull (i.deletedAt))
-                AND activityRelations_bucket_MV_ds_7.segmentId IN (
-                    SELECT segmentId
+                AND (channel, activityRelations_bucket_MV_ds_7.segmentId) IN (
+                    SELECT r.url, r.segmentId
                     FROM repositories r FINAL
-                    WHERE (r.excluded IS NULL OR r.excluded = false)
+                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
+                    WHERE
+                        isNull (r.deletedAt)
+                        AND isNull (i.deletedAt)
+                        AND r.excluded = false
+                        AND r.enabled = true
                 )
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_8.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_8.pipe
@@ -36,12 +36,15 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND channel
-                IN (SELECT arrayJoin(i.repositories) FROM insightsProjects i where isNull (i.deletedAt))
-                AND activityRelations_bucket_MV_ds_8.segmentId IN (
-                    SELECT segmentId
+                AND (channel, activityRelations_bucket_MV_ds_8.segmentId) IN (
+                    SELECT r.url, r.segmentId
                     FROM repositories r FINAL
-                    WHERE (r.excluded IS NULL OR r.excluded = false)
+                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
+                    WHERE
+                        isNull (r.deletedAt)
+                        AND isNull (i.deletedAt)
+                        AND r.excluded = false
+                        AND r.enabled = true
                 )
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_9.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_9.pipe
@@ -36,12 +36,15 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND channel
-                IN (SELECT arrayJoin(i.repositories) FROM insightsProjects i where isNull (i.deletedAt))
-                AND activityRelations_bucket_MV_ds_9.segmentId IN (
-                    SELECT segmentId
+                AND (channel, activityRelations_bucket_MV_ds_9.segmentId) IN (
+                    SELECT r.url, r.segmentId
                     FROM repositories r FINAL
-                    WHERE (r.excluded IS NULL OR r.excluded = false)
+                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
+                    WHERE
+                        isNull (r.deletedAt)
+                        AND isNull (i.deletedAt)
+                        AND r.excluded = false
+                        AND r.enabled = true
                 )
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
diff --git a/services/libs/tinybird/pipes/activityRelations_deduplicated_cleaned_copy_pipe.pipe b/services/libs/tinybird/pipes/activityRelations_deduplicated_cleaned_copy_pipe.pipe
diff --git a/services/libs/tinybird/pipes/activityRepositories_filtered.pipe b/services/libs/tinybird/pipes/activityRepositories_filtered.pipe
@@ -12,22 +12,27 @@ DESCRIPTION >
 NODE activity_repositories_filtered_0
 SQL >
     %
-    select
-        id as "projectId", name as "projectName", slug as "projectSlug", arrayJoin(repositories) as repo
-    from insightsProjects final
+    select i.id as "projectId", i.name as "projectName", i.slug as "projectSlug", r.url as repo
+    from insightsProjects i final
+    JOIN
+        repositories r FINAL
+        ON r.insightsProjectId = i.id
+        AND isNull (r.deletedAt)
+        AND r.enabled = true
+        AND r.excluded = false
     where
-        isNull (deletedAt)
+        isNull (i.deletedAt)
         {% if defined(repos) %}
-            AND repo
+            AND r.url
             IN {{ Array(repos, 'String', description="Filter activity repo list", required=False) }}
         {% end %}
         {% if defined(search) %}
-            AND repo
+            AND r.url
             like 'https://%'
             || {{ String(search, description="Search repos open ended wildcard ", required=False) }}
             || '%'
         {% end %}
         {% if defined(projectSlug) %}
-            AND projectSlug
+            AND i.slug
             = {{ String(projectSlug, description="Search repos using projectSlug", required=False) }}
         {% end %}
diff --git a/services/libs/tinybird/pipes/insights_projects_populated_copy.pipe b/services/libs/tinybird/pipes/insights_projects_populated_copy.pipe
@@ -66,10 +66,15 @@ SQL >
 
 NODE insights_projects_populated_copy_flatten_projects
 SQL >
-    SELECT id, repo AS repository
-    FROM insightsProjects FINAL ARRAY
-    JOIN repositories AS repo
-    where isNull (insightsProjects.deletedAt)
+    SELECT i.id, r.url AS repository
+    FROM insightsProjects i FINAL
+    JOIN
+        repositories r FINAL
+        ON r.insightsProjectId = i.id
+        AND isNull (r.deletedAt)
+        AND r.enabled = true
+        AND r.excluded = false
+    WHERE isNull (i.deletedAt)
 
 NODE insights_projects_populated_copy_repository_criticality
 SQL >
@@ -90,7 +95,8 @@ SQL >
         id as id,
         max(score) as projectScore,
         argMax(rank, score) AS projectRank,
-        groupArray((repository, score, rank)) as repoData
+        groupArray((repository, score, rank)) as repoData,
+        groupArray(repository) as repositories
     FROM insights_projects_populated_copy_repository_criticality
     GROUP BY id
 
@@ -154,7 +160,7 @@ SQL >
         insightsProjects.linkedin as linkedin,
         insightsProjects.twitter as twitter,
         insightsProjects.widgets as widgets,
-        insightsProjects.repositories as repositories,
+        insights_projects_populated_copy_project_repo_data.repositories as repositories,
         insightsProjects.enabled as enabled,
         insightsProjects.isLF as isLF,
         insightsProjects.keywords as keywords,
diff --git a/services/libs/tinybird/pipes/segments_filtered.pipe b/services/libs/tinybird/pipes/segments_filtered.pipe
@@ -15,17 +15,25 @@ TAGS "Infrastructure", "Core", "Project scoping", "Security"
 NODE segments_filtered_0
 SQL >
     %
-    SELECT id as insightsProjectId, segmentId, repositories
-    FROM insightsProjects FINAL
-    where
-        isNull (insightsProjects.deletedAt) AND insightsProjects.enabled
+    SELECT i.id as insightsProjectId, i.segmentId, groupArrayIf(r.url, r.url != '') as repositories
+    FROM insightsProjects i FINAL
+    LEFT JOIN
+        repositories r FINAL
+        ON r.insightsProjectId = i.id
+        AND isNull (r.deletedAt)
+        AND r.excluded = false
+        AND r.enabled = true
+    WHERE
+        isNull (i.deletedAt) AND i.enabled
         {% if defined(project) %}
-            AND slug = {{ String(project, description="Filter by project slug", required=True) }}
+            AND i.slug = {{ String(project, description="Filter by project slug", required=True) }}
         {% else %} AND false
         {% end %}
-        {% if defined(repos) %}
-            AND arrayAll(
-                r -> has(insightsProjects.repositories, r),
+    GROUP BY i.id, i.segmentId
+    {% if defined(repos) %}
+        HAVING
+            arrayAll(
+                repo -> has(repositories, repo),
                 {{ Array(repos, 'String', description="Filter activity repo list", required=False) }}
             )
-        {% end %}
+    {% end %}
