Merge remote-tracking branch 'origin/main' into judicial-snake

Author: max-ostapenko

Makefile

@@ -13,7 +13,7 @@ tf_apply:
 	terraform -chdir=infra/tf init && terraform -chdir=infra/tf apply -auto-approve
 
 bigquery_export_deploy:
-	cd infra/bigquery-export && npm run buildpack
+	cd infra/bigquery-export && npm install && npm run buildpack
 
 bigquery_export_spark_deploy:
 	cd infra/bigquery_export_spark && gcloud builds submit --region=global --tag us-docker.pkg.dev/httparchive/bigquery-spark-procedures/firestore_export:latest

definitions/output/core_web_vitals/technologies.js

@@ -9,7 +9,7 @@ publish('technologies', {
     clusterBy: ['geo', 'app', 'rank', 'client'],
     requirePartitionFilter: true
   },
-  tags: ['crux_ready', 'tech_report'],
+  tags: ['crux_ready'],
   dependOnDependencyAssertions: true
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}

definitions/output/reports/cwv_tech_core_web_vitals.js

@@ -8,7 +8,7 @@ publish('cwv_tech_core_web_vitals', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['tech_report']
+  tags: ['crux_ready']
 }).preOps(ctx => `
 CREATE TEMPORARY FUNCTION GET_VITALS(
   records ARRAY<STRUCT<

definitions/output/reports/tech_crux.js

@@ -360,7 +360,7 @@ other_summary AS (
 
     STRUCT(
       SAFE_CAST(APPROX_QUANTILES(accessibility, 1000)[OFFSET(500)] AS NUMERIC) AS accessibility,
-      SAFE_CAST(APPROX_QUANTILES(best_practices, 1000)[OFFSET(500)] AS NUMERIC) AS practices,
+      SAFE_CAST(APPROX_QUANTILES(best_practices, 1000)[OFFSET(500)] AS NUMERIC) AS best_practices,
       SAFE_CAST(APPROX_QUANTILES(performance, 1000)[OFFSET(500)] AS NUMERIC) AS performance,
       SAFE_CAST(APPROX_QUANTILES(seo, 1000)[OFFSET(500)] AS NUMERIC) AS seo
     ) AS median_lighthouse_score,

definitions/output/reports/tech_report_categories.js

@@ -51,7 +51,7 @@ technology_stats AS (
   SELECT
     technology,
     category_obj AS categories,
-    SUM(origins) AS total_origins
+    SUM(origins.mobile + origins.desktop) AS total_origins
   FROM ${ctx.ref('reports', 'tech_report_technologies')}
   GROUP BY
     technology,

infra/tf/functions.tf

@@ -3,7 +3,7 @@ locals {
 }
 
 resource "google_project_iam_member" "project" {
-  for_each = toset(["roles/bigquery.user", "roles/dataform.serviceAgent", "roles/run.invoker", "roles/run.jobsExecutorWithOverrides", "roles/datastore.user", "roles/storage.objectAdmin"])
+  for_each = toset(["roles/bigquery.jobUser", "roles/dataform.serviceAgent", "roles/run.invoker", "roles/run.jobsExecutorWithOverrides", "roles/datastore.user", "roles/storage.objectUser"])
 
   project = local.project
   role    = each.value

package-lock.json

@@ -2,7 +2,7 @@
   "name": "crawl-data",
   "author": "@max-ostapenko",
   "dependencies": {
-    "@dataform/core": "3.0.14"
+    "@dataform/core": "^3.0.15"
   },
   "scripts": {
     "format": "npx standard --fix; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint . --fix; terraform -chdir=infra/tf fmt -recursive",