lint

Author: max-ostapenko

.gitignore

@@ -1,5 +1,6 @@
 node_modules/
 .DS_Store
+.venv/
 
 # Terraform
 infra/tf/.terraform/

README.md

@@ -78,18 +78,19 @@ In order to unify the workflow triggering mechanism, we use [a Cloud Run functio
 
 1. Install dependencies:
 
-   ```bash
-   npm install
-   ```
+    ```bash
+    npm install
+    ```
 
 2. Available Scripts:
-   - `npm run format` - Format code using Standard.js, fix markdown issues, and format Terraform files
-   - `npm run lint` - Run linting checks on JavaScript, markdown files, and compile Dataform configs
+
+    - `npm run format` - Format code using Standard.js, fix Markdown issues, and format Terraform files
+    - `npm run lint` - Run linting checks on JavaScript, Markdown files, and compile Dataform configs
 
 ## Code Quality
 
 This repository uses:
 
 - Standard.js for JavaScript code style
-- Markdownlint for markdown file formatting
+- Markdownlint for Markdown file formatting
 - Dataform's built-in compiler for SQL validation

infra/bigquery_export_spark/Dockerfile

@@ -7,7 +7,7 @@ FROM debian:12-slim
 ENV DEBIAN_FRONTEND=noninteractive
 
 # Install utilities required by Spark scripts.
-RUN apt update && apt install -y procps tini libjemalloc2
+RUN apt-get update && apt-get install -y procps tini libjemalloc2
 
 # Enable jemalloc2 as default memory allocator
 ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2
@@ -22,10 +22,11 @@ RUN bash Miniforge3-Linux-x86_64.sh -b -p /opt/miniforge3 \
   && ${CONDA_HOME}/bin/conda config --system --set auto_update_conda False \
   && ${CONDA_HOME}/bin/conda config --system --set channel_priority strict
 
+WORKDIR /app
 COPY . .
 
 # Install pip packages.
-RUN ${PYSPARK_PYTHON} -m pip install -r requirements.txt
+RUN ${PYSPARK_PYTHON} -m pip install --no-cache-dir -r app/requirements.txt
 
 # Create the 'spark' group/user.
 # The GID and UID must be 1099. Home directory is required.

infra/tf/bigquery_export/variables.tf

@@ -2,10 +2,6 @@ variable "project" {
   type = string
 }
 
-variable "project_number" {
-  type = string
-}
-
 variable "region" {
   type = string
 }

infra/tf/dataform_export/variables.tf

@@ -1,7 +1,3 @@
-variable "project" {
-  type = string
-}
-
 variable "project_number" {
   type = string
 }
@@ -18,10 +14,6 @@ variable "function_name" {
   type = string
 }
 
-variable "location" {
-  type = string
-}
-
 variable "remote_functions_connection" {
   type = string
 }

infra/tf/main.tf

@@ -27,10 +27,8 @@ provider "google" {
 module "dataform_export" {
   source = "./dataform_export"
 
-  project           = local.project
   project_number    = local.project_number
   region            = local.region
-  location          = local.location
   function_identity = "cloud-function@httparchive.iam.gserviceaccount.com"
   function_name     = "dataform-export"
   remote_functions_connection  = google_bigquery_connection.remote-functions.id
@@ -50,7 +48,6 @@ module "bigquery_export" {
   source = "./bigquery_export"
 
   project           = local.project
-  project_number    = local.project_number
   region            = local.region
   location          = local.location
   function_identity = "cloud-function@httparchive.iam.gserviceaccount.com"