Merge pull request #6675 from Flowminder/parquet_fdw

Parquet fdw

Author: mergify[bot]

CHANGELOG.md

@@ -6,6 +6,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
 
+### Added
+- Added support for Parquet foreign tables using [parquet_fdw](https://github.com/adjust/parquet_fdw)
+
+### Changed
+- FlowKit test and synthetic data now uses parquet foreign tables.
+> [!WARNING]
+> The location of the parquet files in the container is `/parquet_data`, if you are testing with larger amounts of data you may wish to add an additional bind mount for this location.
+
+### Fixed
+
+### Removed
+
 ## [1.27.0]
 
 ### Added
@@ -19,11 +31,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - The location inside the container of FlowDB's automatically generated config file has changed to `/flowdb_autoconf/$AUTO_CONFIG_FILE_NAME`.
 
 
-
-### Fixed
-
-### Removed
-
 ## [1.26.0]
 
 ### Changed

docker-compose-syntheticdata.yml

@@ -29,3 +29,4 @@ services:
       DISASTER_START: ${DISASTER_START:?Must set DISASTER_START env var}
       DISASTER_END: ${DISASTER_END:?Must set DISASTER_END env var}
       DISASTER_REGION_PCOD: ${DISASTER_REGION_PCOD:-NPL.1.1_1}
+

flowdb.Dockerfile

@@ -82,13 +82,25 @@ RUN apt-get update \
         && mv pldebugger /usr/local/src \
         && make -C /usr/local/src/pldebugger \
         && make -C /usr/local/src/pldebugger install \
-        && apt-get remove -y libssl-dev \
-        libkrb5-dev \
-        build-essential \
-        git \
+        && apt-get remove -y build-essential git \
         && apt purge -y --auto-remove \
         && rm -rf /var/lib/apt/lists/*
 
+# Parquet foreign tables
+RUN apt-get update -y && apt-get install -y --no-install-recommends git build-essential ca-certificates lsb-release wget \
+        && wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \
+        && apt-get install -y --no-install-recommends ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \
+        && rm ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \
+        && apt-get update -y \
+        && apt-get install -y --no-install-recommends libarrow-dev libparquet-dev libparquet1600 libarrow1600 \
+        && pip3 install pyarrow \
+        && git clone --branch pg16-compatibility --single-branch https://github.com/adjust/parquet_fdw.git \
+        && mv parquet_fdw /usr/local/src \
+        && make -C /usr/local/src/parquet_fdw \
+        && make -C /usr/local/src/parquet_fdw install \
+        && apt-get remove -y build-essential git wget libarrow-dev libparquet-dev \
+        && apt purge -y --auto-remove \
+        && rm -rf /var/lib/apt/lists/*
 
 
 #

flowdb/bin/build/0010_create_extensions.sh

@@ -5,7 +5,7 @@
 
 
 
-set -e
+set -euo pipefail
 
 #
 #  Perform all actions as $POSTGRES_USER
@@ -14,7 +14,7 @@ export PGUSER="$POSTGRES_USER"
 EXTENSIONS=('postgis' 'postgis_raster' 'postgis_topology' 'fuzzystrmatch' \
             'file_fdw' 'uuid-ossp' 'plpython3u' \
             'tsm_system_rows' 'pgrouting' 'pldbgapi' 'pg_median_utils'\
-            'ogr_fdw' 'tds_fdw' 'btree_gist')
+            'ogr_fdw' 'tds_fdw' 'btree_gist' 'parquet_fdw')
 
 #
 #  Create the 'template_postgis' template db
@@ -52,6 +52,7 @@ echo "Creating extension servers in $DB."
 psql --dbname="$DB" <<-EOSQL
     CREATE SERVER csv_fdw 
         FOREIGN DATA WRAPPER file_fdw;
+    CREATE SERVER parquet_srv FOREIGN DATA WRAPPER parquet_fdw;
 EOSQL
 
 done

flowdb/testdata/bin/9900_ingest_test_data.sh

@@ -6,6 +6,7 @@
 
 set -e
 export PGUSER="$POSTGRES_USER"
+SKIP_TEST_QA_CHECK=${SKIP_TEST_QA_CHECK:-"false"}
 
 #
 #  Ingest test data.

flowdb/testdata/bin/9920_run_convert_events_to_parquet.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+set -e
+export PGUSER="$POSTGRES_USER"
+
+#
+#  Converts Python .
+#
+#  Note that the only purpose of this script is to
+#  call the Python script which does the actual data
+#  data generation, but we need this shell script as
+#  a wrapper because the PostgreSQL entrypoint script
+#  does not pick up .py files on its own.
+#
+
+export DIR=/docker-entrypoint-initdb.d/py/testdata
+
+echo "Running Python script to convert events tables to Parquet partitions."
+pipenv run python ${DIR}/zz_convert_events_to_parquet.py

flowdb/testdata/synthetic_data/Pipfile

@@ -12,6 +12,8 @@ tohu = "==0.6.7"
 numpy = "<=2.0.0" # Tohu uses float division where it should be using int division, and hence passes a float where numpy expects an int (https://github.com/maxalbert/tohu/blob/3adf0c58b13ef1e1d716d7d613484d2adc58fb60/tohu/v6/primitive_generators.py#L335)
                    # This used to work, but doesn't as of numpy 1.26.0 (although I haven't managed to track down the relevant change or find a corresponding issue or changelog entry)
 jinja2 = "*"
+pyarrow = "*"
+
 [dev-packages]
 black = {extras = ["jupyter"],version = "==24.4.2"}