JSON operators pushing down implementation + new SQLite err codes

Author: mkgrgis

.github/workflows/CI.yml

@@ -9,7 +9,7 @@ on:
       - main
 jobs:
   detect-pgversion:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     outputs:
       pgversion: ${{ steps.detect-pgversion.outputs.targets }}
     steps:
@@ -36,7 +36,7 @@ jobs:
         config: ["default", "postgis"]
 
     name: Test on PostgreSQL ${{ matrix.pg }}, ${{ matrix.config }} mode
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
 
@@ -46,10 +46,16 @@ jobs:
       - name: set_proxy
         run: bash GitHubActions/env.sh
 
+      - name: download PostGIS, ${{ matrix.config }} mode
+        run: |
+          if [[ "${{ matrix.config }}" == "postgis" ]]; then
+            bash GitHubActions/download_postgis.sh ${{ env.POSTGIS_VERSION }}
+          fi
+
       - name: install locales
         run: bash GitHubActions/install_locales.sh
 
-      - name: build PostgreSQL ${{ matrix.pg }} with ${{ matrix.config }}
+      - name: build PostgreSQL ${{ matrix.pg }}
         run: bash GitHubActions/build_postgres.sh ${{ matrix.pg }}
 
       - name: build PostGIS ${{ env.POSTGIS_VERSION }} for PostgreSQL ${{ matrix.pg }}, ${{ matrix.config }} mode
@@ -83,3 +89,4 @@ jobs:
             workdir/postgresql-${{ matrix.pg }}/contrib/sqlite_fdw/regression.out
             workdir/postgresql-${{ matrix.pg }}/contrib/sqlite_fdw/results
           retention-days: 7
+

GitHubActions/build_postgis.sh

@@ -2,7 +2,7 @@
 
 ################################################################################
 #
-# This script downloads PostgreSQL from the official web site into ./workdir
+# This script downloads PostGIS from the official web site into ./workdir
 # then builds it.
 #
 # Usage: ./build_postgis.sh pg_version postgis_version
@@ -17,23 +17,18 @@
 POSTGRESQL_VERSION=$1
 POSTGIS_VERSION=$2
 
-cd ./workdir
-cd postgresql-${POSTGRESQL_VERSION}
-
 # Install necessary dependencies
 sudo apt update
 sudo apt install -y build-essential libxml2-dev libgeos-dev libproj-dev libgdal-dev libjson-c-dev libprotobuf-c-dev protobuf-c-compiler
 
-GEOS_CONFIG_PATH=$(which geos-config)
-
+cd ./workdir
 # Download and compile PostGIS
-cd contrib
-wget http://download.osgeo.org/postgis/source/postgis-${POSTGIS_VERSION}.tar.gz
-tar -xzf postgis-${POSTGIS_VERSION}.tar.gz
-mv postgis-${POSTGIS_VERSION} postgis -v
-cd postgis
+cp -vr postgis postgresql-${POSTGRESQL_VERSION}/contrib
+cd postgresql-${POSTGRESQL_VERSION}/contrib/postgis
 echo " - PostGIS directory"
+GEOS_CONFIG_PATH=$(which geos-config)
 export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH
 ./configure --with-pgconfig=/usr/local/pgsql/bin/pg_config --with-geosconfig=$GEOS_CONFIG_PATH
 make
 sudo make install
+

GitHubActions/build_sqlite_fdw.sh

@@ -18,14 +18,16 @@ VERSION="$1"
 MODE="$2"
 
 mkdir -p ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw
-tar zxvf ./sqlite_fdw.tar.gz -C ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw/
+tar zxf ./sqlite_fdw.tar.gz -C ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw/
 cd ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw
-export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH
+
+# show locally compiled sqlite library
+ls -la /usr/local/lib
 
 if [ "$MODE" == "postgis" ]; then
-  make ENABLE_GIS=1
+  make CI_ENV=1 ENABLE_GIS=1
 else
-  make
+  make CI_ENV=1
 fi
 
 sudo make install

GitHubActions/download_postgis.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+################################################################################
+#
+# This script downloads PostGIS from the official web site into ./workdir
+# then builds it.
+#
+# Usage: ./download_postgis.sh postgis_version
+#     postgis_version is a PostGIS version to be installed.
+#
+# Requirements
+# - be able to connect to the PostGIS official web site by wget.
+#
+################################################################################
+
+POSTGIS_VERSION=$1
+
+mkdir -p ./workdir
+cd ./workdir
+pgisfile="postgis-${POSTGIS_VERSION}.tar.gz"
+if [ ! -f "$pgisfile" ]; then
+  wget -nv "http://download.osgeo.org/postgis/source/$pgisfile"
+  tar -xzf "$pgisfile"
+  mv postgis-${POSTGIS_VERSION} postgis -v
+  echo "PostGIS source code directory " $(dirname $(readlink -f postgis))
+else
+  echo "PostGIS downloaded"
+fi

GitHubActions/install_locales.sh

@@ -17,5 +17,6 @@ sudo apt-get install locales language-pack-ja
 sudo locale-gen ja_JP.EUC-JP
 sudo apt-get install language-pack-ko-base language-pack-ko
 sudo locale-gen ko_KR.EUC-KR
-sudo apt -get install language-pack-bg-base language-pack-bg
+sudo apt-get install language-pack-bg-base language-pack-bg
 sudo locale-gen bg_BG
+sudo apt-get install libreadline8 libreadline-dev

GitHubActions/install_sqlite.sh

@@ -32,15 +32,19 @@ done
 
 mkdir -p ./workdir
 cd ./workdir
-curl -O https://www.sqlite.org/${YEAR}/sqlite-src-${VERSION}.zip
-unzip sqlite-src-${VERSION}.zip > /dev/null
-cd sqlite-src-${VERSION}
-
-if [ -z "$CONFIGURE_OPTIONS" ]; then
-  ./configure --enable-fts5
-else
-  ./configure --enable-fts5 $CONFIGURE_OPTIONS
+vsrc="sqlite-src-${VERSION}"
+adr="https://www.sqlite.org/${YEAR}/$vsrc.zip"
+echo "SQLite source code archive: $adr"
+curl -O "$adr"
+unzip "$vsrc.zip" > /dev/null
+cd "$vsrc"
+
+confcom="./configure --enable-fts5"
+if [ ! -z "$CONFIGURE_OPTIONS" ]; then
+  confcom+="$CONFIGURE_OPTIONS"  
 fi
+echo "SQLite configure call: $confcom"
+$confcom
 
 make
 sudo make install

Makefile

@@ -15,21 +15,29 @@ OBJS = connection.o option.o deparse.o sqlite_query.o sqlite_fdw.o sqlite_data_n
 EXTENSION = sqlite_fdw
 DATA = sqlite_fdw--1.0.sql sqlite_fdw--1.0--1.1.sql
 
+# Use local downloaded and compiled SQLite library instead of system
+ifdef CI_ENV
+override PG_LDFLAGS += -L/usr/local/lib
+endif
+
 ifdef ENABLE_GIS
 override PG_CFLAGS += -DSQLITE_FDW_GIS_ENABLE
-GISTEST=postgis
+GISTEST = postgis
 else
-GISTEST=nogis
+GISTEST = nogis
 endif
 
+# Tests for PostgreSQL data types support
+TYPETESTS = types/bitstring types/bool types/float4 types/float8 types/int4 types/int8 types/json types/numeric types/$(GISTEST) types/macaddr types/macaddr8 types/out_of_range types/timestamp types/uuid
+
 ifndef REGRESS
-REGRESS = extra/sqlite_fdw_post types/bitstring types/bool types/float4 types/float8 types/int4 types/int8 types/numeric types/$(GISTEST) types/macaddr types/macaddr8 types/out_of_range types/timestamp types/uuid extra/join extra/limit extra/aggregates extra/prepare extra/select_having extra/select extra/insert extra/update extra/encodings sqlite_fdw type_$(GISTEST) aggregate selectfunc 
+# System tests, full default sequence
+REGRESS = extra/sqlite_fdw_post $(TYPETESTS) extra/join extra/limit extra/aggregates extra/prepare extra/select_having extra/select extra/insert extra/update extra/encodings sqlite_fdw type_$(GISTEST) aggregate selectfunc
 endif
 
+# Other encodings also sre tested. Client encoding should be UTF-8-
 REGRESS_OPTS = --encoding=utf8
 
-SQLITE_LIB = sqlite3
-
 UNAME = uname
 OS := $(shell $(UNAME))
 ifeq ($(OS), Darwin)
@@ -71,8 +79,17 @@ REGRESS := $(addprefix $(REGRESS_PREFIX_SUB)/,$(REGRESS))
 $(shell mkdir -p results/$(REGRESS_PREFIX_SUB)/extra)
 $(shell mkdir -p results/$(REGRESS_PREFIX_SUB)/types)
 
+# $(info    ENABLE_GIS      is $(ENABLE_GIS))
+$(info    CI_ENV          is $(CI_ENV))
+# $(info    SHLIB_LINK      is $(SHLIB_LINK))
+# $(info    LD_LIBRARY_PATH is $(LD_LIBRARY_PATH))
+$(info    PG_CFLAGS       is $(PG_CFLAGS))
+# $(info    REGRESS         is $(REGRESS))
+# $(info    DLSUFFIX        is $(DLSUFFIX))
+
 ifdef ENABLE_GIS
 check: temp-install
 temp-install: EXTRA_INSTALL+=contrib/postgis
 checkprep: EXTRA_INSTALL+=contrib/postgis
 endif
+

README.md

@@ -4,7 +4,7 @@ SQLite Foreign Data Wrapper for PostgreSQL
 This is a foreign data wrapper (FDW) to connect [PostgreSQL](https://www.postgresql.org/)
 to [SQLite](https://sqlite.org/) database file. This FDW works with PostgreSQL 13, 14, 15, 16, 17 and confirmed with SQLite 3.46.0.
 
-<img src="https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg" align="center" height="100" alt="PostgreSQL"/>	+	<img src="https://upload.wikimedia.org/wikipedia/commons/3/38/SQLite370.svg" align="center" height="100" alt="SQLite"/>
+<img src="https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg" align="center" height="100" alt="PostgreSQL"/>	+	<img src="SQLite.png" align="center" height="100" alt="SQLite"/>
 
 Also this foreign data wrapper (FDW) can connect PostgreSQL with [PostGIS](https://www.postgis.net/)
 to [SpatiaLite](https://www.gaia-gis.it/fossil/libspatialite/index) SQLite database file. This FDW works with PostGIS 2+ and confirmed with SpatiaLite 5.1. See [GIS support description](GIS.md).

SQLite.png

@@ -199,6 +199,7 @@ sqlite_open_db(const char *dbpath, int flags)
 		ereport(ERROR,
 				(errcode(ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION),
 				 errmsg("Failed to open SQLite DB, file '%s', result code %d", dbpath, rc)));
+	sqlite3_extended_result_codes(conn, true);
 	/* make 'LIKE' of SQLite case sensitive like PostgreSQL */
 	rc = sqlite3_exec(conn, "pragma case_sensitive_like=1",
 					  NULL, NULL, &err);