From c732592c6687214185dd76066a080c1ac986c593 Mon Sep 17 00:00:00 2001 From: reshke Date: Mon, 2 Feb 2026 13:54:13 +0500 Subject: [PATCH 01/14] Yezzey test (#8) This import old CI job from open-gpdb/gpdb + yezzey. This merely checks that yezzey and cloudberry compiles together with no issues. As discussed privately we will redesign it later in another PR. Right now we copy existing code from open-gpdb in order to make sure all out future PR is Ok --- .github/workflows/yezzey-test.yml | 21 ++++ docker/yezzey/.dockerignore | 2 + docker/yezzey/Dockerfile | 131 +++++++++++++++++++++++ docker/yezzey/docker-compose.yaml | 45 ++++++++ yezzey_test/.minio/certs/CAs/ca.cert.pem | 29 +++++ yezzey_test/.minio/certs/CAs/ca.cert.srl | 1 + yezzey_test/.minio/certs/CAs/ca.key | 51 +++++++++ yezzey_test/.minio/certs/private.key | 51 +++++++++ yezzey_test/.minio/certs/public.crt | 29 +++++ yezzey_test/.minio/certs/server.csr | 27 +++++ yezzey_test/generate_ssh_key.sh | 6 ++ yezzey_test/import_gpg_keys.sh | 5 + yezzey_test/install-wal-g.sh | 20 ++++ yezzey_test/install_yproxy.sh | 20 ++++ yezzey_test/priv.gpg | 105 ++++++++++++++++++ yezzey_test/pub.gpg | 52 +++++++++ yezzey_test/run_tests.sh | 75 +++++++++++++ yezzey_test/wal-g-conf.yaml | 12 +++ yezzey_test/yproxy.conf | 25 +++++ 19 files changed, 707 insertions(+) create mode 100644 .github/workflows/yezzey-test.yml create mode 100644 docker/yezzey/.dockerignore create mode 100644 docker/yezzey/Dockerfile create mode 100644 docker/yezzey/docker-compose.yaml create mode 100755 yezzey_test/.minio/certs/CAs/ca.cert.pem create mode 100755 yezzey_test/.minio/certs/CAs/ca.cert.srl create mode 100755 yezzey_test/.minio/certs/CAs/ca.key create mode 100755 yezzey_test/.minio/certs/private.key create mode 100755 yezzey_test/.minio/certs/public.crt create mode 100755 yezzey_test/.minio/certs/server.csr create mode 100755 yezzey_test/generate_ssh_key.sh create mode 100755 yezzey_test/import_gpg_keys.sh create mode 100755 yezzey_test/install-wal-g.sh create mode 100755 yezzey_test/install_yproxy.sh create mode 100644 yezzey_test/priv.gpg create mode 100644 yezzey_test/pub.gpg create mode 100755 yezzey_test/run_tests.sh create mode 100644 yezzey_test/wal-g-conf.yaml create mode 100644 yezzey_test/yproxy.conf diff --git a/.github/workflows/yezzey-test.yml b/.github/workflows/yezzey-test.yml new file mode 100644 index 00000000000..4922f6eca84 --- /dev/null +++ b/.github/workflows/yezzey-test.yml @@ -0,0 +1,21 @@ +name: Yezzey testing + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + + build_and_run_yezzey: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Test Yezzey + run: docker compose -f docker/yezzey/docker-compose.yaml run --build --remove-orphans yezzey + + diff --git a/docker/yezzey/.dockerignore b/docker/yezzey/.dockerignore new file mode 100644 index 00000000000..4e92238e609 --- /dev/null +++ b/docker/yezzey/.dockerignore @@ -0,0 +1,2 @@ +Dockerfile +docker-compose.yaml \ No newline at end of file diff --git a/docker/yezzey/Dockerfile b/docker/yezzey/Dockerfile new file mode 100644 index 00000000000..0b84f13d7bc --- /dev/null +++ b/docker/yezzey/Dockerfile @@ -0,0 +1,131 @@ +FROM ubuntu:focal + +ARG accessKeyId +ARG secretAccessKey +ARG bucketName +ARG s3endpoint +ARG yezzeyRef + +ENV YEZZEY_REF=${yezzeyRef:-v1.8_opengpdb} + +ENV AWS_ACCESS_KEY_ID=${accessKeyId} +ENV AWS_SECRET_ACCESS_KEY=${secretAccessKey} +ENV S3_BUCKET=${bucketName} +ENV WALG_S3_PREFIX=s3://${bucketName}/yezzey-test-files +ENV S3_ENDPOINT=${s3endpoint} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ENV DEBIAN_FRONTEND=noninteractive + +RUN useradd -rm -d /home/gpadmin -s /bin/bash -g root -G sudo -u 1001 gpadmin + +RUN ln -snf /usr/share/zoneinfo/Europe/London /etc/localtime && echo Europe/London > /etc/timezone \ +&& apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install -y --no-install-recommends --allow-unauthenticated \ + build-essential libssl-dev gnupg devscripts \ + openssl libssl-dev debhelper debootstrap \ + make equivs bison ca-certificates-java ca-certificates \ + cmake curl cgroup-tools flex gcc-8 g++-8 g++-8-multilib \ + git krb5-multidev libapr1-dev libbz2-dev libcurl4-gnutls-dev \ + libevent-dev libkrb5-dev libldap2-dev libperl-dev libreadline6-dev \ + libssl-dev libxml2-dev libyaml-dev libzstd-dev libaprutil1-dev \ + libpam0g-dev libpam0g libcgroup1 libyaml-0-2 libldap-2.4-2 libssl1.1 \ + ninja-build python-dev python-setuptools quilt unzip wget zlib1g-dev libuv1-dev \ + libgpgme-dev libgpgme11 sudo iproute2 less software-properties-common \ + openssh-client openssh-server + +COPY yezzey_test/install_yproxy.sh /home/gpadmin + +RUN ["/home/gpadmin/install_yproxy.sh"] + +RUN apt-get install -y locales \ +&& locale-gen "en_US.UTF-8" \ +&& update-locale LC_ALL="en_US.UTF-8" + +RUN echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers + +USER gpadmin +WORKDIR /home/gpadmin + +COPY yezzey_test/import_gpg_keys.sh /home/gpadmin/ +COPY yezzey_test/priv.gpg /home/gpadmin/yezzey_test/priv.gpg +COPY yezzey_test/pub.gpg /home/gpadmin/yezzey_test/pub.gpg + +RUN ["/home/gpadmin/import_gpg_keys.sh"] + +COPY yezzey_test/generate_ssh_key.sh /home/gpadmin/ + +RUN ["/home/gpadmin/generate_ssh_key.sh"] + + +RUN cd /tmp/ \ +&& git clone https://github.com/boundary/sigar.git \ +&& cd ./sigar/ \ +&& mkdir build && cd build && cmake .. && make \ +&& sudo make install + +COPY . /home/gpadmin + +RUN sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ + bison \ + ccache \ + cmake \ + curl \ + flex \ + git-core \ + gcc \ + g++ \ + inetutils-ping \ + krb5-kdc \ + krb5-admin-server \ + libapr1-dev \ + libbz2-dev \ + libcurl4-gnutls-dev \ + libevent-dev \ + libkrb5-dev \ + libpam-dev \ + libperl-dev \ + libreadline-dev \ + libssl-dev \ + libxml2-dev \ + libyaml-dev \ + libzstd-dev \ + locales \ + net-tools \ + ninja-build \ + openssh-client \ + openssh-server \ + openssl \ + python3-dev \ + python3-pip \ + python3-psutil \ + python3-pygresql \ + python-yaml \ + zlib1g-dev \ + rsync \ +&& sudo apt install -y libhyperic-sigar-java libaprutil1-dev libuv1-dev + +RUN sudo mkdir /usr/local/gpdb \ +&& sudo chown gpadmin:root /usr/local/gpdb + +RUN sudo chown -R gpadmin:root /home/gpadmin \ +&& git status + +RUN git submodule update --init +RUN rm -fr gpcontrib/yezzey + +# Fetch latest yezzey version +RUN git clone https://github.com/open-gpdb/yezzey.git gpcontrib/yezzey && cd gpcontrib/yezzey && git fetch origin $YEZZEY_REF:test_branch && git checkout test_branch && cd /home/gpadmin +RUN sed -i '/^trusted/d' gpcontrib/yezzey/yezzey.control +RUN ./configure --with-perl --with-python --with-libxml --disable-orca --prefix=/usr/local/gpdb \ +--enable-depend --enable-cassert --enable-debug --without-mdblocales --without-zstd CFLAGS='-fno-omit-frame-pointer -Wno-implicit-fallthrough -O3 -pthread' +RUN make -j8 && make -j8 install && make -C gpcontrib/yezzey -j8 install + + +RUN echo ${s3endpoint} + +RUN sed -i "s/\$AWS_ACCESS_KEY_ID/${accessKeyId}/g" yezzey_test/yproxy.conf \ +&& sed -i "s/\$AWS_SECRET_ACCESS_KEY/${secretAccessKey}/g" yezzey_test/yproxy.conf \ +&& sed -i "s/\$AWS_ENDPOINT/${s3endpoint}/g" yezzey_test/yproxy.conf \ +&& sed -i "s/\$WALG_S3_PREFIX/${bucketName}\/yezzey-test-files/g" yezzey_test/yproxy.conf && cp yezzey_test/yproxy.conf /tmp/yproxy.yaml + +ENTRYPOINT ["./yezzey_test/run_tests.sh"] diff --git a/docker/yezzey/docker-compose.yaml b/docker/yezzey/docker-compose.yaml new file mode 100644 index 00000000000..2562ebeaa0b --- /dev/null +++ b/docker/yezzey/docker-compose.yaml @@ -0,0 +1,45 @@ +services: + minio: + image: quay.io/minio/minio + command: server --console-address ":9001" /data + expose: + - "9000" + - "9001" + environment: + MINIO_ROOT_USER: some_key + MINIO_ROOT_PASSWORD: some_key + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + hostname: minio + + setup-minio: + image: quay.io/minio/mc + depends_on: + minio: + condition: service_healthy + entrypoint: | + /bin/sh -c " + /usr/bin/mc alias set myminio http://minio:9000 some_key some_key + /usr/bin/mc mb myminio/gpyezzey + /usr/bin/mc mb myminio/gpyezzey2 + /usr/bin/mc mb myminio/gpyezzey3 + " + + yezzey: + image: yezzey + build: + context: ../.. + dockerfile: docker/yezzey/Dockerfile + args: + accessKeyId: some_key + secretAccessKey: some_key + bucketName: gpyezzey + s3endpoint: "http:\\/\\/minio:9000" + depends_on: + minio: + condition: service_healthy + setup-minio: + condition: service_completed_successfully diff --git a/yezzey_test/.minio/certs/CAs/ca.cert.pem b/yezzey_test/.minio/certs/CAs/ca.cert.pem new file mode 100755 index 00000000000..e9f2f1d73ee --- /dev/null +++ b/yezzey_test/.minio/certs/CAs/ca.cert.pem @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE/TCCAuWgAwIBAgIUU9e6chP84r3iZk3JtvnWb1V2N1YwDQYJKoZIhvcNAQEL +BQAwDTELMAkGA1UEBhMCUlUwIBcNMjMwMzEwMDgzNTUzWhgPMzAyMjA3MTEwODM1 +NTNaMA0xCzAJBgNVBAYTAlJVMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC +AgEAwJuy394cK127yT8nGHVPKF6TG6xL0WpxahyaKwIYp5lbv9wDvzjMPE7KmONU +8GhCFUdEJTRqBkaRdZNYxnOUxufU3+jIf1hq1Csg8q1NXICVWVwfFL2F5mKHgeHQ +n3FaJM2pZQ5iIWFY1c18MgV8qqNWbtyLeppcyZOL9duLM9A8XpYb0JOZis82d+lh +kcxzE1XM+MZEgZfHImh0zod9OMtSAOwQzVXpiA3JO/eHkLQGYcy6KNTm42mubVlX +kBcu/BplnP7gXGOYDt/JyRhGSLAfn762+jRbAlAvbPzOy67hc4pW7aloU5zPBhYf +BaTxM9UPqPtyp7Lxkp9HL68QXtm5MobDuDtZ6ePQtHgHrl7P7PXvEUPwK7BZzgZy +MerVhxIssutA2yBCuu5T7dMSwIsUdvXtgdHRdHDwn1D/V1CxnujDv9l6/T3sCmRv +tWPwTOCUf5BLLw6N6TnSsVR5I9NALKCLYE8LsfCuLdyi363JZqubkdJr1Ro8yI5J +m0GX5pypwZJPV2Ivt6kKVTQiN2hoWNe+3TNPS+7ysqit37s71YRDajZaZ55DopmF ++oIYdA3MqUZEVZyKFifWvo/l2gYarlEtcEJl++OwydirWLAjCPHh9UvDhjKS43bQ +zSlRC+d4CfRqXftmETHVAxMokai3WvAdUpJrW2RrjiuR0MkCAwEAAaNTMFEwHQYD +VR0OBBYEFJGDr6xmoKJFU6cgS90aFg6lUGbhMB8GA1UdIwQYMBaAFJGDr6xmoKJF +U6cgS90aFg6lUGbhMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB +ACj87ymjBlgY9UZTUbudHREPPXfqMi2TgWt5hygQSiTrNeQOodnq+Swp86qX/y8w +xtnvc+iILfFnh9ZevHKmLx+JziN4kD4ywEpHW7zS7c3+2QEjIZUwj5qlIg0ByOBd +0M/kpimmuTwlDylBaY12GcFlZcsbuezzm4hU+0qoCV/zi2DvSdAPKXMAeZ3lOkde +PUYJUpRz/QkkxEhSdM3BQYI51mUiltCHMhe6COoN4MHV7tix0Pj9vPjhAVN/4sot +2PgUiCwY8eNQugZhpTosMTSBLZvg/EKG+4slY75/voNTIxWHAHmnPMOAzVgNTya0 +/eP6NB3MCjFuY2E+fGox9YTomjI5oxBr+1LlwVy7wbwXTrgBz9Z4izScAsVbPrk6 +jSrqNeNWK1f+JVnYZkjgPGgPaQVCJ22vdLmkW7U/ATdeedQS3RCApMnb9VCRTUaO +eY4ccuEvj0huhdcUguw6fBjrhPjoPxKMn6S93ginW8Wz9vo8qLkEg2NtQDFu1Omb +cJM5F8uLRr8NotPV5QPg1koHeBv/N2WTRZiUoavAogR9XdyOtrB8+MBu1nsp4Goi +7/suv9XzMJ7IpgXiQfCM++1x7oooyWWdeFTCzqNDJ1IbQDeOCc9cQgeOAPWcIqWO +nAWt08+eToI1YUvjl6UT0bpVaJEACv+/HfBr1T26u4Jh +-----END CERTIFICATE----- diff --git a/yezzey_test/.minio/certs/CAs/ca.cert.srl b/yezzey_test/.minio/certs/CAs/ca.cert.srl new file mode 100755 index 00000000000..977dab4a3e7 --- /dev/null +++ b/yezzey_test/.minio/certs/CAs/ca.cert.srl @@ -0,0 +1 @@ +53DAAE29A25C5BE967A9E9631F0572E17AC92211 diff --git a/yezzey_test/.minio/certs/CAs/ca.key b/yezzey_test/.minio/certs/CAs/ca.key new file mode 100755 index 00000000000..a0fc44ee422 --- /dev/null +++ b/yezzey_test/.minio/certs/CAs/ca.key @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAwJuy394cK127yT8nGHVPKF6TG6xL0WpxahyaKwIYp5lbv9wD +vzjMPE7KmONU8GhCFUdEJTRqBkaRdZNYxnOUxufU3+jIf1hq1Csg8q1NXICVWVwf +FL2F5mKHgeHQn3FaJM2pZQ5iIWFY1c18MgV8qqNWbtyLeppcyZOL9duLM9A8XpYb +0JOZis82d+lhkcxzE1XM+MZEgZfHImh0zod9OMtSAOwQzVXpiA3JO/eHkLQGYcy6 +KNTm42mubVlXkBcu/BplnP7gXGOYDt/JyRhGSLAfn762+jRbAlAvbPzOy67hc4pW +7aloU5zPBhYfBaTxM9UPqPtyp7Lxkp9HL68QXtm5MobDuDtZ6ePQtHgHrl7P7PXv +EUPwK7BZzgZyMerVhxIssutA2yBCuu5T7dMSwIsUdvXtgdHRdHDwn1D/V1CxnujD +v9l6/T3sCmRvtWPwTOCUf5BLLw6N6TnSsVR5I9NALKCLYE8LsfCuLdyi363JZqub +kdJr1Ro8yI5Jm0GX5pypwZJPV2Ivt6kKVTQiN2hoWNe+3TNPS+7ysqit37s71YRD +ajZaZ55DopmF+oIYdA3MqUZEVZyKFifWvo/l2gYarlEtcEJl++OwydirWLAjCPHh +9UvDhjKS43bQzSlRC+d4CfRqXftmETHVAxMokai3WvAdUpJrW2RrjiuR0MkCAwEA +AQKCAgAgemC4RTDE00J2FfMWublGWmQ991i1kFhdh0Mr22ei40ZIXOY42W/+/15E +V5kcDMiP4/uGtobmVgHzLIx8skK1I6SOuScN6i/hZQBiS3zPC1OjxNfs3GR2y8iD +yzstl6SWriNRShKcBFlBfCvkF27FK1PIz+GpI9xflUS1iXa4nvV/EZrRGgJ7GKPb +pnvwZORGr2In1O76V0iZ8bk4ljo0WHyUcToIFeOSMJjtRrkSWnj1BtuhRP1F/a0O +/VC5mF8w3Zai2YulqJmccHoLMc+wNBqxCiy6lhd+lVzZ6OtKB0w2+m3cF4PjDX8P +TK2gewa9McE5QmU8B/2aNsd/L+r3eGEvWAF/1vRq6NcrFwigq8uCTtgw9edRlDnm +RvICkfAbrwhNaixWwqBVQHoy53H29TohxGNNKa6TTKeJvYEdYKgHx55TxkB9X9jc +iSisqb3fgEl4Yh1Izpu+6nULOqdlldfkKPgKJqVB1AT/avR8J09zmMvW5fPa6fFx +alZ1iVahR5bIFEu1lXygsrBP6N+K/ogyztg7ZKLTIN/FguwMKnXMaUbN/Y/ZZXV1 +oGil9vHKnDrRnUGfcm9tyH2Ddcy6RDoDz+O4cYgMGxDhHran2cicVY1q+Yi08q5h +Napk1phNra5HIHnNHwMxQ75ZKZZ3TOGJL+HMF4yRDj19C/6sAQKCAQEA8a9ZQhWw +0vhZENmSYZgGZLa7RZLSbBzQOX/cetdI6/kvmZVcMvNz4q0/UI9XLkqokL1wJiku +O0zXkaVrBVAsgozp4I3oFqwtcAAGw0KwF4FDAS36k4gkE4SmIUl2eI0XMZCPQIKp +3TB81+XdBITtwfPl5yG+IZDkXNu16qUHEhnhvs/kKhMr8flhFC1J4gdrrQhfuRHY +Jv8e1RLJzMhu/ErRjh82LkzB6m3jp0YxBeIA+9Kkw+OX6SlzRbJPirKxJTaZnB8o +wQmzOy1kTRG4qjKswjdTbzf6549721i8QHwSpwPI3NZQhlSkfsvZ5QL4qPW0nRta +m76YeLlS12yQSQKCAQEAzAQz6OcE6yS2q5UfTZluGaU54Zkm0YSnS394pitJpHoh +JSZlvkL1DzpacquDxa3uQLDikai5TqpNnkuufeMJf7I2ygg4n/v4OFaE+/qj5uNA +3QnL3BVT9DCJ0JvQ1qA5Q/6P5WpUHYB7JHBM9BpaE8e4xocJyWSdcSJDaEXns4Hx +WzhpBdVpPSamqB0VHYg1bv6OGFPfwUaRafWhNzljtxbY8RYcz7IfPmnLImFePTtZ +AjzIoAwUIRFzvmoduda0kQKogRVoEeaW1q6ebPUjYjIZvohnpe27EvgCiTNkcaSf +C96uIxHrSvI8114z9CBXer60xQ0Kz+ds18LtY6w8gQKCAQEAkP/JxlsrHje/f9t4 +9jJ2S4BSNLiUpCZZStYKWmzFJEX5J+SzTyI+uZWFcfi9rlk+brApE8wLH6rHfmtH +HQXv3ldajc21m7yq+hIZ/JYK/d8gaxnBxzebpVYlMb1YZZUIgEUhnOuHq9vGWuVe +x7JUztNccGIPJyY9y/RJXUCrUFHU3Vzun8umxuL+OlO9iu02zbZDb85j52mSfvVp +uwHZjGX6+ZCCOh71DIfnWFlFWikwu+Sx05C9eDbVINCM5kK1AwWR/Ve4ZLBEJtHh +5lcmen4ypcb5uLVWRA0SmxPOxcVqj2c24D94Sk+H7UayMLKqqvvW45cgsmYUJgHR +0MsieQKCAQB9goBk4erWtmliuYTeemuPf2RSc6O79b3t5mfU4oCVnUTS1AJ3wD1+ +tsl6DiYs8MnIJoncTk5iJMdHgQvCCnCHjJ3EQLaFRb/4+NErK5C1tEztLt+pb72M +VmgSXCloQH26ZNslqfpBhA895ZCSA7wyuwXjrKPKsAlj1k5d0dOvTVusYNHLcvUh +V6vjdLDO0EL/G79THBZlkwJWi3Q4wyejNX0VJCNpaw1pmjAL4JbXWLFzfO13+LZR +eakZFbNf5sSDCX2cnAzAJnnZbOet5El2WZgY7VXGcLBMBSOaQHGksD/gT4gVrypv +mwLvA9c2cscejkArkdB7AsalHhho30cBAoIBAFJBO0RU7o0S+F6KHIP5aFbItcUd +NfUgoJTAFUD3EnBirvDv0pu8T8zkgKf7PRFkZQIOXocvpX0Zy6N7fiPbvzTA/vH3 +mFqias89pTUAgv43R8ZsAC/qlozUuByegigEz2zeVd34w7MdkgGo1jnqmijAIXZE +INBo0swkxAbix+W1Pur/yvGUpC6xu3ISmdrn0p20B7QhyuoqC3ea/az7ePwx+Pu9 +Jl8tzMujbHNHhw+OQAQOPHi6EUPs/H37euj3G7oBaVUwXJq3Tbwg95W5Jih+CgTB +Sbe6eYpR/j/SYGwbS6/DbHi3IjvblN+2pSPI05JvXMhLC/lAeqcdVJAgTvw= +-----END RSA PRIVATE KEY----- diff --git a/yezzey_test/.minio/certs/private.key b/yezzey_test/.minio/certs/private.key new file mode 100755 index 00000000000..9d12f84dfa7 --- /dev/null +++ b/yezzey_test/.minio/certs/private.key @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEA1WMTewnKOrmE6ceR+rDY2+IK57DktPou0KlJ+Cir5RjsCrh/ +mD8rNCJCVQ0ytV8EHUlnfJBcSnZRuKZHDTeaYAmKYe0Wcqey+bzAKG9+W3kNvFm0 +7Q/MIEAU4eGoeMtZpG91ZU9Jml45siJz7pZArnIrZdOwixBiu8oa66b88Dz/mHDK +JTgex9q27mYFERWu51ORAIchCLIQ9xKdMqR99irHKvGtC6TN5o6ANfTXmhVuVm4X +7Q6aeeDdmsOoP/KKmK/0yYvTunyfEhyaoEBw/aQmltfuEilMxrOzGoAY1iJbHuXV +tzYqkEomrmGXa7pStUoq3Ruu5PLVU+zEYqxVurYQTz0cWQZxo8BQdGbdmKiwu/NL +oc6WA2dUHdVKB9cDnRQXfKXWO8TZOKCIJiQLzGsUW8Jq6kstiaE7IQL9LBZbSWxG +ScgkneDE+CucgSjD5PwDMZ5R6RXPVxdh9CHfNIytESa1zb1H/d+U51xDRk6qkQbO +Ah5gt241mlekv1HYarRkwx8v0iiRa6ecmaGmnAA3SYRZYV6yYf1co+0Svm9EEG9f +bZMHLmTExpz6xPwg5EsJ2jMgsJP6My2LKJ5TVoUCV7S+Cz5tXFyJyHmtedSTaeC+ +flt5A+9nYQKuk92mh2t/XaSCRDFugERUFAlEAvag0+06qhbVyt69fMGozzkCAwEA +AQKCAgEAsWK7PvzUcBzosK6GW6/HloJCLniOpyOS50LTisfEnZ4qGn9lElrwv1X7 +bliaXsutz+rFbHdVQVE6fhU723DtlAhaUS2WC5n83j5aP0Lv93qaQIkSLj+DoQuk +UGIWetQQoPFG1gEjXoAV1k9tsFiXTGz8RpnDmNb2PMW1u1AF1G/gygh5Ape0fs8C +YwvMCnfL/eEqGRY8D8526+09YGv9ijXle32MLLHDuHWdfz0aPayzHIZIvXf2Unrr +vUwJAZ/ONz+Obj0etVgDpDrDD5SCWVer/Jlj/xT2Dfg0W0NBYkENHpJRJwyQNYJu +xWe7SIKLXslY+JWavhhf3nRkjOJWIGtgix6Sbb6K3TFNu65t5nOVr9dyZFcW4sSD +JpyUjDK6mbpUKgh3yaU9QXQSY0bD48mTt1UB2vmh/dA9OIRRUXr+V4MKPQBtJt0V +7ay2P95oWSHM+zpzPw2Bv6R6s78kaFisD/IqVwt95NMSiVTb3rNsJLs+KNKjuoV0 +EPeZuiQvDZDNi2+pKC+3tT65+sECno6ZSZdvn7naRYy3X+QWbOY/zG9mXIMyRXSs +oDXPH17838swDO7YrmWDvHO2Coz72eEftsovDql+D+4w6lq5DZygbHiSO3yyB4cR +AbI1hQ0nneQC8Gi+YPNEguAz5Sdvys91urZ1awllsTpG6gFPuAECggEBAPROYIyi +4mjr3MAUU0h6FHQ8uCARknM2NOZGDpyHmOtFjdTW47xGB3E8Dhyci0iOYjgwhcNe +lx/gZySKuCbC4HbxWUzLFYOOYRpj41oIZW3kvnkc7vVEst726CO+cVtTc8cjYXQL +pFQO3wN9C+S7OXIZOSO/P83jHm3bG9vQ1Kgh7gMBgZkwx3NMt2d1slEo4NViXZ5n +1960wWer5J6lQtpgwJYik7tZZBXkGA8QrhvzqfAGLYUQc3chWSHqlRt/+YJzFXZc +JYhpmCOs3jefC1I0T4wsxTJAhv3xnlGY5FDFIsHXqkCgwsqU9rYNfNQLLhuuwG3t +kR/sZp1eKmkDCjECggEBAN+Z1DqZIgd9rXTeUICbzlQy4891BhCWy3F20CTbzdfv +7EN49uhaD/OltJ8LJcDqm2F38Xjz4+svGfsYkqe0EMV1IOkdhnf29uGcWb/kWGGX +FtNRL3QhKntouVqqsJdeFNcvbPF4RZPUOQTiHjH2U+nTA/KIKU/nxSqJwR3fj9nc +v05k2jv+eoodDx4Fs/zS/cYzA0bjEXZlO6fS+MSWJwQGVbd3lqieo0FVuD3Y2RVs +nQidKUOm/qTE1/r//ggr0nX/GD6n2gRyUTV5yHIoZ/ENCOuxu38qyOk5ko8knPeo +IGqluaaTCyFav72vS9IbWVUKicdmzaLaYVLG1EzMS4kCggEAbHhoMckYUZF3f+kG +WUWq0zkqX0KuDW1h62PrlOA3qy5EnN2UW8GUCFirw1RWGy7suRoCKg5TdxnBcd4N +iVg5JVZfWdNJiBGtV3RGO3FC55oKX+fSyR9pc8mYpFYoKm5RF3fECywoGBJKPlhE +8ZeXF+vPDOobQCS5G/mO0vhTkxsNfFQs4IFh7PWA8PS26YYG3XdBLdXZaM5EmO0f +Irkph6AxofdUNHiqEXYmoHm9dDepmiKED3KQcbJiysZG0eDuPVcvjk+3cmu/2wcj +2vpIooULVKeEHp270gB3VK9Xn+0HU52Xv/4gvqWJLKesZtFX9X6Pfb9fEhd6MCDh +H6Lz4QKCAQBVTk/cp02HRBhoDOTzm/2ku+GT5VaR/6XQPP7+AOnQZ1VhDIZOcQXC +88YsI0mdBySOk/8JISskL80esepJlvYLDzumFECYBh18R3UqM4jQep8XsdKD4J9f +g72rbJGAvkD/M7XBjhwlYQL77PSOJScwfvPzlKUGOitLplCKAB/Wg1RtKSblWpoP +lTSORKi9XRW6hv3KDpk77TVMOr3z6kEqVOxg2XweCp/ILlbjKZbwiIaEocj7aXuf +CwZHWWDmZymdINx0Ev+nUKHQ4AxsGSRGn612llEtG/qYmAFlIRfNmFP9vApPFaxa +Zk/eo0EvAIYdXq7f50Wuytf0h6y5O/vZAoIBADrF4jHV+kbwWso1Rj4SA9xjmDSS +8v99Cr8YReoEwJoRo7sLa4XssY+deIvvR8OUwHmbPWjzDXeJFSMvJd9H7hytPvTW +wEPXd4eTRwcVo4d2ZUxszmJyLp5Mx/0qMtI2XkYPRfulPhXReYjkezjJ1mIzQiwL +RS4vwafBs1LvmbAM9stJt1K/XoI0e4kTYd9KyruKRW+JeuCrxcSV+O4nMRyOiCIQ +cHFXSlSHDyQo4z46rp2+IhIt9/Vq5DQ8hhbAtIjqa/ndFUj8FaQs6sOQq4410+LK +OCT7UGhKdY9I6fV/O4MAiLzyXcXNo6SRZ0YmPfam2v3gCqdDHLneLsbMoJ8= +-----END RSA PRIVATE KEY----- diff --git a/yezzey_test/.minio/certs/public.crt b/yezzey_test/.minio/certs/public.crt new file mode 100755 index 00000000000..4da1db77d25 --- /dev/null +++ b/yezzey_test/.minio/certs/public.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE6jCCAtICFFParimiXFvpZ6npYx8FcuF6ySIRMA0GCSqGSIb3DQEBCwUAMA0x +CzAJBgNVBAYTAlJVMB4XDTIzMDgxMTA5MzA1MFoXDTI0MDgxMDA5MzA1MFowVjEL +MAkGA1UEBhMCUlUxDTALBgNVBAgMBFRlc3QxDTALBgNVBAcMBFRlc3QxDTALBgNV +BAoMBFRlc3QxDTALBgNVBAsMBFRlc3QxCzAJBgNVBAMMAnMzMIICIjANBgkqhkiG +9w0BAQEFAAOCAg8AMIICCgKCAgEA1WMTewnKOrmE6ceR+rDY2+IK57DktPou0KlJ ++Cir5RjsCrh/mD8rNCJCVQ0ytV8EHUlnfJBcSnZRuKZHDTeaYAmKYe0Wcqey+bzA +KG9+W3kNvFm07Q/MIEAU4eGoeMtZpG91ZU9Jml45siJz7pZArnIrZdOwixBiu8oa +66b88Dz/mHDKJTgex9q27mYFERWu51ORAIchCLIQ9xKdMqR99irHKvGtC6TN5o6A +NfTXmhVuVm4X7Q6aeeDdmsOoP/KKmK/0yYvTunyfEhyaoEBw/aQmltfuEilMxrOz +GoAY1iJbHuXVtzYqkEomrmGXa7pStUoq3Ruu5PLVU+zEYqxVurYQTz0cWQZxo8BQ +dGbdmKiwu/NLoc6WA2dUHdVKB9cDnRQXfKXWO8TZOKCIJiQLzGsUW8Jq6kstiaE7 +IQL9LBZbSWxGScgkneDE+CucgSjD5PwDMZ5R6RXPVxdh9CHfNIytESa1zb1H/d+U +51xDRk6qkQbOAh5gt241mlekv1HYarRkwx8v0iiRa6ecmaGmnAA3SYRZYV6yYf1c +o+0Svm9EEG9fbZMHLmTExpz6xPwg5EsJ2jMgsJP6My2LKJ5TVoUCV7S+Cz5tXFyJ +yHmtedSTaeC+flt5A+9nYQKuk92mh2t/XaSCRDFugERUFAlEAvag0+06qhbVyt69 +fMGozzkCAwEAATANBgkqhkiG9w0BAQsFAAOCAgEAipaTRZxvDImsu/pwxSHEkKFz +ndKt84oEeuWRBo+sT5wZAcUJicz0tHVULwhXiqwMRC06LBqFxGzI/srokK2QiN5B +eikQvWEsr08tZuVm1ewNit6JzlwbaLOJc4DSTgJtWXKjWaIzqWsFl36ViLO55yTb +NgMgmpDhBl6hQ7yKtaSVJ+xiMKSd9nz6gRmJMz7sLchFmy3fTYtayUJcaDsFjEm6 +yTYM1oWj05xVcZJtASB8Bcp+XgeOdNbwvvTGrM2ctDC1GkPCK4X63GJfAuCv6lnV +ggzr2Z8dg/YVLCbmyvJq4n+rwNbVDebQBzRzyU9id0dRccyfPQzB69LPYcrMFzd7 +h97i5Vk7Ar66VeKxdw3Lbyl91yTUy7e7EdYkBrlPzMohRcuCEsFPaUdOoiy3dmUX +PeaubgtJpaNlVbp08rdGAgcrY4aqmvVM+tlKWfgiEnVmQ4vKTcrITnhintvttXyh +GrddsvKAE0m1rDpfd9BqXH4FEydFHoL7oMpKHnu9LUQsPHnwGvpq75KUN4j2nP4P +NIEmmqOr5SY9zcp9HQApxWPscQckb9aCIDZ8MMrceNSxRtU+bw0xNs+IILGx6dNA +cK+bDLPtMneno6XXDqqiitE2ohXf5WIeB3kPLOsp3awVhr/g+pqjMSSjyUEC+KXr +dkqHW3oKT52eCNzR350= +-----END CERTIFICATE----- diff --git a/yezzey_test/.minio/certs/server.csr b/yezzey_test/.minio/certs/server.csr new file mode 100755 index 00000000000..c223c34efb9 --- /dev/null +++ b/yezzey_test/.minio/certs/server.csr @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEmzCCAoMCAQAwVjELMAkGA1UEBhMCUlUxDTALBgNVBAgMBFRlc3QxDTALBgNV +BAcMBFRlc3QxDTALBgNVBAoMBFRlc3QxDTALBgNVBAsMBFRlc3QxCzAJBgNVBAMM +AnMzMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1WMTewnKOrmE6ceR ++rDY2+IK57DktPou0KlJ+Cir5RjsCrh/mD8rNCJCVQ0ytV8EHUlnfJBcSnZRuKZH +DTeaYAmKYe0Wcqey+bzAKG9+W3kNvFm07Q/MIEAU4eGoeMtZpG91ZU9Jml45siJz +7pZArnIrZdOwixBiu8oa66b88Dz/mHDKJTgex9q27mYFERWu51ORAIchCLIQ9xKd +MqR99irHKvGtC6TN5o6ANfTXmhVuVm4X7Q6aeeDdmsOoP/KKmK/0yYvTunyfEhya +oEBw/aQmltfuEilMxrOzGoAY1iJbHuXVtzYqkEomrmGXa7pStUoq3Ruu5PLVU+zE +YqxVurYQTz0cWQZxo8BQdGbdmKiwu/NLoc6WA2dUHdVKB9cDnRQXfKXWO8TZOKCI +JiQLzGsUW8Jq6kstiaE7IQL9LBZbSWxGScgkneDE+CucgSjD5PwDMZ5R6RXPVxdh +9CHfNIytESa1zb1H/d+U51xDRk6qkQbOAh5gt241mlekv1HYarRkwx8v0iiRa6ec +maGmnAA3SYRZYV6yYf1co+0Svm9EEG9fbZMHLmTExpz6xPwg5EsJ2jMgsJP6My2L +KJ5TVoUCV7S+Cz5tXFyJyHmtedSTaeC+flt5A+9nYQKuk92mh2t/XaSCRDFugERU +FAlEAvag0+06qhbVyt69fMGozzkCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQCw +gv4AQ9xh+LP16NQ0QCZ7QM8y6cA7jb7yqc8G+gvTjbacsuxpfI7qiJHx/Aon/ReL +D3Lra+HKzQticJaqFQEiXE6kXWB4DndP7hY7MF0R3xlBvNWyS8Q6Nr3m6IQiJx1Q +qd2Fgc7341wOMf3XMW/E1XUEsnzL8pWrKI+lvHAMI3u2KvsqwTtWtFJ4HMQoab7D +Xi4QYVk/DMF/lirPgcm1xnDZP07f7lzIGasO42XOBW1nV33w3bqVjRfKLZnbCs0Q +IOecmfseUBER9ycTUnFHN99BGx1SmTtXmqoeQIwijIvMoBNtEsOskqhW1+snFlNO +st9pyjwhuASvSWDFbEsaC1tl+5oTK31XMrxs99TDRoPMY1UwVFbrXrc+XWP1MSwg +0SB2c2DnLwGT9Lp9w1+epn9oa95B5JskwRp86Lbf5y+XV7W3Vp/vCzQ+db2sUHuT +bEUpNJl/KcxxseMMkpIJy83w17RhlRsgMKvAQtftYWX/z/RHyGKWHHPPoxEJaKyc +hm+X65QJd8QoOd/IXnkCMTQZocpQxoKBbYqdTBDxYfXnTILWHRUb0OfmjS9fEmBE +UqYdwNawOIXKlPsI/JseO132C3TtxjmkqMSNixTFyRvpoqy+/wFul3QA+yF2eCM0 +pfKcr8QsHn8WEVqjEmyQiM1ixrDIgIDc1jurvLX2cA== +-----END CERTIFICATE REQUEST----- diff --git a/yezzey_test/generate_ssh_key.sh b/yezzey_test/generate_ssh_key.sh new file mode 100755 index 00000000000..1df4cf14b60 --- /dev/null +++ b/yezzey_test/generate_ssh_key.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -ex + +ssh-keygen -f ~/.ssh/id_rsa -N '' +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys diff --git a/yezzey_test/import_gpg_keys.sh b/yezzey_test/import_gpg_keys.sh new file mode 100755 index 00000000000..fc351a3b416 --- /dev/null +++ b/yezzey_test/import_gpg_keys.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -ex + +gpg --import yezzey_test/pub.gpg +gpg --import yezzey_test/priv.gpg diff --git a/yezzey_test/install-wal-g.sh b/yezzey_test/install-wal-g.sh new file mode 100755 index 00000000000..bfad4e82a58 --- /dev/null +++ b/yezzey_test/install-wal-g.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -ex + +# Install latest Go compiler +sudo add-apt-repository ppa:longsleep/golang-backports +sudo apt update +sudo apt install -y golang-go + +# Install lib dependencies +sudo apt install -y libbrotli-dev liblzo2-dev libsodium-dev curl cmake + +# Fetch project and build +git clone https://github.com/wal-g/wal-g.git +cd wal-g +make deps +make gp_build +mv main/gp/wal-g /usr/bin/wal-g + +#Check the installation +wal-g --version diff --git a/yezzey_test/install_yproxy.sh b/yezzey_test/install_yproxy.sh new file mode 100755 index 00000000000..9e3abd1d9cd --- /dev/null +++ b/yezzey_test/install_yproxy.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -ex + +# Install latest Go compiler +sudo add-apt-repository ppa:longsleep/golang-backports +sudo apt update +sudo apt install -y golang-go + +# Install lib dependencies +sudo apt install -y libbrotli-dev liblzo2-dev libsodium-dev curl cmake + +# Fetch project and build +git clone https://github.com/open-gpdb/yproxy.git +cd yproxy +make build + +mv devbin/yproxy /usr/bin/yproxy + +#Check the installation +yproxy --version diff --git a/yezzey_test/priv.gpg b/yezzey_test/priv.gpg new file mode 100644 index 00000000000..1b9727a319c --- /dev/null +++ b/yezzey_test/priv.gpg @@ -0,0 +1,105 @@ +-----BEGIN PGP PRIVATE KEY BLOCK----- + +lQcYBGTbjzEBEADOjtlqSaBe89Pg/orabZALZWi3lRfVqDaBjb/po5jOPlsefBSb +aa7hyAGhMfncmVaqZJXH1ikT2oHx430GcBXD8gjMZbil7U01DK8XBr6iSw/EPWpA +0jD2FdmPCG6vzF58KJ5tv2uPjZUShhYymIwDMZuQAzDMFgGP3dhDQNp4TUXX0AyH +vRGMEeckyjyXhheu1alJ1XNzT9s3dAq3LGFgHuUH4BEwSljpKFP+BARXTGq776wP +JXt7YaJLDB/RXgfMNOKZR18eUYEcckjziMkptBvbD6sG/h9B7ldeYCP91jDqR51Q +ls4eFf6C0dqoJ5ELv8GBW3BZWsnFGffGRA2a6kAk14eSWtcF7w0N5J7o4kPYJSPa +OXq+gIvBz9KeJgQKDu6DrHZYdZh5cCQKFHdLeKzaEsrUPvSOvjfTB8+ywb3QiJPt +/yKYfcZU41zB7dCfbbEFaKTGEvjRUm2XZH9maJvQ5fNUEmtXwQd05nBwN8aW+00d +w5Jy35aSwogCLrJuHMpwI56TRAU81d1R9RaTxNbcZUro8IdOW78BaB8RGrvGT+z1 ++TBPlJOpzJgAYJe6wZp8M4xk0z2y+6ia1X+yUvZXr51U8qiHM1wr1uBZx66EdFv+ +HQf3grEnW3sPCS3kF4Yn6Xm/xIWqbRlXf988RqZc6fKa201DhriEOhk9SQARAQAB +AA/8D1gsTYmHqNYbLNEr9nsCIt6XL2Abg6s51Vj24z/cd6HJOhQEoeDxdWNav6NL +SPh5KDJNsCk4JvnqAlKgdIx6CXLe9XQvCeB9hk3Zae/91GeYFb9sN8GFkppf7ZC2 +TEf10upfjo6USnc9lkEwv/R2Hjyerk2WX+rYldvol2JneF1hLV2OuLyyY4Hm9mQx +qR30f2/u4gXBzFSn19A+hYn999qDsBiUnk4/mZ590tZwjh2mpixxTzS9HhbMLCAL +8w01rhVVqgZTyhhK65aP1cuZCgiPsy/QSDYzkz3F7sRcmBxDyPul85koWbLPk99M +C4bY7xMBKH9m8rRcXdCGiAuBKRlIZFQcyI7Lw4WxqxdJROAjhYb+FXDKSst+cyw8 +c0qP/UbqnNaZmWQ0w95UMlwu8MZ6sJK+CzaLkjt4itkN27/DCfxyfuETqEiTO1ba +yJoVDE5MH0CQLIlByvJAU9AmNd1vLL9CTzm39m8B+eOrPR5n2khu85D+VR3vWZo7 +XnM5RTXhKTMfGFCSunNVlnABEy1eFZYFOogZOkVYdqxRTiZccDIZgV+spN/op4kA +SofCehr1xxdd3DXRar/GOz8J2jrWyqaQzhsD4kKl5nBM3NH7fAPsBBv0VH1DE0OM +ZsiB0yQFyHTWeQOcNla37pD1H1DZa4fpsL6m3BqSK38cKcEIAOUr9faEnjYPTAoW +TKWpoqt2ghyGsklrdYVcgb7pINam1UV0f4uvMBv702ZnLwoC0Rn5ITgPXH0a0MDM +H9yfWZgb6L/mmE+B81D+EDUkb88qafXXq9TMspbwbSuY/DWFTZPVfRDWYacr/IPq +XdtZqNBR2BWDeckgBcnEwg/PBn2HrrpfWab/D9Lt1QFY5baCuFevib7/D1DxO9qq +57iA0IDT+YfydYawwixvarkCNmUfkDowhR7DqHWDppWt5n7rxiTuE7I7C88pV4mV +rVTvXeMh6ooB3m2yW1FhsTgT0DrWxul1glCASQ+TfOks6dcDMvcFZFSzWT1r2rR+ +LqNNpDkIAOa9LcZEaRV86QLLcBYpPnaDAFlZYU/1b6QRbU3L77rULvtHSVegEApp +7a1mDrx1bTp5Z6TGq5ls9bMHF4tg5IcBLDHsO6dYAl2vAbYuCwjfPa3hu/oK+J5F +9IQLrvS2vM2shqhIfIbKglXwAWySLQoR/CsAMgH4vk/R/08AJQRyXndaY3rLImQg +ofUQaRxl4950wO/RJSQ9RD5q1bvseJ6bZNlYu1SoEJ/uOGqlMbMP9DAUb7LnTwlt ++9Mdv/ibYBABnUh5hLLCc5lBgeN+Pd3AAEygvbK6rW9a2CV4vO5U0aBU7Um8WAUz +suVEQM/rMTJanxHAeHgRYMDccadTAZEH/2i/xd20U2Taorj+/Lrq21hQJ1jKpyN+ +CnglPV83pGX1V1j007C2zhO/YB5qTh9ztZvtwkuX5EoUN9mGRZ0t1K3xbmwlDQbB +D1hofRcN0U2h+ZXQ+aID+gswLhw6jWatHiDmhIGSPzopR/1oZrQwS0CJuqb3XQFP +lAgeMPEuFufIf7tijFLL3PDuTPnG5lndI6+uCw+BhkfgN7jAHHREQlolGgzo3mwW +htGr6cOd9W7a9tGN7nHb/yVH22gnXs++wyC8OoeuEgAqQL2CyNOEo3EyN1K79QH0 +7qaKg+XHJy/NymQdcALxp3zUNh3UuOd0CZxPudTVfD5j3cD5fST9Owx+CrQsUm9v +dCBTdXBlcnVzZXIgPHJvb3RAaGFuZGJvb2sud2VzdGFyZXRlLmNvbT6JAk4EEwEK +ADgWIQQBHXUy4S6x61I7K1L/Gb3KATgePQUCZNuPMQIbAwULCQgHAgYVCgkICwIE +FgIDAQIeAQIXgAAKCRD/Gb3KATgePXGWEACRiwzgx5EVSpQEKe79sRAkO9hmAB33 +D4dN4y+D+2kUY/paPg3etCdPIJQs6qFFu+HIWnCTGiUepcsBeUvRpTDwguP6MtXx +TObmaU20ZoSbZZjqnGZ5V9VlwCQjJWj3RN+7BYVsECPJGfEMW+M2nHX3MrGpohfo +KpacoL2kjVswvWocGHNIMHpVHlyGGUtwVHz3kO5HqwcZvWwZj189dKTMUSsfbXdz +723BDp+m/NLUi8Ki/BZpsRwSXqGbpdQ8LYaSJvtS0DY/rIILIOyIfhnHK74rgCip +0vfZsfypc58YrPrj9+pfF7Rx00m0RfGll+lD1Pha4GStS1PvtpdtRTvf6qsmArON +q1N7ha8YJHtPueTKln/7ik3jqir7vO7SLL+CTa/XfS1jaJhonHORfW+fekndXK34 +zrQ75cFn7D5BqqcTXu2hkTDbPqd9aBorWFU/vbqZRVH1a/RRcVU9SODnIwdCm06x +XLoujcr1daZd9GWT0PMvFHaehQ/4zbdd/pYJ9efU1zlfpdwxUpBWyfUEOGbwpHGv +7FjXIHSkHkTNgaOK5AP0FsSGfjAzTxq/2yLRQKm5f3GqWNOZ7nprLg85Tyxh/gdZ +rk8o3/pW7P/XDLU6jfhWV5YR2mxddaFs6P7D+jvRf3EVHbJHtcK30usIWjGDWyNk +310w6Q3CumfEo50HGARk248xARAA6LnhBgnfn0B405FGDbb+MOj/0dmmb81XwL52 +ZqbAIsomMUfrzzFHJb0PtexSsIr7I+NweVCSvc3UaGcMLF8pqOVnP4pkPXi+hBlG +eNeE4KQvMpn56nLsZ90GtpKYLLsMQB6VMm98+v4wCJ6mPkM89SPQlc7q9QFfC31g +trbb5Y//eDqCAATiVIdZ6nEFNKgNDMKC0UmRY+D3RaeElPpWO5lWDDldSjOZ9/Or +dg/TSdRviK7+NzdyOM8xUc6lQunG26qRH/bqQzHluBYMp4W3l71Ojf8YlSATkziB +dAdwpQ1MVx4rCk0o5nYuGf58sgxTIr6qIR4777Z7lanc9utcLeSbltkid+zATW8x +PFPG1ID6pNZ0JX2ttQ7HQAb3RfNDkZxGijqEnJyjY0nJON0/8Jcz46GkeUzkae0r +I/TOS7OASs13NWMQV8TsqDy9VMT+pcTDjy1Rad8krXtraK93WlIPWhb2vCSJkCMr +412bl2SOrtYUdu093yLYkEkrq6vZsl4zkIPrCbuEdJZgSn5d23nHd2ARsaj6D4KZ +RP9dg7A7RpEezNLAh0FDwMLe/nJpNfQdd9MXITALc0CwJ/x0sFBrj1+q7jELYtSp +zh993C4i9ySOaWKfOGV9UjeSLBYfExuxisMQyAIoD8hhNOHnqWVgcYqJ7R3w0ymU +96M29M8AEQEAAQAP/1WZpSiM8ilH1AlxmlRKFjYURaBAz6S44UmeZLt+IxbIxwKC +Yzxq8jHx1/kAyytve09ohULB/a99qV6bZJFfkVmzw2XON++aXW0GRPMGxrPAADI7 +C38ONWFAnYsC4aE2TZu6BAOwmUZSv4U0IY6uOZorSboIiUiD8BswSyX5nWlTLVLi +JlXudfdEb7C5UIJdO6uRUf+78RPNN/ZxVuVbLOOwE0Pcx7EWyM+4Wz1KNdumnT2n +rA7QQJ2frBLckNHLXh8HHmkk72a20DmFNrNZjj1sXpwBE+AqE7knZAozAF5dRVKX +4Jnh5qTaHDvobKIqwVt6yOX0knQp6UwT1hgmWtk8584hLoxuQbzr5yM3/QNiPMka ++aMGu1COS34ppIQfETt1C3Lv7tGcFXkadyV7JfVl0u8qAJjGzKgmrBcVkDBZdKT6 +rP3QAAwpcwdDaY0rXR6FDrHKZWZhvsSNjX9vXf6ThCG4/Wb07bMia1Y6UdtNWYAB +uDQkFJszng03tQP8P1p1J3XouzOb0XA3tkusOoRnMlR47oDb1TlUJKuJv7ztK3AI +5v7FyJO9TJbwhBWWno171KKGn7R1vciNYxWB1IqGamLQFKajTFKYVikn7mCdfDNe +zVgGcK9DAgoJXtclmjtKzUnqfW6ctJKrNKXkyEgkbWKGv33skwDSMZxakTpFCADt +dzG+RbDlZrwlSMhv+33vjV6arulEY32vj3Thbphb4v1JwOTdXNVwF5pC+pf7sUgx +ZAbEuVix4zu+AznCdRGPYjAHZgUe8ldp7JrQuVzlknBJ0Z+DedXcvL8RAs7rFedo +BO9FTqIolTqFZyBJeaDJcMTwf8qQdE71+n/r9oigKi/8UXSAZInnvgYMTUjgegqx +UyQJTEXRQYzUfCUHFpYqel1NwcsLO4JciDwb0ak1QUvIy0HulLrRKrRYUZ7Xy3U9 +XBjt8BPWKdNfN6qKWK+7tYzPNB2BLQT2Z+l0L+Ir9TaWxJTZdfRysYytAhTl3ZoL +99KlA7qIxOs7+DnuR8jdCAD64/v/xNl02ck3pBnnxjnaspe3xEz5D9s/a9Ob+2MU +pDp5b0ZwPJ0tpk6CguiVhi1gzRXzjABPrCbsBOXk2gQ+Pilk3jkugG+wqg4LOLzy +3AiDbD+GVmTL9gdhPNoNIjPBz1tkZEGdTVsUOWT5Kuom3KvbmzowdlTVb4x/kkRM +oOTfO1Uhna3b/pcCsMiHXkM3ZME+DFEy6Imh38742ljllECCDCUAvd4drbx4VaNU +rXyI+j3NP8j/vHhWCld0Bg/C5Xq9ioDOzyrvApTat2OZF2Z9K4nYYTTwniAHJWNk +hIPDKyIKeE4Y/WBrSfVQbdtTNi0fj+3RTtS1j4uSwMObCADf3miT0aX2pBxy7N6D +krhJwq5Usaeu3fVNOOm/HVQsouqJ0BYPX412YIDtL4iY/YibVdzVlNg2hQuPJYGx +hZF/isY+qpckLvdABbtreBDxoqtqBgdOI0EGXpBuJTq9iuh2kiKFWATEchCMND9p +wZK0jrVtQ7JnXwW4CRBnN0wV2LO8+VF6How9FSdVRykG4cRed7JB5U9B11PK3tay +v82+YnmTEr7YuMiJoXHt7+9nA1ftd/Au0IegwvmRO89MxvpGeS8GVZFdYLP5UQiH +bdrnFmLyQEsPPIM3yvFQCfb2W4WmxH3PycTrNDs7mone1L/pk/eMdg7QYF+E/QZD +1N1pfKeJAjYEGAEKACAWIQQBHXUy4S6x61I7K1L/Gb3KATgePQUCZNuPMQIbDAAK +CRD/Gb3KATgePW6sD/sHl+DskI2dz4Ym7ZKng39QHOKFdzoK9xtnLQSynqfrwzvz +UEzsqWNjOHONfzat9savfGBvWB6ULz3vQ4yZ731Y4Tdc4UmVo+79hIgeArNiLyou +p6tvcl+bKOv72xlrYn43lYy51Aj5TsU6TBGVvRETnmOWvzGzsQrhcZ+IA8p4JXoy +fiAM+reuc1lwWYZWS5n+XotskFS3L6BIXLPW1QIHxvcMkZAI8/VeloQej1z4t50i +7mAAXB/RwjXGCLnhFxhjWLHglrxpg6JpsuCkI7j5csYQuzhKrSWa3mxKv7rysRh3 +VpIkHfRRu/kyfOwM4vYCRsQH2lBl/JVgAn1koZKdnGomoq/Gw7+psuYNGX9+CQ5E +ShHGWGE2TUjCi88FDLDpOOmpKMoK19HbNGkeVeKXY80D5LhjSrOL61IY+3Hzyb0a +JhHA7dXAqMHak38Ps5sUrEtgDBjkhGtctCUGV9OsvlqAXwIDLB96W5LKJKw48osE +vcfquO0z3bMKyr0irCyBzn3sPd2nubFy/o1umF8efg2OqWybQ3PZLyMojQ9uu8Yb +SLM3+Q1vX/yAY8+KmaAvvUuTCS1dRWAM0s1g86AUCZ5LbMa0HwIGjCSA2nUw8BD8 +l3sVHJsSKOhB0XTUBVkIDBPdJcj/K6G8E0PGO19UQEMRis0/jnPPL9t8G63xQg== +=uAYd +-----END PGP PRIVATE KEY BLOCK----- diff --git a/yezzey_test/pub.gpg b/yezzey_test/pub.gpg new file mode 100644 index 00000000000..139122aea2a --- /dev/null +++ b/yezzey_test/pub.gpg @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGTbjzEBEADOjtlqSaBe89Pg/orabZALZWi3lRfVqDaBjb/po5jOPlsefBSb +aa7hyAGhMfncmVaqZJXH1ikT2oHx430GcBXD8gjMZbil7U01DK8XBr6iSw/EPWpA +0jD2FdmPCG6vzF58KJ5tv2uPjZUShhYymIwDMZuQAzDMFgGP3dhDQNp4TUXX0AyH +vRGMEeckyjyXhheu1alJ1XNzT9s3dAq3LGFgHuUH4BEwSljpKFP+BARXTGq776wP +JXt7YaJLDB/RXgfMNOKZR18eUYEcckjziMkptBvbD6sG/h9B7ldeYCP91jDqR51Q +ls4eFf6C0dqoJ5ELv8GBW3BZWsnFGffGRA2a6kAk14eSWtcF7w0N5J7o4kPYJSPa +OXq+gIvBz9KeJgQKDu6DrHZYdZh5cCQKFHdLeKzaEsrUPvSOvjfTB8+ywb3QiJPt +/yKYfcZU41zB7dCfbbEFaKTGEvjRUm2XZH9maJvQ5fNUEmtXwQd05nBwN8aW+00d +w5Jy35aSwogCLrJuHMpwI56TRAU81d1R9RaTxNbcZUro8IdOW78BaB8RGrvGT+z1 ++TBPlJOpzJgAYJe6wZp8M4xk0z2y+6ia1X+yUvZXr51U8qiHM1wr1uBZx66EdFv+ +HQf3grEnW3sPCS3kF4Yn6Xm/xIWqbRlXf988RqZc6fKa201DhriEOhk9SQARAQAB +tCxSb290IFN1cGVydXNlciA8cm9vdEBoYW5kYm9vay53ZXN0YXJldGUuY29tPokC +TgQTAQoAOBYhBAEddTLhLrHrUjsrUv8ZvcoBOB49BQJk248xAhsDBQsJCAcCBhUK +CQgLAgQWAgMBAh4BAheAAAoJEP8ZvcoBOB49cZYQAJGLDODHkRVKlAQp7v2xECQ7 +2GYAHfcPh03jL4P7aRRj+lo+Dd60J08glCzqoUW74chacJMaJR6lywF5S9GlMPCC +4/oy1fFM5uZpTbRmhJtlmOqcZnlX1WXAJCMlaPdE37sFhWwQI8kZ8Qxb4zacdfcy +samiF+gqlpygvaSNWzC9ahwYc0gwelUeXIYZS3BUfPeQ7kerBxm9bBmPXz10pMxR +Kx9td3PvbcEOn6b80tSLwqL8FmmxHBJeoZul1DwthpIm+1LQNj+sggsg7Ih+Gccr +viuAKKnS99mx/Klznxis+uP36l8XtHHTSbRF8aWX6UPU+FrgZK1LU++2l21FO9/q +qyYCs42rU3uFrxgke0+55MqWf/uKTeOqKvu87tIsv4JNr9d9LWNomGicc5F9b596 +Sd1crfjOtDvlwWfsPkGqpxNe7aGRMNs+p31oGitYVT+9uplFUfVr9FFxVT1I4Ocj +B0KbTrFcui6NyvV1pl30ZZPQ8y8Udp6FD/jNt13+lgn159TXOV+l3DFSkFbJ9QQ4 +ZvCkca/sWNcgdKQeRM2Bo4rkA/QWxIZ+MDNPGr/bItFAqbl/capY05nuemsuDzlP +LGH+B1muTyjf+lbs/9cMtTqN+FZXlhHabF11oWzo/sP6O9F/cRUdske1wrfS6wha +MYNbI2TfXTDpDcK6Z8SjuQINBGTbjzEBEADoueEGCd+fQHjTkUYNtv4w6P/R2aZv +zVfAvnZmpsAiyiYxR+vPMUclvQ+17FKwivsj43B5UJK9zdRoZwwsXymo5Wc/imQ9 +eL6EGUZ414TgpC8ymfnqcuxn3Qa2kpgsuwxAHpUyb3z6/jAInqY+Qzz1I9CVzur1 +AV8LfWC2ttvlj/94OoIABOJUh1nqcQU0qA0MwoLRSZFj4PdFp4SU+lY7mVYMOV1K +M5n386t2D9NJ1G+Irv43N3I4zzFRzqVC6cbbqpEf9upDMeW4FgynhbeXvU6N/xiV +IBOTOIF0B3ClDUxXHisKTSjmdi4Z/nyyDFMivqohHjvvtnuVqdz261wt5JuW2SJ3 +7MBNbzE8U8bUgPqk1nQlfa21DsdABvdF80ORnEaKOoScnKNjSck43T/wlzPjoaR5 +TORp7Ssj9M5Ls4BKzXc1YxBXxOyoPL1UxP6lxMOPLVFp3ySte2tor3daUg9aFva8 +JImQIyvjXZuXZI6u1hR27T3fItiQSSurq9myXjOQg+sJu4R0lmBKfl3becd3YBGx +qPoPgplE/12DsDtGkR7M0sCHQUPAwt7+cmk19B130xchMAtzQLAn/HSwUGuPX6ru +MQti1KnOH33cLiL3JI5pYp84ZX1SN5IsFh8TG7GKwxDIAigPyGE04eepZWBxiont +HfDTKZT3ozb0zwARAQABiQI2BBgBCgAgFiEEAR11MuEusetSOytS/xm9ygE4Hj0F +AmTbjzECGwwACgkQ/xm9ygE4Hj1urA/7B5fg7JCNnc+GJu2Sp4N/UBzihXc6Cvcb +Zy0Esp6n68M781BM7KljYzhzjX82rfbGr3xgb1gelC8970OMme99WOE3XOFJlaPu +/YSIHgKzYi8qLqerb3Jfmyjr+9sZa2J+N5WMudQI+U7FOkwRlb0RE55jlr8xs7EK +4XGfiAPKeCV6Mn4gDPq3rnNZcFmGVkuZ/l6LbJBUty+gSFyz1tUCB8b3DJGQCPP1 +XpaEHo9c+LedIu5gAFwf0cI1xgi54RcYY1ix4Ja8aYOiabLgpCO4+XLGELs4Sq0l +mt5sSr+68rEYd1aSJB30Ubv5MnzsDOL2AkbEB9pQZfyVYAJ9ZKGSnZxqJqKvxsO/ +qbLmDRl/fgkOREoRxlhhNk1IwovPBQyw6TjpqSjKCtfR2zRpHlXil2PNA+S4Y0qz +i+tSGPtx88m9GiYRwO3VwKjB2pN/D7ObFKxLYAwY5IRrXLQlBlfTrL5agF8CAywf +eluSyiSsOPKLBL3H6rjtM92zCsq9Iqwsgc597D3dp7mxcv6NbphfHn4Njqlsm0Nz +2S8jKI0PbrvGG0izN/kNb1/8gGPPipmgL71LkwktXUVgDNLNYPOgFAmeS2zGtB8C +BowkgNp1MPAQ/Jd7FRybEijoQdF01AVZCAwT3SXI/yuhvBNDxjtfVEBDEYrNP45z +zy/bfBut8UI= +=x2ib +-----END PGP PUBLIC KEY BLOCK----- diff --git a/yezzey_test/run_tests.sh b/yezzey_test/run_tests.sh new file mode 100755 index 00000000000..68dff441c49 --- /dev/null +++ b/yezzey_test/run_tests.sh @@ -0,0 +1,75 @@ +#!/bin/bash +set -ex + +eval "$(ssh-agent -s)" +ssh-add ~/.ssh/id_rsa +sudo service ssh start +ssh -o StrictHostKeyChecking=no gpadmin@$(hostname) "echo 'Hello world'" + +sudo bash -c 'cat >> /etc/ld.so.conf <<-EOF +/usr/local/lib + +EOF' +sudo ldconfig + +sudo bash -c 'cat >> /etc/sysctl.conf <<-EOF +kernel.shmmax = 500000000 +kernel.shmmni = 4096 +kernel.shmall = 4000000000 +kernel.sem = 500 1024000 200 4096 +kernel.sysrq = 1 +kernel.core_uses_pid = 1 +kernel.msgmnb = 65536 +kernel.msgmax = 65536 +kernel.msgmni = 2048 +net.ipv4.tcp_syncookies = 1 +net.ipv4.ip_forward = 0 +net.ipv4.conf.default.accept_source_route = 0 +net.ipv4.tcp_tw_recycle = 1 +net.ipv4.tcp_max_syn_backlog = 4096 +net.ipv4.conf.all.arp_filter = 1 +net.ipv4.ip_local_port_range = 1025 65535 +net.core.netdev_max_backlog = 10000 +net.core.rmem_max = 2097152 +net.core.wmem_max = 2097152 +vm.overcommit_memory = 2 + +EOF' + +sudo bash -c 'cat >> /etc/security/limits.conf <<-EOF +* soft nofile 65536 +* hard nofile 65536 +* soft nproc 131072 +* hard nproc 131072 + +EOF' + +export GPHOME=/usr/local/gpdb +source $GPHOME/cloudberry-env.sh +ulimit -n 65536 +make destroy-demo-cluster && make create-demo-cluster +export USER=gpadmin +source gpAux/gpdemo/gpdemo-env.sh + +gpconfig -c shared_preload_libraries -v yezzey + +gpstop -a -i && gpstart -a + +createdb $USER + + +gpconfig -c yezzey.yproxy_socket -v "'/tmp/yproxy.sock'" +psql -c "ALTER SYSTEM SET yezzey.use_gpg_crypto TO false" +gpconfig -c yezzey.use_otm_feature -v "true" +gpconfig -c yezzey.use_gpg_crypto -v "false" + +gpstop -a -i && gpstart -a + +#run yproxy in daemon mode +/usr/bin/yproxy -c /tmp/yproxy.yaml -ldebug > yproxy.log 2>&1 & + +i=0 +while (! [ -S /tmp/yproxy.sock ]) && [ $i -lt 20 ]; do sleep 1; i=$(($i+1)) ; done + +cd gpcontrib/yezzey +make installcheck || (echo Yproxy logs; cat ../../yproxy.log; cat /home/gpadmin/gpcontrib/yezzey/regression.diffs && exit 1) diff --git a/yezzey_test/wal-g-conf.yaml b/yezzey_test/wal-g-conf.yaml new file mode 100644 index 00000000000..8de5c4c2159 --- /dev/null +++ b/yezzey_test/wal-g-conf.yaml @@ -0,0 +1,12 @@ +AWS_ACCESS_KEY_ID: "$AWS_ACCESS_KEY_ID" +AWS_SECRET_ACCESS_KEY: "$AWS_SECRET_ACCESS_KEY" +AWS_ENDPOINT: "$AWS_ENDPOINT" +AWS_S3_FORCE_PATH_STYLE: true + +WALG_COMPRESSION_METHOD: "brotli" +WALG_DELTA_MAX_STEPS: 6 +WALG_UPLOAD_CONCURRENCY: 10 +WALG_DISK_RATE_LIMIT: 41943040 +WALG_NETWORK_RATE_LIMIT: 10485760 +WALG_S3_PREFIX: "$WALG_S3_PREFIX" +WALG_PGP_KEY_PATH: "/home/gpadmin/yezzey_test/priv.gpg" diff --git a/yezzey_test/yproxy.conf b/yezzey_test/yproxy.conf new file mode 100644 index 00000000000..855c3fce7d7 --- /dev/null +++ b/yezzey_test/yproxy.conf @@ -0,0 +1,25 @@ +socket_path: "/tmp/yproxy.sock" +interconnect_socket_path: "/tmp/ic.sock" +log_level: debug + +storage: + access_key_id: "$AWS_ACCESS_KEY_ID" + secret_access_key: "$AWS_SECRET_ACCESS_KEY" + storage_endpoint: "$AWS_ENDPOINT" + storage_prefix: "" + storage_bucket: "gpyezzey" + storage_region: "us-west-2" + storage_type: "s3" + tablespace_map: + "pg_default": "gpyezzey" + "tab1": "gpyezzey2" + "tab2": "gpyezzey3" + +backup_storage: + access_key_id: "$AWS_ACCESS_KEY_ID" + secret_access_key: "$AWS_SECRET_ACCESS_KEY" + storage_endpoint: "$AWS_ENDPOINT" + storage_prefix: "" + storage_bucket: "gpyezzey" + storage_region: "us-west-2" + storage_type: "s3" From 2d43adc1665ef326345175b96b1e2b9ec51a647c Mon Sep 17 00:00:00 2001 From: Leonid <63977577+leborchuk@users.noreply.github.com> Date: Wed, 18 Feb 2026 16:41:03 +0300 Subject: [PATCH 02/14] Add group access to CBDB (#12) * Allow group access for init CBDB * Allow group access for segments CBDB --------- Co-authored-by: Leonid Borchuk --- gpMgmt/bin/gpinitsystem | 1 + gpMgmt/bin/lib/gpcreateseg.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/gpMgmt/bin/gpinitsystem b/gpMgmt/bin/gpinitsystem index fa85d42ae3f..f8d42e87b11 100755 --- a/gpMgmt/bin/gpinitsystem +++ b/gpMgmt/bin/gpinitsystem @@ -1272,6 +1272,7 @@ CREATE_QD_DB () { if [ x"$HEAP_CHECKSUM" == x"on" ]; then cmd="$cmd --data-checksums" fi + cmd="$cmd --allow-group-access" LOG_MSG "[INFO]:-Commencing local $cmd" $cmd >> $LOG_FILE 2>&1 RETVAL=$? diff --git a/gpMgmt/bin/lib/gpcreateseg.sh b/gpMgmt/bin/lib/gpcreateseg.sh index 5dd0f5b0006..89f5f405295 100755 --- a/gpMgmt/bin/lib/gpcreateseg.sh +++ b/gpMgmt/bin/lib/gpcreateseg.sh @@ -106,6 +106,7 @@ CREATE_QES_PRIMARY () { cmd="$cmd $LC_ALL_SETTINGS" cmd="$cmd --max_connections=$QE_MAX_CONNECT" cmd="$cmd --shared_buffers=$QE_SHARED_BUFFERS" + cmd="$cmd --allow-group-access" if [ x"$HEAP_CHECKSUM" == x"on" ]; then cmd="$cmd --data-checksums" fi From 7d2fc20a3aa124f38da1e63fab3b834aa88d37cc Mon Sep 17 00:00:00 2001 From: reshke Date: Wed, 11 Feb 2026 00:43:38 +0500 Subject: [PATCH 03/14] Add yezzey as submodule (#23) --- .gitmodules | 3 +++ gpcontrib/yezzey | 1 + 2 files changed, 4 insertions(+) create mode 160000 gpcontrib/yezzey diff --git a/.gitmodules b/.gitmodules index a7b61644ee2..9ebe8907c58 100644 --- a/.gitmodules +++ b/.gitmodules @@ -15,3 +15,6 @@ path = dependency/yyjson url = https://github.com/ibireme/yyjson.git +[submodule "gpcontrib/yezzey"] + path = gpcontrib/yezzey + url = git@github.com:open-gpdb/yezzey.git diff --git a/gpcontrib/yezzey b/gpcontrib/yezzey new file mode 160000 index 00000000000..ba7467211db --- /dev/null +++ b/gpcontrib/yezzey @@ -0,0 +1 @@ +Subproject commit ba7467211db94f0017e0028a6541fc201f02634e From 2de10ec33a8e67e5add51a509fee9c27073682be Mon Sep 17 00:00:00 2001 From: Leonid <63977577+leborchuk@users.noreply.github.com> Date: Thu, 12 Feb 2026 17:11:42 +0300 Subject: [PATCH 04/14] UseAnonymousAddress (#24) Co-authored-by: Leonid Borchuk --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 9ebe8907c58..f900edb6807 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,4 +17,4 @@ [submodule "gpcontrib/yezzey"] path = gpcontrib/yezzey - url = git@github.com:open-gpdb/yezzey.git + url = https://github.com/open-gpdb/yezzey.git From 8958fe31f59d8139649ee513ca3155efb0b26f60 Mon Sep 17 00:00:00 2001 From: Leonid <63977577+leborchuk@users.noreply.github.com> Date: Wed, 11 Mar 2026 10:21:49 +0300 Subject: [PATCH 05/14] Add yezzey build option (#26) * Add yezzey build option * Move yezey to commit 4c6b5b8 --------- Co-authored-by: Leonid Borchuk --- .github/workflows/yezzey-ci.yaml | 334 +++++++++++++++++++++++ .github/workflows/yezzey-test.yml | 21 -- configure | 54 ++++ configure.ac | 7 + docker/yezzey/.dockerignore | 2 - docker/yezzey/Dockerfile | 131 --------- docker/yezzey/docker-compose.yaml | 45 --- gpcontrib/Makefile | 10 + gpcontrib/yezzey | 2 +- src/Makefile.global.in | 3 + yezzey_test/.minio/certs/CAs/ca.cert.pem | 29 -- yezzey_test/.minio/certs/CAs/ca.cert.srl | 1 - yezzey_test/.minio/certs/CAs/ca.key | 51 ---- yezzey_test/.minio/certs/private.key | 51 ---- yezzey_test/.minio/certs/public.crt | 29 -- yezzey_test/.minio/certs/server.csr | 27 -- yezzey_test/generate_ssh_key.sh | 6 - yezzey_test/import_gpg_keys.sh | 5 - yezzey_test/install-wal-g.sh | 20 -- yezzey_test/install_yproxy.sh | 20 -- yezzey_test/priv.gpg | 105 ------- yezzey_test/pub.gpg | 52 ---- yezzey_test/run_tests.sh | 75 ----- yezzey_test/wal-g-conf.yaml | 12 - yezzey_test/yproxy.conf | 25 -- 25 files changed, 409 insertions(+), 708 deletions(-) create mode 100644 .github/workflows/yezzey-ci.yaml delete mode 100644 .github/workflows/yezzey-test.yml delete mode 100644 docker/yezzey/.dockerignore delete mode 100644 docker/yezzey/Dockerfile delete mode 100644 docker/yezzey/docker-compose.yaml delete mode 100755 yezzey_test/.minio/certs/CAs/ca.cert.pem delete mode 100755 yezzey_test/.minio/certs/CAs/ca.cert.srl delete mode 100755 yezzey_test/.minio/certs/CAs/ca.key delete mode 100755 yezzey_test/.minio/certs/private.key delete mode 100755 yezzey_test/.minio/certs/public.crt delete mode 100755 yezzey_test/.minio/certs/server.csr delete mode 100755 yezzey_test/generate_ssh_key.sh delete mode 100755 yezzey_test/import_gpg_keys.sh delete mode 100755 yezzey_test/install-wal-g.sh delete mode 100755 yezzey_test/install_yproxy.sh delete mode 100644 yezzey_test/priv.gpg delete mode 100644 yezzey_test/pub.gpg delete mode 100755 yezzey_test/run_tests.sh delete mode 100644 yezzey_test/wal-g-conf.yaml delete mode 100644 yezzey_test/yproxy.conf diff --git a/.github/workflows/yezzey-ci.yaml b/.github/workflows/yezzey-ci.yaml new file mode 100644 index 00000000000..c1c41497a64 --- /dev/null +++ b/.github/workflows/yezzey-ci.yaml @@ -0,0 +1,334 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# Yezzey CI Workflow +# -------------------------------------------------------------------- +name: Yezzey CI Pipeline + +on: + push: + branches: [ main ] + pull_request: + types: [opened, synchronize, reopened, edited] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +env: + CLOUDBERRY_HOME: "/usr/local/cloudberry-db" + CLOUDBERRY_VERSION: "main" + +jobs: + + ## Stage 1: Build artifacts and run tests for cloudberry + + test-cloudberry: + name: Build and Test Yezzey Cloudberry + runs-on: ubuntu-latest + container: + image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest + options: >- + --user root + -h cdw + -v /usr/share:/host_usr_share + -v /usr/local:/host_usr_local + -v /opt:/host_opt + + services: + # Define the MinIO service container + minio: + image: lazybit/minio # Use a specific MinIO image tag + ports: + - 9000:9000 # Expose MinIO's API port (9000) + - 9001:9001 # Expose MinIO's console port (optional, for web UI) + env: + # MinIO root credentials (required for admin access) + MINIO_ROOT_USER: some_key + MINIO_ROOT_PASSWORD: some_key + # Healthcheck to ensure MinIO is ready before the job proceeds + options: >- + --name minio + --health-cmd "curl --fail http://localhost:9000/minio/health/live" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + volumes: + - ${{ github.workspace }}/data:/data + + steps: + - name: Checkout Cloudberry source + uses: actions/checkout@v4 + with: + path: cloudberry + submodules: true + + - name: Checkout Yproxy source + uses: actions/checkout@v4 + with: + repository: open-gpdb/yproxy + ref: master + path: yproxy + + - name: Cloudberry Environment Initialization + shell: bash + env: + LOGS_DIR: build-logs + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + + mkdir -p "${SRC_DIR}/build-logs" + chown -R gpadmin:gpadmin "${SRC_DIR}/build-logs" + mkdir -p "${LOGS_DIR}/details" + chown -R gpadmin:gpadmin . + chmod -R 755 . + chmod 777 "${LOGS_DIR}" + + df -kh / + rm -rf /__t/* + df -kh / + + df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" + free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" + + { + echo "=== Environment Information ===" + uname -a + df -h + free -h + env + } | tee -a "${LOGS_DIR}/details/environment.log" + + echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" + + - name: Install MinIO Client (mc) + run: | + set -ex pipefail + # Download mc for Linux (amd64) + curl -O https://dl.min.io/client/mc/release/linux-amd64/mc + chmod +x mc + sudo mv mc /usr/local/bin/mc # Make mc available system-wide + + - name: Configure MinIO service + run: | + set -ex pipefail + # Add the MinIO service as an "alias" in mc (name it "minio-ci") + mc alias set minio-ci http://minio:9000 some_key some_key + + # Verify the connection + mc admin info minio-ci + + # Create buckets + mc mb minio-ci/gpyezzey + mc mb minio-ci/gpyezzey2 + mc mb minio-ci/gpyezzey3 + + - name: Run Apache Cloudberry configure script + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + echo "::error::Configure script failed" + exit 1 + fi + + - name: Run Apache Cloudberry build script + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then + echo "::error::Build script failed" + exit 1 + fi + + - name: Run Yezzey build script + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + + if ! time su - gpadmin -c "cd ${SRC_DIR}/gpcontrib/yezzey && make && make install"; then + echo "::error::Build yezzey failed" + exit 1 + fi + + - name: Deploy yezzey config + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + + chmod +x "${SRC_DIR}"/gpcontrib/yezzey/devops/scripts/prepare_test_yezzey.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && gpcontrib/yezzey/devops/scripts/prepare_test_yezzey.sh"; then + echo "::error::Config yezzey failed" + exit 1 + fi + + - name: Install yproxy + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/yproxy + run: | + set -eo pipefail + + # Install latest Go compiler + sudo apt update + sudo apt install -y software-properties-common + sudo add-apt-repository ppa:longsleep/golang-backports + sudo apt update + sudo apt install -y golang-go + + # Install lib dependencies + + sudo apt install -y libbrotli-dev liblzo2-dev libsodium-dev curl cmake + + # Fetch project and build + git config --global --add safe.directory ${SRC_DIR} + cd ${SRC_DIR} + make build + + mv devbin/yproxy /usr/bin/yproxy + + #Check the installation + yproxy --version + + - name: Create demo cluster with yezzey + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + + if ! time su - gpadmin -c "cd ${SRC_DIR} && gpcontrib/yezzey/devops/scripts/create_demo_yezzey_cloudberry.sh"; then + echo "::error::Create cluster with yezzey failed" + exit 1 + fi + + - name: Run tests + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + set -x + + chmod +x "${SRC_DIR}"/gpcontrib/yezzey/devops/scripts/launch_yproxy.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && gpcontrib/yezzey/devops/scripts/launch_yproxy.sh && cd ${SRC_DIR}/gpcontrib/yezzey && source /usr/local/cloudberry-db/cloudberry-env.sh && source ../../gpAux/gpdemo/gpdemo-env.sh && IS_CLOUDBERRY=true make installcheck"; then + echo "::error::Test yezzey failed" + cat ${SRC_DIR}/gpcontrib/yezzey/regression.diffs + exit 1 + fi + + - name: Upload test logs + uses: actions/upload-artifact@v4 + with: + name: test-logs-cloudberry-${{ needs.build.outputs.build_timestamp }} + path: | + build-logs/ + retention-days: 7 + + - name: Upload test results files + uses: actions/upload-artifact@v4 + with: + name: results-cloudberry-${{ needs.build.outputs.build_timestamp }} + path: | + **/regression.out + **/regression.diffs + **/results/ + retention-days: 7 + + - name: Upload test regression logs + if: failure() || cancelled() + uses: actions/upload-artifact@v4 + with: + name: regression-logs-cloudberry-${{ needs.build.outputs.build_timestamp }} + path: | + **/regression.out + **/regression.diffs + **/results/ + **/yproxy.log + cloudberry/gpAux/gpdemo/datadirs/standby/log/ + cloudberry/gpAux/gpdemo/datadirs/qddir/demoDataDir-1/log/ + cloudberry/gpAux/gpdemo/datadirs/dbfast1/demoDataDir0/log/ + cloudberry/gpAux/gpdemo/datadirs/dbfast2/demoDataDir1/log/ + cloudberry/gpAux/gpdemo/datadirs/dbfast3/demoDataDir2/log/ + cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror1/demoDataDir0/log/ + cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror2/demoDataDir1/log/ + cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror3/demoDataDir2/log/ + retention-days: 7 + + ## ====================================================================== + ## Job: report + ## ====================================================================== + + report: + name: Generate Apache Cloudberry Build Report + needs: [test-cloudberry] + if: always() + runs-on: ubuntu-22.04 + steps: + - name: Generate Final Report + run: | + { + echo "# Yezzey Test Pipeline Report" + + echo "## Job Status" + echo "- Cloudberry Job: ${{ needs.test-cloudberry.result }}" + echo "- Completion Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + + if [[ "${{ needs.test-cloudberry.result }}" == "success" ]]; then + echo "✅ Pipeline completed successfully" + else + echo "⚠️ Pipeline completed with failures" + + if [[ "${{ needs.test-cloudberry.result }}" != "success" ]]; then + echo "### Cloudberry Test Failure" + echo "Check build logs for details" + fi + + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Notify on failure + if: | + (needs.test-cloudberry.result != 'success') + run: | + echo "::error::Build/Test pipeline failed! Check job summaries and logs for details" + echo "Timestamp: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + echo "Cloudberry Result: ${{ needs.test-cloudberry.result }}" + + diff --git a/.github/workflows/yezzey-test.yml b/.github/workflows/yezzey-test.yml deleted file mode 100644 index 4922f6eca84..00000000000 --- a/.github/workflows/yezzey-test.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Yezzey testing - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -jobs: - - build_and_run_yezzey: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Test Yezzey - run: docker compose -f docker/yezzey/docker-compose.yaml run --build --remove-orphans yezzey - - diff --git a/configure b/configure index febd1b30169..6c94bce46bd 100755 --- a/configure +++ b/configure @@ -722,6 +722,7 @@ with_apr_config with_libcurl with_rt with_zstd +with_yezzey with_libbz2 LZ4_LIBS LZ4_CFLAGS @@ -943,6 +944,9 @@ with_zlib with_lz4 with_libbz2 with_zstd +with_diskquota +with_gp_stats_collector +with_yezzey with_rt with_libcurl with_apr_config @@ -11250,6 +11254,56 @@ $as_echo "yes" >&6; } fi fi +# +# gp_stats_collector +# + + + +# Check whether --with-gp-stats-collector was given. +if test "${with_gp_stats_collector+set}" = set; then : + withval=$with_gp_stats_collector; + case $withval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-gp-stats-collector option" "$LINENO" 5 + ;; + esac + +else + with_gp_stats_collector=no + +fi + +# +# yezzey +# + +# Check whether --with-yezzey was given. +if test "${with_yezzey+set}" = set; then : + withval=$with_yezzey; + case $withval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-yezzey option" "$LINENO" 5 + ;; + esac + +else + with_yezzey=no + +fi + # # Realtime library # diff --git a/configure.ac b/configure.ac index 6f6ba21bbd3..89876e69d4f 100644 --- a/configure.ac +++ b/configure.ac @@ -1373,6 +1373,13 @@ if test "$with_zstd" = yes; then PKG_CHECK_MODULES([ZSTD], [libzstd >= 1.4.0]) fi +# +# yezzey +# +PGAC_ARG_BOOL(with, yezzey, no, + [build with Yezzey extension]) +AC_SUBST(with_yezzey) + # # Realtime library # diff --git a/docker/yezzey/.dockerignore b/docker/yezzey/.dockerignore deleted file mode 100644 index 4e92238e609..00000000000 --- a/docker/yezzey/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -Dockerfile -docker-compose.yaml \ No newline at end of file diff --git a/docker/yezzey/Dockerfile b/docker/yezzey/Dockerfile deleted file mode 100644 index 0b84f13d7bc..00000000000 --- a/docker/yezzey/Dockerfile +++ /dev/null @@ -1,131 +0,0 @@ -FROM ubuntu:focal - -ARG accessKeyId -ARG secretAccessKey -ARG bucketName -ARG s3endpoint -ARG yezzeyRef - -ENV YEZZEY_REF=${yezzeyRef:-v1.8_opengpdb} - -ENV AWS_ACCESS_KEY_ID=${accessKeyId} -ENV AWS_SECRET_ACCESS_KEY=${secretAccessKey} -ENV S3_BUCKET=${bucketName} -ENV WALG_S3_PREFIX=s3://${bucketName}/yezzey-test-files -ENV S3_ENDPOINT=${s3endpoint} - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] -ENV DEBIAN_FRONTEND=noninteractive - -RUN useradd -rm -d /home/gpadmin -s /bin/bash -g root -G sudo -u 1001 gpadmin - -RUN ln -snf /usr/share/zoneinfo/Europe/London /etc/localtime && echo Europe/London > /etc/timezone \ -&& apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install -y --no-install-recommends --allow-unauthenticated \ - build-essential libssl-dev gnupg devscripts \ - openssl libssl-dev debhelper debootstrap \ - make equivs bison ca-certificates-java ca-certificates \ - cmake curl cgroup-tools flex gcc-8 g++-8 g++-8-multilib \ - git krb5-multidev libapr1-dev libbz2-dev libcurl4-gnutls-dev \ - libevent-dev libkrb5-dev libldap2-dev libperl-dev libreadline6-dev \ - libssl-dev libxml2-dev libyaml-dev libzstd-dev libaprutil1-dev \ - libpam0g-dev libpam0g libcgroup1 libyaml-0-2 libldap-2.4-2 libssl1.1 \ - ninja-build python-dev python-setuptools quilt unzip wget zlib1g-dev libuv1-dev \ - libgpgme-dev libgpgme11 sudo iproute2 less software-properties-common \ - openssh-client openssh-server - -COPY yezzey_test/install_yproxy.sh /home/gpadmin - -RUN ["/home/gpadmin/install_yproxy.sh"] - -RUN apt-get install -y locales \ -&& locale-gen "en_US.UTF-8" \ -&& update-locale LC_ALL="en_US.UTF-8" - -RUN echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers - -USER gpadmin -WORKDIR /home/gpadmin - -COPY yezzey_test/import_gpg_keys.sh /home/gpadmin/ -COPY yezzey_test/priv.gpg /home/gpadmin/yezzey_test/priv.gpg -COPY yezzey_test/pub.gpg /home/gpadmin/yezzey_test/pub.gpg - -RUN ["/home/gpadmin/import_gpg_keys.sh"] - -COPY yezzey_test/generate_ssh_key.sh /home/gpadmin/ - -RUN ["/home/gpadmin/generate_ssh_key.sh"] - - -RUN cd /tmp/ \ -&& git clone https://github.com/boundary/sigar.git \ -&& cd ./sigar/ \ -&& mkdir build && cd build && cmake .. && make \ -&& sudo make install - -COPY . /home/gpadmin - -RUN sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ - bison \ - ccache \ - cmake \ - curl \ - flex \ - git-core \ - gcc \ - g++ \ - inetutils-ping \ - krb5-kdc \ - krb5-admin-server \ - libapr1-dev \ - libbz2-dev \ - libcurl4-gnutls-dev \ - libevent-dev \ - libkrb5-dev \ - libpam-dev \ - libperl-dev \ - libreadline-dev \ - libssl-dev \ - libxml2-dev \ - libyaml-dev \ - libzstd-dev \ - locales \ - net-tools \ - ninja-build \ - openssh-client \ - openssh-server \ - openssl \ - python3-dev \ - python3-pip \ - python3-psutil \ - python3-pygresql \ - python-yaml \ - zlib1g-dev \ - rsync \ -&& sudo apt install -y libhyperic-sigar-java libaprutil1-dev libuv1-dev - -RUN sudo mkdir /usr/local/gpdb \ -&& sudo chown gpadmin:root /usr/local/gpdb - -RUN sudo chown -R gpadmin:root /home/gpadmin \ -&& git status - -RUN git submodule update --init -RUN rm -fr gpcontrib/yezzey - -# Fetch latest yezzey version -RUN git clone https://github.com/open-gpdb/yezzey.git gpcontrib/yezzey && cd gpcontrib/yezzey && git fetch origin $YEZZEY_REF:test_branch && git checkout test_branch && cd /home/gpadmin -RUN sed -i '/^trusted/d' gpcontrib/yezzey/yezzey.control -RUN ./configure --with-perl --with-python --with-libxml --disable-orca --prefix=/usr/local/gpdb \ ---enable-depend --enable-cassert --enable-debug --without-mdblocales --without-zstd CFLAGS='-fno-omit-frame-pointer -Wno-implicit-fallthrough -O3 -pthread' -RUN make -j8 && make -j8 install && make -C gpcontrib/yezzey -j8 install - - -RUN echo ${s3endpoint} - -RUN sed -i "s/\$AWS_ACCESS_KEY_ID/${accessKeyId}/g" yezzey_test/yproxy.conf \ -&& sed -i "s/\$AWS_SECRET_ACCESS_KEY/${secretAccessKey}/g" yezzey_test/yproxy.conf \ -&& sed -i "s/\$AWS_ENDPOINT/${s3endpoint}/g" yezzey_test/yproxy.conf \ -&& sed -i "s/\$WALG_S3_PREFIX/${bucketName}\/yezzey-test-files/g" yezzey_test/yproxy.conf && cp yezzey_test/yproxy.conf /tmp/yproxy.yaml - -ENTRYPOINT ["./yezzey_test/run_tests.sh"] diff --git a/docker/yezzey/docker-compose.yaml b/docker/yezzey/docker-compose.yaml deleted file mode 100644 index 2562ebeaa0b..00000000000 --- a/docker/yezzey/docker-compose.yaml +++ /dev/null @@ -1,45 +0,0 @@ -services: - minio: - image: quay.io/minio/minio - command: server --console-address ":9001" /data - expose: - - "9000" - - "9001" - environment: - MINIO_ROOT_USER: some_key - MINIO_ROOT_PASSWORD: some_key - healthcheck: - test: ["CMD", "mc", "ready", "local"] - interval: 5s - timeout: 5s - retries: 5 - hostname: minio - - setup-minio: - image: quay.io/minio/mc - depends_on: - minio: - condition: service_healthy - entrypoint: | - /bin/sh -c " - /usr/bin/mc alias set myminio http://minio:9000 some_key some_key - /usr/bin/mc mb myminio/gpyezzey - /usr/bin/mc mb myminio/gpyezzey2 - /usr/bin/mc mb myminio/gpyezzey3 - " - - yezzey: - image: yezzey - build: - context: ../.. - dockerfile: docker/yezzey/Dockerfile - args: - accessKeyId: some_key - secretAccessKey: some_key - bucketName: gpyezzey - s3endpoint: "http:\\/\\/minio:9000" - depends_on: - minio: - condition: service_healthy - setup-minio: - condition: service_completed_successfully diff --git a/gpcontrib/Makefile b/gpcontrib/Makefile index 8d95a14f876..af9862530d6 100644 --- a/gpcontrib/Makefile +++ b/gpcontrib/Makefile @@ -35,6 +35,16 @@ else diskquota endif +ifeq "$(with_diskquota)" "yes" + recurse_targets += diskquota +endif + +ifeq "$(with_gp_stats_collector)" "yes" + recurse_targets += gp_stats_collector +endif +ifeq "$(with_yezzey)" "yes" + recurse_targets += yezzey +endif ifeq "$(with_zstd)" "yes" recurse_targets += zstd endif diff --git a/gpcontrib/yezzey b/gpcontrib/yezzey index ba7467211db..4c6b5b83735 160000 --- a/gpcontrib/yezzey +++ b/gpcontrib/yezzey @@ -1 +1 @@ -Subproject commit ba7467211db94f0017e0028a6541fc201f02634e +Subproject commit 4c6b5b83735320dda01e042631a851336300a3ca diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 77b58e7aa76..457a4a0944e 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -271,6 +271,9 @@ with_zstd = @with_zstd@ ZSTD_CFLAGS = @ZSTD_CFLAGS@ ZSTD_LIBS = @ZSTD_LIBS@ EVENT_LIBS = @EVENT_LIBS@ +with_diskquota = @with_diskquota@ +with_gp_stats_collector = @with_gp_stats_collector@ +with_yezzey = @with_yezzey@ ########################################################################## # diff --git a/yezzey_test/.minio/certs/CAs/ca.cert.pem b/yezzey_test/.minio/certs/CAs/ca.cert.pem deleted file mode 100755 index e9f2f1d73ee..00000000000 --- a/yezzey_test/.minio/certs/CAs/ca.cert.pem +++ /dev/null @@ -1,29 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIE/TCCAuWgAwIBAgIUU9e6chP84r3iZk3JtvnWb1V2N1YwDQYJKoZIhvcNAQEL -BQAwDTELMAkGA1UEBhMCUlUwIBcNMjMwMzEwMDgzNTUzWhgPMzAyMjA3MTEwODM1 -NTNaMA0xCzAJBgNVBAYTAlJVMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC -AgEAwJuy394cK127yT8nGHVPKF6TG6xL0WpxahyaKwIYp5lbv9wDvzjMPE7KmONU -8GhCFUdEJTRqBkaRdZNYxnOUxufU3+jIf1hq1Csg8q1NXICVWVwfFL2F5mKHgeHQ -n3FaJM2pZQ5iIWFY1c18MgV8qqNWbtyLeppcyZOL9duLM9A8XpYb0JOZis82d+lh -kcxzE1XM+MZEgZfHImh0zod9OMtSAOwQzVXpiA3JO/eHkLQGYcy6KNTm42mubVlX -kBcu/BplnP7gXGOYDt/JyRhGSLAfn762+jRbAlAvbPzOy67hc4pW7aloU5zPBhYf -BaTxM9UPqPtyp7Lxkp9HL68QXtm5MobDuDtZ6ePQtHgHrl7P7PXvEUPwK7BZzgZy -MerVhxIssutA2yBCuu5T7dMSwIsUdvXtgdHRdHDwn1D/V1CxnujDv9l6/T3sCmRv -tWPwTOCUf5BLLw6N6TnSsVR5I9NALKCLYE8LsfCuLdyi363JZqubkdJr1Ro8yI5J -m0GX5pypwZJPV2Ivt6kKVTQiN2hoWNe+3TNPS+7ysqit37s71YRDajZaZ55DopmF -+oIYdA3MqUZEVZyKFifWvo/l2gYarlEtcEJl++OwydirWLAjCPHh9UvDhjKS43bQ -zSlRC+d4CfRqXftmETHVAxMokai3WvAdUpJrW2RrjiuR0MkCAwEAAaNTMFEwHQYD -VR0OBBYEFJGDr6xmoKJFU6cgS90aFg6lUGbhMB8GA1UdIwQYMBaAFJGDr6xmoKJF -U6cgS90aFg6lUGbhMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB -ACj87ymjBlgY9UZTUbudHREPPXfqMi2TgWt5hygQSiTrNeQOodnq+Swp86qX/y8w -xtnvc+iILfFnh9ZevHKmLx+JziN4kD4ywEpHW7zS7c3+2QEjIZUwj5qlIg0ByOBd -0M/kpimmuTwlDylBaY12GcFlZcsbuezzm4hU+0qoCV/zi2DvSdAPKXMAeZ3lOkde -PUYJUpRz/QkkxEhSdM3BQYI51mUiltCHMhe6COoN4MHV7tix0Pj9vPjhAVN/4sot -2PgUiCwY8eNQugZhpTosMTSBLZvg/EKG+4slY75/voNTIxWHAHmnPMOAzVgNTya0 -/eP6NB3MCjFuY2E+fGox9YTomjI5oxBr+1LlwVy7wbwXTrgBz9Z4izScAsVbPrk6 -jSrqNeNWK1f+JVnYZkjgPGgPaQVCJ22vdLmkW7U/ATdeedQS3RCApMnb9VCRTUaO -eY4ccuEvj0huhdcUguw6fBjrhPjoPxKMn6S93ginW8Wz9vo8qLkEg2NtQDFu1Omb -cJM5F8uLRr8NotPV5QPg1koHeBv/N2WTRZiUoavAogR9XdyOtrB8+MBu1nsp4Goi -7/suv9XzMJ7IpgXiQfCM++1x7oooyWWdeFTCzqNDJ1IbQDeOCc9cQgeOAPWcIqWO -nAWt08+eToI1YUvjl6UT0bpVaJEACv+/HfBr1T26u4Jh ------END CERTIFICATE----- diff --git a/yezzey_test/.minio/certs/CAs/ca.cert.srl b/yezzey_test/.minio/certs/CAs/ca.cert.srl deleted file mode 100755 index 977dab4a3e7..00000000000 --- a/yezzey_test/.minio/certs/CAs/ca.cert.srl +++ /dev/null @@ -1 +0,0 @@ -53DAAE29A25C5BE967A9E9631F0572E17AC92211 diff --git a/yezzey_test/.minio/certs/CAs/ca.key b/yezzey_test/.minio/certs/CAs/ca.key deleted file mode 100755 index a0fc44ee422..00000000000 --- a/yezzey_test/.minio/certs/CAs/ca.key +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJKAIBAAKCAgEAwJuy394cK127yT8nGHVPKF6TG6xL0WpxahyaKwIYp5lbv9wD -vzjMPE7KmONU8GhCFUdEJTRqBkaRdZNYxnOUxufU3+jIf1hq1Csg8q1NXICVWVwf -FL2F5mKHgeHQn3FaJM2pZQ5iIWFY1c18MgV8qqNWbtyLeppcyZOL9duLM9A8XpYb -0JOZis82d+lhkcxzE1XM+MZEgZfHImh0zod9OMtSAOwQzVXpiA3JO/eHkLQGYcy6 -KNTm42mubVlXkBcu/BplnP7gXGOYDt/JyRhGSLAfn762+jRbAlAvbPzOy67hc4pW -7aloU5zPBhYfBaTxM9UPqPtyp7Lxkp9HL68QXtm5MobDuDtZ6ePQtHgHrl7P7PXv -EUPwK7BZzgZyMerVhxIssutA2yBCuu5T7dMSwIsUdvXtgdHRdHDwn1D/V1CxnujD -v9l6/T3sCmRvtWPwTOCUf5BLLw6N6TnSsVR5I9NALKCLYE8LsfCuLdyi363JZqub -kdJr1Ro8yI5Jm0GX5pypwZJPV2Ivt6kKVTQiN2hoWNe+3TNPS+7ysqit37s71YRD -ajZaZ55DopmF+oIYdA3MqUZEVZyKFifWvo/l2gYarlEtcEJl++OwydirWLAjCPHh -9UvDhjKS43bQzSlRC+d4CfRqXftmETHVAxMokai3WvAdUpJrW2RrjiuR0MkCAwEA -AQKCAgAgemC4RTDE00J2FfMWublGWmQ991i1kFhdh0Mr22ei40ZIXOY42W/+/15E -V5kcDMiP4/uGtobmVgHzLIx8skK1I6SOuScN6i/hZQBiS3zPC1OjxNfs3GR2y8iD -yzstl6SWriNRShKcBFlBfCvkF27FK1PIz+GpI9xflUS1iXa4nvV/EZrRGgJ7GKPb -pnvwZORGr2In1O76V0iZ8bk4ljo0WHyUcToIFeOSMJjtRrkSWnj1BtuhRP1F/a0O -/VC5mF8w3Zai2YulqJmccHoLMc+wNBqxCiy6lhd+lVzZ6OtKB0w2+m3cF4PjDX8P -TK2gewa9McE5QmU8B/2aNsd/L+r3eGEvWAF/1vRq6NcrFwigq8uCTtgw9edRlDnm -RvICkfAbrwhNaixWwqBVQHoy53H29TohxGNNKa6TTKeJvYEdYKgHx55TxkB9X9jc -iSisqb3fgEl4Yh1Izpu+6nULOqdlldfkKPgKJqVB1AT/avR8J09zmMvW5fPa6fFx -alZ1iVahR5bIFEu1lXygsrBP6N+K/ogyztg7ZKLTIN/FguwMKnXMaUbN/Y/ZZXV1 -oGil9vHKnDrRnUGfcm9tyH2Ddcy6RDoDz+O4cYgMGxDhHran2cicVY1q+Yi08q5h -Napk1phNra5HIHnNHwMxQ75ZKZZ3TOGJL+HMF4yRDj19C/6sAQKCAQEA8a9ZQhWw -0vhZENmSYZgGZLa7RZLSbBzQOX/cetdI6/kvmZVcMvNz4q0/UI9XLkqokL1wJiku -O0zXkaVrBVAsgozp4I3oFqwtcAAGw0KwF4FDAS36k4gkE4SmIUl2eI0XMZCPQIKp -3TB81+XdBITtwfPl5yG+IZDkXNu16qUHEhnhvs/kKhMr8flhFC1J4gdrrQhfuRHY -Jv8e1RLJzMhu/ErRjh82LkzB6m3jp0YxBeIA+9Kkw+OX6SlzRbJPirKxJTaZnB8o -wQmzOy1kTRG4qjKswjdTbzf6549721i8QHwSpwPI3NZQhlSkfsvZ5QL4qPW0nRta -m76YeLlS12yQSQKCAQEAzAQz6OcE6yS2q5UfTZluGaU54Zkm0YSnS394pitJpHoh -JSZlvkL1DzpacquDxa3uQLDikai5TqpNnkuufeMJf7I2ygg4n/v4OFaE+/qj5uNA -3QnL3BVT9DCJ0JvQ1qA5Q/6P5WpUHYB7JHBM9BpaE8e4xocJyWSdcSJDaEXns4Hx -WzhpBdVpPSamqB0VHYg1bv6OGFPfwUaRafWhNzljtxbY8RYcz7IfPmnLImFePTtZ -AjzIoAwUIRFzvmoduda0kQKogRVoEeaW1q6ebPUjYjIZvohnpe27EvgCiTNkcaSf -C96uIxHrSvI8114z9CBXer60xQ0Kz+ds18LtY6w8gQKCAQEAkP/JxlsrHje/f9t4 -9jJ2S4BSNLiUpCZZStYKWmzFJEX5J+SzTyI+uZWFcfi9rlk+brApE8wLH6rHfmtH -HQXv3ldajc21m7yq+hIZ/JYK/d8gaxnBxzebpVYlMb1YZZUIgEUhnOuHq9vGWuVe -x7JUztNccGIPJyY9y/RJXUCrUFHU3Vzun8umxuL+OlO9iu02zbZDb85j52mSfvVp -uwHZjGX6+ZCCOh71DIfnWFlFWikwu+Sx05C9eDbVINCM5kK1AwWR/Ve4ZLBEJtHh -5lcmen4ypcb5uLVWRA0SmxPOxcVqj2c24D94Sk+H7UayMLKqqvvW45cgsmYUJgHR -0MsieQKCAQB9goBk4erWtmliuYTeemuPf2RSc6O79b3t5mfU4oCVnUTS1AJ3wD1+ -tsl6DiYs8MnIJoncTk5iJMdHgQvCCnCHjJ3EQLaFRb/4+NErK5C1tEztLt+pb72M -VmgSXCloQH26ZNslqfpBhA895ZCSA7wyuwXjrKPKsAlj1k5d0dOvTVusYNHLcvUh -V6vjdLDO0EL/G79THBZlkwJWi3Q4wyejNX0VJCNpaw1pmjAL4JbXWLFzfO13+LZR -eakZFbNf5sSDCX2cnAzAJnnZbOet5El2WZgY7VXGcLBMBSOaQHGksD/gT4gVrypv -mwLvA9c2cscejkArkdB7AsalHhho30cBAoIBAFJBO0RU7o0S+F6KHIP5aFbItcUd -NfUgoJTAFUD3EnBirvDv0pu8T8zkgKf7PRFkZQIOXocvpX0Zy6N7fiPbvzTA/vH3 -mFqias89pTUAgv43R8ZsAC/qlozUuByegigEz2zeVd34w7MdkgGo1jnqmijAIXZE -INBo0swkxAbix+W1Pur/yvGUpC6xu3ISmdrn0p20B7QhyuoqC3ea/az7ePwx+Pu9 -Jl8tzMujbHNHhw+OQAQOPHi6EUPs/H37euj3G7oBaVUwXJq3Tbwg95W5Jih+CgTB -Sbe6eYpR/j/SYGwbS6/DbHi3IjvblN+2pSPI05JvXMhLC/lAeqcdVJAgTvw= ------END RSA PRIVATE KEY----- diff --git a/yezzey_test/.minio/certs/private.key b/yezzey_test/.minio/certs/private.key deleted file mode 100755 index 9d12f84dfa7..00000000000 --- a/yezzey_test/.minio/certs/private.key +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJKAIBAAKCAgEA1WMTewnKOrmE6ceR+rDY2+IK57DktPou0KlJ+Cir5RjsCrh/ -mD8rNCJCVQ0ytV8EHUlnfJBcSnZRuKZHDTeaYAmKYe0Wcqey+bzAKG9+W3kNvFm0 -7Q/MIEAU4eGoeMtZpG91ZU9Jml45siJz7pZArnIrZdOwixBiu8oa66b88Dz/mHDK -JTgex9q27mYFERWu51ORAIchCLIQ9xKdMqR99irHKvGtC6TN5o6ANfTXmhVuVm4X -7Q6aeeDdmsOoP/KKmK/0yYvTunyfEhyaoEBw/aQmltfuEilMxrOzGoAY1iJbHuXV -tzYqkEomrmGXa7pStUoq3Ruu5PLVU+zEYqxVurYQTz0cWQZxo8BQdGbdmKiwu/NL -oc6WA2dUHdVKB9cDnRQXfKXWO8TZOKCIJiQLzGsUW8Jq6kstiaE7IQL9LBZbSWxG -ScgkneDE+CucgSjD5PwDMZ5R6RXPVxdh9CHfNIytESa1zb1H/d+U51xDRk6qkQbO -Ah5gt241mlekv1HYarRkwx8v0iiRa6ecmaGmnAA3SYRZYV6yYf1co+0Svm9EEG9f -bZMHLmTExpz6xPwg5EsJ2jMgsJP6My2LKJ5TVoUCV7S+Cz5tXFyJyHmtedSTaeC+ -flt5A+9nYQKuk92mh2t/XaSCRDFugERUFAlEAvag0+06qhbVyt69fMGozzkCAwEA -AQKCAgEAsWK7PvzUcBzosK6GW6/HloJCLniOpyOS50LTisfEnZ4qGn9lElrwv1X7 -bliaXsutz+rFbHdVQVE6fhU723DtlAhaUS2WC5n83j5aP0Lv93qaQIkSLj+DoQuk -UGIWetQQoPFG1gEjXoAV1k9tsFiXTGz8RpnDmNb2PMW1u1AF1G/gygh5Ape0fs8C -YwvMCnfL/eEqGRY8D8526+09YGv9ijXle32MLLHDuHWdfz0aPayzHIZIvXf2Unrr -vUwJAZ/ONz+Obj0etVgDpDrDD5SCWVer/Jlj/xT2Dfg0W0NBYkENHpJRJwyQNYJu -xWe7SIKLXslY+JWavhhf3nRkjOJWIGtgix6Sbb6K3TFNu65t5nOVr9dyZFcW4sSD -JpyUjDK6mbpUKgh3yaU9QXQSY0bD48mTt1UB2vmh/dA9OIRRUXr+V4MKPQBtJt0V -7ay2P95oWSHM+zpzPw2Bv6R6s78kaFisD/IqVwt95NMSiVTb3rNsJLs+KNKjuoV0 -EPeZuiQvDZDNi2+pKC+3tT65+sECno6ZSZdvn7naRYy3X+QWbOY/zG9mXIMyRXSs -oDXPH17838swDO7YrmWDvHO2Coz72eEftsovDql+D+4w6lq5DZygbHiSO3yyB4cR -AbI1hQ0nneQC8Gi+YPNEguAz5Sdvys91urZ1awllsTpG6gFPuAECggEBAPROYIyi -4mjr3MAUU0h6FHQ8uCARknM2NOZGDpyHmOtFjdTW47xGB3E8Dhyci0iOYjgwhcNe -lx/gZySKuCbC4HbxWUzLFYOOYRpj41oIZW3kvnkc7vVEst726CO+cVtTc8cjYXQL -pFQO3wN9C+S7OXIZOSO/P83jHm3bG9vQ1Kgh7gMBgZkwx3NMt2d1slEo4NViXZ5n -1960wWer5J6lQtpgwJYik7tZZBXkGA8QrhvzqfAGLYUQc3chWSHqlRt/+YJzFXZc -JYhpmCOs3jefC1I0T4wsxTJAhv3xnlGY5FDFIsHXqkCgwsqU9rYNfNQLLhuuwG3t -kR/sZp1eKmkDCjECggEBAN+Z1DqZIgd9rXTeUICbzlQy4891BhCWy3F20CTbzdfv -7EN49uhaD/OltJ8LJcDqm2F38Xjz4+svGfsYkqe0EMV1IOkdhnf29uGcWb/kWGGX -FtNRL3QhKntouVqqsJdeFNcvbPF4RZPUOQTiHjH2U+nTA/KIKU/nxSqJwR3fj9nc -v05k2jv+eoodDx4Fs/zS/cYzA0bjEXZlO6fS+MSWJwQGVbd3lqieo0FVuD3Y2RVs -nQidKUOm/qTE1/r//ggr0nX/GD6n2gRyUTV5yHIoZ/ENCOuxu38qyOk5ko8knPeo -IGqluaaTCyFav72vS9IbWVUKicdmzaLaYVLG1EzMS4kCggEAbHhoMckYUZF3f+kG -WUWq0zkqX0KuDW1h62PrlOA3qy5EnN2UW8GUCFirw1RWGy7suRoCKg5TdxnBcd4N -iVg5JVZfWdNJiBGtV3RGO3FC55oKX+fSyR9pc8mYpFYoKm5RF3fECywoGBJKPlhE -8ZeXF+vPDOobQCS5G/mO0vhTkxsNfFQs4IFh7PWA8PS26YYG3XdBLdXZaM5EmO0f -Irkph6AxofdUNHiqEXYmoHm9dDepmiKED3KQcbJiysZG0eDuPVcvjk+3cmu/2wcj -2vpIooULVKeEHp270gB3VK9Xn+0HU52Xv/4gvqWJLKesZtFX9X6Pfb9fEhd6MCDh -H6Lz4QKCAQBVTk/cp02HRBhoDOTzm/2ku+GT5VaR/6XQPP7+AOnQZ1VhDIZOcQXC -88YsI0mdBySOk/8JISskL80esepJlvYLDzumFECYBh18R3UqM4jQep8XsdKD4J9f -g72rbJGAvkD/M7XBjhwlYQL77PSOJScwfvPzlKUGOitLplCKAB/Wg1RtKSblWpoP -lTSORKi9XRW6hv3KDpk77TVMOr3z6kEqVOxg2XweCp/ILlbjKZbwiIaEocj7aXuf -CwZHWWDmZymdINx0Ev+nUKHQ4AxsGSRGn612llEtG/qYmAFlIRfNmFP9vApPFaxa -Zk/eo0EvAIYdXq7f50Wuytf0h6y5O/vZAoIBADrF4jHV+kbwWso1Rj4SA9xjmDSS -8v99Cr8YReoEwJoRo7sLa4XssY+deIvvR8OUwHmbPWjzDXeJFSMvJd9H7hytPvTW -wEPXd4eTRwcVo4d2ZUxszmJyLp5Mx/0qMtI2XkYPRfulPhXReYjkezjJ1mIzQiwL -RS4vwafBs1LvmbAM9stJt1K/XoI0e4kTYd9KyruKRW+JeuCrxcSV+O4nMRyOiCIQ -cHFXSlSHDyQo4z46rp2+IhIt9/Vq5DQ8hhbAtIjqa/ndFUj8FaQs6sOQq4410+LK -OCT7UGhKdY9I6fV/O4MAiLzyXcXNo6SRZ0YmPfam2v3gCqdDHLneLsbMoJ8= ------END RSA PRIVATE KEY----- diff --git a/yezzey_test/.minio/certs/public.crt b/yezzey_test/.minio/certs/public.crt deleted file mode 100755 index 4da1db77d25..00000000000 --- a/yezzey_test/.minio/certs/public.crt +++ /dev/null @@ -1,29 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIE6jCCAtICFFParimiXFvpZ6npYx8FcuF6ySIRMA0GCSqGSIb3DQEBCwUAMA0x -CzAJBgNVBAYTAlJVMB4XDTIzMDgxMTA5MzA1MFoXDTI0MDgxMDA5MzA1MFowVjEL -MAkGA1UEBhMCUlUxDTALBgNVBAgMBFRlc3QxDTALBgNVBAcMBFRlc3QxDTALBgNV -BAoMBFRlc3QxDTALBgNVBAsMBFRlc3QxCzAJBgNVBAMMAnMzMIICIjANBgkqhkiG -9w0BAQEFAAOCAg8AMIICCgKCAgEA1WMTewnKOrmE6ceR+rDY2+IK57DktPou0KlJ -+Cir5RjsCrh/mD8rNCJCVQ0ytV8EHUlnfJBcSnZRuKZHDTeaYAmKYe0Wcqey+bzA -KG9+W3kNvFm07Q/MIEAU4eGoeMtZpG91ZU9Jml45siJz7pZArnIrZdOwixBiu8oa -66b88Dz/mHDKJTgex9q27mYFERWu51ORAIchCLIQ9xKdMqR99irHKvGtC6TN5o6A -NfTXmhVuVm4X7Q6aeeDdmsOoP/KKmK/0yYvTunyfEhyaoEBw/aQmltfuEilMxrOz -GoAY1iJbHuXVtzYqkEomrmGXa7pStUoq3Ruu5PLVU+zEYqxVurYQTz0cWQZxo8BQ -dGbdmKiwu/NLoc6WA2dUHdVKB9cDnRQXfKXWO8TZOKCIJiQLzGsUW8Jq6kstiaE7 -IQL9LBZbSWxGScgkneDE+CucgSjD5PwDMZ5R6RXPVxdh9CHfNIytESa1zb1H/d+U -51xDRk6qkQbOAh5gt241mlekv1HYarRkwx8v0iiRa6ecmaGmnAA3SYRZYV6yYf1c -o+0Svm9EEG9fbZMHLmTExpz6xPwg5EsJ2jMgsJP6My2LKJ5TVoUCV7S+Cz5tXFyJ -yHmtedSTaeC+flt5A+9nYQKuk92mh2t/XaSCRDFugERUFAlEAvag0+06qhbVyt69 -fMGozzkCAwEAATANBgkqhkiG9w0BAQsFAAOCAgEAipaTRZxvDImsu/pwxSHEkKFz -ndKt84oEeuWRBo+sT5wZAcUJicz0tHVULwhXiqwMRC06LBqFxGzI/srokK2QiN5B -eikQvWEsr08tZuVm1ewNit6JzlwbaLOJc4DSTgJtWXKjWaIzqWsFl36ViLO55yTb -NgMgmpDhBl6hQ7yKtaSVJ+xiMKSd9nz6gRmJMz7sLchFmy3fTYtayUJcaDsFjEm6 -yTYM1oWj05xVcZJtASB8Bcp+XgeOdNbwvvTGrM2ctDC1GkPCK4X63GJfAuCv6lnV -ggzr2Z8dg/YVLCbmyvJq4n+rwNbVDebQBzRzyU9id0dRccyfPQzB69LPYcrMFzd7 -h97i5Vk7Ar66VeKxdw3Lbyl91yTUy7e7EdYkBrlPzMohRcuCEsFPaUdOoiy3dmUX -PeaubgtJpaNlVbp08rdGAgcrY4aqmvVM+tlKWfgiEnVmQ4vKTcrITnhintvttXyh -GrddsvKAE0m1rDpfd9BqXH4FEydFHoL7oMpKHnu9LUQsPHnwGvpq75KUN4j2nP4P -NIEmmqOr5SY9zcp9HQApxWPscQckb9aCIDZ8MMrceNSxRtU+bw0xNs+IILGx6dNA -cK+bDLPtMneno6XXDqqiitE2ohXf5WIeB3kPLOsp3awVhr/g+pqjMSSjyUEC+KXr -dkqHW3oKT52eCNzR350= ------END CERTIFICATE----- diff --git a/yezzey_test/.minio/certs/server.csr b/yezzey_test/.minio/certs/server.csr deleted file mode 100755 index c223c34efb9..00000000000 --- a/yezzey_test/.minio/certs/server.csr +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIIEmzCCAoMCAQAwVjELMAkGA1UEBhMCUlUxDTALBgNVBAgMBFRlc3QxDTALBgNV -BAcMBFRlc3QxDTALBgNVBAoMBFRlc3QxDTALBgNVBAsMBFRlc3QxCzAJBgNVBAMM -AnMzMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1WMTewnKOrmE6ceR -+rDY2+IK57DktPou0KlJ+Cir5RjsCrh/mD8rNCJCVQ0ytV8EHUlnfJBcSnZRuKZH -DTeaYAmKYe0Wcqey+bzAKG9+W3kNvFm07Q/MIEAU4eGoeMtZpG91ZU9Jml45siJz -7pZArnIrZdOwixBiu8oa66b88Dz/mHDKJTgex9q27mYFERWu51ORAIchCLIQ9xKd -MqR99irHKvGtC6TN5o6ANfTXmhVuVm4X7Q6aeeDdmsOoP/KKmK/0yYvTunyfEhya -oEBw/aQmltfuEilMxrOzGoAY1iJbHuXVtzYqkEomrmGXa7pStUoq3Ruu5PLVU+zE -YqxVurYQTz0cWQZxo8BQdGbdmKiwu/NLoc6WA2dUHdVKB9cDnRQXfKXWO8TZOKCI -JiQLzGsUW8Jq6kstiaE7IQL9LBZbSWxGScgkneDE+CucgSjD5PwDMZ5R6RXPVxdh -9CHfNIytESa1zb1H/d+U51xDRk6qkQbOAh5gt241mlekv1HYarRkwx8v0iiRa6ec -maGmnAA3SYRZYV6yYf1co+0Svm9EEG9fbZMHLmTExpz6xPwg5EsJ2jMgsJP6My2L -KJ5TVoUCV7S+Cz5tXFyJyHmtedSTaeC+flt5A+9nYQKuk92mh2t/XaSCRDFugERU -FAlEAvag0+06qhbVyt69fMGozzkCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQCw -gv4AQ9xh+LP16NQ0QCZ7QM8y6cA7jb7yqc8G+gvTjbacsuxpfI7qiJHx/Aon/ReL -D3Lra+HKzQticJaqFQEiXE6kXWB4DndP7hY7MF0R3xlBvNWyS8Q6Nr3m6IQiJx1Q -qd2Fgc7341wOMf3XMW/E1XUEsnzL8pWrKI+lvHAMI3u2KvsqwTtWtFJ4HMQoab7D -Xi4QYVk/DMF/lirPgcm1xnDZP07f7lzIGasO42XOBW1nV33w3bqVjRfKLZnbCs0Q -IOecmfseUBER9ycTUnFHN99BGx1SmTtXmqoeQIwijIvMoBNtEsOskqhW1+snFlNO -st9pyjwhuASvSWDFbEsaC1tl+5oTK31XMrxs99TDRoPMY1UwVFbrXrc+XWP1MSwg -0SB2c2DnLwGT9Lp9w1+epn9oa95B5JskwRp86Lbf5y+XV7W3Vp/vCzQ+db2sUHuT -bEUpNJl/KcxxseMMkpIJy83w17RhlRsgMKvAQtftYWX/z/RHyGKWHHPPoxEJaKyc -hm+X65QJd8QoOd/IXnkCMTQZocpQxoKBbYqdTBDxYfXnTILWHRUb0OfmjS9fEmBE -UqYdwNawOIXKlPsI/JseO132C3TtxjmkqMSNixTFyRvpoqy+/wFul3QA+yF2eCM0 -pfKcr8QsHn8WEVqjEmyQiM1ixrDIgIDc1jurvLX2cA== ------END CERTIFICATE REQUEST----- diff --git a/yezzey_test/generate_ssh_key.sh b/yezzey_test/generate_ssh_key.sh deleted file mode 100755 index 1df4cf14b60..00000000000 --- a/yezzey_test/generate_ssh_key.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -ex - -ssh-keygen -f ~/.ssh/id_rsa -N '' -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys -chmod 600 ~/.ssh/authorized_keys diff --git a/yezzey_test/import_gpg_keys.sh b/yezzey_test/import_gpg_keys.sh deleted file mode 100755 index fc351a3b416..00000000000 --- a/yezzey_test/import_gpg_keys.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -ex - -gpg --import yezzey_test/pub.gpg -gpg --import yezzey_test/priv.gpg diff --git a/yezzey_test/install-wal-g.sh b/yezzey_test/install-wal-g.sh deleted file mode 100755 index bfad4e82a58..00000000000 --- a/yezzey_test/install-wal-g.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -set -ex - -# Install latest Go compiler -sudo add-apt-repository ppa:longsleep/golang-backports -sudo apt update -sudo apt install -y golang-go - -# Install lib dependencies -sudo apt install -y libbrotli-dev liblzo2-dev libsodium-dev curl cmake - -# Fetch project and build -git clone https://github.com/wal-g/wal-g.git -cd wal-g -make deps -make gp_build -mv main/gp/wal-g /usr/bin/wal-g - -#Check the installation -wal-g --version diff --git a/yezzey_test/install_yproxy.sh b/yezzey_test/install_yproxy.sh deleted file mode 100755 index 9e3abd1d9cd..00000000000 --- a/yezzey_test/install_yproxy.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -set -ex - -# Install latest Go compiler -sudo add-apt-repository ppa:longsleep/golang-backports -sudo apt update -sudo apt install -y golang-go - -# Install lib dependencies -sudo apt install -y libbrotli-dev liblzo2-dev libsodium-dev curl cmake - -# Fetch project and build -git clone https://github.com/open-gpdb/yproxy.git -cd yproxy -make build - -mv devbin/yproxy /usr/bin/yproxy - -#Check the installation -yproxy --version diff --git a/yezzey_test/priv.gpg b/yezzey_test/priv.gpg deleted file mode 100644 index 1b9727a319c..00000000000 --- a/yezzey_test/priv.gpg +++ /dev/null @@ -1,105 +0,0 @@ ------BEGIN PGP PRIVATE KEY BLOCK----- - -lQcYBGTbjzEBEADOjtlqSaBe89Pg/orabZALZWi3lRfVqDaBjb/po5jOPlsefBSb -aa7hyAGhMfncmVaqZJXH1ikT2oHx430GcBXD8gjMZbil7U01DK8XBr6iSw/EPWpA -0jD2FdmPCG6vzF58KJ5tv2uPjZUShhYymIwDMZuQAzDMFgGP3dhDQNp4TUXX0AyH -vRGMEeckyjyXhheu1alJ1XNzT9s3dAq3LGFgHuUH4BEwSljpKFP+BARXTGq776wP -JXt7YaJLDB/RXgfMNOKZR18eUYEcckjziMkptBvbD6sG/h9B7ldeYCP91jDqR51Q -ls4eFf6C0dqoJ5ELv8GBW3BZWsnFGffGRA2a6kAk14eSWtcF7w0N5J7o4kPYJSPa -OXq+gIvBz9KeJgQKDu6DrHZYdZh5cCQKFHdLeKzaEsrUPvSOvjfTB8+ywb3QiJPt -/yKYfcZU41zB7dCfbbEFaKTGEvjRUm2XZH9maJvQ5fNUEmtXwQd05nBwN8aW+00d -w5Jy35aSwogCLrJuHMpwI56TRAU81d1R9RaTxNbcZUro8IdOW78BaB8RGrvGT+z1 -+TBPlJOpzJgAYJe6wZp8M4xk0z2y+6ia1X+yUvZXr51U8qiHM1wr1uBZx66EdFv+ -HQf3grEnW3sPCS3kF4Yn6Xm/xIWqbRlXf988RqZc6fKa201DhriEOhk9SQARAQAB -AA/8D1gsTYmHqNYbLNEr9nsCIt6XL2Abg6s51Vj24z/cd6HJOhQEoeDxdWNav6NL -SPh5KDJNsCk4JvnqAlKgdIx6CXLe9XQvCeB9hk3Zae/91GeYFb9sN8GFkppf7ZC2 -TEf10upfjo6USnc9lkEwv/R2Hjyerk2WX+rYldvol2JneF1hLV2OuLyyY4Hm9mQx -qR30f2/u4gXBzFSn19A+hYn999qDsBiUnk4/mZ590tZwjh2mpixxTzS9HhbMLCAL -8w01rhVVqgZTyhhK65aP1cuZCgiPsy/QSDYzkz3F7sRcmBxDyPul85koWbLPk99M -C4bY7xMBKH9m8rRcXdCGiAuBKRlIZFQcyI7Lw4WxqxdJROAjhYb+FXDKSst+cyw8 -c0qP/UbqnNaZmWQ0w95UMlwu8MZ6sJK+CzaLkjt4itkN27/DCfxyfuETqEiTO1ba -yJoVDE5MH0CQLIlByvJAU9AmNd1vLL9CTzm39m8B+eOrPR5n2khu85D+VR3vWZo7 -XnM5RTXhKTMfGFCSunNVlnABEy1eFZYFOogZOkVYdqxRTiZccDIZgV+spN/op4kA -SofCehr1xxdd3DXRar/GOz8J2jrWyqaQzhsD4kKl5nBM3NH7fAPsBBv0VH1DE0OM -ZsiB0yQFyHTWeQOcNla37pD1H1DZa4fpsL6m3BqSK38cKcEIAOUr9faEnjYPTAoW -TKWpoqt2ghyGsklrdYVcgb7pINam1UV0f4uvMBv702ZnLwoC0Rn5ITgPXH0a0MDM -H9yfWZgb6L/mmE+B81D+EDUkb88qafXXq9TMspbwbSuY/DWFTZPVfRDWYacr/IPq -XdtZqNBR2BWDeckgBcnEwg/PBn2HrrpfWab/D9Lt1QFY5baCuFevib7/D1DxO9qq -57iA0IDT+YfydYawwixvarkCNmUfkDowhR7DqHWDppWt5n7rxiTuE7I7C88pV4mV -rVTvXeMh6ooB3m2yW1FhsTgT0DrWxul1glCASQ+TfOks6dcDMvcFZFSzWT1r2rR+ -LqNNpDkIAOa9LcZEaRV86QLLcBYpPnaDAFlZYU/1b6QRbU3L77rULvtHSVegEApp -7a1mDrx1bTp5Z6TGq5ls9bMHF4tg5IcBLDHsO6dYAl2vAbYuCwjfPa3hu/oK+J5F -9IQLrvS2vM2shqhIfIbKglXwAWySLQoR/CsAMgH4vk/R/08AJQRyXndaY3rLImQg -ofUQaRxl4950wO/RJSQ9RD5q1bvseJ6bZNlYu1SoEJ/uOGqlMbMP9DAUb7LnTwlt -+9Mdv/ibYBABnUh5hLLCc5lBgeN+Pd3AAEygvbK6rW9a2CV4vO5U0aBU7Um8WAUz -suVEQM/rMTJanxHAeHgRYMDccadTAZEH/2i/xd20U2Taorj+/Lrq21hQJ1jKpyN+ -CnglPV83pGX1V1j007C2zhO/YB5qTh9ztZvtwkuX5EoUN9mGRZ0t1K3xbmwlDQbB -D1hofRcN0U2h+ZXQ+aID+gswLhw6jWatHiDmhIGSPzopR/1oZrQwS0CJuqb3XQFP -lAgeMPEuFufIf7tijFLL3PDuTPnG5lndI6+uCw+BhkfgN7jAHHREQlolGgzo3mwW -htGr6cOd9W7a9tGN7nHb/yVH22gnXs++wyC8OoeuEgAqQL2CyNOEo3EyN1K79QH0 -7qaKg+XHJy/NymQdcALxp3zUNh3UuOd0CZxPudTVfD5j3cD5fST9Owx+CrQsUm9v -dCBTdXBlcnVzZXIgPHJvb3RAaGFuZGJvb2sud2VzdGFyZXRlLmNvbT6JAk4EEwEK -ADgWIQQBHXUy4S6x61I7K1L/Gb3KATgePQUCZNuPMQIbAwULCQgHAgYVCgkICwIE -FgIDAQIeAQIXgAAKCRD/Gb3KATgePXGWEACRiwzgx5EVSpQEKe79sRAkO9hmAB33 -D4dN4y+D+2kUY/paPg3etCdPIJQs6qFFu+HIWnCTGiUepcsBeUvRpTDwguP6MtXx -TObmaU20ZoSbZZjqnGZ5V9VlwCQjJWj3RN+7BYVsECPJGfEMW+M2nHX3MrGpohfo -KpacoL2kjVswvWocGHNIMHpVHlyGGUtwVHz3kO5HqwcZvWwZj189dKTMUSsfbXdz -723BDp+m/NLUi8Ki/BZpsRwSXqGbpdQ8LYaSJvtS0DY/rIILIOyIfhnHK74rgCip -0vfZsfypc58YrPrj9+pfF7Rx00m0RfGll+lD1Pha4GStS1PvtpdtRTvf6qsmArON -q1N7ha8YJHtPueTKln/7ik3jqir7vO7SLL+CTa/XfS1jaJhonHORfW+fekndXK34 -zrQ75cFn7D5BqqcTXu2hkTDbPqd9aBorWFU/vbqZRVH1a/RRcVU9SODnIwdCm06x -XLoujcr1daZd9GWT0PMvFHaehQ/4zbdd/pYJ9efU1zlfpdwxUpBWyfUEOGbwpHGv -7FjXIHSkHkTNgaOK5AP0FsSGfjAzTxq/2yLRQKm5f3GqWNOZ7nprLg85Tyxh/gdZ -rk8o3/pW7P/XDLU6jfhWV5YR2mxddaFs6P7D+jvRf3EVHbJHtcK30usIWjGDWyNk -310w6Q3CumfEo50HGARk248xARAA6LnhBgnfn0B405FGDbb+MOj/0dmmb81XwL52 -ZqbAIsomMUfrzzFHJb0PtexSsIr7I+NweVCSvc3UaGcMLF8pqOVnP4pkPXi+hBlG -eNeE4KQvMpn56nLsZ90GtpKYLLsMQB6VMm98+v4wCJ6mPkM89SPQlc7q9QFfC31g -trbb5Y//eDqCAATiVIdZ6nEFNKgNDMKC0UmRY+D3RaeElPpWO5lWDDldSjOZ9/Or -dg/TSdRviK7+NzdyOM8xUc6lQunG26qRH/bqQzHluBYMp4W3l71Ojf8YlSATkziB -dAdwpQ1MVx4rCk0o5nYuGf58sgxTIr6qIR4777Z7lanc9utcLeSbltkid+zATW8x -PFPG1ID6pNZ0JX2ttQ7HQAb3RfNDkZxGijqEnJyjY0nJON0/8Jcz46GkeUzkae0r -I/TOS7OASs13NWMQV8TsqDy9VMT+pcTDjy1Rad8krXtraK93WlIPWhb2vCSJkCMr -412bl2SOrtYUdu093yLYkEkrq6vZsl4zkIPrCbuEdJZgSn5d23nHd2ARsaj6D4KZ -RP9dg7A7RpEezNLAh0FDwMLe/nJpNfQdd9MXITALc0CwJ/x0sFBrj1+q7jELYtSp -zh993C4i9ySOaWKfOGV9UjeSLBYfExuxisMQyAIoD8hhNOHnqWVgcYqJ7R3w0ymU -96M29M8AEQEAAQAP/1WZpSiM8ilH1AlxmlRKFjYURaBAz6S44UmeZLt+IxbIxwKC -Yzxq8jHx1/kAyytve09ohULB/a99qV6bZJFfkVmzw2XON++aXW0GRPMGxrPAADI7 -C38ONWFAnYsC4aE2TZu6BAOwmUZSv4U0IY6uOZorSboIiUiD8BswSyX5nWlTLVLi -JlXudfdEb7C5UIJdO6uRUf+78RPNN/ZxVuVbLOOwE0Pcx7EWyM+4Wz1KNdumnT2n -rA7QQJ2frBLckNHLXh8HHmkk72a20DmFNrNZjj1sXpwBE+AqE7knZAozAF5dRVKX -4Jnh5qTaHDvobKIqwVt6yOX0knQp6UwT1hgmWtk8584hLoxuQbzr5yM3/QNiPMka -+aMGu1COS34ppIQfETt1C3Lv7tGcFXkadyV7JfVl0u8qAJjGzKgmrBcVkDBZdKT6 -rP3QAAwpcwdDaY0rXR6FDrHKZWZhvsSNjX9vXf6ThCG4/Wb07bMia1Y6UdtNWYAB -uDQkFJszng03tQP8P1p1J3XouzOb0XA3tkusOoRnMlR47oDb1TlUJKuJv7ztK3AI -5v7FyJO9TJbwhBWWno171KKGn7R1vciNYxWB1IqGamLQFKajTFKYVikn7mCdfDNe -zVgGcK9DAgoJXtclmjtKzUnqfW6ctJKrNKXkyEgkbWKGv33skwDSMZxakTpFCADt -dzG+RbDlZrwlSMhv+33vjV6arulEY32vj3Thbphb4v1JwOTdXNVwF5pC+pf7sUgx -ZAbEuVix4zu+AznCdRGPYjAHZgUe8ldp7JrQuVzlknBJ0Z+DedXcvL8RAs7rFedo -BO9FTqIolTqFZyBJeaDJcMTwf8qQdE71+n/r9oigKi/8UXSAZInnvgYMTUjgegqx -UyQJTEXRQYzUfCUHFpYqel1NwcsLO4JciDwb0ak1QUvIy0HulLrRKrRYUZ7Xy3U9 -XBjt8BPWKdNfN6qKWK+7tYzPNB2BLQT2Z+l0L+Ir9TaWxJTZdfRysYytAhTl3ZoL -99KlA7qIxOs7+DnuR8jdCAD64/v/xNl02ck3pBnnxjnaspe3xEz5D9s/a9Ob+2MU -pDp5b0ZwPJ0tpk6CguiVhi1gzRXzjABPrCbsBOXk2gQ+Pilk3jkugG+wqg4LOLzy -3AiDbD+GVmTL9gdhPNoNIjPBz1tkZEGdTVsUOWT5Kuom3KvbmzowdlTVb4x/kkRM -oOTfO1Uhna3b/pcCsMiHXkM3ZME+DFEy6Imh38742ljllECCDCUAvd4drbx4VaNU -rXyI+j3NP8j/vHhWCld0Bg/C5Xq9ioDOzyrvApTat2OZF2Z9K4nYYTTwniAHJWNk -hIPDKyIKeE4Y/WBrSfVQbdtTNi0fj+3RTtS1j4uSwMObCADf3miT0aX2pBxy7N6D -krhJwq5Usaeu3fVNOOm/HVQsouqJ0BYPX412YIDtL4iY/YibVdzVlNg2hQuPJYGx -hZF/isY+qpckLvdABbtreBDxoqtqBgdOI0EGXpBuJTq9iuh2kiKFWATEchCMND9p -wZK0jrVtQ7JnXwW4CRBnN0wV2LO8+VF6How9FSdVRykG4cRed7JB5U9B11PK3tay -v82+YnmTEr7YuMiJoXHt7+9nA1ftd/Au0IegwvmRO89MxvpGeS8GVZFdYLP5UQiH -bdrnFmLyQEsPPIM3yvFQCfb2W4WmxH3PycTrNDs7mone1L/pk/eMdg7QYF+E/QZD -1N1pfKeJAjYEGAEKACAWIQQBHXUy4S6x61I7K1L/Gb3KATgePQUCZNuPMQIbDAAK -CRD/Gb3KATgePW6sD/sHl+DskI2dz4Ym7ZKng39QHOKFdzoK9xtnLQSynqfrwzvz -UEzsqWNjOHONfzat9savfGBvWB6ULz3vQ4yZ731Y4Tdc4UmVo+79hIgeArNiLyou -p6tvcl+bKOv72xlrYn43lYy51Aj5TsU6TBGVvRETnmOWvzGzsQrhcZ+IA8p4JXoy -fiAM+reuc1lwWYZWS5n+XotskFS3L6BIXLPW1QIHxvcMkZAI8/VeloQej1z4t50i -7mAAXB/RwjXGCLnhFxhjWLHglrxpg6JpsuCkI7j5csYQuzhKrSWa3mxKv7rysRh3 -VpIkHfRRu/kyfOwM4vYCRsQH2lBl/JVgAn1koZKdnGomoq/Gw7+psuYNGX9+CQ5E -ShHGWGE2TUjCi88FDLDpOOmpKMoK19HbNGkeVeKXY80D5LhjSrOL61IY+3Hzyb0a -JhHA7dXAqMHak38Ps5sUrEtgDBjkhGtctCUGV9OsvlqAXwIDLB96W5LKJKw48osE -vcfquO0z3bMKyr0irCyBzn3sPd2nubFy/o1umF8efg2OqWybQ3PZLyMojQ9uu8Yb -SLM3+Q1vX/yAY8+KmaAvvUuTCS1dRWAM0s1g86AUCZ5LbMa0HwIGjCSA2nUw8BD8 -l3sVHJsSKOhB0XTUBVkIDBPdJcj/K6G8E0PGO19UQEMRis0/jnPPL9t8G63xQg== -=uAYd ------END PGP PRIVATE KEY BLOCK----- diff --git a/yezzey_test/pub.gpg b/yezzey_test/pub.gpg deleted file mode 100644 index 139122aea2a..00000000000 --- a/yezzey_test/pub.gpg +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- - -mQINBGTbjzEBEADOjtlqSaBe89Pg/orabZALZWi3lRfVqDaBjb/po5jOPlsefBSb -aa7hyAGhMfncmVaqZJXH1ikT2oHx430GcBXD8gjMZbil7U01DK8XBr6iSw/EPWpA -0jD2FdmPCG6vzF58KJ5tv2uPjZUShhYymIwDMZuQAzDMFgGP3dhDQNp4TUXX0AyH -vRGMEeckyjyXhheu1alJ1XNzT9s3dAq3LGFgHuUH4BEwSljpKFP+BARXTGq776wP -JXt7YaJLDB/RXgfMNOKZR18eUYEcckjziMkptBvbD6sG/h9B7ldeYCP91jDqR51Q -ls4eFf6C0dqoJ5ELv8GBW3BZWsnFGffGRA2a6kAk14eSWtcF7w0N5J7o4kPYJSPa -OXq+gIvBz9KeJgQKDu6DrHZYdZh5cCQKFHdLeKzaEsrUPvSOvjfTB8+ywb3QiJPt -/yKYfcZU41zB7dCfbbEFaKTGEvjRUm2XZH9maJvQ5fNUEmtXwQd05nBwN8aW+00d -w5Jy35aSwogCLrJuHMpwI56TRAU81d1R9RaTxNbcZUro8IdOW78BaB8RGrvGT+z1 -+TBPlJOpzJgAYJe6wZp8M4xk0z2y+6ia1X+yUvZXr51U8qiHM1wr1uBZx66EdFv+ -HQf3grEnW3sPCS3kF4Yn6Xm/xIWqbRlXf988RqZc6fKa201DhriEOhk9SQARAQAB -tCxSb290IFN1cGVydXNlciA8cm9vdEBoYW5kYm9vay53ZXN0YXJldGUuY29tPokC -TgQTAQoAOBYhBAEddTLhLrHrUjsrUv8ZvcoBOB49BQJk248xAhsDBQsJCAcCBhUK -CQgLAgQWAgMBAh4BAheAAAoJEP8ZvcoBOB49cZYQAJGLDODHkRVKlAQp7v2xECQ7 -2GYAHfcPh03jL4P7aRRj+lo+Dd60J08glCzqoUW74chacJMaJR6lywF5S9GlMPCC -4/oy1fFM5uZpTbRmhJtlmOqcZnlX1WXAJCMlaPdE37sFhWwQI8kZ8Qxb4zacdfcy -samiF+gqlpygvaSNWzC9ahwYc0gwelUeXIYZS3BUfPeQ7kerBxm9bBmPXz10pMxR -Kx9td3PvbcEOn6b80tSLwqL8FmmxHBJeoZul1DwthpIm+1LQNj+sggsg7Ih+Gccr -viuAKKnS99mx/Klznxis+uP36l8XtHHTSbRF8aWX6UPU+FrgZK1LU++2l21FO9/q -qyYCs42rU3uFrxgke0+55MqWf/uKTeOqKvu87tIsv4JNr9d9LWNomGicc5F9b596 -Sd1crfjOtDvlwWfsPkGqpxNe7aGRMNs+p31oGitYVT+9uplFUfVr9FFxVT1I4Ocj -B0KbTrFcui6NyvV1pl30ZZPQ8y8Udp6FD/jNt13+lgn159TXOV+l3DFSkFbJ9QQ4 -ZvCkca/sWNcgdKQeRM2Bo4rkA/QWxIZ+MDNPGr/bItFAqbl/capY05nuemsuDzlP -LGH+B1muTyjf+lbs/9cMtTqN+FZXlhHabF11oWzo/sP6O9F/cRUdske1wrfS6wha -MYNbI2TfXTDpDcK6Z8SjuQINBGTbjzEBEADoueEGCd+fQHjTkUYNtv4w6P/R2aZv -zVfAvnZmpsAiyiYxR+vPMUclvQ+17FKwivsj43B5UJK9zdRoZwwsXymo5Wc/imQ9 -eL6EGUZ414TgpC8ymfnqcuxn3Qa2kpgsuwxAHpUyb3z6/jAInqY+Qzz1I9CVzur1 -AV8LfWC2ttvlj/94OoIABOJUh1nqcQU0qA0MwoLRSZFj4PdFp4SU+lY7mVYMOV1K -M5n386t2D9NJ1G+Irv43N3I4zzFRzqVC6cbbqpEf9upDMeW4FgynhbeXvU6N/xiV -IBOTOIF0B3ClDUxXHisKTSjmdi4Z/nyyDFMivqohHjvvtnuVqdz261wt5JuW2SJ3 -7MBNbzE8U8bUgPqk1nQlfa21DsdABvdF80ORnEaKOoScnKNjSck43T/wlzPjoaR5 -TORp7Ssj9M5Ls4BKzXc1YxBXxOyoPL1UxP6lxMOPLVFp3ySte2tor3daUg9aFva8 -JImQIyvjXZuXZI6u1hR27T3fItiQSSurq9myXjOQg+sJu4R0lmBKfl3becd3YBGx -qPoPgplE/12DsDtGkR7M0sCHQUPAwt7+cmk19B130xchMAtzQLAn/HSwUGuPX6ru -MQti1KnOH33cLiL3JI5pYp84ZX1SN5IsFh8TG7GKwxDIAigPyGE04eepZWBxiont -HfDTKZT3ozb0zwARAQABiQI2BBgBCgAgFiEEAR11MuEusetSOytS/xm9ygE4Hj0F -AmTbjzECGwwACgkQ/xm9ygE4Hj1urA/7B5fg7JCNnc+GJu2Sp4N/UBzihXc6Cvcb -Zy0Esp6n68M781BM7KljYzhzjX82rfbGr3xgb1gelC8970OMme99WOE3XOFJlaPu -/YSIHgKzYi8qLqerb3Jfmyjr+9sZa2J+N5WMudQI+U7FOkwRlb0RE55jlr8xs7EK -4XGfiAPKeCV6Mn4gDPq3rnNZcFmGVkuZ/l6LbJBUty+gSFyz1tUCB8b3DJGQCPP1 -XpaEHo9c+LedIu5gAFwf0cI1xgi54RcYY1ix4Ja8aYOiabLgpCO4+XLGELs4Sq0l -mt5sSr+68rEYd1aSJB30Ubv5MnzsDOL2AkbEB9pQZfyVYAJ9ZKGSnZxqJqKvxsO/ -qbLmDRl/fgkOREoRxlhhNk1IwovPBQyw6TjpqSjKCtfR2zRpHlXil2PNA+S4Y0qz -i+tSGPtx88m9GiYRwO3VwKjB2pN/D7ObFKxLYAwY5IRrXLQlBlfTrL5agF8CAywf -eluSyiSsOPKLBL3H6rjtM92zCsq9Iqwsgc597D3dp7mxcv6NbphfHn4Njqlsm0Nz -2S8jKI0PbrvGG0izN/kNb1/8gGPPipmgL71LkwktXUVgDNLNYPOgFAmeS2zGtB8C -BowkgNp1MPAQ/Jd7FRybEijoQdF01AVZCAwT3SXI/yuhvBNDxjtfVEBDEYrNP45z -zy/bfBut8UI= -=x2ib ------END PGP PUBLIC KEY BLOCK----- diff --git a/yezzey_test/run_tests.sh b/yezzey_test/run_tests.sh deleted file mode 100755 index 68dff441c49..00000000000 --- a/yezzey_test/run_tests.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash -set -ex - -eval "$(ssh-agent -s)" -ssh-add ~/.ssh/id_rsa -sudo service ssh start -ssh -o StrictHostKeyChecking=no gpadmin@$(hostname) "echo 'Hello world'" - -sudo bash -c 'cat >> /etc/ld.so.conf <<-EOF -/usr/local/lib - -EOF' -sudo ldconfig - -sudo bash -c 'cat >> /etc/sysctl.conf <<-EOF -kernel.shmmax = 500000000 -kernel.shmmni = 4096 -kernel.shmall = 4000000000 -kernel.sem = 500 1024000 200 4096 -kernel.sysrq = 1 -kernel.core_uses_pid = 1 -kernel.msgmnb = 65536 -kernel.msgmax = 65536 -kernel.msgmni = 2048 -net.ipv4.tcp_syncookies = 1 -net.ipv4.ip_forward = 0 -net.ipv4.conf.default.accept_source_route = 0 -net.ipv4.tcp_tw_recycle = 1 -net.ipv4.tcp_max_syn_backlog = 4096 -net.ipv4.conf.all.arp_filter = 1 -net.ipv4.ip_local_port_range = 1025 65535 -net.core.netdev_max_backlog = 10000 -net.core.rmem_max = 2097152 -net.core.wmem_max = 2097152 -vm.overcommit_memory = 2 - -EOF' - -sudo bash -c 'cat >> /etc/security/limits.conf <<-EOF -* soft nofile 65536 -* hard nofile 65536 -* soft nproc 131072 -* hard nproc 131072 - -EOF' - -export GPHOME=/usr/local/gpdb -source $GPHOME/cloudberry-env.sh -ulimit -n 65536 -make destroy-demo-cluster && make create-demo-cluster -export USER=gpadmin -source gpAux/gpdemo/gpdemo-env.sh - -gpconfig -c shared_preload_libraries -v yezzey - -gpstop -a -i && gpstart -a - -createdb $USER - - -gpconfig -c yezzey.yproxy_socket -v "'/tmp/yproxy.sock'" -psql -c "ALTER SYSTEM SET yezzey.use_gpg_crypto TO false" -gpconfig -c yezzey.use_otm_feature -v "true" -gpconfig -c yezzey.use_gpg_crypto -v "false" - -gpstop -a -i && gpstart -a - -#run yproxy in daemon mode -/usr/bin/yproxy -c /tmp/yproxy.yaml -ldebug > yproxy.log 2>&1 & - -i=0 -while (! [ -S /tmp/yproxy.sock ]) && [ $i -lt 20 ]; do sleep 1; i=$(($i+1)) ; done - -cd gpcontrib/yezzey -make installcheck || (echo Yproxy logs; cat ../../yproxy.log; cat /home/gpadmin/gpcontrib/yezzey/regression.diffs && exit 1) diff --git a/yezzey_test/wal-g-conf.yaml b/yezzey_test/wal-g-conf.yaml deleted file mode 100644 index 8de5c4c2159..00000000000 --- a/yezzey_test/wal-g-conf.yaml +++ /dev/null @@ -1,12 +0,0 @@ -AWS_ACCESS_KEY_ID: "$AWS_ACCESS_KEY_ID" -AWS_SECRET_ACCESS_KEY: "$AWS_SECRET_ACCESS_KEY" -AWS_ENDPOINT: "$AWS_ENDPOINT" -AWS_S3_FORCE_PATH_STYLE: true - -WALG_COMPRESSION_METHOD: "brotli" -WALG_DELTA_MAX_STEPS: 6 -WALG_UPLOAD_CONCURRENCY: 10 -WALG_DISK_RATE_LIMIT: 41943040 -WALG_NETWORK_RATE_LIMIT: 10485760 -WALG_S3_PREFIX: "$WALG_S3_PREFIX" -WALG_PGP_KEY_PATH: "/home/gpadmin/yezzey_test/priv.gpg" diff --git a/yezzey_test/yproxy.conf b/yezzey_test/yproxy.conf deleted file mode 100644 index 855c3fce7d7..00000000000 --- a/yezzey_test/yproxy.conf +++ /dev/null @@ -1,25 +0,0 @@ -socket_path: "/tmp/yproxy.sock" -interconnect_socket_path: "/tmp/ic.sock" -log_level: debug - -storage: - access_key_id: "$AWS_ACCESS_KEY_ID" - secret_access_key: "$AWS_SECRET_ACCESS_KEY" - storage_endpoint: "$AWS_ENDPOINT" - storage_prefix: "" - storage_bucket: "gpyezzey" - storage_region: "us-west-2" - storage_type: "s3" - tablespace_map: - "pg_default": "gpyezzey" - "tab1": "gpyezzey2" - "tab2": "gpyezzey3" - -backup_storage: - access_key_id: "$AWS_ACCESS_KEY_ID" - secret_access_key: "$AWS_SECRET_ACCESS_KEY" - storage_endpoint: "$AWS_ENDPOINT" - storage_prefix: "" - storage_bucket: "gpyezzey" - storage_region: "us-west-2" - storage_type: "s3" From c4173a5828ff371b0a081b7f65972a4031e575b8 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Wed, 25 Feb 2026 13:36:13 +0300 Subject: [PATCH 06/14] Fix gpexpand not changing content-id for wal-g cmd When expanding a cluster, gpexpand copies the postgresql.conf file directly from the template segment (content 0). This causes issues for tools like wal-g which use a --content-id flag in archive_command and restore_command. Previously, new segments inherited --content-id=0 from the template. This caused them to push WAL segments to the wrong location, potentially overwriting segment 0's segments. This fix ensures the content ID in archive_command and restore_command is updated to match the new segment's ID during expansion. If the commands do not contain the --content-id flag, they remain unchanged. --- gpMgmt/bin/lib/gpconfigurenewsegment | 20 +- .../gpexpand_archive_restore_conf.out | 1047 +++++++++++++++++ .../isolation2_expandshrink_schedule | 3 +- .../sql/gpexpand_archive_restore_conf.sql | 48 + 4 files changed, 1116 insertions(+), 2 deletions(-) create mode 100644 src/test/isolation2/expected/gpexpand_archive_restore_conf.out create mode 100644 src/test/isolation2/sql/gpexpand_archive_restore_conf.sql diff --git a/gpMgmt/bin/lib/gpconfigurenewsegment b/gpMgmt/bin/lib/gpconfigurenewsegment index c37c70bf1fa..a372ad291ec 100755 --- a/gpMgmt/bin/lib/gpconfigurenewsegment +++ b/gpMgmt/bin/lib/gpconfigurenewsegment @@ -9,7 +9,7 @@ import shutil from optparse import Option, OptionGroup, OptionParser, OptionValueError, SUPPRESS_USAGE from gppylib.gpparseopts import OptParser, OptChecker -from gppylib.commands.gp import ModifyConfSetting, SegmentStart, SegmentStop +from gppylib.commands.gp import GpConfigHelper, ModifyConfSetting, SegmentStart, SegmentStop from gppylib.commands.pg import PgBaseBackup from gppylib.db import dbconn from gppylib.commands import unix @@ -268,6 +268,24 @@ class ConfExpSegCmd(Command): self.set_results(modifyConfCmd.get_results()) raise + # Update --content-id flag in wal-g for archive-restore GUCs if present + for guc in ['archive_command', 'restore_command']: + read_cmd = GpConfigHelper('Read %s' % guc, self.datadir, guc, getParameter=True) + read_cmd.run(validateAfter=True) + + val = read_cmd.get_value() + if not read_cmd.was_successful() or not val: + continue + + new_val = re.sub(r'(--content-id(?:=|\s+))-?\d+', r'\g<1>' + str(self.contentid), val) + if new_val != val: + write_cmd = GpConfigHelper('Update %s' % guc, self.datadir, guc, value=new_val) + try: + write_cmd.run(validateAfter=True) + except Exception: + self.set_results(write_cmd.get_results()) + raise + # We might need to stop the segment if the last setup failed past this point if os.path.exists('%s/postmaster.pid' % self.datadir): logger.info('%s/postmaster.pid exists. Stopping segment' % self.datadir) diff --git a/src/test/isolation2/expected/gpexpand_archive_restore_conf.out b/src/test/isolation2/expected/gpexpand_archive_restore_conf.out new file mode 100644 index 00000000000..08d97a49c47 --- /dev/null +++ b/src/test/isolation2/expected/gpexpand_archive_restore_conf.out @@ -0,0 +1,1047 @@ +-- Verify that gpexpand correctly updates segment-specific flags for wal-g in +-- archive_command and restore_command when initializing new segments. +-- +-- Previously, these GUCs were copied verbatim from the template segment +-- (content 0), causing new segments to invoke archiving with an incorrect +-- content-id (possible overwriting archive or restoring other's data). + +-- Cleanup any previous state +!\retcode yes | gpexpand -c; +-- start_ignore +20260225:10:13:28:029615 gpexpand:cdw:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:13:28:029615 gpexpand:cdw:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 14.4 (Apache Cloudberry 3.0.0-devel+dev.4.g01c21009c35 build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.4.0-1ubuntu1~22.04.2) 11.4.0, 64-bit compiled on Feb 25 2026 10:08:48 (with assert checking)' +20260225:10:13:28:029615 gpexpand:cdw:gpadmin-[INFO]:-Querying gpexpand schema for current expansion state + +System Expansion is used to add segments to an existing CBDB array. +gpexpand did not detect a System Expansion that is in progress. + +Before initiating a System Expansion, you need to provision and burn-in +the new hardware. Please be sure to run gpcheckperf to make sure the +new hardware is working properly. + +Please refer to the Admin Guide for more information. + +Would you like to initiate a new System Expansion Yy|Nn (default=N): +> +Enter a comma separated list of new hosts you want +to add to your array. Do not include interface hostnames. +**Enter a blank line to only add segments to existing hosts**[]: +> 20260225:10:13:28:029615 gpexpand:cdw:gpadmin-[ERROR]:-gpexpand failed: You must be adding two or more hosts when expanding a system with mirroring enabled. + +Exiting... +20260225:10:13:28:029615 gpexpand:cdw:gpadmin-[INFO]:-Shutting down gpexpand... + +-- end_ignore +(exited with code 3) +!\retcode gpshrink -c; +-- start_ignore +20260225:10:13:28:029633 gpshrink:cdw:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:13:28:029633 gpshrink:cdw:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 14.4 (Apache Cloudberry 3.0.0-devel+dev.4.g01c21009c35 build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.4.0-1ubuntu1~22.04.2) 11.4.0, 64-bit compiled on Feb 25 2026 10:08:48 (with assert checking)' +20260225:10:13:28:029633 gpshrink:cdw:gpadmin-[INFO]:-Removing gpshrink schema +20260225:10:13:28:029633 gpshrink:cdw:gpadmin-[INFO]:-Cleanup Finished. exiting... + +-- end_ignore +(exited with code 0) +!\retcode rm -r /tmp/datadirs/; +-- start_ignore +rm: cannot remove '/tmp/datadirs/': No such file or directory + +-- end_ignore +(exited with code 1) + +-- Set GUCs on all segments, hardcode --content-id to 0 +!\retcode gpconfig -c restore_command -v '/bin/true'; +-- start_ignore +20260225:10:13:29:029665 gpconfig:cdw:gpadmin-[INFO]:-completed successfully with parameters '-c restore_command -v /bin/true' + +-- end_ignore +(exited with code 0) +!\retcode gpconfig -c archive_command -v 'wal-g seg wal-push %p --content-id=0'; +-- start_ignore +20260225:10:13:29:029714 gpconfig:cdw:gpadmin-[INFO]:-completed successfully with parameters '-c archive_command -v 'wal-g seg wal-push %p --content-id=0'' + +-- end_ignore +(exited with code 0) +!\retcode gpstop -u; +-- start_ignore +20260225:10:13:29:029763 gpstop:cdw:gpadmin-[INFO]:-Starting gpstop with args: -u +20260225:10:13:29:029763 gpstop:cdw:gpadmin-[INFO]:-Gathering information and validating the environment... +20260225:10:13:29:029763 gpstop:cdw:gpadmin-[INFO]:-Obtaining Cloudberry Coordinator catalog information +20260225:10:13:29:029763 gpstop:cdw:gpadmin-[INFO]:-Obtaining Segment details from coordinator... +20260225:10:13:29:029763 gpstop:cdw:gpadmin-[INFO]:-Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:13:29:029763 gpstop:cdw:gpadmin-[INFO]:-Signalling all postmaster processes to reload + +-- end_ignore +(exited with code 0) + +-- Prepare expansion configuration +!\retcode echo "localhost|localhost|7008|/tmp/datadirs/dbfast4/demoDataDir3|9|3|p localhost|localhost|7009|/tmp/datadirs/dbfast_mirror4/demoDataDir3|10|3|m" > /tmp/testexpand; +-- start_ignore + +-- end_ignore +(exited with code 0) + +-- Expand +!\retcode gpexpand -i /tmp/testexpand; +-- start_ignore +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 14.4 (Apache Cloudberry 3.0.0-devel+dev.4.g01c21009c35 build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.4.0-1ubuntu1~22.04.2) 11.4.0, 64-bit compiled on Feb 25 2026 10:08:48 (with assert checking)' +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Querying gpexpand schema for current expansion state +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast1/demoDataDir0 for dbid 2: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540553465864 +Database cluster state: in production +pg_control last modified: Wed 25 Feb 2026 10:13:25 AM GMT +Latest checkpoint location: 0/C067FA8 +Latest checkpoint's REDO location: 0/C067F70 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:877 +Latest checkpoint's NextGxid: 9 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 876 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:13:25 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/0 +Min recovery ending loc's timeline: 0 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: 4ab1899cbfd68b59268d02e0d5a1966c0ee3f4b463979bc5372053c2b14ece58 +File encryption method: + +stderr: +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror1/demoDataDir0 for dbid 5: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540553465864 +Database cluster state: in archive recovery +pg_control last modified: Wed 25 Feb 2026 10:12:19 AM GMT +Latest checkpoint location: 0/C005578 +Latest checkpoint's REDO location: 0/C005578 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:872 +Latest checkpoint's NextGxid: 3 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 0 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:12:16 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/C005600 +Min recovery ending loc's timeline: 1 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: 4ab1899cbfd68b59268d02e0d5a1966c0ee3f4b463979bc5372053c2b14ece58 +File encryption method: + +stderr: +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast2/demoDataDir1 for dbid 3: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540653109260 +Database cluster state: in production +pg_control last modified: Wed 25 Feb 2026 10:13:25 AM GMT +Latest checkpoint location: 0/C067FA8 +Latest checkpoint's REDO location: 0/C067F70 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:877 +Latest checkpoint's NextGxid: 9 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 876 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:13:25 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/0 +Min recovery ending loc's timeline: 0 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: 91cf6a7a7826dd0c9f8ff23a43d226c7a1a72752a9b1544a78c7cfbe62970991 +File encryption method: + +stderr: +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror2/demoDataDir1 for dbid 6: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540653109260 +Database cluster state: in archive recovery +pg_control last modified: Wed 25 Feb 2026 10:12:19 AM GMT +Latest checkpoint location: 0/C005578 +Latest checkpoint's REDO location: 0/C005578 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:872 +Latest checkpoint's NextGxid: 3 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 0 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:12:16 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/C005600 +Min recovery ending loc's timeline: 1 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: 91cf6a7a7826dd0c9f8ff23a43d226c7a1a72752a9b1544a78c7cfbe62970991 +File encryption method: + +stderr: +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast3/demoDataDir2 for dbid 4: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540636610571 +Database cluster state: in production +pg_control last modified: Wed 25 Feb 2026 10:13:25 AM GMT +Latest checkpoint location: 0/C067FA8 +Latest checkpoint's REDO location: 0/C067F70 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:877 +Latest checkpoint's NextGxid: 9 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 876 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:13:25 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/0 +Min recovery ending loc's timeline: 0 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: deb1a8b64d8bc3ddf938a6ec69c7c9785d8f32343c42e3878fe207fbee3ac992 +File encryption method: + +stderr: +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror3/demoDataDir2 for dbid 7: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540636610571 +Database cluster state: in archive recovery +pg_control last modified: Wed 25 Feb 2026 10:12:19 AM GMT +Latest checkpoint location: 0/C005578 +Latest checkpoint's REDO location: 0/C005578 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:872 +Latest checkpoint's NextGxid: 3 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 0 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:12:16 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/C005600 +Min recovery ending loc's timeline: 1 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: deb1a8b64d8bc3ddf938a6ec69c7c9785d8f32343c42e3878fe207fbee3ac992 +File encryption method: + +stderr: +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Heap checksum setting consistent across cluster +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Syncing Apache Cloudberry extensions +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[ERROR]:-Syncing of Apache Cloudberry extensions has failed. +Traceback (most recent call last): + File "/usr/local/cloudberry-db/bin/gpexpand", line 1935, in sync_packages + operation.get_ret() + File "/usr/local/cloudberry-db/lib/python/gppylib/operations/__init__.py", line 64, in get_ret + if isinstance(self.ret, Exception): +AttributeError: 'SyncPackages' object has no attribute 'ret' +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[WARNING]:-Please run gppkg --clean after successful expansion. +20260225:10:13:30:029792 gpexpand:cdw:gpadmin-[INFO]:-Locking catalog +20260225:10:13:30:029792 gpexpand:cdw:gpadmin-[INFO]:-Locked catalog +20260225:10:13:30:029792 gpexpand:cdw:gpadmin-[INFO]:-Creating segment template +20260225:10:13:33:029792 gpexpand:cdw:gpadmin-[INFO]:-Copying postgresql.conf from existing segment into template +20260225:10:13:33:029792 gpexpand:cdw:gpadmin-[INFO]:-Copying pg_hba.conf from existing segment into template +20260225:10:13:34:029792 gpexpand:cdw:gpadmin-[INFO]:-Creating schema tar file +20260225:10:13:36:029792 gpexpand:cdw:gpadmin-[INFO]:-Distributing template tar file to new hosts +20260225:10:13:52:029792 gpexpand:cdw:gpadmin-[INFO]:-Configuring new segments (primary) +20260225:10:13:52:029792 gpexpand:cdw:gpadmin-[INFO]:-{'localhost': '/tmp/datadirs/dbfast4/demoDataDir3:7008:true:false:9:3::-1:'} +20260225:10:13:54:029792 gpexpand:cdw:gpadmin-[INFO]:-Starting to create new pg_hba.conf on primary segments +20260225:10:13:54:029792 gpexpand:cdw:gpadmin-[INFO]:-None of the reachable segments require update to pg_hba.conf +20260225:10:13:54:029792 gpexpand:cdw:gpadmin-[INFO]:-SegmentStart pg_ctl cmd is env GPSESSID=0000000000 GPERA=None $GPHOME/bin/pg_ctl -D /tmp/datadirs/dbfast4/demoDataDir3 -l /tmp/datadirs/dbfast4/demoDataDir3/log/startup.log -w -t 600 -o " -p 7008 -c gp_role=utility -M " start +20260225:10:13:55:029792 gpexpand:cdw:gpadmin-[INFO]:-Cleaning up temporary template files +20260225:10:13:56:029792 gpexpand:cdw:gpadmin-[INFO]:-Cleaning up databases in new segments. +20260225:10:13:56:029792 gpexpand:cdw:gpadmin-[INFO]:-SegmentStart pg_ctl cmd is env GPSESSID=0000000000 GPERA=None $GPHOME/bin/pg_ctl -D /tmp/datadirs/dbfast4/demoDataDir3 -l /tmp/datadirs/dbfast4/demoDataDir3/log/startup.log -w -t 600 -o " -p 7008 -c gp_role=utility " start +20260225:10:13:56:029792 gpexpand:cdw:gpadmin-[INFO]:-Unlocking catalog +20260225:10:13:56:029792 gpexpand:cdw:gpadmin-[INFO]:-Unlocked catalog +20260225:10:13:57:029792 gpexpand:cdw:gpadmin-[INFO]:-Creating expansion schema +20260225:10:13:57:029792 gpexpand:cdw:gpadmin-[INFO]:-Populating gpexpand.status_detail with data from database postgres +20260225:10:13:57:029792 gpexpand:cdw:gpadmin-[INFO]:-Populating gpexpand.status_detail with data from database template1 +20260225:10:13:57:029792 gpexpand:cdw:gpadmin-[INFO]:-Populating gpexpand.status_detail with data from database isolation2parallelretrcursor +20260225:10:13:57:029792 gpexpand:cdw:gpadmin-[INFO]:-Populating gpexpand.status_detail with data from database isolation2test +20260225:10:13:58:029792 gpexpand:cdw:gpadmin-[INFO]:-Starting new mirror segment synchronization +20260225:10:14:08:029792 gpexpand:cdw:gpadmin-[INFO]:-************************************************ +20260225:10:14:08:029792 gpexpand:cdw:gpadmin-[INFO]:-Initialization of the system expansion complete. +20260225:10:14:08:029792 gpexpand:cdw:gpadmin-[INFO]:-To begin table expansion onto the new segments +20260225:10:14:08:029792 gpexpand:cdw:gpadmin-[INFO]:-rerun gpexpand +20260225:10:14:08:029792 gpexpand:cdw:gpadmin-[INFO]:-************************************************ +20260225:10:14:08:029792 gpexpand:cdw:gpadmin-[INFO]:-Exiting... + +-- end_ignore +(exited with code 0) +!\retcode gpexpand -i /tmp/testexpand; +-- start_ignore +20260225:10:14:08:030678 gpexpand:cdw:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:14:08:030678 gpexpand:cdw:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 14.4 (Apache Cloudberry 3.0.0-devel+dev.4.g01c21009c35 build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.4.0-1ubuntu1~22.04.2) 11.4.0, 64-bit compiled on Feb 25 2026 10:08:48 (with assert checking)' +20260225:10:14:08:030678 gpexpand:cdw:gpadmin-[INFO]:-Querying gpexpand schema for current expansion state +20260225:10:14:09:030678 gpexpand:cdw:gpadmin-[INFO]:-EXPANSION COMPLETED SUCCESSFULLY +20260225:10:14:09:030678 gpexpand:cdw:gpadmin-[INFO]:-Exiting... + +-- end_ignore +(exited with code 0) + +-- Get the new segment's datadir (content=3) +!\retcode psql -d postgres -Aqt -c "SELECT datadir FROM gp_segment_configuration WHERE content = 3 AND role = 'p'" > /tmp/new_segment_datadir; +-- start_ignore + +-- end_ignore +(exited with code 0) + +-- Confirm that the --content-id flag within archive_command has been +-- updated to match the new segment's content. The restore_command +-- lacks --content-id flag and should be unchanged. +! grep "^archive_command" $(cat /tmp/new_segment_datadir)/postgresql.conf; +archive_command='wal-g seg wal-push %p --content-id=3' + +! grep "^restore_command" $(cat /tmp/new_segment_datadir)/postgresql.conf; +restore_command='/bin/true' + + +-- Cleanup +!\retcode gpconfig -r restore_command; +-- start_ignore +20260225:10:14:09:030725 gpconfig:cdw:gpadmin-[INFO]:-completed successfully with parameters '-r restore_command' + +-- end_ignore +(exited with code 0) +!\retcode gpconfig -r archive_command; +-- start_ignore +20260225:10:14:10:030829 gpconfig:cdw:gpadmin-[INFO]:-completed successfully with parameters '-r archive_command' + +-- end_ignore +(exited with code 0) +!\retcode gpstop -u; +-- start_ignore +20260225:10:14:10:030933 gpstop:cdw:gpadmin-[INFO]:-Starting gpstop with args: -u +20260225:10:14:10:030933 gpstop:cdw:gpadmin-[INFO]:-Gathering information and validating the environment... +20260225:10:14:10:030933 gpstop:cdw:gpadmin-[INFO]:-Obtaining Cloudberry Coordinator catalog information +20260225:10:14:10:030933 gpstop:cdw:gpadmin-[INFO]:-Obtaining Segment details from coordinator... +20260225:10:14:10:030933 gpstop:cdw:gpadmin-[INFO]:-Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:14:10:030933 gpstop:cdw:gpadmin-[INFO]:-Signalling all postmaster processes to reload + +-- end_ignore +(exited with code 0) + +!\retcode gpshrink -i /tmp/testexpand; +-- start_ignore +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 14.4 (Apache Cloudberry 3.0.0-devel+dev.4.g01c21009c35 build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.4.0-1ubuntu1~22.04.2) 11.4.0, 64-bit compiled on Feb 25 2026 10:08:48 (with assert checking)' +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast1/demoDataDir0 for dbid 2: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540553465864 +Database cluster state: in production +pg_control last modified: Wed 25 Feb 2026 10:13:25 AM GMT +Latest checkpoint location: 0/C067FA8 +Latest checkpoint's REDO location: 0/C067F70 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:877 +Latest checkpoint's NextGxid: 9 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 876 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:13:25 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/0 +Min recovery ending loc's timeline: 0 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: 4ab1899cbfd68b59268d02e0d5a1966c0ee3f4b463979bc5372053c2b14ece58 +File encryption method: + +stderr: +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror1/demoDataDir0 for dbid 5: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540553465864 +Database cluster state: in archive recovery +pg_control last modified: Wed 25 Feb 2026 10:12:19 AM GMT +Latest checkpoint location: 0/C005578 +Latest checkpoint's REDO location: 0/C005578 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:872 +Latest checkpoint's NextGxid: 3 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 0 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:12:16 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/C005600 +Min recovery ending loc's timeline: 1 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: 4ab1899cbfd68b59268d02e0d5a1966c0ee3f4b463979bc5372053c2b14ece58 +File encryption method: + +stderr: +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast2/demoDataDir1 for dbid 3: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540653109260 +Database cluster state: in production +pg_control last modified: Wed 25 Feb 2026 10:13:25 AM GMT +Latest checkpoint location: 0/C067FA8 +Latest checkpoint's REDO location: 0/C067F70 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:877 +Latest checkpoint's NextGxid: 9 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 876 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:13:25 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/0 +Min recovery ending loc's timeline: 0 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: 91cf6a7a7826dd0c9f8ff23a43d226c7a1a72752a9b1544a78c7cfbe62970991 +File encryption method: + +stderr: +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror2/demoDataDir1 for dbid 6: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540653109260 +Database cluster state: in archive recovery +pg_control last modified: Wed 25 Feb 2026 10:12:19 AM GMT +Latest checkpoint location: 0/C005578 +Latest checkpoint's REDO location: 0/C005578 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:872 +Latest checkpoint's NextGxid: 3 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 0 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:12:16 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/C005600 +Min recovery ending loc's timeline: 1 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: 91cf6a7a7826dd0c9f8ff23a43d226c7a1a72752a9b1544a78c7cfbe62970991 +File encryption method: + +stderr: +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast3/demoDataDir2 for dbid 4: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540636610571 +Database cluster state: in production +pg_control last modified: Wed 25 Feb 2026 10:13:25 AM GMT +Latest checkpoint location: 0/C067FA8 +Latest checkpoint's REDO location: 0/C067F70 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:877 +Latest checkpoint's NextGxid: 9 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 876 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:13:25 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/0 +Min recovery ending loc's timeline: 0 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: deb1a8b64d8bc3ddf938a6ec69c7c9785d8f32343c42e3878fe207fbee3ac992 +File encryption method: + +stderr: +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror3/demoDataDir2 for dbid 7: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540636610571 +Database cluster state: in archive recovery +pg_control last modified: Wed 25 Feb 2026 10:12:19 AM GMT +Latest checkpoint location: 0/C005578 +Latest checkpoint's REDO location: 0/C005578 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:872 +Latest checkpoint's NextGxid: 3 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 0 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:12:16 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/C005600 +Min recovery ending loc's timeline: 1 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: deb1a8b64d8bc3ddf938a6ec69c7c9785d8f32343c42e3878fe207fbee3ac992 +File encryption method: + +stderr: +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /tmp/datadirs/dbfast_mirror4/demoDataDir3 for dbid 10: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743519254633466 +Database cluster state: in archive recovery +pg_control last modified: Wed 25 Feb 2026 10:14:07 AM GMT +Latest checkpoint location: 0/1C000060 +Latest checkpoint's REDO location: 0/1C000028 +Latest checkpoint's REDO WAL file: 000000010000000000000007 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:966 +Latest checkpoint's NextGxid: 8196 +Latest checkpoint's NextOID: 33402 +Latest checkpoint's NextRelfilenode: 24576 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 966 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:14:02 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/1C000110 +Min recovery ending loc's timeline: 1 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: ec79bca6b7c99a5ea2ac78e41778e68c225231739152db9d7d59cc19b9096a29 +File encryption method: + +stderr: +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /tmp/datadirs/dbfast4/demoDataDir3 for dbid 9: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743519254633466 +Database cluster state: in production +pg_control last modified: Wed 25 Feb 2026 10:14:02 AM GMT +Latest checkpoint location: 0/1C000060 +Latest checkpoint's REDO location: 0/1C000028 +Latest checkpoint's REDO WAL file: 000000010000000000000007 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:966 +Latest checkpoint's NextGxid: 8196 +Latest checkpoint's NextOID: 33402 +Latest checkpoint's NextRelfilenode: 24576 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 966 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:14:02 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/0 +Min recovery ending loc's timeline: 0 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: ec79bca6b7c99a5ea2ac78e41778e68c225231739152db9d7d59cc19b9096a29 +File encryption method: + +stderr: +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Heap checksum setting consistent across cluster +20260225:10:14:11:030997 gpshrink:cdw:gpadmin-[INFO]:-Locking catalog +20260225:10:14:11:030997 gpshrink:cdw:gpadmin-[INFO]:-Locked catalog +20260225:10:14:11:030997 gpshrink:cdw:gpadmin-[INFO]:-Unlocking catalog +20260225:10:14:11:030997 gpshrink:cdw:gpadmin-[INFO]:-Unlocked catalog +20260225:10:14:11:030997 gpshrink:cdw:gpadmin-[INFO]:-Creating shrink schema +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-Populating gpshrink.status_detail with data from database postgres +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-Populating gpshrink.status_detail with data from database template1 +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-Populating gpshrink.status_detail with data from database isolation2parallelretrcursor +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-Populating gpshrink.status_detail with data from database isolation2test +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-************************************************ +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-Initialization of the system shrink complete. +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-To begin table shrink onto the new segments +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-rerun gpshrink +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-************************************************ +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-Exiting... + +-- end_ignore +(exited with code 0) +!\retcode gpshrink -i /tmp/testexpand; +-- start_ignore +20260225:10:14:13:031255 gpshrink:cdw:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:14:13:031255 gpshrink:cdw:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 14.4 (Apache Cloudberry 3.0.0-devel+dev.4.g01c21009c35 build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.4.0-1ubuntu1~22.04.2) 11.4.0, 64-bit compiled on Feb 25 2026 10:08:48 (with assert checking)' +20260225:10:14:13:031255 gpshrink:cdw:gpadmin-[INFO]:-Shrinking postgres.gpexpand.status_detail +20260225:10:14:13:031255 gpshrink:cdw:gpadmin-[INFO]:-Finished shrinking postgres.gpexpand.status_detail +20260225:10:14:14:031255 gpshrink:cdw:gpadmin-[INFO]:-Shrinking postgres.gpshrink.status_detail +20260225:10:14:14:031255 gpshrink:cdw:gpadmin-[INFO]:-Finished shrinking postgres.gpshrink.status_detail +20260225:10:14:14:031255 gpshrink:cdw:gpadmin-[INFO]:-Shrinking postgres.gpexpand.status +20260225:10:14:14:031255 gpshrink:cdw:gpadmin-[INFO]:-Finished shrinking postgres.gpexpand.status +20260225:10:14:14:031255 gpshrink:cdw:gpadmin-[INFO]:-Shrinking postgres.gpshrink.status +20260225:10:14:14:031255 gpshrink:cdw:gpadmin-[INFO]:-Finished shrinking postgres.gpshrink.status +20260225:10:14:18:031255 gpshrink:cdw:gpadmin-[INFO]:-SHRINK COMPLETED SUCCESSFULLY +20260225:10:14:18:031255 gpshrink:cdw:gpadmin-[INFO]:-Locking catalog +20260225:10:14:18:031255 gpshrink:cdw:gpadmin-[INFO]:-Locked catalog +20260225:10:14:19:031255 gpshrink:cdw:gpadmin-[INFO]:-Unlocking catalog +20260225:10:14:19:031255 gpshrink:cdw:gpadmin-[INFO]:-Unlocked catalog +20260225:10:14:19:031255 gpshrink:cdw:gpadmin-[INFO]:-Exiting... + +-- end_ignore +(exited with code 0) + +!\retcode yes | gpexpand -c; +-- start_ignore +20260225:10:14:19:031525 gpexpand:cdw:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:14:19:031525 gpexpand:cdw:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 14.4 (Apache Cloudberry 3.0.0-devel+dev.4.g01c21009c35 build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.4.0-1ubuntu1~22.04.2) 11.4.0, 64-bit compiled on Feb 25 2026 10:08:48 (with assert checking)' +20260225:10:14:19:031525 gpexpand:cdw:gpadmin-[INFO]:-Querying gpexpand schema for current expansion state + + +Do you want to dump the gpexpand.status_detail table to file? Yy|Nn (default=Y): +> 20260225:10:14:19:031525 gpexpand:cdw:gpadmin-[INFO]:-Dumping gpexpand.status_detail to /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/qddir/demoDataDir-1/gpexpand.status_detail +20260225:10:14:19:031525 gpexpand:cdw:gpadmin-[INFO]:-Removing gpexpand schema +20260225:10:14:19:031525 gpexpand:cdw:gpadmin-[INFO]:-Cleanup Finished. exiting... + +-- end_ignore +(exited with code 0) +!\retcode gpshrink -c; +-- start_ignore +20260225:10:14:20:031554 gpshrink:cdw:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:14:20:031554 gpshrink:cdw:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 14.4 (Apache Cloudberry 3.0.0-devel+dev.4.g01c21009c35 build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.4.0-1ubuntu1~22.04.2) 11.4.0, 64-bit compiled on Feb 25 2026 10:08:48 (with assert checking)' +20260225:10:14:20:031554 gpshrink:cdw:gpadmin-[INFO]:-Removing gpshrink schema +20260225:10:14:20:031554 gpshrink:cdw:gpadmin-[INFO]:-Cleanup Finished. exiting... + +-- end_ignore +(exited with code 0) +!\retcode rm -r /tmp/datadirs/; +-- start_ignore + +-- end_ignore +(exited with code 0) +!\retcode rm /tmp/new_segment_datadir; +-- start_ignore + +-- end_ignore +(exited with code 0) + diff --git a/src/test/isolation2/isolation2_expandshrink_schedule b/src/test/isolation2/isolation2_expandshrink_schedule index 3fd75a5eff0..89787bc1818 100644 --- a/src/test/isolation2/isolation2_expandshrink_schedule +++ b/src/test/isolation2/isolation2_expandshrink_schedule @@ -1,3 +1,4 @@ # Tests for gpexpand and gpshrink # Keep for single schedule due to redistribute all the table in all database -test: gpexpand_gpshrink \ No newline at end of file +test: gpexpand_gpshrink +test: gpexpand_archive_restore_conf diff --git a/src/test/isolation2/sql/gpexpand_archive_restore_conf.sql b/src/test/isolation2/sql/gpexpand_archive_restore_conf.sql new file mode 100644 index 00000000000..39cdf458f22 --- /dev/null +++ b/src/test/isolation2/sql/gpexpand_archive_restore_conf.sql @@ -0,0 +1,48 @@ +-- Verify that gpexpand correctly updates segment-specific flags for wal-g in +-- archive_command and restore_command when initializing new segments. +-- +-- Previously, these GUCs were copied verbatim from the template segment +-- (content 0), causing new segments to invoke archiving with an incorrect +-- content-id (possible overwriting archive or restoring other's data). + +-- Cleanup any previous state +!\retcode yes | gpexpand -c; +!\retcode gpshrink -c; +!\retcode rm -r /tmp/datadirs/; + +-- Set GUCs on all segments, hardcode --content-id to 0 +!\retcode gpconfig -c restore_command -v '/bin/true'; +!\retcode gpconfig -c archive_command -v 'wal-g seg wal-push %p --content-id=0'; +!\retcode gpstop -u; + +-- Prepare expansion configuration +!\retcode echo "localhost|localhost|7008|/tmp/datadirs/dbfast4/demoDataDir3|9|3|p +localhost|localhost|7009|/tmp/datadirs/dbfast_mirror4/demoDataDir3|10|3|m" > /tmp/testexpand; + +-- Expand +!\retcode gpexpand -i /tmp/testexpand; +!\retcode gpexpand -i /tmp/testexpand; + +-- Get the new segment's datadir (content=3) +!\retcode psql -d postgres -Aqt -c "SELECT datadir FROM gp_segment_configuration +WHERE content = 3 AND role = 'p'" > /tmp/new_segment_datadir; + +-- Confirm that the --content-id flag within archive_command has been +-- updated to match the new segment's content. The restore_command +-- lacks --content-id flag and should be unchanged. +! grep "^archive_command" $(cat /tmp/new_segment_datadir)/postgresql.conf; +! grep "^restore_command" $(cat /tmp/new_segment_datadir)/postgresql.conf; + +-- Cleanup +!\retcode gpconfig -r restore_command; +!\retcode gpconfig -r archive_command; +!\retcode gpstop -u; + +!\retcode gpshrink -i /tmp/testexpand; +!\retcode gpshrink -i /tmp/testexpand; + +!\retcode yes | gpexpand -c; +!\retcode gpshrink -c; +!\retcode rm -r /tmp/datadirs/; +!\retcode rm /tmp/new_segment_datadir; + From baa696e33f557338ef8a05c574abad7b92d1ab5a Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 21 Apr 2026 13:54:35 +0500 Subject: [PATCH 07/14] Bump yezzey. (#33) To the MWP cbdb version --- gpcontrib/yezzey | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpcontrib/yezzey b/gpcontrib/yezzey index 4c6b5b83735..642726e074f 160000 --- a/gpcontrib/yezzey +++ b/gpcontrib/yezzey @@ -1 +1 @@ -Subproject commit 4c6b5b83735320dda01e042631a851336300a3ca +Subproject commit 642726e074f8a553af05e1eab8e5d460b45a93ca From 5241b08a411aa116442ded080ae04f8df990ef6d Mon Sep 17 00:00:00 2001 From: Leonid <63977577+leborchuk@users.noreply.github.com> Date: Fri, 15 May 2026 17:35:47 +0300 Subject: [PATCH 08/14] Move yezzey forward to full support Cloudberry (#35) * Move yezzey forward to full support Cloudberry --- .github/workflows/yezzey-ci.yaml | 4 ++-- gpcontrib/yezzey | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/yezzey-ci.yaml b/.github/workflows/yezzey-ci.yaml index c1c41497a64..1d4402b0ad1 100644 --- a/.github/workflows/yezzey-ci.yaml +++ b/.github/workflows/yezzey-ci.yaml @@ -130,7 +130,7 @@ jobs: run: | set -ex pipefail # Download mc for Linux (amd64) - curl -O https://dl.min.io/client/mc/release/linux-amd64/mc + curl -fsSL -o mc https://dl.min.io/client/mc/release/linux-amd64/mc chmod +x mc sudo mv mc /usr/local/bin/mc # Make mc available system-wide @@ -193,7 +193,7 @@ jobs: set -eo pipefail chmod +x "${SRC_DIR}"/gpcontrib/yezzey/devops/scripts/prepare_test_yezzey.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && gpcontrib/yezzey/devops/scripts/prepare_test_yezzey.sh"; then + if ! time su - gpadmin -c "cd ${SRC_DIR}/gpcontrib/yezzey && devops/scripts/prepare_test_yezzey.sh"; then echo "::error::Config yezzey failed" exit 1 fi diff --git a/gpcontrib/yezzey b/gpcontrib/yezzey index 642726e074f..0d88f66a5fd 160000 --- a/gpcontrib/yezzey +++ b/gpcontrib/yezzey @@ -1 +1 @@ -Subproject commit 642726e074f8a553af05e1eab8e5d460b45a93ca +Subproject commit 0d88f66a5fd0dba82681eef5929529cb153cb325 From 215a3497776b8e83eddc895d0319adcd2441c942 Mon Sep 17 00:00:00 2001 From: reshke Date: Wed, 3 Jun 2026 11:49:10 +0500 Subject: [PATCH 09/14] Bump yezzey for PAX fix (#37) https://github.com/open-gpdb/yezzey/compare/0d88f66a5fd0dba82681eef5929529cb153cb325...2b5dcadd45b4183a4aa5ab976e50c97f0d4c7057 --- gpcontrib/yezzey | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpcontrib/yezzey b/gpcontrib/yezzey index 0d88f66a5fd..2b5dcadd45b 160000 --- a/gpcontrib/yezzey +++ b/gpcontrib/yezzey @@ -1 +1 @@ -Subproject commit 0d88f66a5fd0dba82681eef5929529cb153cb325 +Subproject commit 2b5dcadd45b4183a4aa5ab976e50c97f0d4c7057 From 0744b3b94415bdf500d0dba75beebd394353fb33 Mon Sep 17 00:00:00 2001 From: Alena Rybakina <58230554+Alena0704@users.noreply.github.com> Date: Tue, 16 Jun 2026 18:46:11 +0300 Subject: [PATCH 10/14] Allow non-superuser role to manage resource groups. Historically Yandex Greenplum allows non-superuser no managed resource groups. So, a regular non-superuser role allowed to run pg_resgroup_move_query(), and tune CPU/memory limits if granted with mdb_admin. Such feature was introduced as early as 6.22, see also gpdb commit 3ac99962. This commit introduces same feature for managed Cloudberry. To disallow altering predefined roles, fixed-OID hardening is used, reserving 8067 OID to be an mdb_admin role OID. We choose this (efficiently a catalog change) over complex bookkeeping what CREATEROLE can do and what is disallowed. We use Yandex managed predefined roles bootstrap util via auxiliary contrib extension, based on what Yandex Postgres fork does, see also pg-sharding/cpg repo. Co-authored-by: Andrey Borodin Co-authored-by: reshke --- .github/workflows/build-cloudberry-rocky8.yml | 1 + .github/workflows/build-cloudberry.yml | 1 + .github/workflows/build-deb-cloudberry.yml | 1 + contrib/Makefile | 1 + contrib/pg_aux_catalog/.gitignore | 6 + contrib/pg_aux_catalog/Makefile | 37 ++++++ contrib/pg_aux_catalog/README.md | 71 ++++++++++ .../expected/pg_aux_catalog.out | 46 +++++++ .../expected/resgroup_mdb_admin.out | 123 ++++++++++++++++++ .../isolation2/isolation2_schedule | 1 + .../isolation2/sql/resgroup_mdb_admin.sql | 91 +++++++++++++ .../pg_aux_catalog/pg_aux_catalog--1.0.sql | 10 ++ contrib/pg_aux_catalog/pg_aux_catalog.c | 88 +++++++++++++ contrib/pg_aux_catalog/pg_aux_catalog.control | 5 + contrib/pg_aux_catalog/sql/pg_aux_catalog.sql | 38 ++++++ pom.xml | 3 + src/backend/catalog/oid_dispatch.c | 50 ++++++- src/backend/commands/resgroupcmds.c | 51 +++++--- src/backend/utils/resgroup/resgroup_helper.c | 5 +- src/include/access/transam.h | 10 ++ src/include/catalog/oid_dispatch.h | 7 + src/include/utils/acl.h | 8 ++ 22 files changed, 631 insertions(+), 23 deletions(-) create mode 100644 contrib/pg_aux_catalog/.gitignore create mode 100644 contrib/pg_aux_catalog/Makefile create mode 100644 contrib/pg_aux_catalog/README.md create mode 100644 contrib/pg_aux_catalog/expected/pg_aux_catalog.out create mode 100644 contrib/pg_aux_catalog/isolation2/expected/resgroup_mdb_admin.out create mode 100644 contrib/pg_aux_catalog/isolation2/isolation2_schedule create mode 100644 contrib/pg_aux_catalog/isolation2/sql/resgroup_mdb_admin.sql create mode 100644 contrib/pg_aux_catalog/pg_aux_catalog--1.0.sql create mode 100644 contrib/pg_aux_catalog/pg_aux_catalog.c create mode 100644 contrib/pg_aux_catalog/pg_aux_catalog.control create mode 100644 contrib/pg_aux_catalog/sql/pg_aux_catalog.sql diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml index 2abf88060e3..c8068f098cb 100644 --- a/.github/workflows/build-cloudberry-rocky8.yml +++ b/.github/workflows/build-cloudberry-rocky8.yml @@ -311,6 +311,7 @@ jobs: "contrib/pgstattuple:installcheck", "contrib/tablefunc:installcheck", "contrib/passwordcheck:installcheck", + "contrib/pg_aux_catalog:installcheck", "contrib/pg_buffercache:installcheck", "contrib/sslinfo:installcheck"] }, diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index ca75f7b42e7..99bc67c99b7 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -304,6 +304,7 @@ jobs: "contrib/pgstattuple:installcheck", "contrib/tablefunc:installcheck", "contrib/passwordcheck:installcheck", + "contrib/pg_aux_catalog:installcheck", "contrib/pg_buffercache:installcheck", "contrib/sslinfo:installcheck"] }, diff --git a/.github/workflows/build-deb-cloudberry.yml b/.github/workflows/build-deb-cloudberry.yml index 85d917b8ff0..fee69b073f7 100644 --- a/.github/workflows/build-deb-cloudberry.yml +++ b/.github/workflows/build-deb-cloudberry.yml @@ -243,6 +243,7 @@ jobs: "contrib/pgstattuple:installcheck", "contrib/tablefunc:installcheck", "contrib/passwordcheck:installcheck", + "contrib/pg_aux_catalog:installcheck", "contrib/pg_buffercache:installcheck", "contrib/sslinfo:installcheck"] }, diff --git a/contrib/Makefile b/contrib/Makefile index b14600e3557..01315b1f6f8 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -34,6 +34,7 @@ SUBDIRS = \ old_snapshot \ pageinspect \ passwordcheck \ + pg_aux_catalog \ postgres_fdw \ pg_buffercache \ pg_freespacemap \ diff --git a/contrib/pg_aux_catalog/.gitignore b/contrib/pg_aux_catalog/.gitignore new file mode 100644 index 00000000000..c4ec060cefb --- /dev/null +++ b/contrib/pg_aux_catalog/.gitignore @@ -0,0 +1,6 @@ +# Generated test output +/log/ +/results/ +/tmp_check/ +/isolation2/results/ +/isolation2/output_iso/ diff --git a/contrib/pg_aux_catalog/Makefile b/contrib/pg_aux_catalog/Makefile new file mode 100644 index 00000000000..439ecf31a76 --- /dev/null +++ b/contrib/pg_aux_catalog/Makefile @@ -0,0 +1,37 @@ +# contrib/pg_aux_catalog/Makefile + +MODULE_big = pg_aux_catalog +OBJS = \ + $(WIN32RES) \ + pg_aux_catalog.o + +EXTENSION = pg_aux_catalog +DATA = pg_aux_catalog--1.0.sql + +PGFILEDESC = "pg_aux_catalog - auxiliary catalog management" + +REGRESS = pg_aux_catalog +REGRESS_OPTS = --init-file=$(top_srcdir)/src/test/regress/init_file + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/pg_aux_catalog +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif + +# Multi-session resource-group permission tests, run with the GPDB isolation2 +# harness. Requires a running cluster with resource groups enabled +# (gp_resource_manager=group); see installcheck-resgroup in +# src/test/isolation2. Not part of the default "installcheck". +installcheck-isolation2: install + $(pg_isolation2_regress_installcheck) \ + --init-file=$(top_builddir)/src/test/regress/init_file \ + --inputdir=$(srcdir)/isolation2 \ + --outputdir=isolation2 \ + --schedule=$(srcdir)/isolation2/isolation2_schedule + diff --git a/contrib/pg_aux_catalog/README.md b/contrib/pg_aux_catalog/README.md new file mode 100644 index 00000000000..a89535838ee --- /dev/null +++ b/contrib/pg_aux_catalog/README.md @@ -0,0 +1,71 @@ +# pg_aux_catalog + +Auxiliary catalog management for Apache Cloudberry. + +This extension provisions the **`mdb_admin`** privilege role, which lets a +non-superuser manage resource groups in managed-service deployments where the +client is never given superuser. + +## Background + +In Greenplum/Cloudberry only a superuser may `CREATE`/`ALTER`/`DROP` resource +groups or move a running query between groups with `pg_resgroup_move_query()`. +The server gates those four entry points on membership of `mdb_admin`, +identified by a **fixed OID (8067)** rather than by name, so the privilege is +recognised reliably across the coordinator and all segments. + +A fixed OID cannot be obtained from a plain `CREATE ROLE` (that assigns an +ordinary OID). This extension provides the one supported way to create the +role at OID 8067. + +## Functions + +### `pg_create_mdb_admin_role() returns oid` + +Creates the `mdb_admin` role with its fixed OID (8067). +Returns the OID of the created role (8067). Errors if a role with that OID or +the name `mdb_admin` already exists. The OID assignment is dispatched to the +segments, so the role has the same OID cluster-wide. + +## Usage + +```sql +CREATE EXTENSION pg_aux_catalog; + +-- Provision the role (the control plane does this once per cluster). +SELECT pg_create_mdb_admin_role(); + +-- Grant the capability to a tenant admin. +GRANT mdb_admin TO cloud_admin; + +-- cloud_admin can now manage resource groups without superuser: +SET ROLE cloud_admin; +CREATE RESOURCE GROUP rg_tenant WITH (concurrency = 4, cpu_max_percent = 20); +ALTER RESOURCE GROUP rg_tenant SET cpu_max_percent 30; +DROP RESOURCE GROUP rg_tenant; +``` + +`admin_group` and `system_group` remain superuser-only for `ALTER`/`DROP`: +they are infrastructure, not user-tunable groups. + +## Building and testing + +```sh +make -C contrib/pg_aux_catalog install +make -C contrib/pg_aux_catalog installcheck +``` + +`installcheck` runs a single-session regression test (role creation and the +resource-group permission gate). A multi-session isolation2 test covering the +dispatched / cross-session behaviour lives under `isolation2/` and is run +separately, against a cluster with resource groups enabled +(`gp_resource_manager=group`): + +```sh +make -C contrib/pg_aux_catalog installcheck-isolation2 +``` + +## Credits + +Based on [pg-sharding/cpg](https://github.com/pg-sharding/cpg) commit +`7b8c912`. Some tests are adapted from open-gpdb/gpdb commit `3ac99962ad2`. diff --git a/contrib/pg_aux_catalog/expected/pg_aux_catalog.out b/contrib/pg_aux_catalog/expected/pg_aux_catalog.out new file mode 100644 index 00000000000..32e52b9728d --- /dev/null +++ b/contrib/pg_aux_catalog/expected/pg_aux_catalog.out @@ -0,0 +1,46 @@ +-- Tests for the pg_aux_catalog extension: creation of the fixed-OID +-- mdb_admin role and the resource-group permission gate it enables. +CREATE EXTENSION pg_aux_catalog; +-- --------------------------------------------------------------------- +-- pg_create_mdb_admin_role() creates the mdb_admin role with its fixed OID. +-- --------------------------------------------------------------------- +SELECT pg_create_mdb_admin_role() AS mdb_admin_oid; + mdb_admin_oid +--------------- + 8067 +(1 row) + +-- The role exists with the fixed OID and is a non-login, non-superuser, +-- connection-limited role. +SELECT oid = 8067 AS has_fixed_oid, rolcanlogin, rolsuper, + rolcreaterole, rolcreatedb, rolconnlimit + FROM pg_authid WHERE rolname = 'mdb_admin'; + has_fixed_oid | rolcanlogin | rolsuper | rolcreaterole | rolcreatedb | rolconnlimit +---------------+-------------+----------+---------------+-------------+-------------- + t | f | f | f | f | 0 +(1 row) + +-- Creating it a second time is rejected. +SELECT pg_create_mdb_admin_role(); +ERROR: role with OID 8067 already exists +-- --------------------------------------------------------------------- +-- Resource-group permission gate: a role that is not a member of mdb_admin +-- is rejected on every entry point. These checks run before the "resource +-- group is enabled" check, so they are deterministic regardless of the +-- resource manager in use. +-- --------------------------------------------------------------------- +CREATE ROLE regress_rg_noadmin; +SET ROLE regress_rg_noadmin; +CREATE RESOURCE GROUP regress_rg_x WITH (concurrency=1, cpu_max_percent=5); +ERROR: must be mdb_admin to create resource groups +ALTER RESOURCE GROUP regress_rg_x SET cpu_max_percent 6; +ERROR: must be mdb_admin to alter resource groups +DROP RESOURCE GROUP regress_rg_x; +ERROR: must be mdb_admin to drop resource groups +RESET ROLE; +DROP ROLE regress_rg_noadmin; +-- --------------------------------------------------------------------- +-- Cleanup. +-- --------------------------------------------------------------------- +DROP ROLE mdb_admin; +DROP EXTENSION pg_aux_catalog; diff --git a/contrib/pg_aux_catalog/isolation2/expected/resgroup_mdb_admin.out b/contrib/pg_aux_catalog/isolation2/expected/resgroup_mdb_admin.out new file mode 100644 index 00000000000..c8fd232faea --- /dev/null +++ b/contrib/pg_aux_catalog/isolation2/expected/resgroup_mdb_admin.out @@ -0,0 +1,123 @@ +-- Tests permission checks for the mdb_admin role with +-- resource groups enabled. + +-- start_matchsubs +-- m/ERROR: cannot find process: \d+/ +-- s/\d+/XXX/g +-- end_matchsubs + +DROP ROLE IF EXISTS role_rg_admin; +DROP +DROP ROLE IF EXISTS role_rg_noadmin; +DROP +DROP ROLE IF EXISTS mdb_admin; +DROP +-- start_ignore +DROP RESOURCE GROUP rg_perm_admin1; +DROP RESOURCE GROUP rg_perm_admin2; +DROP RESOURCE GROUP rg_perm_revoke1; +DROP RESOURCE GROUP rg_perm_revoke2; +DROP RESOURCE GROUP rg_perm_test; +-- end_ignore + +-- --------------------------------------------------------------------- +-- Setup. The mdb_admin role is not predefined in the catalog; it is +-- created here the same way the control plane provisions it at runtime. +-- --------------------------------------------------------------------- +CREATE RESOURCE GROUP rg_perm_test WITH (concurrency=2, cpu_max_percent=10); +CREATE +CREATE ROLE mdb_admin; +CREATE +CREATE ROLE role_rg_admin RESOURCE GROUP rg_perm_test; +CREATE +CREATE ROLE role_rg_noadmin RESOURCE GROUP rg_perm_test; +CREATE +GRANT mdb_admin TO role_rg_admin; +GRANT + +-- --------------------------------------------------------------------- +-- 1. Member of mdb_admin can CREATE/ALTER/DROP resource groups +-- (statements are dispatched to segments). +-- --------------------------------------------------------------------- +1: SET ROLE role_rg_admin; +SET +1: CREATE RESOURCE GROUP rg_perm_admin1 WITH (concurrency=1, cpu_max_percent=5); +CREATE +1: ALTER RESOURCE GROUP rg_perm_admin1 SET cpu_max_percent 6; +ALTER +1: DROP RESOURCE GROUP rg_perm_admin1; +DROP + +-- 2. Even a member cannot ALTER or DROP the system admin_group. +1: ALTER RESOURCE GROUP admin_group SET cpu_max_percent 99; +ERROR: must be superuser to alter resource group "admin_group" +1: DROP RESOURCE GROUP admin_group; +ERROR: must be superuser to drop resource group "admin_group" +1q: ... + +-- --------------------------------------------------------------------- +-- 3. A non-member is rejected on every entry point. +-- --------------------------------------------------------------------- +2: SET ROLE role_rg_noadmin; +SET +2: CREATE RESOURCE GROUP rg_perm_admin2 WITH (concurrency=1, cpu_max_percent=5); +ERROR: must be mdb_admin to create resource groups +2: ALTER RESOURCE GROUP rg_perm_test SET cpu_max_percent 7; +ERROR: must be mdb_admin to alter resource groups +2: DROP RESOURCE GROUP rg_perm_test; +ERROR: must be mdb_admin to drop resource groups +2q: ... + +-- --------------------------------------------------------------------- +-- 4. pg_resgroup_move_query() honours the same permission check. +-- The first call (non-member) must fail with "must be mdb_admin". +-- The second call (member) gets past the permission gate and +-- fails on the pid lookup (masked by start_matchsubs above). +-- --------------------------------------------------------------------- +3: SET ROLE role_rg_noadmin; +SET +3: SELECT pg_resgroup_move_query(999999999, 'admin_group'); +ERROR: must be mdb_admin to move query +3: RESET ROLE; +RESET +3: SET ROLE role_rg_admin; +SET +3: SELECT pg_resgroup_move_query(999999999, 'admin_group'); +ERROR: cannot find process: XXX +3q: ... + +-- --------------------------------------------------------------------- +-- 5. Cross-session REVOKE takes effect on the granted session's +-- next statement (the privilege is re-checked per command, not +-- cached at SET ROLE time). +-- --------------------------------------------------------------------- +4: SET ROLE role_rg_admin; +SET +4: CREATE RESOURCE GROUP rg_perm_revoke1 WITH (concurrency=1, cpu_max_percent=5); +CREATE +5: REVOKE mdb_admin FROM role_rg_admin; +REVOKE +4: CREATE RESOURCE GROUP rg_perm_revoke2 WITH (concurrency=1, cpu_max_percent=5); +ERROR: must be mdb_admin to create resource groups +4: DROP RESOURCE GROUP rg_perm_revoke1; +ERROR: must be mdb_admin to drop resource groups +4q: ... +5q: ... + +-- --------------------------------------------------------------------- +-- Cleanup. Roles must be dropped before the resource group they +-- reference, otherwise DROP RESOURCE GROUP fails with +-- "resource group is used by at least one role". +-- --------------------------------------------------------------------- +RESET ROLE; +RESET +DROP ROLE role_rg_admin; +DROP +DROP ROLE role_rg_noadmin; +DROP +DROP ROLE mdb_admin; +DROP +DROP RESOURCE GROUP rg_perm_revoke1; +DROP +DROP RESOURCE GROUP rg_perm_test; +DROP diff --git a/contrib/pg_aux_catalog/isolation2/isolation2_schedule b/contrib/pg_aux_catalog/isolation2/isolation2_schedule new file mode 100644 index 00000000000..73b2a8a95a5 --- /dev/null +++ b/contrib/pg_aux_catalog/isolation2/isolation2_schedule @@ -0,0 +1 @@ +test: resgroup_mdb_admin diff --git a/contrib/pg_aux_catalog/isolation2/sql/resgroup_mdb_admin.sql b/contrib/pg_aux_catalog/isolation2/sql/resgroup_mdb_admin.sql new file mode 100644 index 00000000000..1b7ea19fb3e --- /dev/null +++ b/contrib/pg_aux_catalog/isolation2/sql/resgroup_mdb_admin.sql @@ -0,0 +1,91 @@ +-- Tests permission checks for the mdb_admin role with +-- resource groups enabled. + +-- start_matchsubs +-- m/ERROR: cannot find process: \d+/ +-- s/\d+/XXX/g +-- end_matchsubs + +DROP ROLE IF EXISTS role_rg_admin; +DROP ROLE IF EXISTS role_rg_noadmin; +DROP ROLE IF EXISTS mdb_admin; +-- start_ignore +DROP RESOURCE GROUP rg_perm_admin1; +DROP RESOURCE GROUP rg_perm_admin2; +DROP RESOURCE GROUP rg_perm_revoke1; +DROP RESOURCE GROUP rg_perm_revoke2; +DROP RESOURCE GROUP rg_perm_test; +-- end_ignore + +-- --------------------------------------------------------------------- +-- Setup. mdb_admin is identified by its fixed OID, so it must be created +-- through contrib/pg_aux_catalog (a plain CREATE ROLE would assign a +-- different OID and the permission checks would not recognise its members). +-- --------------------------------------------------------------------- +CREATE RESOURCE GROUP rg_perm_test WITH (concurrency=2, cpu_max_percent=10); +CREATE EXTENSION IF NOT EXISTS pg_aux_catalog; +SELECT pg_create_mdb_admin_role(); +CREATE ROLE role_rg_admin RESOURCE GROUP rg_perm_test; +CREATE ROLE role_rg_noadmin RESOURCE GROUP rg_perm_test; +GRANT mdb_admin TO role_rg_admin; + +-- --------------------------------------------------------------------- +-- 1. Member of mdb_admin can CREATE/ALTER/DROP resource groups +-- (statements are dispatched to segments). +-- --------------------------------------------------------------------- +1: SET ROLE role_rg_admin; +1: CREATE RESOURCE GROUP rg_perm_admin1 WITH (concurrency=1, cpu_max_percent=5); +1: ALTER RESOURCE GROUP rg_perm_admin1 SET cpu_max_percent 6; +1: DROP RESOURCE GROUP rg_perm_admin1; + +-- 2. Even a member cannot ALTER or DROP the system admin_group. +1: ALTER RESOURCE GROUP admin_group SET cpu_max_percent 99; +1: DROP RESOURCE GROUP admin_group; +1q: + +-- --------------------------------------------------------------------- +-- 3. A non-member is rejected on every entry point. +-- --------------------------------------------------------------------- +2: SET ROLE role_rg_noadmin; +2: CREATE RESOURCE GROUP rg_perm_admin2 WITH (concurrency=1, cpu_max_percent=5); +2: ALTER RESOURCE GROUP rg_perm_test SET cpu_max_percent 7; +2: DROP RESOURCE GROUP rg_perm_test; +2q: + +-- --------------------------------------------------------------------- +-- 4. pg_resgroup_move_query() honours the same permission check. +-- The first call (non-member) must fail with "must be mdb_admin". +-- The second call (member) gets past the permission gate and +-- fails on the pid lookup (masked by start_matchsubs above). +-- --------------------------------------------------------------------- +3: SET ROLE role_rg_noadmin; +3: SELECT pg_resgroup_move_query(999999999, 'admin_group'); +3: RESET ROLE; +3: SET ROLE role_rg_admin; +3: SELECT pg_resgroup_move_query(999999999, 'admin_group'); +3q: + +-- --------------------------------------------------------------------- +-- 5. Cross-session REVOKE takes effect on the granted session's +-- next statement (the privilege is re-checked per command, not +-- cached at SET ROLE time). +-- --------------------------------------------------------------------- +4: SET ROLE role_rg_admin; +4: CREATE RESOURCE GROUP rg_perm_revoke1 WITH (concurrency=1, cpu_max_percent=5); +5: REVOKE mdb_admin FROM role_rg_admin; +4: CREATE RESOURCE GROUP rg_perm_revoke2 WITH (concurrency=1, cpu_max_percent=5); +4: DROP RESOURCE GROUP rg_perm_revoke1; +4q: +5q: + +-- --------------------------------------------------------------------- +-- Cleanup. Roles must be dropped before the resource group they +-- reference, otherwise DROP RESOURCE GROUP fails with +-- "resource group is used by at least one role". +-- --------------------------------------------------------------------- +RESET ROLE; +DROP ROLE role_rg_admin; +DROP ROLE role_rg_noadmin; +DROP ROLE mdb_admin; +DROP RESOURCE GROUP rg_perm_revoke1; +DROP RESOURCE GROUP rg_perm_test; diff --git a/contrib/pg_aux_catalog/pg_aux_catalog--1.0.sql b/contrib/pg_aux_catalog/pg_aux_catalog--1.0.sql new file mode 100644 index 00000000000..a1e1b00fcce --- /dev/null +++ b/contrib/pg_aux_catalog/pg_aux_catalog--1.0.sql @@ -0,0 +1,10 @@ +/* contrib/pg_aux_catalog/pg_aux_catalog--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION pg_aux_catalog" to load this file. \quit + +-- Create the mdb_admin role with fixed OID 8067 +CREATE FUNCTION pg_create_mdb_admin_role() +RETURNS OID +AS 'MODULE_PATHNAME', 'pg_create_mdb_admin_role' +LANGUAGE C PARALLEL SAFE STRICT; diff --git a/contrib/pg_aux_catalog/pg_aux_catalog.c b/contrib/pg_aux_catalog/pg_aux_catalog.c new file mode 100644 index 00000000000..91685561cca --- /dev/null +++ b/contrib/pg_aux_catalog/pg_aux_catalog.c @@ -0,0 +1,88 @@ +/*------------------------------------------------------------------------- + * + * pg_aux_catalog.c + * Extension for auxiliary catalog management + * + * contrib/pg_aux_catalog/pg_aux_catalog.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "catalog/oid_dispatch.h" +#include "catalog/pg_authid.h" +#include "commands/user.h" +#include "fmgr.h" +#include "miscadmin.h" +#include "nodes/makefuncs.h" +#include "nodes/parsenodes.h" +#include "utils/acl.h" +#include "utils/builtins.h" +#include "utils/syscache.h" + +PG_MODULE_MAGIC; + +/* Name of the mdb_admin role; its OID is MDB_ADMIN_ROLEID (see acl.h). */ +#define MDB_ADMIN_ROLE_NAME "mdb_admin" + +PG_FUNCTION_INFO_V1(pg_create_mdb_admin_role); + +/* + * Create the mdb_admin role with its fixed OID (MDB_ADMIN_ROLEID, 8067). + * + * The core privilege checks identify mdb_admin by this fixed OID (see acl.c + * and resgroupcmds.c), so the role must always be created with it. On a + * Cloudberry cluster the OID is dispatched to the segments so the role ends + * up with the same OID everywhere. Returns the new role's OID. + */ +Datum +pg_create_mdb_admin_role(PG_FUNCTION_ARGS) +{ + CreateRoleStmt stmt; + List *options = NIL; + Oid roleid; + + /* + * Only a superuser may establish the mdb_admin privilege role. Otherwise + * a CREATEROLE user could drop mdb_admin and re-create it (CreateRole only + * requires CREATEROLE), taking ownership of the fixed-OID role and + * granting the capability to itself. + */ + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to create the mdb_admin role"))); + + /* Check if a role with the fixed OID already exists. */ + if (SearchSysCacheExists1(AUTHOID, ObjectIdGetDatum(MDB_ADMIN_ROLEID))) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("role with OID %u already exists", MDB_ADMIN_ROLEID))); + + /* Check if a role named "mdb_admin" already exists. */ + if (SearchSysCacheExists1(AUTHNAME, CStringGetDatum(MDB_ADMIN_ROLE_NAME))) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("role \"%s\" already exists", MDB_ADMIN_ROLE_NAME))); + + /* Build options for CreateRole: connection limit = 0. */ + options = list_make1(makeDefElem("connectionlimit", + (Node *) makeInteger(0), -1)); + + /* Prepare the CreateRoleStmt. */ + memset(&stmt, 0, sizeof(stmt)); + stmt.type = T_CreateRoleStmt; + stmt.stmt_type = ROLESTMT_ROLE; + stmt.role = MDB_ADMIN_ROLE_NAME; + stmt.options = options; + + /* + * Request the fixed OID for the role. GetNewOidForAuthId() consumes and + * clears this override. + */ + next_aux_pg_authid_oid = MDB_ADMIN_ROLEID; + + roleid = CreateRole(NULL, &stmt); + + PG_RETURN_OID(roleid); +} diff --git a/contrib/pg_aux_catalog/pg_aux_catalog.control b/contrib/pg_aux_catalog/pg_aux_catalog.control new file mode 100644 index 00000000000..aff4205f7eb --- /dev/null +++ b/contrib/pg_aux_catalog/pg_aux_catalog.control @@ -0,0 +1,5 @@ +# pg_aux_catalog extension +comment = 'auxiliary catalog management (mdb_admin role creation)' +default_version = '1.0' +module_pathname = '$libdir/pg_aux_catalog' +relocatable = false diff --git a/contrib/pg_aux_catalog/sql/pg_aux_catalog.sql b/contrib/pg_aux_catalog/sql/pg_aux_catalog.sql new file mode 100644 index 00000000000..aa16ffe4e13 --- /dev/null +++ b/contrib/pg_aux_catalog/sql/pg_aux_catalog.sql @@ -0,0 +1,38 @@ +-- Tests for the pg_aux_catalog extension: creation of the fixed-OID +-- mdb_admin role and the resource-group permission gate it enables. + +CREATE EXTENSION pg_aux_catalog; + +-- --------------------------------------------------------------------- +-- pg_create_mdb_admin_role() creates the mdb_admin role with its fixed OID. +-- --------------------------------------------------------------------- +SELECT pg_create_mdb_admin_role() AS mdb_admin_oid; + +-- The role exists with the fixed OID and is a non-login, non-superuser, +-- connection-limited role. +SELECT oid = 8067 AS has_fixed_oid, rolcanlogin, rolsuper, + rolcreaterole, rolcreatedb, rolconnlimit + FROM pg_authid WHERE rolname = 'mdb_admin'; + +-- Creating it a second time is rejected. +SELECT pg_create_mdb_admin_role(); + +-- --------------------------------------------------------------------- +-- Resource-group permission gate: a role that is not a member of mdb_admin +-- is rejected on every entry point. These checks run before the "resource +-- group is enabled" check, so they are deterministic regardless of the +-- resource manager in use. +-- --------------------------------------------------------------------- +CREATE ROLE regress_rg_noadmin; +SET ROLE regress_rg_noadmin; +CREATE RESOURCE GROUP regress_rg_x WITH (concurrency=1, cpu_max_percent=5); +ALTER RESOURCE GROUP regress_rg_x SET cpu_max_percent 6; +DROP RESOURCE GROUP regress_rg_x; +RESET ROLE; +DROP ROLE regress_rg_noadmin; + +-- --------------------------------------------------------------------- +-- Cleanup. +-- --------------------------------------------------------------------- +DROP ROLE mdb_admin; +DROP EXTENSION pg_aux_catalog; diff --git a/pom.xml b/pom.xml index 0e000093399..6eaa095fa37 100644 --- a/pom.xml +++ b/pom.xml @@ -352,6 +352,9 @@ code or new licensing patterns. contrib/indexscan/indexscan.c contrib/indexscan/indexscan.sql.in + contrib/pg_aux_catalog/pg_aux_catalog.c + contrib/pg_aux_catalog/isolation2/isolation2_schedule + contrib/file_fdw/init_file contrib/file_fdw/data/** diff --git a/src/backend/catalog/oid_dispatch.c b/src/backend/catalog/oid_dispatch.c index 6f39a07857e..eaa2e099876 100644 --- a/src/backend/catalog/oid_dispatch.c +++ b/src/backend/catalog/oid_dispatch.c @@ -156,6 +156,18 @@ static MemoryContext oids_context = NULL; static bool preserve_oids_on_commit = false; +/* + * OID to assign to the next auxiliary pg_authid role created through + * GetNewOidForAuthId(), or InvalidOid for the normal allocation path. + * + * This is the GPDB analogue of upstream PostgreSQL's + * binary_upgrade_next_pg_authid_oid (which is disabled here in favour of the + * generic OID pre-assignment machinery). It lets contrib/pg_aux_catalog + * create roles such as mdb_admin with a fixed, well-known OID. It is reset + * to InvalidOid as soon as it is consumed. + */ +Oid next_aux_pg_authid_oid = InvalidOid; + /* * These will be used by the schema restoration process during binary upgrade, * so any new object must not use any Oid in this structure or else there will @@ -423,6 +435,7 @@ GetNewOrPreassignedOid(Relation relation, Oid indexId, AttrNumber oidcolumn, OidAssignment *searchkey) { Oid oid; + Oid forcedOid = searchkey->oid; searchkey->catalog = RelationGetRelid(relation); @@ -461,8 +474,18 @@ GetNewOrPreassignedOid(Relation relation, Oid indexId, AttrNumber oidcolumn, { MemoryContext oldcontext; - /* Assign a new oid, and memorize it in the list of OIDs to dispatch */ - oid = GetNewOidWithIndex(relation, indexId, oidcolumn); + /* + * Assign a new oid, and memorize it in the list of OIDs to dispatch. + * + * A caller may request a fixed, well-known OID by passing it in + * searchkey->oid (e.g. the mdb_admin auxiliary role, see + * GetNewOidForAuthId()). In that case use the requested OID instead + * of allocating a fresh one, but still record it for dispatch so the + * QEs end up with the same OID. + */ + oid = OidIsValid(forcedOid) + ? forcedOid + : GetNewOidWithIndex(relation, indexId, oidcolumn); oldcontext = MemoryContextSwitchTo(get_oids_context()); searchkey->oid = oid; @@ -479,7 +502,9 @@ GetNewOrPreassignedOid(Relation relation, Oid indexId, AttrNumber oidcolumn, } else { - oid = GetNewOidWithIndex(relation, indexId, oidcolumn); + oid = OidIsValid(forcedOid) + ? forcedOid + : GetNewOidWithIndex(relation, indexId, oidcolumn); } return oid; @@ -572,6 +597,25 @@ GetNewOidForAuthId(Relation relation, Oid indexId, AttrNumber oidcolumn, memset(&key, 0, sizeof(OidAssignment)); key.type = T_OidAssignment; key.objname = rolname; + + /* + * Allow auxiliary roles (such as mdb_admin, see contrib/pg_aux_catalog) + * to be created with a fixed, well-known OID. The OID is supplied through + * the next_aux_pg_authid_oid override, mirroring how upstream PostgreSQL + * assigns role OIDs during binary upgrade. We only honor it for OIDs in + * the auxiliary range to avoid clashing with normal OID allocation, and + * reset it immediately so it affects a single role only. + */ + if (OidIsValid(next_aux_pg_authid_oid)) + { + if (!IsAuxOid(next_aux_pg_authid_oid)) + elog(ERROR, "pre-assigned auxiliary role OID %u is out of the auxiliary OID range", + next_aux_pg_authid_oid); + + key.oid = next_aux_pg_authid_oid; + next_aux_pg_authid_oid = InvalidOid; + } + return GetNewOrPreassignedOid(relation, indexId, oidcolumn, &key); } diff --git a/src/backend/commands/resgroupcmds.c b/src/backend/commands/resgroupcmds.c index 384675edb7f..b746a3db49b 100644 --- a/src/backend/commands/resgroupcmds.c +++ b/src/backend/commands/resgroupcmds.c @@ -34,6 +34,7 @@ #include "commands/resgroupcmds.h" #include "miscadmin.h" #include "nodes/pg_list.h" +#include "utils/acl.h" #include "utils/builtins.h" #include "utils/datetime.h" #include "utils/fmgroids.h" @@ -103,11 +104,11 @@ CreateResourceGroup(CreateResourceGroupStmt *stmt) int nResGroups; MemoryContext oldContext; - /* Permission check - only superuser can create groups. */ - if (!superuser()) + /* Permission check - only superuser or mdb_admin can create groups. */ + if (!is_member_of_role(GetUserId(), MDB_ADMIN_ROLEID)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("must be superuser to create resource groups"))); + errmsg("must be mdb_admin to create resource groups"))); /* * Check for an illegal name ('none' is used to signify no group in ALTER ROLE). @@ -269,11 +270,11 @@ DropResourceGroup(DropResourceGroupStmt *stmt) Oid groupid; ResourceGroupCallbackContext *callbackCtx; - /* Permission check - only superuser can drop resource groups. */ - if (!superuser()) + /* Permission check - only superuser or mdb_admin can drop resource groups. */ + if (!is_member_of_role(GetUserId(), MDB_ADMIN_ROLEID)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("must be superuser to drop resource groups"))); + errmsg("must be mdb_admin to drop resource groups"))); /* * Check the pg_resgroup relation to be certain the resource group already @@ -302,6 +303,13 @@ DropResourceGroup(DropResourceGroupStmt *stmt) */ groupid = ((Form_pg_resgroup) GETSTRUCT(tuple))->oid; + /* Permission check - only superuser can drop the admin/system resource groups. */ + if (!superuser() && (groupid == ADMINRESGROUP_OID || groupid == SYSTEMRESGROUP_OID)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to drop resource group \"%s\"", + stmt->name))); + /* cannot DROP default resource groups */ if (groupid == DEFAULTRESGROUP_OID || groupid == ADMINRESGROUP_OID @@ -375,11 +383,24 @@ AlterResourceGroup(AlterResourceGroupStmt *stmt) ResourceGroupCallbackContext *callbackCtx; MemoryContext oldContext; - /* Permission check - only superuser can alter resource groups. */ - if (!superuser()) + /* Permission check - only mdb_admin can alter resource groups. */ + if (!is_member_of_role(GetUserId(), MDB_ADMIN_ROLEID)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be mdb_admin to alter resource groups"))); + + /* + * Check the pg_resgroup relation to be certain the resource group already + * exists. + */ + groupid = get_resgroup_oid(stmt->name, false); + + /* Permission check - only superuser can alter the admin/system resource groups. */ + if (!superuser() && (groupid == ADMINRESGROUP_OID || groupid == SYSTEMRESGROUP_OID)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("must be superuser to alter resource groups"))); + errmsg("must be superuser to alter resource group \"%s\"", + stmt->name))); /* Currently we only support to ALTER one limit at one time */ Assert(list_length(stmt->options) == 1); @@ -406,12 +427,6 @@ AlterResourceGroup(AlterResourceGroupStmt *stmt) checkResgroupCapLimit(limitType, value); } - /* - * Check the pg_resgroup relation to be certain the resource group already - * exists. - */ - groupid = get_resgroup_oid(stmt->name, false); - if (limitType == RESGROUP_LIMIT_TYPE_CONCURRENCY && value == 0 && groupid == ADMINRESGROUP_OID) @@ -500,7 +515,7 @@ AlterResourceGroup(AlterResourceGroupStmt *stmt) RESGROUP_DEFAULT_CPU_WEIGHT, ""); updateResgroupCapabilityEntry(pg_resgroupcapability_rel, - groupid, RESGROUP_LIMIT_TYPE_CPUSET, + groupid, RESGROUP_LIMIT_TYPE_CPUSET, 0, caps.cpuset); } else if (limitType == RESGROUP_LIMIT_TYPE_CPU) @@ -1007,7 +1022,7 @@ parseStmtOptions(CreateResourceGroupStmt *stmt, ResGroupCaps *caps) else mask |= 1 << type; - if (type == RESGROUP_LIMIT_TYPE_CPUSET) + if (type == RESGROUP_LIMIT_TYPE_CPUSET) { const char *cpuset = defGetString(defel); strlcpy(caps->cpuset, cpuset, sizeof(caps->cpuset)); @@ -1611,7 +1626,7 @@ checkCpuSetByRole(const char *cpuset) * ex: * cpuset = "1;4" * then we should assign '1' to corrdinator and '4' to segment - * + * * cpuset = "1" * assign '1' to both coordinator and segment */ diff --git a/src/backend/utils/resgroup/resgroup_helper.c b/src/backend/utils/resgroup/resgroup_helper.c index 00aaded168d..5acca0be5e7 100644 --- a/src/backend/utils/resgroup/resgroup_helper.c +++ b/src/backend/utils/resgroup/resgroup_helper.c @@ -21,6 +21,7 @@ #include "cdb/cdbvars.h" #include "commands/resgroupcmds.h" #include "storage/procarray.h" +#include "utils/acl.h" #include "utils/builtins.h" #include "utils/datetime.h" #include "utils/resgroup.h" @@ -464,10 +465,10 @@ pg_resgroup_move_query(PG_FUNCTION_ARGS) (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("resource group is not enabled")))); - if (!superuser()) + if (!is_member_of_role(GetUserId(), MDB_ADMIN_ROLEID)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - (errmsg("must be superuser to move query")))); + (errmsg("must be mdb_admin to move query")))); if (Gp_role == GP_ROLE_DISPATCH) { diff --git a/src/include/access/transam.h b/src/include/access/transam.h index 687799bec9f..a1bf330b762 100644 --- a/src/include/access/transam.h +++ b/src/include/access/transam.h @@ -208,6 +208,16 @@ FullTransactionIdAdvance(FullTransactionId *dest) #define FirstBinaryUpgradeReservedObjectId 9000 #define LastBinaryUpgradeReservedObjectId 9100 +/* + * Reserve a block of OIDs for auxiliary catalog objects (such as the + * mdb_admin role created by contrib/pg_aux_catalog). These need fixed, + * well-known OIDs that are stable across clusters, so they live in their own + * range below FirstBinaryUpgradeReservedObjectId. + */ +#define FirstAuxObjectId 8000 +#define LastAuxObjectId 9000 +#define IsAuxOid(oid) ((oid) >= FirstAuxObjectId && (oid) < LastAuxObjectId) + /* * VariableCache is a data structure in shared memory that is used to track * OID and XID assignment state. For largely historical reasons, there is diff --git a/src/include/catalog/oid_dispatch.h b/src/include/catalog/oid_dispatch.h index fbb7a14f59e..d9c543bee5d 100644 --- a/src/include/catalog/oid_dispatch.h +++ b/src/include/catalog/oid_dispatch.h @@ -16,6 +16,13 @@ #include "utils/relcache.h" #include "access/htup.h" +/* + * OID to assign to the next auxiliary pg_authid role created through + * GetNewOidForAuthId(), or InvalidOid for normal allocation. Set by + * contrib/pg_aux_catalog to create roles such as mdb_admin with a fixed OID. + */ +extern PGDLLIMPORT Oid next_aux_pg_authid_oid; + /* Functions used in master */ extern List *GetAssignedOidsForDispatch(void); diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index 49068f04b2f..e3949dd8769 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -213,6 +213,14 @@ extern bool is_member_of_role_nosuper(Oid member, Oid role); extern bool is_admin_of_role(Oid member, Oid role); // -- non-upstream patch begin +/* + * Fixed, well-known OID of the mdb_admin role. The role is created by + * contrib/pg_aux_catalog (pg_create_mdb_admin_role()) with this OID, and the + * resource-group permission checks identify it by OID. It lives in the + * auxiliary OID range (see IsAuxOid()). + */ +#define MDB_ADMIN_ROLEID 8067 + extern bool mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId); extern void check_mdb_admin_is_member_of_role(Oid member, Oid role); From b6a83ac270c6c0e03e358296de267af5dbb97903 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 19 Jun 2026 12:22:35 +0000 Subject: [PATCH 11/14] Fix duplicate mdb_admin_allow_bypass_owner_checks definition in acl.c --- src/backend/utils/adt/acl.c | 42 ------------------------------------- 1 file changed, 42 deletions(-) diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index 1baf148f987..906480c5137 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -5129,48 +5129,6 @@ mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) // -- non-upstream patch end -// -- non-upstream patch begin -/* - * Is userId allowed to bypass ownership check - * and tranfer onwership to ownerId role? - */ -bool -mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) -{ - Oid mdb_admin_roleoid; - /* - * Never allow nobody to grant objects to - * superusers. - * This can result in various CVE. - * For paranoic reasons, check this even before - * membership of mdb_admin role. - */ - if (superuser_arg(ownerId)) { - return false; - } - - mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); - /* Is userId actually member of mdb admin? */ - if (!is_member_of_role(userId, mdb_admin_roleoid)) { - /* if no, disallow. */ - return false; - } - - /* - * Now, we need to check if ownerId - * is some dangerous role to trasfer membership to. - * - * For now, we check that ownerId does not have - * priviledge to execute server program or/and - * read/write server files, or/and pg read/write all data - */ - - /* All checks passed, hope will not be hacked here (again) */ - return !has_privs_of_unwanted_system_role(ownerId); -} - -// -- non-upstream patch end - /* * Is member a member of role (directly or indirectly)? * From f0dd39d6cf8968b26f7e45b883c00907e7a272e0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 19 Jun 2026 12:27:59 +0000 Subject: [PATCH 12/14] Address review comments: fix includes, configure options, pipefail, license headers, expected output --- .github/workflows/yezzey-ci.yaml | 4 +- configure.ac | 14 +++ .../expected/resgroup_mdb_admin.out | 103 +++++++++++++++++- .../isolation2/isolation2_schedule | 17 +++ contrib/pg_aux_catalog/pg_aux_catalog.c | 17 +++ gpcontrib/Makefile | 4 - pom.xml | 3 - src/backend/catalog/oid_dispatch.c | 1 + 8 files changed, 150 insertions(+), 13 deletions(-) diff --git a/.github/workflows/yezzey-ci.yaml b/.github/workflows/yezzey-ci.yaml index 1d4402b0ad1..9913a6196c7 100644 --- a/.github/workflows/yezzey-ci.yaml +++ b/.github/workflows/yezzey-ci.yaml @@ -128,7 +128,7 @@ jobs: - name: Install MinIO Client (mc) run: | - set -ex pipefail + set -exo pipefail # Download mc for Linux (amd64) curl -fsSL -o mc https://dl.min.io/client/mc/release/linux-amd64/mc chmod +x mc @@ -136,7 +136,7 @@ jobs: - name: Configure MinIO service run: | - set -ex pipefail + set -exo pipefail # Add the MinIO service as an "alias" in mc (name it "minio-ci") mc alias set minio-ci http://minio:9000 some_key some_key diff --git a/configure.ac b/configure.ac index 89876e69d4f..308e0872f07 100644 --- a/configure.ac +++ b/configure.ac @@ -1380,6 +1380,20 @@ PGAC_ARG_BOOL(with, yezzey, no, [build with Yezzey extension]) AC_SUBST(with_yezzey) +# +# diskquota +# +PGAC_ARG_BOOL(with, diskquota, yes, + [build with diskquota extension]) +AC_SUBST(with_diskquota) + +# +# gp_stats_collector +# +PGAC_ARG_BOOL(with, gp_stats_collector, yes, + [build with gp_stats_collector extension]) +AC_SUBST(with_gp_stats_collector) + # # Realtime library # diff --git a/contrib/pg_aux_catalog/isolation2/expected/resgroup_mdb_admin.out b/contrib/pg_aux_catalog/isolation2/expected/resgroup_mdb_admin.out index c8fd232faea..7d8ed663485 100644 --- a/contrib/pg_aux_catalog/isolation2/expected/resgroup_mdb_admin.out +++ b/contrib/pg_aux_catalog/isolation2/expected/resgroup_mdb_admin.out @@ -21,13 +21,20 @@ DROP RESOURCE GROUP rg_perm_test; -- end_ignore -- --------------------------------------------------------------------- --- Setup. The mdb_admin role is not predefined in the catalog; it is --- created here the same way the control plane provisions it at runtime. +-- Setup. mdb_admin is identified by its fixed OID, so it must be created +-- through contrib/pg_aux_catalog (a plain CREATE ROLE would assign a +-- different OID and the permission checks would not recognise its members). -- --------------------------------------------------------------------- CREATE RESOURCE GROUP rg_perm_test WITH (concurrency=2, cpu_max_percent=10); CREATE -CREATE ROLE mdb_admin; -CREATE +CREATE EXTENSION IF NOT EXISTS pg_aux_catalog; +CREATE EXTENSION +SELECT pg_create_mdb_admin_role(); + pg_create_mdb_admin_role +-------------------------- + 8067 +(1 row) + CREATE ROLE role_rg_admin RESOURCE GROUP rg_perm_test; CREATE CREATE ROLE role_rg_noadmin RESOURCE GROUP rg_perm_test; @@ -121,3 +128,91 @@ DROP RESOURCE GROUP rg_perm_revoke1; DROP DROP RESOURCE GROUP rg_perm_test; DROP + + +-- --------------------------------------------------------------------- +-- 1. Member of mdb_admin can CREATE/ALTER/DROP resource groups +-- (statements are dispatched to segments). +-- --------------------------------------------------------------------- +1: SET ROLE role_rg_admin; +SET +1: CREATE RESOURCE GROUP rg_perm_admin1 WITH (concurrency=1, cpu_max_percent=5); +CREATE +1: ALTER RESOURCE GROUP rg_perm_admin1 SET cpu_max_percent 6; +ALTER +1: DROP RESOURCE GROUP rg_perm_admin1; +DROP + +-- 2. Even a member cannot ALTER or DROP the system admin_group. +1: ALTER RESOURCE GROUP admin_group SET cpu_max_percent 99; +ERROR: must be superuser to alter resource group "admin_group" +1: DROP RESOURCE GROUP admin_group; +ERROR: must be superuser to drop resource group "admin_group" +1q: ... + +-- --------------------------------------------------------------------- +-- 3. A non-member is rejected on every entry point. +-- --------------------------------------------------------------------- +2: SET ROLE role_rg_noadmin; +SET +2: CREATE RESOURCE GROUP rg_perm_admin2 WITH (concurrency=1, cpu_max_percent=5); +ERROR: must be mdb_admin to create resource groups +2: ALTER RESOURCE GROUP rg_perm_test SET cpu_max_percent 7; +ERROR: must be mdb_admin to alter resource groups +2: DROP RESOURCE GROUP rg_perm_test; +ERROR: must be mdb_admin to drop resource groups +2q: ... + +-- --------------------------------------------------------------------- +-- 4. pg_resgroup_move_query() honours the same permission check. +-- The first call (non-member) must fail with "must be mdb_admin". +-- The second call (member) gets past the permission gate and +-- fails on the pid lookup (masked by start_matchsubs above). +-- --------------------------------------------------------------------- +3: SET ROLE role_rg_noadmin; +SET +3: SELECT pg_resgroup_move_query(999999999, 'admin_group'); +ERROR: must be mdb_admin to move query +3: RESET ROLE; +RESET +3: SET ROLE role_rg_admin; +SET +3: SELECT pg_resgroup_move_query(999999999, 'admin_group'); +ERROR: cannot find process: XXX +3q: ... + +-- --------------------------------------------------------------------- +-- 5. Cross-session REVOKE takes effect on the granted session's +-- next statement (the privilege is re-checked per command, not +-- cached at SET ROLE time). +-- --------------------------------------------------------------------- +4: SET ROLE role_rg_admin; +SET +4: CREATE RESOURCE GROUP rg_perm_revoke1 WITH (concurrency=1, cpu_max_percent=5); +CREATE +5: REVOKE mdb_admin FROM role_rg_admin; +REVOKE +4: CREATE RESOURCE GROUP rg_perm_revoke2 WITH (concurrency=1, cpu_max_percent=5); +ERROR: must be mdb_admin to create resource groups +4: DROP RESOURCE GROUP rg_perm_revoke1; +ERROR: must be mdb_admin to drop resource groups +4q: ... +5q: ... + +-- --------------------------------------------------------------------- +-- Cleanup. Roles must be dropped before the resource group they +-- reference, otherwise DROP RESOURCE GROUP fails with +-- "resource group is used by at least one role". +-- --------------------------------------------------------------------- +RESET ROLE; +RESET +DROP ROLE role_rg_admin; +DROP +DROP ROLE role_rg_noadmin; +DROP +DROP ROLE mdb_admin; +DROP +DROP RESOURCE GROUP rg_perm_revoke1; +DROP +DROP RESOURCE GROUP rg_perm_test; +DROP diff --git a/contrib/pg_aux_catalog/isolation2/isolation2_schedule b/contrib/pg_aux_catalog/isolation2/isolation2_schedule index 73b2a8a95a5..cb32fe3e768 100644 --- a/contrib/pg_aux_catalog/isolation2/isolation2_schedule +++ b/contrib/pg_aux_catalog/isolation2/isolation2_schedule @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + test: resgroup_mdb_admin diff --git a/contrib/pg_aux_catalog/pg_aux_catalog.c b/contrib/pg_aux_catalog/pg_aux_catalog.c index 91685561cca..958bd2f55be 100644 --- a/contrib/pg_aux_catalog/pg_aux_catalog.c +++ b/contrib/pg_aux_catalog/pg_aux_catalog.c @@ -5,6 +5,23 @@ * * contrib/pg_aux_catalog/pg_aux_catalog.c * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * *------------------------------------------------------------------------- */ #include "postgres.h" diff --git a/gpcontrib/Makefile b/gpcontrib/Makefile index af9862530d6..c62855d3089 100644 --- a/gpcontrib/Makefile +++ b/gpcontrib/Makefile @@ -35,10 +35,6 @@ else diskquota endif -ifeq "$(with_diskquota)" "yes" - recurse_targets += diskquota -endif - ifeq "$(with_gp_stats_collector)" "yes" recurse_targets += gp_stats_collector endif diff --git a/pom.xml b/pom.xml index 6eaa095fa37..0e000093399 100644 --- a/pom.xml +++ b/pom.xml @@ -352,9 +352,6 @@ code or new licensing patterns. contrib/indexscan/indexscan.c contrib/indexscan/indexscan.sql.in - contrib/pg_aux_catalog/pg_aux_catalog.c - contrib/pg_aux_catalog/isolation2/isolation2_schedule - contrib/file_fdw/init_file contrib/file_fdw/data/** diff --git a/src/backend/catalog/oid_dispatch.c b/src/backend/catalog/oid_dispatch.c index eaa2e099876..95afd25d0b0 100644 --- a/src/backend/catalog/oid_dispatch.c +++ b/src/backend/catalog/oid_dispatch.c @@ -129,6 +129,7 @@ #include "catalog/pg_type.h" #include "catalog/pg_user_mapping.h" #include "catalog/oid_dispatch.h" +#include "access/transam.h" #include "cdb/cdbvars.h" #include "executor/execdesc.h" #include "lib/rbtree.h" From 27090049453af94c1da132412cd8c18692f241e3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 19 Jun 2026 12:37:48 +0000 Subject: [PATCH 13/14] Fix Yezzey CI shell for pipefail steps --- .github/workflows/yezzey-ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/yezzey-ci.yaml b/.github/workflows/yezzey-ci.yaml index 9913a6196c7..82511c4f701 100644 --- a/.github/workflows/yezzey-ci.yaml +++ b/.github/workflows/yezzey-ci.yaml @@ -127,6 +127,7 @@ jobs: echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" - name: Install MinIO Client (mc) + shell: bash run: | set -exo pipefail # Download mc for Linux (amd64) @@ -135,6 +136,7 @@ jobs: sudo mv mc /usr/local/bin/mc # Make mc available system-wide - name: Configure MinIO service + shell: bash run: | set -exo pipefail # Add the MinIO service as an "alias" in mc (name it "minio-ci") @@ -331,4 +333,3 @@ jobs: echo "Timestamp: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" echo "Cloudberry Result: ${{ needs.test-cloudberry.result }}" - From 21cf26c7daccfae0ea514e0755c0eeeffda0a696 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 19 Jun 2026 14:26:11 +0000 Subject: [PATCH 14/14] Fix failing ic-deb-good-opt-on CI tests: privileges and misc expected outputs - privileges.out: update expected output for terminate_nothrow test to show 4 background processes (autovacuum launcher, dtx recovery process, logical replication launcher, login monitor) instead of 0 rows, matching Cloudberry's actual pg_terminate_backend behavior - output/misc.source: change expected value from 't' to 'f' for mdb_locale_enabled() since CI builds without --with-mdblocales (ENABLE_MDBLOCALES defaults to false in configure-cloudberry.sh) --- src/test/regress/expected/privileges.out | 10 +++++++--- src/test/regress/output/misc.source | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/test/regress/expected/privileges.out b/src/test/regress/expected/privileges.out index ee9f8fa1530..de323f54114 100644 --- a/src/test/regress/expected/privileges.out +++ b/src/test/regress/expected/privileges.out @@ -2007,9 +2007,13 @@ END$$; ALTER FUNCTION terminate_nothrow OWNER TO pg_signal_backend; SELECT backend_type FROM pg_stat_activity WHERE CASE WHEN COALESCE(usesysid, 10) = 10 THEN terminate_nothrow(pid) END; - backend_type --------------- -(0 rows) + backend_type +------------------------------ + autovacuum launcher + dtx recovery process + logical replication launcher + login monitor +(4 rows) ROLLBACK; -- test default ACLs diff --git a/src/test/regress/output/misc.source b/src/test/regress/output/misc.source index a0c63418446..f2f7c0dee32 100644 --- a/src/test/regress/output/misc.source +++ b/src/test/regress/output/misc.source @@ -613,6 +613,6 @@ CONTEXT: SQL function "equipment" during startup SELECT mdb_locale_enabled(); mdb_locale_enabled -------------------- - t + f (1 row)