go/vt/mysqlctl: run backup init sql after catchup, disable super read… #42776
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Backups - Manual - Upgrade Downgrade Testing | |
| on: | |
| push: | |
| branches: | |
| - "main" | |
| - "release-[0-9]+.[0-9]" | |
| tags: '**' | |
| pull_request: | |
| branches: '**' | |
| concurrency: | |
| group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing - Backups - Manual') | |
| cancel-in-progress: true | |
| permissions: read-all | |
| jobs: | |
| # This job usually execute in ± 20 minutes | |
| upgrade_downgrade_test_manual: | |
| timeout-minutes: 40 | |
| name: Run Upgrade Downgrade Test - Backups - Manual | |
| runs-on: oracle-vm-16cpu-64gb-x86-64 | |
| steps: | |
| - name: Harden the runner (Audit all outbound calls) | |
| uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 | |
| with: | |
| egress-policy: audit | |
| - name: Skip CI | |
| run: | | |
| if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then | |
| echo "skipping CI due to the 'Skip CI' label" | |
| exit 1 | |
| fi | |
| # Checkout to this build's commit | |
| - name: Checkout to commit's code | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| fetch-depth: 0 | |
| persist-credentials: 'false' | |
| - name: Check for changes in relevant files | |
| uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 | |
| id: changes | |
| with: | |
| token: '' | |
| filters: | | |
| end_to_end: | |
| - 'test/config.json' | |
| - 'go/**' | |
| - 'go/**/*.go' | |
| - 'test.go' | |
| - 'Makefile' | |
| - 'build.env' | |
| - 'go.sum' | |
| - 'go.mod' | |
| - 'proto/*.proto' | |
| - 'tools/**' | |
| - 'config/**' | |
| - 'bootstrap.sh' | |
| - '.github/workflows/upgrade_downgrade_test_backups_manual.yml' | |
| - 'examples/**' | |
| - name: Set output with latest release branch | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| id: output-previous-release-ref | |
| run: | | |
| previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) | |
| echo $previous_release_ref | |
| echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT | |
| - name: Tune the OS | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| uses: ./.github/actions/tune-os | |
| - name: Set up python | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| - name: Setup MySQL | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| uses: ./.github/actions/setup-mysql | |
| with: | |
| flavor: mysql-8.4 | |
| - name: Setup Percona Apt repository | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| uses: ./.github/actions/setup-percona-repo | |
| - name: Get base dependencies | |
| timeout-minutes: 10 | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| sudo DEBIAN_FRONTEND="noninteractive" apt-get update | |
| # Install everything else we need, and configure | |
| sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep | |
| sudo service etcd stop | |
| sudo percona-release enable tools | |
| sudo percona-release setup pxb-80 | |
| sudo apt-get update | |
| sudo apt-get install -y percona-xtrabackup-80 | |
| # Checkout to the last release of Vitess | |
| - name: Checkout to the other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} | |
| persist-credentials: 'false' | |
| - name: Set up Go | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 | |
| with: | |
| go-version-file: go.mod | |
| cache: false | |
| - name: Get dependencies for the last release | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| go mod download | |
| - name: Building last release's binaries | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 5 | |
| run: | | |
| source build.env | |
| NOVTADMINBUILD=1 make build | |
| mkdir -p /tmp/vitess-build-other/ | |
| cp -R bin /tmp/vitess-build-other/ | |
| rm -Rf bin/* | |
| # Checkout to this build's commit | |
| - name: Checkout to commit's code | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| persist-credentials: 'false' | |
| - name: Set up Go | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 | |
| with: | |
| go-version-file: go.mod | |
| cache: false | |
| - name: Get dependencies for this commit | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| go mod download | |
| - name: Run make minimaltools | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| make minimaltools | |
| - name: Building the binaries for this commit | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 5 | |
| run: | | |
| source build.env | |
| NOVTADMINBUILD=1 make build | |
| mkdir -p /tmp/vitess-build-current/ | |
| cp -R bin /tmp/vitess-build-current/ | |
| # We create a sharded Vitess cluster following the local example. | |
| # We also insert a few rows in our three tables. | |
| - name: Create the example Vitess cluster with all components using version N | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 10 | |
| run: | | |
| source build.env ; cd examples/backups | |
| ./start_cluster.sh | |
| # Taking a backup | |
| - name: Take a backup of all the shards | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 5 | |
| run: | | |
| source build.env ; cd examples/backups | |
| ./take_backups.sh | |
| # We insert more data in every table after the backup. | |
| # When we restore the backup made in the previous step, we do not want to see the rows we are about to insert now. | |
| # The initial number of rows for each table is: | |
| # - customer: 5 | |
| # - product: 2 | |
| # - corder: 5 | |
| # We shall see the same number of rows after restoring the backup. | |
| - name: Insert more data after the backup | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| source build.env ; cd examples; source ./common/env.sh | |
| echo "insert into customer(email) values('new_user_1@domain.com');" | mysql | |
| echo "insert into product(sku, description, price) values('SKU-1009', 'description', 89);" | mysql | |
| echo "insert into corder(customer_id, sku, price) values(1, 'SKU-1009', 100);" | mysql | |
| # Stop all the tablets and remove their data | |
| - name: Stop tablets | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 10 | |
| run: | | |
| source build.env ; cd examples/backups | |
| ./stop_tablets.sh | |
| # We downgrade: we use the version N-1 of vttablet | |
| - name: Downgrade - Swap binaries, use VTTablet N-1 | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| source build.env | |
| rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld | |
| cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet | |
| cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl | |
| cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld | |
| vttablet --version | |
| # Starting the tablets again, they will automatically start restoring the last backup. | |
| - name: Start new tablets and restore | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 10 | |
| run: | | |
| source build.env ; cd examples/backups | |
| ./restart_tablets.sh | |
| # give enough time to the tablets to restore the backup | |
| sleep 90 | |
| # Count the number of rows in each table to make sure the restoration is successful. | |
| - name: Assert the number of rows in every table | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| source build.env ; cd examples ; source ./common/env.sh | |
| echo "select count(sku) from product;" | mysql 2>&1| grep 2 | |
| echo "select count(email) from customer;" | mysql 2>&1| grep 5 | |
| echo "select count(sku) from corder;" | mysql 2>&1| grep 5 | |
| # We insert one more row in every table. | |
| - name: Insert more rows in the tables | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| source build.env ; cd examples ; source ./common/env.sh | |
| echo "insert into customer(email) values('new_user_2@domain.com');" | mysql | |
| echo "insert into product(sku, description, price) values('SKU-1011', 'description', 111);" | mysql | |
| echo "insert into corder(customer_id, sku, price) values(1, 'SKU-1011', 111);" | mysql | |
| # Taking a second backup of the cluster. | |
| - name: Take a second backup of all the shards | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 10 | |
| run: | | |
| source build.env ; cd examples/backups | |
| ./take_backups.sh | |
| # We upgrade: we swap binaries and use the version N of the tablet. | |
| - name: Upgrade - Swap binaries, use VTTablet N | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| source build.env | |
| rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld | |
| cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet | |
| cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl | |
| cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld | |
| vttablet --version | |
| # Starting the tablets again and restoring the previous backup. | |
| - name: Start new tablets and restore | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 10 | |
| run: | | |
| source build.env ; cd examples/backups | |
| ./upgrade_cluster.sh | |
| # We count the number of rows in every table to check that the restore step was successful. | |
| - name: Assert the number of rows in every table | |
| if: steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| source build.env ; cd examples ; source ./common/env.sh | |
| echo "select count(sku) from product;" | mysql 2>&1| grep 3 | |
| echo "select count(email) from customer;" | mysql 2>&1| grep 6 | |
| echo "select count(sku) from corder;" | mysql 2>&1| grep 6 | |
| - name: Stop the Vitess cluster | |
| if: steps.changes.outputs.end_to_end == 'true' && !cancelled() | |
| run: | | |
| source build.env ; cd examples/local | |
| ./401_teardown.sh || true |