diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 995bf5d..a4fa23d 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -37,5 +37,15 @@ 'skip-review', ], }, + { + description: 'help renovate fetch changelogs for apko', + matchDatasources: [ + 'custom.wolfi' + ], + matchDepNames: [ + 'apko' + ], + changelogUrl: 'https://github.com/chainguard-dev/apko', + }, ] } diff --git a/.gitignore b/.gitignore index e53ec5c..c992b5d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,9 @@ cover/ *.egg-info/ .coverage coverage.xml +melange.rsa +melange.rsa.pub +packages/ +.melange.yaml +.melange-source +*.tar diff --git a/.melangeignore b/.melangeignore new file mode 100644 index 0000000..28839a4 --- /dev/null +++ b/.melangeignore @@ -0,0 +1,4 @@ +.venv/ +__pycache__/ +packages/ +.git/ diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 45dc172..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,325 +0,0 @@ -# Changelog -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -# [1.5.0] - 2023-11-10 - -### Changed -- If a contact point already exits it will be updated. Previous behaviour was to only to create which failed for existing contact points. - - -# [1.4.2] - 2023-11-01 - -### Added -- enable aws s3 role assumption (use default providers chain) by @DerekTBrown in #245 - -### Changed -- chore: organize s3 utils by @DerekTBrown in #245 - -### Removed - -# [1.4.1] - 2023-09-21 - -### Added - -### Changed -- add missing argument http_get_headers #242 #243 - -### Removed - -# [1.4.0] - 2023-09-20 - -### Added -- add contact points and notifcation policy backup functionalities by @ysde in #238 -- added http headers to get_grafana_version request by @Mar8x in #239 - -### Changed -- added py3-packaging to slim docker image, reported by @tasiotas in #241 :tada: - -### Removed - -# [1.3.3] - 2023-07-27 - -### Added - -### Changed -- verify-ssl: only convert if string contains bool by @chriz-active in #234 -- Validate version using regex in get_grafana_version by @acjohnson in #232 - -### Removed - -# [1.3.2] - 2023-07-19 - -### Added - -### Changed -- fix versions/dashboard_versions hiccup by @chriz-active in #229 -- hide_version-config: add version as config/env var by @chriz-active in #228 -- Expose BACKUP_FILE_FORMAT to env by @vindex10 in #223 -- add slug suffix for uid dashboards by @vindex10 in #222 -- allow disabling auth check by @vindex10 in #221 - -### Removed -- [cicd] github-actions Python 2.7 support is no longer available #230 - -# [1.3.1] - 2023-05-31 - -### Added - -### Changed -- import StringIO and pass data through StringIO read by @Keimille in #173 -- Pass verify_ssl to get_grafana_version method by @yg265 in #207 -- Update alert-rules key to match convention set in cli.py by @relaytheurgency in #208 -- add the x-disable-provenance header for create_alert_rule by @yg265 in #211 -- Perform upsert of alert rules by @mt3593 in #217 - -### Removed - -# [1.3.0] - 2023-04-12 - -### Added - -### Changed -- Only support alert rules if grafana is above version 9.4.0 by @mt3593 in #205 - -### Removed - -# [1.2.6] - 2023-04-12 - -### Added - -### Changed -- Add alert rules to backup by @mt3593 in #201 -- Fix restoring library elements that have more than one folder by @antifuchs in #199 -- Fixed GCP bucket upload example in readme by @IldarGalikov in #200 -- Create a more compact docker image (remove dev packages from image) by @lavirott in #196 - -### Removed - -# [1.2.5] - 2023-02-21 - -### Added - -### Changed -- Backup, restore, and delete teams, team-members, and library-elements by @nileger in #181 - -### Removed - -# [1.2.4] - 2022-08-04 - -### Added -- #190 InfluxDB Support -- #184 (feat): enable/disable api health check not supported by amazon managed grafana - -### Changed -- #176 fix `folder_uid` when dashboard has no folder #175 -- #183 Change the docker image in README.md examples -- #186 Require only necessary permissions when saving backup to GCS bucket - -### Removed - -# [1.2.3] - 2022-01-30 - -### Added - -### Changed -- #174 Fixed ValueError when auth not properly configured - -### Removed - -# [1.2.2] - 2021-11-30 - -### Added - -### Changed -- #158 added uid_support to datasource backups - -### Removed - -# [1.2.1] - 2021-10-15 - -### Added - -### Changed -- 42755ef moved pause and unpause alerts to tools subcommand -- af51467 fixes #140 and ensures folder_permissions are restored - -### Removed -- 30a937f removed unused restore_from_dir function - -# [1.2.0] - 2021-10-11 -- Publish to PyPi - -### Added - -### Changed -- cli changes are coming in 1.2.x which will introduce breaking arg parsing changes - -### Removed - -# [1.1.10] - 2021-10-04 - -### Added -- #150 Option to pause and unpause alerts -- #140 Add folder permissions backup -- #148 Option to save and restore snapshots -- #148 Option to save dashboard versions (restore isn't really possible with the API) -- #148 Option to save and restore annotations - -### Changed - -### Removed - -# [1.1.9] - 2021-06-27 - -### Added -- c6b6f68 Create python-publish.yml -- #100 GCS support - -### Changed -- #92 better error message when the user specifies a bad S3 key -- #135 Update main organization instead of creating a new one -- #133 Creating the docker container without these packages didn't work -- #139 windows env var fix - -### Removed - -# [1.1.8] - 2021-04-14 - -### Added - -### Changed -- #124 fixed #123 -- #121 multi arch docker support -- #127 add azure storage support - -### Removed - -# [1.1.7] - 2021-01-13 - -### Added - -### Changed -- #123 fixed Crash on save_folders.py - -### Removed - -# [1.1.6] - 2020-12-28 - -### Added - -### Changed -- #94 Key Error in api_checks.py (ensure compat with hide_version Grafana setting) -- #117 replace api version check with specific feature checks - -### Removed - -# [1.1.5] - 2020-12-13 - -### Added - -### Changed -- #104 added python2 support to restore functions -- #105 add configuration attribute to set backup file name -- #112 changed restore_functions to ordered dict -- #113 [Fixed] Dashboards with same name in different folders not restored - -### Removed -- #110 remove useless cleanup method within tempfile - -# [1.1.4] - 2020-10-25 - -### Added - -### Changed -- #102 add AWS_ENDPOINT_URL config option - -### Removed - -# [1.1.3] - 2020-08-24 - -### Added -- #57 Backup users, organizations - -### Changed -- #83 Fix tarfile options for python2 -- #64 Remove empty folders when backup file (.tar.gz) created. - - -# [1.1.2] - 2020-08-14 - -### Added - -### Changed -- #70 Fix parameters for get_folder_id_from_old_folder_url -- #74 Fixed Bug folders.txt does not contain all folders -- #75 Introduces the settings option client_cert -- #76 require api checks to succeed before save or restore -- #78 fixed issue #77 TypeError: health_check - -### Removed - -## [1.1.0] - 2020-07-28 - -### Added -- boto3 added to package dependencies -- environment variables and configuration for native AWS S3 support -- s3_upload.py -- s3_download.py - -### Changed - -### Removed - -## [1.0.0] - 2020-07-23 - -### Added -- add setup.py -- add Makefile (docker build) -- add grafanaSettings.json -- add constants.py -- add archive.py - -### Changed -- rename src directory -> grafana_backup -- refactor Dockerfile - - switched to alpine base - - use CMD instead of ENTRYPOINT - - run as non-root user -- README.md updates -- implemented cli "console script" using docopt - - added "save" command - - added "restore" command -- refactored variable passing - - removed global variable passing in favor of positional arguments - - all code lives inside functions now, no need for globalized code -- refactored grafanaSettings module - - grafanaSettings can now be parameterized via an external json config file (~/.grafana-backup.json) -- refactored backup/restore shell scripts -- individual components can be backed up and restored using console_script -- archiving of backup can now be skipped using --no-archive - -### Removed -- delete Pipenv -- delete docker_entry.sh -- delete requirements.txt - -## [0.1.0] - 2019-05-22 -Initial release, tentative. - -### Added -- this `CHANGELOG.md` -- `requirements.txt` and `Pipfile` for better packaging hygiene - -### Changed -- reorganize code layout, move all python into `src/` -- update README.md to reflect changes - -### Removed -- delete boilerplate `restore_SOMETHING.sh` scripts - - -[Unreleased]: https://github.com/ysde/grafana-backup-tool/compare/v0.1.0...HEAD -[0.1.0]: https://github.com/ysde/grafana-backup-tool/releases/tag/v0.1.0 diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 4b14992..0000000 --- a/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -# Stage 1: Build the wheel file -FROM docker.io/python:3.10-alpine@sha256:cb0f3c7df8d980aed5c2a84cc6b83cdcd3ef5a359a2ff68ec750946e86fe281a AS builder - -WORKDIR /build - -ADD . /build - -# Build the wheel file -RUN pip install poetry && \ - poetry build && \ - ls -l /build/dist - -# Stage 2: Create the final image -FROM docker.io/python:3.10-alpine@sha256:cb0f3c7df8d980aed5c2a84cc6b83cdcd3ef5a359a2ff68ec750946e86fe281a - -ENV RESTORE=false -ENV ARCHIVE_FILE="" -ENV PY_COLORS=1 -ENV TZ=UTC -ENV PIP_ROOT_USER_ACTION=ignore - -WORKDIR /opt/grafana-backup-tool - -# Copy the wheel file from the builder stage -COPY --from=builder /build/dist/*.whl / - -# Install the wheel file -RUN pip install --upgrade --no-cache-dir pip && \ - pip install --no-cache-dir $(find / -name "grafana_backup-*.whl") && \ - rm -f grafana_backup-*.whl && \ - chown -R 1337:1337 /opt/grafana-backup-tool && \ - chmod -R 755 /opt/grafana-backup-tool - -USER 1337 -CMD ["sh", "-c", "if [ \"$RESTORE\" = true ]; then if [ ! -z \"$AWS_S3_BUCKET_NAME\" ] || [ ! -z \"$AZURE_STORAGE_CONTAINER_NAME\" ] || [ ! -z \"$GCS_BUCKET_NAME\" ]; then grafana-backup restore $ARCHIVE_FILE; else grafana-backup restore _OUTPUT_/$ARCHIVE_FILE; fi else grafana-backup save; fi"] diff --git a/Makefile b/Makefile index 8f4e08c..fa68256 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,15 @@ SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec VERSION ?= $(shell git describe --dirty --tags --match='v*' 2>/dev/null || git rev-parse --short HEAD) +MELANGE_VERSION ?= $(patsubst v%,%,$(word 1,$(subst -, ,$(VERSION)))) REGISTRY ?= ghcr.io REPO ?= stackitcloud/grafana-backup-tool -PLATFORMS ?= amd64 arm64 +PLATFORMS ?= amd64 +MELANGE_KEY ?= melange.rsa +PACKAGES_DIR ?= packages +IMAGE_TAG ?= $(REGISTRY)/$(REPO):$(VERSION) +IMAGE_TAR ?= grafana-backup-tool.tar +LOCAL ?= false .PHONY: all all: verify @@ -16,25 +22,44 @@ all: verify build: poetry install +.PHONY: key +key: ## Generate melange signing key. + @test -f $(MELANGE_KEY) || melange keygen $(MELANGE_KEY) + +.PHONY: melange-build +melange-build: key ## Build APK package with melange. + sed 's/^ version:.*/ version: $(MELANGE_VERSION)/' melange.yaml > .melange.yaml + @for arch in $$(echo $(PLATFORMS) | tr ',' ' '); do \ + echo "Building architecture: $$arch"; \ + melange build .melange.yaml \ + --source-dir . \ + --out-dir $(PACKAGES_DIR) \ + --arch $$arch \ + --signing-key $(MELANGE_KEY); \ + done + rm -f .melange.yaml + .PHONY: image -image: ## Build Docker image. - docker buildx build \ - --output type=image,name=$(REGISTRY)/$(REPO),push=false \ - --platform $(PLATFORMS) \ - --tag $(REGISTRY)/$(REPO):$(VERSION) \ - --file Dockerfile . - -.PHONY: push -push: image ## Build and push Docker image. - docker push $(REGISTRY)/$(REPO):$(VERSION) - -.PHONY: buildx-and-push -buildx-and-push: ## Build multi-platform and push Docker image. - docker buildx build \ - --output type=image,name=$(REGISTRY)/$(REPO),push=true \ - --platform $(PLATFORMS) \ - --tag $(REGISTRY)/$(REPO):$(VERSION) \ - --file Dockerfile . +image: melange-build ## Build OCI image. Set LOCAL=true to load to local Docker, false (default) to push. +ifeq ($(LOCAL),true) + @echo "Building image locally and loading into Docker..." + apko build apko.yaml \ + $(IMAGE_TAG) \ + $(IMAGE_TAR) \ + --sbom=false \ + --arch $(PLATFORMS) \ + -r "@local ./$(PACKAGES_DIR)" \ + --keyring-append $(MELANGE_KEY).pub + docker load -i $(IMAGE_TAR) +else + @echo "Publishing image to registry $(REGISTRY)..." + apko publish apko.yaml \ + $(IMAGE_TAG) \ + --sbom=false \ + --arch $(PLATFORMS) \ + -r "@local ./$(PACKAGES_DIR)" \ + --keyring-append $(MELANGE_KEY).pub +endif ##@ Python @@ -71,6 +96,8 @@ verify: verify-fmt check clean: ## Remove build artifacts. rm -rf dist/ build/ *.egg-info grafana_backup.egg-info rm -rf .pytest_cache/ .coverage coverage.xml + rm -rf $(PACKAGES_DIR)/ .melange.yaml $(IMAGE_TAR) + rm -f $(MELANGE_KEY) $(MELANGE_KEY).pub .PHONY: help help: ## Display this help. diff --git a/README.md b/README.md index 4d63841..da65334 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ There are three ways to setup the configuration: **NOTE** If you use `environment variables`, you need to add the following to your `.bashrc` or execute once before using the tool (please change variables according to your setup): -(`GRAFANA_HEADERS` is optional, use it if necessary. please see [#45](https://github.com/ysde/grafana-backup-tool/issues/45)) +(`GRAFANA_HEADERS` is optional, use it if necessary) ```bash ### Do not use a trailing slash on GRAFANA_URL @@ -161,7 +161,7 @@ docker run --user $(id -u):$(id -g) --rm --name grafana-backup-tool \ -e GRAFANA_ADMIN_PASSWORD={YOUR_GRAFANA_ADMIN_PASSWORD} \ -e VERIFY_SSL={True/False} \ -v {YOUR_BACKUP_FOLDER_ON_THE_HOST}:/opt/grafana-backup-tool/_OUTPUT_ \ - ysde/docker-grafana-backup-tool + ghcr.io/stackitcloud/grafana-backup-tool ``` **_Example:_** @@ -174,10 +174,10 @@ docker run --user $(id -u):$(id -g) --rm --name grafana-backup-tool \ -e GRAFANA_ADMIN_PASSWORD=adminpassword \ -e VERIFY_SSL=False \ -v /tmp/backup/:/opt/grafana-backup-tool/_OUTPUT_ \ - ysde/docker-grafana-backup-tool + ghcr.io/stackitcloud/grafana-backup-tool ``` -**_S3 Example:_** Set S3 configurations in `-e` or `grafanaSettings.json`([example](https://github.com/ysde/grafana-backup-tool/blob/master/examples/grafana-backup.example.json)) +**_S3 Example:_** Set S3 configurations in `-e` or `grafanaSettings.json`([example](https://github.com/stackitcloud/grafana-backup-tool/blob/master/examples/grafana-backup.example.json)) ```shell -e AWS_S3_BUCKET_NAME="my-backups-bucket" \ @@ -187,14 +187,14 @@ docker run --user $(id -u):$(id -g) --rm --name grafana-backup-tool \ -e AWS_SECRET_ACCESS_KEY="secret" \ ``` -**_Azure Example:_** Set Azure configurations in `-e` or `grafanaSettings.json`([example](https://github.com/ysde/grafana-backup-tool/blob/master/examples/grafana-backup.example.json)) +**_Azure Example:_** Set Azure configurations in `-e` or `grafanaSettings.json`([example](https://github.com/stackitcloud/grafana-backup-tool/blob/master/examples/grafana-backup.example.json)) ```shell -e AZURE_STORAGE_CONTAINER_NAME="azure-storage-container-name" \ -e AZURE_STORAGE_CONNECTION_STRING="azure-storage-connection-string" ``` -**_GCS Example:_** Set GCS configurations in `-e` or `grafanaSettings.json`([example](https://github.com/ysde/grafana-backup-tool/blob/master/examples/grafana-backup.example.json)) +**_GCS Example:_** Set GCS configurations in `-e` or `grafanaSettings.json`([example](https://github.com/stackitcloud/grafana-backup-tool/blob/master/examples/grafana-backup.example.json)) ```shell -e GCS_BUCKET_NAME="backups-bucket-name" \ @@ -215,7 +215,7 @@ docker run --user $(id -u):$(id -g) --rm --name grafana-backup-tool \ -e RESTORE="true" \ -e ARCHIVE_FILE={THE_ARCHIVED_FILE_NAME} \ -v {YOUR_BACKUP_FOLDER_ON_THE_HOST}:/opt/grafana-backup-tool/_OUTPUT_ \ - ysde/docker-grafana-backup-tool + ghcr.io/stackitcloud/grafana-backup-tool ``` **_Example:_** @@ -230,12 +230,12 @@ docker run --user $(id -u):$(id -g) --rm --name grafana-backup-tool \ -e RESTORE="true" \ -e ARCHIVE_FILE="202006280247.tar.gz" \ -v /tmp/backup/:/opt/grafana-backup-tool/_OUTPUT_ \ - ysde/docker-grafana-backup-tool + ghcr.io/stackitcloud/grafana-backup-tool ``` ### Building -You can build the docker image simply by executing `make` in the root of this repo. The image will get tagged as `ysde:grafana-backup` +You can build the container image locally by `make image LOCAL=true` in the root of this repo. ### Monitoring diff --git a/apko.yaml b/apko.yaml new file mode 100644 index 0000000..6b5a98a --- /dev/null +++ b/apko.yaml @@ -0,0 +1,33 @@ +contents: + repositories: + - https://packages.wolfi.dev/os + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + packages: + - wolfi-base + - python-3.10=~3.10.20 + - grafana-backup@local + +accounts: + groups: + - groupname: grafana-backup + gid: 1337 + users: + - username: grafana-backup + uid: 1337 + run-as: "1337" + +environment: + RESTORE: "false" + ARCHIVE_FILE: "" + PY_COLORS: "1" + TZ: "UTC" + PYTHONWARNINGS: "ignore::FutureWarning" + +entrypoint: + command: /usr/bin/grafana-backup-entrypoint + +work-dir: /opt/grafana-backup-tool + +archs: + - amd64 diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..e758507 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +if [ $# -gt 0 ]; then + if [[ "$1" == -* ]] || [[ "$1" == "save" ]] || [[ "$1" == "restore" ]]; then + exec grafana-backup "$@" + fi + exec "$@" +fi + +if [ "$RESTORE" = "true" ]; then + if [ -n "$AWS_S3_BUCKET_NAME" ] || [ -n "$AZURE_STORAGE_CONTAINER_NAME" ] || [ -n "$GCS_BUCKET_NAME" ]; then + exec grafana-backup restore "$ARCHIVE_FILE" + else + exec grafana-backup restore "_OUTPUT_/$ARCHIVE_FILE" + fi +else + exec grafana-backup save +fi diff --git a/examples/grafana-backup-k8s-cronjob.yaml b/examples/grafana-backup-k8s-cronjob.yaml index cc30701..59bb8d2 100644 --- a/examples/grafana-backup-k8s-cronjob.yaml +++ b/examples/grafana-backup-k8s-cronjob.yaml @@ -42,7 +42,7 @@ spec: restartPolicy: "Never" containers: - name: grafana-backup-tool - image: "ysde/docker-grafana-backup-tool:1.2.4" + image: "ghcr.io/stackitcloud/grafana-backup-tool:1.2.4" imagePullPolicy: Always envFrom: - configMapRef: diff --git a/melange.yaml b/melange.yaml new file mode 100644 index 0000000..c9bf1d0 --- /dev/null +++ b/melange.yaml @@ -0,0 +1,40 @@ +package: + name: grafana-backup + version: 0.0.0 + epoch: 0 + description: "A Python-based tool to backup and restore Grafana settings" + copyright: + - license: MIT + dependencies: + runtime: + - python-3.10 + +environment: + contents: + repositories: + - https://packages.wolfi.dev/os + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + packages: + - build-base + - busybox=~1.37.0 + - python-3.10=~3.10.20 + - python-3.10-dev=~3.10.20 + - py3.10-pip=~26.1.2 + - py3.10-poetry-bin=~2.4.1 + +pipeline: + - runs: | + cd /home/build + poetry build + pip install --ignore-installed --prefix=/usr --root="${{targets.destdir}}" dist/*.whl + install -Dm755 entrypoint.sh "${{targets.destdir}}/usr/bin/grafana-backup-entrypoint" + +test: + environment: + contents: + packages: + - busybox + pipeline: + - runs: | + test -f /usr/bin/grafana-backup