diff --git a/.agents/skills/openwrt-package-update/SKILL.md b/.agents/skills/openwrt-package-update/SKILL.md new file mode 100644 index 000000000..7f9164525 --- /dev/null +++ b/.agents/skills/openwrt-package-update/SKILL.md @@ -0,0 +1,60 @@ +--- +name: openwrt-package-update +description: > + Use when updating any forked OpenWrt package in a NethSecurity workspace from + the upstream openwrt/packages feed. Covers the full update cycle: version bump, + merging upstream file changes, updating dependent ns-api handlers and migration + scripts, UCI overlay defaults, and correlated documentation. Triggers on updating + a package by name (adblock, mwan3, banip, rsyslog, snort3, keepalived, + python-jinja2, python-semver, and similar forks), syncing with upstream, or any + request to align a local package fork with openwrt/packages — even if upstream + is not mentioned explicitly. +compatibility: Requires git and curl. Works in a NethSecurity workspace with build.conf.defaults present. +metadata: + domain: nethsecurity-packages + type: package-update +allowed-tools: Bash(git:*) Bash(curl:*) Bash(diff:*) Bash(tar:*) Bash(grep:*) Bash(sed:*) Bash(awk:*) Read Write +--- + +## What I do + +Update a forked upstream OpenWrt package from the openwrt/packages feed. Auto-discovers the package path, compares the current version against the upstream target, extracts snapshots for comparison, applies upstream changes, propagates impact to ns-api handlers and migration scripts, and updates correlated documentation. + +## Quick start + +1. **Setup**: `bash scripts/setup-package.sh ` — auto-discovers upstream path, finds version commits, extracts old/new snapshots into `assets/_old/` and `assets/_new/` +2. **Compare**: `bash scripts/diff-package.sh ` — shows what changed upstream +3. **Merge**: Read both snapshots, apply upstream changes to local files in `packages//`; update PKG_VERSION and PKG_RELEASE in the Makefile +4. **Cross-package**: Grep for any changed config keys or function names in ns-api, migration scripts, and files/ overlay; update as needed +5. **Documentation**: Update `docs/` and `packages/ns-api/README.md` if user-facing behavior changed +6. **Verify**: Run `ruff check` on any changed Python files; build the package inside the container + +For per-step detail on any of the above, read [references/WORKFLOW.md](references/WORKFLOW.md). + +## How it works + +The skill uses three helper scripts: + +- **get-target-commit.sh** — reads `build.conf.defaults` for OWRT_VERSION, fetches `feeds.conf.default` from that tag, extracts the openwrt/packages commit hash +- **setup-package.sh** — auto-discovers package upstream path, uses git pickaxe to find the commit matching current PKG_VERSION then PKG_RELEASE, clones/fetches openwrt/packages bare repo into `assets/.openwrt-packages-repo/` (persistent cache), extracts both versions into named snapshots +- **diff-package.sh** — convenience wrapper: `diff -rN assets/_old/ assets/_new/` + +## Snapshot structure + +Each snapshot has the exact upstream package files at root level: +``` +assets/adblock_old/ +├── Makefile +├── files/ +│ ├── adblock.sh +│ ├── adblock.init +│ ├── adblock.conf +│ └── ... +``` + +## Edge cases + +- **mwan3**: Deep fork — apply upstream changes conservatively, one by one +- **snort3**: Has local patches — verify patches still apply after update +- **Version not found**: If script fails, local version is very old; check `assets/.openwrt-packages-repo/` git history +- **Empty diff**: Versions match; only update PKG_VERSION/PKG_RELEASE if tracking newer release diff --git a/.agents/skills/openwrt-package-update/assets/.gitignore b/.agents/skills/openwrt-package-update/assets/.gitignore new file mode 100644 index 000000000..72e8ffc0d --- /dev/null +++ b/.agents/skills/openwrt-package-update/assets/.gitignore @@ -0,0 +1 @@ +* diff --git a/.agents/skills/openwrt-package-update/references/WORKFLOW.md b/.agents/skills/openwrt-package-update/references/WORKFLOW.md new file mode 100644 index 000000000..ec4af820c --- /dev/null +++ b/.agents/skills/openwrt-package-update/references/WORKFLOW.md @@ -0,0 +1,88 @@ +# Detailed Workflow: Updating Upstream Packages + +## 1. Identify the package + +Confirm the package exists and is in scope. Read `packages//Makefile` to understand the current version: +- Note PKG_VERSION and PKG_RELEASE +- Check if `packages//patches/` exists (indicates local patches) +- Scan `files/` directory to understand what configuration files and scripts are included + +## 2. Setup snapshots + +Run the setup script: +```bash +bash scripts/setup-package.sh +``` + +This script will: +- Auto-discover the upstream path in openwrt/packages (e.g., `net/adblock`) +- Use git pickaxe to find the commit that introduced the current PKG_VERSION +- Then find the commit that set the current PKG_RELEASE +- Extract both versions into `assets/_old/` and `assets/_new/` +- Output summary with commit hashes and paths + +## 3. Compare and classify + +Examine both snapshots using your file reading tools. Run: +```bash +bash scripts/diff-package.sh +``` + +Classify each change: +- **Makefile**: version bumps, new dependencies, build flags +- **files/*.sh** / **files/*.init**: behavioral changes to shell scripts +- **files/*.conf**: UCI configuration schema changes (new/removed/renamed options) +- **files/*.sources**, **files/*.categories**, data files: content-only updates +- **New files** added / **files removed** + +## 4. Apply upstream changes + +For each changed file: +- If the local copy is identical to old version or has no NethSecurity-specific changes → take upstream as-is +- If the local copy has customizations → merge carefully: + - Use the diff tool to understand what changed upstream + - Manually integrate the upstream changes into the local file + - Preserve any NethSecurity-specific logic or configuration + +Always: +- Update PKG_VERSION and PKG_RELEASE in `packages//Makefile` +- If `packages//patches/` exists, re-verify each patch still applies cleanly with the new version +- Test build: `make package/feeds/nethsecurity//compile V=sc` inside the build container + +## 5. Cross-package impact detection + +For each changed UCI option name, function name, or config key in the diff: +- Grep the workspace for references: + ```bash + grep -r "" packages/ files/ docs/ --include="*.py" --include="*.sh" --include="*.json" --include="*.conf" --include="*.md" + ``` +- Grep `packages/ns-migration/` for migration scripts that reference old option names +- Grep `packages/ns-api/` for API scripts that read/write this package's configuration +- Present findings to user before making changes + +## 6. Apply cross-package fixes (after user confirmation) + +Update affected files: +- **ns-api scripts**: if UCI schema changed, update handlers in `packages/ns-api/files/ns.` +- **Migration scripts**: if old config options are removed, add migration logic to `packages/ns-migration/files/` +- **files/ overlay defaults**: if default values changed, update `files/etc/uci-defaults/` or `files/etc/config/` +- **Documentation**: update `docs/` if user-facing behavior changed + +## Edge cases + +**mwan3**: This is a deep fork with significant local modifications. Apply upstream changes one by one, preferring manual review for any behavioral change. Verify the result against firewall rules and failover behavior. + +**snort3, openvpn-easy-rsa**: These packages have local patches (see `packages//patches/`). After updating upstream files, re-run `quilt refresh` or regenerate patches. If patches no longer apply cleanly, update them to match the new upstream. + +**Empty diff**: If upstream and local versions are the same, only update PKG_VERSION/PKG_RELEASE in the Makefile if tracking a newer release. Otherwise, the package is up-to-date. + +**Version not found**: If the script cannot find the current PKG_VERSION in upstream history, it means the local version is very old or from a different source. Check git history in the bare repo (`assets/.openwrt-packages-repo/`) to understand the version trajectory. + +## Helpful commands + +Inside the skill directory: +- `bash scripts/get-target-commit.sh` — print the target packages feed commit hash from feeds.conf.default +- `bash scripts/setup-package.sh ` — extract old/new snapshots +- `bash scripts/diff-package.sh ` — show recursive diff +- `find assets/_old/ -type f` — list all files in old snapshot +- `cat assets/_old/Makefile | grep PKG_` — check old version details diff --git a/.agents/skills/openwrt-package-update/scripts/diff-package.sh b/.agents/skills/openwrt-package-update/scripts/diff-package.sh new file mode 100755 index 000000000..a32cc4ee8 --- /dev/null +++ b/.agents/skills/openwrt-package-update/scripts/diff-package.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# Diff old and new package snapshots +# +# Usage: diff-package.sh +# +# Outputs a recursive diff between assets/_old/ and assets/_new/ +# + +set -euo pipefail + +PACKAGE_NAME="${1:-}" + +if [[ -z "$PACKAGE_NAME" ]]; then + echo "Usage: diff-package.sh " >&2 + exit 1 +fi + +# Find skill root +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SKILL_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ASSETS_DIR="$SKILL_ROOT/assets" + +OLD_DIR="$ASSETS_DIR/${PACKAGE_NAME}_old" +NEW_DIR="$ASSETS_DIR/${PACKAGE_NAME}_new" + +if [[ ! -d "$OLD_DIR" ]]; then + echo "Error: $OLD_DIR not found. Run setup-package.sh first." >&2 + exit 1 +fi + +if [[ ! -d "$NEW_DIR" ]]; then + echo "Error: $NEW_DIR not found. Run setup-package.sh first." >&2 + exit 1 +fi + +# Run recursive diff, showing added/removed/modified files +diff -rN "$OLD_DIR" "$NEW_DIR" || true diff --git a/.agents/skills/openwrt-package-update/scripts/get-target-commit.sh b/.agents/skills/openwrt-package-update/scripts/get-target-commit.sh new file mode 100755 index 000000000..687a5ee67 --- /dev/null +++ b/.agents/skills/openwrt-package-update/scripts/get-target-commit.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Get the target OpenWrt packages feed commit hash from feeds.conf.default +# at the version specified in build.conf.defaults +# +# Outputs the commit hash to stdout, or exits with error if fetch/parse fails. +# + +set -euo pipefail + +# Find workspace root +WORKSPACE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && git rev-parse --show-toplevel 2>/dev/null || echo ".") + +# Read OWRT_VERSION from build.conf.defaults +if [[ ! -f "$WORKSPACE_ROOT/build.conf.defaults" ]]; then + echo "Error: build.conf.defaults not found at $WORKSPACE_ROOT" >&2 + exit 1 +fi + +OWRT_VERSION=$(grep '^OWRT_VERSION=' "$WORKSPACE_ROOT/build.conf.defaults" | cut -d'=' -f2 | tr -d ' ') + +if [[ -z "$OWRT_VERSION" ]]; then + echo "Error: OWRT_VERSION not found or empty in build.conf.defaults" >&2 + exit 1 +fi + +# Fetch feeds.conf.default from OpenWrt at the specified tag +FEEDS_URL="https://raw.githubusercontent.com/openwrt/openwrt/$OWRT_VERSION/feeds.conf.default" + +FEEDS_CONTENT=$(curl -fsSL "$FEEDS_URL" 2>/dev/null || { + echo "Error: Failed to fetch $FEEDS_URL" >&2 + exit 1 +}) + +# Parse the packages line to extract the commit hash (after the ^) +# Expected format: src-git packages https://git.openwrt.org/feed/packages.git^ +PACKAGES_COMMIT=$(echo "$FEEDS_CONTENT" | grep '^src-git packages' | sed 's/.*\^//g' | head -1) + +if [[ -z "$PACKAGES_COMMIT" ]]; then + echo "Error: Could not parse packages commit hash from feeds.conf.default" >&2 + exit 1 +fi + +echo "$PACKAGES_COMMIT" diff --git a/.agents/skills/openwrt-package-update/scripts/setup-package.sh b/.agents/skills/openwrt-package-update/scripts/setup-package.sh new file mode 100755 index 000000000..0ce5bf135 --- /dev/null +++ b/.agents/skills/openwrt-package-update/scripts/setup-package.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# +# Setup old and new package snapshots for comparison +# +# Usage: setup-package.sh +# +# Outputs: +# - Creates assets/_old/ with the current pinned upstream version +# - Creates assets/_new/ with the target upstream version (from feeds.conf.default) +# - Prints to stdout: package name, CURRENT_COMMIT, TARGET_COMMIT, upstream path +# + +set -euo pipefail + +PACKAGE_NAME="${1:-}" + +if [[ -z "$PACKAGE_NAME" ]]; then + echo "Usage: setup-package.sh " >&2 + exit 1 +fi + +# Find workspace root (convert to absolute path) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORKSPACE_ROOT=$(cd "$SCRIPT_DIR/../../../.." && git rev-parse --show-toplevel 2>/dev/null || pwd) +SKILL_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ASSETS_DIR="$SKILL_ROOT/assets" + +# Ensure assets dir exists +mkdir -p "$ASSETS_DIR" + +# Read current package version from local Makefile +LOCAL_MAKEFILE="$WORKSPACE_ROOT/packages/$PACKAGE_NAME/Makefile" +if [[ ! -f "$LOCAL_MAKEFILE" ]]; then + echo "Error: $LOCAL_MAKEFILE not found" >&2 + exit 1 +fi + +PKG_VERSION=$(grep '^PKG_VERSION:=' "$LOCAL_MAKEFILE" | cut -d'=' -f2 | tr -d ' ') +PKG_RELEASE=$(grep '^PKG_RELEASE:=' "$LOCAL_MAKEFILE" | cut -d'=' -f2 | tr -d ' ') + +if [[ -z "$PKG_VERSION" ]]; then + echo "Error: PKG_VERSION not found in $LOCAL_MAKEFILE" >&2 + exit 1 +fi + +echo "Package: $PACKAGE_NAME (version $PKG_VERSION-$PKG_RELEASE)" >&2 + +# Get target commit from feeds.conf.default +TARGET_COMMIT=$("$SKILL_ROOT/scripts/get-target-commit.sh") +echo "Target commit: $TARGET_COMMIT" >&2 + +# Clone/fetch openwrt/packages bare repo +BARE_REPO="$ASSETS_DIR/.openwrt-packages-repo" + +if [[ ! -d "$BARE_REPO" ]]; then + echo "Cloning openwrt/packages (bare repo)..." >&2 + git clone --bare --filter=blob:none https://github.com/openwrt/packages.git "$BARE_REPO" 2>/dev/null || { + echo "Error: Failed to clone openwrt/packages" >&2 + exit 1 + } +else + echo "Fetching openwrt/packages..." >&2 + cd "$BARE_REPO" + git fetch origin 2>/dev/null || { + echo "Error: Failed to fetch openwrt/packages" >&2 + exit 1 + } + cd - > /dev/null +fi + +# Auto-discover the upstream path by searching for /Makefile +# Look in the target commit first to find the exact path +echo "Discovering upstream package path..." >&2 +UPSTREAM_PATH=$(cd "$BARE_REPO" && git ls-tree -r --name-only "$TARGET_COMMIT" | grep -E "^[^/]+/$PACKAGE_NAME/Makefile$" | sed 's|/Makefile$||' | head -1) + +if [[ -z "$UPSTREAM_PATH" ]]; then + echo "Error: Could not find $PACKAGE_NAME in upstream openwrt/packages at $TARGET_COMMIT" >&2 + exit 1 +fi + +echo "Upstream path: $UPSTREAM_PATH" >&2 + +# Find the commit that introduced the current version +# Step 1: Find when PKG_VERSION was introduced using pickaxe +# Step 2: Find when PKG_RELEASE was set using pickaxe +echo "Finding commit that matches local version ($PKG_VERSION-$PKG_RELEASE)..." >&2 + +CURRENT_COMMIT="" + +# Step 1: Find commits that introduced PKG_VERSION +echo " Step 1: Finding commits with PKG_VERSION:=$PKG_VERSION..." >&2 +version_commits=$(cd "$BARE_REPO" && git log -S "PKG_VERSION:=$PKG_VERSION" --all --format="%H" -- "$UPSTREAM_PATH/Makefile" 2>/dev/null) + +if [[ -z "$version_commits" ]]; then + echo "Error: Could not find PKG_VERSION:=$PKG_VERSION in git history" >&2 + exit 1 +fi + +# Get the oldest commit where PKG_VERSION was introduced (last in the list when searching backwards) +version_commit=$(echo "$version_commits" | tail -1) +echo " PKG_VERSION introduced at: $version_commit" >&2 + +# Step 2: From that commit forward, find when PKG_RELEASE was set +# We search commits from version_commit onwards for PKG_RELEASE change +echo " Step 2: Finding commits with PKG_RELEASE:=$PKG_RELEASE after version introduction..." >&2 +release_commits=$(cd "$BARE_REPO" && git log -S "PKG_RELEASE:=$PKG_RELEASE" --all --format="%H" -- "$UPSTREAM_PATH/Makefile" 2>/dev/null) + +if [[ -n "$release_commits" ]]; then + # Find the first commit in release_commits that is an ancestor of or after version_commit + # Also verify both PKG_VERSION and PKG_RELEASE are present + while IFS= read -r commit; do + makefile_content=$(cd "$BARE_REPO" && git show "$commit:$UPSTREAM_PATH/Makefile" 2>/dev/null || echo "") + + if echo "$makefile_content" | grep -q "^PKG_VERSION:=$PKG_VERSION" && \ + echo "$makefile_content" | grep -q "^PKG_RELEASE:=$PKG_RELEASE"; then + CURRENT_COMMIT="$commit" + break + fi + done <<< "$release_commits" +fi + +if [[ -z "$CURRENT_COMMIT" ]]; then + echo "Error: Could not find a commit matching $PKG_VERSION-$PKG_RELEASE in upstream history" >&2 + echo "Available version range (last 20 commits):" >&2 + cd "$BARE_REPO" && git log --all --oneline -20 -- "$UPSTREAM_PATH/Makefile" >&2 + exit 1 +fi + +echo "Current commit: $CURRENT_COMMIT" >&2 + +# Extract the old version (current commit) +OLD_DIR="$ASSETS_DIR/${PACKAGE_NAME}_old" +rm -rf "$OLD_DIR" +mkdir -p "$OLD_DIR" + +echo "Extracting old version to $OLD_DIR..." >&2 +strip_level=$(echo "$UPSTREAM_PATH" | awk -F/ '{print NF}') +cd "$BARE_REPO" && git archive "$CURRENT_COMMIT" "$UPSTREAM_PATH/" | tar -x --strip-components=$strip_level -C "$OLD_DIR" 2>/dev/null || { + echo "Error: Failed to extract old version" >&2 + exit 1 +} + +# Extract the new version (target commit) +NEW_DIR="$ASSETS_DIR/${PACKAGE_NAME}_new" +rm -rf "$NEW_DIR" +mkdir -p "$NEW_DIR" + +echo "Extracting new version to $NEW_DIR..." >&2 +cd "$BARE_REPO" && git archive "$TARGET_COMMIT" "$UPSTREAM_PATH/" | tar -x --strip-components=$strip_level -C "$NEW_DIR" 2>/dev/null || { + echo "Error: Failed to extract new version" >&2 + exit 1 +} + +# Output summary to stdout +echo "" +echo "===== SETUP COMPLETE =====" +echo "Package: $PACKAGE_NAME" +echo "Current upstream commit: $CURRENT_COMMIT" +echo "Target upstream commit: $TARGET_COMMIT" +echo "Upstream path: $UPSTREAM_PATH" +echo "Old version snapshot: $OLD_DIR" +echo "New version snapshot: $NEW_DIR" diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index eae1f39af..c52f863b8 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -11,10 +11,8 @@ on: - 'files/**' - 'packages/**' - 'patches/**' - - 'build.conf.example' + - 'build.conf.defaults' - 'build-nethsec.sh' - tags: - - '*' pull_request: paths: - 'builder/**' @@ -22,7 +20,7 @@ on: - 'files/**' - 'packages/**' - 'patches/**' - - 'build.conf.example' + - 'build.conf.defaults' - 'build-nethsec.sh' jobs: @@ -33,8 +31,8 @@ jobs: NETHSECURITY_VERSION: ${{ steps.build_vars.outputs.NETHSECURITY_VERSION }} REPO_CHANNEL: ${{ steps.build_vars.outputs.REPO_CHANNEL }} env: - USIGN_PUB_KEY: ${{ secrets.USIGN_PUB_KEY }} - USIGN_PRIV_KEY: ${{ secrets.USIGN_PRIV_KEY }} + APK_PUB_KEY: ${{ secrets.APK_PUB_KEY }} + APK_PRIV_KEY: ${{ secrets.APK_PRIV_KEY }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} steps: @@ -42,32 +40,34 @@ jobs: - name: Generate build variables id: build_vars run: | - # export OWRT_VERSION from build.conf.example - echo "OWRT_VERSION=$(grep -oP 'OWRT_VERSION=\K.*' build.conf.example)" >> $GITHUB_OUTPUT + # export OWRT_VERSION from build.conf.defaults + echo "OWRT_VERSION=$(grep -oP 'OWRT_VERSION=\K.*' build.conf.defaults)" >> $GITHUB_OUTPUT - # export TARGET from build.conf.example - echo "TARGET=$(grep -oP 'TARGET=\K.*' build.conf.example)" >> $GITHUB_OUTPUT + # export TARGET from build.conf.defaults + echo "TARGET=$(grep -oP 'TARGET=\K.*' build.conf.defaults)" >> $GITHUB_OUTPUT - # export NETHSECURITY_VERSION from build - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.example)" >> $GITHUB_OUTPUT - - # When pushing a tag, set REPO_CHANNEL to stable - if [[ "${{ github.ref }}" == refs/tags/* ]]; then - echo "REPO_CHANNEL=stable" >> $GITHUB_OUTPUT - # save NETHSECURITY_VERSION to env - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.example)" >> $GITHUB_OUTPUT + # export base NETHSECURITY_VERSION from build.conf.defaults + BASE_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.defaults) + COMMIT_HASH=$(git rev-parse --short HEAD) + TIMESTAMP=$(date +'%Y%m%d%H%M%S') + echo "NETHSECURITY_VERSION=${BASE_VERSION}" >> $GITHUB_OUTPUT # When pushing to main branch, set REPO_CHANNEL to dev - elif [[ "${{ github.ref }}" == refs/heads/main ]]; then + if [[ "${{ github.ref }}" == refs/heads/main ]]; then echo "REPO_CHANNEL=dev" >> $GITHUB_OUTPUT - # save NETHSECURITY_VERSION to env and append -dev to it - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.example)-dev+$(git rev-parse --short HEAD).$(date +'%Y%m%d%H%M%S')" >> $GITHUB_OUTPUT + echo "BUILD_SEMVER_SUFFIX=-dev.${TIMESTAMP}.${COMMIT_HASH}" >> $GITHUB_OUTPUT + + # For pull requests and workflow_dispatch, add timestamp suffix + elif [[ "${{ github.event_name }}" == 'pull_request' ]]; then + BRANCH_NAME="${{ github.head_ref }}" + echo "REPO_CHANNEL=${BRANCH_NAME}" >> $GITHUB_OUTPUT + echo "BUILD_SEMVER_SUFFIX=-${BRANCH_NAME}.${TIMESTAMP}.${COMMIT_HASH}" >> $GITHUB_OUTPUT + + elif [[ "${{ github.event_name }}" == 'workflow_dispatch' ]]; then + BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's|refs/heads/||') + echo "REPO_CHANNEL=${BRANCH_NAME}" >> $GITHUB_OUTPUT + echo "BUILD_SEMVER_SUFFIX=-.${TIMESTAMP}.${COMMIT_HASH}" >> $GITHUB_OUTPUT - # Otherwise, get the branch name of the PR pushing if REPO_CHANNEL is not set - elif [[ "${{ github.event_name }}" == 'pull_request' && ! -v REPO_CHANNEL ]]; then - echo "REPO_CHANNEL=${{ github.head_ref }}" >> $GITHUB_OUTPUT - # save NETHSECURITY_VERSION to env and append last commit hash to it - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.example)-${{ github.head_ref }}+$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT fi - name: Build the image env: @@ -75,12 +75,14 @@ jobs: NETHSECURITY_VERSION: ${{ steps.build_vars.outputs.NETHSECURITY_VERSION }} REPO_CHANNEL: ${{ steps.build_vars.outputs.REPO_CHANNEL }} TARGET: ${{ steps.build_vars.outputs.TARGET }} + BUILD_SEMVER_SUFFIX: ${{ steps.build_vars.outputs.BUILD_SEMVER_SUFFIX }} run: ./build-nethsec.sh - name: Update latest_release file run: | - # Create release file pointing to 8-VERSION - echo "${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}" > latest_release - echo "::notice title='Image published':: ${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}" + # Create release file with the full display version (base + suffix) + FULL_VERSION="${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}${{ steps.build_vars.outputs.BUILD_SEMVER_SUFFIX }}" + echo "${FULL_VERSION}" > latest_release + echo "::notice title='Image published':: ${FULL_VERSION}" - uses: actions/upload-artifact@v7 name: Upload image with: @@ -117,8 +119,14 @@ jobs: RCLONE_CONFIG_REPO_ACCESS_KEY_ID: ${{ secrets.DO_SPACE_ACCESS_KEY }} RCLONE_CONFIG_REPO_SECRET_ACCESS_KEY: ${{ secrets.DO_SPACE_SECRET_KEY }} run: | + # Sync binaries to channel/version/ rclone sync bin/ repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/${{ steps.build_vars.outputs.NETHSECURITY_VERSION }} --progress --create-empty-src-dirs - rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/ --progress --create-empty-src-dirs + + # Place latest_release inside the version directory + rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}/ --progress + + # Also place latest_release at channel root as fallback + rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/ --progress tools: name: 'Run tools' diff --git a/.gitignore b/.gitignore index f54bf6bc0..40665e2e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ -key-build* -build.conf.override +private-key.pem +public-key.pem /bin build-logs build.conf netify-flow-actions netify-agent-stats-plugin +scripts/netifyd-apks diff --git a/AGENTS.md b/AGENTS.md index 6425615b1..2b4437365 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,7 @@ This project has domain-specific skills available. You MUST activate the relevan - `ns-api` — **ACTIVATE** when writing, modifying, or reviewing Python RPCD API scripts. Triggers: creating or updating `ns.*` RPCD API endpoints, handling UCI configuration changes, managing pre/post-commit hooks, defining ACL permissions, documenting methods in OpenAPI 3.1.0, or when user mentions ns-api, API endpoints, hooks, or references `/usr/libexec/rpcd/ns.` files. Covers stdin/stdout JSON protocol, error handling, naming conventions, code style, and spec file updates. - `openwrt-package` — **ACTIVATE** when creating or modifying OpenWrt `ns-*` packages. Triggers: building new packages for NethSecurity, managing package dependencies, patching upstream feeds, modifying Makefiles, or when user mentions Makefile, package structure, config fragments, or upstream patches. Covers naming conventions, required Makefile fields, architecture selection, external version management, and patch workflows. +- `openwrt-package-update` — **ACTIVATE** when updating forked OpenWrt packages from the upstream feed (adblock, mwan3, banip, etc.). Triggers: updating non-ns- packages, comparing local forks against openwrt/packages, merging upstream improvements, or when user mentions upstream package updates. Auto-discovers packages; extracts old/new snapshots for side-by-side comparison; guides cross-package impact detection. - `python-nethsecurity` — **ACTIVATE** when writing or modifying Python scripts for NethSecurity packages. Triggers: creating new Python scripts, configuring package build systems, writing utilities, or when user mentions Python code in packages/ns-* or references Python scripts in the NethSecurity package tree. Covers shebang, license headers, extension handling, ruff compliance, available modules, and UCI commit conventions. --- diff --git a/build-nethsec.sh b/build-nethsec.sh index 9e570ea92..426025776 100755 --- a/build-nethsec.sh +++ b/build-nethsec.sh @@ -7,23 +7,36 @@ set -e -# Source build files if it exists +# Snapshot current environment so it can be restored with highest precedence +_env_snapshot=$(export -p) + +# Source versioned defaults first set -o allexport +if [ -f build.conf.defaults ]; then + echo "Loading build.conf.defaults..." + . ./build.conf.defaults +fi + +# Source local overrides second (can override defaults) if [ -f build.conf ]; then - echo "Loading build.conf file..." + echo "Loading build.conf (local overrides)..." . ./build.conf fi set +o allexport +# Re-apply original environment variables so they take final precedence over config files +eval "$_env_snapshot" + # Check required environment variables OWRT_VERSION=${OWRT_VERSION:?Missing OWRT_VERSION environment variable} NETHSECURITY_VERSION=${NETHSECURITY_VERSION:?Missing NETHSECURITY_VERSION environment variable} REPO_CHANNEL=${REPO_CHANNEL:-dev} TARGET=${TARGET:-x86_64} +BUILD_SEMVER_SUFFIX=${BUILD_SEMVER_SUFFIX:-} -if [ -f "./key-build" ] && [ -f "./key-build.pub" ]; then - USIGN_PRIV_KEY="$(cat ./key-build)" - USIGN_PUB_KEY="$(cat ./key-build.pub)" +if [ -f "./private-key.pem" ] && [ -f "./public-key.pem" ]; then + APK_PRIV_KEY="$(cat ./private-key.pem)" + APK_PUB_KEY="$(cat ./public-key.pem)" fi @@ -32,20 +45,20 @@ podman build \ --layers \ --file builder/Containerfile \ --tag nethsecurity-next \ - --target builder \ --jobs 0 \ --build-arg OWRT_VERSION="$OWRT_VERSION" \ --build-arg REPO_CHANNEL="$REPO_CHANNEL" \ --build-arg TARGET="$TARGET" \ --build-arg NETHSECURITY_VERSION="$NETHSECURITY_VERSION" \ + --build-arg BUILD_SEMVER_SUFFIX="$BUILD_SEMVER_SUFFIX" \ . set +e status=0 podman run \ - --env USIGN_PRIV_KEY="$USIGN_PRIV_KEY" \ - --env USIGN_PUB_KEY="$USIGN_PUB_KEY" \ + --env APK_PRIV_KEY="$APK_PRIV_KEY" \ + --env APK_PUB_KEY="$APK_PUB_KEY" \ --name nethsecurity-builder \ --interactive \ --tty \ diff --git a/build.conf.defaults b/build.conf.defaults new file mode 100644 index 000000000..eabd4d1d1 --- /dev/null +++ b/build.conf.defaults @@ -0,0 +1,4 @@ +OWRT_VERSION=v25.12.4 +NETHSECURITY_VERSION=8.8.0 +TARGET=x86_64 +REPO_CHANNEL=dev diff --git a/build.conf.example b/build.conf.example deleted file mode 100644 index 0f05f1e39..000000000 --- a/build.conf.example +++ /dev/null @@ -1,4 +0,0 @@ -OWRT_VERSION=v24.10.5 -NETHSECURITY_VERSION=8.7.2 -TARGET=x86_64 -REPO_CHANNEL=dev diff --git a/builder/Containerfile b/builder/Containerfile index 351dda0ef..a8e05b581 100644 --- a/builder/Containerfile +++ b/builder/Containerfile @@ -1,46 +1,50 @@ # -# Copyright (C) 2025 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # -# 2025-06-30 -FROM debian:12.11 AS base -ARG SOURCE_DATE_EPOCH=1751241600 +FROM debian:13.4 AS base RUN apt-get update \ && apt-get install --yes --no-install-recommends --no-install-suggests \ + # openwrt build dependencies https://openwrt.org/docs/guide-developer/toolchain/install-buildsystem#debianubuntumint bison \ build-essential \ - ca-certificates \ clang \ - cmake \ - curl \ file \ flex \ g++ \ + g++-multilib \ gawk \ gcc-multilib \ gettext \ git \ libncurses5-dev \ libssl-dev \ - python3-distutils \ + python3-setuptools \ rsync \ - sudo \ + swig \ unzip \ - vim \ wget \ - zlib1g-dev - -FROM base AS usign_build -RUN git clone --depth 1 https://git.openwrt.org/project/usign.git /tmp/usign \ - && cd /tmp/usign \ - && cmake . \ - && make + zlib1g-dev \ + # other dependencies + ca-certificates \ + cmake \ + curl \ + less \ + quilt \ + sudo \ + vim -FROM base AS builder RUN groupadd -g 1000 'buildbot' \ && useradd -m -s '/bin/bash' -u 1000 -g 1000 'buildbot' \ - && echo 'buildbot ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/buildbot + && echo 'buildbot ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/buildbot \ + && echo "QUILT_DIFF_ARGS=--no-timestamps --no-index -p ab --color=auto" \ + "QUILT_REFRESH_ARGS=--no-timestamps --no-index -p ab" \ + "QUILT_SERIES_ARGS=--color=auto" \ + "QUILT_PATCH_OPTS=--unified" \ + "QUILT_DIFF_OPTS=-p" \ + "EDITOR=vim" \ + > /home/buildbot/.quiltrc USER buildbot WORKDIR /home/buildbot # OpenWRT repo @@ -63,19 +67,19 @@ COPY --chown=buildbot:buildbot config config ARG REPO_CHANNEL ARG TARGET ARG NETHSECURITY_VERSION +ARG BUILD_SEMVER_SUFFIX COPY --chmod=777 builder/configure-build.sh /usr/local/bin/configure-build RUN /usr/local/bin/configure-build COPY --chmod=777 builder/entrypoint.sh /usr/local/bin/entrypoint.sh ENTRYPOINT [ "/usr/local/bin/entrypoint.sh" ] -COPY --from=usign_build /tmp/usign/usign /usr/local/bin/usign RUN mkdir -p \ - .ccache \ - build_dir \ - dl \ - download \ - staging_dir + .ccache \ + build_dir \ + dl \ + download \ + staging_dir VOLUME "/home/buildbot/openwrt/.ccache" \ - "/home/buildbot/openwrt/build_dir" \ - "/home/buildbot/openwrt/dl" \ - "/home/buildbot/openwrt/download" \ - "/home/buildbot/openwrt/staging_dir" + "/home/buildbot/openwrt/build_dir" \ + "/home/buildbot/openwrt/dl" \ + "/home/buildbot/openwrt/download" \ + "/home/buildbot/openwrt/staging_dir" diff --git a/builder/configure-build.sh b/builder/configure-build.sh index 8f1cf189a..f21b34fe1 100644 --- a/builder/configure-build.sh +++ b/builder/configure-build.sh @@ -12,6 +12,13 @@ nethsecurity_version=${NETHSECURITY_VERSION:?Missing NETHSECURITY_VERSION enviro repo_channel=${REPO_CHANNEL:?Missing REPO_CHANNEL environment variable} target=${TARGET:?Missing TARGET environment variable} owrt_version=${OWRT_VERSION:?Missing OWRT_VERSION environment variable} +build_semver_suffix=${BUILD_SEMVER_SUFFIX:-} + +if [ -n "$build_semver_suffix" ]; then + image_version="${nethsecurity_version}${build_semver_suffix}" +else + image_version="${nethsecurity_version}" +fi # For each file inside the config directory, cat the content into a .config file for file in config/*.conf; do @@ -31,7 +38,7 @@ CONFIG_VERSION_DIST="NethSecurity" CONFIG_VERSION_HOME_URL="https://github.com/nethserver/nethsecurity" CONFIG_VERSION_MANUFACTURER="Nethesis" CONFIG_VERSION_MANUFACTURER_URL="https://www.nethesis.it" -CONFIG_VERSION_NUMBER="${nethsecurity_version}" +CONFIG_VERSION_NUMBER="${image_version}" CONFIG_VERSION_CODE="${owrt_version}" CONFIG_VERSION_PRODUCT="NethSecurity" CONFIG_VERSION_REPO="https://updates.nethsecurity.nethserver.org/${repo_channel}/${nethsecurity_version}" diff --git a/builder/entrypoint.sh b/builder/entrypoint.sh index ff3b2a27f..09f4f5409 100644 --- a/builder/entrypoint.sh +++ b/builder/entrypoint.sh @@ -7,12 +7,9 @@ set -e -if [ -n "$USIGN_PUB_KEY" ] && [ -n "$USIGN_PRIV_KEY" ]; then - echo "$USIGN_PUB_KEY" > /home/buildbot/openwrt/key-build.pub - echo "$USIGN_PRIV_KEY" > /home/buildbot/openwrt/key-build -else - echo "No signing keys found. Generating dummy keys..." - usign -G -s ./key-build -p ./key-build.pub -c "Local build key" +if [ -n "$APK_PRIV_KEY" ] && [ -n "$APK_PUB_KEY" ]; then + echo "$APK_PRIV_KEY" > /home/buildbot/openwrt/private-key.pem + echo "$APK_PUB_KEY" > /home/buildbot/openwrt/public-key.pem fi # if command $1 is a file or a executable, run it diff --git a/config/avahi.conf b/config/avahi.conf new file mode 100644 index 000000000..c06fb1e5c --- /dev/null +++ b/config/avahi.conf @@ -0,0 +1 @@ +CONFIG_PACKAGE_avahi-nodbus-daemon=m diff --git a/config/monitoring.conf b/config/monitoring.conf new file mode 100644 index 000000000..f5e4fa381 --- /dev/null +++ b/config/monitoring.conf @@ -0,0 +1,3 @@ +CONFIG_PACKAGE_victoria-metrics=y +CONFIG_PACKAGE_victoria-logs=m +CONFIG_PACKAGE_telegraf=y diff --git a/config/netdata.conf b/config/netdata.conf deleted file mode 100644 index e4636c07f..000000000 --- a/config/netdata.conf +++ /dev/null @@ -1,2 +0,0 @@ -CONFIG_PACKAGE_netdata=y -CONFIG_PACKAGE_fping=y diff --git a/docs/build/index.md b/docs/build/index.md index 1afbbc4dc..a0c4e9ea9 100644 --- a/docs/build/index.md +++ b/docs/build/index.md @@ -39,7 +39,15 @@ By default, the CI will build the `x86_64` target. To build a different target, To build locally, it's recommended to populate the `build.conf` file with the options you want to use for the build. This file is ignored by Git and should not be committed to the repository. -You can use the `build.conf.example` file as a starting point. Refer to [Environment variables](#environment-variables) for more details on the available options. +The `build.conf.defaults` file contains the versioned defaults and is always tracked by Git. + +You can create a local `build.conf` override that inherits from `build.conf.defaults`: +```bash +cp build.conf.defaults build.conf +# Edit build.conf to override any variables as needed +``` + +Refer to [Environment variables](#environment-variables) for more details on the available options. To build images locally on your machine, make sure these minimum requirements are met: @@ -65,23 +73,30 @@ During the start-up, the container will download netifyd plugins if configuratio ### Environment variables -The `build-nethsec.sh` script behavior can be changed by giving the following environment variables or setting them inside the `build.conf` file: +The `build-nethsec.sh` script behavior can be changed by setting environment variables or by populating the `build.conf` file (git-ignored, local overrides only). + +**Variable loading order:** +1. `build.conf.defaults` (versioned, always loaded first — contains canonical defaults) +2. `build.conf` (git-ignored, optional — can override any variable) +3. Environment variables set before calling `./build-nethsec.sh` (highest priority) + +**Available variables:** - `OWRT_VERSION`: specify the OpenWrt version to build, it can be either a TAG or a branch in the [GitHub OpenWRT repo](https://github.com/openwrt/openwrt); **required** - `NETHSECURITY_VERSION`: specify what to call the NethSecurity image; **required** - `TARGET`: specify the target to build; if not set default is `x86_64` - `REPO_CHANNEL`: specify the channel to publish the image to; if not set default is `dev` -- `USIGN_PUB_KEY` and `USIGN_PRIV_KEY`: see [package signing section](#package-signing) - with the given keys +- `BUILD_SEMVER_SUFFIX`: optional semver suffix appended to the image version only (not the distfeed URL). Use pre-release format (`-rc.1`, `-beta.2`) or metadata format (`+hotfix.1`, `+testing`) or both (`-rc.1+fix.1`). +- `APK_PUB_KEY` and `APK_PRIV_KEY`: see [package signing section](#package-signing) -The `USIGN_PUB_KEY`, `USIGN_PRIV_KEY` variables are always set as secrets inside the CI pipeline, but +The `APK_PUB_KEY`, `APK_PRIV_KEY` variables are always set as secrets inside the CI pipeline, but for [security reasons](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#accessing-secrets) they are not accessible when building pull requests from forks. ### Build locally for a release If you need to build some packages locally for a release, make sure the following environment variables are set: -- `USIGN_PUB_KEY` and `USIGN_PRIV_KEY`: refer to the [package signing section](#package-signing) for more info +- `APK_PUB_KEY` and `APK_PRIV_KEY`: refer to the [package signing section](#package-signing) for more info Then execute the build as described in the [Build locally](#build-locally) section. @@ -133,7 +148,8 @@ Development version example: ## Upstream version change -To change the OpenWrt version used by NethSecurity, you can just replace the `OWRT_VERSION` variable inside the `build.conf.example` file with the new OpenWrt version. +To change the OpenWrt version used by NethSecurity, update the `OWRT_VERSION` variable inside the `build.conf.defaults` file (versioned, always tracked by Git). +This ensures all developers and CI get the same default version. ## Release new image checklist @@ -256,26 +272,26 @@ To replace an upstream package just create a new package with the same name insi ### Package signing -All packages are signed with the following public key generated with [OpenBSD signify](nethsecurity-pub.key). - -Public key fingerprint: `7640d16662de3b89` +Packages are signed using an EC prime256v1 key pair (PEM format) via the APK package manager. -Public key content: +To generate a new signing key pair: ``` -untrusted comment: NethSecurity sign key -RWR2QNFmYt47ieK7g/zEPwgk+MN8bHsA2vFnPThSpnLZ48L7sh6wxB/f +openssl ecparam -name prime256v1 -genkey -noout -out private-key.pem +openssl ec -in private-key.pem -pubout -out public-key.pem ``` -To sign the packages, just execute the `build-nethsec.sh` script with the following environment variables: -- `USIGN_PUB_KEY` -- `USIGN_PRIV_KEY` +To sign the packages, execute the `build-nethsec.sh` script with the following environment variables: +- `APK_PUB_KEY` +- `APK_PRIV_KEY` Usage example: ``` -USIGN_PUB_KEY=$(cat nethsecurity-pub.key) USIGN_PRIV_KEY=$(cat nethsecurity-priv.key) ./build-nethsec.sh +APK_PUB_KEY=$(cat public-key.pem) APK_PRIV_KEY=$(cat private-key.pem) ./build-nethsec.sh ``` -Or you can have the keys as two files named `key-build` and `key-build.pub` in the root of the repository. They will be automatically used by the build script. +Or you can place the keys as two files named `private-key.pem` and `public-key.pem` in the root of the repository. They will be automatically used by the build script. + +If no keys are provided, OpenWrt will auto-generate a throwaway key pair at build time. Builds executed inside CI will sign the packages with the correct key. diff --git a/files/bin/ipcalc.sh b/files/bin/ipcalc.sh deleted file mode 100755 index e8c7a07df..000000000 --- a/files/bin/ipcalc.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/sh - -. /lib/functions/ipv4.sh - -PROG="$(basename "$0")" - -# wrapper to convert an integer to an address, unless we're using -# decimal output format. -# hook for library function -_ip2str() { - local var="$1" n="$2" - assert_uint32 "$n" || exit 1 - - if [ "$decimal" -ne 0 ]; then - export -- "$var=$n" - elif [ "$hexadecimal" -ne 0 ]; then - export -- "$var=$(printf "%x" "$n")" - else - ip2str "$@" - fi -} - -usage() { - echo "Usage: $PROG [ -d | -x ] address/prefix [ start limit ]" >&2 - exit 1 -} - -decimal=0 -hexadecimal=0 -if [ "$1" = "-d" ]; then - decimal=1 - shift -elif [ "$1" = "-x" ]; then - hexadecimal=1 - shift -fi - -if [ $# -eq 0 ]; then - usage -fi - -case "$1" in -*/*.*) - # data is n.n.n.n/m.m.m.m format, like on a Cisco router - str2ip ipaddr "${1%/*}" || exit 1 - str2ip netmask "${1#*/}" || exit 1 - netmask2prefix prefix "$netmask" || exit 1 - shift - ;; -*/*) - # more modern prefix notation of n.n.n.n/p - str2ip ipaddr "${1%/*}" || exit 1 - prefix="${1#*/}" - assert_uint32 "$prefix" || exit 1 - if [ "$prefix" -gt 32 ]; then - printf "Prefix out of range (%s)\n" "$prefix" >&2 - exit 1 - fi - prefix2netmask netmask "$prefix" || exit 1 - shift - ;; -*) - # address and netmask as two separate arguments - str2ip ipaddr "$1" || exit 1 - str2ip netmask "$2" || exit 1 - netmask2prefix prefix "$netmask" || exit 1 - shift 2 - ;; -esac - -# we either have no arguments left, or we have a range start and length -if [ $# -ne 0 ] && [ $# -ne 2 ]; then - usage -fi - -# complement of the netmask, i.e. the hostmask -hostmask=$((netmask ^ 0xffffffff)) -network=$((ipaddr & netmask)) -broadcast=$((network | hostmask)) -count=$((hostmask + 1)) - -_ip2str IP "$ipaddr" -_ip2str NETMASK "$netmask" -_ip2str NETWORK "$network" - -echo "IP=$IP" -echo "NETMASK=$NETMASK" -# don't include this-network or broadcast addresses -if [ "$prefix" -le 30 ]; then - _ip2str BROADCAST "$broadcast" - echo "BROADCAST=$BROADCAST" -fi -echo "NETWORK=$NETWORK" -echo "PREFIX=$prefix" -echo "COUNT=$count" - -# if there's no range, we're done -[ $# -eq 0 ] && exit 0 -[ -z "$1$2" ] && exit 0 - -if [ "$prefix" -le 30 ]; then - lower=$((network + 1)) -else - lower="$network" -fi - -start="$1" -assert_uint32 "$start" || exit 1 -start=$((network | (start & hostmask))) -[ "$start" -lt "$lower" ] && start="$lower" -[ "$start" -eq "$ipaddr" ] && start=$((start + 1)) - -if [ "$prefix" -le 30 ]; then - upper=$(((network | hostmask) - 1)) -elif [ "$prefix" -eq 31 ]; then - upper=$((network | hostmask)) -else - upper="$network" -fi - -range="$2" -assert_uint32 "$range" || exit 1 -end=$((start + range - 1)) -[ "$end" -gt "$upper" ] && end="$upper" -[ "$end" -eq "$ipaddr" ] && end=$((end - 1)) - -if [ "$start" -gt "$end" ]; then - echo "network ($NETWORK/$prefix) too small" >&2 - exit 1 -fi - -_ip2str START "$start" -_ip2str END "$end" - -if [ "$start" -le "$ipaddr" ] && [ "$ipaddr" -le "$end" ]; then - echo "warning: address $IP inside range $START..$END" >&2 -fi - -echo "START=$START" -echo "END=$END" - -exit 0 diff --git a/files/etc/profile.d/busybox-history-file.sh b/files/etc/profile.d/busybox-history-file.sh new file mode 100644 index 000000000..9f2c9ab5f --- /dev/null +++ b/files/etc/profile.d/busybox-history-file.sh @@ -0,0 +1 @@ +export HISTFILE=~/.bash_history diff --git a/packages/adblock/Makefile b/packages/adblock/Makefile index 9cfeef42c..2a9f153fb 100644 --- a/packages/adblock/Makefile +++ b/packages/adblock/Makefile @@ -1,13 +1,13 @@ -# -# Copyright (c) 2015-2023 Dirk Brenken (dev@brenken.org) +# dns based ad/abuse domain blocking +# Copyright (c) 2015-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. # include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=4.1.5 -PKG_RELEASE:=9 +PKG_VERSION:=4.5.5 +PKG_RELEASE:=3 PKG_LICENSE:=GPL-3.0-or-later PKG_MAINTAINER:=Dirk Brenken @@ -16,22 +16,21 @@ include $(INCLUDE_DIR)/package.mk define Package/adblock SECTION:=net CATEGORY:=Network - TITLE:=Powerful adblock script to block ad/abuse domains by using DNS - DEPENDS:=+jshn +jsonfilter +coreutils +coreutils-sort +ca-bundle +opkg + TITLE:=adblock blocks ad/abuse domains by using DNS + DEPENDS:=+jshn +jsonfilter +firewall4 +coreutils +coreutils-sort +gawk +ca-bundle +rpcd +rpcd-mod-rpcsys PKGARCH:=all endef define Package/adblock/description -Powerful adblock solution to block ad/abuse domains via dnsmasq, unbound, named or kresd. -The script supports many domain blacklist sites plus manual black- and whitelist overrides. +adblock blocks ad/abuse domains via dnsmasq, unbound, named, smartdns or kresd. +adblock consumes a minimum of memory, is very fast and supports many domain blocklist sites plus local block- and allowlist overrides. Please see https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md for further information. endef define Package/adblock/conffiles /etc/config/adblock -/etc/adblock/adblock.whitelist -/etc/adblock/adblock.blacklist +/etc/adblock/adblock.custom.feeds endef define Build/Prepare @@ -55,11 +54,13 @@ define Package/adblock/install $(INSTALL_DIR) $(1)/etc/adblock $(INSTALL_BIN) ./files/adblock.mail $(1)/etc/adblock - $(INSTALL_CONF) ./files/adblock.blacklist $(1)/etc/adblock - $(INSTALL_CONF) ./files/adblock.whitelist $(1)/etc/adblock $(INSTALL_CONF) ./files/adblock.categories $(1)/etc/adblock - $(INSTALL_CONF) ./files/adblock.sources $(1)/etc/adblock - gzip -9n $(1)/etc/adblock/adblock.sources + $(INSTALL_CONF) ./files/adblock.feeds $(1)/etc/adblock + $(INSTALL_CONF) ./files/adblock.custom.feeds $(1)/etc/adblock + + $(INSTALL_DIR) $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/95-adblock-housekeeping $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/99_adblock_migrate_lists.sh $(1)/etc/uci-defaults/99_adblock_migrate_lists endef $(eval $(call BuildPackage,adblock)) diff --git a/packages/adblock/files/95-adblock-housekeeping b/packages/adblock/files/95-adblock-housekeeping new file mode 100755 index 000000000..3cef57d79 --- /dev/null +++ b/packages/adblock/files/95-adblock-housekeeping @@ -0,0 +1,114 @@ +#!/bin/sh +# Copyright (c) 2015-2026 Dirk Brenken (dev@brenken.org) +# This is free software, licensed under the GNU General Public License v3. + +# (s)hellcheck exceptions +# shellcheck disable=all + +export LC_ALL=C +export PATH="/usr/sbin:/usr/bin:/sbin:/bin" + +config="adblock" +old_options="adb_sources adb_forcedns adb_fetchutil adb_hag_sources adb_hst_sources adb_stb_sources adb_utc_sources \ + adb_maxqueue adb_backup adb_dnsfilereset adb_tmpbase adb_mailcnt adb_safesearchmod adb_srcfile adb_srcarc adb_nice \ + adb_hag_feed adb_jaildir adb_dnsdenyip adb_dnsallowip adb_zonelist adb_portlist adb_dnsforce adb_replisten" + +for option in ${old_options}; do + inplace="0" + if uci -q get ${config}.global.${option} >/dev/null 2>&1; then + old_values="$(uci -q get ${config}.global.${option})" + for value in ${old_values}; do + case "${option}" in + "adb_sources") + if ! uci -q get ${config}.global.adb_feed | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_feed="${value}" + fi + ;; + "adb_hag_sources") + if ! uci -q get ${config}.global.adb_hag_feed | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_hag_feed="${value}" + fi + ;; + "adb_hst_sources") + if ! uci -q get ${config}.global.adb_hst_feed | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_hst_feed="${value}" + fi + ;; + "adb_stb_sources") + if ! uci -q get ${config}.global.adb_stb_feed | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_stb_feed="${value}" + fi + ;; + "adb_utc_sources") + if ! uci -q get ${config}.global.adb_utc_feed | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_utc_feed="${value}" + fi + ;; + "adb_fetchutil") + uci -q set ${config}.global.adb_fetchcmd="${value}" + ;; + "adb_tmpbase") + uci -q set ${config}.global.adb_basedir="${value}" + ;; + "adb_nice") + uci -q set ${config}.global.adb_nicelimit="${value}" + ;; + "adb_hag_feed") + inplace="1" + if ! printf '%s' "${value}" | grep -qE "^(wildcard/|domains/)"; then + uci -q del_list ${config}.global.adb_hag_feed="${value}" + uci -q add_list ${config}.global.adb_hag_feed="wildcard/${value}" + fi + ;; + "adb_forcedns" | "adb_dnsforce") + uci -q set ${config}.global.adb_nftforce="${value}" + ;; + "adb_zonelist") + if ! uci -q get ${config}.global.adb_nftdevforce | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_nftdevforce="${value}" + fi + ;; + "adb_portlist") + if ! uci -q get ${config}.global.adb_nftportforce | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_nftportforce="${value}" + fi + ;; + "adb_replisten") + if ! uci -q get ${config}.global.adb_repport | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_repport="${value}" + fi + ;; + esac + done + [ "${inplace}" = "0" ] && uci -q delete ${config}.global.${option} + fi +done + +# NethSecurity: migrate adb_bypass to ns_tsdns_bypass +# +if uci -q get ${config}.global.adb_bypass >/dev/null 2>&1; then + old_bypass="$(uci -q get ${config}.global.adb_bypass)" + for value in ${old_bypass}; do + if ! uci -q get ${config}.global.ns_tsdns_bypass | grep -q "${value}"; then + uci -q add_list ${config}.global.ns_tsdns_bypass="${value}" + fi + done + uci -q delete ${config}.global.adb_bypass +fi + +[ -n "$(uci -q changes ${config})" ] && uci -q commit ${config} + +# remove former adblock-related firewall sections (adblock_* redirects and tsdns_bypass ipset) +# +fwcfg="$(uci -qNX show "firewall" | awk 'BEGIN{FS="[.=]"};/adblock_/{if(zone==$2){next}else{ORS=" ";zone=$2;print zone}}')" +for section in ${fwcfg}; do + uci -q delete firewall."${section}" +done +if uci -q get firewall.tsdns_bypass >/dev/null 2>&1; then + uci -q delete firewall.tsdns_bypass +fi +if [ -n "$(uci -q changes firewall)" ]; then + uci -q commit firewall + /etc/init.d/firewall reload +fi +exit 0 diff --git a/packages/adblock/files/99_adblock_migrate_lists.sh b/packages/adblock/files/99_adblock_migrate_lists.sh new file mode 100644 index 000000000..ceea7796d --- /dev/null +++ b/packages/adblock/files/99_adblock_migrate_lists.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Migrate local allow/block list files to staged UCI storage. +# Skip once the dedicated section is already present. +uci -q get adblock.ns_lists >/dev/null 2>&1 && exit 0 + +uci set adblock.ns_lists=ns_lists + +for type in allowlist blocklist; do + file="/etc/adblock/adblock.${type}" + [ -f "${file}" ] || continue + + while IFS= read -r line || [ -n "${line}" ]; do + [ -z "${line}" ] && continue + uci add_list "adblock.ns_lists.${type}=${line}" + done < "${file}" +done + +uci commit adblock diff --git a/packages/adblock/files/README.md b/packages/adblock/files/README.md index 688bdb7cc..725aade52 100644 --- a/packages/adblock/files/README.md +++ b/packages/adblock/files/README.md @@ -2,54 +2,41 @@ # DNS based ad/abuse domain blocking + ## Description -A lot of people already use adblocker plugins within their desktop browsers, but what if you are using your (smart) phone, tablet, watch or any other (wlan) gadget!? Getting rid of annoying ads, trackers and other abuse sites (like facebook) is simple: block them with your router. When the DNS server on your router receives DNS requests, you will sort out queries that ask for the resource records of ad servers and return a simple 'NXDOMAIN'. This is nothing but **N**on-e**X**istent Internet or Intranet domain name, if domain name is unable to resolved using the DNS server, a condition called the 'NXDOMAIN' occurred. +A lot of people already use adblocker plugins within their desktop browsers, but what if you are using your (smart) phone, tablet, watch or any other (wlan) gadget!? Getting rid of annoying ads, trackers and other abuse sites (like facebook) is simple: block them with your router. +When the DNS server on your router receives DNS requests, you will sort out queries that ask for the resource records of ad servers and return a simple 'NXDOMAIN'. This is nothing but **N**on-e**X**istent Internet or Intranet domain name, if a domain name cannot be resolved using the DNS server, a condition called the 'NXDOMAIN' occurred. + + ## Main Features -* Support of the following fully pre-configured domain blocklist sources (free for private usage, for commercial use please check their individual licenses) +* Support of the following fully pre-configured domain blocklist feeds (free for private usage, for commercial use please check their individual licenses) -| Source | Enabled | Size | Focus | Information | +| Feed | Enabled | Size | Focus | Information | | :------------------ | :-----: | :--- | :--------------- | :-------------------------------------------------------------------------------- | -| adaway | x | S | mobile | [Link](https://github.com/AdAway/adaway.github.io) | -| adguard | x | L | general | [Link](https://adguard.com) | -| adguard_tracking | | S | tracking | [Link](https://github.com/AdguardTeam/cname-trackers) | +| 1Hosts | | VAR | compilation | [Link](https://github.com/badmojr/1Hosts) | +| adguard | x | L | general | [Link](https://adguard.com) | +| adguard_tracking | x | L | tracking | [Link](https://github.com/AdguardTeam/cname-trackers) | | android_tracking | | S | tracking | [Link](https://github.com/Perflyst/PiHoleBlocklist) | | andryou | | L | compilation | [Link](https://gitlab.com/andryou/block/-/blob/master/readme.md) | | anti_ad | | L | compilation | [Link](https://github.com/privacy-protection-tools/anti-AD/blob/master/README.md) | -| antipopads | | L | compilation | [Link](https://github.com/AdroitAdorKhan/antipopads-re) | | anudeep | | M | compilation | [Link](https://github.com/anudeepND/blacklist) | | bitcoin | | S | mining | [Link](https://github.com/hoshsadiq/adblock-nocoin-list) | +| certpl | x | L | phishing | [Link](https://cert.pl/en/warning-list/) | | cpbl | | XL | compilation | [Link](https://github.com/bongochong/CombinedPrivacyBlockLists) | -| disconnect | x | S | general | [Link](https://disconnect.me) | +| disconnect | | S | general | [Link](https://disconnect.me) | +| divested | | XXL | compilation | [Link](https://divested.dev/pages/dnsbl) | | doh_blocklist | | S | doh_server | [Link](https://github.com/dibdot/DoH-IP-blocklists) | -| easylist | | M | compilation | [Link](https://easylist.to) | -| easyprivacy | | M | tracking | [Link](https://easylist.to) | | firetv_tracking | | S | tracking | [Link](https://github.com/Perflyst/PiHoleBlocklist) | | games_tracking | | S | tracking | [Link](https://www.gameindustry.eu) | +| hagezi | | VAR | compilation | [Link](https://github.com/hagezi/dns-blocklists) | | hblock | | XL | compilation | [Link](https://hblock.molinero.dev) | -| lightswitch05 | | XL | compilation | [Link](https://github.com/lightswitch05/hosts) | -| notracking | | XL | tracking | [Link](https://github.com/notracking/hosts-blocklists) | +| ipfire_dbl | | VAR | compilation | [Link](https://www.ipfire.org/dbl) | | oisd_big | | XXL | general | [Link](https://oisd.nl) | | oisd_nsfw | | XXL | porn | [Link](https://oisd.nl) | +| oisd_nsfw_small | | M | porn | [Link](https://oisd.nl) | | oisd_small | | L | general | [Link](https://oisd.nl) | -| openphish | | S | phishing | [Link](https://openphish.com) | | phishing_army | | S | phishing | [Link](https://phishing.army) | -| reg_cn | | S | reg_china | [Link](https://easylist.to) | -| reg_cz | | S | reg_czech+slovak | [Link](https://easylist.to) | -| reg_de | | S | reg_germany | [Link](https://easylist.to) | -| reg_es | | S | reg_espania | [Link](https://easylist.to) | -| reg_fi | | S | reg_finland | [Link](https://github.com/finnish-easylist-addition) | -| reg_fr | | M | reg_france | [Link](https://forums.lanik.us/viewforum.php?f=91) | -| reg_id | | S | reg_indonesia | [Link](https://easylist.to) | -| reg_it | | S | reg_italy | [Link](https://easylist.to) | -| reg_jp | | S | reg_japan | [Link](https://github.com/k2jp/abp-japanese-filters) | -| reg_kr | | S | reg_korea | [Link](https://github.com/List-KR/List-KR) | -| reg_nl | | S | reg_netherlands | [Link](https://easylist.to) | -| reg_pl | | M | reg_poland | [Link](https://kadantiscam.netlify.com) | -| reg_ro | | S | reg_romania | [Link](https://easylist.to) | -| reg_ru | | S | reg_russia | [Link](https://easylist.to) | -| reg_se | | S | reg_sweden | [Link](https://github.com/lassekongo83/Frellwits-filter-lists) | -| reg_vn | | S | reg_vietnam | [Link](https://bigdargon.github.io/hostsVN) | | smarttv_tracking | | S | tracking | [Link](https://github.com/Perflyst/PiHoleBlocklist) | | spam404 | | S | general | [Link](https://github.com/Dawsey21) | | stevenblack | | VAR | compilation | [Link](https://github.com/StevenBlack/hosts) | @@ -57,68 +44,76 @@ A lot of people already use adblocker plugins within their desktop browsers, but | utcapitole | | VAR | general | [Link](https://dsi.ut-capitole.fr/blacklists/index_en.php) | | wally3k | | S | compilation | [Link](https://firebog.net/about) | | whocares | | M | general | [Link](https://someonewhocares.org) | -| winhelp | | S | general | [Link](https://winhelp2002.mvps.org) | | winspy | | S | win_telemetry | [Link](https://github.com/crazy-max/WindowsSpyBlocker) | -| yoyo | x | S | general | [Link](https://pgl.yoyo.org/adservers) | - -* List of supported and fully pre-configured adblock sources, already active sources are pre-selected. - To avoid OOM errors, please do not select too many lists! - List size information with the respective domain ranges as follows: - • S (-10k), M (10k-30k) and L (30k-80k) should work for 128 MByte devices, - • XL (80k-200k) should work for 256-512 MByte devices, - • XXL (200k-) needs more RAM and Multicore support, e.g. x86 or raspberry devices. - • VAR (50k-500k) variable size depending on the selection. +| yoyo | | S | general | [Link](https://pgl.yoyo.org/adservers) | + +* List of supported and fully pre-configured adblock sources, already active sources are pre-selected. + To avoid OOM errors, please do not select too many lists! + List size information with the respective domain ranges as follows: + • S (-10k), M (10k-30k) and L (30k-80k) should work for 128 MByte devices + • XL (80k-200k) should work for 256-512 MByte devices + • XXL (200k-) needs more RAM and Multicore support, e.g. x86 or raspberry devices + • VAR (50k-900k) variable size depending on the selection * Zero-conf like automatic installation & setup, usually no manual changes needed * Simple but yet powerful adblock engine: adblock does not use error prone external iptables rulesets, http pixel server instances and things like that -* Supports five different DNS backend formats: dnsmasq, unbound, named (bind), kresd or raw (e.g. used by dnscrypt-proxy) -* Supports four different SSL-enabled download utilities: uclient-fetch, wget, curl or aria2c -* Supports SafeSearch for google, bing, duckduckgo, yandex, youtube and pixabay -* Supports RPZ-trigger 'RPZ-CLIENT-IP' to always allow/deny certain DNS clients based on their IP address (currently only supported by bind dns backend) +* Supports six different DNS backend formats: dnsmasq, unbound, named (bind), kresd, smartdns or raw (e.g. used by dnscrypt-proxy) +* Supports three different SSL-enabled download utilities: uclient-fetch, full wget or curl +* Supports SafeSearch for google, bing, brave, duckduckgo, yandex, youtube and pixabay * Fast downloads & list processing as they are handled in parallel running background jobs with multicore support +* The download engine supports ETAG headers to download only updated feeds * Supports a wide range of router modes, even AP modes are supported * Full IPv4 and IPv6 support * Provides top level domain compression ('tld compression'), this feature removes thousands of needless host entries from the blocklist and lowers the memory footprint for the DNS backend -* Provides a 'DNS File Reset', where the generated DNS blocklist file will be purged after DNS backend loading to save storage space -* Source parsing by fast & flexible regex rulesets, all rules and source information are placed in an external/compredd JSON file ('/etc/adblock/adblock.sources.gz') +* Provides a 'DNS Blocklist Shift', where the generated final DNS blocklist is moved to the backup directory and only a soft link to this file is set in memory. As long as your backup directory is located on an external drive, you should activate this option to save valuable RAM. +* Feed parsing by a very fast & secure domain validator, all domain rules and feed information are placed in an external JSON file ('/etc/adblock/adblock.feeds') * Overall duplicate removal in generated blocklist file 'adb_list.overall' -* Additional local blacklist for manual overrides, located in '/etc/adblock/adblock.blacklist' -* Additional local whitelist for manual overrides, located in '/etc/adblock/adblock.whitelist' -* Quality checks during blocklist update to ensure a reliable DNS backend service +* Additional local allowlist for manual overrides, located in '/etc/adblock/adblock.allowlist' (only exact matches). +* Additional local blocklist for manual overrides, located in '/etc/adblock/adblock.blocklist' +* Implements firewall‑based DNS Control to force DNS interfaces/ports and to redirect to external unfiltered/filtered DNS server +* Includes firewall‑based Remote DNS Allow, a CGI-Interface to allow certain MACs temporary bypass the local adblock DNS +* Supports firewall‑based temporary DNS Bridging, to ensure a Zero‑Downtime during adblock-related DNS Restarts +* Connection checks during blocklist update to ensure a reliable DNS backend service * Minimal status & error logging to syslog, enable debug logging to receive more output -* Procd based init system support ('start', 'stop', 'restart', 'reload', 'enable', 'disable', 'running', 'status', 'suspend', 'resume', 'query', 'report', 'list', 'timer') +* Procd based init system support ('start', 'stop', 'restart', 'reload', 'enable', 'disable', 'running', 'status', 'suspend', 'resume', 'search', 'report') * Auto-Startup via procd network interface trigger or via classic time based startup -* Suspend & Resume adblock temporarily without blocklist reloading +* Suspend & Resume adblock temporarily without blocklist re-processing * Provides comprehensive runtime information -* Provides a detailed DNS Query Report with DNS related information about client requests, top (blocked) domains and more -* Provides a powerful query function to quickly find blocked (sub-)domains, e.g. for whitelisting -* Provides an easily configurable blocklist update scheduler called 'Refresh Timer' -* Includes an option to generate an additional, restrictive 'adb_list.jail' to block access to all domains except those listed in the whitelist file. You can use this restrictive blocklist manually e.g. for guest wifi or kidsafe configurations -* Includes an option to force DNS requests to the local resolver +* Provides a detailed DNS Report with DNS related information about client requests, top (blocked) domains and more +* Provides a powerful search function to quickly find blocked (sub-)domains, e.g. to allow certain domains +* Implements a jail mode - only domains on the allowlist are permitted, all other DNS requests are rejected * Automatic blocklist backup & restore, these backups will be used in case of download errors and during startup -* Send notification E-Mails in case of a processing error or if the overall domain count is ≤ 0 -* Add new adblock sources on your own, see example below +* Send notification E-Mails, see example configuration below +* Add new adblock feeds on your own with the 'Custom Feed Editor' in LuCI or via CLI, see example below * Strong LuCI support, all relevant options are exposed to the web frontend + ## Prerequisites -* [OpenWrt](https://openwrt.org), tested with the stable release series and with the latest rolling snapshot releases. - Please note: Devices with less than 128 MByte RAM are _not_ supported! -* A usual setup with an enabled DNS backend at minimum - dumb AP modes without a working DNS backend are _not_ supported -* A download utility with SSL support: 'wget', 'uclient-fetch' with one of the 'libustream-*' ssl libraries, 'aria2c' or 'curl' is required +* **[OpenWrt](https://openwrt.org)**, latest stable release or a development snapshot +* A usual setup with a working DNS backend +* A download utility with SSL support: 'wget', 'uclient-fetch' with one of the 'libustream-*' ssl libraries or 'curl' is required * A certificate store such as 'ca-bundle' or 'ca-certificates', as adblock checks the validity of the SSL certificates of all download sites by default -* Optional E-Mail notification support: for E-Mail notifications you need to install the additional 'msmtp' package -* Optional DNS Query Report support: for DNS reporting you need to install the additional package 'tcpdump-mini' or 'tcpdump' -* Optional support for gnu awk as alternative to the busybox default, install the additional package 'gawk' +* For E-Mail notifications you need to install and setup the additional 'msmtp' package +* For DNS reporting you need to install the additional package 'tcpdump-mini' or 'tcpdump' +**Please note:** +* Devices with less than 128MB of RAM are **_not_** supported +* For performance reasons, adblock depends on gnu sort and gawk +* Before update from former adblock releases please make a backup of your local allow- and blocklists. In the latest adblock these lists have been renamed to '/etc/adblock/adblock.allowlist' and '/etc/adblock/adblock.blocklist'. There is no automatic content transition to the new files. +* The uci configuration of adblock is automatically migrated during package installation via the uci-defaults mechanism using a housekeeping script + + ## Installation & Usage -* Update your local opkg repository (_opkg update_) -* Install 'adblock' (_opkg install adblock_). The adblock service is enabled by default -* Install the LuCI companion package 'luci-app-adblock' (_opkg install luci-app-adblock_) +* Make a backup and update your local opkg/apk repository +* Install the LuCI companion package 'luci-app-adblock' which also installs the main 'adblock' package as a dependency +* Enable the adblock system service (System -> Startup) and enable adblock itself (adblock -> General Settings) * It's strongly recommended to use the LuCI frontend to easily configure all aspects of adblock, the application is located in LuCI under the 'Services' menu -* Update from a former adblock version is easy. During the update a backup is made of the old configuration '/etc/config/adblock-backup' and replaced by the new config - that's all +* It's also strongly recommended to configure a ‘Startup Trigger Interface’ to ensure automatic adblock startup on WAN-ifup events during boot or reboot of your router -## Adblock CLI Options -* All important adblock functions are accessible via CLI as well. -

+
+## Adblock CLI interface
+* The most important adblock functions are accessible via CLI as well.
+
+```
 ~# /etc/init.d/adblock
 Syntax: /etc/init.d/adblock [command]
 
@@ -132,73 +127,91 @@ Available commands:
 	enabled         Check if service is started on boot
 	suspend         Suspend adblock processing
 	resume          Resume adblock processing
-	query           <domain> Query active blocklists and backups for a specific domain
-	report          [<search>] Print DNS statistics with an optional search parameter
-	list            [<add>|<add_utc>|<add_eng>|<add_stb>|<remove>|<remove_utc>|<remove_eng>|<remove_stb>] <source(s)> List/Edit available sources
-	timer           [<add> <tasks> <hour> [<minute>] [<weekday>]]|[<remove> <line no.>] List/Edit cron update intervals
-	version         Print version information
+	search           Search active blocklists and backups for a specific domain
+	report          [|||] Print DNS statistics
 	running         Check if service is running
 	status          Service status
 	trace           Start with syscall trace
-
+ info Dump procd service info +``` + ## Adblock Config Options * Usually the auto pre-configured adblock setup works quite well and no manual overrides are needed -| Option | Default | Description/Valid Values | -| :----------------- | :--------------------------------- | :--------------------------------------------------------------------------------------------- | -| adb_enabled | 1, enabled | set to 0 to disable the adblock service | -| adb_srcarc | -, /etc/adblock/adblock.sources.gz | full path to the used adblock source archive | -| adb_srcfile | -, /tmp/adb_sources.json | full path to the used adblock source file, which has a higher precedence than the archive file | -| adb_dns | -, auto-detected | 'dnsmasq', 'unbound', 'named', 'kresd' or 'raw' | -| adb_fetchutil | -, auto-detected | 'uclient-fetch', 'wget', 'curl' or 'aria2c' | -| adb_fetchparm | -, auto-detected | manually override the config options for the selected download utility | -| adb_fetchinsecure | 0, disabled | don't check SSL server certificates during download | -| adb_trigger | -, not set | trigger network interface or 'not set' to use a time-based startup | -| adb_triggerdelay | 2 | additional trigger delay in seconds before adblock processing begins | -| adb_debug | 0, disabled | set to 1 to enable the debug output | -| adb_nice | 0, standard prio. | valid nice level range 0-19 of the adblock processes | -| adb_forcedns | 0, disabled | set to 1 to force DNS requests to the local resolver | -| adb_bypass | -, not set | list of IP addresses excluded from `adb_forcedns` option | -| adb_dnsdir | -, auto-detected | path for the generated blocklist file 'adb_list.overall' | -| adb_dnstimeout | 10 | timeout in seconds to wait for a successful DNS backend restart | -| adb_dnsinstance | 0, first instance | set to the relevant dns backend instance used by adblock (dnsmasq only) | -| adb_dnsflush | 0, disabled | set to 1 to flush the DNS Cache before & after adblock processing | -| adb_dnsallow | -, not set | set to 1 to disable selective DNS whitelisting (RPZ-PASSTHRU) | -| adb_lookupdomain | example.com | external domain to check for a successful DNS backend restart or 'false' to disable this check | -| adb_portlist | 53 853 5353 | space separated list of firewall ports which should be redirected locally | -| adb_report | 0, disabled | set to 1 to enable the background tcpdump gathering process for reporting | -| adb_reportdir | /tmp | path for DNS related report files | -| adb_repiface | -, auto-detected | name of the reporting interface or 'any' used by tcpdump | -| adb_replisten | 53 | space separated list of reporting port(s) used by tcpdump | -| adb_repchunkcnt | 5 | report chunk count used by tcpdump | -| adb_repchunksize | 1 | report chunk size used by tcpdump in MB | -| adb_represolve | 0, disabled | resolve reporting IP addresses using reverse DNS (PTR) lookups | -| adb_backup | 1, enabled | set to 0 to disable the backup function | -| adb_backupdir | /tmp | path for adblock backups | -| adb_tmpbase | /tmp | path for all adblock related runtime operations, e.g. downloading, sorting, merging etc. | -| adb_safesearch | 0, disabled | set to 1 to enforce SafeSearch for google, bing, duckduckgo, yandex, youtube and pixabay | -| adb_safesearchlist | -, not set | Limit SafeSearch to certain provider (see above) | -| adb_safesearchmod | 0, disabled | set to 1 to enable moderate SafeSearch filters for youtube | -| adb_mail | 0, disabled | set to 1 to enable notification E-Mails in case of a processing errors | -| adb_mailreceiver | -, not set | receiver address for adblock notification E-Mails | -| adb_mailsender | no-reply@adblock | sender address for adblock notification E-Mails | -| adb_mailtopic | adblock notification | topic for adblock notification E-Mails | -| adb_mailprofile | adb_notify | mail profile used in 'msmtp' for adblock notification E-Mails | -| adb_mailcnt | 0 | minimum domain count to trigger E-Mail notifications | -| adb_jail | 0 | set to 1 to enable the additional, restrictive 'adb_list.jail' creation | -| adb_jaildir | /tmp | path for the generated jail list | +| Option | Default | Description/Valid Values | +| :------------------- | :--------------------------------- | :------------------------------------------------------------------------------------------------- | +| adb_enabled | 1, enabled | set to 0 to disable the adblock service | +| adb_feedfile | /etc/adblock/adblock.feeds | full path to the used adblock feed file | +| adb_dns | -, auto-detected | 'dnsmasq', 'unbound', 'named', 'kresd', 'smartdns' or 'raw' | +| adb_cores | -, auto-detected | limit the cpu cores used by adblock to save RAM | +| adb_fetchcmd | -, auto-detected | 'uclient-fetch', 'wget' or 'curl' | +| adb_fetchparm | -, auto-detected | manually override the config options for the selected download utility | +| adb_fetchinsecure | 0, disabled | don't check SSL server certificates during download | +| adb_trigger | -, not set | trigger network interface or 'not set' to use a time-based startup | +| adb_triggerdelay | 5 | additional trigger delay in seconds before adblock processing begins | +| adb_debug | 0, disabled | set to 1 to enable the debug output | +| adb_nicelimit | 0, standard prio. | valid nice level range 0-19 of the adblock processes | +| adb_dnsshift | 0, disabled | shift the blocklist to the backup directory and only set a soft link to this file in memory | +| adb_dnsdir | -, auto-detected | path for the generated blocklist file 'adb_list.overall' | +| adb_dnstimeout | 20 | timeout in seconds to wait for a successful DNS backend restart | +| adb_dnsinstance | 0, first instance | set the relevant dnsmasq backend instance used by adblock | +| adb_dnsflush | 0, disabled | set to 1 to flush the DNS Cache before & after adblock processing | +| adb_lookupdomain | localhost | domain to check for a successful DNS backend restart | +| adb_report | 0, disabled | set to 1 to enable the background tcpdump gathering process for reporting | +| adb_map | 0, disabled | enable a GeoIP Map with blocked domains | +| adb_reportdir | /tmp/adblock-report | path for DNS related report files | +| adb_repiface | -, auto-detected | name of the reporting interface or 'any' used by tcpdump | +| adb_repport | 53 | list of reporting port(s) used by tcpdump | +| adb_repchunkcnt | 5 | report chunk count used by tcpdump | +| adb_repchunksize | 1 | report chunk size used by tcpdump in MB | +| adb_represolve | 0, disabled | resolve reporting IP addresses using reverse DNS (PTR) lookups | +| adb_tld | 1, enabled | set to 0 to disable the top level domain compression (tld) function | +| adb_basedir | /tmp | path for all adblock related runtime operations, e.g. downloading, sorting, merging etc. | +| adb_backupdir | /tmp/adblock-backup | path for adblock backups | +| adb_safesearch | 0, disabled | enforce SafeSearch for google, bing, brave, duckduckgo, yandex, youtube and pixabay | +| adb_safesearchlist | -, not set | limit SafeSearch to certain provider (see above) | +| adb_mail | 0, disabled | set to 1 to enable notification E-Mails in case of a processing errors | +| adb_mailreceiver | -, not set | receiver address for adblock notification E-Mails | +| adb_mailsender | no-reply@adblock | sender address for adblock notification E-Mails | +| adb_mailtopic | adblock notification | topic for adblock notification E-Mails | +| adb_mailprofile | adb_notify | mail profile used in 'msmtp' for adblock notification E-Mails | +| adb_jail | 0 | jail mode - only domains on the allowlist are permitted, all other DNS requests are rejected | +| adb_nftforce | 0, disabled | redirect all local DNS queries from specified LAN zones to the local DNS resolver | +| adb_nftdevforce | -, not set | firewall LAN Devices/VLANs that should be forced locally | +| adb_nftportforce | -, not set | firewall ports that should be forced locally | +| adb_nftallow | 0, disabled | routes MACs or interfaces to an unfiltered external DNS resolver, bypassing local adblock | +| adb_nftmacallow | -, not set | listed MAC addresses will always use the configured unfiltered DNS server | +| adb_nftdevallow | -, not set | entire interfaces or VLANs will be routed to the unfiltered DNS server | +| adb_allowdnsv4 | -, not set | external IPv4 DNS resolver applied to MACs and interfaces using the unfiltered DNS policy | +| adb_allowdnsv6 | -, not set | external IPv6 DNS resolver applied to MACs and interfaces using the unfiltered DNS policy | +| adb_nftremote | 0, disabled | routes MACs to an unfiltered external DNS resolver, bypassing local adblock | +| adb_nftmacremote | -, not set | Allows listed MACs to remotely access an unfiltered external DNS resolver, bypassing local adblock | +| adb_nftremotetimeout | 15 | Time limit in minutes for remote DNS access of the listed MAC addresses | +| adb_remotednsv4 | -, not set | external IPv4 DNS resolver applied to MACs using the unfiltered remote DNS policy | +| adb_remotednsv6 | -, not set | external IPv6 DNS resolver applied to MACs using the unfiltered remote DNS policy | +| adb_nftblock | 0, disabled | routes MACs or interfaces to a filtered external DNS resolver, bypassing local adblock | +| adb_nftmacblock | -, not set | listed MAC addresses will always use the configured filtered DNS server | +| adb_nftdevblock | -, not set | entire interfaces or VLANs will be routed to the filtered DNS server | +| adb_blockdnsv4 | -, not set | external IPv4 DNS resolver applied to MACs and interfaces using the filtered DNS policy | +| adb_blockdnsv6 | -, not set | external IPv6 DNS resolver applied to MACs and interfaces using the filtered DNS policy | +| adb_nftbridge | -, not set | enables a temporary DNS bridge to an external DNS resolver during local DNS restarts | +| adb_bridgednsv4 | -, not set | external IPv4 DNS resolver used during bridging | +| adb_bridgednsv6 | -, not set | external IPv6 DNS resolver used during bridging | + ## Examples -**Change the DNS backend to 'unbound':** -No further configuration is needed, adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/unbound' by default. + +**Change the DNS backend to 'unbound':** +No further configuration is needed, adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/unbound' by default. To preserve the DNS cache after adblock processing please install the additional package 'unbound-control'. -**Change the DNS backend to 'bind':** -Adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/bind' by default. -To preserve the DNS cache after adblock processing please install the additional package 'bind-rdnc'. +**Change the DNS backend to 'bind':** +Adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/bind' by default. +To preserve the DNS cache after adblock processing please install the additional package 'bind-rndc'. To use the blocklist please modify '/etc/bind/named.conf': -

+
+```
 in the 'options' namespace add:
   response-policy { zone "rpz"; };
 
@@ -209,26 +222,108 @@ and at the end of the file add:
     allow-query { none; };
     allow-transfer { none; };
   };
-
+``` + +**Change the DNS backend to 'kresd':** +Adblock deposits the final blocklist 'adb_list.overall' in '/tmp/kresd', no further configuration needed. + +**Change the DNS backend to 'smartdns':** +No further configuration is needed, adblock deposits the final blocklist 'adb_list.overall' in '/tmp/smartdns' by default. + +**Service status output:** +In LuCI you'll see the realtime status in the 'Runtime' section on the overview page. +To get the status in the CLI, just call _/etc/init.d/adblock status_ or _/etc/init.d/adblock status\_service_: + +```sh +root@blackhole:~# /etc/init.d/adblock status +::: adblock runtime information + + adblock_status : enabled + + frontend_ver : 4.5.2-r4 + + backend_ver : 4.5.2-r4 + + blocked_domains : 888 135 + + active_feeds : 1hosts, adguard, adguard_tracking, bitcoin, certpl, doh_blocklist, hagezi, ipfire_dbl, phishing_army, smarttv_tracking, stevenblack, winspy + + dns_backend : unbound (1.24.2-r1), /mnt/data/adblock/backup, 346.57 MB + + run_ifaces : trigger: wan, report: br-lan + + run_information : base: /mnt/data/adblock, dns: /var/lib/unbound, backup: /mnt/data/adblock/backup, report: /mnt/data/adblock/report, error: /dev/null + + run_flags : shift: ✔, custom feed: ✘, ext. DNS (std/prot/remote/bridge): ✘/✔/✔/✔, force: ✔, flush: ✘, tld: ✔, search: ✘, report: ✔, mail: ✔, jail: ✘, debug: ✘ + + last_run : mode: reload, 2026-03-12T19:08:41+01:00, duration: 0m 57s, 1337.01 MB available + + system_info : cores: 4, fetch: curl, Bananapi BPI-R3, mediatek/filogic, OpenWrt SNAPSHOT (r33360-ab0872a734) +``` + + +## Best practice and tweaks + +**Recommendation for low memory systems** +adblock keeps all working data in RAM to avoid unnecessary flash wear. On devices with only 128–256 MB RAM, you can reduce memory pressure with the following optimizations: +* Use external storage: Set adb_basedir, adb_backupdir and adb_reportdir to a USB drive or SSD to offload temporary and persistent data +* Limit CPU parallelism: Set adb_cores=1 to reduce peak memory usage during feed processing +* Enable blocklist shifting: Activate adb_dnsshift to store the generated blocklist on external storage and keep only a symlink in RAM +* Use firewall‑based DNS redirection: Route DNS queries via nftables to external filtered DNS resolvers and keep only a minimal local blocklist active + +**Sensible choice of blocklists** +The following feeds are just my personal recommendation as an initial setup: +* 'adguard', 'adguard_tracking' and 'certpl' + +In total, this feed selection blocks about 280K domains. It may also be useful to include compilations like hagezi, stevenblack or oisd. +Please note: don't just blindly activate too many feeds at once, sooner or later this will lead to OOM conditions. + +**DNS reporting, enable the GeoIP Map** +adblock includes a powerful reporting tool on the DNS Report tab which shows the latest DNS statistics generated by tcpdump. To get the latest statistics always press the "Refresh" button. +In addition to a tabular overview adblock reporting includes a GeoIP map in a modal popup window/iframe that shows the geolocation of your own uplink addresses (in green) and the locations of blocked domains in red. To enable the GeoIP Map set the following option in "Advanced Report Settings" config tab: set 'adb_map' to '1' to include the external components listed below and activate the GeoIP map. + +To make this work, adblock uses the following external components: +* [Leaflet](https://leafletjs.com/) is a lightweight open-source JavaScript library for interactive maps +* [OpenStreetMap](https://www.openstreetmap.org/) provides the map data under an open-source license +* [CARTO basemap styles](https://github.com/CartoDB/basemap-styles) based on [OpenMapTiles](https://openmaptiles.org/schema) +* The free and quite fast [IP Geolocation API](https://ip-api.com/) to resolve the required IP/geolocation information (max. 45 blocked Domains per request) + +**External adblock test** +In addition to the built‑in DNS reporting and GeoIP map, adblock users can verify the effectiveness of their configuration with an external test page. The [Adblock Test](https://adblock.turtlecute.org/) provides a simple way to check whether your current adblock setup is working as expected. It loads a series of test elements (ads, trackers, and other resources) and reports whether they are successfully blocked by your configuration. + +The test runs entirely in the browser and does not require additional configuration. For best results, open the page in the same environment where adblock is active and review the results displayed. + +**Firewall‑Based DNS Control** +adblock provides several advanced firewall‑integrated features that allow you to enforce DNS policies directly at the network layer. These mechanisms operate independently of the local DNS resolver and ensure that DNS traffic follows your filtering rules, even when clients attempt to bypass them. +* Unfiltered external DNS Routing: routes DNS queries from selected devices or interfaces to an external unfiltered DNS resolver +* Filtered external DNS Routing: routes DNS queries from selected devices or interfaces to an external filtered DNS resolver +* Force DNS: blocks or redirects all external DNS traffic to ensure that clients use the local resolver + +The DNS routing allows you to apply external DNS (unfiltered and/or filtered) to specific devices or entire network segments. DNS queries from these targets are transparently redirected to a chosen external resolver (IPv4 and/or IPv6): +* MAC‑based targeting for individual devices +* Interface/VLAN targeting for entire segments +* Separate IPv4/IPv6 resolver selection +* Transparent DNS redirection without client‑side configuration +This mode is ideal for guest networks, IoT devices, or environments where certain clients require stricter/lesser DNS filtering. + +force DNS ensures that all DNS traffic on your network by specific devices or entire network segments is processed by the local resolver. Any attempt to use external DNS servers is blocked or redirected. +* Blocks external DNS on port 53 and redirects DNS queries to the local resolver when appropriate +* Also prevents DNS bypassing by clients with hardcoded DNS settings on other ports, e.g. on port 853 +This mode guarantees that adblock’s filtering pipeline is always applied. + +adblock's firewall rules are based on nftables in a separate isolated nftables table (inet adblock) and chains (prerouting), with MAC addresses stored in a nftables set. The configuration is carried out centrally in LuCI on the ‘Firewall Settings’ tab in adblock. + +**Remote DNS Allow (Temporary MAC‑Based Bypass)** +This additional firewall feature lets selected client devices temporarily bypass local DNS blocking and use an external, unfiltered DNS resolver. It is designed for situations where a device needs short‑term access to content normally blocked by the adblock rules. -**Change the DNS backend to 'kresd':** -Adblock deposits the final blocklist 'adb_list.overall' in '/etc/kresd', no further configuration needed. -Please note: The knot-resolver (kresd) is only available on Turris devices and does not support the SafeSearch functionality yet. +A lightweight CGI endpoint handles the workflow: +* The client opens the URL, e.g. https://\cgi-bin/adblock (preferably transferred via QR code shown in LuCI) +* The script automatically detects the device’s MAC address +* If the MAC is authorized, the script displays the current status: + * Not in the nftables set → option to request a temporary allow (“Renew”) + * Already active → shows remaining timeout +* When renewing, the CGI adds the MAC to an nftables Set with a per‑entry timeout -**Use restrictive jail modes:** -You can enable a restrictive 'adb_list.jail' to block access to all domains except those listed in the whitelist file. Usually this list will be generated as an additional list for guest or kidsafe configurations (for a separate dns server instance). If the jail directory points to your primary dns directory, adblock enables the restrictive jail mode automatically (jail mode only). +The CGI interface is mobile‑friendly and includes a LuCI‑style loading spinner during the renew process, giving immediate visual feedback while the nftables entry is created. All operations are atomic and safe even when multiple devices renew access in parallel. -**Manually override the download options:** -By default adblock uses the following pre-configured download options: -* aria2c: --timeout=20 --allow-overwrite=true --auto-file-renaming=false --log-level=warn --dir=/ -o -* curl: --connect-timeout 20 --silent --show-error --location -o -* uclient-fetch: --timeout=20 -O -* wget: --no-cache --no-cookies --max-redirect=0 --timeout=20 -O +**Temporary DNS Bridging (Zero‑Downtime during DNS Restarts)** +Adblock can optionally enable a temporary DNS bridging mode to avoid DNS downtime during DNS backend restarts. +When this feature is enabled, all DNS queries from LAN clients are briefly redirected to an external fallback resolver until the local DNS backend becomes available again. This ensures that DNS resolution continues to work seamlessly for all clients, even while adblock reloads blocklists or restarts the DNS service. Just set the options 'adb_nftbridge', 'adb_bridgednsv4' and 'adb_bridgednsv6' accordingly. -To override the default set 'adb_fetchparm' manually to your needs. +**Jail mode (allowlist-only):** +Enforces a strict allowlist‑only DNS policy in which only domains listed in the allowlist file are resolved, while every other query is rejected. This mode is intended for highly restrictive environments and depends on a carefully maintained allowlist, typically managed manually. -**Enable E-Mail notification via 'msmtp':** -To use the email notification you have to install & configure the package 'msmtp'. +**Enable E-Mail notification via 'msmtp':** +To use the email notification you have to install & configure the package 'msmtp'. Modify the file '/etc/msmtprc': 

 [...]
@@ -246,70 +341,102 @@ from            dev.adblock@gmail.com
 user            dev.adblock
 password        xxx
-Finally enable E-Mail support and add a valid E-Mail receiver address in LuCI. +Finally enable E-Mail support, add a valid E-Mail receiver address in LuCI and setup an appropriate cron job. -**Service status output:** -In LuCI you'll see the realtime status in the 'Runtime' section on the overview page. -To get the status in the CLI, just call _/etc/init.d/adblock status_ or _/etc/init.d/adblock status\_service_: -

-~#@blackhole:~# /etc/init.d/adblock status
-::: adblock runtime information
-  + adblock_status  : enabled
-  + adblock_version : 4.1.4
-  + blocked_domains : 268355
-  + active_sources  : adaway, adguard, adguard_tracking, android_tracking, bitcoin, disconnect, firetv_tracking, games_t
-                      racking, hblock, oisd_basic, phishing_army, smarttv_tracking, stopforumspam, wally3k, winspy, yoyo
-  + dns_backend     : unbound (unbound-control), /var/lib/unbound
-  + run_utils       : download: /usr/bin/curl, sort: /usr/libexec/sort-coreutils, awk: /bin/busybox
-  + run_ifaces      : trigger: wan, report: br-lan
-  + run_directories : base: /tmp, backup: /mnt/data/adblock-Backup, report: /mnt/data/adblock-Report, jail: /tmp
-  + run_flags       : backup: ✔, flush: ✘, force: ✔, search: ✘, report: ✔, mail: ✔, jail: ✘
-  + last_run        : restart, 3m 17s, 249/73/68, 2022-09-10T13:43:07+02:00
-  + system          : ASUS RT-AX53U, OpenWrt SNAPSHOT r20535-2ca5602864
-
-The 'last\_run' line includes the used start type, the run duration, the memory footprint after DNS backend loading (total/free/available) and the date/time of the last run. +**Automatic adblock feed updates and E-Mail reports** +For a regular, automatic update of the used feeds or other regular adblock tasks set up a cron job. In LuCI you find the cron settings under 'System' => 'Scheduled Tasks'. On the command line the cron file is located at '/etc/crontabs/root': -**Edit, add new adblock sources:** -The adblock blocklist sources are stored in an external, compressed JSON file '/etc/adblock/adblock.sources.gz'. -This file is directly parsed in LuCI and accessible via CLI, just call _/etc/init.d/adblock list_: -

-/etc/init.d/adblock list
-::: Available adblock sources
-:::
-    Name                 Enabled   Size   Focus               Info URL
-    ------------------------------------------------------------------
-  + adaway               x         S      mobile              https://adaway.org
-  + adguard              x         L      general             https://adguard.com
-  + andryou              x         L      compilation         https://gitlab.com/andryou/block/-/blob/master/readme.md
-  + bitcoin              x         S      mining              https://github.com/hoshsadiq/adblock-nocoin-list
-  + disconnect           x         S      general             https://disconnect.me
-  + dshield                        XL     general             https://www.dshield.org
-[...]
-  + winhelp                        S      general             http://winhelp2002.mvps.org
-  + winspy               x         S      win_telemetry       https://github.com/crazy-max/WindowsSpyBlocker
-  + yoyo                 x         S      general             https://pgl.yoyo.org
-
+Example 1 +```sh +# update the adblock feeds every morning at 4 o'clock +00 04 * * * /etc/init.d/adblock reload +``` -To add new or edit existing sources extract the compressed JSON file _gunzip /etc/adblock/adblock.sources.gz_. -A valid JSON source object contains the following required information, e.g.: -

+Example 2
+```sh
+# update the adblock feeds every hour
+0 */1 * * * /etc/init.d/adblock reload
+```
+
+Example 3
+```sh
+# send an adblock E-Mail report every morning at 3 o'clock
+00 03 * * * /etc/init.d/adblock report mail
+```
+
+**Change/add adblock feeds**
+The adblock blocklist feeds are stored in an external JSON file '/etc/adblock/adblock.feeds'. All custom changes should be stored in an external JSON file '/etc/adblock/adblock.custom.feeds' (empty by default). It's recommended to use the LuCI based Custom Feed Editor to make changes to this file.
+A valid JSON source object contains the following information, e.g.:
+
+```json
 	[...]
-	"adaway": {
-		"url": "https://raw.githubusercontent.com/AdAway/adaway.github.io/master/hosts.txt",
-		"rule": "/^127\\.0\\.0\\.1[[:space:]]+([[:alnum:]_-]+\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}",
-		"size": "S",
-		"focus": "mobile",
-		"descurl": "https://github.com/AdAway/adaway.github.io"
+	"stevenblack": {
+		"url": "https://raw.githubusercontent.com/StevenBlack/hosts/master/",
+		"rule": "feed 0.0.0.0 2",
+		"size": "VAR",
+		"descr": "compilation"
 	},
 	[...]
-
-Add an unique object name, make the required changes to 'url', 'rule', 'size' and 'descurl' and finally compress the changed JSON file _gzip /etc/adblock/adblock.sources_ to use the new source object in adblock. -Please note: if you're going to add new sources on your own, please make a copy of the default file and work with that copy further on, cause the default will be overwritten with every adblock update. To reference your copy set the option 'adb\_srcarc' which points by default to '/etc/adblock/adblock.sources.gz' -Please note: when adblock starts, it looks for the uncompressed 'adb\_srcfile', only if this file is not found the archive 'adb\_srcarc' is unpacked once and then the uncompressed file is used +``` + +Add a unique feed name (no spaces, no special chars) and make the required changes: adapt at least the URL, check/change the rule, the size and the description for a new feed. +The rule consist of max. 4 individual, space separated parameters: +1. type: always 'feed' (required) +2. prefix: an optional search term (a string literal, no regex) to identify valid domain list entries, e.g. '0.0.0.0' +3. column: the domain column within the feed file, e.g. '2' (required) +4. separator: an optional field separator, default is the character class '[[:space:]]' + +**Enable debug mode** +Adblock provides an optional debug mode that writes diagnostic information to the system log and captures internal error output in a dedicated error logfile - by default located in the adblock base directory as '/tmp/adb_error.log'. The log file is automatically cleared at the beginning of each run. Under normal conditions, all error messages are discarded to keep regular runs clean and silent. To enable debug mode, set the option 'adb_debug' to '1'. When enabled, the script produces significantly more log output to assist with troubleshooting. ## Support Please join the adblock discussion in this [forum thread](https://forum.openwrt.org/t/adblock-support-thread/507) or contact me by mail -Have fun! +## Removal +Stop all adblock related services with _/etc/init.d/adblock stop_ and remove the adblock package if necessary. + +## Donations +You like this project - is there a way to donate? Generally speaking "No" - I have a well-paying full-time job and my OpenWrt projects are just a hobby of mine in my spare time. + +If you still insist to donate some bucks ... +* I would be happy if you put your money in kind into other, social projects in your area, e.g. a children's hospice +* Let's meet and invite me for a coffee if you are in my area, the “Markgräfler Land” in southern Germany or in Switzerland (Basel) +* Send your money to my [PayPal account](https://www.paypal.me/DirkBrenken) and I will collect your donations over the year to support various social projects in my area + +No matter what you decide - thank you very much for your support! + +Have fun! Dirk +--- + +## NethSecurity Integration + +NethSecurity ships adblock as the DNS-blocking engine for the Threat Shield DNS feature. + +### Threat Shield DNS + +The Threat Shield DNS integration is controlled by the `ts_enabled` option in `adblock.global`. When enabled, `ts-dns` populates `/etc/adblock/adblock.custom.feeds` with Nethesis enterprise feeds (if a subscription is active) and community free feeds, which adblock reads automatically. + +Relevant UCI options set by NethSecurity (in addition to standard adblock options): + +| Option | Default | Description | +| :--- | :--- | :--- | +| `ts_enabled` | `0` | Set to `1` by NethSecurity to activate Threat Shield DNS mode | +| `ns_tsdns_bypass` | -, not set | List of IP addresses or subnets excluded from `adb_nftforce` DNS redirection | + +### IP-based DNS bypass + +The standard adblock `adb_nftforce` feature forces DNS queries from specified LAN devices/VLANs through the local resolver. NethSecurity extends this with an **IP-based bypass** list (`ns_tsdns_bypass`): any source IP or subnet in that list is exempt from DNS redirection, even when DNS enforcement is active. + +To add a bypass address or subnet: +```sh +uci add_list adblock.global.ns_tsdns_bypass=192.168.100.2 +uci commit adblock +/etc/init.d/adblock restart +``` + +The bypass rules are injected into the `inet adblock pre-routing` chain (adblock's own nftables table) as `return` rules that take effect before the DNS redirect rules. No fw4/firewall rules are involved. + +### Disabling the CGI remote allow page +The upstream `adblock.cgi` CGI endpoint is **not installed** in NethSecurity. All adblock management is handled through the NethSecurity API (`ns.threatshield`). diff --git a/packages/adblock/files/adblock.categories b/packages/adblock/files/adblock.categories index 1d1118837..fad1e4abc 100644 --- a/packages/adblock/files/adblock.categories +++ b/packages/adblock/files/adblock.categories @@ -1,21 +1,100 @@ -stb;fakenews;alternates/fakenews/hosts -stb;fakenews-gambling;alternates/fakenews-gambling/hosts -stb;fakenews-gambling-porn;alternates/fakenews-gambling-porn/hosts -stb;fakenews-gambling-porn-social;alternates/fakenews-porn-social/hosts -stb;fakenews-gambling-social;alternates/fakenews-gambling-social/hosts -stb;fakenews-porn;alternates/fakenews-porn/hosts -stb;fakenews-porn-social;alternates/fakenews-porn-social/hosts -stb;fakenews-social;alternates/fakenews-social/hosts -stb;gambling;alternates/gambling/hosts -stb;gambling-porn;alternates/gambling-porn/hosts -stb;gambling-porn-social;alternates/gambling-porn-social/hosts -stb;gambling-social;alternates/gambling-social/hosts -stb;porn;alternates/porn/hosts -stb;porn-social;alternates/porn-social/hosts -stb;social;alternates/social/hosts +hag;multi-light;wildcard/light-onlydomains.txt +hag;multi-normal;wildcard/multi-onlydomains.txt +hag;multi-pro;wildcard/pro-onlydomains.txt +hag;multi-pro.mini;wildcard/pro.mini-onlydomains.txt +hag;multi-pro.plus;wildcard/pro.plus-onlydomains.txt +hag;multi-pro.plus.mini;wildcard/pro.plus.mini-onlydomains.txt +hag;multi-ultimate;wildcard/ultimate-onlydomains.txt +hag;multi-ultimate.mini;wildcard/ultimate.mini-onlydomains.txt +hag;threat-intelligence;wildcard/tif-onlydomains.txt +hag;threat-intelligence.medium;wildcard/tif.medium-onlydomains.txt +hag;threat-intelligence.mini;wildcard/tif.mini-onlydomains.txt +hag;anti.piracy;wildcard/anti.piracy-onlydomains.txt +hag;blocklist-referral;wildcard/blocklist-referral-onlydomains.txt +hag;doh;wildcard/doh-onlydomains.txt +hag;doh-vpn-proxy-bypass;wildcard/doh-vpn-proxy-bypass-onlydomains.txt +hag;dyndns;wildcard/dyndns-onlydomains.txt +hag;fake;wildcard/fake-onlydomains.txt +hag;gambling;wildcard/gambling-onlydomains.txt +hag;gambling.medium;wildcard/gambling.medium-onlydomains.txt +hag;gambling.mini;wildcard/gambling.mini-onlydomains.txt +hag;hoster;wildcard/hoster-onlydomains.txt +hag;nsfw;wildcard/nsfw-onlydomains.txt +hag;tracker.amazon;wildcard/native.amazon-onlydomains.txt +hag;tracker.apple;wildcard/native.apple-onlydomains.txt +hag;tracker.huawei;wildcard/native.huawei-onlydomains.txt +hag;tracker.lgwebos;wildcard/native.lgwebos-onlydomains.txt +hag;tracker.oppo-realme;wildcard/native.oppo-realme-onlydomains.txt +hag;tracker.roku;wildcard/native.roku-onlydomains.txt +hag;tracker.samsung;wildcard/native.samsung-onlydomains.txt +hag;tracker.tiktok;wildcard/native.tiktok-onlydomains.txt +hag;tracker.tiktok.extended;wildcard/native.tiktok.extended-onlydomains.txt +hag;tracker.vivo;wildcard/native.vivo-onlydomains.txt +hag;tracker.winoffice;wildcard/native.winoffice-onlydomains.txt +hag;tracker.xiaomi;wildcard/native.xiaomi-onlydomains.txt +hag;nosafesearch;wildcard/nosafesearch-onlydomains.txt +hag;popupads;wildcard/popupads-onlydomains.txt +hag;urlshortener;wildcard/urlshortener-onlydomains.txt +hag;abusetlds;wildcard/spam-tlds-onlydomains.txt +hag;social;wildcard/social-onlydomains.txt +hag;dga-7days;domains/dga7.txt +hag;dga-14days;domains/dga14.txt +hag;dga-30days;domains/dga30.txt +hag;nrd-7days;domains/nrd7.txt +hag;nrd-14days;domains/nrd14-8.txt +hag;nrd-21days;domains/nrd21-15.txt +hag;nrd-28days;domains/nrd28-22.txt +hag;nrd-35days;domains/nrd35-29.txt +hst;lite;Lite/domains.wildcards +hst;xtra;Xtra/domains.wildcards +ipf;ads;ads/domains.txt +ipf;dating;dating/domains.txt +ipf;doh;doh/domains.txt +ipf;gambling;gambling/domains.txt +ipf;games;games/domains.txt +ipf;malware;malware/domains.txt +ipf;phishing;phishing/domains.txt +ipf;piracy;piracy/domains.txt +ipf;porn;porn/domains.txt +ipf;shopping;shopping/domains.txt +ipf;smart-tv;smart-tv/domains.txt +ipf;social;social/domains.txt +ipf;streaming;streaming/domains.txt +ipf;violence;violence/domains.txt stb;standard;hosts +stb;standard-fakenews;alternates/fakenews/hosts +stb;standard-fakenews-gambling;alternates/fakenews-gambling/hosts +stb;standard-fakenews-gambling-porn;alternates/fakenews-gambling-porn/hosts +stb;standard-fakenews-gambling-porn-social;alternates/fakenews-gambling-porn-social/hosts +stb;standard-fakenews-gambling-social;alternates/fakenews-gambling-social/hosts +stb;standard-fakenews-porn;alternates/fakenews-porn/hosts +stb;standard-fakenews-porn-social;alternates/fakenews-porn-social/hosts +stb;standard-fakenews-social;alternates/fakenews-social/hosts +stb;standard-gambling;alternates/gambling/hosts +stb;standard-gambling-porn;alternates/gambling-porn/hosts +stb;standard-gambling-porn-social;alternates/gambling-porn-social/hosts +stb;standard-gambling-social;alternates/gambling-social/hosts +stb;standard-porn;alternates/porn/hosts +stb;standard-porn-social;alternates/porn-social/hosts +stb;standard-social;alternates/social/hosts +stb;fakenews;alternates/fakenews-only/hosts +stb;fakenews-gambling;alternates/fakenews-gambling-only/hosts +stb;fakenews-gambling-porn;alternates/fakenews-gambling-porn-only/hosts +stb;fakenews-gambling-porn-social;alternates/fakenews-gambling-porn-social-only/hosts +stb;fakenews-gambling-social;alternates/fakenews-gambling-social-only/hosts +stb;fakenews-porn;alternates/fakenews-porn-only/hosts +stb;fakenews-porn-social;alternates/fakenews-porn-social-only/hosts +stb;fakenews-social;alternates/fakenews-social-only/hosts +stb;gambling;alternates/gambling-only/hosts +stb;gambling-porn;alternates/gambling-porn-only/hosts +stb;gambling-porn-social;alternates/gambling-porn-social-only/hosts +stb;gambling-social;alternates/gambling-social-only/hosts +stb;porn;alternates/porn-only/hosts +stb;porn-social;alternates/porn-social-only/hosts +stb;social;alternates/social-only/hosts utc;adult utc;agressif +utc;ai utc;arjel utc;associations_religieuses utc;astrology @@ -36,9 +115,11 @@ utc;dialer utc;doh utc;download utc;drogue +utc;dynamic-dns utc;educational_games utc;examen_pix utc;exceptions_liste_bu +utc;fakenews utc;filehosting utc;financial utc;forums @@ -61,6 +142,7 @@ utc;radio utc;reaffected utc;redirector utc;remote-control +utc;residential-proxies utc;sect utc;sexual_education utc;shopping @@ -73,7 +155,9 @@ utc;strict_redirector utc;strong_redirector utc;translation utc;tricheur +utc;tricheur_pix utc;update utc;vpn utc;warez +utc;webhosting utc;webmail diff --git a/packages/adblock/files/adblock.conf b/packages/adblock/files/adblock.conf index 8b50b4211..37fd67aa6 100644 --- a/packages/adblock/files/adblock.conf +++ b/packages/adblock/files/adblock.conf @@ -2,13 +2,11 @@ config adblock 'global' option adb_enabled '0' option adb_debug '0' - option adb_forcedns '0' + option adb_dnsforce '0' + option adb_dnsshift '0' option adb_safesearch '0' - option adb_dnsfilereset '0' option adb_mail '0' option adb_report '0' - option adb_backup '1' - list adb_sources 'adaway' - list adb_sources 'adguard' - list adb_sources 'disconnect' - list adb_sources 'yoyo' + list adb_feed 'adguard' + list adb_feed 'adguard_tracking' + list adb_feed 'certpl' diff --git a/packages/adblock/files/adblock.blacklist b/packages/adblock/files/adblock.custom.feeds similarity index 100% rename from packages/adblock/files/adblock.blacklist rename to packages/adblock/files/adblock.custom.feeds diff --git a/packages/adblock/files/adblock.feeds b/packages/adblock/files/adblock.feeds new file mode 100644 index 000000000..34a437261 --- /dev/null +++ b/packages/adblock/files/adblock.feeds @@ -0,0 +1,194 @@ +{ + "1hosts": { + "url": "https://raw.githubusercontent.com/badmojr/1Hosts/master/", + "rule": "feed 1", + "size": "VAR", + "descr": "compilation" + }, + "adguard": { + "url": "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt", + "rule": "feed || 3 [|^]", + "size": "L", + "descr": "general" + }, + "adguard_tracking": { + "url": "https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_trackers_justdomains.txt", + "rule": "feed 1", + "size": "L", + "descr": "tracking" + }, + "android_tracking": { + "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/android-tracking.txt", + "rule": "feed 1", + "size": "S", + "descr": "tracking" + }, + "andryou": { + "url": "https://gitlab.com/andryou/block/raw/master/kouhai-compressed-domains", + "rule": "feed 1", + "size": "L", + "descr": "compilation" + }, + "anti_ad": { + "url": "https://raw.githubusercontent.com/privacy-protection-tools/anti-AD/master/anti-ad-domains.txt", + "rule": "feed 1", + "size": "L", + "descr": "compilation" + }, + "anudeep": { + "url": "https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt", + "rule": "feed 0.0.0.0 2", + "size": "M", + "descr": "compilation" + }, + "bitcoin": { + "url": "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt", + "rule": "feed 0.0.0.0 2", + "size": "S", + "descr": "mining" + }, + "certpl": { + "url": "https://hole.cert.pl/domains/v2/domains.txt", + "rule": "feed 1", + "size": "L", + "descr": "phishing" + }, + "cpbl": { + "url": "https://raw.githubusercontent.com/bongochong/CombinedPrivacyBlockLists/master/NoFormatting/cpbl-ctld.txt", + "rule": "feed 1", + "size": "XL", + "descr": "compilation" + }, + "disconnect": { + "url": "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt", + "rule": "feed 1", + "size": "S", + "descr": "general" + }, + "divested": { + "url": "https://divested.dev/hosts-domains-wildcards", + "rule": "feed 1", + "size": "XXL", + "descr": "compilation" + }, + "doh_blocklist": { + "url": "https://raw.githubusercontent.com/dibdot/DoH-IP-blocklists/master/doh-domains_overall.txt", + "rule": "feed 1", + "size": "S", + "descr": "doh_server" + }, + "firetv_tracking": { + "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/AmazonFireTV.txt", + "rule": "feed 1", + "size": "S", + "descr": "tracking" + }, + "games_tracking": { + "url": "https://raw.githubusercontent.com/KodoPengin/GameIndustry-hosts-Template/master/Main-Template/hosts", + "rule": "feed 0.0.0.0 2", + "size": "S", + "descr": "tracking" + }, + "hagezi": { + "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/", + "rule": "feed 1", + "size": "VAR", + "descr": "compilation" + }, + "hblock": { + "url": "https://hblock.molinero.dev/hosts_domains.txt", + "rule": "feed 1", + "size": "XL", + "descr": "compilation" + }, + "ipfire_dbl": { + "url": "https://dbl.ipfire.org/lists/", + "rule": "feed 1", + "size": "VAR", + "descr": "compilation" + }, + "oisd_big": { + "url": "https://big.oisd.nl/domainswild2", + "rule": "feed 1", + "size": "XXL", + "descr": "general" + }, + "oisd_nsfw": { + "url": "https://nsfw.oisd.nl/domainswild2", + "rule": "feed 1", + "size": "XXL", + "descr": "porn" + }, + "oisd_nsfw_small": { + "url": "https://nsfw-small.oisd.nl/domainswild2", + "rule": "feed 1", + "size": "M", + "descr": "porn" + }, + "oisd_small": { + "url": "https://small.oisd.nl/domainswild2", + "rule": "feed 1", + "size": "L", + "descr": "general" + }, + "phishing_army": { + "url": "https://phishing.army/download/phishing_army_blocklist_extended.txt", + "rule": "feed 1", + "size": "S", + "descr": "phishing" + }, + "smarttv_tracking": { + "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV.txt", + "rule": "feed 1", + "size": "S", + "descr": "tracking" + }, + "spam404": { + "url": "https://raw.githubusercontent.com/Dawsey21/Lists/master/main-blacklist.txt", + "rule": "feed 1", + "size": "S", + "descr": "general" + }, + "stevenblack": { + "url": "https://raw.githubusercontent.com/StevenBlack/hosts/master/", + "rule": "feed 0.0.0.0 2", + "size": "VAR", + "descr": "compilation" + }, + "stopforumspam": { + "url": "https://www.stopforumspam.com/downloads/toxic_domains_whole.txt", + "rule": "feed 1", + "size": "S", + "descr": "spam" + }, + "utcapitole": { + "url": "https://dsi.ut-capitole.fr/blacklists/download/blacklists.tar.gz", + "rule": "feed 1", + "size": "VAR", + "descr": "general" + }, + "wally3k": { + "url": "https://v.firebog.net/hosts/static/w3kbl.txt", + "rule": "feed 1", + "size": "S", + "descr": "compilation" + }, + "whocares": { + "url": "https://someonewhocares.org/hosts/hosts", + "rule": "feed 127.0.0.1 2", + "size": "M", + "descr": "general" + }, + "winspy": { + "url": "https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt", + "rule": "feed 0.0.0.0 2", + "size": "S", + "descr": "win_telemetry" + }, + "yoyo": { + "url": "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml&showintro=0&mimetype=plaintext", + "rule": "feed 1", + "size": "S", + "descr": "general" + } +} diff --git a/packages/adblock/files/adblock.init b/packages/adblock/files/adblock.init index 44e6228f8..14dbc1190 100755 --- a/packages/adblock/files/adblock.init +++ b/packages/adblock/files/adblock.init @@ -1,8 +1,8 @@ #!/bin/sh /etc/rc.common -# Copyright (c) 2015-2022 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2015-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. -# disable (s)hellcheck in release +# (s)hellcheck exceptions # shellcheck disable=all START=30 @@ -10,25 +10,69 @@ USE_PROCD=1 extra_command "suspend" "Suspend adblock processing" extra_command "resume" "Resume adblock processing" -extra_command "query" " Query active blocklists and backups for a specific domain" -extra_command "report" "[[|||] [] [] []] Print DNS statistics with an optional search parameter" -extra_command "list" "[|||||||] List/Edit available sources" -extra_command "timer" "[ [] []]|[ ] List/Edit cron update intervals" +extra_command "search" " Search active blocklists and backups for a specific domain" +extra_command "report" "[|||] Print DNS statistics" adb_init="/etc/init.d/adblock" adb_script="/usr/bin/adblock.sh" -adb_pidfile="/var/run/adblock.pid" - -if [ -s "${adb_pidfile}" ] && { [ "${action}" = "start" ] || [ "${action}" = "stop" ] || - [ "${action}" = "restart" ] || [ "${action}" = "reload" ] || [ "${action}" = "report" ] || - [ "${action}" = "suspend" ] || [ "${action}" = "resume" ] || [ "${action}" = "query" ] || - { [ "${action}" = "list" ] && [ -n "${1}" ]; }; }; then - return 0 +adb_rundir="/var/run/adblock" +adb_pidfile="/var/run/adblock/adblock.pid" + +if [ -z "${IPKG_INSTROOT}" ]; then + + # ensure runtime directory exists + # + [ ! -d "${adb_rundir}" ] && mkdir -p "${adb_rundir}" + + # check for running instance and handle boot trigger + # + case "${action}" in + "boot") + "${adb_init}" running && exit 0 + ;; + esac + + # reset pidfile if no/stale process is found, + # otherwise exit with error to prevent multiple instances + # + if [ -s "${adb_pidfile}" ]; then + pid="$(cat "${adb_pidfile}" 2>/dev/null)" + if [ -n "${pid}" ] && kill -0 "${pid}" 2>/dev/null; then + case "${action}" in + "start" | "stop" | "restart" | "reload" | "report" | "suspend" | "resume" | "search") + exit 1 + ;; + esac + else + : >"${adb_pidfile}" + fi + fi fi boot() { - [ -s "${adb_pidfile}" ] && : >"${adb_pidfile}" - rc_procd start_service + rc_procd start_service boot +} + +f_append_local_list_entry() { + local entry="${1}" + local file="${2}" + + printf '%s\n' "${entry}" >> "${file}" +} + +f_write_local_lists() { + local allowlist_file='/etc/adblock/adblock.allowlist' + local blocklist_file='/etc/adblock/adblock.blocklist' + + uci -q get adblock.ns_lists >/dev/null 2>&1 || return 0 + + config_load adblock + + : > "${allowlist_file}" + config_list_foreach ns_lists allowlist f_append_local_list_entry "${allowlist_file}" + + : > "${blocklist_file}" + config_list_foreach ns_lists blocklist f_append_local_list_entry "${blocklist_file}" } start_service() { @@ -37,30 +81,38 @@ start_service() { if [ "${action}" = "boot" ]; then [ -n "$(uci_get adblock global adb_trigger)" ] && return 0 fi + # Start NethSecurity patch + f_write_local_lists + # End NethSecurity patch procd_open_instance "adblock" - procd_set_param command "${adb_script}" "${@}" + procd_set_param command "${adb_script}" "${@:-"${action}"}" procd_set_param pidfile "${adb_pidfile}" - procd_set_param nice "$(uci_get adblock global adb_nice "0")" - procd_set_param stdout 1 + procd_set_param nice "$(uci_get adblock global adb_nicelimit "0")" + procd_set_param stdout 0 procd_set_param stderr 1 procd_close_instance fi } +restart() { + # Start NethSecurity patch + /usr/sbin/ts-dns # configure threat shield dns, if needed + # End NethSecurity patch + stop_service "restart" + rc_procd start_service restart +} + reload_service() { + # Start NethSecurity patch /usr/sbin/ts-dns # configure threat shield dns, if needed + f_write_local_lists + ${adb_script} nft-reload + # End NethSecurity patch rc_procd start_service reload - /etc/init.d/firewall restart } stop_service() { - rc_procd "${adb_script}" stop -} - -restart() { - /usr/sbin/ts-dns # configure threat shield dns, if needed - rc_procd start_service restart - /etc/init.d/firewall restart + [ -z "${1}" ] && rc_procd "${adb_script}" stop } suspend() { @@ -71,204 +123,52 @@ resume() { rc_procd start_service resume } -query() { - rc_procd "${adb_script}" query "${1}" +search() { + rc_procd "${adb_script}" search "${1}" } report() { rc_procd "${adb_script}" report "${1:-"cli"}" "${2}" "${3}" "${4}" } -list() { - local src_archive src_file src_enabled enabled name utc_list size focus descurl action="${1}" - - if [ "${action%_*}" = "add" ] || [ "${action%_*}" = "remove" ]; then - shift - for name in "${@}"; do - case "${action}" in - "add") - if ! uci_get adblock global adb_sources | grep -q "${name}"; then - uci_add_list adblock global adb_sources "${name}" - printf "%s\n" "::: adblock source '${name}' added to config" - fi - ;; - "remove") - if uci_get adblock global adb_sources | grep -q "${name}"; then - uci_remove_list adblock global adb_sources "${name}" - printf "%s\n" "::: adblock source '${name}' removed from config" - fi - ;; - "add_utc") - if ! uci_get adblock global adb_utc_sources | grep -q "${name}"; then - uci_add_list adblock global adb_utc_sources "${name}" - printf "%s\n" "::: adblock utcapitole '${name}' added to config" - fi - ;; - "remove_utc") - if uci_get adblock global adb_utc_sources | grep -q "${name}"; then - uci_remove_list adblock global adb_utc_sources "${name}" - printf "%s\n" "::: adblock utcapitole '${name}' removed from config" - fi - ;; - "add_eng") - if ! uci_get adblock global adb_eng_sources | grep -q "${name}"; then - uci_add_list adblock global adb_eng_sources "${name}" - printf "%s\n" "::: adblock energized '${name}' added to config" - fi - ;; - "remove_eng") - if uci_get adblock global adb_eng_sources | grep -q "${name}"; then - uci_remove_list adblock global adb_eng_sources "${name}" - printf "%s\n" "::: adblock energized '${name}' removed from config" - fi - ;; - "add_stb") - if ! uci_get adblock global adb_stb_sources | grep -q "${name}"; then - uci_add_list adblock global adb_stb_sources "${name}" - printf "%s\n" "::: adblock stevenblack '${name}' added to config" - fi - ;; - "remove_stb") - if uci_get adblock global adb_stb_sources | grep -q "${name}"; then - uci_remove_list adblock global adb_stb_sources "${name}" - printf "%s\n" "::: adblock stevenblack '${name}' removed from config" - fi - ;; - esac - done - [ -n "$(uci -q changes adblock)" ] && { uci_commit adblock; "${adb_init}" start; } - else - src_archive="$(uci_get adblock global adb_srcarc "/etc/adblock/adblock.sources.gz")" - src_file="$(uci_get adblock global adb_srcfile "/tmp/adb_sources.json")" - src_enabled="$(uci -q show adblock.global.adb_sources)" - [ -r "${src_archive}" ] && zcat "${src_archive}" >"${src_file}" || printf "%s\n" "::: adblock source archive '${src_archive}' not found" - - if [ -r "${src_file}" ]; then - src_enabled="${src_enabled#*=}" - src_enabled="${src_enabled//\'}" - printf "%s\n" "::: Available adblock sources" - printf "%s\n" ":::" - printf "%-25s%-10s%-7s%-21s%s\n" " Name" "Enabled" "Size" "Focus" "Info URL" - printf "%s\n" " -------------------------------------------------------------------" - json_load_file "${src_file}" - json_get_keys keylist - for key in ${keylist}; do - json_select "${key}" - json_get_var size "size" - json_get_var focus "focus" - json_get_var descurl "descurl" - json_get_var url "url" - json_get_var rule "rule" - if [ -n "${url}" ] && [ -n "${rule}" ]; then - if printf "%s" "${src_enabled}" | grep -q "${key}"; then - enabled="x" - else - enabled=" " - fi - src_enabled="${src_enabled/${key}}" - printf " + %-21s%-10s%-7s%-21s%s\n" "${key:0:20}" "${enabled}" "${size:0:3}" "${focus:0:20}" "${descurl:0:50}" - else - src_enabled="${src_enabled} ${key}" - fi - json_select .. - done - utc_list="$(uci_get adblock global adb_utc_sources "-")" - eng_list="$(uci_get adblock global adb_eng_sources "-")" - stb_list="$(uci_get adblock global adb_stb_sources "-")" - printf "%s\n" " ---------------------------------------------------------------------------" - printf " * %s\n" "Configured utcapitole categories: ${utc_list// /, }" - printf " * %s\n" "Configured energized variants: ${eng_list// /, }" - printf " * %s\n" "Configured stevenblack variants: ${stb_list// /, }" - - if [ -n "${src_enabled// }" ]; then - printf "%s\n" " ---------------------------------------------------------------------------" - printf "%s\n" " Sources with invalid configuration" - printf "%s\n" " ---------------------------------------------------------------------------" - for key in ${src_enabled}; do - printf " - %s\n" "${key:0:20}" - done - fi - else - printf "%s\n" "::: adblock source file '${src_file}' not found" - fi - fi -} - status() { status_service } status_service() { - local key keylist value idxval values type rtfile - - rtfile="$(uci_get adblock global adb_rtfile "/tmp/adb_runtime.json")" + local key keylist type value values - json_load_file "${rtfile}" >/dev/null 2>&1 + json_init + json_load_file "/var/run/adblock/adblock.runtime.json" >/dev/null 2>&1 json_get_keys keylist if [ -n "${keylist}" ]; then - printf "%s\n" "::: adblock runtime information" + printf '%s\n' "::: adblock runtime information" for key in ${keylist}; do - json_get_var value "${key}" >/dev/null 2>&1 - if [ "${key%_*}" = "active" ]; then - printf " + %-15s : " "${key}" - json_select "${key}" >/dev/null 2>&1 - values="" - index="1" - while json_get_type type "${index}" && [ "${type}" = "object" ]; do - json_get_values idxval "${index}" >/dev/null 2>&1 - if [ "${index}" = "1" ]; then - values="${idxval}" - else - values="${values}, ${idxval}" - fi - index="$((index + 1))" - done - values="$(printf "%s" "${values}" | awk '{NR=1;max=98;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max)}else{printf"%-22s%s\n","",substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}')" - printf "%s\n" "${values:-"-"}" - json_select ".." + json_get_type type "${key}" >/dev/null 2>&1 + if [ "${type}" = "array" ]; then + json_get_values values "${key}" >/dev/null 2>&1 + value="${values}" else - printf " + %-15s : %s\n" "${key}" "${value:-"-"}" + json_get_var value "${key}" >/dev/null 2>&1 fi + printf ' + %-15s : %s\n' "${key}" "${value:-"-"}" done else - printf "%s\n" "::: no adblock runtime information available" - fi -} - -timer() { - local cron_file cron_content cron_lineno action="${1:-"list"}" cron_tasks="${2}" hour="${3}" minute="${4:-0}" weekday="${5:-"*"}" - - cron_file="/etc/crontabs/root" - - if [ -s "${cron_file}" ] && [ "${action}" = "list" ]; then - awk '{print NR "> " $0}' "${cron_file}" - elif [ -x "/etc/init.d/cron" ] && [ "${action}" = "add" ]; then - hour="${hour//[[:alpha:]]/}" - minute="${minute//[[:alpha:]]/}" - if [ -n "${cron_tasks}" ] && [ -n "${hour}" ] && [ -n "${minute}" ] && [ -n "${weekday}" ] && - [ "${hour}" -ge 0 ] && [ "${hour}" -le 23 ] && - [ "${minute}" -ge 0 ] && [ "${minute}" -le 59 ]; then - printf "%02d %02d %s\n" "${minute}" "${hour}" "* * ${weekday} ${adb_init} ${cron_tasks}" >>"${cron_file}" - /etc/init.d/cron restart - fi - elif [ -x "/etc/init.d/cron" ] && [ -s "${cron_file}" ] && [ "${action}" = "remove" ]; then - cron_tasks="${cron_tasks//[[:alpha:]]/}" - cron_lineno="$(awk 'END{print NR}' "${cron_file}")" - cron_content="$(awk '{print $0}' "${cron_file}")" - if [ "${cron_tasks:-"0"}" -le "${cron_lineno:-"1"}" ] && [ -n "${cron_content}" ]; then - printf "%s\n" "${cron_content}" | awk "NR!~/^${cron_tasks}$/" >"${cron_file}" - /etc/init.d/cron restart - fi + printf '%s\n' "::: no adblock runtime information available" fi } service_triggers() { - local iface delay + local iface delay trigger - iface="$(uci_get adblock global adb_trigger)" delay="$(uci_get adblock global adb_triggerdelay "5")" - PROCD_RELOAD_DELAY="$((delay * 1000))" + trigger="$(uci_get adblock global adb_trigger)" - [ -n "${iface}" ] && procd_add_interface_trigger "interface.*.up" "${iface}" "${adb_init}" "start" - procd_add_reload_trigger "adblock" + PROCD_RELOAD_DELAY="$((delay * 1000))" + for iface in ${trigger}; do + procd_add_interface_trigger "interface.*.up" "${iface}" "${adb_init}" start + done + # Start NethSecurity patch + procd_add_reload_trigger adblock + # End NethSecurity patch } diff --git a/packages/adblock/files/adblock.mail b/packages/adblock/files/adblock.mail index 67fc011aa..98f4bb1cc 100755 --- a/packages/adblock/files/adblock.mail +++ b/packages/adblock/files/adblock.mail @@ -1,6 +1,6 @@ #!/bin/sh # send mail script for adblock notifications -# Copyright (c) 2015-2022 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2015-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. # Please note: you have to manually install and configure the package 'msmtp' before using this script @@ -8,70 +8,66 @@ # set (s)hellcheck exceptions # shellcheck disable=all -LC_ALL=C -PATH="/usr/sbin:/usr/bin:/sbin:/bin" - -[ -r "/lib/functions.sh" ] && . "/lib/functions.sh" +[ -r "/usr/bin/adblock.sh" ] && . "/usr/bin/adblock.sh" "mail" adb_debug="$(uci_get adblock global adb_debug "0")" adb_mailsender="$(uci_get adblock global adb_mailsender "no-reply@adblock")" adb_mailreceiver="$(uci_get adblock global adb_mailreceiver)" adb_mailtopic="$(uci_get adblock global adb_mailtopic "adblock notification")" adb_mailprofile="$(uci_get adblock global adb_mailprofile "adb_notify")" -adb_ver="${1}" -adb_mail="$(command -v msmtp)" -adb_logger="$(command -v logger)" -adb_logread="$(command -v logread)" -adb_rc="1" - -f_log() { - local class="${1}" log_msg="${2}" - - if [ -x "${adb_logger}" ]; then - "${adb_logger}" -p "${class}" -t "adblock-${adb_ver}[${$}]" "${log_msg}" - else - printf "%s %s %s\n" "${class}" "adblock-${adb_ver}[${$}]" "${log_msg}" - fi -} -if [ -z "${adb_mailreceiver}" ]; then - f_log "err" "please set the mail receiver with the 'adb_mailreceiver' option" - exit ${adb_rc} -fi +[ -z "${adb_mailreceiver}" ] && f_log "info" "please set the mail receiver with the 'adb_mailreceiver' option" [ "${adb_debug}" = "1" ] && debug="--debug" -adb_mailhead="From: ${adb_mailsender}\nTo: ${adb_mailreceiver}\nSubject: ${adb_mailtopic}\nReply-to: ${adb_mailsender}\nMime-Version: 1.0\nContent-Type: text/html;charset=utf-8\nContent-Disposition: inline\n\n" - # info preparation # sys_info="$( strings /etc/banner 2>/dev/null - ubus call system board | sed -e 's/\"release\": {//' | sed -e 's/^[ \t]*//' | sed -e 's/[{}\",]//g' | sed -e 's/[ ]/ \t/' | sed '/^$/d' 2>/dev/null + "${adb_ubuscmd}" call system board | "${adb_awkcmd}" 'BEGIN{FS="[{}\"]"}{if($2=="kernel"||$2=="hostname"||$2=="system"||$2=="model"||$2=="description")printf " + %-12s: %s\n",$2,$4}' 2>/dev/null )" adb_info="$(/etc/init.d/adblock status 2>/dev/null)" -rep_info="${2}" -if [ -x "${adb_logread}" ]; then - log_info="$("${adb_logread}" -l 100 -e "adblock-" | awk '{NR=1;max=120;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max)}else{print substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}')" +rep_info="${1}" +if [ -x "${adb_logreadcmd}" ]; then + log_info="$("${adb_logreadcmd}" -l 100 -e "adblock-" 2>/dev/null)" fi # mail body # -adb_mailtext="
"
-adb_mailtext="${adb_mailtext}\n++\n++ System Information ++\n++\n${sys_info}"
-adb_mailtext="${adb_mailtext}\n\n++\n++ Adblock Information ++\n++\n${adb_info}"
-if [ -n "${rep_info}" ]; then
-	adb_mailtext="${adb_mailtext}\n\n++\n++ Report Information ++\n++\n${rep_info}"
-fi
-adb_mailtext="${adb_mailtext}\n\n++\n++ Logfile Information ++\n++\n${log_info}"
-adb_mailtext="${adb_mailtext}
" +adb_mailtext="$( + printf '%s\n' "
"
+	printf '\n%s\n' "++
+++ System Information ++
+++"
+	printf '%s\n' "${sys_info:-"-"}"
+	printf '\n%s\n' "++
+++ Adblock Information ++
+++"
+	printf '%s\n' "${adb_info:-"-"}"
+	if [ -n "${rep_info}" ]; then
+		printf '\n%s\n' "++
+++ Report Information ++
+++"
+		printf '%s\n' "${rep_info}"
+	fi
+	printf '\n%s\n' "++
+++ Logfile Information ++
+++"
+	printf '%s\n' "${log_info:-"-"}"
+	printf '%s\n' "
" +)" # send mail # -if [ -x "${adb_mail}" ]; then - printf "%b" "${adb_mailhead}${adb_mailtext}" 2>/dev/null | "${adb_mail}" ${debug} -a "${adb_mailprofile}" "${adb_mailreceiver}" >/dev/null 2>&1 - adb_rc=${?} - f_log "info" "mail sent to '${adb_mailreceiver}' with rc '${adb_rc}'" +if [ -x "${adb_mailcmd}" ]; then + adb_mailhead="From: ${adb_mailsender}\nTo: ${adb_mailreceiver}\nSubject: ${adb_mailtopic}\nReply-to: ${adb_mailsender}\nMime-Version: 1.0\nContent-Type: text/html;charset=utf-8\nContent-Disposition: inline\n\n" + printf '%b' "${adb_mailhead}${adb_mailtext}" 2>/dev/null | "${adb_mailcmd}" ${debug} -a "${adb_mailprofile}" "${adb_mailreceiver}" 2>>"${adb_errorlog}" + mail_rc="${?}" + if [ "${mail_rc}" = "0" ]; then + f_log "info" "mail successfully sent to '${adb_mailreceiver}'" + else + f_log "info" "failed to send mail to '${adb_mailreceiver}' with rc '${mail_rc}'" + fi else - f_log "err" "msmtp mail daemon not found" + f_log "info" "msmtp mail daemon not found" fi -exit ${adb_rc} +exit 0 diff --git a/packages/adblock/files/adblock.sh b/packages/adblock/files/adblock.sh index fe6fc10fe..155c6d713 100755 --- a/packages/adblock/files/adblock.sh +++ b/packages/adblock/files/adblock.sh @@ -1,9 +1,9 @@ #!/bin/sh # dns based ad/abuse domain blocking -# Copyright (c) 2015-2023 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2015-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. -# disable (s)hellcheck in release +# (s)hellcheck exceptions # shellcheck disable=all # set initial defaults @@ -11,125 +11,179 @@ export LC_ALL=C export PATH="/usr/sbin:/usr/bin:/sbin:/bin" -adb_ver="4.1.5" adb_enabled="0" adb_debug="0" -adb_forcedns="0" +adb_nftforce="0" +adb_nftdevforce="" +adb_nftportforce="" +ns_tsdns_bypass="" +adb_nftallow="0" +adb_nftmacallow="" +adb_nftdevallow="" +adb_nftblock="0" +adb_nftmacblock="" +adb_nftdevblock="" +adb_nftremote="0" +adb_nftremotetimeout="15" +adb_nftmacremote="" +adb_nftbridge="0" +adb_allowdnsv4="" +adb_allowdnsv6="" +adb_remotednsv4="" +adb_remotednsv6="" +adb_blockdnsv4="" +adb_blockdnsv6="" +adb_bridgednsv4="" +adb_bridgednsv6="" +adb_dnsshift="0" adb_dnsflush="0" -adb_dnstimeout="20" +adb_dnstimeout="30" adb_safesearch="0" -adb_safesearchlist="" -adb_safesearchmod="0" adb_report="0" adb_trigger="" -adb_triggerdelay="0" -adb_backup="1" +adb_triggerdelay="5" adb_mail="0" -adb_mailcnt="0" adb_jail="0" +adb_map="0" +adb_tld="1" adb_dns="" -adb_dnsprefix="adb_list" -adb_locallist="blacklist whitelist iplist" -adb_tmpbase="/tmp" -adb_backupdir="${adb_tmpbase}/adblock-Backup" -adb_reportdir="${adb_tmpbase}/adblock-Report" -adb_jaildir="/tmp" -adb_pidfile="/var/run/adblock.pid" -adb_blacklist="/etc/adblock/adblock.blacklist" -adb_whitelist="/etc/adblock/adblock.whitelist" +adb_dnspid="" +adb_locallist="allowlist blocklist" +adb_basedir="/tmp" +adb_finaldir="" +adb_backupdir="/tmp/adblock-backup" +adb_reportdir="/tmp/adblock-report" +adb_rundir="/var/run/adblock" +adb_pidfile="${adb_rundir}/adblock.pid" +adb_rtfile="${adb_rundir}/adblock.runtime.json" +adb_etaglock="${adb_rundir}/adblock.etag.lock" +adb_allowlist="/etc/adblock/adblock.allowlist" +adb_blocklist="/etc/adblock/adblock.blocklist" adb_mailservice="/etc/adblock/adblock.mail" -adb_dnsfile="${adb_dnsprefix}.overall" -adb_dnsjail="${adb_dnsprefix}.jail" -adb_srcarc="/etc/adblock/adblock.sources.gz" -adb_srcfile="${adb_tmpbase}/adb_sources.json" -adb_rtfile="${adb_tmpbase}/adb_runtime.json" -adb_loggercmd="$(command -v logger)" -adb_dumpcmd="$(command -v tcpdump)" -adb_lookupcmd="$(command -v nslookup)" -adb_fetchutil="" +adb_dnsfile="adb_list.overall" +adb_feedfile="/etc/adblock/adblock.feeds" +adb_customfeedfile="/etc/adblock/adblock.custom.feeds" +adb_errorlog="/dev/null" +adb_fetchcmd="" adb_fetchinsecure="" -adb_zonelist="" -adb_portlist="" +adb_fetchretry="5" +adb_fetchparm="" +adb_etagparm="" +adb_geoparm="" +adb_geourl="http://ip-api.com/json" adb_repiface="" -adb_replisten="53" +adb_repport="53" adb_repchunkcnt="5" adb_repchunksize="1" adb_represolve="0" -adb_lookupdomain="example.com" -adb_action="${1:-"start"}" +adb_lookupdomain="localhost" +adb_action="${1}" adb_packages="" -adb_sources="" adb_cnt="" -# load & check adblock environment +# helper function to find a command in the system and check if it's executable, +# if not try alternative command or log an error if not found # -f_load() { - local bg_pid iface port ports cpu core +f_cmd() { + local cmd pri_cmd="${1}" sec_cmd="${2}" + + cmd="$(command -v "${pri_cmd}" 2>/dev/null)" + if [ -z "${cmd}" ]; then + if [ -n "${sec_cmd}" ]; then + [ "${sec_cmd}" = "optional" ] && return + cmd="$(command -v "${sec_cmd}" 2>/dev/null)" + fi + if [ -n "${cmd}" ]; then + printf '%s' "${cmd}" + else + f_log "emerg" "command '${pri_cmd:-"-"}'/'${sec_cmd:-"-"}' not found" + fi + else + printf '%s' "${cmd}" + fi +} - adb_sysver="$(ubus -S call system board 2>/dev/null | jsonfilter -q -e '@.model' -e '@.release.description' | - "${adb_awk}" 'BEGIN{RS="";FS="\n"}{printf "%s, %s",$1,$2}')" - adb_memory="$("${adb_awk}" '/^MemTotal|^MemFree|^MemAvailable/{ORS="/"; print int($2/1000)}' "/proc/meminfo" 2>/dev/null | - "${adb_awk}" '{print substr($0,1,length($0)-1)}')" +# load adblock environment +# +f_load() { + local cnt bg_pid port filter tcpdump_filter cpu + # load adblock config and set debug log file + # f_conf + if [ "${adb_debug}" = "1" ] && [ -d "${adb_basedir}" ]; then + adb_errorlog="${adb_basedir}/adb_error.log" + else + adb_errorlog="/dev/null" + fi - cpu="$(grep -c '^processor' /proc/cpuinfo 2>/dev/null)" - core="$(grep -cm1 '^core id' /proc/cpuinfo 2>/dev/null)" - [ "${cpu}" = "0" ] && cpu="1" - [ "${core}" = "0" ] && core="1" - adb_cores="$((cpu * core))" + # fetch installed packages and system information + # + adb_packages="$("${adb_ubuscmd}" -S call rpc-sys packagelist '{ "all": true }' 2>>"${adb_errorlog}")" + adb_bver="$(printf '%s' "${adb_packages}" | "${adb_jsoncmd}" -ql1 -e '@.packages.adblock')" + adb_fver="$(printf '%s' "${adb_packages}" | "${adb_jsoncmd}" -ql1 -e '@.packages["luci-app-adblock"]')" + adb_sysver="$("${adb_ubuscmd}" -S call system board 2>>"${adb_errorlog}" | + "${adb_jsoncmd}" -ql1 -e '@.model' -e '@.release.target' -e '@.release.distribution' -e '@.release.version' -e '@.release.revision' | + "${adb_awkcmd}" 'BEGIN{RS="";FS="\n"}{printf "%s, %s, %s %s (%s)",$1,$2,$3,$4,$5}')" + + # detect cpu cores for parallel processing + # + if [ -z "${adb_cores}" ]; then + cpu="$("${adb_grepcmd}" -cm16 '^processor' /proc/cpuinfo 2>>"${adb_errorlog}")" + [ "${cpu}" = "0" ] && cpu="1" + adb_cores="${cpu}" + fi - if [ "${adb_action}" != "report" ]; then + # load dns backend and fetch utility + # + if [ "${adb_action}" != "report" ] && [ "${adb_action}" != "mail" ]; then f_dns f_fetch fi - if [ "${adb_enabled}" = "0" ]; then - f_extconf - f_temp - f_rmdns - f_jsnup "disabled" - f_log "info" "adblock is currently disabled, please set the config option 'adb_enabled' to '1' to use this service" - exit 0 - fi - + # check if reporting is enabled and tcpdump is available + # if [ "${adb_report}" = "1" ] && [ ! -x "${adb_dumpcmd}" ]; then - f_log "info" "Please install the package 'tcpdump' or 'tcpdump-mini' to use the reporting feature" - elif [ "${adb_report}" = "0" ] && [ "${adb_action}" = "report" ]; then - f_log "info" "Please enable the 'DNS Report' option to use the reporting feature" - exit 0 - fi - - bg_pid="$(pgrep -f "^${adb_dumpcmd}.*adb_report\\.pcap$" | "${adb_awk}" '{ORS=" "; print $1}')" - if [ -x "${adb_dumpcmd}" ] && { [ "${adb_report}" = "0" ] || { [ -n "${bg_pid}" ] && { [ "${adb_action}" = "stop" ] || [ "${adb_action}" = "restart" ]; }; }; }; then - if [ -n "${bg_pid}" ]; then - kill -HUP "${bg_pid}" 2>/dev/null - while kill -0 "${bg_pid}" 2>/dev/null; do - sleep 1 - done - unset bg_pid + f_log "info" "please install the package 'tcpdump' or 'tcpdump-mini' to use the reporting feature" + elif [ -x "${adb_dumpcmd}" ]; then + bg_pid="$("${adb_pgrepcmd}" -nf "${adb_reportdir}/adb_report.pcap")" + if [ -n "${bg_pid}" ] && { [ "${adb_report}" = "0" ] || [ "${adb_action}" = "stop" ] || [ "${adb_action}" = "restart" ]; }; then + if kill -HUP "${bg_pid}" 2>>"${adb_errorlog}"; then + for cnt in 1 2 3; do + kill -0 "${bg_pid}" >/dev/null 2>&1 || break + sleep 1 + done + fi + bg_pid="$("${adb_pgrepcmd}" -nf "${adb_reportdir}/adb_report.pcap")" + "${adb_rmcmd}" -f "${adb_reportdir}"/adb_report.pcap* fi - fi - if [ -x "${adb_dumpcmd}" ] && [ "${adb_report}" = "1" ] && [ -z "${bg_pid}" ] && [ "${adb_action}" != "report" ] && [ "${adb_action}" != "stop" ]; then - for port in ${adb_replisten}; do - [ -z "${ports}" ] && ports="port ${port}" || ports="${ports} or port ${port}" - done - if [ -z "${adb_repiface}" ]; then - network_get_device iface "lan" - [ -z "${iface}" ] && network_get_physdev iface "lan" - [ -n "${iface}" ] && adb_repiface="${iface}" - [ -n "${adb_repiface}" ] && { uci_set adblock global adb_repiface "${adb_repiface}"; f_uci "adblock"; } - fi - if [ -n "${adb_reportdir}" ] && [ ! -d "${adb_reportdir}" ]; then - mkdir -p "${adb_reportdir}" - f_log "info" "report directory '${adb_reportdir}' created" - fi - if [ -n "${adb_repiface}" ] && [ -d "${adb_reportdir}" ]; then - ("${adb_dumpcmd}" -nn -p -s0 -l -i ${adb_repiface} ${ports} -C${adb_repchunksize} -W${adb_repchunkcnt} -w "${adb_reportdir}/adb_report.pcap" >/dev/null 2>&1 &) - bg_pid="$(pgrep -f "^${adb_dumpcmd}.*adb_report\\.pcap$" | "${adb_awk}" '{ORS=" "; print $1}')" - else - f_log "info" "Please set the name of the reporting network device 'adb_repiface' manually" + if [ "${adb_report}" = "1" ] && [ -z "${bg_pid}" ] && [ "${adb_action}" != "report" ] && [ "${adb_action}" != "stop" ]; then + [ ! -d "${adb_reportdir}" ] && mkdir -p "${adb_reportdir}" + if [ -z "${adb_repiface}" ]; then + network_get_device adb_repiface "lan" + [ -z "${adb_repiface}" ] && network_get_physdev adb_repiface "lan" + uci_set adblock global adb_repiface "${adb_repiface:-"any"}" + f_uci "adblock" + fi + for port in ${adb_repport}; do + [ -n "${filter}" ] && filter="${filter} or " + filter="${filter}(udp port ${port}) or (tcp port ${port})" + done + tcpdump_filter="(${filter}) and greater 28" + if [ -n "${adb_repiface}" ] && [ -d "${adb_reportdir}" ]; then + ( + "${adb_dumpcmd}" --immediate-mode -nn -p -s0 -i "${adb_repiface}" \ + "${tcpdump_filter}" \ + -C "${adb_repchunksize}" -W "${adb_repchunkcnt}" \ + -w "${adb_reportdir}/adb_report.pcap" >/dev/null 2>&1 & + ) + sleep 1 + bg_pid="$("${adb_pgrepcmd}" -nf "${adb_reportdir}/adb_report.pcap")" + f_log "info" "tcpdump background process started for interface: ${adb_repiface}, port: ${adb_repport}, dir: ${adb_reportdir}, pid: ${bg_pid}" + else + f_log "info" "please set the reporting interface 'adb_repiface' and reporting directory 'adb_reportdir' manually" + fi fi fi } @@ -137,514 +191,591 @@ f_load() { # check & set environment # f_env() { - adb_starttime="$(date "+%s")" - f_log "info" "adblock instance started ::: action: ${adb_action}, priority: ${adb_nice:-"0"}, pid: ${$}" - f_jsnup "running" + : >"${adb_errorlog}" + read -r adb_starttime _ <"/proc/uptime" + adb_starttime="${adb_starttime%.*}" + f_log "info" "adblock instance started ::: action: ${adb_action}, priority: ${adb_nicelimit:-"0"}, pid: ${$}" + f_jsnup "processing" f_extconf f_temp - - if [ "${adb_dnsflush}" = "1" ] || [ "${adb_memory##*/}" -lt "64" ]; then - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" - f_dnsup - fi - - if [ ! -r "${adb_srcfile}" ]; then - if [ -r "${adb_srcarc}" ]; then - zcat "${adb_srcarc}" >"${adb_srcfile}" + f_nftadd + json_init + if [ -s "${adb_customfeedfile}" ]; then + if json_load_file "${adb_customfeedfile}" >/dev/null 2>&1; then + return else - f_log "err" "adblock source archive not found" + f_log "info" "can't load adblock custom feed file" fi fi - if [ -r "${adb_srcfile}" ] && [ "${adb_action}" != "report" ]; then - json_init - json_load_file "${adb_srcfile}" + if [ -s "${adb_feedfile}" ] && json_load_file "${adb_feedfile}" >/dev/null 2>&1; then + return else - f_log "err" "adblock source file not found" + f_log "err" "can't load adblock feed file" fi } # load adblock config # f_conf() { - local cnt="0" cnt_max="10" - - [ ! -r "/etc/config/adblock" ] && f_log "err" "no valid adblock config found, please re-install the package via opkg with the '--force-reinstall --force-maintainer' options" - config_cb() { option_cb() { - local option="${1}" - local value="${2}" - eval "${option}=\"${value}\"" + local option="${1}" value="${2//\"/\\\"}" + + case "${option}" in + *[!a-zA-Z0-9_]*) ;; + + *) + eval "${option}=\"\${value}\"" + ;; + esac } list_cb() { - local option="${1}" - local value="${2}" - if [ "${option}" = "adb_sources" ]; then - eval "${option}=\"$(printf "%s" "${adb_sources}") ${value}\"" - elif [ "${option}" = "adb_eng_sources" ]; then - eval "${option}=\"$(printf "%s" "${adb_eng_sources}") ${value}\"" - elif [ "${option}" = "adb_stb_sources" ]; then - eval "${option}=\"$(printf "%s" "${adb_stb_sources}") ${value}\"" - elif [ "${option}" = "adb_utc_sources" ]; then - eval "${option}=\"$(printf "%s" "${adb_utc_sources}") ${value}\"" - elif [ "${option}" = "adb_denyip" ]; then - eval "${option}=\"$(printf "%s" "${adb_denyip}") ${value}\"" - elif [ "${option}" = "adb_allowip" ]; then - eval "${option}=\"$(printf "%s" "${adb_allowip}") ${value}\"" - elif [ "${option}" = "adb_safesearchlist" ]; then - eval "${option}=\"$(printf "%s" "${adb_safesearchlist}") ${value}\"" - elif [ "${option}" = "adb_zonelist" ]; then - eval "${option}=\"$(printf "%s" "${adb_zonelist}") ${value}\"" - elif [ "${option}" = "adb_portlist" ]; then - eval "${option}=\"$(printf "%s" "${adb_portlist}") ${value}\"" - elif [ "${option}" = "adb_bypass" ]; then - eval "${option}=\"$(printf "%s" "${adb_bypass}") ${value}\"" - fi + local append option="${1}" value="${2//\"/\\\"}" + + case "${option}" in + *[!a-zA-Z0-9_]*) ;; + + *) + eval "append=\"\${${option}}\"" + if [ -n "${append}" ]; then + eval "${option}=\"\${append} \${value}\"" + else + eval "${option}=\"\${value}\"" + fi + ;; + esac } } config_load adblock - - if [ -z "${adb_fetchutil}" ] || [ -z "${adb_dns}" ]; then - while [ -z "${adb_packages}" ] && [ "${cnt}" -le "${cnt_max}" ]; do - adb_packages="$(opkg list-installed 2>/dev/null)" - cnt="$((cnt + 1))" - sleep 1 - done - [ -z "${adb_packages}" ] && f_log "err" "local opkg package repository is not available, please set 'adb_fetchutil' and 'adb_dns' manually" - fi } -# status helper function +# domain validation # -f_char() { - local result input="${1}" +f_chkdom() { + local type prefix column separator + + case "${1}" in + "feed" | "local") + type="${1}" + case "${2}" in + [0-9]) + prefix="" + column="${2}" + separator="${3:-[[:space:]]+}" + ;; + *) + prefix="${2}" + column="${3}" + separator="${4:-[[:space:]]+}" + ;; + esac + ;; + "google") + type="${1}" + prefix="" + column="${2}" + separator="${3:-[[:space:]]+}" + ;; + esac - if [ "${input}" = "1" ]; then - result="✔" - else - result="✘" - fi - printf "%s" "${result}" + "${adb_awkcmd}" -v type="${type}" -v pre="${prefix}" -v col="${column}" -v chk="${adb_lookupdomain}" -F "${separator}" ' + { + domain = $col + # remove carriage returns and trim the input + gsub(/\r|^[[:space:]]+|[[:space:]]+$/, "", domain) + # add www. for google safe search + if (type=="google" && domain ~ /^\.+/) { sub(/^\.+/, "", domain); domain="www."domain } + # check optional search prefix + if (pre != "" && index($0, pre) != 1) next + # skip empty lines, comments and special domains + if (domain == "" || domain ~ /^(#|localhost|loopback)/ || index(domain, chk) == 1) next + # no domain with trailing dot + if (substr(domain, length(domain), 1) == ".") next + # check total length (253 characters) + if (length(domain) > 253) next + n = split(domain, L, ".") + valid = 1 + for (i = 1; i <= n; i++) { + l = L[i] + len = length(l) + # label length 1–63 + if (len < 1 || len > 63) { valid = 0; break } + # no leading/trailing hyphen + if (l ~ /^-/ || l ~ /-$/) { valid = 0; break } + # ASCII + hyphen + if (l !~ /^[A-Za-z0-9-]+$/) { valid = 0; break } + } + # TLD must start with a letter or "xn--" + if (valid && L[n] !~ /^[A-Za-z]/ && L[n] !~ /^xn--/) valid = 0 + if (valid) print tolower(domain) + }' + + f_log "debug" "f_chkdom ::: name: ${src_name}, type: ${type}, prefix: ${prefix:-"-"}, column: ${column:-"-"}, separator: ${separator:-"-"}" } # load dns backend config # f_dns() { - local util utils dns_up cnt="0" + local dns dns_list dns_section dns_info free_mem dir + + free_mem="$("${adb_awkcmd}" '/^MemAvailable/{printf "%s",int($2/1000)}' "/proc/meminfo" 2>>"${adb_errorlog}")" + if [ "${adb_action}" = "boot" ] && [ -z "${adb_trigger}" ]; then + sleep ${adb_triggerdelay:-"5"} + fi if [ -z "${adb_dns}" ]; then - utils="knot-resolver bind unbound dnsmasq raw" - for util in ${utils}; do - if [ "${util}" = "raw" ] || printf "%s" "${adb_packages}" | grep -q "^${util}"; then - if [ "${util}" = "knot-resolver" ]; then - util="kresd" - elif [ "${util}" = "bind" ]; then - util="named" - fi - if [ "${util}" = "raw" ] || [ -x "$(command -v "${util}")" ]; then - adb_dns="${util}" - uci_set adblock global adb_dns "${util}" + dns_list="knot-resolver bind-server unbound-daemon smartdns dnsmasq-full dnsmasq-dhcpv6 dnsmasq" + for dns in ${dns_list}; do + if printf '%s' "${adb_packages}" | "${adb_jsoncmd}" -ql1 -e "@.packages[\"${dns}\"]" >/dev/null 2>&1; then + case "${dns}" in + "knot-resolver") + dns="kresd" + ;; + "bind-server") + dns="named" + ;; + "unbound-daemon") + dns="unbound" + ;; + "dnsmasq-full" | "dnsmasq-dhcpv6") + dns="dnsmasq" + ;; + esac + + if command -v "${dns}" >/dev/null 2>&1; then + adb_dns="${dns}" + uci_set adblock global adb_dns "${dns}" f_uci "adblock" break fi fi done - elif [ "${adb_dns}" != "raw" ] && [ ! -x "$(command -v "${adb_dns}")" ]; then - unset adb_dns fi - if [ -n "${adb_dns}" ]; then - case "${adb_dns}" in - "dnsmasq") - adb_dnscachecmd="-" - adb_dnsinstance="${adb_dnsinstance:-"0"}" - adb_dnsuser="${adb_dnsuser:-"dnsmasq"}" - adb_dnsdir="${adb_dnsdir:-"/tmp/dnsmasq.d"}" - adb_dnsheader="${adb_dnsheader:-""}" - adb_dnsdeny="${adb_dnsdeny:-"${adb_awk} '{print \"local=/\"\$0\"/\"}'"}" - adb_dnsallow="${adb_dnsallow:-"${adb_awk} '{print \"local=/\"\$0\"/#\"}'"}" - adb_dnssafesearch="${adb_dnssafesearch:-"${adb_awk} -v item=\"\$item\" '{print \"address=/\"\$0\"/\"item\"\"}'"}" - adb_dnsstop="${adb_dnsstop:-"address=/#/"}" - ;; - "unbound") - adb_dnscachecmd="$(command -v unbound-control || printf "%s" "-")" - adb_dnsinstance="${adb_dnsinstance:-"0"}" - adb_dnsuser="${adb_dnsuser:-"unbound"}" - adb_dnsdir="${adb_dnsdir:-"/var/lib/unbound"}" - adb_dnsheader="${adb_dnsheader:-""}" - adb_dnsdeny="${adb_dnsdeny:-"${adb_awk} '{print \"local-zone: \\042\"\$0\"\\042 always_nxdomain\"}'"}" - adb_dnsallow="${adb_dnsallow:-"${adb_awk} '{print \"local-zone: \\042\"\$0\"\\042 always_transparent\"}'"}" - adb_dnssafesearch="${adb_dnssafesearch:-"${adb_awk} -v item=\"\$item\" '{type=\"AAAA\";if(match(item,/^([0-9]{1,3}\.){3}[0-9]{1,3}$/)){type=\"A\"}}{print \"local-data: \\042\"\$0\" \"type\" \"item\"\\042\"}'"}" - adb_dnsstop="${adb_dnsstop:-"local-zone: \".\" always_nxdomain"}" - ;; - "named") - adb_dnscachecmd="$(command -v rndc || printf "%s" "-")" - adb_dnsinstance="${adb_dnsinstance:-"0"}" - adb_dnsuser="${adb_dnsuser:-"bind"}" - adb_dnsdir="${adb_dnsdir:-"/var/lib/bind"}" - adb_dnsheader="${adb_dnsheader:-"\$TTL 2h\n@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)\n IN NS localhost.\n"}" - adb_dnsdeny="${adb_dnsdeny:-"${adb_awk} '{print \"\"\$0\" CNAME .\\n*.\"\$0\" CNAME .\"}'"}" - adb_dnsallow="${adb_dnsallow:-"${adb_awk} '{print \"\"\$0\" CNAME rpz-passthru.\\n*.\"\$0\" CNAME rpz-passthru.\"}'"}" - adb_dnsdenyip="${adb_dnsdenyip:-"${adb_awk} '{print \"\"\$0\".rpz-client-ip CNAME .\"}'"}" - adb_dnsallowip="${adb_dnsallowip:-"${adb_awk} '{print \"\"\$0\".rpz-client-ip CNAME rpz-passthru.\"}'"}" - adb_dnssafesearch="${adb_dnssafesearch:-"${adb_awk} -v item=\"\$item\" '{print \"\"\$0\" CNAME \"item\".\\n*.\"\$0\" CNAME \"item\".\"}'"}" - adb_dnsstop="${adb_dnsstop:-"* CNAME ."}" - ;; - "kresd") - adb_dnscachecmd="-" - adb_dnsinstance="${adb_dnsinstance:-"0"}" - adb_dnsuser="${adb_dnsuser:-"root"}" - adb_dnsdir="${adb_dnsdir:-"/etc/kresd"}" - adb_dnsheader="${adb_dnsheader:-"\$TTL 2h\n@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)\n"}" - adb_dnsdeny="${adb_dnsdeny:-"${adb_awk} '{print \"\"\$0\" CNAME .\\n*.\"\$0\" CNAME .\"}'"}" - adb_dnsallow="${adb_dnsallow:-"${adb_awk} '{print \"\"\$0\" CNAME rpz-passthru.\\n*.\"\$0\" CNAME rpz-passthru.\"}'"}" - adb_dnssafesearch="${adb_dnssafesearch:-"${adb_awk} -v item=\"\$item\" '{type=\"AAAA\";if(match(item,/^([0-9]{1,3}\.){3}[0-9]{1,3}$/)){type=\"A\"}}{print \"\"\$0\" \"type\" \"item\"\"}'"}" - adb_dnsstop="${adb_dnsstop:-"* CNAME ."}" - ;; - "raw") - adb_dnscachecmd="-" - adb_dnsinstance="${adb_dnsinstance:-"0"}" - adb_dnsuser="${adb_dnsuser:-"root"}" - adb_dnsdir="${adb_dnsdir:-"/tmp"}" - adb_dnsheader="${adb_dnsheader:-""}" - adb_dnsdeny="${adb_dnsdeny:-"0"}" - adb_dnsallow="${adb_dnsallow:-"1"}" - adb_dnssafesearch="${adb_dnssafesearch:-"0"}" - adb_dnsstop="${adb_dnsstop:-"0"}" - ;; - esac - fi - - if [ "${adb_dns}" != "raw" ] && { [ -z "${adb_dns}" ] || [ ! -x "$(command -v "${adb_dns}")" ]; }; then + if [ "${adb_dns}" != "raw" ] && ! command -v "${adb_dns}" >/dev/null 2>&1; then f_log "err" "dns backend not found, please set 'adb_dns' manually" fi - if [ "${adb_dns}" != "raw" ] && { [ "${adb_dnsdir}" = "${adb_tmpbase}" ] || [ "${adb_dnsdir}" = "${adb_backupdir}" ] || [ "${adb_dnsdir}" = "${adb_reportdir}" ]; }; then - f_log "err" "dns directory '${adb_dnsdir}' has been misconfigured, it must not point to the 'adb_tmpbase', 'adb_backupdir', 'adb_reportdir'" - fi - - if [ "${adb_action}" = "start" ] && [ -z "${adb_trigger}" ]; then - sleep ${adb_triggerdelay} - fi - - if [ "${adb_dns}" != "raw" ] && [ "${adb_action}" != "stop" ]; then - while [ "${cnt}" -le 30 ]; do - dns_up="$(ubus -S call service list "{\"name\":\"${adb_dns}\"}" 2>/dev/null | jsonfilter -l1 -e "@[\"${adb_dns}\"].instances.*.running" 2>/dev/null)" - if [ "${dns_up}" = "true" ]; then - break - fi - sleep 1 - cnt="$((cnt + 1))" - done - fi - - if [ "${adb_action}" != "stop" ]; then - if [ -n "${adb_dnsdir}" ] && [ ! -d "${adb_dnsdir}" ]; then - if mkdir -p "${adb_dnsdir}"; then - f_log "info" "dns backend directory '${adb_dnsdir}' created" + case "${adb_dns}" in + "dnsmasq") + adb_dnscachecmd="" + adb_dnsinstance="${adb_dnsinstance:-"0"}" + adb_dnsuser="dnsmasq" + adb_dnsdir="${adb_dnsdir:-""}" + if [ -z "${adb_dnsdir}" ]; then + dns_section="$("${adb_ubuscmd}" -S call uci get "{\"config\":\"dhcp\", \"section\":\"@dnsmasq[${adb_dnsinstance}]\", \"type\":\"dnsmasq\"}" 2>>"${adb_errorlog}")" + dns_info="$(printf '%s' "${dns_section}" | "${adb_jsoncmd}" -l1 -e '@.values["confdir"]')" + if [ -n "${dns_info}" ]; then + adb_dnsdir="${dns_info}" else - f_log "err" "dns backend directory '${adb_dnsdir}' could not be created" + dns_info="$(printf '%s' "${dns_section}" | "${adb_jsoncmd}" -l1 -e '@.values[".name"]')" + [ -n "${dns_info}" ] && adb_dnsdir="/tmp/dnsmasq.${dns_info}.d" fi fi - [ ! -f "${adb_dnsdir}/${adb_dnsfile}" ] && printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" + adb_dnsheader="" + adb_dnsdeny="1" + adb_dnsallow="1" + adb_dnssafesearch="1" + adb_dnsstop="address=/#/\nlocal=/#/" + f_dnsdeny() { + "${adb_awkcmd}" '{print "local=/"$0"/"}' "${@}" + } + f_dnsallow() { + "${adb_awkcmd}" '{print "local=/"$0"/#"}' "${@}" + } + f_dnssafesearch() { + local item="${1}" - if [ "${dns_up}" != "true" ]; then - if ! f_dnsup 4; then - f_log "err" "dns backend '${adb_dns}' not running or executable" - fi - fi + shift + "${adb_awkcmd}" -v item="${item}" '{print "address=/"$0"/"item"";print "local=/"$0"/"}' "${@}" + } + ;; + "unbound") + adb_dnscachecmd="$(f_cmd unbound-control optional)" + adb_dnsinstance="" + adb_dnsuser="unbound" + adb_dnsdir="${adb_dnsdir:-"/var/lib/unbound"}" + adb_dnsheader="" + adb_dnsdeny="1" + adb_dnsallow="1" + adb_dnssafesearch="1" + adb_dnsstop="local-zone: \".\" always_nxdomain" + f_dnsdeny() { + "${adb_awkcmd}" '{print "local-zone: \042"$0"\042 always_nxdomain"}' "${@}" + } + f_dnsallow() { + "${adb_awkcmd}" '{print "local-zone: \042"$0"\042 always_transparent"}' "${@}" + } + f_dnssafesearch() { + local item="${1}" - if [ "${adb_backup}" = "1" ] && [ -n "${adb_backupdir}" ] && [ ! -d "${adb_backupdir}" ]; then - if mkdir -p "${adb_backupdir}"; then - f_log "info" "backup directory '${adb_backupdir}' created" - else - f_log "err" "backup directory '${adb_backupdir}' could not be created" - fi - fi + shift + "${adb_awkcmd}" -v item="${item}" \ + '{type="AAAA";if(match(item,/^([0-9]{1,3}\.){3}[0-9]{1,3}$/)){type="A"}}{print "local-data: \042"$0" "type" "item"\042"}' "${@}" + } + ;; + "named") + adb_dnscachecmd="$(f_cmd rndc optional)" + adb_dnsinstance="" + adb_dnsuser="bind" + adb_dnsdir="${adb_dnsdir:-"/var/lib/bind"}" + adb_dnsheader="\$TTL 2h\n@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)\n IN NS localhost.\n" + adb_dnsdeny="1" + adb_dnsallow="1" + adb_dnssafesearch="1" + adb_dnsstop="* CNAME ." + f_dnsdeny() { + "${adb_awkcmd}" '{print ""$0" CNAME .\n*."$0" CNAME ."}' "${@}" + } + f_dnsallow() { + "${adb_awkcmd}" '{print ""$0" CNAME rpz-passthru.\n*."$0" CNAME rpz-passthru."}' "${@}" + } + f_dnssafesearch() { + local item="${1}" - if [ -n "${adb_jaildir}" ] && [ ! -d "${adb_jaildir}" ]; then - if mkdir -p "${adb_jaildir}"; then - f_log "info" "jail directory '${adb_jaildir}' created" - else - f_log "err" "jail directory '${adb_jaildir}' could not be created" - fi + shift + "${adb_awkcmd}" -v item="${item}" '{print ""$0" CNAME "item".\n*."$0" CNAME "item"."}' "${@}" + } + ;; + "kresd") + adb_dnscachecmd="" + adb_dnsinstance="" + adb_dnsuser="root" + adb_dnsdir="${adb_dnsdir:-"/tmp/kresd"}" + adb_dnsheader="\$TTL 2h\n@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)\n" + adb_dnsdeny="1" + adb_dnsallow="1" + adb_dnssafesearch="1" + adb_dnsstop="* CNAME ." + f_dnsdeny() { + "${adb_awkcmd}" '{print ""$0" CNAME .\n*."$0" CNAME ."}' "${@}" + } + f_dnsallow() { + "${adb_awkcmd}" '{print ""$0" CNAME rpz-passthru.\n*."$0" CNAME rpz-passthru."}' "${@}" + } + f_dnssafesearch() { + local item="${1}" + + shift + "${adb_awkcmd}" -v item="${item}" '{print ""$0" CNAME "item".\n*."$0" CNAME "item"."}' "${@}" + } + ;; + "smartdns") + adb_dnscachecmd="" + adb_dnsinstance="${adb_dnsinstance:-"0"}" + adb_dnsuser="root" + adb_dnsdir="${adb_dnsdir:-"/tmp/smartdns"}" + adb_dnsheader="" + adb_dnsdeny="1" + adb_dnsallow="1" + adb_dnssafesearch="1" + adb_dnsstop="address #" + f_dnsdeny() { + "${adb_awkcmd}" '{print "address /"$0"/#"}' "${@}" + } + f_dnsallow() { + "${adb_awkcmd}" '{print "address /"$0"/-"}' "${@}" + } + f_dnssafesearch() { + local item="${1}" + + shift + "${adb_awkcmd}" -v item="${item}" '{print "cname /"$0"/"item""}' "${@}" + } + ;; + "raw") + adb_dnscachecmd="" + adb_dnsinstance="" + adb_dnsuser="root" + adb_dnsdir="${adb_dnsdir:-"/tmp"}" + adb_dnsheader="" + adb_dnsdeny="" + adb_dnsallow="" + adb_dnssafesearch="" + adb_dnsstop="" + ;; + esac + + # determine final dns file directory based on dns shifting + # + if [ "${adb_dnsshift}" = "0" ]; then + adb_finaldir="${adb_dnsdir}" + [ -L "${adb_dnsdir}/${adb_dnsfile}" ] && "${adb_rmcmd}" -f "${adb_dnsdir}/${adb_dnsfile}" + else + adb_finaldir="${adb_backupdir}" + fi + + # create dns file with header if it doesn't exist or dns flushing is enabled, also create backup and final directories if they don't exist + # + if [ "${adb_action}" != "stop" ]; then + for dir in "${adb_dnsdir:-"/tmp"}" "${adb_backupdir:-"/tmp"}"; do + [ ! -d "${dir}" ] && mkdir -p "${dir}" + done + if [ "${adb_dnsflush}" = "1" ] || [ "${free_mem:-"0"}" -lt "64" ]; then + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" + f_dnsup + elif [ ! -f "${adb_finaldir}/${adb_dnsfile}" ]; then + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" fi fi - f_log "debug" "f_dns ::: dns: ${adb_dns}, dns_dir: ${adb_dnsdir}, dns_file: ${adb_dnsfile}, dns_user: ${adb_dnsuser}, dns_instance: ${adb_dnsinstance}, backup: ${adb_backup}, backup_dir: ${adb_backupdir}, jail_dir: ${adb_jaildir}" + + # check if adblock is enabled + # + if [ "${adb_enabled}" = "0" ]; then + f_extconf + f_temp + f_nftremove + f_rmdns + f_jsnup "disabled" + f_log "info" "adblock is currently disabled, please set the config option 'adb_enabled' to '1' to use this service" + exit 0 + fi + + f_log "debug" "f_dns ::: dns: ${adb_dns}, dns_instance: ${adb_dnsinstance:-"-"}, dns_user: ${adb_dnsuser}, dns_dir: ${adb_dnsdir}, backup_dir: ${adb_backupdir}, final_dir: ${adb_finaldir}" } # load fetch utility # f_fetch() { - local util utils insecure cnt="0" - - if [ -z "${adb_fetchutil}" ]; then - utils="aria2c curl wget uclient-fetch" - for util in ${utils}; do - if { [ "${util}" = "uclient-fetch" ] && printf "%s" "${adb_packages}" | grep -q "^libustream-"; } || - { [ "${util}" = "wget" ] && printf "%s" "${adb_packages}" | grep -q "^wget -"; } || - [ "${util}" = "curl" ] || [ "${util}" = "aria2c" ]; then - if [ -x "$(command -v "${util}")" ]; then - adb_fetchutil="${util}" - uci_set adblock global adb_fetchutil "${util}" + local fetch fetch_list insecure update="0" + + adb_fetchcmd="$(command -v "${adb_fetchcmd}" 2>/dev/null)" + if [ -z "${adb_fetchcmd}" ]; then + fetch_list="curl wget-ssl libustream-openssl libustream-wolfssl libustream-mbedtls" + for fetch in ${fetch_list}; do + case "${adb_packages}" in *"\"${fetch}\""*) + case "${fetch}" in + "wget-ssl") + fetch="wget" + ;; + "libustream-openssl" | "libustream-wolfssl" | "libustream-mbedtls") + fetch="uclient-fetch" + ;; + esac + adb_fetchcmd="$(command -v "${fetch}" 2>/dev/null)" + if [ -n "${adb_fetchcmd}" ]; then + update="1" + uci_set adblock global adb_fetchcmd "${fetch}" f_uci "adblock" break fi - fi + ;; + esac done - elif [ ! -x "$(command -v "${adb_fetchutil}")" ]; then - unset adb_fetchutil fi - case "${adb_fetchutil}" in - "aria2c") - [ "${adb_fetchinsecure}" = "1" ] && insecure="--check-certificate=false" - adb_fetchparm="${adb_fetchparm:-"${insecure} --timeout=20 --allow-overwrite=true --auto-file-renaming=false --log-level=warn --dir=/ -o"}" - ;; - "curl") - [ "${adb_fetchinsecure}" = "1" ] && insecure="--insecure" - adb_fetchparm="${adb_fetchparm:-"${insecure} --connect-timeout 20 --fail --silent --show-error --location -o"}" - ;; - "uclient-fetch") - [ "${adb_fetchinsecure}" = "1" ] && insecure="--no-check-certificate" - adb_fetchparm="${adb_fetchparm:-"${insecure} --timeout=20 -O"}" - ;; - "wget") - [ "${adb_fetchinsecure}" = "1" ] && insecure="--no-check-certificate" - adb_fetchparm="${adb_fetchparm:-"${insecure} --no-cache --no-cookies --max-redirect=0 --timeout=20 -O"}" - ;; + + [ -z "${adb_fetchcmd}" ] && f_log "err" "download utility with SSL support not found, please set 'adb_fetchcmd' manually" + + case "${adb_fetchcmd##*/}" in + "curl") + [ "${adb_fetchinsecure}" = "1" ] && insecure="--insecure" + adb_fetchparm="${adb_fetchparm:-"${insecure} --connect-timeout 20 --retry-delay 10 --retry $((adb_fetchretry - 1)) --retry-max-time $(((adb_fetchretry - 1) * 20)) --retry-all-errors --fail --silent --show-error --location -o"}" + adb_etagparm="--connect-timeout 5 --silent --location --head" + adb_geoparm="--connect-timeout 5 --silent --location" + ;; + "wget") + [ "${adb_fetchinsecure}" = "1" ] && insecure="--no-check-certificate" + adb_fetchparm="${adb_fetchparm:-"${insecure} --no-cache --no-cookies --timeout=20 --waitretry=10 --tries=${adb_fetchretry} --retry-connrefused -O"}" + adb_etagparm="--timeout=5 --spider --server-response" + adb_geoparm="--timeout=5 --quiet -O-" + ;; + "uclient-fetch") + [ "${adb_fetchinsecure}" = "1" ] && insecure="--no-check-certificate" + adb_fetchparm="${adb_fetchparm:-"${insecure} --timeout=20 -O"}" + adb_geoparm="--timeout=5 --quiet -O-" + ;; esac - if [ -n "${adb_fetchutil}" ] && [ -n "${adb_fetchparm}" ]; then - adb_fetchutil="$(command -v "${adb_fetchutil}")" - else - f_log "err" "download utility with SSL support not found, please install 'uclient-fetch' with a 'libustream-*' variant or another download utility like 'wget', 'curl' or 'aria2'" - fi - f_log "debug" "f_fetch ::: fetch_util: ${adb_fetchutil:-"-"}, fetch_parm: ${adb_fetchparm:-"-"}" + + f_log "debug" "f_fetch ::: update: ${update}, cmd: ${adb_fetchcmd:-"-"}, parm: ${adb_fetchparm:-"-"}, etag_parm: ${adb_etagparm:-"-"}, geo_parm: ${adb_geoparm:-"-"}" } # create temporary files, directories and set dependent options # f_temp() { - if [ -d "${adb_tmpbase}" ]; then - adb_tmpdir="$(mktemp -p "${adb_tmpbase}" -d)" + if [ -d "${adb_basedir}" ]; then + adb_tmpdir="$(mktemp -p "${adb_basedir}" -d)" adb_tmpload="$(mktemp -p "${adb_tmpdir}" -tu)" adb_tmpfile="$(mktemp -p "${adb_tmpdir}" -tu)" adb_srtopts="--temporary-directory=${adb_tmpdir} --compress-program=gzip --parallel=${adb_cores}" else - f_log "err" "the temp base directory '${adb_tmpbase}' does not exist/is not mounted yet, please create the directory or raise the 'adb_triggerdelay' to defer the adblock start" + f_log "err" "the base directory '${adb_basedir}' does not exist/is not mounted yet, please create the directory or raise the 'adb_triggerdelay' to defer the adblock start" fi - [ ! -s "${adb_pidfile}" ] && printf "%s" "${$}" >"${adb_pidfile}" - f_log "debug" "f_temp ::: tmp_base: ${adb_tmpbase:-"-"}, tmp_dir: ${adb_tmpdir:-"-"}, sort_options: ${adb_srtopts}, pid_file: ${adb_pidfile:-"-"}" + [ ! -s "${adb_pidfile}" ] && printf '%s' "${$}" >"${adb_pidfile}" } # remove temporary files and directories # f_rmtemp() { - [ -d "${adb_tmpdir}" ] && rm -rf "${adb_tmpdir}" - rm -f "${adb_srcfile}" + [ -f "${adb_errorlog}" ] && [ ! -s "${adb_errorlog}" ] && "${adb_rmcmd}" -f "${adb_errorlog}" + [ -d "${adb_tmpdir}" ] && "${adb_rmcmd}" -rf "${adb_tmpdir}" : >"${adb_pidfile}" - f_log "debug" "f_rmtemp ::: tmp_dir: ${adb_tmpdir:-"-"}, src_file: ${adb_srcfile:-"-"}, pid_file: ${adb_pidfile:-"-"}" } # remove dns related files # f_rmdns() { - local status - - status="$(ubus -S call service list '{"name":"adblock"}' 2>/dev/null | jsonfilter -l1 -e '@["adblock"].instances.*.running' 2>/dev/null)" - if [ "${adb_dns}" = "raw" ] || { [ -n "${adb_dns}" ] && [ -n "${status}" ]; }; then - : >"${adb_rtfile}" - [ "${adb_backup}" = "1" ] && rm -f "${adb_backupdir}/${adb_dnsprefix}".*.gz - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" - f_dnsup 4 + if [ -n "${adb_finaldir}" ]; then + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" + f_dnsup fi f_rmtemp - f_log "debug" "f_rmdns ::: dns: ${adb_dns}, status: ${status:-"-"}, dns_dir: ${adb_dnsdir}, dns_file: ${adb_dnsfile}, rt_file: ${adb_rtfile}, backup_dir: ${adb_backupdir:-"-"}" + if [ -d "${adb_backupdir}" ] && { [ "${adb_action}" = "stop" ] || [ "${adb_enabled}" = "0" ]; }; then + "${adb_findcmd}" "${adb_backupdir}" -maxdepth 1 -type f -name '*.gz' -exec "${adb_rmcmd}" -f {} + + fi } # commit uci changes # f_uci() { - local change config="${1}" - - if [ -n "${config}" ]; then - change="$(uci -q changes "${config}" | "${adb_awk}" '{ORS=" "; print $0}')" - if [ -n "${change}" ]; then - uci_commit "${config}" - case "${config}" in - "firewall") - "/etc/init.d/firewall" reload >/dev/null 2>&1 - ;; - "resolver") - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" - f_count - f_jsnup "running" - "/etc/init.d/${adb_dns}" reload >/dev/null 2>&1 - ;; - esac + local config="${1}" + + if [ -n "$(uci -q changes "${config}")" ]; then + uci_commit "${config}" + if [ "${config}" = "resolver" ]; then + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" + adb_cnt="0" + f_jsnup "processing" + "/etc/init.d/${adb_dns}" reload >/dev/null 2>&1 fi - f_log "debug" "f_uci ::: config: ${config}, change: ${change}" fi } # get list counter # f_count() { - local file mode="${1}" name="${2}" + local files mode="${1}" file="${2}" var="${3}" adb_cnt="0" - case "${mode}" in - "iplist") - [ -s "${adb_tmpdir}/tmp.add.${name}" ] && adb_cnt="$(wc -l 2>/dev/null <"${adb_tmpdir}/tmp.add.${name}")" - ;; - "blacklist") - [ -s "${adb_tmpfile}.${name}" ] && adb_cnt="$(wc -l 2>/dev/null <"${adb_tmpfile}.${name}")" - ;; - "whitelist") - [ -s "${adb_tmpdir}/tmp.raw.${name}" ] && { adb_cnt="$(wc -l 2>/dev/null <"${adb_tmpdir}/tmp.raw.${name}")"; rm -f "${adb_tmpdir}/tmp.raw.${name}"; } - ;; - "safesearch") - [ -s "${adb_tmpdir}/tmp.safesearch.${name}" ] && adb_cnt="$(wc -l 2>/dev/null <"${adb_tmpdir}/tmp.safesearch.${name}")" - ;; - "merge") - [ -s "${adb_tmpdir}/${adb_dnsfile}" ] && adb_cnt="$(wc -l 2>/dev/null <"${adb_tmpdir}/${adb_dnsfile}")" - ;; - "download" | "backup" | "restore") - [ -s "${src_tmpfile}" ] && adb_cnt="$(wc -l 2>/dev/null <"${src_tmpfile}")" - ;; - "final") - if [ -s "${adb_dnsdir}/${adb_dnsfile}" ]; then - adb_cnt="$(wc -l 2>/dev/null <"${adb_dnsdir}/${adb_dnsfile}")" - if [ -s "${adb_tmpdir}/tmp.add.whitelist" ]; then - adb_cnt="$((adb_cnt - $(wc -l 2>/dev/null <"${adb_tmpdir}/tmp.add.whitelist")))" - fi - for file in "${adb_tmpdir}/tmp.safesearch".*; do - if [ -r "${file}" ]; then - adb_cnt="$((adb_cnt - $(wc -l 2>/dev/null <"${file}")))" - fi - done - [ -n "${adb_dnsheader}" ] && adb_cnt="$(((adb_cnt - $(printf "%b" "${adb_dnsheader}" | grep -c "^")) / 2))" - fi - ;; - esac + if [ -s "${file}" ]; then + if [ -n "${var}" ] || [ "${mode}" != "final" ]; then + adb_cnt="$("${adb_awkcmd}" 'END{print NR}' "${file}")" + [ -n "${var}" ] && printf '%s' "${adb_cnt}" + else + # build argument list: main file first, then all subtraction files + # + files="${file}" + [ -s "${adb_tmpdir}/tmp.add.allowlist" ] && files="${files} ${adb_tmpdir}/tmp.add.allowlist" + for file in "${adb_tmpdir}/tmp.safesearch".*; do + [ -s "${file}" ] && files="${files} ${file}" + done + adb_cnt="$("${adb_awkcmd}" -v hdr="${adb_dnsheader}" ' + NR == FNR { main++; next } + { sub_cnt++ } + END { + cnt = main - sub_cnt + if (hdr != "") { + hlines = gsub(/\n/, "", hdr) + cnt = int((cnt - hlines) / 2) + } + if (cnt < 0) cnt = 0 + res = "" + pos = 0 + s = sprintf("%d", cnt) + for (i = length(s); i > 0; i--) { + res = substr(s, i, 1) res + if (++pos == 3 && i > 1) { res = " " res; pos = 0 } + } + print res + } + ' ${files})" + fi + fi } # set external config options # f_extconf() { - local config config_dir config_file section zone port fwcfg + local config case "${adb_dns}" in - "dnsmasq") - config="dhcp" - config_dir="$(uci_get dhcp "@dnsmasq[${adb_dnsinstance}]" confdir | grep -Fo "${adb_dnsdir}")" - if [ "${adb_enabled}" = "1" ] && [ -z "${config_dir}" ]; then - uci_set dhcp "@dnsmasq[${adb_dnsinstance}]" confdir "${adb_dnsdir}" 2>/dev/null - fi - ;; - "kresd") - config="resolver" - config_file="$(uci_get resolver kresd rpz_file | grep -Fo "${adb_dnsdir}/${adb_dnsfile}")" - if [ "${adb_enabled}" = "1" ] && [ -z "${config_file}" ]; then - uci -q add_list resolver.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" - elif [ "${adb_enabled}" = "0" ] && [ -n "${config_file}" ]; then - uci -q del_list resolver.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" - fi - ;; - esac - f_uci "${config}" - - config="firewall" - fwcfg="$(uci -qNX show "${config}" | "${adb_awk}" 'BEGIN{FS="[.=]"};/adblock_/{if(zone==$2){next}else{ORS=" ";zone=$2;print zone}}')" - if [ "${adb_enabled}" = "1" ] && [ "${adb_forcedns}" = "1" ] && - /etc/init.d/firewall enabled; then - for zone in ${adb_zonelist}; do - for port in ${adb_portlist}; do - if ! printf "%s" "${fwcfg}" | grep -q "adblock_${zone}${port}[ |\$]"; then - uci -q batch <<-EOC - set firewall."adblock_${zone}${port}"="redirect" - set firewall."adblock_${zone}${port}".name="Adblock DNS (${zone}, ${port})" - set firewall."adblock_${zone}${port}".src="${zone}" - set firewall."adblock_${zone}${port}".proto="tcp udp" - set firewall."adblock_${zone}${port}".src_dport="${port}" - set firewall."adblock_${zone}${port}".dest_port="${port}" - set firewall."adblock_${zone}${port}".target="DNAT" - set firewall."adblock_${zone}${port}".family="any" - set firewall."adblock_${zone}${port}".ipset="!tsdns_bypass" - add_list firewall."adblock_${zone}${port}".ns_tag="automated" - EOC - fi - fwcfg="${fwcfg/adblock_${zone}${port}[ |\$]/}" - done - done - fwcfg="${fwcfg#"${fwcfg%%[![:space:]]*}"}" - fwcfg="${fwcfg%"${fwcfg##*[![:space:]]}"}" - fi - if [ "${adb_enabled}" = "0" ] || [ "${adb_forcedns}" = "0" ] || [ -n "${fwcfg}" ]; then - for section in ${fwcfg}; do - uci_remove firewall "${section}" - done - fi - - # add adb_bypass - if [ "${adb_enabled}" = "1" ] && [ "${adb_forcedns}" = "1" ] && /etc/init.d/firewall enabled; then - if ! uci -q get firewall.tsdns_bypass >/dev/null; then - uci -q batch <<-EOC - set firewall.tsdns_bypass="ipset" - set firewall.tsdns_bypass.name="tsdns_bypass" - set firewall.tsdns_bypass.match="src_net" - set firewall.tsdns_bypass.enabled="1" - EOC + "dnsmasq") + config="dhcp" + if [ "${adb_dnsshift}" = "1" ] && + ! uci_get ${config} @dnsmasq[${adb_dnsinstance}] addnmount | "${adb_grepcmd}" -q "${adb_backupdir}"; then + uci -q add_list ${config}.@dnsmasq[${adb_dnsinstance}].addnmount="${adb_backupdir}" + elif [ "${adb_dnsshift}" = "0" ] && + uci_get ${config} @dnsmasq[${adb_dnsinstance}] addnmount | "${adb_grepcmd}" -q "${adb_backupdir}"; then + uci -q del_list ${config}.@dnsmasq[${adb_dnsinstance}].addnmount="${adb_backupdir}" fi - # note: adb_bypass var contains an extra space at the beginning - if [ " $(uci -q get firewall.tsdns_bypass.entry)" != "${adb_bypass}" ]; then - # make sure bypass list is always empty - uci -q delete firewall.tsdns_bypass.entry - for src in ${adb_bypass} - do - uci -q add_list firewall.tsdns_bypass.entry="${src}" - done + ;; + "kresd") + config="resolver" + if [ "${adb_enabled}" = "1" ] && + ! uci_get ${config} kresd rpz_file | "${adb_grepcmd}" -q "${adb_dnsdir}/${adb_dnsfile}"; then + uci -q add_list ${config}.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" + elif [ "${adb_enabled}" = "0" ] && + uci_get ${config} kresd rpz_file | "${adb_grepcmd}" -q "${adb_dnsdir}/${adb_dnsfile}"; then + uci -q del_list ${config}.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" fi - fi - - # remove adb_bypass - if [ "${adb_enabled}" = "0" ] || [ "${adb_forcedns}" = "0" ]; then - uci -q delete firewall.tsdns_bypass - fi - + ;; + "smartdns") + config="smartdns" + if [ "${adb_enabled}" = "1" ] && + ! uci_get ${config} @${config}[${adb_dnsinstance}] conf_files | "${adb_grepcmd}" -q "${adb_dnsdir}/${adb_dnsfile}"; then + uci -q add_list ${config}.@${config}[${adb_dnsinstance}].conf_files="${adb_dnsdir}/${adb_dnsfile}" + elif [ "${adb_enabled}" = "0" ] && + uci_get ${config} @${config}[${adb_dnsinstance}] conf_files | "${adb_grepcmd}" -q "${adb_dnsdir}/${adb_dnsfile}"; then + uci -q del_list ${config}.@${config}[${adb_dnsinstance}].conf_files="${adb_dnsdir}/${adb_dnsfile}" + fi + ;; + esac f_uci "${config}" } # restart dns backend # f_dnsup() { - local rset dns_service dns_up dns_pid restart_rc cnt="0" out_rc="4" in_rc="${1:-0}" + local restart_rc nft_rc cnt="0" out_rc="4" - if [ "${adb_dns}" = "raw" ]; then + if [ "${adb_dns}" = "raw" ] || [ -z "${adb_dns}" ]; then out_rc="0" else - if [ "${in_rc}" = "0" ] && [ "${adb_dnsflush}" = "0" ]; then + + # load external dns bridge + # + if { [ -n "${adb_bridgednsv4}" ] || [ -n "${adb_bridgednsv6}" ]; } && [ "${adb_nftbridge}" = "1" ]; then + if "${adb_nftcmd}" list chain inet adblock dns-bridge >/dev/null 2>&1; then + if [ -n "${adb_bridgednsv4}" ]; then + "${adb_nftcmd}" add rule inet adblock dns-bridge meta nfproto ipv4 meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_bridgednsv4}:53 2>>"${adb_errorlog}" + nft_rc="${?}" + fi + if [ -n "${adb_bridgednsv6}" ]; then + "${adb_nftcmd}" add rule inet adblock dns-bridge meta nfproto ipv6 meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_bridgednsv6}]:53 2>>"${adb_errorlog}" + nft_rc="$((nft_rc + $?))" + fi + if [ "${nft_rc}" = "0" ]; then + f_log "debug" "external DNS bridge loaded: ${adb_bridgednsv4:-"-"} / ${adb_bridgednsv6:-"-"}" + else + f_log "err" "failed to load external DNS bridge: ${adb_bridgednsv4:-"-"} / ${adb_bridgednsv6:-"-"}" + fi + fi + fi + + # restart dns backend + # + if [ "${adb_dnsflush}" = "0" ]; then case "${adb_dns}" in - "unbound") - if [ -x "${adb_dnscachecmd}" ] && [ -d "${adb_tmpdir}" ] && [ -f "${adb_dnsdir}/unbound.conf" ]; then - "${adb_dnscachecmd}" -c "${adb_dnsdir}/unbound.conf" dump_cache >"${adb_tmpdir}/adb_cache.dump" 2>/dev/null - fi - "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 + "unbound") + if [ -x "${adb_dnscachecmd}" ] && [ -d "${adb_tmpdir}" ] && [ -f "${adb_dnsdir}/unbound.conf" ]; then + "${adb_dnscachecmd}" -c "${adb_dnsdir}/unbound.conf" dump_cache >"${adb_tmpdir}/adb_cache.dump" 2>>"${adb_errorlog}" + fi + "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 + restart_rc="${?}" + ;; + "named") + if [ -x "${adb_dnscachecmd}" ] && [ -f "/etc/bind/rndc.conf" ]; then + "${adb_dnscachecmd}" -c "/etc/bind/rndc.conf" reload >/dev/null 2>&1 restart_rc="${?}" - ;; - "named") - if [ -x "${adb_dnscachecmd}" ] && [ -f "/etc/bind/rndc.conf" ]; then - "${adb_dnscachecmd}" -c "/etc/bind/rndc.conf" reload >/dev/null 2>&1 - restart_rc="${?}" - fi - if [ -z "${restart_rc}" ] || { [ -n "${restart_rc}" ] && [ "${restart_rc}" != "0" ]; }; then - "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 - restart_rc="${?}" - fi - ;; - *) + fi + if [ -z "${restart_rc}" ] || { [ -n "${restart_rc}" ] && [ "${restart_rc}" != "0" ]; }; then "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 restart_rc="${?}" - ;; + fi + ;; + *) + "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 + restart_rc="${?}" + ;; esac fi if [ -z "${restart_rc}" ]; then @@ -652,24 +783,14 @@ f_dnsup() { restart_rc="${?}" fi fi + + # check if dns backend is responsive, restore dns cache for unbound and get dns backend pid + # if [ "${restart_rc}" = "0" ]; then - rset="/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}" while [ "${cnt}" -le "${adb_dnstimeout}" ]; do - dns_service="$(ubus -S call service list "{\"name\":\"${adb_dns}\"}")" - dns_up="$(printf "%s" "${dns_service}" | jsonfilter -l1 -e "@[\"${adb_dns}\"].instances.*.running")" - dns_pid="$(printf "%s" "${dns_service}" | jsonfilter -l1 -e "@[\"${adb_dns}\"].instances.*.pid")" - if [ "${dns_up}" = "true" ] && [ -n "${dns_pid}" ] && ! ls "/proc/${dns_pid}/fd/${adb_dnsdir}/${adb_dnsfile}" >/dev/null 2>&1; then - if [ -x "${adb_lookupcmd}" ] && [ -n "$(printf "%s" "${adb_lookupdomain}" | "${adb_awk}" "${rset}")" ]; then - if "${adb_lookupcmd}" "${adb_lookupdomain}" >/dev/null 2>&1; then - out_rc="0" - break - fi - else - sleep ${adb_dnstimeout} - cnt=${adb_dnstimeout} - out_rc="0" - break - fi + if "${adb_lookupcmd}" "${adb_lookupdomain}." >/dev/null 2>&1; then + out_rc="0" + break fi cnt="$((cnt + 1))" sleep 1 @@ -681,231 +802,519 @@ f_dnsup() { fi fi fi - f_log "debug" "f_dnsup ::: dns: ${adb_dns}, cache_cmd: ${adb_dnscachecmd:-"-"}, lookup_cmd: ${adb_lookupcmd:-"-"}, lookup_domain: ${adb_lookupdomain:-"-"}, restart_rc: ${restart_rc:-"-"}, dns_flush: ${adb_dnsflush}, dns_timeout: ${adb_dnstimeout}, dns_cnt: ${cnt}, in_rc: ${in_rc}, out_rc: ${out_rc}" + + # remove external dns bridge + # + if [ "${adb_nftbridge}" = "1" ] && "${adb_nftcmd}" list chain inet adblock dns-bridge >/dev/null 2>&1; then + "${adb_nftcmd}" flush chain inet adblock dns-bridge 2>>"${adb_errorlog}" + nft_rc="${?}" + if [ "${nft_rc}" = "0" ]; then + f_log "debug" "external DNS bridge removed" + else + f_log "err" "failed to remove external DNS bridge" + fi + fi + + f_log "debug" "f_dnsup ::: dns: ${adb_dns}, cache_cmd: ${adb_dnscachecmd:-"-"}, lookup_domain: ${adb_lookupdomain:-"-"}, restart_rc: ${restart_rc:-"-"}, dns_flush: ${adb_dnsflush}, dns_timeout: ${adb_dnstimeout}, dns_cnt: ${cnt}, nft_rc: ${nft_rc:-"-"}, rc: ${out_rc}" return "${out_rc}" } +# handle etag http header +# +f_etag() { + local http_head http_code etag_id etag_cnt out_rc="4" feed="${1}" feed_url="${2}" feed_suffix="${3}" feed_cnt="${4:-"1"}" + + if [ -n "${adb_etagparm}" ]; then + + # ensure etag file exists + # + [ ! -f "${adb_backupdir}/adblock.etag" ] && : >"${adb_backupdir}/adblock.etag" + + # fetch http headers and extract http code and etag/last-modified header + # + http_head="$("${adb_fetchcmd}" ${adb_etagparm} "${feed_url}${feed_suffix}" 2>&1)" + http_code="$(printf '%s' "${http_head}" | "${adb_awkcmd}" 'tolower($0)~/^[[:space:]]*http\/[0-9.]+ /{printf "%s",$2}')" + etag_id="$(printf '%s' "${http_head}" | "${adb_awkcmd}" 'tolower($0)~/^[[:space:]]*etag: /{gsub("\"","");printf "%s",$2}')" + + # if etag header is not present, try to use last-modified header as fallback for change detection + # + if [ -z "${etag_id}" ]; then + etag_id="$(printf '%s' "${http_head}" | "${adb_awkcmd}" 'tolower($0)~/^[[:space:]]*last-modified: /{gsub(/[Ll]ast-[Mm]odified:|[[:space:]]|,|:/,"");printf "%s\n",$1}')" + fi + + # acquire exclusive lock on etag file to serialize concurrent read-modify-write from parallel feeds + # + exec 9>"${adb_etaglock}" + "${adb_flockcmd}" -x 9 + + # compare http code and etag id with stored values, update etag file and return code accordingly + # + etag_cnt="$("${adb_awkcmd}" -v f="${feed}" '$1 == f { n++ } END { print n+0 }' "${adb_backupdir}/adblock.etag")" + if [ "${http_code}" = "200" ] && [ "${etag_cnt}" = "${feed_cnt}" ] && [ -n "${etag_id}" ] && + "${adb_awkcmd}" -v f="${feed}" -v s="${feed_suffix}" -v e="${etag_id}" ' + BEGIN { rc = 1; p = f " " s } + index($0, p) == 1 { + rest = substr($0, length(p) + 1) + sub(/^[[:space:]]+/, "", rest) + if (rest == e) { rc = 0; exit } + } + END { exit rc }' "${adb_backupdir}/adblock.etag"; then + out_rc="0" + elif [ -n "${etag_id}" ]; then + + # if feed count is less than etag count, it means the feed source has been removed or disabled, so remove all entries for this feed, + # otherwise only remove the entry with the matching feed suffix (feed url) to allow multiple sources for the same feed + # + if [ "${feed_cnt}" -lt "${etag_cnt}" ]; then + "${adb_awkcmd}" -v f="${feed}" '$1 != f' \ + "${adb_backupdir}/adblock.etag" >"${adb_backupdir}/adblock.etag.new" + else + "${adb_awkcmd}" -v f="${feed}" -v s="${feed_suffix}" ' + BEGIN { p = f " " s } + index($0, p) != 1' \ + "${adb_backupdir}/adblock.etag" >"${adb_backupdir}/adblock.etag.new" + fi + "${adb_mvcmd}" -f "${adb_backupdir}/adblock.etag.new" "${adb_backupdir}/adblock.etag" + printf '%s\t%s\n' "${feed} ${feed_suffix}" "${etag_id}" >>"${adb_backupdir}/adblock.etag" + out_rc="2" + fi + + # release lock + # + exec 9>&- + fi + + f_log "debug" "f_etag ::: feed: ${feed}, suffix: ${feed_suffix:-"-"}, http_code: ${http_code:-"-"}, feed/etag: ${feed_cnt}/${etag_cnt:-"0"}, rc: ${out_rc}" + return "${out_rc}" +} + +# add adblock-related nft rules +# +f_nftadd() { + local devices device port file="${adb_tmpdir}/adb_nft.add" + + # only proceed if at least one feature is enabled + # + if [ "${adb_nftallow}" = "0" ] && [ "${adb_nftblock}" = "0" ] && + [ "${adb_nftremote}" = "0" ] && [ "${adb_nftforce}" = "0" ] && + [ "${adb_nftbridge}" = "0" ]; then + return + fi + + # remove existing adblock-related nft rules on restart + # + [ "${adb_action}" = "restart" ] && f_nftremove + + # do not proceed if adblock-related nft rules already exists + # + if "${adb_nftcmd}" -t list table inet adblock >/dev/null 2>&1; then + return + fi + + # do not proceed if action is stop/suspend/resume + # + if [ "${adb_action}" = "stop" ] || [ "${adb_action}" = "suspend" ] || [ "${adb_action}" = "resume" ]; then + return + fi + + # prepare nftables rules for adblock features + # + { + # nft header (tables, sets, base and regular chains) + # + printf '%s\n\n' "#!${adb_nftcmd} -f" + printf '%s\n' "add table inet adblock" + + # allow Set + # + if [ "${adb_nftallow}" = "1" ] && [ -n "${adb_nftmacallow}" ]; then + printf '%s\n' "add set inet adblock mac_allow { type ether_addr; flags interval; auto-merge; elements = { ${adb_nftmacallow// /, } }; }" + fi + + # remote allow Set with timeout, for MACs that should be temporary allowed to bypass dns blocking + # + if [ "${adb_nftremote}" = "1" ] && [ -n "${adb_nftmacremote}" ]; then + printf '%s\n' "add set inet adblock mac_remote { type ether_addr; flags timeout; timeout ${adb_nftremotetimeout}m; }" + fi + + # adblock pre-routing chain for allow/block rules + # + if [ "${adb_nftblock}" = "1" ] && [ -n "${adb_nftmacblock}" ]; then + printf '%s\n' "add set inet adblock mac_block { type ether_addr; flags interval; auto-merge; elements = { ${adb_nftmacblock// /, } }; }" + fi + printf '%s\n' "add chain inet adblock pre-routing { type nat hook prerouting priority -150; policy accept; }" + printf '%s\n' "add chain inet adblock _reject" + + # dns-bridge base chain + # + printf '%s\n' "add chain inet adblock dns-bridge { type nat hook prerouting priority -160; policy accept; }" + + # reject chain rules + # + printf '%s\n' "add rule inet adblock _reject meta l4proto tcp counter reject with tcp reset" + printf '%s\n' "add rule inet adblock _reject counter reject with icmpx host-unreachable" + + # external allow rules + # + if [ "${adb_nftallow}" = "1" ]; then + if [ -n "${adb_nftmacallow}" ]; then + [ -n "${adb_allowdnsv4}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv4 ether saddr @mac_allow meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_allowdnsv4}:53" + [ -n "${adb_allowdnsv6}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv6 ether saddr @mac_allow meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_allowdnsv6}]:53" + fi + for device in ${adb_nftdevallow}; do + [ -n "${adb_allowdnsv4}" ] && printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto ipv4 meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_allowdnsv4}:53" + [ -n "${adb_allowdnsv6}" ] && printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto ipv6 meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_allowdnsv6}]:53" + done + f_log "debug" "adblock-related nft allow rules prepared for external DNS ${adb_allowdnsv4:-"-"} / ${adb_allowdnsv6:-"-"}" + fi + + # external remote allow rules + # + if [ "${adb_nftremote}" = "1" ] && [ -n "${adb_nftmacremote}" ]; then + [ -n "${adb_remotednsv4}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv4 ether saddr @mac_remote meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_remotednsv4}:53" + [ -n "${adb_remotednsv6}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv6 ether saddr @mac_remote meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_remotednsv6}]:53" + f_log "debug" "adblock-related nft remote allow rules prepared for external DNS ${adb_remotednsv4:-"-"} / ${adb_remotednsv6:-"-"} with timeout of ${adb_nftremotetimeout} minutes" + fi + + # external block rules + # + if [ "${adb_nftblock}" = "1" ]; then + if [ -n "${adb_nftmacblock}" ]; then + [ -n "${adb_blockdnsv4}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv4 ether saddr @mac_block meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_blockdnsv4}:53" + [ -n "${adb_blockdnsv6}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv6 ether saddr @mac_block meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_blockdnsv6}]:53" + fi + for device in ${adb_nftdevblock}; do + [ -n "${adb_blockdnsv4}" ] && printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto ipv4 meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_blockdnsv4}:53" + [ -n "${adb_blockdnsv6}" ] && printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto ipv6 meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_blockdnsv6}]:53" + done + f_log "debug" "adblock-related nft block rules prepared for external DNS ${adb_blockdnsv4:-"-"} / ${adb_blockdnsv6:-"-"}" + fi + + # local dns enforcement + # + if [ "${adb_nftforce}" = "1" ]; then + + # device/vlan exceptions + # + for device in ${adb_nftdevallow} ${adb_nftdevblock}; do + case " ${devices} " in + *" ${device} "*) ;; + + *) + [ -n "${devices}" ] && devices="${devices} ${device}" || devices="${device}" + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" return" + ;; + esac + done + + # mac exceptions + # + for device in ${adb_nftdevforce}; do + if [ "${adb_nftallow}" = "1" ] && [ -n "${adb_nftmacallow}" ]; then + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" ether saddr @mac_allow return" + fi + if [ "${adb_nftblock}" = "1" ] && [ -n "${adb_nftmacblock}" ]; then + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" ether saddr @mac_block return" + fi + + # NethSecurity: IP-based bypass exceptions + # + for bypass_ip in ${ns_tsdns_bypass}; do + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" ip saddr ${bypass_ip} return" + done + + # dns enforce rules + # + for port in ${adb_nftportforce}; do + if [ "${port}" = "53" ]; then + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto { ipv4, ipv6 } meta l4proto { udp, tcp } th dport ${port} counter redirect to :${port}" + else + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto { ipv4, ipv6 } meta l4proto { udp, tcp } th dport ${port} counter goto _reject" + fi + done + done + f_log "debug" "adblock-related nft local DNS enforcement rules prepared for devices: ${adb_nftdevforce// /, } and ports: ${adb_nftportforce// /, }" + fi + } >"${file}" + if "${adb_nftcmd}" -f "${file}" 2>>"${adb_errorlog}"; then + f_log "info" "adblock-related nft rules loaded" + else + f_log "err" "failed to load adblock-related nft rules" + fi +} + +# remove adblock-related nft rules +# +f_nftremove() { + local file="${adb_tmpdir}/adb_nft.remove" + + if "${adb_nftcmd}" -t list table inet adblock >/dev/null 2>&1; then + { + printf '%s\n' "#!${adb_nftcmd} -f" + printf '%s\n' "delete table inet adblock" + } >"${file}" + + if "${adb_nftcmd}" -f "${file}" 2>>"${adb_errorlog}"; then + f_log "info" "adblock-related nft rules removed" + else + f_log "err" "failed to remove adblock-related nft rules" + fi + fi +} + # backup/restore/remove blocklists # f_list() { - local hold file rset item array safe_url safe_ips safe_cname safe_domains ip out_rc mode="${1}" src_name="${2:-"${src_name}"}" in_rc="${src_rc:-0}" cnt ffiles="-maxdepth 1 -name ${adb_dnsprefix}.*.gz" + local file files item array safe_url safe_ips safe_cname safe_domains ip out_rc file_name name + local mode="${1}" src_name="${2:-"${src_name}"}" in_rc="${src_rc:-0}" use_cname="0" case "${mode}" in - "iplist") - src_name="${mode}" - if [ "${adb_dns}" = "named" ]; then - rset="BEGIN{FS=\"[.:]\";pfx=\"32\"}{if(match(\$0,/:/))pfx=\"128\"}{printf \"%s.\",pfx;for(seg=NF;seg>=1;seg--)if(seg==1)printf \"%s\n\",\$seg;else if(\$seg>=0)printf \"%s.\",\$seg; else printf \"%s.\",\"zz\"}" - if [ -n "${adb_allowip}" ]; then - : >"${adb_tmpdir}/tmp.raw.${src_name}" - for ip in ${adb_allowip}; do - printf "%s" "${ip}" | "${adb_awk}" "${rset}" >>"${adb_tmpdir}/tmp.raw.${src_name}" - done - eval "${adb_dnsallowip}" "${adb_tmpdir}/tmp.raw.${src_name}" >"${adb_tmpdir}/tmp.add.${src_name}" + "blocklist" | "allowlist") + src_name="${mode}" + case "${src_name}" in + "blocklist") + if [ -f "${adb_blocklist}" ]; then + file_name="${adb_tmpfile}.${src_name}" + f_chkdom local 1 <"${adb_blocklist}" >"${adb_tmpdir}/tmp.raw.${src_name}" + if [ "${adb_tld}" = "1" ]; then + if [ -s "${adb_tmpdir}/tmp.rem.allowlist" ]; then + "${adb_awkcmd}" ' + NR==FNR { member[$1]; next } + !($1 in member) { + n = split($1, seg, ".") + for (f = n; f > 1; f--) printf "%s.", seg[f] + print seg[1] + } + ' "${adb_tmpdir}/tmp.rem.allowlist" "${adb_tmpdir}/tmp.raw.${src_name}" + else + "${adb_awkcmd}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' "${adb_tmpdir}/tmp.raw.${src_name}" + fi | "${adb_sortcmd}" ${adb_srtopts} -u >"${file_name}" out_rc="${?}" - fi - if [ -n "${adb_denyip}" ] && { [ -z "${out_rc}" ] || [ "${out_rc}" = "0" ]; }; then - : >"${adb_tmpdir}/tmp.raw.${src_name}" - for ip in ${adb_denyip}; do - printf "%s" "${ip}" | "${adb_awk}" "${rset}" >>"${adb_tmpdir}/tmp.raw.${src_name}" - done - eval "${adb_dnsdenyip}" "${adb_tmpdir}/tmp.raw.${src_name}" >>"${adb_tmpdir}/tmp.add.${src_name}" + else + if [ -s "${adb_tmpdir}/tmp.rem.allowlist" ]; then + "${adb_awkcmd}" 'NR==FNR{member[$1];next}!($1 in member)' "${adb_tmpdir}/tmp.rem.allowlist" "${adb_tmpdir}/tmp.raw.${src_name}" | + "${adb_sortcmd}" ${adb_srtopts} -u >"${file_name}" 2>>"${adb_errorlog}" + else + "${adb_sortcmd}" ${adb_srtopts} -u "${adb_tmpdir}/tmp.raw.${src_name}" 2>>"${adb_errorlog}" >"${file_name}" + fi out_rc="${?}" fi - rm -f "${adb_tmpdir}/tmp.raw.${src_name}" fi ;; - "blacklist" | "whitelist") - src_name="${mode}" - if [ "${src_name}" = "blacklist" ] && [ -f "${adb_blacklist}" ]; then - rset="/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}" - "${adb_awk}" "${rset}" "${adb_blacklist}" >"${adb_tmpdir}/tmp.raw.${src_name}" - if [ -s "${adb_whitelist}" ]; then - "${adb_awk}" 'NR==FNR{member[$1];next}!($1 in member)' "${adb_whitelist}" "${adb_tmpdir}/tmp.raw.${src_name}" >"${adb_tmpdir}/tmp.deduplicate.${src_name}" - else - cat "${adb_tmpdir}/tmp.raw.${src_name}" >"${adb_tmpdir}/tmp.deduplicate.${src_name}" - fi - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' "${adb_tmpdir}/tmp.deduplicate.${src_name}" >"${adb_tmpdir}/tmp.raw.${src_name}" - "${adb_sort}" ${adb_srtopts} -u "${adb_tmpdir}/tmp.raw.${src_name}" 2>/dev/null >"${adb_tmpfile}.${src_name}" - out_rc="${?}" - rm -f "${adb_tmpdir}/tmp.raw.${src_name}" - elif [ "${src_name}" = "whitelist" ] && [ -f "${adb_whitelist}" ]; then - rset="/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}" - printf "%s\n" "${adb_lookupdomain}" | "${adb_awk}" "${rset}" >"${adb_tmpdir}/tmp.raw.${src_name}" - "${adb_awk}" "${rset}" "${adb_whitelist}" >>"${adb_tmpdir}/tmp.raw.${src_name}" + "allowlist") + if [ -f "${adb_allowlist}" ] && [ "${adb_dnsallow}" = "1" ]; then + file_name="${adb_tmpdir}/tmp.raw.${src_name}" + [ "${adb_lookupdomain}" != "localhost" ] && { printf '%s\n' "${adb_lookupdomain}" | f_chkdom local 1; } >"${file_name}" + f_chkdom local 1 <"${adb_allowlist}" >>"${file_name}" + "${adb_catcmd}" "${file_name}" >"${adb_tmpdir}/tmp.rem.${src_name}" + f_dnsallow "${file_name}" >"${adb_tmpdir}/tmp.add.${src_name}" out_rc="${?}" - if [ "${out_rc}" = "0" ]; then - rset="/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{gsub(\"\\\\.\",\"\\\\.\",\$1);print tolower(\"^(|.*\\\\.)\"\$1\"$\")}" - "${adb_awk}" "${rset}" "${adb_tmpdir}/tmp.raw.${src_name}" >"${adb_tmpdir}/tmp.rem.${src_name}" - out_rc="${?}" - if [ "${out_rc}" = "0" ] && [ "${adb_dnsallow}" != "1" ]; then - eval "${adb_dnsallow}" "${adb_tmpdir}/tmp.raw.${src_name}" >"${adb_tmpdir}/tmp.add.${src_name}" - out_rc="${?}" - if [ "${out_rc}" = "0" ] && [ "${adb_jail}" = "1" ] && [ "${adb_dnsstop}" != "0" ]; then - : >"${adb_jaildir}/${adb_dnsjail}" - [ -n "${adb_dnsheader}" ] && printf "%b" "${adb_dnsheader}" >>"${adb_jaildir}/${adb_dnsjail}" - cat "${adb_tmpdir}/tmp.add.${src_name}" >>"${adb_jaildir}/${adb_dnsjail}" - printf "%s\n" "${adb_dnsstop}" >>"${adb_jaildir}/${adb_dnsjail}" - fi - fi + if [ "${adb_jail}" = "1" ] && [ -n "${adb_dnsstop}" ]; then + printf '%b' "${adb_dnsheader}" >"${adb_tmpdir}/${adb_dnsfile}" + "${adb_catcmd}" "${adb_tmpdir}/tmp.add.${src_name}" >>"${adb_tmpdir}/${adb_dnsfile}" + printf '%b\n' "${adb_dnsstop}" >>"${adb_tmpdir}/${adb_dnsfile}" fi fi ;; - "safesearch") - case "${src_name}" in - "google") - rset="/^\\.([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{printf \"%s\n%s\n\",tolower(\"www\"\$1),tolower(substr(\$1,2,length(\$1)))}" - safe_url="https://www.google.com/supported_domains" - safe_cname="forcesafesearch.google.com" - safe_domains="${adb_tmpdir}/tmp.load.safesearch.${src_name}" - if [ "${adb_backup}" = "1" ] && [ -s "${adb_backupdir}/safesearch.${src_name}.gz" ]; then - zcat "${adb_backupdir}/safesearch.${src_name}.gz" >"${safe_domains}" - out_rc="${?}" - else - "${adb_fetchutil}" ${adb_fetchparm} "${safe_domains}" "${safe_url}" 2>/dev/null - out_rc="${?}" - if [ "${adb_backup}" = "1" ] && [ "${out_rc}" = "0" ]; then - gzip -cf "${safe_domains}" >"${adb_backupdir}/safesearch.${src_name}.gz" - out_rc="${?}" - fi - fi - if [ "${out_rc}" = "0" ]; then - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && "${adb_awk}" "${rset}" "${safe_domains}" >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - fi - ;; - "bing") - safe_cname="strict.bing.com" - safe_domains="www.bing.com" - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && printf "%s\n" ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - ;; - "duckduckgo") - safe_cname="safe.duckduckgo.com" - safe_domains="duckduckgo.com" - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && printf "%s\n" ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - ;; - "pixabay") - safe_cname="safesearch.pixabay.com" - safe_domains="pixabay.com" - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && printf "%s\n" ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - ;; - "yandex") - safe_cname="familysearch.yandex.ru" - safe_domains="ya.ru yandex.ru yandex.com yandex.com.tr yandex.ua yandex.by yandex.ee yandex.lt yandex.lv yandex.md yandex.uz yandex.tm yandex.tj yandex.az yandex.kz" - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && printf "%s\n" ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - ;; - "youtube") - if [ "${adb_safesearchmod}" = "0" ]; then - safe_cname="restrict.youtube.com" - else - safe_cname="restrictmoderate.youtube.com" - fi - safe_domains="www.youtube.com m.youtube.com youtubei.googleapis.com youtube.googleapis.com www.youtube-nocookie.com" - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && printf "%s\n" ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - ;; - esac - if [ "${out_rc}" = "0" ] && [ -s "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" ]; then - : >"${adb_tmpdir}/tmp.safesearch.${src_name}" - [ "${adb_dns}" = "named" ] && array="${safe_cname}" || array="${safe_ips}" - for item in ${array}; do - if ! eval "${adb_dnssafesearch}" "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" >>"${adb_tmpdir}/tmp.safesearch.${src_name}"; then - rm -f "${adb_tmpdir}/tmp.safesearch.${src_name}" - break - fi - done - out_rc="${?}" - rm -f "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" + esac + ;; + "safesearch") + file_name="${adb_tmpdir}/tmp.safesearch.${src_name}" + case "${adb_dns}" in + "named" | "kresd" | "smartdns") + use_cname="1" + ;; + esac + case "${src_name}" in + "google") + safe_url="https://www.google.com/supported_domains" + safe_cname="forcesafesearch.google.com" + + # refresh if cache is missing or older than 30 days + # + if [ -s "${adb_backupdir}/safesearch.${src_name}.gz" ] && + [ -z "$("${adb_findcmd}" "${adb_backupdir}/safesearch.${src_name}.gz" -mtime +30 2>>"${adb_errorlog}")" ]; then + "${adb_zcatcmd}" "${adb_backupdir}/safesearch.${src_name}.gz" >"${adb_tmpdir}/tmp.load.safesearch.${src_name}" + else + "${adb_fetchcmd}" ${adb_fetchparm} "${adb_tmpdir}/tmp.load.safesearch.${src_name}" "${safe_url}" 2>>"${adb_errorlog}" + if [ -s "${adb_tmpdir}/tmp.load.safesearch.${src_name}" ]; then + "${adb_gzipcmd}" -cf "${adb_tmpdir}/tmp.load.safesearch.${src_name}" >"${adb_backupdir}/safesearch.${src_name}.gz" + elif [ -s "${adb_backupdir}/safesearch.${src_name}.gz" ]; then + "${adb_zcatcmd}" "${adb_backupdir}/safesearch.${src_name}.gz" >"${adb_tmpdir}/tmp.load.safesearch.${src_name}" + fi fi ;; - "backup") - ( - gzip -cf "${src_tmpfile}" >"${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" - out_rc="${?}" - ) & + "bing") + safe_cname="strict.bing.com" + safe_domains="www.bing.com" ;; - "restore") - if [ -n "${src_name}" ] && [ -s "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" ]; then - zcat "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" >"${src_tmpfile}" - out_rc="${?}" - elif [ -z "${src_name}" ]; then - cnt="1" - for file in "${adb_backupdir}/${adb_dnsprefix}".*.gz; do - if [ -r "${file}" ]; then - name="${file##*/}" - name="${name%.*}" - zcat "${file}" >"${adb_tmpfile}.${name}" & - hold="$((cnt % adb_cores))" - if [ "${hold}" = "0" ]; then - wait - fi - cnt="$((cnt + 1))" - fi - done - wait - out_rc="${?}" + "brave") + safe_cname="forcesafe.search.brave.com" + safe_domains="search.brave.com" + ;; + "duckduckgo") + safe_cname="safe.duckduckgo.com" + safe_domains="duckduckgo.com" + ;; + "pixabay") + safe_cname="safesearch.pixabay.com" + safe_domains="pixabay.com" + ;; + "yandex") + safe_cname="familysearch.yandex.ru" + safe_domains="ya.ru yandex.ru yandex.com yandex.com.tr yandex.ua yandex.by yandex.ee yandex.lt yandex.lv yandex.md yandex.uz yandex.tm yandex.tj yandex.az yandex.kz" + ;; + "youtube") + safe_cname="restrict.youtube.com" + safe_domains="www.youtube.com m.youtube.com youtubei.googleapis.com youtube.googleapis.com www.youtube-nocookie.com" + ;; + esac + if [ -n "${safe_domains}" ] && [ -n "${safe_cname}" ]; then + if [ "${use_cname}" = "0" ]; then + safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>>"${adb_errorlog}" | "${adb_awkcmd}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" + fi + if [ -n "${safe_ips}" ] || [ "${use_cname}" = "1" ]; then + printf '%s\n' ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" + [ "${use_cname}" = "1" ] && array="${safe_cname}" || array="${safe_ips}" + fi + fi + if [ -s "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" ]; then + : >"${file_name}" + for item in ${array}; do + if ! f_dnssafesearch "${item}" "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" >>"${file_name}"; then + : >"${file_name}" + break + fi + done + : >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" + out_rc="0" + fi + ;; + "prepare") + file_name="${src_tmpfile}" + if [ -s "${src_tmpload}" ]; then + if [ "${adb_tld}" = "1" ]; then + f_chkdom ${src_rset} <"${src_tmpload}" | + "${adb_awkcmd}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' | + "${adb_sortcmd}" ${adb_srtopts} -u >"${src_tmpfile}" 2>>"${adb_errorlog}" else - out_rc=4 + f_chkdom ${src_rset} <"${src_tmpload}" | + "${adb_sortcmd}" ${adb_srtopts} -u >"${src_tmpfile}" 2>>"${adb_errorlog}" fi - if [ "${adb_action}" != "start" ] && [ "${adb_action}" != "resume" ] && [ -n "${src_name}" ] && [ "${out_rc}" != "0" ]; then - adb_sources="${adb_sources/${src_name}}" + out_rc="${?}" + if [ "${out_rc}" = "0" ] && [ -s "${src_tmpfile}" ]; then + f_list backup + elif [ "${adb_action}" != "boot" ] && [ "${adb_action}" != "start" ]; then + f_log "info" "preparation of '${src_name}' failed, rc: ${src_rc}" + f_list restore + out_rc="${?}" + : >"${src_tmpfile}" fi - ;; - "remove") - [ "${adb_backup}" = "1" ] && rm "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" 2>/dev/null + else + f_log "info" "download of '${src_name}' failed, url: ${src_url}, rule: ${src_rset:-"-"}, categories: ${src_cat:-"-"}, rc: ${src_rc}" + if [ "${adb_action}" != "boot" ] && [ "${adb_action}" != "start" ]; then + f_list restore + out_rc="${?}" + fi + fi + ;; + "backup") + file_name="${src_tmpfile}" + "${adb_gzipcmd}" -cf "${src_tmpfile}" >"${adb_backupdir}/adb_list.${src_name}.gz" + out_rc="${?}" + ;; + "restore") + file_name="${src_tmpfile}" + if [ -n "${src_name}" ] && [ -s "${adb_backupdir}/adb_list.${src_name}.gz" ]; then + "${adb_zcatcmd}" "${adb_backupdir}/adb_list.${src_name}.gz" >"${src_tmpfile}" out_rc="${?}" - adb_sources="${adb_sources/${src_name}}" - ;; - "merge") - if [ "${adb_backup}" = "1" ]; then - for src_name in ${adb_sources}; do - ffiles="${ffiles} -a ! -name ${adb_dnsprefix}.${src_name}.gz" - done - if [ "${adb_safesearch}" = "1" ] && [ "${adb_dnssafesearch}" != "0" ]; then - ffiles="${ffiles} -a ! -name safesearch.google.gz" + elif [ -z "${src_name}" ]; then + for file in "${adb_backupdir}/adb_list."*.gz; do + if [ -r "${file}" ]; then + name="${file##*/}" + name="${name%.*}" + "${adb_zcatcmd}" "${file}" >"${adb_tmpfile}.${name}" + out_rc="${?}" + [ "${out_rc}" != "0" ] && break fi - find "${adb_backupdir}" ${ffiles} -print0 2>/dev/null | xargs -0 rm 2>/dev/null + done + else + out_rc="4" + fi + case "${adb_action}" in + "boot" | "start" | "restart" | "resume") ;; + + *) + if [ -n "${src_name}" ] && [ "${out_rc}" != "0" ]; then + adb_feed=" ${adb_feed} " + adb_feed="${adb_feed// ${src_name} / }" + adb_feed="${adb_feed# }" + adb_feed="${adb_feed% }" fi - unset src_name - "${adb_sort}" ${adb_srtopts} -mu "${adb_tmpfile}".* 2>/dev/null >"${adb_tmpdir}/${adb_dnsfile}" - out_rc="${?}" - rm -f "${adb_tmpfile}".* ;; - "final") - unset src_name - { [ -n "${adb_dnsheader}" ] && printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}"; } || : >"${adb_dnsdir}/${adb_dnsfile}" - [ -s "${adb_tmpdir}/tmp.add.iplist" ] && cat "${adb_tmpdir}/tmp.add.iplist" >>"${adb_dnsdir}/${adb_dnsfile}" - [ -s "${adb_tmpdir}/tmp.add.whitelist" ] && cat "${adb_tmpdir}/tmp.add.whitelist" >>"${adb_dnsdir}/${adb_dnsfile}" - for file in "${adb_tmpdir}/tmp.safesearch".*; do - [ -r "${file}" ] && cat "${file}" >>"${adb_dnsdir}/${adb_dnsfile}" - done - { [ "${adb_dnsdeny}" != "0" ] && eval "${adb_dnsdeny}" "${adb_tmpdir}/${adb_dnsfile}" >>"${adb_dnsdir}/${adb_dnsfile}"; } || mv "${adb_tmpdir}/${adb_dnsfile}" "${adb_dnsdir}/${adb_dnsfile}" + esac + ;; + "remove") + "${adb_rmcmd}" "${adb_backupdir}/adb_list.${src_name}.gz" 2>>"${adb_errorlog}" + out_rc="${?}" + adb_feed=" ${adb_feed} " + adb_feed="${adb_feed// ${src_name} / }" + adb_feed="${adb_feed# }" + adb_feed="${adb_feed% }" + ;; + "merge") + src_name="" + file_name="${adb_tmpdir}/${adb_dnsfile}" + + # remove stale backup files + # + files="" + for file in ${adb_feed}; do + files="${files} ! -name adb_list.${file}.gz" + done + if [ "${adb_safesearch}" = "1" ] && [ "${adb_dnssafesearch}" = "1" ]; then + files="${files} ! -name safesearch.google.gz" + fi + "${adb_findcmd}" "${adb_backupdir}" -maxdepth 1 -type f -name '*.gz' ${files} -print0 2>>"${adb_errorlog}" | "${adb_xargscmd}" -0r "${adb_rmcmd}" -f + + # merge files + # + for file in "${adb_tmpfile}".*; do + [ -e "${file}" ] || continue + "${adb_sortcmd}" ${adb_srtopts} -mu "${adb_tmpfile}".* 2>>"${adb_errorlog}" >"${file_name}" out_rc="${?}" - ;; + break + done + [ -z "${out_rc}" ] && { + : >"${file_name}" + out_rc="4" + } + "${adb_rmcmd}" -f "${adb_tmpfile}".* + ;; + "final") + src_name="" + file_name="${adb_finaldir}/${adb_dnsfile}" + { + [ -n "${adb_dnsheader}" ] && printf '%b' "${adb_dnsheader}" + [ -s "${adb_tmpdir}/tmp.add.allowlist" ] && "${adb_sortcmd}" ${adb_srtopts} -u "${adb_tmpdir}/tmp.add.allowlist" + [ "${adb_safesearch}" = "1" ] && "${adb_catcmd}" "${adb_tmpdir}/tmp.safesearch."* 2>>"${adb_errorlog}" + if [ "${adb_dnsdeny}" = "1" ]; then + f_dnsdeny "${adb_tmpdir}/${adb_dnsfile}" + else + "${adb_catcmd}" "${adb_tmpdir}/${adb_dnsfile}" + fi + } >"${file_name}" + if [ "${adb_dnsshift}" = "1" ] && [ ! -L "${adb_dnsdir}/${adb_dnsfile}" ]; then + "${adb_lncmd}" -fs "${file_name}" "${adb_dnsdir}/${adb_dnsfile}" + elif [ "${adb_dnsshift}" = "0" ] && [ -s "${adb_backupdir}/${adb_dnsfile}" ]; then + "${adb_rmcmd}" -f "${adb_backupdir}/${adb_dnsfile}" + fi + out_rc="0" + ;; esac - f_count "${mode}" "${src_name}" + f_count "${mode}" "${file_name}" out_rc="${out_rc:-"${in_rc}"}" + f_log "debug" "f_list ::: name: ${src_name:-"-"}, mode: ${mode}, cnt: ${adb_cnt}, in_rc: ${in_rc}, out_rc: ${out_rc}" return "${out_rc}" } @@ -913,188 +1322,404 @@ f_list() { # top level domain compression # f_tld() { - local cnt cnt_tld source="${1}" temp_tld="${1}.tld" + local cnt_tld cnt_rem source="${1}" temp_tld="${1}.tld" - if "${adb_awk}" '{if(NR==1){tld=$NF};while(getline){if(index($NF,tld".")==0){print tld;tld=$NF}}print tld}' "${source}" | - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' >"${temp_tld}"; then - mv -f "${temp_tld}" "${source}" - cnt_tld="$(wc -l 2>/dev/null <"${source}")" - else - rm -f "${temp_tld}" + # reverse domain, get unique tlds and unreverse them back to original form + # + if "${adb_awkcmd}" ' + function unreverse(dom, n, seg, out, i) { + n = split(dom, seg, ".") + out = seg[n] + for (i = n-1; i >= 1; i--) out = out "." seg[i] + return out + } + { + if (NR == 1) { parent = $NF } + else if (index($NF, parent ".") == 0) { print unreverse(parent); parent = $NF } + } + END { if (parent) print unreverse(parent) } + ' "${source}" >"${temp_tld}"; then + [ "${adb_debug}" = "1" ] && cnt_tld="$(f_count tld "${temp_tld}" "var")" + + # remove allowlisted (sub-) domains from tld list if allowlist is enabled and not empty + # + if [ -s "${adb_tmpdir}/tmp.rem.allowlist" ]; then + "${adb_awkcmd}" ' + NR==FNR { + del[$0] + next + } + { + dominated = 0 + n = split($0, seg, ".") + for (i = 1; i <= n; i++) { + parent = seg[i] + for (j = i + 1; j <= n; j++) parent = parent "." seg[j] + if (parent in del) { dominated = 1; break } + } + if (!dominated) print + } + ' "${adb_tmpdir}/tmp.rem.allowlist" "${temp_tld}" >"${source}" + "${adb_rmcmd}" -f "${temp_tld}" + [ "${adb_debug}" = "1" ] && cnt_rem="$(f_count tld "${source}" "var")" + else + "${adb_mvcmd}" -f "${temp_tld}" "${source}" + fi fi - f_log "debug" "f_tld ::: source: ${source}, cnt: ${adb_cnt:-"-"}, cnt_tld: ${cnt_tld:-"-"}" + + f_log "debug" "f_tld ::: name: -, cnt: ${adb_cnt:-"-"}, cnt_tld: ${cnt_tld:-"-"}, cnt_rem: ${cnt_rem:-"-"}" } -# suspend/resume adblock processing -# -f_switch() { - local status entry done="false" mode="${1}" +# suspend/resume adblock processing +# +f_switch() { + local status switch rc="0" mode="${1}" + + json_init + json_load_file "${adb_rtfile}" >/dev/null 2>&1 + json_get_var status "adblock_status" + f_env + + # suspend adblock processing + # + if [ "${status}" = "enabled" ] && [ "${mode}" = "suspend" ]; then + + # suspend via external DNS bridge + # + if { [ -n "${adb_bridgednsv4}" ] || [ -n "${adb_bridgednsv6}" ]; } && [ "${adb_nftbridge}" = "1" ]; then + if "${adb_nftcmd}" list chain inet adblock dns-bridge >/dev/null 2>&1; then + if [ -n "${adb_bridgednsv4}" ]; then + "${adb_nftcmd}" add rule inet adblock dns-bridge meta nfproto ipv4 meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_bridgednsv4}:53 2>>"${adb_errorlog}" + rc="${?}" + fi + if [ -n "${adb_bridgednsv6}" ]; then + "${adb_nftcmd}" add rule inet adblock dns-bridge meta nfproto ipv6 meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_bridgednsv6}]:53 2>>"${adb_errorlog}" + rc="$((rc + $?))" + fi + [ "${rc}" = "0" ] && switch="nft" + fi + + # suspend via local DNS + # + else + if [ "${adb_dnsshift}" = "0" ] && [ -f "${adb_finaldir}/${adb_dnsfile}" ]; then + "${adb_mvcmd}" -f "${adb_finaldir}/${adb_dnsfile}" "${adb_backupdir}/${adb_dnsfile}" + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" + switch="dns" + elif [ "${adb_dnsshift}" = "1" ] && [ -L "${adb_dnsdir}/${adb_dnsfile}" ]; then + "${adb_rmcmd}" -f "${adb_dnsdir}/${adb_dnsfile}" + printf '%b' "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" + switch="dns" + fi + fi + + # resume adblock processing + # + elif [ "${status}" = "paused" ] && [ "${mode}" = "resume" ]; then - json_init - json_load_file "${adb_rtfile}" >/dev/null 2>&1 - json_select "data" >/dev/null 2>&1 - json_get_var status "adblock_status" - if [ "${mode}" = "suspend" ] && [ "${status}" = "enabled" ]; then - f_env - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" - if [ "${adb_jail}" = "1" ] && [ "${adb_jaildir}" = "${adb_dnsdir}" ]; then - printf "%b" "${adb_dnsheader}" >"${adb_jaildir}/${adb_dnsjail}" - elif [ -f "${adb_dnsdir}/${adb_dnsjail}" ]; then - rm -f "${adb_dnsdir}/${adb_dnsjail}" + # resume via external DNS bridge + # + if [ "${adb_nftbridge}" = "1" ] && "${adb_nftcmd}" list chain inet adblock dns-bridge >/dev/null 2>&1; then + if "${adb_nftcmd}" flush chain inet adblock dns-bridge 2>>"${adb_errorlog}"; then + switch="nft" + fi + f_count "final" "${adb_finaldir}/${adb_dnsfile}" + + # resume via local DNS + # + else + if [ "${adb_dnsshift}" = "0" ] && [ -f "${adb_backupdir}/${adb_dnsfile}" ]; then + "${adb_mvcmd}" -f "${adb_backupdir}/${adb_dnsfile}" "${adb_finaldir}/${adb_dnsfile}" + f_count "final" "${adb_finaldir}/${adb_dnsfile}" + switch="dns" + elif [ "${adb_dnsshift}" = "1" ] && [ ! -L "${adb_dnsdir}/${adb_dnsfile}" ]; then + "${adb_lncmd}" -fs "${adb_finaldir}/${adb_dnsfile}" "${adb_dnsdir}/${adb_dnsfile}" + f_count "final" "${adb_finaldir}/${adb_dnsfile}" + switch="dns" + fi fi - f_count - done="true" - elif [ "${mode}" = "resume" ] && [ "${status}" = "paused" ]; then - f_env - f_main - done="true" fi - if [ "${done}" = "true" ]; then - [ "${mode}" = "suspend" ] && f_dnsup + + # update runtime information and log action + # + if [ "${switch}" = "nft" ]; then + f_jsnup "${mode}" + f_log "info" "${mode} adblock service via external DNS bridge" + elif [ "${switch}" = "dns" ]; then + f_dnsup f_jsnup "${mode}" - f_log "info" "${mode} adblock processing" + f_log "info" "${mode} adblock service via local DNS" + else + f_count "final" "${adb_finaldir}/${adb_dnsfile}" + f_jsnup "${status}" fi f_rmtemp } -# query blocklist for certain (sub-)domains +# search blocklist for certain (sub-)domains # -f_query() { - local search result prefix suffix field query_start query_end query_timeout=30 domain="${1}" tld="${1#*.}" +f_search() { + local rc search res result tmp_result prefix suffix field search_start search_end search_timeout=30 domain="${1}" tld="${1#*.}" - if [ -z "${domain}" ] || [ "${domain}" = "${tld}" ]; then - printf "%s\n" "::: invalid input, please submit a single (sub-)domain :::" - else - case "${adb_dns}" in - "dnsmasq") - prefix=".*[\\/\\.]" - suffix="(\\/)" - field="2" - ;; - "unbound") - prefix=".*[\"\\.]" - suffix="(always_nxdomain)" - field="3" - ;; - "named") - prefix="[^\\*].*[\\.]" - suffix="( \\.)" - field="1" - ;; - "kresd") - prefix="[^\\*].*[\\.]" - suffix="( \\.)" - field="1" - ;; - "raw") - prefix=".*[\\.]" - suffix="" - field="1" - ;; - esac - query_start="$(date "+%s")" - while [ "${domain}" != "${tld}" ]; do - search="${domain//[+*~%\$&\"\']/}" - search="${search//./\\.}" - result="$("${adb_awk}" -F '/|\"|\t| ' "/^(${search}|${prefix}+${search}.*${suffix})$/{i++;if(i<=9){printf \" + %s\n\",\$${field}}else if(i==10){printf \" + %s\n\",\"[...]\";exit}}" "${adb_dnsdir}/${adb_dnsfile}")" - printf "%s\n%s\n%s\n" ":::" "::: domain '${domain}' in active blocklist" ":::" - printf "%s\n\n" "${result:-" - no match"}" - domain="${tld}" - tld="${domain#*.}" - done - if [ "${adb_backup}" = "1" ] && [ -d "${adb_backupdir}" ]; then - search="${1//[+*~%\$&\"\']/}" - search="${search//./\\.}" - printf "%s\n%s\n%s\n" ":::" "::: domain '${1}' in backups and black-/whitelist" ":::" - for file in "${adb_backupdir}/${adb_dnsprefix}".*.gz "${adb_blacklist}" "${adb_whitelist}"; do - suffix="${file##*.}" - if [ "${suffix}" = "gz" ]; then - zcat "${file}" 2>/dev/null | - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' | "${adb_awk}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" + # prepare result file + # + tmp_result="${adb_rundir}/adblock.search.tmp" + result="${adb_rundir}/adblock.search" + + # input validation + # + case "${domain}" in + "" | *[!a-zA-Z0-9.-]* | -* | *- | *..* | *.) + printf '%s\n' "::: invalid input, please submit a single (sub-)domain :::" + printf '%s\n' "::: invalid input, please submit a single (sub-)domain :::" >"${result}" + return + ;; + esac + + # length validation for domain part, max. 253 characters according to RFC 1035 + # + case "${#domain}" in + [0-9] | [1-9][0-9] | 1[0-9][0-9] | 2[0-4][0-9] | 25[0-3]) ;; + + *) + printf '%s\n' "::: invalid input, domain exceeds 253 characters :::" + printf '%s\n' "::: invalid input, domain exceeds 253 characters :::" >"${result}" + return + ;; + esac + + # search blocklist + # + case "${adb_dns}" in + "dnsmasq") + prefix='local=.*[\/\.]' + suffix='\/' + field="2" + ;; + "unbound") + prefix='local-zone: .*["\.]' + suffix='" always_nxdomain' + field="3" + ;; + "named") + prefix="" + suffix=' CNAME \.' + field="1" + ;; + "kresd") + prefix="" + suffix=' CNAME \.' + field="1" + ;; + "smartdns") + prefix='address .*.*[\/\.]' + suffix='\/#' + field="3" + ;; + "raw") + prefix="" + suffix="" + field="1" + ;; + esac + + # initialize tmp_result and start search + # + : >"${tmp_result}" + read -r search_start _ <"/proc/uptime" + search_start="${search_start%.*}" + + # search recursively for domain and its parent domains until tld is reached + # + while :; do + search="${domain//./\\.}" + res="$("${adb_awkcmd}" -F '/|\"|\t| ' "/^(${prefix}${search}${suffix})$/{i++;if(i<=9){printf \" + %s\n\",\$${field}}else if(i==10){printf \" + %s\n\",\"[...]\";exit}}" "${adb_finaldir}/${adb_dnsfile}")" + printf '%s\n%s\n%s\n' ":::" "::: domain '${domain}' in active blocklist" ":::" >>"${tmp_result}" + printf '%s\n\n' "${res:-" - no match"}" >>"${tmp_result}" + [ "${domain}" = "${tld}" ] && break + domain="${tld}" + tld="${domain#*.}" + done + + # search exactly for domain in backup files and local block-/allowlist + # + if [ -d "${adb_backupdir}" ]; then + search="${1//./\\.}" + printf '%s\n%s\n%s\n' ":::" "::: domain '${1}' in backups and in local block-/allowlist" ":::" >>"${tmp_result}" + for file in "${adb_backupdir}/adb_list".*.gz "${adb_blocklist}" "${adb_allowlist}"; do + suffix="${file##*.}" + if [ "${suffix}" = "gz" ]; then + if [ "${adb_tld}" = "1" ]; then + "${adb_zcatcmd}" "${file}" 2>>"${adb_errorlog}" | + "${adb_awkcmd}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' | + "${adb_awkcmd}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" >>"${tmp_result}" else - "${adb_awk}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" "${file}" + "${adb_zcatcmd}" "${file}" 2>>"${adb_errorlog}" | + "${adb_awkcmd}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" >>"${tmp_result}" fi - if [ "${?}" = "0" ]; then - result="true" - query_end="$(date "+%s")" - if [ "$((query_end - query_start))" -gt "${query_timeout}" ]; then - printf "%s\n\n" " - [...]" - break - fi + rc="${?}" + else + "${adb_awkcmd}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" "${file}" >>"${tmp_result}" + rc="${?}" + fi + if [ "${rc}" = "0" ]; then + res="true" + read -r search_end _ <"/proc/uptime" + search_end="${search_end%.*}" + if [ "$((search_end - search_start))" -gt "${search_timeout}" ]; then + printf '%s\n\n' " - [...]" >>"${tmp_result}" + break fi - done - [ "${result}" != "true" ] && printf "%s\n\n" " - no match" - fi + fi + done + [ "${res}" != "true" ] && printf '%s\n\n' " - no match" >>"${tmp_result}" fi + "${adb_mvcmd}" -f "${tmp_result}" "${result}" + "${adb_catcmd}" "${result}" 2>>"${adb_errorlog}" } # update runtime information # f_jsnup() { - local entry sources runtime utils bg_pid status="${1:-"enabled"}" + local s_shift s_custom s_unfiltered s_filtered s_remote s_bridge s_force s_flush s_tld s_search s_report s_mail + local s_jail s_debug pid pids object feeds end_time runtime dns dns_ver free_mem custom_feed="0" status="${1:-"enabled"}" + local vm_mem dns_mem="0" duration jail="0" nft_unfiltered="0" nft_filtered="0" nft_remote="0" nft_bridge="0" nft_force="0" - adb_memory="$("${adb_awk}" '/^MemTotal|^MemFree|^MemAvailable/{ORS="/"; print int($2/1000)}' "/proc/meminfo" 2>/dev/null | - "${adb_awk}" '{print substr($0,1,length($0)-1)}')" - - case "${status}" in - "enabled" | "error") - adb_endtime="$(date "+%s")" - if [ "$(((adb_endtime - adb_starttime) / 60))" -lt 60 ]; then - runtime="${adb_action}, $(((adb_endtime - adb_starttime) / 60))m $(((adb_endtime - adb_starttime) % 60))s, ${adb_memory:-0}, $(date -Iseconds)" - else - runtime="${adb_action}, n/a, ${adb_memory:-0}, $(date -Iseconds)" - fi - [ "${status}" = "error" ] && adb_cnt="0" + # get DNS memory usage and version + # + if adb_dnspid="$("${adb_ubuscmd}" -S call service list 2>>"${adb_errorlog}" | + "${adb_jsoncmd}" -l1 -e "@[\"${adb_dns}\"].instances.*.pid")" && [ -n "${adb_dnspid}" ]; then + pids="$("${adb_pgrepcmd}" -P "${adb_dnspid}" 2>>"${adb_errorlog}")" + for pid in ${adb_dnspid} ${pids}; do + vm_mem="$("${adb_awkcmd}" '/^VmRSS/{printf "%d", $2}' "/proc/${pid}/status" 2>>"${adb_errorlog}")" + dns_mem="$((dns_mem + ${vm_mem:-0}))" + done + case "${adb_dns}" in + "kresd") + dns="knot-resolver" ;; - "suspend") - status="paused" + "named") + dns="bind-server" ;; - "resume") - status="" + "unbound") + dns="unbound-daemon" + ;; + "dnsmasq") + dns='dnsmasq", "dnsmasq-full", "dnsmasq-dhcpv6' ;; + esac + dns_ver="$(printf '%s' "${adb_packages}" | "${adb_jsoncmd}" -ql1 -e "@.packages[\"${dns:-"${adb_dns}"}\"]")" + dns_mem="$("${adb_awkcmd}" -v mem="${dns_mem}" 'BEGIN{printf "%.2f", mem/1024}' 2>>"${adb_errorlog}")" + fi + free_mem="$("${adb_awkcmd}" '/^MemAvailable/{printf "%.2f", $2/1024}' "/proc/meminfo" 2>>"${adb_errorlog}")" + + # check for custom feed and nft rules + # + [ -s "${adb_customfeedfile}" ] && custom_feed="1" + if [ "${adb_nftforce}" = "1" ] && [ -n "${adb_nftdevforce}" ] && [ -n "${adb_nftportforce}" ]; then + nft_force="1" + fi + if [ "${adb_nftallow}" = "1" ] && + { [ -n "${adb_nftmacallow}" ] || [ -n "${adb_nftdevallow}" ]; } && + { [ -n "${adb_allowdnsv4}" ] || [ -n "${adb_allowdnsv6}" ]; }; then + nft_unfiltered="1" + fi + if [ "${adb_nftblock}" = "1" ] && + { [ -n "${adb_nftmacblock}" ] || [ -n "${adb_nftdevblock}" ]; } && + { [ -n "${adb_blockdnsv4}" ] || [ -n "${adb_blockdnsv6}" ]; }; then + nft_filtered="1" + fi + if [ "${adb_nftremote}" = "1" ] && [ -n "${adb_nftmacremote}" ] && + { [ -n "${adb_remotednsv4}" ] || [ -n "${adb_remotednsv6}" ]; }; then + nft_remote="1" + fi + if [ "${adb_nftbridge}" = "1" ] && + { [ -n "${adb_bridgednsv4}" ] || [ -n "${adb_bridgednsv6}" ]; }; then + nft_bridge="1" + fi + + # update runtime information based on status + # + case "${status}" in + "enabled") + if [ -n "${adb_starttime}" ]; then + read -r end_time _ <"/proc/uptime" + end_time="${end_time%.*}" + duration="$(((end_time - adb_starttime) / 60))m $(((end_time - adb_starttime) % 60))s" + fi + runtime="mode: ${adb_action}, date / time: $(date "+%d/%m/%Y %H:%M:%S"), duration: ${duration:-"-"}, memory: ${free_mem:-0} MB available" + ;; + "resume") + status="enabled" + ;; + "suspend") + adb_cnt="0" + status="paused" + ;; + *) + adb_cnt="0" + ;; esac + + # update runtime file and send mail if enabled + # json_init if json_load_file "${adb_rtfile}" >/dev/null 2>&1; then - utils="download: $(readlink -fn "${adb_fetchutil}"), sort: $(readlink -fn "${adb_sort}"), awk: $(readlink -fn "${adb_awk}")" - [ -z "${adb_cnt}" ] && { json_get_var adb_cnt "blocked_domains"; adb_cnt="${adb_cnt%% *}"; } + [ -z "${adb_cnt}" ] && json_get_var adb_cnt "blocked_domains" [ -z "${runtime}" ] && json_get_var runtime "last_run" - fi - if [ "${adb_jail}" = "1" ] && [ "${adb_jaildir}" = "${adb_dnsdir}" ]; then - adb_cnt="0" - sources="restrictive_jail" - else - sources="$(printf "%s\n" ${adb_sources} | "${adb_sort}" | "${adb_awk}" '{ORS=" ";print $0}')" + if [ "${status}" = "enabled" ]; then + if [ "${adb_jail}" = "1" ] && [ -n "${adb_dnsstop}" ]; then + jail="1" + adb_cnt="0" + feeds="restrictive jail (allowlist-only)" + else + feeds="$(printf '%s\n' ${adb_feed// /, } | "${adb_sortcmd}" | "${adb_xargscmd}")" + fi + fi fi - : >"${adb_rtfile}" + # map flag values to status characters + # + case "${adb_dnsshift}" in "1") s_shift="✔" ;; *) s_shift="✘" ;; esac + case "${custom_feed}" in "1") s_custom="✔" ;; *) s_custom="✘" ;; esac + case "${nft_unfiltered}" in "1") s_unfiltered="✔" ;; *) s_unfiltered="✘" ;; esac + case "${nft_filtered}" in "1") s_filtered="✔" ;; *) s_filtered="✘" ;; esac + case "${nft_remote}" in "1") s_remote="✔" ;; *) s_remote="✘" ;; esac + case "${nft_bridge}" in "1") s_bridge="✔" ;; *) s_bridge="✘" ;; esac + case "${nft_force}" in "1") s_force="✔" ;; *) s_force="✘" ;; esac + case "${adb_dnsflush}" in "1") s_flush="✔" ;; *) s_flush="✘" ;; esac + case "${adb_tld}" in "1") s_tld="✔" ;; *) s_tld="✘" ;; esac + case "${adb_safesearch}" in "1") s_search="✔" ;; *) s_search="✘" ;; esac + case "${adb_report}" in "1") s_report="✔" ;; *) s_report="✘" ;; esac + case "${adb_mail}" in "1") s_mail="✔" ;; *) s_mail="✘" ;; esac + case "${jail}" in "1") s_jail="✔" ;; *) s_jail="✘" ;; esac + case "${adb_debug}" in "1") s_debug="✔" ;; *) s_debug="✘" ;; esac + + # update runtime file + # + printf '%s\n' "{}" >"${adb_rtfile}" json_init json_load_file "${adb_rtfile}" >/dev/null 2>&1 - json_init - json_add_string "adblock_status" "${status:-"enabled"}" - json_add_string "adblock_version" "${adb_ver}" - json_add_string "blocked_domains" "${adb_cnt:-0}" - json_add_array "active_sources" - for entry in ${sources}; do - json_add_object - json_add_string "source" "${entry}" - json_close_object + json_add_string "adblock_status" "${status}" + json_add_string "frontend_ver" "${adb_fver}" + json_add_string "backend_ver" "${adb_bver}" + json_add_string "blocked_domains" "${adb_cnt:-"0"}" + json_add_array "active_feeds" + for object in ${feeds:-"-"}; do + json_add_string "${object}" "${object}" done json_close_array - json_add_string "dns_backend" "${adb_dns:-"-"} (${adb_dnscachecmd##*/}), ${adb_dnsdir:-"-"}" - json_add_string "run_utils" "${utils:-"-"}" + json_add_string "dns_backend" "${adb_dns:-"-"} (${dns_ver:-"-"}), ${adb_finaldir:-"-"}, ${dns_mem:-"0"} MB" json_add_string "run_ifaces" "trigger: ${adb_trigger:-"-"}, report: ${adb_repiface:-"-"}" - json_add_string "run_directories" "base: ${adb_tmpbase}, backup: ${adb_backupdir}, report: ${adb_reportdir}, jail: ${adb_jaildir}" - json_add_string "run_flags" "backup: $(f_char ${adb_backup}), flush: $(f_char ${adb_dnsflush}), force: $(f_char ${adb_forcedns}), search: $(f_char ${adb_safesearch}), report: $(f_char ${adb_report}), mail: $(f_char ${adb_mail}), jail: $(f_char ${adb_jail})" + json_add_string "run_information" "base: ${adb_basedir}, dns: ${adb_dnsdir}, backup: ${adb_backupdir}, report: ${adb_reportdir}, error: ${adb_errorlog}" + json_add_string "run_flags" "shift: ${s_shift}, custom feed: ${s_custom}, ext. DNS (std/prot/remote/bridge): ${s_unfiltered}/${s_filtered}/${s_remote}/${s_bridge}, force: ${s_force}, flush: ${s_flush}, tld: ${s_tld}, search: ${s_search}, report: ${s_report}, mail: ${s_mail}, jail: ${s_jail}, debug: ${s_debug}" json_add_string "last_run" "${runtime:-"-"}" - json_add_string "system" "${adb_sysver}" + json_add_string "system_info" "cores: ${adb_cores}, fetch: ${adb_fetchcmd##*/}, ${adb_sysver}" json_dump >"${adb_rtfile}" - if [ "${adb_mail}" = "1" ] && [ -x "${adb_mailservice}" ] && - { [ "${status}" = "error" ] || { [ "${status}" = "enabled" ] && [ "${adb_cnt}" -le "${adb_mailcnt}" ]; }; }; then - ("${adb_mailservice}" "${adb_ver}" >/dev/null 2>&1) & - bg_pid="${!}" + if [ "${adb_mail}" = "1" ] && [ -x "${adb_mailservice}" ] && [ "${status}" = "enabled" ] && [ "${adb_action}" != "resume" ]; then + "${adb_mailservice}" >/dev/null 2>&1 fi - f_log "debug" "f_jsnup ::: status: ${status:-"-"}, cnt: ${adb_cnt}, mail: ${adb_mail}, mail_service: ${adb_mailservice}, mail_cnt: ${adb_mailcnt}, mail_pid: ${bg_pid:-"-"}" } # write to syslog @@ -1103,10 +1728,13 @@ f_log() { local class="${1}" log_msg="${2}" if [ -n "${log_msg}" ] && { [ "${class}" != "debug" ] || [ "${adb_debug}" = "1" ]; }; then - [ -x "${adb_loggercmd}" ] && "${adb_loggercmd}" -p "${class}" -t "adblock-${adb_ver}[${$}]" "${log_msg}" || \ - printf "%s %s %s\n" "${class}" "adblock-${adb_ver}[${$}]" "${log_msg}" - if [ "${class}" = "err" ]; then - f_rmdns + if [ -x "${adb_loggercmd}" ]; then + "${adb_loggercmd}" -p "${class}" -t "adblock-${adb_bver}[${$}]" "${log_msg::512}" + else + printf '%s %s %s\n' "${class}" "adblock-${adb_bver}[${$}]" "${log_msg::512}" + fi + if [ "${class}" = "err" ] || [ "${class}" = "emerg" ]; then + [ "${adb_action}" != "mail" ] && f_rmdns f_jsnup "error" exit 1 fi @@ -1116,19 +1744,33 @@ f_log() { # main function for blocklist processing # f_main() { - local src_tmpload src_tmpfile src_name src_rset src_url src_log src_arc src_cat src_item src_list src_entries src_suffix src_rc entry cnt + local src_name src_domain src_rset src_url src_cat src_item src_list src_entries src_suffix src_rc entry cnt + local src_tmpcat src_tmparchive src_tmpload src_tmpfile seen_domains feed_restore map_domain - f_log "debug" "f_main ::: memory: ${adb_memory:-0}, cores: ${adb_cores}, safe_search: ${adb_safesearch}, force_dns: ${adb_forcedns}, awk: ${adb_awk}" - - # white- and blacklist preparation + # allow- and blocklist preparation # for entry in ${adb_locallist}; do - (f_list "${entry}" "${entry}") & + f_list "${entry}" "${entry}" done - if [ "${adb_dns}" != "raw" ] && [ "${adb_jail}" = "1" ] && [ "${adb_jaildir}" = "${adb_dnsdir}" ]; then - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" - chown "${adb_dnsuser}" "${adb_jaildir}/${adb_dnsjail}" 2>/dev/null + # jail mode preparation + # + if [ "${adb_jail}" = "1" ] && [ -n "${adb_dnsstop}" ]; then + if [ -s "${adb_tmpdir}/${adb_dnsfile}" ]; then + "${adb_mvcmd}" -f "${adb_tmpdir}/${adb_dnsfile}" "${adb_finaldir}/${adb_dnsfile}" + else + f_log "info" "jail mode active without allowlist, blocking all queries" + { + printf '%b' "${adb_dnsheader}" + printf '%b\n' "${adb_dnsstop}" + } >"${adb_finaldir}/${adb_dnsfile}" + fi + chown "${adb_dnsuser}" "${adb_finaldir}/${adb_dnsfile}" 2>>"${adb_errorlog}" + if [ "${adb_dnsshift}" = "1" ] && [ ! -L "${adb_dnsdir}/${adb_dnsfile}" ]; then + "${adb_lncmd}" -fs "${adb_finaldir}/${adb_dnsfile}" "${adb_dnsdir}/${adb_dnsfile}" + elif [ "${adb_dnsshift}" = "0" ] && [ -s "${adb_backupdir}/${adb_dnsfile}" ]; then + "${adb_rmcmd}" -f "${adb_backupdir}/${adb_dnsfile}" + fi if f_dnsup; then if [ "${adb_action}" != "resume" ]; then f_jsnup "enabled" @@ -1139,160 +1781,233 @@ f_main() { fi f_rmtemp return - elif [ -f "${adb_dnsdir}/${adb_dnsjail}" ]; then - rm -f "${adb_dnsdir}/${adb_dnsjail}" - f_dnsup fi # safe search preparation # - if [ "${adb_safesearch}" = "1" ] && [ "${adb_dnssafesearch}" != "0" ]; then - [ -z "${adb_safesearchlist}" ] && adb_safesearchlist="google bing duckduckgo pixabay yandex youtube" + if [ "${adb_safesearch}" = "1" ] && [ "${adb_dnssafesearch}" = "1" ]; then + [ -z "${adb_safesearchlist}" ] && adb_safesearchlist="google bing brave duckduckgo pixabay yandex youtube" cnt="1" for entry in ${adb_safesearchlist}; do - (f_list safesearch "${entry}") & - hold="$((cnt % adb_cores))" - [ "${hold}" = "0" ] && wait + ( + f_list safesearch "${entry}" + ) & + [ "${cnt}" -gt "${adb_cores}" ] && wait -n cnt="$((cnt + 1))" done + wait + fi + + # add map service domain to allowlist if map is enabled + # + if [ "${adb_map}" = "1" ] && [ "${adb_dnsallow}" = "1" ] && [ -n "${adb_geourl}" ]; then + map_domain="${adb_geourl#*://}" + map_domain="${map_domain%%/*}" + if [ -n "${map_domain}" ]; then + printf '%s\n' "${map_domain}" | f_dnsallow >>"${adb_tmpdir}/tmp.add.allowlist" + printf '%s\n' "${map_domain}" >>"${adb_tmpdir}/tmp.rem.allowlist" + fi fi - wait # main loop # cnt="1" - for src_name in ${adb_sources}; do + seen_domains="${map_domain}" + + # determine if feed restore should be attempted based on action + # + case "${adb_action}" in + "boot" | "start" | "restart" | "resume") + feed_restore="1" + ;; + *) + feed_restore="0" + ;; + esac + + # loop through feeds defined in configuration and process them + # + for src_name in ${adb_feed}; do + + # check if feed is defined in configuration, if not remove it from feed list and continue with next one + # if ! json_select "${src_name}" >/dev/null 2>&1; then - adb_sources="${adb_sources/${src_name}/}" + adb_feed=" ${adb_feed} " + adb_feed="${adb_feed// ${src_name} / }" + adb_feed="${adb_feed# }" + adb_feed="${adb_feed% }" continue fi + + # get feed information + # json_get_var src_url "url" >/dev/null 2>&1 json_get_var src_rset "rule" >/dev/null 2>&1 json_select .. src_tmpcat="${adb_tmpload}.${src_name}.cat" src_tmpload="${adb_tmpload}.${src_name}.load" - src_tmpsort="${adb_tmpload}.${src_name}.sort" + src_tmparchive="${adb_tmpload}.${src_name}.archive" src_tmpfile="${adb_tmpfile}.${src_name}" src_rc=4 # basic pre-checks # - if [ -z "${src_url}" ] || [ -z "${src_rset}" ]; then + if [ -z "${src_url}" ] || [ -z "${src_rset}" ] || + [ "${src_rset%% *}" != "feed" ]; then f_list remove continue fi - # backup mode + # add domains of active feed URLs to the allowlist # - if [ "${adb_backup}" = "1" ] && { [ "${adb_action}" = "start" ] || [ "${adb_action}" = "resume" ]; }; then - if f_list restore && [ -s "${src_tmpfile}" ]; then - continue - fi + src_domain="${src_url#*://}" + src_domain="${src_domain%%/*}" + if [ -n "${src_domain}" ] && [ "${adb_dnsallow}" = "1" ]; then + case " ${seen_domains} " in + *" ${src_domain} "*) ;; + + *) + seen_domains="${seen_domains} ${src_domain}" + printf '%s\n' "${src_domain}" | f_dnsallow >>"${adb_tmpdir}/tmp.add.allowlist" + printf '%s\n' "${src_domain}" >>"${adb_tmpdir}/tmp.rem.allowlist" + ;; + esac fi # download queue processing # - unset src_cat src_entries - if [ "${src_name}" = "utcapitole" ] && [ -n "${adb_utc_sources}" ]; then - src_cat="${adb_utc_sources}" - if [ -n "${src_cat}" ]; then - ( - src_arc="${adb_tmpdir}/${src_url##*/}" - src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_arc}" "${src_url}" 2>&1)" - src_rc="${?}" - if [ "${src_rc}" = "0" ] && [ -s "${src_arc}" ]; then - src_suffix="$(eval printf "%s" \"\$\{adb_src_suffix_${src_name}:-\"domains\"\}\")" - src_list="$(tar -tzf "${src_arc}" 2>/dev/null)" - for src_item in ${src_cat}; do - src_entries="${src_entries} $(printf "%s" "${src_list}" | grep -E "${src_item}/${src_suffix}$")" - done - if [ -n "${src_entries}" ]; then - tar -xOzf "${src_arc}" ${src_entries} 2>/dev/null >"${src_tmpload}" - src_rc="${?}" - fi - : >"${src_arc}" - else - src_log="$(printf "%s" "${src_log}" | "${adb_awk}" '{ORS=" ";print $0}')" - f_log "info" "download of '${src_name}' failed, url: ${src_url}, rule: ${src_rset:-"-"}, categories: ${src_cat:-"-"}, rc: ${src_rc}, log: ${src_log:-"-"}" - fi - if [ "${src_rc}" = "0" ] && [ -s "${src_tmpload}" ]; then - if [ -s "${adb_tmpdir}/tmp.rem.whitelist" ]; then - "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | - grep -Evf "${adb_tmpdir}/tmp.rem.whitelist" | "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' >"${src_tmpsort}" - else - "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' >"${src_tmpsort}" - fi - : >"${src_tmpload}" - "${adb_sort}" ${adb_srtopts} -u "${src_tmpsort}" 2>/dev/null >"${src_tmpfile}" - src_rc="${?}" - : >"${src_tmpsort}" - if [ "${src_rc}" = "0" ] && [ -s "${src_tmpfile}" ]; then - f_list download - [ "${adb_backup}" = "1" ] && f_list backup - elif [ "${adb_backup}" = "1" ] && [ "${adb_action}" != "start" ]; then - f_log "info" "archive preparation of '${src_name}' failed, categories: ${src_cat:-"-"}, entries: ${src_entries}, rc: ${src_rc}" - f_list restore - rm -f "${src_tmpfile}" - fi - elif [ "${adb_backup}" = "1" ] && [ "${adb_action}" != "start" ]; then - f_log "info" "archive extraction of '${src_name}' failed, categories: ${src_cat:-"-"}, entries: ${src_entries}, rc: ${src_rc}" - f_list restore - fi - ) & + src_cat="" + src_entries="" + + # category handling for feeds with fixed categories + # + case "${src_name}" in + "1hosts") + src_cat="${adb_hst_feed}" + if [ -z "${src_cat}" ]; then + f_log "info" "feed '${src_name}' requires category configuration, skipping" + continue fi - else - if [ "${src_name}" = "energized" ] && [ -n "${adb_eng_sources}" ]; then - src_cat="${adb_eng_sources}" - elif [ "${src_name}" = "stevenblack" ] && [ -n "${adb_stb_sources}" ]; then - src_cat="${adb_stb_sources}" - elif { [ "${src_name}" = "energized" ] && [ -z "${adb_eng_sources}" ]; } || - { [ "${src_name}" = "stevenblack" ] && [ -z "${adb_stb_sources}" ]; }; then + ;; + "hagezi") + src_cat="${adb_hag_feed}" + if [ -z "${src_cat}" ]; then + f_log "info" "feed '${src_name}' requires category configuration, skipping" + continue + fi + ;; + "ipfire_dbl") + src_cat="${adb_ipf_feed}" + if [ -z "${src_cat}" ]; then + f_log "info" "feed '${src_name}' requires category configuration, skipping" + continue + fi + ;; + "stevenblack") + src_cat="${adb_stb_feed}" + if [ -z "${src_cat}" ]; then + f_log "info" "feed '${src_name}' requires category configuration, skipping" continue fi + ;; + esac + + # category handling for feeds with multiple categories + # + if [ -n "${src_cat}" ]; then ( - for suffix in ${src_cat:-${src_url}}; do - if [ "${src_url}" != "${suffix}" ]; then - src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_tmpcat}" "${src_url}${suffix}" 2>&1)" - src_rc="${?}" - if [ "${src_rc}" = "0" ] && [ -s "${src_tmpcat}" ]; then - cat "${src_tmpcat}" >>"${src_tmpload}" - : >"${src_tmpcat}" + # restore handling on boot, resume or (re-)start + # + if [ "${feed_restore}" = "1" ]; then + if f_list restore && [ -s "${src_tmpfile}" ]; then + exit 0 + fi + fi + + # etag handling on reload + # + if [ -n "${adb_etagparm}" ] && [ "${adb_action}" = "reload" ]; then + etag_rc="0" + src_cnt="0" + for _ in ${src_cat}; do + src_cnt="$((src_cnt + 1))" + done + for suffix in ${src_cat}; do + if ! f_etag "${src_name}" "${src_url}" "${suffix}" "${src_cnt}"; then + etag_rc="$((etag_rc + 1))" fi - else - src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_tmpload}" "${src_url}" 2>&1)" - src_rc="${?}" + done + if [ "${etag_rc}" = "0" ]; then + if f_list restore && [ -s "${src_tmpfile}" ]; then + exit 0 + fi + fi + fi + + # category download + # + for suffix in ${src_cat}; do + "${adb_fetchcmd}" ${adb_fetchparm} "${src_tmpcat}" "${src_url}${suffix}" 2>>"${adb_errorlog}" + src_rc="${?}" + if [ "${src_rc}" = "0" ] && [ -s "${src_tmpcat}" ]; then + "${adb_catcmd}" "${src_tmpcat}" >>"${src_tmpload}" + : >"${src_tmpcat}" fi done - if [ "${src_rc}" = "0" ] && [ -s "${src_tmpload}" ]; then - if [ -s "${adb_tmpdir}/tmp.rem.whitelist" ]; then - "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | - grep -Evf "${adb_tmpdir}/tmp.rem.whitelist" | "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' >"${src_tmpsort}" - else - "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' >"${src_tmpsort}" + f_list prepare + ) & + + # normal handling for feeds without categories + # + else + ( + [ "${src_name}" = "utcapitole" ] && src_cat="${adb_utc_feed}" + + # restore handling on boot, resume or (re-)start + # + if [ "${feed_restore}" = "1" ]; then + if f_list restore && [ -s "${src_tmpfile}" ]; then + exit 0 fi - : >"${src_tmpload}" - "${adb_sort}" ${adb_srtopts} -u "${src_tmpsort}" 2>/dev/null >"${src_tmpfile}" - src_rc="${?}" - : >"${src_tmpsort}" - if [ "${src_rc}" = "0" ] && [ -s "${src_tmpfile}" ]; then - f_list download - [ "${adb_backup}" = "1" ] && f_list backup - elif [ "${adb_backup}" = "1" ] && [ "${adb_action}" != "start" ]; then - f_log "info" "preparation of '${src_name}' failed, rc: ${src_rc}" - f_list restore - rm -f "${src_tmpfile}" + fi + + # etag handling on reload + # + if [ -n "${adb_etagparm}" ] && [ "${adb_action}" = "reload" ]; then + if f_etag "${src_name}" "${src_url}"; then + if f_list restore && [ -s "${src_tmpfile}" ]; then + exit 0 + fi + fi + fi + + # download feed and extract categories if necessary + # + if [ "${src_name}" = "utcapitole" ]; then + if [ -n "${src_cat}" ]; then + "${adb_fetchcmd}" ${adb_fetchparm} "${src_tmparchive}" "${src_url}" 2>>"${adb_errorlog}" + src_rc="${?}" + if [ "${src_rc}" = "0" ] && [ -s "${src_tmparchive}" ]; then + src_suffix="${adb_src_suffix_utcapitole:-"domains"}" + src_entries="$(tar -tzf "${src_tmparchive}" 2>>"${adb_errorlog}" | "${adb_awkcmd}" \ + -v cats="${src_cat}" -v sfx="${src_suffix}" ' + BEGIN { n = split(cats, c, " ") } + { for (i = 1; i <= n; i++) if ($0 ~ "(^|/)" c[i] "/" sfx "$") print }')" + if [ -n "${src_entries}" ]; then + tar -xOzf "${src_tmparchive}" ${src_entries} 2>>"${adb_errorlog}" >"${src_tmpload}" + src_rc="${?}" + fi + : >"${src_tmparchive}" + fi fi else - src_log="$(printf "%s" "${src_log}" | "${adb_awk}" '{ORS=" ";print $0}')" - f_log "info" "download of '${src_name}' failed, url: ${src_url}, rule: ${src_rset:-"-"}, categories: ${src_cat:-"-"}, rc: ${src_rc}, log: ${src_log:-"-"}" - [ "${adb_backup}" = "1" ] && [ "${adb_action}" != "start" ] && f_list restore + "${adb_fetchcmd}" ${adb_fetchparm} "${src_tmpload}" "${src_url}" 2>>"${adb_errorlog}" + src_rc="${?}" fi + f_list prepare ) & fi - hold="$((cnt % adb_cores))" - [ "${hold}" = "0" ] && wait + [ "${cnt}" -gt "${adb_cores}" ] && wait -n cnt="$((cnt + 1))" done wait @@ -1300,15 +2015,16 @@ f_main() { # tld compression and dns restart # if f_list merge && [ -s "${adb_tmpdir}/${adb_dnsfile}" ]; then - f_tld "${adb_tmpdir}/${adb_dnsfile}" + [ "${adb_tld}" = "1" ] && f_tld "${adb_tmpdir}/${adb_dnsfile}" f_list final else - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" + f_log "info" "no merge input, only header written to ${adb_finaldir}/${adb_dnsfile}" fi - chown "${adb_dnsuser}" "${adb_dnsdir}/${adb_dnsfile}" 2>/dev/null + chown "${adb_dnsuser}" "${adb_finaldir}/${adb_dnsfile}" 2>>"${adb_errorlog}" if f_dnsup; then - [ "${adb_action}" != "resume" ] && f_jsnup "enabled" f_log "info" "blocklist with overall ${adb_cnt} blocked domains loaded successfully (${adb_sysver})" + [ "${adb_action}" != "resume" ] && f_jsnup "enabled" else f_log "err" "dns backend restart with adblock blocklist failed" fi @@ -1318,149 +2034,412 @@ f_main() { # trace dns queries via tcpdump and prepare a report # f_report() { - local report_raw report_txt content status total start end start_date start_time end_date end_time blocked percent top_list top array item index hold ports value key key_list cnt="0" resolve="-nn" action="${1}" top_count="${2:-"10"}" res_count="${3:-"50"}" search="${4:-"+"}" + local report_raw report_txt content status total start end start_date start_time end_date end_time blocked percent top_list top array item index value key key_list + local domain rc map_seen ip request requests iface_v4 iface_v6 ip_v4 ip_v6 map_jsn cnt report_srt report_jsn top_tmpclients top_tmpdomains top_tmpblocked + local file jsn resolve="-nn" action="${1}" top_count="${2:-"10"}" res_count="${3:-"50"}" search="${4:-"+"}" report_raw="${adb_reportdir}/adb_report.raw" report_srt="${adb_reportdir}/adb_report.srt" - report_jsn="${adb_reportdir}/adb_report.json" + report_jsn="${adb_reportdir}/adb_report.jsn" report_txt="${adb_reportdir}/adb_mailreport.txt" + top_tmpclients="${adb_reportdir}/top_clients.tmp" + top_tmpdomains="${adb_reportdir}/top_domains.tmp" + top_tmpblocked="${adb_reportdir}/top_blocked.tmp" + map_jsn="${adb_reportdir}/adb_map.jsn" - # build json file + # build report # if [ "${action}" != "json" ]; then - : >"${report_raw}" - : >"${report_srt}" - : >"${report_txt}" - : >"${report_jsn}" + : >"${report_srt}" >"${report_txt}" >"${report_jsn}" >"${map_jsn}" + : >"${top_tmpclients}" >"${top_tmpdomains}" >"${top_tmpblocked}" [ "${adb_represolve}" = "1" ] && resolve="" + cnt="1" for file in "${adb_reportdir}/adb_report.pcap"*; do + [ -s "${file}" ] || continue ( - "${adb_dumpcmd}" "${resolve}" -tttt -r "${file}" 2>/dev/null | - "${adb_awk}" -v cnt="${cnt}" '!/\.lan\. |PTR\? | SOA\? /&&/ A[\? ]+|NXDomain|0\.0\.0\.0/{a=$1;b=substr($2,0,8);c=$4;sub(/\.[0-9]+$/,"",c);gsub(/[^[:alnum:]\.:-]/,"",c);d=cnt $7;sub(/\*$/,"",d); - e=$(NF-1);sub(/[0-9]\/[0-9]\/[0-9]|0\.0\.0\.0/,"NX",e);sub(/\.$/,"",e);sub(/([0-9]{1,3}\.){3}[0-9]{1,3}/,"OK",e);gsub(/[^[:alnum:]\.-]/,"",e);if(e==""){e="err"};printf "%s\t%s\t%s\t%s\t%s\n",d,e,a,b,c}' >>"${report_raw}" + "${adb_dumpcmd}" ${resolve} --immediate-mode -tttt -T domain -r "${file}" 2>/dev/null | + "${adb_awkcmd}" -v repiface="${adb_repiface}" ' + BEGIN { + pending = 0 + } + # ignore Reverse DNS + /\.in-addr\.arpa/ || /\.ip6\.arpa/ { next } + # domain request parser (with optional EDNS marker support) + /\+[[:space:]]+(\[[0-9a-z]*\][[:space:]]+)?(A\?|AAAA\?)/ { + # drop unresolved previous query + if (pending) + pending = 0 + date = $1 + split($2, t, ":") + time = t[1] ":" t[2] ":" substr(t[3],1,2) + interface = repiface + client = $4 + if (repiface == "any") { + interface = $3 + client = $6 + } + sub(/\.[0-9]+$/, "", client) + domain = $(NF-1) + sub(/[,\.]+$/, "", domain) + if (domain ~ /\.lan$/) next + if (domain !~ /\./) next + if (domain ~ /[\/:]/) next + qtype = $(NF-2) + sub(/\?$/, "", qtype) + last_date = date + last_time = time + last_client = client + last_interface = interface + last_domain = domain + last_qtype = qtype + pending = 1 + next + } + # ok answer + / (A|AAAA|CNAME) / && !/NXDomain/ && !/ServFail/ { + if (pending) { + printf "%s\t%s\t%s\t%s\t%s\t%s\tOK\n", + last_date, last_time, last_client, last_interface, last_qtype, last_domain + pending = 0 + } + next + } + # nxdomain answer + / NXDomain/ { + if (pending) { + printf "%s\t%s\t%s\t%s\t%s\t%s\tNX\n", + last_date, last_time, last_client, last_interface, last_qtype, last_domain + pending = 0 + } + next + } + # servfail answer + / ServFail/ { + if (pending) { + printf "%s\t%s\t%s\t%s\t%s\t%s\tSF\n", + last_date, last_time, last_client, last_interface, last_qtype, last_domain + pending = 0 + } + next + } + ' >"${report_raw}.${cnt}" ) & - hold="$((cnt % adb_cores))" - [ "${hold}" = "0" ] && wait + [ "${cnt}" -gt "${adb_cores}" ] && wait -n cnt="$((cnt + 1))" done wait - if [ -s "${report_raw}" ]; then - "${adb_sort}" ${adb_srtopts} -k1 -k3 -k4 -k5 -k1 -ur "${report_raw}" | - "${adb_awk}" '{currA=($1+0);currB=$1;currC=substr($1,length($1),1);if(reqA==currB){reqA=0;printf "%s\t%s\n",d,$2}else if(currC=="+"){reqA=currA;d=$3"\t"$4"\t"$5"\t"$2}}' | - "${adb_sort}" ${adb_srtopts} -k1 -k2 -k3 -k4 -ur >"${report_srt}" - rm -f "${report_raw}" - fi + for file in "${report_raw}".*; do + [ -s "${file}" ] || continue + "${adb_sortcmd}" ${adb_srtopts} -ru "${report_raw}".* >"${report_srt}" + "${adb_rmcmd}" -f "${report_raw}".* + break + done + # build json + # if [ -s "${report_srt}" ]; then - start="$("${adb_awk}" 'END{printf "%s_%s",$1,$2}' "${report_srt}")" - end="$("${adb_awk}" 'NR==1{printf "%s_%s",$1,$2}' "${report_srt}")" - total="$(wc -l <"${report_srt}")" - blocked="$("${adb_awk}" '{if($5=="NX")cnt++}END{printf "%s",cnt}' "${report_srt}")" - percent="$("${adb_awk}" -v t="${total}" -v b="${blocked}" 'BEGIN{printf "%.2f%s",b/t*100,"%"}')" - : >"${report_jsn}" + start="$("${adb_awkcmd}" 'END{printf "%s_%s",$1,$2}' "${report_srt}")" + end="$("${adb_awkcmd}" 'NR==1{printf "%s_%s",$1,$2}' "${report_srt}")" + total="$(f_count tld "${report_srt}" "var")" + blocked="$("${adb_awkcmd}" '{if($7=="NX")cnt++}END{printf "%s",cnt}' "${report_srt}")" + percent="$("${adb_awkcmd}" -v t="${total}" -v b="${blocked}" 'BEGIN{ if(t>0) printf "%.2f%s",b/t*100,"%"; else printf "0.00%%"}')" { - printf "%s\n" "{ " - printf "\t%s\n" "\"start_date\": \"${start%_*}\", " - printf "\t%s\n" "\"start_time\": \"${start#*_}\", " - printf "\t%s\n" "\"end_date\": \"${end%_*}\", " - printf "\t%s\n" "\"end_time\": \"${end#*_}\", " - printf "\t%s\n" "\"total\": \"${total}\", " - printf "\t%s\n" "\"blocked\": \"${blocked}\", " - printf "\t%s\n" "\"percent\": \"${percent}\", " - } >>"${report_jsn}" + printf '%s\n' "{ " + printf '\t%s\n' "\"start_date\": \"${start%_*}\", " + printf '\t%s\n' "\"start_time\": \"${start#*_}\", " + printf '\t%s\n' "\"end_date\": \"${end%_*}\", " + printf '\t%s\n' "\"end_time\": \"${end#*_}\", " + printf '\t%s\n' "\"total\": \"${total}\", " + printf '\t%s\n' "\"blocked\": \"${blocked}\", " + printf '\t%s\n' "\"percent\": \"${percent}\", " + } >"${report_jsn}" + + # build top list counters + # + "${adb_awkcmd}" ' + { + if (NF < 7) { + next + } + client = $3 + domain = $6 + rc = $7 + + if (domain == "" || domain == "-") { + next + } + + sub(/[\.]+$/, "", domain) + domain = tolower(domain) + + clients[client]++ + if (rc == "OK") { + ok_domain[domain]++ + } + else if (rc == "NX") { + nx_domain[domain]++ + } + all_domain[domain]++ + } + END { + for (c in clients) { + printf "%d %s\n", clients[c], c > "'"${top_tmpclients}"'" + } + for (d in all_domain) { + if (d in ok_domain) { + printf "%d %s\n", ok_domain[d], d > "'"${top_tmpdomains}"'" + } + if (d in nx_domain) { + printf "%d %s\n", nx_domain[d], d > "'"${top_tmpblocked}"'" + } + } + } + ' "${report_srt}" + + # build json top lists + # top_list="top_clients top_domains top_blocked" for top in ${top_list}; do - printf "\t%s" "\"${top}\": [ " >>"${report_jsn}" + printf '\t"%s": [ ' "${top}" >>"${report_jsn}" case "${top}" in - "top_clients") - "${adb_awk}" '{print $3}' "${report_srt}" | "${adb_sort}" ${adb_srtopts} | uniq -c | - "${adb_sort}" ${adb_srtopts} -nr | - "${adb_awk}" "{ORS=\" \";if(NR==1)printf \"\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2; else if(NR<=${top_count})printf \",\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2}" >>"${report_jsn}" - ;; - "top_domains") - "${adb_awk}" '{if($5!="NX")print $4}' "${report_srt}" | "${adb_sort}" ${adb_srtopts} | uniq -c | - "${adb_sort}" ${adb_srtopts} -nr | - "${adb_awk}" "{ORS=\" \";if(NR==1)printf \"\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2; else if(NR<=${top_count})printf \",\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2}" >>"${report_jsn}" - ;; - "top_blocked") - "${adb_awk}" '{if($5=="NX")print $4}' "${report_srt}" | - "${adb_sort}" ${adb_srtopts} | uniq -c | "${adb_sort}" ${adb_srtopts} -nr | - "${adb_awk}" "{ORS=\" \";if(NR==1)printf \"\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2; else if(NR<=${top_count})printf \",\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2}" >>"${report_jsn}" - ;; + top_clients) + "${adb_sortcmd}" ${adb_srtopts} -nr "${top_tmpclients}" | + "${adb_awkcmd}" -v top_count="${top_count}" ' + BEGIN { ORS=""; OFS="" } + NR==1 { + printf "\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + NR>1 && NR<=top_count { + printf ",\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + ' >>"${report_jsn}" + ;; + top_domains) + "${adb_sortcmd}" ${adb_srtopts} -nr "${top_tmpdomains}" | + "${adb_awkcmd}" -v top_count="${top_count}" ' + BEGIN { ORS=""; OFS="" } + NR==1 { + printf "\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + NR>1 && NR<=top_count { + printf ",\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + ' >>"${report_jsn}" + ;; + top_blocked) + "${adb_sortcmd}" ${adb_srtopts} -nr "${top_tmpblocked}" | + "${adb_awkcmd}" -v top_count="${top_count}" ' + BEGIN { ORS=""; OFS="" } + NR==1 { + printf "\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + NR>1 && NR<=top_count { + printf ",\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + ' >>"${report_jsn}" + ;; esac - printf "\n\t%s\n" "]," >>"${report_jsn}" + printf '\n\t],\n' >>"${report_jsn}" + done + "${adb_rmcmd}" -f "${top_tmpclients}" "${top_tmpdomains}" "${top_tmpblocked}" + + # build json request list + # + search="${search//[!a-zA-Z0-9._-]/}" + case "${res_count}" in + '' | *[!0-9]*) + res_count="50" + ;; + esac + "${adb_awkcmd}" -v search="${search}" -v res_count="${res_count}" ' + BEGIN { + i = 0 + printf "\t\"requests\": [\n" + } + + # only match if search is empty or non-empty and NF == 7 + ((search == "" || index($0, search)) && NF == 7) { + i++ + if (res_count > 0 && i > res_count) { + next + } + if (i > 1) { + printf ",\n" + } + + printf "\n\t\t{\n" + printf "\t\t\t\"date\": \"%s\",\n", $1 + printf "\t\t\t\"time\": \"%s\",\n", $2 + printf "\t\t\t\"client\": \"%s\",\n", $3 + printf "\t\t\t\"iface\": \"%s\",\n", $4 + printf "\t\t\t\"type\": \"%s\",\n", $5 + printf "\t\t\t\"domain\": \"%s\",\n", $6 + printf "\t\t\t\"rc\": \"%s\"\n", $7 + printf "\t\t}" + } + END { + printf "\n\t]\n}\n" + }' "${report_srt}" >>"${report_jsn}" + "${adb_rmcmd}" -f "${report_srt}" + fi + + # retrieve/prepare map data + # + if [ "${adb_map}" = "1" ] && [ -s "${report_jsn}" ]; then + cnt="1" + network_find_wan iface_v4 && network_get_ipaddr ip_v4 "${iface_v4}" + network_find_wan6 iface_v6 && network_get_ipaddr6 ip_v6 "${iface_v6}" + if [ -n "${ip_v4}" ] || [ -n "${ip_v6}" ]; then + f_fetch + printf '%s' ",[{}" >"${map_jsn}" + fi + for ip in ${ip_v4} ${ip_v6}; do + ( + "${adb_fetchcmd}" ${adb_geoparm} "${adb_geourl}/${ip}" 2>>"${adb_errorlog}" | + "${adb_awkcmd}" -v feed="homeIP" '{printf ",{\"%s\": %s}\n",feed,$0}' >"${map_jsn}.${cnt}" + ) & + [ "${cnt}" -gt "${adb_cores}" ] && wait -n + cnt="$((cnt + 1))" + done + wait + if [ -s "${map_jsn}" ] && [ "${cnt}" -lt "45" ]; then + map_seen="" + json_init + if json_load_file "${report_jsn}" >/dev/null 2>&1; then + json_select "requests" >/dev/null 2>&1 + json_get_keys requests >/dev/null 2>&1 + for request in ${requests}; do + json_select "${request}" >/dev/null 2>&1 + json_get_var rc "rc" >/dev/null 2>&1 + json_get_var domain "domain" >/dev/null 2>&1 + if [ "${rc}" = "NX" ]; then + case " ${map_seen} " in + *" ${domain} "*) ;; + + *) + map_seen="${map_seen} ${domain} " + ( + "${adb_fetchcmd}" ${adb_geoparm} "${adb_geourl}/${domain}" 2>>"${adb_errorlog}" | + "${adb_awkcmd}" -v feed="${domain}" '{printf ",{\"%s\": %s}\n",feed,$0}' >"${map_jsn}.${cnt}" + ) & + [ "${cnt}" -gt "${adb_cores}" ] && wait -n + cnt="$((cnt + 1))" + [ "${cnt}" -ge "45" ] && break + ;; + esac + fi + json_select ".." + done + wait + fi + fi + for file in "${map_jsn}".*; do + [ -s "${file}" ] || continue + "${adb_catcmd}" "${map_jsn}".* >>"${map_jsn}" 2>/dev/null + "${adb_rmcmd}" -f "${map_jsn}".* + break done - search="${search//./\\.}" - search="${search//[+*~%\$&\"\' ]/}" - "${adb_awk}" "BEGIN{i=0;printf \"\t\\\"requests\\\": [\n\"}/(${search})/{i++;if(i==1)printf \"\n\t\t{\n\t\t\t\\\"date\\\": \\\"%s\\\",\n\t\t\t\\\"time\\\": \\\"%s\\\",\n\t\t\t\\\"client\\\": \\\"%s\\\",\n\t\t\t\\\"domain\\\": \\\"%s\\\",\n\t\t\t\\\"rc\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2,\$3,\$4,\$5;else if(i<=${res_count})printf \",\n\t\t{\n\t\t\t\\\"date\\\": \\\"%s\\\",\n\t\t\t\\\"time\\\": \\\"%s\\\",\n\t\t\t\\\"client\\\": \\\"%s\\\",\n\t\t\t\\\"domain\\\": \\\"%s\\\",\n\t\t\t\\\"rc\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2,\$3,\$4,\$5}END{printf \"\n\t]\n}\n\"}" "${adb_reportdir}/adb_report.srt" >>"${report_jsn}" - rm -f "${report_srt}" fi fi # output preparation # if [ -s "${report_jsn}" ] && { [ "${action}" = "cli" ] || [ "${action}" = "mail" ]; }; then - printf "%s\n%s\n%s\n" ":::" "::: Adblock DNS-Query Report" ":::" >>"${report_txt}" + printf '%s\n%s\n%s\n' ":::" "::: Adblock DNS Report" ":::" >>"${report_txt}" json_init json_load_file "${report_jsn}" json_get_keys key_list for key in ${key_list}; do json_get_var value "${key}" - eval "${key}=\"${value}\"" + case "${key}" in + "start_date") + start_date="${value}" + ;; + "start_time") + start_time="${value}" + ;; + "end_date") + end_date="${value}" + ;; + "end_time") + end_time="${value}" + ;; + "total") + total="${value}" + ;; + "blocked") + blocked="${value}" + ;; + "percent") + percent="${value}" + ;; + esac done - printf " + %s\n + %s\n" "Start ::: ${start_date}, ${start_time}" "End ::: ${end_date}, ${end_time}" >>"${report_txt}" - printf " + %s\n + %s %s\n" "Total ::: ${total}" "Blocked ::: ${blocked}" "(${percent})" >>"${report_txt}" + printf ' + %s\n + %s\n' "Start ::: ${start_date}, ${start_time}" "End ::: ${end_date}, ${end_time}" >>"${report_txt}" + printf ' + %s\n + %s %s\n' "Total ::: ${total}" "Blocked ::: ${blocked}" "(${percent})" >>"${report_txt}" top_list="top_clients top_domains top_blocked requests" for top in ${top_list}; do case "${top}" in - "top_clients") - item="::: Top Clients" - ;; - "top_domains") - item="::: Top Domains" - ;; - "top_blocked") - item="::: Top Blocked Domains" - ;; + "top_clients") + item="::: Top Clients" + ;; + "top_domains") + item="::: Top Domains" + ;; + "top_blocked") + item="::: Top Blocked Domains" + ;; esac if json_get_type status "${top}" && [ "${top}" != "requests" ] && [ "${status}" = "array" ]; then - printf "%s\n%s\n%s\n" ":::" "${item}" ":::" >>"${report_txt}" + printf '%s\n%s\n%s\n' ":::" "${item}" ":::" >>"${report_txt}" json_select "${top}" index="1" item="" while json_get_type status "${index}" && [ "${status}" = "object" ]; do json_get_values item "${index}" - printf " + %-9s::: %s\n" ${item} >>"${report_txt}" + printf ' + %-9s::: %s\n' ${item} >>"${report_txt}" index="$((index + 1))" done elif json_get_type status "${top}" && [ "${top}" = "requests" ] && [ "${status}" = "array" ]; then - printf "%s\n%s\n%s\n" ":::" "::: Latest DNS Queries" ":::" >>"${report_txt}" - printf "%-15s%-15s%-45s%-80s%s\n" "Date" "Time" "Client" "Domain" "Answer" >>"${report_txt}" + printf '%s\n%s\n%s\n' ":::" "::: Latest DNS Queries" ":::" >>"${report_txt}" + printf '%-11s%-9s%-40s%-15s%-5s%-70s%s\n' "Date" "Time" "Client" "Interface" "Type" "Domain" "Answer" >>"${report_txt}" json_select "${top}" index="1" while json_get_type status "${index}" && [ "${status}" = "object" ]; do json_get_values item "${index}" - printf "%-15s%-15s%-45s%-80s%s\n" ${item} >>"${report_txt}" + printf '%-11s%-9s%-40s%-15s%-5s%-70s%s\n' ${item} >>"${report_txt}" index="$((index + 1))" done fi json_select ".." done - content="$(cat "${report_txt}" 2>/dev/null)" - rm -f "${report_txt}" + content="$("${adb_catcmd}" "${report_txt}" 2>>"${adb_errorlog}")" + "${adb_rmcmd}" -f "${report_txt}" fi # report output # - if [ "${action}" = "cli" ]; then - printf "%s\n" "${content}" - elif [ "${action}" = "json" ]; then - cat "${report_jsn}" - elif [ "${action}" = "mail" ] && [ "${adb_mail}" = "1" ] && [ -x "${adb_mailservice}" ]; then - ("${adb_mailservice}" "${adb_ver}" "${content}" >/dev/null 2>&1) & - bg_pid="${!}" - fi - f_log "debug" "f_report ::: action: ${action}, top_count: ${top_count}, res_count: ${res_count}, search: ${search}, dump_util: ${adb_dumpcmd}, rep_dir: ${adb_reportdir}, rep_iface: ${adb_repiface:-"-"}, rep_listen: ${adb_replisten}, rep_chunksize: ${adb_repchunksize}, rep_chunkcnt: ${adb_repchunkcnt}, rep_resolve: ${adb_represolve}" + case "${action}" in + "cli") + printf '%s\n' "${content}" + ;; + "json") + if [ "${adb_map}" = "1" ] && [ -s "${map_jsn}" ]; then + jsn="$("${adb_catcmd}" ${report_jsn} ${map_jsn} 2>>"${adb_errorlog}")" + [ -n "${jsn}" ] && printf '[%s]]\n' "${jsn}" + else + jsn="$("${adb_catcmd}" "${report_jsn}" 2>>"${adb_errorlog}")" + [ -n "${jsn}" ] && printf '[%s]\n' "${jsn}" + fi + ;; + "mail") + [ "${adb_mail}" = "1" ] && [ -x "${adb_mailservice}" ] && "${adb_mailservice}" "${content}" >/dev/null 2>&1 + "${adb_rmcmd}" -f "${report_txt}" + ;; + "gen") + printf '%s\n' "1" >"${adb_rundir}/adblock.report" + ;; + esac } # source required system libraries @@ -1473,47 +2452,72 @@ else f_log "err" "system libraries not found" fi -# awk check +# create runtime directory if it doesn't exist # -adb_awk="$(command -v gawk)" -if [ ! -x "${adb_awk}" ]; then - adb_awk="$(command -v awk)" - [ ! -x "${adb_awk}" ] && f_log "err" "awk not found or not executable" -fi +[ ! -d "${adb_rundir}" ] && mkdir -p "${adb_rundir}" -# sort check +# reference required system utilities # -adb_sort="$(command -v sort)" -if [ ! -x "${adb_sort}" ] || ! "${adb_sort}" --version 2>/dev/null | grep -q "coreutils"; then - f_log "err" "coreutils sort not found or not executable" -fi +adb_mvcmd="$(f_cmd mv)" +adb_lncmd="$(f_cmd ln)" +adb_rmcmd="$(f_cmd rm)" +adb_catcmd="$(f_cmd cat)" +adb_zcatcmd="$(f_cmd zcat)" +adb_awkcmd="$(f_cmd gawk awk)" +adb_sortcmd="$(f_cmd sort)" +adb_grepcmd="$(f_cmd grep)" +adb_gzipcmd="$(f_cmd gzip)" +adb_pgrepcmd="$(f_cmd pgrep)" +adb_sedcmd="$(f_cmd sed)" +adb_findcmd="$(f_cmd find)" +adb_jsoncmd="$(f_cmd jsonfilter)" +adb_ubuscmd="$(f_cmd ubus)" +adb_loggercmd="$(f_cmd logger)" +adb_lookupcmd="$(f_cmd nslookup)" +adb_xargscmd="$(f_cmd xargs)" +adb_flockcmd="$(f_cmd flock)" +adb_dumpcmd="$(f_cmd tcpdump optional)" +adb_mailcmd="$(f_cmd msmtp optional)" +adb_logreadcmd="$(f_cmd logread optional)" +adb_nftcmd="$(f_cmd nft)" # handle different adblock actions # f_load case "${adb_action}" in - "stop") - f_rmdns - ;; - "restart") - f_rmdns - f_env - f_main - ;; - "suspend") - [ "${adb_dns}" != "raw" ] && f_switch suspend - ;; - "resume") - [ "${adb_dns}" != "raw" ] && f_switch resume - ;; - "report") - f_report "${2}" "${3}" "${4}" "${5}" - ;; - "query") - f_query "${2}" - ;; - "start" | "reload") - f_env - f_main - ;; +"stop") + f_temp + f_nftremove + f_rmdns + f_jsnup "stopped" + ;; +"suspend") + [ "${adb_dns}" != "raw" ] && f_switch suspend + ;; +"resume") + [ "${adb_dns}" != "raw" ] && f_switch resume + ;; +"report") + f_report "${2}" "${3}" "${4}" "${5}" + ;; +"search") + f_search "${2}" + ;; +"boot" | "start" | "reload") + f_env + f_main + ;; +# Start NethSecurity patch +"nft-reload") + f_nftremove + f_nftadd + ;; +# End NethSecurity patch +"restart") + f_temp + f_jsnup "processing" + f_rmdns + f_env + f_main + ;; esac diff --git a/packages/adblock/files/adblock.sources b/packages/adblock/files/adblock.sources deleted file mode 100644 index 85af8602b..000000000 --- a/packages/adblock/files/adblock.sources +++ /dev/null @@ -1,352 +0,0 @@ -{ - "adaway": { - "url": "https://raw.githubusercontent.com/AdAway/adaway.github.io/master/hosts.txt", - "rule": "/^127\\.0\\.0\\.1[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "mobile", - "descurl": "https://github.com/AdAway/adaway.github.io" - }, - "adguard": { - "url": "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt", - "rule": "BEGIN{FS=\"[\/|^|\\r]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+[\\/\\^\\r]+$/{print tolower($3)}", - "size": "L", - "focus": "general", - "descurl": "https://adguard.com" - }, - "adguard_tracking": { - "url": "https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/combined_disguised_trackers_justdomains.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "tracking", - "descurl": "https://github.com/AdguardTeam/cname-trackers" - }, - "android_tracking": { - "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/android-tracking.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "tracking", - "descurl": "https://github.com/Perflyst/PiHoleBlocklist" - }, - "andryou": { - "url": "https://gitlab.com/andryou/block/raw/master/kouhai-compressed-domains", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "L", - "focus": "compilation", - "descurl": "https://gitlab.com/andryou/block/-/blob/master/readme.md" - }, - "anti_ad": { - "url": "https://raw.githubusercontent.com/privacy-protection-tools/anti-AD/master/anti-ad-domains.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "L", - "focus": "compilation", - "descurl": "https://github.com/privacy-protection-tools/anti-AD/blob/master/README.md" - }, - "antipopads": { - "url": "https://raw.githubusercontent.com/AdroitAdorKhan/antipopads-re/master/formats/domains.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "L", - "focus": "compilation", - "descurl": "https://github.com/AdroitAdorKhan/antipopads-re" - }, - "anudeep": { - "url": "https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "M", - "focus": "compilation", - "descurl": "https://github.com/anudeepND/blacklist" - }, - "bitcoin": { - "url": "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "mining", - "descurl": "https://github.com/hoshsadiq/adblock-nocoin-list" - }, - "cpbl": { - "url": "https://raw.githubusercontent.com/bongochong/CombinedPrivacyBlockLists/master/NoFormatting/cpbl-ctld.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "XL", - "focus": "compilation", - "descurl": "https://github.com/bongochong/CombinedPrivacyBlockLists" - }, - "disconnect": { - "url": "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "general", - "descurl": "https://disconnect.me" - }, - "doh_blocklist": { - "url": "https://raw.githubusercontent.com/dibdot/DoH-IP-blocklists/master/doh-domains_overall.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "doh_server", - "descurl": "https://github.com/dibdot/DoH-IP-blocklists" - }, - "easylist": { - "url": "https://easylist-downloads.adblockplus.org/easylist.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "M", - "focus": "compilation", - "descurl": "https://easylist.to" - }, - "easyprivacy": { - "url": "https://easylist-downloads.adblockplus.org/easyprivacy.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "M", - "focus": "tracking", - "descurl": "https://easylist.to" - }, - "firetv_tracking": { - "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/AmazonFireTV.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "tracking", - "descurl": "https://github.com/Perflyst/PiHoleBlocklist" - }, - "games_tracking": { - "url": "https://raw.githubusercontent.com/KodoPengin/GameIndustry-hosts-Template/master/Main-Template/hosts", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "tracking", - "descurl": "https://www.gameindustry.eu" - }, - "hblock": { - "url": "https://hblock.molinero.dev/hosts_domains.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "XL", - "focus": "compilation", - "descurl": "https://hblock.molinero.dev" - }, - "lightswitch05": { - "url": "https://www.github.developerdan.com/hosts/lists/ads-and-tracking-extended.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "XL", - "focus": "compilation", - "descurl": "https://github.com/lightswitch05/hosts" - }, - "notracking": { - "url": "https://raw.githubusercontent.com/notracking/hosts-blocklists/master/dnscrypt-proxy/dnscrypt-proxy.blacklist.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "XL", - "focus": "tracking", - "descurl": "https://github.com/notracking/hosts-blocklists" - }, - "oisd_big": { - "url": "https://big.oisd.nl/domainswild", - "rule": "BEGIN{FS=\"\\\\*.\"}/^\\*\\.([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "XXL", - "focus": "general", - "descurl": "https://oisd.nl" - }, - "oisd_nsfw": { - "url": "https://nsfw.oisd.nl/domainswild", - "rule": "BEGIN{FS=\"\\\\*.\"}/^\\*\\.([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "XXL", - "focus": "porn", - "descurl": "https://oisd.nl" - }, - "oisd_small": { - "url": "https://small.oisd.nl/domainswild", - "rule": "BEGIN{FS=\"\\\\*.\"}/^\\*\\.([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "L", - "focus": "general", - "descurl": "https://oisd.nl" - }, - "openphish": { - "url": "https://openphish.com/feed.txt", - "rule": "BEGIN{FS=\"\/\"}/^http[s]?:\\/\\/([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+(\\/|$)/{print tolower($3)}", - "size": "S", - "focus": "phishing", - "descurl": "https://openphish.com" - }, - "phishing_army": { - "url": "https://phishing.army/download/phishing_army_blocklist_extended.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "phishing", - "descurl": "https://phishing.army" - }, - "reg_cn": { - "url": "https://easylist-downloads.adblockplus.org/easylistchina.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_china", - "descurl": "https://easylist.to" - }, - "reg_cz": { - "url": "https://easylist-downloads.adblockplus.org/easylistczechslovak.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_czech+slovak", - "descurl": "https://easylist.to" - }, - "reg_de": { - "url": "https://easylist-downloads.adblockplus.org/easylistgermany.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_germany", - "descurl": "https://easylist.to" - }, - "reg_es": { - "url": "https://easylist-downloads.adblockplus.org/easylistspanish.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_spain", - "descurl": "https://easylist.to" - }, - "reg_fi": { - "url": "https://raw.githubusercontent.com/finnish-easylist-addition/finnish-easylist-addition/master/Finland_adb.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_finland", - "descurl": "https://github.com/finnish-easylist-addition" - }, - "reg_fr": { - "url": "https://easylist-downloads.adblockplus.org/liste_fr.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "M", - "focus": "reg_france", - "descurl": "https://forums.lanik.us/viewforum.php?f=91" - }, - "reg_id": { - "url": "https://easylist-downloads.adblockplus.org/abpindo.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_indonesia", - "descurl": "https://easylist.to" - }, - "reg_it": { - "url": "https://easylist-downloads.adblockplus.org/easylistitaly.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_italy", - "descurl": "https://easylist.to" - }, - "reg_jp": { - "url": "https://raw.githubusercontent.com/k2jp/abp-japanese-filters/master/abpjf.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_japan", - "descurl": "https://github.com/k2jp/abp-japanese-filters" - }, - "reg_kr": { - "url": "https://raw.githubusercontent.com/List-KR/List-KR/master/filters-share/adservice.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_korea", - "descurl": "https://github.com/List-KR/List-KR" - }, - "reg_nl": { - "url": "https://easylist-downloads.adblockplus.org/easylistdutch.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_netherlands", - "descurl": "https://easylist.to" - }, - "reg_pl": { - "url": "https://raw.githubusercontent.com/PolishFiltersTeam/KADhosts/master/KADhosts.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "M", - "focus": "reg_poland", - "descurl": "https://kadantiscam.netlify.app" - }, - "reg_ro": { - "url": "https://easylist-downloads.adblockplus.org/rolist.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_romania", - "descurl": "https://easylist.to" - }, - "reg_ru": { - "url": "https://easylist-downloads.adblockplus.org/ruadlist.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_russia", - "descurl": "https://easylist.to" - }, - "reg_se": { - "url": "https://raw.githubusercontent.com/lassekongo83/Frellwits-filter-lists/master/Frellwits-Swedish-Hosts-File.txt", - "rule": "/^127\\.0\\.0\\.1[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "reg_sweden", - "descurl": "https://github.com/lassekongo83/Frellwits-filter-lists" - }, - "reg_vn": { - "url": "https://raw.githubusercontent.com/bigdargon/hostsVN/master/hosts", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "reg_vietnam", - "descurl": "https://bigdargon.github.io/hostsVN" - }, - "smarttv_tracking": { - "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "tracking", - "descurl": "https://github.com/Perflyst/PiHoleBlocklist" - }, - "spam404": { - "url": "https://raw.githubusercontent.com/Dawsey21/Lists/master/main-blacklist.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "general", - "descurl": "https://github.com/Dawsey21" - }, - "stevenblack": { - "url": "https://raw.githubusercontent.com/StevenBlack/hosts/master/", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "VAR", - "focus": "compilation", - "descurl": "https://github.com/StevenBlack/hosts" - }, - "stopforumspam": { - "url": "https://www.stopforumspam.com/downloads/toxic_domains_whole.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "spam", - "descurl": "https://www.stopforumspam.com" - }, - "utcapitole": { - "url": "https://dsi.ut-capitole.fr/blacklists/download/blacklists.tar.gz", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "VAR", - "focus": "general", - "descurl": "https://dsi.ut-capitole.fr/blacklists/index_en.php" - }, - "wally3k": { - "url": "https://v.firebog.net/hosts/static/w3kbl.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "compilation", - "descurl": "https://firebog.net/about" - }, - "whocares": { - "url": "https://someonewhocares.org/hosts/hosts", - "rule": "/^127\\.0\\.0\\.1[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "M", - "focus": "general", - "descurl": "https://someonewhocares.org" - }, - "winhelp": { - "url": "https://winhelp2002.mvps.org/hosts.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "general", - "descurl": "https://winhelp2002.mvps.org" - }, - "winspy": { - "url": "https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "win_telemetry", - "descurl": "https://github.com/crazy-max/WindowsSpyBlocker" - }, - "yoyo": { - "url": "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml&showintro=0&mimetype=plaintext", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "general", - "descurl": "https://pgl.yoyo.org/as" - } -} diff --git a/packages/adblock/files/adblock.whitelist b/packages/adblock/files/adblock.whitelist deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/checkmk-agent/Makefile b/packages/checkmk-agent/Makefile index 8f33df9f8..30053f9f0 100644 --- a/packages/checkmk-agent/Makefile +++ b/packages/checkmk-agent/Makefile @@ -6,7 +6,9 @@ include $(TOPDIR)/rules.mk PKG_NAME:=checkmk-agent -PKG_VERSION:=2.4.0p24 +# renovate: datasource=github-tags depName=Checkmk/checkmk +CHECKMK_UPSTREAM_VERSION:=2.5.0 +PKG_VERSION:=$(subst p,_p,$(CHECKMK_UPSTREAM_VERSION)) PKG_RELEASE:=1 PKG_BUILD_DIR:=$(BUILD_DIR)/checkmk-agent-$(PKG_VERSION) @@ -32,7 +34,7 @@ endef # Download the Check_MK agent binary define Download/checkmk-agent-binary - URL:=https://raw.githubusercontent.com/Checkmk/checkmk/v$(PKG_VERSION)/agents + URL:=https://raw.githubusercontent.com/Checkmk/checkmk/v$(CHECKMK_UPSTREAM_VERSION)/agents URL_FILE:=check_mk_agent.openwrt FILE:=check_mk_agent.openwrt HASH:=skip diff --git a/packages/netifyd/Makefile b/packages/netifyd/Makefile index 54a598df9..291baa833 100644 --- a/packages/netifyd/Makefile +++ b/packages/netifyd/Makefile @@ -8,8 +8,8 @@ PKG_MAINTAINER:=Darryl Sokoloski PKG_LICENSE:=Unlicensed # Base URL for downloads -NETIFYD_BASE_URL:=https://updates.nethsecurity.nethserver.org/netifyd-dist/netifyd-$(NETIFYD_VERSION)/$(ARCH) -DL_DIR:=$(DL_DIR)/netifyd-$(NETIFYD_VERSION)-$(ARCH) +NETIFYD_BASE_URL:=https://updates.nethsecurity.nethserver.org/netifyd-dist/netifyd-$(PKG_VERSION)/25.12.2/$(ARCH) +DL_DIR:=$(DL_DIR)/netifyd-$(PKG_VERSION)-$(ARCH) include $(INCLUDE_DIR)/package.mk @@ -66,7 +66,7 @@ define Download/libnetifyd URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetifyd.so.4.0.0 FILE:=libnetifyd.so.4.0.0 - HASH:=a7bef78717e200eef177da8ee94557a565828ce796d3fd1a1f078e3f55959dd5 + HASH:=ffbbe5078a2b6575db4c478cf1b4d257f9b7241797c84aa1bcbca17ee1b23f3b endef $(eval $(call Download,libnetifyd)) @@ -74,7 +74,7 @@ define Download/libnetify-plm URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-plm.so.1.0.0 FILE:=libnetify-plm.so.1.0.0 - HASH:=b5c8994dc5f497ef1ccf8aae3685c1541702031726bc9f2819489cacfcb67e8f + HASH:=4320235873539f561238c92094702e74b75b8939a301b694255a5512fa036a00 endef $(eval $(call Download,libnetify-plm)) @@ -82,7 +82,7 @@ define Download/libnetify-proc-aggregator URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-proc-aggregator.so.0.0.0 FILE:=libnetify-proc-aggregator.so.0.0.0 - HASH:=c9e5981fd4410776a1808902fbe284f40882823fc4c711f54e14421c80e94c6b + HASH:=736acb4c22d891da66694eee5507ea70679885f0747dec7a07eec8c926dd76fc endef $(eval $(call Download,libnetify-proc-aggregator)) @@ -90,7 +90,7 @@ define Download/libnetify-proc-core URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-proc-core.so.0.0.0 FILE:=libnetify-proc-core.so.0.0.0 - HASH:=725415498d05fc29695a6f1fe14aed4c6b5967ff0d7d4703d06b80489c852222 + HASH:=8c9a1f26a498a6d1c88d76e6aa0367749779ca5f44ea3fa614bc1814387258ed endef $(eval $(call Download,libnetify-proc-core)) @@ -98,7 +98,7 @@ define Download/libnetify-proc-dev-discovery URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-proc-dev-discovery.so.0.0.0 FILE:=libnetify-proc-dev-discovery.so.0.0.0 - HASH:=c25c4a497925cc0cf9d48f353de76fd2b5c9aa223390c87f2163feb1b0d61efb + HASH:=756c01e22c3724311eb6952ca6d580c728592771948207b93bab9b9795c38d1c endef $(eval $(call Download,libnetify-proc-dev-discovery)) @@ -106,7 +106,7 @@ define Download/libnetify-proc-flow-actions URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-proc-flow-actions.so.0.0.0 FILE:=libnetify-proc-flow-actions.so.0.0.0 - HASH:=06ed65a3ec5d840edff008aaebbe294e4b0f7545ab295b287f6ed286bf28b7d7 + HASH:=1d6f03ead8760e314f98f5fe083515b19bf9c60e720d36bc03f70fa60a9b0d2b endef $(eval $(call Download,libnetify-proc-flow-actions)) @@ -114,7 +114,7 @@ define Download/libnetify-sink-http URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-sink-http.so.0.0.0 FILE:=libnetify-sink-http.so.0.0.0 - HASH:=aaaadb9179f3df215f08d29f3a46db314d3990adafa5f3c4abeff305f4bb9583 + HASH:=95d0d6fa2b47b3764318a2f8680441d126c94bb39c84440b3e056767e330991d endef $(eval $(call Download,libnetify-sink-http)) @@ -122,7 +122,7 @@ define Download/libnetify-sink-log URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-sink-log.so.0.0.0 FILE:=libnetify-sink-log.so.0.0.0 - HASH:=26d136562f7416ce4fe7c70d8023ae3f578a2d2dff4239791c65ad4289d28765 + HASH:=f9cafc30e150109dc3ab5e57eb203d51525a4ad0241fe76724206edf73c575b1 endef $(eval $(call Download,libnetify-sink-log)) @@ -130,7 +130,7 @@ define Download/libnetify-sink-socket URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-sink-socket.so.0.0.0 FILE:=libnetify-sink-socket.so.0.0.0 - HASH:=8772335da50b702edd2a1412c98c82e32058dee2158db6e874ba3964ac66590a + HASH:=7ed9fe8d4d952ccbedca1c2be505fe288080f19c5c60d4db46932078c97d5364 endef $(eval $(call Download,libnetify-sink-socket)) @@ -138,7 +138,7 @@ define Download/libnetify-sink-sqlite URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-sink-sqlite.so.0.0.0 FILE:=libnetify-sink-sqlite.so.0.0.0 - HASH:=6c2879cb8c9da36d592c85b9f8b4042339f0322fe2ebf335a5d15d06562ec65b + HASH:=0de0ef72cbd092fedd5cc278a4b57977891650aa10712f5ae0e6381a5a02fa52 endef $(eval $(call Download,libnetify-sink-sqlite)) @@ -146,7 +146,7 @@ define Download/netifyd URL:=$(NETIFYD_BASE_URL)/usr/sbin URL_FILE:=netifyd FILE:=netifyd - HASH:=02a8647d779f7dc4506fe9dd10295a06f6f1cb7b701edbfe791e6f319c4d7fd3 + HASH:=a1d8f40f87ba58876c7652dc0f82e699d6f8139793b8ede9d33cea043a693c72 endef $(eval $(call Download,netifyd)) diff --git a/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf b/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf index 61266c775..687f9d690 100644 --- a/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf +++ b/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf @@ -158,5 +158,6 @@ all = include [netlink] # Set the Netlink buffer size buffer_size = 32768 +bridge_pvid_discovery = no # vim: set ft=dosini : diff --git a/packages/ns-api/Makefile b/packages/ns-api/Makefile index 6f09ec1cc..6ea6d583d 100644 --- a/packages/ns-api/Makefile +++ b/packages/ns-api/Makefile @@ -22,6 +22,7 @@ define Package/ns-api TITLE:=NethSecurity REST API URL:=https://github.com/NethServer/nethsecurity-controller/ DEPENDS:= \ + +adblock \ +coreutils-date \ +coreutils-stty \ +python3-idna \ @@ -32,6 +33,7 @@ define Package/ns-api +python3-urllib \ +sshpass \ +wireguard-tools + EXTRA_DEPENDS:=adblock (>=4.5.3) PKGARCH:=all endef @@ -118,8 +120,8 @@ define Package/ns-api/install $(INSTALL_DATA) ./files/ns.mwan.json $(1)/usr/share/rpcd/acl.d/ $(INSTALL_BIN) ./files/ns.dpi $(1)/usr/libexec/rpcd/ $(INSTALL_DATA) ./files/ns.dpi.json $(1)/usr/share/rpcd/acl.d/ - $(INSTALL_BIN) ./files/ns.netdata $(1)/usr/libexec/rpcd/ - $(INSTALL_DATA) ./files/ns.netdata.json $(1)/usr/share/rpcd/acl.d/ + $(INSTALL_BIN) ./files/ns.telegraf $(1)/usr/libexec/rpcd/ + $(INSTALL_DATA) ./files/ns.telegraf.json $(1)/usr/share/rpcd/acl.d/ $(INSTALL_BIN) ./files/ns.storage $(1)/usr/libexec/rpcd/ $(INSTALL_DATA) ./files/ns.storage.json $(1)/usr/share/rpcd/acl.d/ $(INSTALL_BIN) ./files/ns.account $(1)/usr/libexec/rpcd/ @@ -186,7 +188,6 @@ define Package/ns-api/install $(INSTALL_CONF) ./files/config/ns-api $(1)/etc/config/ns-api $(INSTALL_CONF) ./files/config/ns-wizard $(1)/etc/config/ns-wizard $(INSTALL_CONF) ./files/templates $(1)/etc/config/ - $(INSTALL_BIN) ./files/post-commit/restart-netdata.py $(1)/usr/libexec/ns-api/post-commit/ $(INSTALL_BIN) ./files/pre-commit/fix-redirect-reflections.py $(1)/usr/libexec/ns-api/pre-commit $(INSTALL_BIN) ./files/pre-commit/update-objects.py $(1)/usr/libexec/ns-api/pre-commit $(INSTALL_BIN) ./files/post-commit/reload-ipsets.py $(1)/usr/libexec/ns-api/post-commit diff --git a/packages/ns-api/README.md b/packages/ns-api/README.md index 6c2e38672..a0265ba4c 100644 --- a/packages/ns-api/README.md +++ b/packages/ns-api/README.md @@ -150,6 +150,124 @@ Response: } ``` +## ns.telegraf + +Read and update Telegraf ping monitoring targets, query historical metrics stored in VictoriaMetrics, and list the current alerts evaluated by vmalert. + +### get-configuration + +Get the current list of hosts monitored by the Telegraf ping input: +``` +api-cli ns.telegraf get-configuration +``` + +Output example: +```json +{ + "hosts": [ + "1.1.1.1", + "google.com" + ] +} +``` + +### set-hosts + +Set the list of hosts monitored by the Telegraf ping input and restart Telegraf: +``` +api-cli ns.telegraf set-hosts --data '{"hosts": ["1.1.1.1", "8.8.8.8"]}' +``` + +Parameters: +- `hosts`: array of hostnames or IP addresses to monitor + +Output example: +```json +{ + "success": true +} +``` + +### metrics-history + +Return historical system and network metrics collected by Telegraf and stored in VictoriaMetrics: +``` +api-cli ns.telegraf metrics-history --data '{"start": 1746607800, "end": 1746608400, "step": 60}' +``` + +Parameters: +- `start`: start of the time range as Unix timestamp +- `end`: end of the time range as Unix timestamp +- `step`: sampling interval in seconds + +Output example: +```json +{ + "connections": { + "labels": [1746608100], + "datasets": [{ "label": "Connections", "data": [123] }] + }, + "traffic": {}, + "cpu": { + "labels": [1746608100], + "datasets": [{ "label": "CPU (%)", "data": [14.2] }] + }, + "load": { + "labels": [1746608100], + "datasets": [ + { "label": "1m", "data": [0.12] }, + { "label": "5m", "data": [0.08] }, + { "label": "15m", "data": [0.05] } + ] + }, + "diskio": { "labels": [], "datasets": [] }, + "disk": { "labels": [], "datasets": [] }, + "processes": { "labels": [], "datasets": [] }, + "memory": { "labels": [], "datasets": [] }, + "packets": { "labels": [], "datasets": [] }, + "latency_quality": {} +} +``` + +### list-alerts + +List the current pending and firing alerts evaluated by vmalert: +``` +api-cli ns.telegraf list-alerts +``` + +Output example: +```json +{ + "alerts": [ + { + "state": "firing", + "name": "BackupEncryptionDisabled", + "value": "0", + "labels": { + "alertgroup": "backup", + "alertname": "BackupEncryptionDisabled", + "severity": "warning", + "service": "backup" + }, + "annotations": { + "summary_en": "Backup encryption is disabled", + "summary_it": "La cifratura dei backup e disattivata", + "description_en": "The backup passphrase file /etc/backup.pass is missing or empty.", + "description_it": "Il file della passphrase dei backup /etc/backup.pass manca o e vuoto." + }, + "activeAt": "2026-05-07T09:18:00Z", + "expression": "backup_encryption_encrypted == 0", + "source": "http://NethSec:8082/vmalert/alert?group_id=10212661952842894290&alert_id=4214684507782533109" + } + ] +} +``` + +Possible errors: +- `cannot_retrieve_alerts` +- `invalid_alerts_response` + ## ns.firewall ### list-forward-rules @@ -2436,7 +2554,10 @@ Response example: ### traffic-interface -Return an array of point describing the network traffic in the last hour: +Return an array of points describing the network traffic in the last hour. +Data is sourced from Victoria Metrics using `net_bytes_recv` and `net_bytes_sent` Telegraf counters, +converted to kb/s (kilobits per second). Labels are Unix timestamps in descending order (newest first), +with one point every 20 seconds (~180 points total). ``` api-cli ns.dashboard interface-traffic --data '{"interface": "eth0"}' ``` @@ -4504,39 +4625,6 @@ Error response example: {"error": "restart_failed"} ``` -## ns.netdata - -Configure netdata reporting daemon. - -### get-configuration - -Get current netdata configuration: -``` -api-cli ns.netdata get-configuration -``` - -Response example: -```json -{ - "hosts": [ - "1.2.3.4", - "google.it" - ] -} -``` - -### set-hosts - -Configure hosts to be monitored by fping: -``` -api-cli ns.netdata set-hosts --data '{"hosts": ["1.1.1.1", "google.com"]}' -``` - -Response example: -```json -{"result": "success"} -``` - ## ns.factoryreset ### reset @@ -6171,12 +6259,15 @@ Response example: { "data": [ { - "address": "nethesis.it" + "address": "nethesis.it", + "description": "my allow1" } ] } ``` +The allow and block list methods work on UCI-staged data. Changes are visible immediately through the API and are written to `/etc/adblock/adblock.allowlist` and `/etc/adblock/adblock.blocklist` during the next adblock reload triggered by `ns.commit` or `reload_config`. + ### dns-add-allowed Add a domain which is always allowed: @@ -7932,7 +8023,7 @@ Output example: ### latency-and-quality-report -Report latency metrics (minimum, maximum and average) and connectivy quality data (packet delivery rate) for every host configured in Netdata fping configuration file, located at `/etc/netdata/fping.conf`. +Report latency metrics (minimum, maximum and average) and connectivity quality data (packet loss percentage) for every host configured in the Telegraf ping plugin configuration file, located at `/etc/telegraf.conf.d/ping.conf`. Usage example: ``` api-cli ns.report latency-and-quality-report @@ -7982,7 +8073,7 @@ Output example: ], [ 1731485262, - 99.8152174 + 100 ], [ 1731484894, @@ -8032,7 +8123,7 @@ Output example: ], [ 1731485262, - 99.8152174 + 100 ], [ 1731484894, diff --git a/packages/ns-api/files/ns.dashboard b/packages/ns-api/files/ns.dashboard index 449fec46e..2a98ddae7 100644 --- a/packages/ns-api/files/ns.dashboard +++ b/packages/ns-api/files/ns.dashboard @@ -12,6 +12,8 @@ import os import sys import json import subprocess +import time +import urllib.parse import urllib.request from euci import EUci from nethsec import utils, ovpn @@ -154,7 +156,7 @@ def check_adblock(): if not adb_enabled: return "disabled" pa = subprocess.run(["service", "adblock", "status"], check=False, capture_output=True, text=True) - if adb_enabled and re.search('adblock_status\s+:\s+enabled', pa.stdout): + if adb_enabled and re.search(r'adblock_status\s+:\s+enabled', pa.stdout): return "ok" else: return "error" @@ -165,7 +167,7 @@ def check_banip(): if not bip_enabled: return "disabled" pa = subprocess.run(["service", "banip", "status"], check=False, capture_output=True, text=True) - if bip_enabled and re.search('status\s+:\s+active', pa.stdout): + if bip_enabled and re.search(r'status\s+:\s+active', pa.stdout): return "ok" def check_ts_ip(): @@ -274,17 +276,24 @@ def system_info(): def interface_traffic(interface): ret = {"labels": [], "data": []} - # retrieve from netdata the traffic for the last hour - url = f'http://127.0.0.1:19999/api/v1/data?chart=net.{interface}&after=-3600&points=180&options=abs' - try: - with urllib.request.urlopen(url, timeout=10) as fu: - data = json.loads(fu.read()) - except: - return ret + vm_url = "http://127.0.0.1:8428/api/v1/query_range" + now = int(time.time()) + one_hour_ago = now - 3600 + + def vm_query(expr): + params = urllib.parse.urlencode({"query": expr, "start": one_hour_ago, "end": now, "step": 20}) + with urllib.request.urlopen(f"{vm_url}?{params}", timeout=5) as resp: + data = json.loads(resp.read()) + result = data.get("data", {}).get("result", []) + return result[0].get("values", []) if result else [] - for record in data["data"]: - ret["labels"].append(record[0]) - ret["data"].append([record[1], record[2]]) + try: + recv = vm_query(f'rate(net_bytes_recv{{interface="{interface}"}}[20s]) * 8 / 1000') + sent = vm_query(f'rate(net_bytes_sent{{interface="{interface}"}}[20s]) * 8 / 1000') + ret["labels"] = [int(ts) for ts, _ in reversed(recv)] + ret["data"] = [[float(r), float(s)] for (_, r), (_, s) in zip(reversed(recv), reversed(sent))] + except Exception: + pass return ret diff --git a/packages/ns-api/files/ns.nathelpers b/packages/ns-api/files/ns.nathelpers index c674078ac..15afe300e 100755 --- a/packages/ns-api/files/ns.nathelpers +++ b/packages/ns-api/files/ns.nathelpers @@ -58,12 +58,12 @@ DEFAULT_PARAMS = { def get_nat_helper_names(): nat_helpers = [] - proc = subprocess.run("/bin/opkg files kmod-nf-nathelper | grep -e '\.ko$' | cut -d'/' -f 5 | cut -d'.' -f1", shell=True, check=True, + proc = subprocess.run("/bin/opkg files kmod-nf-nathelper | grep -e '\\.ko$' | cut -d'/' -f 5 | cut -d'.' -f1", shell=True, check=True, capture_output=True, text=True) nat_helpers = proc.stdout.splitlines() nat_helpers_extra = [] - proc = subprocess.run("/bin/opkg files kmod-nf-nathelper-extra | grep -e '\.ko$' | cut -d'/' -f 5 | cut -d'.' -f1", shell=True, check=True, + proc = subprocess.run("/bin/opkg files kmod-nf-nathelper-extra | grep -e '\\.ko$' | cut -d'/' -f 5 | cut -d'.' -f1", shell=True, check=True, capture_output=True, text=True) nat_helpers_extra = proc.stdout.splitlines() return nat_helpers + nat_helpers_extra diff --git a/packages/ns-api/files/ns.netdata b/packages/ns-api/files/ns.netdata deleted file mode 100755 index bb2309181..000000000 --- a/packages/ns-api/files/ns.netdata +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/python3 - -# -# Copyright (C) 2023 Nethesi3 S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -# Read and set fping configuration for netdata - -import os -import sys -import json -import subprocess -import configparser - -fping_conf_file = "/etc/netdata/fping.conf" -netdata_conf_file = "/etc/netdata/netdata.conf" - -def get_config(): - hosts = [] - # create a simpligied fping.conf if not exists - # the file must contain only one line: hosts="" - if not os.path.exists(fping_conf_file): - with open(fping_conf_file, 'w') as fp: - fp.write('hosts=""\n') - # parse the simplified config file - try: - with open(fping_conf_file, 'r') as fp: - line = fp.readline() - line = line[7:-2] - hosts = line.split(" ") - except: - pass - return {"hosts": hosts} - -def set_config(config): - # Enable and disable fping plugin on netdata - nparser = configparser.ConfigParser() - nparser.read(netdata_conf_file) - if len(config['hosts']) > 0: - nparser['plugins']['fping'] = 'yes' - else: - nparser['plugins']['fping'] = 'no' - with open(netdata_conf_file, 'w') as fpc: - nparser.write(fpc) - - try: - with open(fping_conf_file, 'w') as fp: - hosts = " ".join(config['hosts']) - fp.write(f'hosts="{hosts}"\n') - subprocess.run(["/etc/init.d/netdata", "restart"], check=True) - return {"success": True} - except: - return {"success": False} - -cmd = sys.argv[1] - -if cmd == 'list': - print(json.dumps({"get-configuration": {}, "set-hosts": {"hosts": ["1.1.1.1", "google.com"]}})) -else: - action = sys.argv[2] - if action == "get-configuration": - print(json.dumps(get_config())) - elif action == "set-hosts": - args = json.loads(sys.stdin.read()) - print(json.dumps(set_config(args))) diff --git a/packages/ns-api/files/ns.netdata.json b/packages/ns-api/files/ns.netdata.json deleted file mode 100644 index 5764ef6d4..000000000 --- a/packages/ns-api/files/ns.netdata.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "netdata-manager": { - "description": "Read and set netdata configuration", - "write": {}, - "read": { - "ubus": { - "ns.netdata": [ - "*" - ] - } - } - } -} diff --git a/packages/ns-api/files/ns.ovpnrw b/packages/ns-api/files/ns.ovpnrw index 147b7e86e..a43e24e10 100755 --- a/packages/ns-api/files/ns.ovpnrw +++ b/packages/ns-api/files/ns.ovpnrw @@ -63,8 +63,7 @@ def add_tap_to_bridge(u, bridge, interface): u.save("network") except: pass - finally: - return + return def remove_tap_from_bridge(u, bridge, interface): if bridge is None or interface is None: @@ -85,8 +84,7 @@ def remove_tap_from_bridge(u, bridge, interface): u.save("network") except: pass - finally: - return + return def get_ip_and_mask(bridge): u = EUci() diff --git a/packages/ns-api/files/ns.redirects b/packages/ns-api/files/ns.redirects index 8356c065c..4af2f2837 100755 --- a/packages/ns-api/files/ns.redirects +++ b/packages/ns-api/files/ns.redirects @@ -21,7 +21,7 @@ def get_services(): line = line.strip() if not line: continue - tmp = re.split("\s+", line) + tmp = re.split(r"\s+", line) port = tmp[1][0:tmp[1].index("/")] services[port] = tmp[0] return services diff --git a/packages/ns-api/files/ns.report b/packages/ns-api/files/ns.report index 1eca1eca1..47dc1c315 100755 --- a/packages/ns-api/files/ns.report +++ b/packages/ns-api/files/ns.report @@ -15,6 +15,7 @@ import subprocess from datetime import datetime from collections import defaultdict from nethsec import utils +import urllib.parse import urllib.request from euci import EUci @@ -324,41 +325,51 @@ def ovpnrw_bytes_by_hour_and_user(instance, day, user): return {"hours": hours_bytes} -def get_fping_hosts(): - # read fping hosts from /etc/netdata/fping.conf - try: - with open("/etc/netdata/fping.conf", 'r') as fp: - line = fp.readline() - line = line[7:-2] - hosts = line.split(" ") - return hosts - except: - return [] - - -def get_netdata_chart_data(chart_name): - ret = {"labels": [], "data": []} - # retrieve chart data from netdata - url = f'http://127.0.0.1:19999/api/v1/data?chart={chart_name}&after=-3600&points=180&options=abs' +def get_victoria_metrics_ping_data(host): + """ + Query Victoria Metrics for ping metrics. + Returns: {"latency": {"labels": [...], "data": [...]}, "quality": {"labels": [...], "data": [...]}} + """ + ret_latency = {"labels": ["time", "minimum", "maximum", "average"], "data": []} + ret_quality = {"labels": ["time", "returned"], "data": []} + + vm_url = "http://127.0.0.1:8428/api/v1/query_range" + now = int(time.time()) + one_hour_ago = now - 3600 + timeout = 5 + + def vm_query(metric_expr): + params = urllib.parse.urlencode({'query': metric_expr, 'start': one_hour_ago, 'end': now, 'step': 20}) + with urllib.request.urlopen(f"{vm_url}?{params}", timeout=timeout) as resp: + data = json.loads(resp.read()) + result = data.get('data', {}).get('result', []) + return result[0].get('values', []) if result else [] + try: - with urllib.request.urlopen(url, timeout=10) as fu: - data = json.loads(fu.read()) - except: - return ret - return data + min_values = vm_query(f'ping_minimum_response_ms{{url="{host}"}}') + max_values = vm_query(f'ping_maximum_response_ms{{url="{host}"}}') + avg_values = vm_query(f'ping_average_response_ms{{url="{host}"}}') + + ret_latency["data"] = [ + [int(ts), float(mn), float(mx), float(av)] + for (ts, mn), (_, mx), (_, av) in zip(min_values, max_values, avg_values) + ] + + loss_values = vm_query(f'100 - ping_percent_packet_loss{{url="{host}"}} or 100 - ping_percent_reply_loss{{url="{host}"}}') + ret_quality["data"] = [[int(ts), float(val)] for ts, val in loss_values] + + except Exception as e: + print(f"Error querying Victoria Metrics for {host}: {str(e)}", file=sys.stderr) + + return {"latency": ret_latency, "quality": ret_quality} def latency_and_quality_report(): - hosts = get_fping_hosts() + e_uci = EUci() + hosts = e_uci.get('telegraf', 'internet', 'pings', dtype=str, list=True, default=[]) ret = {} for host in hosts: - host_replaced = host.replace('.', '_') - latency_chart_data = get_netdata_chart_data(f'fping.{host_replaced}_latency') - quality_chart_data = get_netdata_chart_data(f'fping.{host_replaced}_quality') - ret[host] = { - "latency": latency_chart_data, - "quality": quality_chart_data - } + ret[host] = get_victoria_metrics_ping_data(host) return ret diff --git a/packages/ns-api/files/ns.telegraf b/packages/ns-api/files/ns.telegraf new file mode 100755 index 000000000..cd18a11cc --- /dev/null +++ b/packages/ns-api/files/ns.telegraf @@ -0,0 +1,384 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Read and set ping configuration for telegraf, and expose metrics history from VictoriaMetrics + +import os +import sys +import json +import subprocess +import re +import time +import ipaddress +import urllib.error +import urllib.parse +import urllib.request +from euci import EUci +from nethsec.utils import validation_error + + +def _is_valid_ip_or_hostname(host): + """Validate if a string is a valid IPv4, IPv6, or hostname.""" + if not isinstance(host, str) or not host.strip(): + return False + + # Try to parse as IP address (IPv4 or IPv6) + try: + ipaddress.ip_address(host) + return True + except ValueError: + pass + + # Hostname pattern (alphanumeric, dots, hyphens) + hostname_pattern = r'^([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?)(\.([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?))*$' + return bool(re.match(hostname_pattern, host)) + + +def _read_ping_hosts(): + """Read the list of monitored ping hosts from UCI config.""" + e_uci = EUci() + pings = e_uci.get('telegraf', 'internet', 'pings', dtype=str, list=True, default=[]) + return pings + + +def get_config(): + return {"hosts": _read_ping_hosts()} + + +def set_config(config): + if 'hosts' not in config: + return validation_error('hosts', 'required') + + hosts = config.get('hosts') + if not isinstance(hosts, list): + return validation_error('hosts', 'invalid') + + # Validate each host with per-index error reporting + for idx, host in enumerate(hosts): + if not _is_valid_ip_or_hostname(host): + return validation_error(f'hosts.{idx}', 'invalid', value=host) + + e_uci = EUci() + current = set(e_uci.get('telegraf', 'internet', 'pings', dtype=str, list=True, default=[])) + if current != set(hosts): + e_uci.set('telegraf', 'internet', 'pings', hosts) + e_uci.save('telegraf') + return {"success": True} + + +VM_URL = "http://127.0.0.1:8428/api/v1/query_range" +VM_ALERTS_URL = "http://127.0.0.1:8082/api/v1/alerts" + + +def _get_interface_to_zone_map(): + """Map physical interface names to firewall zone names and devices. + Returns: {"eth1": {"zone": "wan", "device": "eth1"}, "br-lan": {"zone": "lan", "device": "br-lan"}} + """ + interface_map = {} + try: + u = EUci() + zones = u.get("firewall") + networks = u.get("network") + + if not zones or not networks: + return interface_map + + # First, build a map of network names to devices + network_to_device = {} + for net_name, net_config in networks.items(): + if isinstance(net_config, dict): + device = net_config.get("device") or net_config.get("ifname") + if device: + network_to_device[net_name] = device + + # Now map zones to networks and then to devices + for zone_name, zone_config in zones.items(): + if isinstance(zone_config, dict) and "name" in zone_config: + zone_label = zone_config.get("name", zone_name) + networks_list = zone_config.get("network", ()) + + # Convert to list if needed + if isinstance(networks_list, str): + networks_list = [networks_list] + elif not isinstance(networks_list, (list, tuple)): + networks_list = [] + + for network in networks_list: + network = str(network).strip() + if network in network_to_device: + device = network_to_device[network] + interface_map[device] = {"zone": zone_label, "device": device} + except Exception: + pass + + return interface_map + + +def _vm_query(expr, start, end, step): + """Execute a single PromQL range query against VictoriaMetrics.""" + params = urllib.parse.urlencode( + {"query": expr, "start": start, "end": end, "step": step} + ) + with urllib.request.urlopen(f"{VM_URL}?{params}", timeout=10) as resp: + data = json.loads(resp.read()) + return data.get("data", {}).get("result", []) + + +def _single_series(results): + """Extract timestamps and values from a single-series result.""" + if not results: + return {"labels": [], "data": []} + values = results[0].get("values", []) + return { + "labels": [int(ts) for ts, _ in values], + "data": [float(v) for _, v in values], + } + + +def list_alerts(): + try: + with urllib.request.urlopen(VM_ALERTS_URL, timeout=10) as resp: + data = json.loads(resp.read()) + except (TimeoutError, urllib.error.URLError): + return {"error": "cannot_retrieve_alerts"} + except json.JSONDecodeError: + return {"error": "invalid_alerts_response"} + + alerts = data.get("data", {}).get("alerts") + if not isinstance(alerts, list): + return {"error": "invalid_alerts_response"} + + return {"alerts": alerts} + + +def metrics_history(args): + now = int(time.time()) + start = int(args.get("start", now - 86400)) + end = int(args.get("end", now)) + step = int(args.get("step", 300)) + + def q(expr): + try: + return _vm_query(expr, start, end, step) + except Exception: + return [] + + def single(expr): + return _single_series(q(expr)) + + # Connections (conntrack) + s = single("conntrack_ip_conntrack_count") + connections = { + "labels": s["labels"], + "datasets": [{"label": "Connections", "data": s["data"]}], + } + + # Traffic per interface – exclude loopback and intermediate functional blocks (ifb*) + # Organize traffic by interface with zone name labels + interface_map = _get_interface_to_zone_map() + recv_results = q('rate(net_bytes_recv{interface!~"lo|ifb.*"}[5m])') + sent_results = q('rate(net_bytes_sent{interface!~"lo|ifb.*"}[5m])') + + traffic_labels: list = [] + for r in recv_results + sent_results: + if r.get("values"): + traffic_labels = [int(ts) for ts, _ in r["values"]] + break + + # Build traffic data organized by interface + traffic_by_interface = {} + for r in recv_results: + iface = r.get("metric", {}).get("interface", "unknown") + iface_info = interface_map.get(iface, {"zone": iface, "device": iface}) + zone_key = f"{iface_info['zone']}|{iface_info['device']}" # Use composite key + if zone_key not in traffic_by_interface: + traffic_by_interface[zone_key] = { + "labels": traffic_labels, + "datasets": [], + "zone": iface_info["zone"], + "device": iface_info["device"], + } + traffic_by_interface[zone_key]["datasets"].append( + {"label": "Download", "data": [float(v) for _, v in r.get("values", [])]} + ) + + for r in sent_results: + iface = r.get("metric", {}).get("interface", "unknown") + iface_info = interface_map.get(iface, {"zone": iface, "device": iface}) + zone_key = f"{iface_info['zone']}|{iface_info['device']}" + if zone_key not in traffic_by_interface: + traffic_by_interface[zone_key] = { + "labels": traffic_labels, + "datasets": [], + "zone": iface_info["zone"], + "device": iface_info["device"], + } + traffic_by_interface[zone_key]["datasets"].append( + {"label": "Upload", "data": [float(v) for _, v in r.get("values", [])]} + ) + + # CPU usage (%) + s = single('100 - (avg(cpu_usage_idle{cpu="cpu-total"}))') + cpu = {"labels": s["labels"], "datasets": [{"label": "CPU (%)", "data": s["data"]}]} + + # System load (1m / 5m / 15m) + s1 = single("system_load1") + s5 = single("system_load5") + s15 = single("system_load15") + load = { + "labels": s1["labels"], + "datasets": [ + {"label": "1m", "data": s1["data"]}, + {"label": "5m", "data": s5["data"]}, + {"label": "15m", "data": s15["data"]}, + ], + } + + # Disk I/O – sum across all non-loop block devices + s_read = single('sum(rate(diskio_read_bytes{name!~"loop.*"}[5m]))') + s_write = single('sum(rate(diskio_write_bytes{name!~"loop.*"}[5m]))') + diskio = { + "labels": s_read["labels"], + "datasets": [ + {"label": "Read", "data": s_read["data"]}, + {"label": "Write", "data": s_write["data"]}, + ], + } + + # Disk usage per real partition (exclude virtual filesystems) + disk_results = q( + 'disk_used_percent{fstype!~"tmpfs|cgroup2|devtmpfs|sysfs|proc|overlay|squashfs"}' + ) + if disk_results: + disk_labels = [int(ts) for ts, _ in disk_results[0].get("values", [])] + disk_datasets = [ + { + "label": r.get("metric", {}).get("path", "unknown"), + "data": [float(v) for _, v in r.get("values", [])], + } + for r in disk_results + ] + disk = {"labels": disk_labels, "datasets": disk_datasets} + else: + disk = {"labels": [], "datasets": []} + + # Total processes + s = single("processes_total") + processes = { + "labels": s["labels"], + "datasets": [{"label": "Processes", "data": s["data"]}], + } + + # RAM usage: Used and Free (MB) + s_used = single("mem_used / 1048576") + s_free = single("mem_free / 1048576") + memory = { + "labels": s_used["labels"], + "datasets": [ + {"label": "Used", "data": s_used["data"]}, + {"label": "Free", "data": s_free["data"]}, + ], + } + + # Packets Rx/Tx – sum across all interfaces + s_rx = single("sum(rate(net_packets_recv[5m]))") + s_tx = single("sum(rate(net_packets_sent[5m]))") + packets = { + "labels": s_rx["labels"], + "datasets": [ + {"label": "Rx", "data": s_rx["data"]}, + {"label": "Tx", "data": s_tx["data"]}, + ], + } + + # Latency and quality reports for monitored hosts + latency_quality = latency_and_quality_report(start, end, step) + + return { + "connections": connections, + "traffic": traffic_by_interface, + "cpu": cpu, + "load": load, + "diskio": diskio, + "disk": disk, + "processes": processes, + "memory": memory, + "packets": packets, + "latency_quality": latency_quality, + } + + +def latency_and_quality_report(start, end, step): + """Fetch latency and packet loss data for monitored hosts.""" + + def q_values(expr): + try: + result = _vm_query(expr, start, end, step) + return result[0].get("values", []) if result else [] + except Exception: + return [] + + ret = {} + for host in _read_ping_hosts(): + latency_data = {"labels": [], "datasets": []} + quality_data = {"labels": [], "datasets": []} + + try: + min_values = q_values(f'ping_minimum_response_ms{{url="{host}"}}') + max_values = q_values(f'ping_maximum_response_ms{{url="{host}"}}') + avg_values = q_values(f'ping_average_response_ms{{url="{host}"}}') + loss_values = q_values( + f'100 - ping_percent_packet_loss{{url="{host}"}} or 100 - ping_percent_reply_loss{{url="{host}"}}' + ) + + if min_values and max_values and avg_values: + labels = [int(ts) for ts, _ in min_values] + latency_data["labels"] = labels + latency_data["datasets"] = [ + {"label": "Min", "data": [float(v) for _, v in min_values]}, + {"label": "Avg", "data": [float(v) for _, v in avg_values]}, + {"label": "Max", "data": [float(v) for _, v in max_values]}, + ] + + if loss_values: + quality_data["labels"] = [int(ts) for ts, _ in loss_values] + quality_data["datasets"] = [ + {"label": "Delivery %", "data": [float(v) for _, v in loss_values]}, + ] + except Exception: + pass + + ret[host] = {"latency": latency_data, "quality": quality_data} + + return ret + + +cmd = sys.argv[1] + +if cmd == "list": + print( + json.dumps( + { + "get-configuration": {}, + "set-hosts": {"hosts": ["1.1.1.1", "google.com"]}, + "metrics-history": {"start": 0, "end": 0, "step": 300}, + "list-alerts": {}, + } + ) + ) +else: + action = sys.argv[2] + if action == "get-configuration": + print(json.dumps(get_config())) + elif action == "set-hosts": + args = json.loads(sys.stdin.read()) + print(json.dumps(set_config(args))) + elif action == "metrics-history": + args = json.loads(sys.stdin.read()) + print(json.dumps(metrics_history(args))) + elif action == "list-alerts": + print(json.dumps(list_alerts())) diff --git a/packages/ns-api/files/ns.telegraf.json b/packages/ns-api/files/ns.telegraf.json new file mode 100644 index 000000000..972119685 --- /dev/null +++ b/packages/ns-api/files/ns.telegraf.json @@ -0,0 +1,13 @@ +{ + "telegraf-manager": { + "description": "Read and set telegraf ping monitor configuration", + "write": {}, + "read": { + "ubus": { + "ns.telegraf": [ + "*" + ] + } + } + } +} diff --git a/packages/ns-api/files/ns.threatshield b/packages/ns-api/files/ns.threatshield index be8ed3de1..7ed3e3638 100644 --- a/packages/ns-api/files/ns.threatshield +++ b/packages/ns-api/files/ns.threatshield @@ -101,14 +101,28 @@ def write_allow_list(allow_list, file='/etc/banip/banip.allowlist'): f.write('\n') subprocess.run(["/etc/init.d/banip", "reload"], capture_output=True) -def dns_write_local_list(local_list, file): - with open(file, 'w') as f: - for x in local_list: - f.write(x['address']) - if x['description']: - f.write(' #' + x['description']) - f.write('\n') - subprocess.run(["/etc/init.d/adblock", "restart"], capture_output=True) +def dns_get_local_list(e_uci, list_type): + values = e_uci.get('adblock', 'ns_lists', list_type, list=True, default=[]) + ret = [] + for value in values: + parts = value.split('#', 1) + ret.append({'address': parts[0].strip(), 'description': parts[1].strip() if len(parts) > 1 else ''}) + return ret + +def dns_write_local_list(e_uci, local_list, list_type): + option = 'allowlist' if list_type == 'allowlist' else 'blocklist' + e_uci.set('adblock', 'ns_lists', 'ns_lists') + + values = tuple( + f"{entry['address']} #{entry['description']}" if entry.get('description') else entry['address'] + for entry in local_list + ) + if values: + e_uci.set('adblock', 'ns_lists', option, values) + elif e_uci.get('adblock', 'ns_lists', option, list=True, default=[]): + e_uci.delete('adblock', 'ns_lists', option) + + e_uci.save('adblock') def write_block_list(block_list): write_allow_list(block_list, '/etc/banip/banip.blocklist') @@ -120,17 +134,24 @@ def read_gz(file): else: return {} -def list_dns_feeds(enterprise=False): - # Decompress and read the JSON file /etc/adblock/combined.sources.gz - ret = {} - sources = '/etc/adblock/combined.sources.gz' - if not os.path.exists(sources): - ret = read_gz('/usr/share/threat_shield/community-dns.sources.gz') - if enterprise: - ret.update(read_gz('/usr/share/threat_shield/nethesis-dns.sources.gz')) - else: - ret = read_gz(sources) +def read_json(file): + if os.path.exists(file) and os.path.getsize(file) > 0: + with open(file, 'r') as f: + try: + return json.load(f) + except Exception: + return {} + return {} +def list_dns_feeds(enterprise=False): + # Read feeds from adblock.feeds (builtin) and adblock.custom.feeds (NethSecurity/user overrides) + ret = read_json('/etc/adblock/adblock.feeds') + custom = read_json('/etc/adblock/adblock.custom.feeds') + if custom: + ret.update(custom) + elif enterprise: + # fallback: merge nethesis sources gz if no custom feeds file yet + ret.update(read_gz('/usr/share/threat_shield/nethesis-dns.sources.gz')) return ret def get_confidence(f, enterprise=False): @@ -391,7 +412,7 @@ def dns_list_blocklist(e_uci): has_bl = has_bl_entitlement(e_uci) feeds = list_dns_feeds(has_bl) try: - enabled_feeds = list(e_uci.get_all('adblock', 'global', 'adb_sources')) + enabled_feeds = list(e_uci.get_all('adblock', 'global', 'adb_feed')) except: enabled_feeds = [] for f in feeds: @@ -413,7 +434,7 @@ def dns_list_blocklist(e_uci): def dns_edit_blocklist(e_uci, payload): try: - enabled = list(e_uci.get_all('adblock', 'global', 'adb_sources')) + enabled = list(e_uci.get_all('adblock', 'global', 'adb_feed')) except: enabled = [] if payload['enabled'] and payload['blocklist'] not in enabled: @@ -424,7 +445,7 @@ def dns_edit_blocklist(e_uci, payload): has_bl = has_bl_entitlement(e_uci) feeds = list_dns_feeds(has_bl) enabled = [feed for feed in enabled if feed in feeds] - e_uci.set('adblock', 'global', 'adb_sources', enabled) + e_uci.set('adblock', 'global', 'adb_feed', enabled) e_uci.save('adblock') return {'message': 'success'} @@ -438,11 +459,11 @@ def dns_list_zones(e_uci): def dns_list_settings(e_uci): ts_enabled = e_uci.get('adblock', 'global', 'ts_enabled', default='0') try: - zones = list(e_uci.get_all('adblock', 'global', 'adb_zonelist')) + zones = list(e_uci.get_all('adblock', 'global', 'adb_nftdevforce')) except: zones = ['lan'] try: - ports = list(e_uci.get_all('adblock', 'global', 'adb_portlist')) + ports = list(e_uci.get_all('adblock', 'global', 'adb_nftportforce')) except: ports = ['53', '853'] return { 'data': {'enabled': ts_enabled == '1', "zones": zones, "ports": ports} } @@ -453,65 +474,64 @@ def dns_edit_settings(e_uci, payload): raise ValidationError('zones', 'wan_zone_not_allowed', payload['zones']) e_uci.set('adblock', 'global', 'ts_enabled', '1') e_uci.set('adblock', 'global', 'adb_enabled', '1') - e_uci.set('adblock', 'global', 'adb_backup', '1') - e_uci.set('adblock', 'global', 'adb_forcedns', '1') - e_uci.set('adblock', 'global', 'adb_zonelist', payload.get('zones', ['lan'])) - e_uci.set('adblock', 'global', 'adb_portlist', payload.get('ports', ['53', '853'])) + e_uci.set('adblock', 'global', 'adb_nftforce', '1') + e_uci.set('adblock', 'global', 'adb_nftdevforce', payload.get('zones', ['lan'])) + e_uci.set('adblock', 'global', 'adb_nftportforce', payload.get('ports', ['53', '853'])) else: e_uci.set('adblock', 'global', 'ts_enabled', '0') e_uci.set('adblock', 'global', 'adb_enabled', '0') - e_uci.set('adblock', 'global', 'adb_forcedns', '0') + e_uci.set('adblock', 'global', 'adb_nftforce', '0') e_uci.save('adblock') return {'message': 'success'} -def dns_list_allowed(): - return { "data": get_allow_list('/etc/adblock/adblock.whitelist') } +def dns_list_allowed(e_uci): + return { "data": dns_get_local_list(e_uci, 'allowlist') } -def dns_list_blocked(): - return { "data": get_allow_list('/etc/adblock/adblock.blacklist') } +def dns_list_blocked(e_uci): + return { "data": dns_get_local_list(e_uci, 'blocklist') } -def dns_add_allowed(payload): - cur = get_allow_list('/etc/adblock/adblock.whitelist') +def dns_add_allowed(e_uci, payload): + cur = dns_get_local_list(e_uci, 'allowlist') # extract address from cur list if payload['address'] in [x['address'] for x in cur]: raise ValidationError('address', 'address_already_present', payload['address']) cur.append({ "address": payload['address'], "description": payload['description'] }) - dns_write_local_list(cur, '/etc/adblock/adblock.whitelist') + dns_write_local_list(e_uci, cur, 'allowlist') return {'message': 'success'} -def dns_add_blocked(payload): - cur = get_allow_list('/etc/adblock/adblock.blacklist') +def dns_add_blocked(e_uci, payload): + cur = dns_get_local_list(e_uci, 'blocklist') # extract address from cur list if payload['address'] in [x['address'] for x in cur]: raise ValidationError('address', 'address_already_present', payload['address']) cur.append({ "address": payload['address'], "description": payload.get('description') }) - dns_write_local_list(cur, '/etc/adblock/adblock.blacklist') + dns_write_local_list(e_uci, cur, 'blocklist') return {'message': 'success'} -def dns_edit_allowed(payload): - cur = get_allow_list('/etc/adblock/adblock.whitelist') +def dns_edit_allowed(e_uci, payload): + cur = dns_get_local_list(e_uci, 'allowlist') if payload['address'] not in [x['address'] for x in cur]: raise ValidationError('address', 'address_not_found', payload['address']) for i in range(len(cur)): if cur[i]['address'] == payload['address']: cur[i]['description'] = payload['description'] break - dns_write_local_list(cur, '/etc/adblock/adblock.whitelist') + dns_write_local_list(e_uci, cur, 'allowlist') return {'message': 'success'} -def dns_edit_blocked(payload): - cur = get_allow_list('/etc/adblock/adblock.blacklist') +def dns_edit_blocked(e_uci, payload): + cur = dns_get_local_list(e_uci, 'blocklist') if payload['address'] not in [x['address'] for x in cur]: raise ValidationError('address', 'address_not_found', payload['address']) for i in range(len(cur)): if cur[i]['address'] == payload['address']: cur[i]['description'] = payload.get('description') break - dns_write_local_list(cur, '/etc/adblock/adblock.blacklist') + dns_write_local_list(e_uci, cur, 'blocklist') return {'message': 'success'} -def dns_delete_allowed(payload): - cur = get_allow_list('/etc/adblock/adblock.whitelist') +def dns_delete_allowed(e_uci, payload): + cur = dns_get_local_list(e_uci, 'allowlist') if payload['address'] not in [x['address'] for x in cur]: raise ValidationError('address', 'address_not_found', payload['address']) # remove address from cur list @@ -519,11 +539,11 @@ def dns_delete_allowed(payload): if cur[i]['address'] == payload['address']: del cur[i] break - dns_write_local_list(cur, '/etc/adblock/adblock.whitelist') + dns_write_local_list(e_uci, cur, 'allowlist') return {'message': 'success'} -def dns_delete_blocked(payload): - cur = get_allow_list('/etc/adblock/adblock.blacklist') +def dns_delete_blocked(e_uci, payload): + cur = dns_get_local_list(e_uci, 'blocklist') if payload['address'] not in [x['address'] for x in cur]: raise ValidationError('address', 'address_not_found', payload['address']) # remove address from cur list @@ -531,38 +551,38 @@ def dns_delete_blocked(payload): if cur[i]['address'] == payload['address']: del cur[i] break - dns_write_local_list(cur, '/etc/adblock/adblock.blacklist') + dns_write_local_list(e_uci, cur, 'blocklist') return {'message': 'success'} def dns_list_bypass(e_uci): - # adblock.global.adb_bypass + # adblock.global.ns_tsdns_bypass try: - bypass = e_uci.get_all('adblock', 'global', 'adb_bypass') + bypass = e_uci.get_all('adblock', 'global', 'ns_tsdns_bypass') except: bypass = [] return { "data": bypass } def dns_add_bypass(e_uci, payload): try: - bypass = list(e_uci.get_all('adblock', 'global', 'adb_bypass')) + bypass = list(e_uci.get_all('adblock', 'global', 'ns_tsdns_bypass')) except: bypass = [] if payload['address'] in bypass: raise ValidationError('address', 'address_already_present', payload['address']) bypass.append(payload['address']) - e_uci.set('adblock', 'global', 'adb_bypass', bypass) + e_uci.set('adblock', 'global', 'ns_tsdns_bypass', bypass) e_uci.save('adblock') return {'message': 'success'} def dns_delete_bypass(e_uci, payload): try: - bypass = list(e_uci.get_all('adblock', 'global', 'adb_bypass')) + bypass = list(e_uci.get_all('adblock', 'global', 'ns_tsdns_bypass')) except: bypass = [] if payload['address'] not in bypass: raise ValidationError('address', 'address_not_found', payload['address']) bypass.remove(payload['address']) - e_uci.set('adblock', 'global', 'adb_bypass', bypass) + e_uci.set('adblock', 'global', 'ns_tsdns_bypass', bypass) e_uci.save('adblock') return {'message': 'success'} @@ -803,15 +823,15 @@ elif cmd == 'call': ret = dns_list_zones(e_uci) elif action == 'dns-add-allowed': payload = json.loads(sys.stdin.read()) - ret = dns_add_allowed(payload) + ret = dns_add_allowed(e_uci, payload) elif action == 'dns-edit-allowed': payload = json.loads(sys.stdin.read()) - ret = dns_edit_allowed(payload) + ret = dns_edit_allowed(e_uci, payload) elif action == 'dns-list-allowed': - ret = dns_list_allowed() + ret = dns_list_allowed(e_uci) elif action == 'dns-delete-allowed': payload = json.loads(sys.stdin.read()) - ret = dns_delete_allowed(payload) + ret = dns_delete_allowed(e_uci, payload) elif action == 'dns-list-bypass': ret = dns_list_bypass(e_uci) elif action == 'dns-add-bypass': @@ -821,16 +841,16 @@ elif cmd == 'call': payload = json.loads(sys.stdin.read()) ret = dns_delete_bypass(e_uci, payload) elif action == 'dns-list-blocked': - ret = dns_list_blocked() + ret = dns_list_blocked(e_uci) elif action == 'dns-add-blocked': payload = json.loads(sys.stdin.read()) - ret = dns_add_blocked(payload) + ret = dns_add_blocked(e_uci, payload) elif action == 'dns-edit-blocked': payload = json.loads(sys.stdin.read()) - ret = dns_edit_blocked(payload) + ret = dns_edit_blocked(e_uci, payload) elif action == 'dns-delete-blocked': payload = json.loads(sys.stdin.read()) - ret = dns_delete_blocked(payload) + ret = dns_delete_blocked(e_uci, payload) print(json.dumps(ret)) except ValidationError as ex: diff --git a/packages/ns-api/files/ns.update b/packages/ns-api/files/ns.update index b1fef000d..9121fdffd 100755 --- a/packages/ns-api/files/ns.update +++ b/packages/ns-api/files/ns.update @@ -86,7 +86,7 @@ def check_system_update(): response = requests.get(f"{url}/latest_release", headers={"Accept": "application/json"}, timeout=5) response.raise_for_status() version = response.text.strip() - if semver.compare(version, current_version) > 0: + if semver.Version.compare(version, current_version) > 0: data["lastVersion"] = f'NethSecurity {version}' except requests.exceptions.ConnectionError: return utils.generic_error("connection_error") diff --git a/packages/ns-api/files/post-commit/restart-netdata.py b/packages/ns-api/files/post-commit/restart-netdata.py deleted file mode 100755 index ae6a7017b..000000000 --- a/packages/ns-api/files/post-commit/restart-netdata.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/python - -# -# Copyright (C) 2024 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -# This script restarts netdata is a WAN has changed to update the multiwan chart. - -import subprocess - -# The changes variable is already within the scope from the caller -if 'mwan3' in changes: - subprocess.run(["/etc/init.d/netdata", "restart"]) diff --git a/packages/ns-api/openapi.yml b/packages/ns-api/openapi.yml index c5372c489..cef18dc97 100644 --- a/packages/ns-api/openapi.yml +++ b/packages/ns-api/openapi.yml @@ -79,6 +79,29 @@ components: items: $ref: "#/components/schemas/ValidationErrorDetail" + SuccessResponse: + type: object + required: [message] + properties: + message: + type: string + example: success + + ThreatShieldDnsListEntry: + type: object + required: + - address + - description + properties: + address: + type: string + description: Domain name present in the local Threat Shield DNS list + example: nethesis.it + description: + type: string + description: Optional free-form description associated with the domain + example: my allow1 + securitySchemes: BearerAuth: type: http @@ -188,3 +211,539 @@ paths: example: success - $ref: "#/components/schemas/ValidationError" - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-list-allowed: + post: + summary: List local Threat Shield DNS allowlist entries + description: Returns the allowlist entries currently staged in UCI. They are written to the adblock file on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-list-allowed + tags: + - threatshield + responses: + "200": + description: Staged allowlist entries + content: + application/json: + schema: + oneOf: + - type: object + required: + - data + properties: + data: + type: array + items: + $ref: "#/components/schemas/ThreatShieldDnsListEntry" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-add-allowed: + post: + summary: Add a local Threat Shield DNS allowlist entry + description: Stages the new allowlist entry in UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-add-allowed + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + - description + properties: + address: + type: string + description: Domain to add to the local allowlist + example: nethesis.it + description: + type: string + description: Free-form description for the domain + example: my allow1 + responses: + "200": + description: Allowlist entry staged or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-edit-allowed: + post: + summary: Edit a local Threat Shield DNS allowlist entry + description: Updates the staged allowlist description in UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-edit-allowed + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + - description + properties: + address: + type: string + description: Existing domain in the local allowlist + example: nethesis.it + description: + type: string + description: Updated description for the domain + example: my new desc + responses: + "200": + description: Allowlist entry updated or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-delete-allowed: + post: + summary: Delete a local Threat Shield DNS allowlist entry + description: Removes the staged allowlist entry from UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-delete-allowed + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Existing domain in the local allowlist + example: nethesis.it + responses: + "200": + description: Allowlist entry deleted or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-list-blocklist: + post: + summary: List available Threat Shield DNS blocklists + operationId: ns.threatshield.dns-list-blocklist + tags: + - threatshield + responses: + "200": + description: Available blocklists + content: + application/json: + schema: + oneOf: + - type: object + required: + - data + properties: + data: + type: array + items: + type: object + required: + - name + - type + - enabled + - confidence + - description + properties: + name: + type: string + description: Blocklist name + example: adguard + type: + type: string + enum: [enterprise, community] + description: Blocklist category + example: enterprise + enabled: + type: boolean + description: Whether the blocklist is enabled + example: true + confidence: + type: integer + description: Entitlement confidence score + example: 10 + description: + type: [string, "null"] + description: Blocklist description + example: OpenDNS family shield + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-list-blocked: + post: + summary: List local Threat Shield DNS blocklist entries + description: Returns the blocklist entries currently staged in UCI. They are written to the adblock file on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-list-blocked + tags: + - threatshield + responses: + "200": + description: Staged blocklist entries + content: + application/json: + schema: + oneOf: + - type: object + required: + - data + properties: + data: + type: array + items: + $ref: "#/components/schemas/ThreatShieldDnsListEntry" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-add-blocked: + post: + summary: Add a local Threat Shield DNS blocklist entry + description: Stages the new blocklist entry in UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-add-blocked + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Domain to add to the local blocklist + example: nastydomain.net + description: + type: string + description: Optional free-form description for the domain + example: my block1 + responses: + "200": + description: Blocklist entry staged or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-edit-blocked: + post: + summary: Edit a local Threat Shield DNS blocklist entry + description: Updates the staged blocklist description in UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-edit-blocked + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Existing domain in the local blocklist + example: nastydomain.net + description: + type: string + description: Updated description for the domain + example: My new desc + responses: + "200": + description: Blocklist entry updated or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-delete-blocked: + post: + summary: Delete a local Threat Shield DNS blocklist entry + description: Removes the staged blocklist entry from UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-delete-blocked + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Existing domain in the local blocklist + example: nastydomain.net + responses: + "200": + description: Blocklist entry deleted or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-list-settings: + post: + summary: Get Threat Shield DNS enforcement settings + operationId: ns.threatshield.dns-list-settings + tags: + - threatshield + responses: + "200": + description: Current DNS enforcement settings + content: + application/json: + schema: + oneOf: + - type: object + required: + - data + properties: + data: + type: object + required: + - enabled + - zones + - ports + properties: + enabled: + type: boolean + description: Whether Threat Shield DNS is enabled + example: true + zones: + type: array + description: Firewall zones where DNS redirection is enforced + items: + type: string + example: [lan] + ports: + type: array + description: DNS ports enforced locally + items: + type: string + example: ["53", "853"] + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-list-zones: + post: + summary: List firewall zones available for Threat Shield DNS + operationId: ns.threatshield.dns-list-zones + tags: + - threatshield + responses: + "200": + description: Available zones + content: + application/json: + schema: + oneOf: + - type: object + required: + - data + properties: + data: + type: array + items: + type: string + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-edit-blocklist: + post: + summary: Enable or disable a Threat Shield DNS blocklist + operationId: ns.threatshield.dns-edit-blocklist + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - blocklist + - enabled + properties: + blocklist: + type: string + description: DNS blocklist name + example: adguard + enabled: + type: boolean + description: Whether the blocklist should be enabled + example: true + responses: + "200": + description: Blocklist updated or validation failed + content: + application/json: + schema: + oneOf: + - type: object + required: + - message + properties: + message: + type: string + example: success + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-edit-settings: + post: + summary: Update Threat Shield DNS enforcement settings + operationId: ns.threatshield.dns-edit-settings + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - enabled + properties: + enabled: + type: boolean + description: Enable or disable Threat Shield DNS + example: true + zones: + type: array + description: Firewall zones where DNS redirection is enforced + items: + type: string + example: [lan] + ports: + type: array + description: DNS ports enforced locally + items: + type: string + example: ["53", "853"] + responses: + "200": + description: Settings updated or validation failed + content: + application/json: + schema: + oneOf: + - type: object + required: + - message + properties: + message: + type: string + example: success + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-list-bypass: + post: + summary: List Threat Shield DNS bypass entries + operationId: ns.threatshield.dns-list-bypass + tags: + - threatshield + responses: + "200": + description: DNS bypass entries + content: + application/json: + schema: + oneOf: + - type: object + required: + - data + properties: + data: + type: array + items: + type: string + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-add-bypass: + post: + summary: Add a Threat Shield DNS bypass entry + operationId: ns.threatshield.dns-add-bypass + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Source IP address or subnet that should bypass DNS redirection + example: 192.168.1.22 + responses: + "200": + description: Bypass entry added or validation failed + content: + application/json: + schema: + oneOf: + - type: object + required: + - message + properties: + message: + type: string + example: success + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-delete-bypass: + post: + summary: Delete a Threat Shield DNS bypass entry + operationId: ns.threatshield.dns-delete-bypass + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Source IP address or subnet to remove from the DNS bypass list + example: 192.168.1.22 + responses: + "200": + description: Bypass entry removed or validation failed + content: + application/json: + schema: + oneOf: + - type: object + required: + - message + properties: + message: + type: string + example: success + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" diff --git a/packages/ns-migration/files/scripts/openvpn b/packages/ns-migration/files/scripts/openvpn index 06908f81a..09cf6a1b2 100755 --- a/packages/ns-migration/files/scripts/openvpn +++ b/packages/ns-migration/files/scripts/openvpn @@ -62,8 +62,7 @@ def add_tap_to_bridge(u, bridge, interface): u.commit("network") except: pass - finally: - return + return iname="ns_roadwarrior1" diff --git a/packages/ns-plug/Makefile b/packages/ns-plug/Makefile index b1715109b..020a61948 100644 --- a/packages/ns-plug/Makefile +++ b/packages/ns-plug/Makefile @@ -44,6 +44,8 @@ if [ -z "$${IPKG_INSTROOT}" ]; then /etc/init.d/cron restart /usr/libexec/ns-plug/40_ns-plug_mwan_hooks /etc/init.d/ns-plug restart + /etc/init.d/ns-plug-alert-proxy enable + /etc/init.d/ns-plug-alert-proxy restart fi exit 0 endef @@ -55,6 +57,8 @@ if [ -z "$${IPKG_INSTROOT}" ]; then crontab -l | grep -v "/usr/sbin/send-inventory" | sort | uniq | crontab - crontab -l | grep -v "/usr/sbin/send-heartbeat" | sort | uniq | crontab - sed -i '/\/usr\/libexec\/ns-plug\/mwan-hooks/d' /etc/mwan3.user + /etc/init.d/ns-plug-alert-proxy stop + /etc/init.d/ns-plug-alert-proxy disable fi exit 0 endef @@ -68,12 +72,13 @@ define Package/ns-plug/install $(INSTALL_DIR) $(1)/etc/init.d $(INSTALL_DIR) $(1)/etc/config $(INSTALL_DIR) $(1)/etc/uci-defaults - $(INSTALL_DIR) $(1)/etc/netdata $(INSTALL_DIR) $(1)/lib/upgrade/keep.d $(INSTALL_DIR) $(1)/usr/libexec/ns-plug $(INSTALL_DIR) $(1)/usr/libexec/mwan-hooks $(INSTALL_BIN) ./files/ns-plug.init $(1)/etc/init.d/ns-plug + $(INSTALL_BIN) ./files/ns-plug-alert-proxy.init $(1)/etc/init.d/ns-plug-alert-proxy $(INSTALL_BIN) ./files/ns-plug $(1)/usr/sbin/ns-plug + $(INSTALL_BIN) ./files/ns-plug-alert-proxy $(1)/usr/sbin/ns-plug-alert-proxy $(INSTALL_BIN) ./files/distfeed-setup $(1)/usr/sbin/distfeed-setup $(INSTALL_BIN) ./files/remote-backup $(1)/usr/sbin $(INSTALL_BIN) ./files/send-backup $(1)/usr/sbin @@ -89,22 +94,17 @@ define Package/ns-plug/install $(INSTALL_BIN) ./files/ns-push-reports $(1)/usr/bin $(INSTALL_BIN) ./files/ns-controller-push-info $(1)/usr/sbin $(INSTALL_BIN) ./files/20_ns-plug $(1)/etc/uci-defaults - $(INSTALL_BIN) ./files/30_ns-plug_alerts $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/40_ns-plug_automatic_updates $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/40_ns-plug_automatic_updates $(1)/usr/libexec/ns-plug $(INSTALL_BIN) ./files/40_ns-plug_mwan_hooks $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/40_ns-plug_mwan_hooks $(1)/usr/libexec/ns-plug - $(INSTALL_BIN) ./files/netadata_enable_alerts $(1)/usr/share/ns-plug/hooks/register/70netadata_enable_alerts - $(INSTALL_BIN) ./files/netadata_disable_alerts $(1)/usr/share/ns-plug/hooks/unregister/70netadata_disable_alerts $(INSTALL_BIN) ./files/enable_automatic_updates $(1)/usr/share/ns-plug/hooks/register/60enable_automatic_updates $(INSTALL_BIN) ./files/disable_automatic_updates $(1)/usr/share/ns-plug/hooks/unregister/60disable_automatic_updates $(INSTALL_CONF) ./files/config $(1)/etc/config/ns-plug $(INSTALL_CONF) files/ns-plug.keep $(1)/lib/upgrade/keep.d/ns-plug - $(INSTALL_CONF) files/health_alarm_notify.conf $(1)/etc/netdata - $(INSTALL_BIN) ./files/send-mwan-alert $(1)/usr/libexec/mwan-hooks - $(INSTALL_BIN) ./files/backup-encryption-alert $(1)/usr/libexec $(INSTALL_BIN) ./files/mwan-hooks $(1)/usr/libexec/ns-plug $(INSTALL_BIN) ./files/ns-plug-rsyslog-fixup.uci-default $(1)/etc/uci-defaults/rsyslog-fixup + $(INSTALL_BIN) ./files/99_ns-plug-netdata-cleanup.uci-default $(1)/etc/uci-defaults/99_ns-plug-netdata-cleanup endef $(eval $(call BuildPackage,ns-plug)) diff --git a/packages/ns-plug/README.md b/packages/ns-plug/README.md index 263ea7a7b..eebc96abb 100644 --- a/packages/ns-plug/README.md +++ b/packages/ns-plug/README.md @@ -110,13 +110,9 @@ the given passphrase: only the encrypted backup will be sent to the remote serve To disable the encryption, just delete the file `/etc/backup.pass`. -If the backup is not encrypted, an alert will be sent to the remote portal (my.nethesis.it or my.nethserver.com). -Unencrypted backups are deprecated and will be removed in the future. -The alert can be disabled using this command: -``` -uci set ns-plug.config.backup_alert_disabled=1 -uci commit ns-plug -``` +Non-encrypted backups are not sent to the remote server for security reasons. +If the backup is not encrypted, an alert will be sent to the remote portal (my.nethesis.it or my.nethserver.com) +so the user can be aware of the risk and take action to secure the backup. ### Restore @@ -133,28 +129,28 @@ remote-backup download $(remote-backup list | jq -r .[0].file) - | gpg --batch - ## Alerts -All system alerts, except MultiWAN ones, are handled by netdata, including those from the multiwan monitoring. -Alerts are disabled by default and enabled only if the machine has a valid subscription. -In this case, alerts are automatically sent to the remote server (either my.nethesis.it or my.nethserver.com) using a -custom sender (`/etc/netdata/health_alarm_notify.conf`). -Alerts are also logged to `/var/log/messages` and are visible within the netdata UI. +System alerts are handled by **vmalert** (Victoria Metrics alert evaluation engine) which evaluates +alert rules against metrics collected by telegraf. -Only the following alerts are sent to the remote system: +When an alert fires or resolves, vmalert sends an Alertmanager-format webhook to `ns-plug-alert-proxy` +running on `127.0.0.1:9095`. The proxy forwards the following alerts to the registered monitoring +portal (my.nethesis.it or my.nethserver.com): -- disk space occupation -- WAN down events +| Alert | Condition | Legacy alert_id | +|---|---|---| +| `WanDown` | WAN interface offline for 2m | `wan::down` | +| `DiskSpaceCritical` | Disk usage > 90% for 2m | `df:root:percent_bytes:free` or `df:boot:percent_bytes:free` | +| `BackupEncryptionDisabled` | Backup passphrase missing | `backup:config:notencrypted` | +| `StorageStatus` | Storage status is error | `storage:status` | -When an alert is resolved, netdata will also send a clear command to remote server. +All other alert are silently dropped by the proxy. +If the machine is not registered, all alerts are silently dropped. -### MultiWAN alerts +The proxy starts automatically at boot regardless of registration state. +Firing/resolved state is determined from the Alertmanager-standard `endsAt` field: +if `endsAt` is in the future (or zero/missing) a **FAILURE** is sent; if `endsAt` is in +the past an **OK** is sent. +A FAILURE is sent when the alert starts firing and an OK is sent when it resolves. -MultiWAN alerts are managed using `/etc/mwan3.user` script. - -When a WAN changes its status, all executable scripts inside the `/usr/libexec/mwan-hooks/` directory will be executed. -If the machine has a valid subscription, the `send-mwan-alert` script will send an alert to my.nethesis.it and my.nethserver.com monitoring portals. -Sent alerts are logged to `/var/log/messages`, example: -``` -Jul 31 12:40:42 NethSec mwan3-alert: Sending alert wan:wanb:down with status FAILURE -... -Jul 31 12:41:04 NethSec mwan3-alert: Sending alert wan:wanb:down with status OK -``` +If Mimir credentials are configured in ns-plug UCI (`my_url`, `my_system_key`, `my_system_secret`), +vmalert also forwards all alerts to the Mimir alertmanager for cloud-side processing. diff --git a/packages/ns-plug/files/20_ns-plug b/packages/ns-plug/files/20_ns-plug index b87f6a92f..4026f4031 100644 --- a/packages/ns-plug/files/20_ns-plug +++ b/packages/ns-plug/files/20_ns-plug @@ -2,7 +2,6 @@ # setup cron jobs for remote servers crontab -l | grep -q '/usr/sbin/send-backup' || echo '02 2 * * * sleep $(( RANDOM % 1800 )); /usr/sbin/send-backup' >> /etc/crontabs/root -crontab -l | grep -q '/usr/libexec/backup-encryption-alert' || echo '02 3 * * * sleep $(( RANDOM % 1800 )); /usr/libexec/backup-encryption-alert' >> /etc/crontabs/root crontab -l | grep -q '/usr/sbin/send-heartbeat' || echo '*/10 * * * * sleep $(( RANDOM % 60 )); /usr/sbin/send-heartbeat' >> /etc/crontabs/root crontab -l | grep -q '/usr/sbin/send-inventory' || echo '05 3 * * * sleep $(( RANDOM % 1800 )); /usr/sbin/send-inventory' >> /etc/crontabs/root diff --git a/packages/ns-plug/files/30_ns-plug_alerts b/packages/ns-plug/files/30_ns-plug_alerts deleted file mode 100644 index dfb62a848..000000000 --- a/packages/ns-plug/files/30_ns-plug_alerts +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/sh - -# Custom disk alerts -disks_f="/etc/netdata/health.d/disks.conf" -if [ ! -f "$disks_f" ]; then - cat << EOF > "$disks_f" -template: disk_space_usage - on: disk.space - class: Utilization - type: System -component: Disk - os: linux freebsd - hosts: * - families: !/dev !/dev/* !/run !/run/* !/overlay * - calc: \$used * 100 / (\$avail + \$used) - units: % - every: 1m - warn: \$this > ((\$status >= \$WARNING ) ? (80) : (90)) - crit: \$this > ((\$status == \$CRITICAL) ? (90) : (98)) - delay: up 1m down 15m multiplier 1.5 max 1h - info: disk $family space utilization - to: sysadmin -EOF -fi - -# Disable unwanted alerts -files="cpu disks entropy ipc load memory net netfilter processes ram softnet tcp_conn tcp_listen tcp_mem tcp_orphans tcp_resets timex udp_errors" -for f in $files -do - file="/etc/netdata/health.d/${f}.conf" - if [ ! -f $file ]; then - > $file - fi -done - -# Enable mwan chart -sed -i 's/python.d = no/python.d = yes/' /etc/netdata/netdata.conf -python_f="/etc/netdata/python.d.conf" -if [ ! -f "$python_f" ]; then - cat << EOF > "$python_f" -enabled: yes -gc_run: yes -gc_interval: 300 -apache_cache: no -chrony: no -example: no -go_expvar: no -gunicorn_log: no -hpssa: no -logind: no -nginx_log: no -EOF -fi - -# Create mwan alert -cat << EOF > /etc/netdata/health.d/mwan.conf -template: wan_status - on: mwan.score -lookup: min -1m foreach * - every: 1m - warn: \$this < 5 - crit: \$this <= 1 - info: The score of the WAN, 0 means down -EOF diff --git a/packages/ns-plug/files/99_ns-plug-netdata-cleanup.uci-default b/packages/ns-plug/files/99_ns-plug-netdata-cleanup.uci-default new file mode 100644 index 000000000..e76fe4356 --- /dev/null +++ b/packages/ns-plug/files/99_ns-plug-netdata-cleanup.uci-default @@ -0,0 +1,10 @@ +#!/bin/sh +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Remove the legacy backup-encryption alert cron entry if it is still present. +if crontab -l 2>/dev/null | grep -q '/usr/libexec/backup-encryption-alert'; then + crontab -l 2>/dev/null | grep -v '/usr/libexec/backup-encryption-alert' | sort | uniq | crontab - +fi diff --git a/packages/ns-plug/files/backup-encryption-alert b/packages/ns-plug/files/backup-encryption-alert deleted file mode 100644 index 1249f5479..000000000 --- a/packages/ns-plug/files/backup-encryption-alert +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2025 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -# Send a backup alert if backup is not encrypted - -lk=$(uci -q get ns-plug.config.system_id) -secret=$(uci -q get ns-plug.config.secret) -url=$(uci -q get ns-plug.config.alerts_url)"alerts/store" - -# Do not send alert if system_id or secret is not set -if [ -z "$lk" ] || [ -z "$secret" ]; then - exit 0 -fi - -# Check if alert is enabled -if [ "$(uci -q get ns-plug.config.backup_alert_disabled)" = "1" ]; then - exit 0 -fi - -# Send the alert -if [ ! -f "/etc/backup.pass" ]; then - status="FAILURE" -else - status="OK" -fi - -alert_id="backup:config:notencrypted" -logger -t backup-alert "Sending alert ${alert_id} with status ${status}" -payload='{"lk": "'$lk'", "alert_id": "'$alert_id'", "status": "'$status'"}' - -/usr/bin/curl -m 30 --retry 3 -L -s \ - --header "Authorization: token ${secret}" --header "Content-Type: application/json" --header "Accept: application/json" \ - --data-raw "${payload}" ${url} > /dev/null diff --git a/packages/ns-plug/files/health_alarm_notify.conf b/packages/ns-plug/files/health_alarm_notify.conf deleted file mode 100644 index 00852cfa9..000000000 --- a/packages/ns-plug/files/health_alarm_notify.conf +++ /dev/null @@ -1,75 +0,0 @@ -# Configuration for alarm notifications - -SEND_EMAIL="NO" -SEND_DYNATRACE="NO" -SEND_STACKPULSE="NO" -SEND_OPSGENIE="NO" -SEND_HANGOUTS="NO" -SEND_PUSHOVER="NO" -SEND_PUSHBULLET="NO" -SEND_TWILIO="NO" -SEND_MESSAGEBIRD="NO" -SEND_KAVENEGAR="NO" -SEND_TELEGRAM="NO" -SEND_SLACK="NO" -SEND_MSTEAMS="NO" -SEND_ROCKETCHAT="NO" -SEND_ALERTA="NO" -SEND_FLOCK="NO" -SEND_DISCORD="NO" -SEND_HIPCHAT="NO" -SEND_KAFKA="NO" -SEND_PD="NO" -SEND_FLEEP="NO" -SEND_IRC="NO" -SEND_SYSLOG="NO" -SEND_PROWL="NO" -SEND_AWSSNS="NO" -SEND_SMS="NO" -SEND_MATRIX="NO" - -# Enable only syslog and custom notification -use_fqdn='YES' -SEND_SYSLOG="YES" -SYSLOG_FACILITY='' -DEFAULT_RECIPIENT_SYSLOG="sysadmin" -SEND_CUSTOM="YES" -DEFAULT_RECIPIENT_CUSTOM="sysadmin" - -# Always generate clear events -clear_alarm_always='YES' - -# Send alerts to my.nethesis.it or my.nethserver.com -custom_sender() { - lk=$(uci -q get ns-plug.config.system_id) - secret=$(uci -q get ns-plug.config.secret) - url=$(uci -q get ns-plug.config.alerts_url)"alerts/store" - alert_id=${name} - if [ "${status}" == "CRITICAL" ]; then - status="FAILURE" - elif [ "${status}" == "CLEAR" ]; then - status="OK" - fi - - # map to old alerts, when possible - if [ "${chart}" == "disk_space._overlay" ] || [ "${chart}" == "disk_space._" ]; then - alert_id="df:root:percent_bytes:free" - elif [ "${chart}" == "disk_space._boot" ]; then - alert_id="df:boot:percent_bytes:free" - else - alert_id="${name}:${chart}" - fi - payload='{"lk": "'$lk'", "alert_id": "'$alert_id'", "status": "'$status'"}' - - # send only if the machine is registered - if [ -z "${lk}" ] || [ -z "${secret}" ]; then - return - fi - - # send to remote server - if [ "${status}" == "FAILURE" ] || [ "${status}" == "OK" ]; then - /usr/bin/curl -m 180 --retry 3 -L -s \ - --header "Authorization: token ${secret}" --header "Content-Type: application/json" --header "Accept: application/json" \ - --data-raw "${payload}" ${url} - fi -} diff --git a/packages/ns-plug/files/netadata_disable_alerts b/packages/ns-plug/files/netadata_disable_alerts deleted file mode 100644 index b21473515..000000000 --- a/packages/ns-plug/files/netadata_disable_alerts +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -# Disable netdata alerts -sed -i 's/enabled = yes/enabled = no/' /etc/netdata/netdata.conf -/etc/init.d/netdata restart diff --git a/packages/ns-plug/files/netadata_enable_alerts b/packages/ns-plug/files/netadata_enable_alerts deleted file mode 100644 index cf066e58a..000000000 --- a/packages/ns-plug/files/netadata_enable_alerts +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -# Enable netdata alerts -sed -i 's/enabled = no/enabled = yes/' /etc/netdata/netdata.conf -/etc/init.d/netdata restart diff --git a/packages/ns-plug/files/ns-plug-alert-proxy b/packages/ns-plug/files/ns-plug-alert-proxy new file mode 100644 index 000000000..f50f70d4b --- /dev/null +++ b/packages/ns-plug/files/ns-plug-alert-proxy @@ -0,0 +1,176 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +""" +Alert proxy: receives Alertmanager-like notifications from vmalert and +forwards selected alerts to the legacy my.nethesis.it / my.nethserver.com +monitoring portals. + +Only the following alerts are forwarded: + - WanDown → wan::down + - DiskSpaceCritical → df:root:percent_bytes:free (path=/) + df:boot:percent_bytes:free (path=/boot) + - BackupEncryptionDisabled → backup:config:notencrypted + - StorageStatus → storage:status + +All other alerts are silently dropped. +If the machine is not registered (no system_id/secret in UCI), all alerts +are silently dropped. + +Firing/resolved state is determined from the Alertmanager-standard endsAt +field: if endsAt is in the future (or zero/missing) the alert is FAILURE; +if endsAt is in the past the alert is OK. +""" + +import json +import re +import sys +import time +import urllib.request +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, HTTPServer +from socketserver import ThreadingMixIn +from euci import EUci + +LISTEN_ADDR = "127.0.0.1" +LISTEN_PORT = 9095 + +_DISK_PATH_MAP = { + "/": "df:root:percent_bytes:free", + "/boot": "df:boot:percent_bytes:free", +} + +_ZERO_TIME = "0001-01-01T00:00:00Z" +# vmalert uses nanosecond precision; strip to microseconds for Python parsing +_NANO_RE = re.compile(r"(\.\d{6})\d+(Z|[+-]\d{2}:\d{2})$") + + +def _is_firing(alert): + """Return True if the alert is currently firing based on endsAt.""" + ends_at_str = alert.get("endsAt", "") + if not ends_at_str or ends_at_str == _ZERO_TIME: + return True + ends_at_str = _NANO_RE.sub(r"\1\2", ends_at_str) + ends_at_str = ends_at_str.replace("Z", "+00:00") + try: + ends_at = datetime.fromisoformat(ends_at_str) + return ends_at > datetime.now(timezone.utc) + except Exception: + return True + + +def _map_alert_id(alert_name, labels): + """Return the legacy alert_id string, or None if the alert is not mapped.""" + if alert_name == "WanDown": + iface = labels.get("interface", "unknown") + return f"wan:{iface}:down" + if alert_name == "DiskSpaceCritical": + path = labels.get("path", "") + return _DISK_PATH_MAP.get(path) + if alert_name == "BackupEncryptionDisabled": + return "backup:config:notencrypted" + if alert_name == "StorageStatus": + return "storage:status" + return None + + +def _send_alert(system_id, secret, alerts_url, alert_id, status, retry=3): + url = alerts_url.rstrip("/") + "/alerts/store" + payload = json.dumps( + {"lk": system_id, "alert_id": alert_id, "status": status} + ).encode() + req = urllib.request.Request( + url, + data=payload, + method="POST", + headers={ + "Authorization": f"token {secret}", + "Content-Type": "application/json", + "Accept": "application/json", + }, + ) + try: + with urllib.request.urlopen(req, timeout=60) as resp: + print(f"Alert sent: {alert_id} {status} → {resp.status}", file=sys.stderr) + except Exception as ex: + if retry > 0: + print( + f"Alert send failed: {alert_id} {ex} — retrying in 20s", file=sys.stderr + ) + time.sleep(20) + _send_alert(system_id, secret, alerts_url, alert_id, status, retry - 1) + else: + print(f"Alert send aborted: {alert_id} {ex}", file=sys.stderr) + + +class _AlertHandler(BaseHTTPRequestHandler): + def log_message(self, format, *args): + # Suppress access log + pass + + def do_GET(self): + self.send_response(200) + self.end_headers() + + def do_POST(self): + if self.system_id is None or self.secret is None or self.alerts_url is None: + # Just drop the alert if not configured + self.send_response(200) + self.end_headers() + return + try: + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + data = json.loads(body) + except Exception as ex: + self.send_response(400) + self.end_headers() + self.wfile.write(str(ex).encode()) + return + + if type(data) is list: + alerts = data + else: + alerts = data.get("alerts", []) + for alert in alerts: + labels = alert.get("labels", {}) + alert_name = labels.get("alertname", "") + legacy_status = "FAILURE" if _is_firing(alert) else "OK" + + alert_id = _map_alert_id(alert_name, labels) + if not alert_id: + print( + f"Alert dropped (no mapping): {alert_name} {labels}", + file=sys.stderr, + ) + continue + + _send_alert(self.system_id, self.secret, self.alerts_url, alert_id, legacy_status) + + self.send_response(200) + self.end_headers() + + def __init__(self, *args, **kwargs): + uci = EUci() + self.system_id = uci.get("ns-plug", "config", "system_id", default=None) + self.secret = uci.get("ns-plug", "config", "secret", default=None) + self.alerts_url = uci.get("ns-plug", "config", "alerts_url", default=None) + super().__init__(*args, **kwargs) + + +class _ThreadingHTTPServer(ThreadingMixIn, HTTPServer): + daemon_threads = True + + +def main(): + server = _ThreadingHTTPServer((LISTEN_ADDR, LISTEN_PORT), _AlertHandler) + print(f"alert-proxy listening on {LISTEN_ADDR}:{LISTEN_PORT}", file=sys.stderr) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/packages/ns-plug/files/ns-plug-alert-proxy.init b/packages/ns-plug/files/ns-plug-alert-proxy.init new file mode 100644 index 000000000..38ce01012 --- /dev/null +++ b/packages/ns-plug/files/ns-plug-alert-proxy.init @@ -0,0 +1,30 @@ +#!/bin/sh /etc/rc.common + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +START=95 +STOP=4 +USE_PROCD=1 + +start_service() { + procd_open_instance + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param command '/usr/sbin/ns-plug-alert-proxy' + procd_set_param respawn 3600 5 0 + procd_close_instance +} + +service_triggers() +{ + procd_add_reload_trigger "ns-plug" +} + +reload_service() +{ + stop + start +} diff --git a/packages/ns-plug/files/send-mwan-alert b/packages/ns-plug/files/send-mwan-alert deleted file mode 100644 index 1a74ec96c..000000000 --- a/packages/ns-plug/files/send-mwan-alert +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2024 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -# Send WAN alert to monitoring portal - -lk=$(uci -q get ns-plug.config.system_id) -secret=$(uci -q get ns-plug.config.secret) -url=$(uci -q get ns-plug.config.alerts_url)"alerts/store" -pidfile="/tmp/mwan3.$INTERFACE" - -# Do not send alert if system_id or secret is not set -if [ -z "$lk" ] || [ -z "$secret" ]; then - exit 0 -fi - -# Ignore ifup and ifdown events, they both triggers connected and disconnected events -if [ "${ACTION}" == "connected" ]; then - pid=$(cat "$pidfile" 2>/dev/null) - # If a wan is connected within 30 seconds from disconnect, assume it's a restart - # and kill the alert sending process - # mwan3 restart should complete within 30 seconds - if [ -n "$pid" ]; then - kill -s SIGHUP "$pid" - rm "$pidfile" - exit 0 - fi - status="OK" -elif [ "${ACTION}" == "disconnected" ]; then - echo $$ > "$pidfile" - # Delay alert by 30 seconds, so that it can be canceled - sleep 30 - rm "$pidfile" - status="FAILURE" -fi - -# Exit if status is not set -if [ -z "$status" ]; then - exit 0 -fi - -alert_id="wan:${INTERFACE}:down" -logger -t mwan3-alert "Sending alert ${alert_id} with status ${status}" -payload='{"lk": "'$lk'", "alert_id": "'$alert_id'", "status": "'$status'"}' -/usr/bin/curl -m 30 --retry 3 -L -s \ - --header "Authorization: token ${secret}" --header "Content-Type: application/json" --header "Accept: application/json" \ - --data-raw "${payload}" ${url} diff --git a/packages/ns-storage/Makefile b/packages/ns-storage/Makefile index 27938cfd1..8bd79ac3b 100644 --- a/packages/ns-storage/Makefile +++ b/packages/ns-storage/Makefile @@ -70,8 +70,6 @@ define Package/ns-storage/install $(INSTALL_BIN) ./files/32-ns-storage-convert-uuid.uci-default $(1)/etc/uci-defaults/32-ns-storage-convert-uuid $(INSTALL_CONF) ./files/data.conf $(1)/etc/logrotate.d $(INSTALL_BIN) ./files/storage-status $(1)/usr/sbin - $(INSTALL_BIN) ./files/storage-alarm $(1)/usr/libexec - $(INSTALL_BIN) ./files/ns-storage-alert.init $(1)/etc/init.d/ns-storage-alert $(INSTALL_BIN) ./files/ns-storage-check.init $(1)/etc/init.d/ns-storage-check endef diff --git a/packages/ns-storage/README.md b/packages/ns-storage/README.md index b9554a8b2..ba92773c3 100644 --- a/packages/ns-storage/README.md +++ b/packages/ns-storage/README.md @@ -69,6 +69,10 @@ the system as follow: - rsyslog will write logs also inside `/mnt/data/logs/messages` file - logrotate will rotate `/mnt/data/logs/messages` once a week (see `/etc/logrotate/data.conf` for more info) +### Storage status alert + +The storage health check is exported to Telegraf by `/usr/libexec/telegraf-storage-status` and evaluated by vmalert as `StorageStatus`. + ## Data sync customization Every night the cron will run a script named `sync-data` to sync data from in-memory diff --git a/packages/ns-storage/files/ns-storage-alert.init b/packages/ns-storage/files/ns-storage-alert.init deleted file mode 100644 index 26a259041..000000000 --- a/packages/ns-storage/files/ns-storage-alert.init +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh /etc/rc.common - -# -# Copyright (C) 2025 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -START=99 -USE_PROCD=1 - -start_service() -{ - procd_open_instance - procd_set_param stdout 1 - procd_set_param stderr 1 - procd_set_param command '/usr/libexec/storage-alarm' - procd_close_instance -} - -reload_service() -{ - start -} - -service_triggers() -{ - procd_add_reload_trigger fstab ns-plug - procd_add_reload_mount_trigger /mnt/data -} diff --git a/packages/ns-storage/files/storage-alarm b/packages/ns-storage/files/storage-alarm deleted file mode 100644 index adaccedad..000000000 --- a/packages/ns-storage/files/storage-alarm +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -# -# Copyright (C) 2025 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -system_id=$(uci -q get ns-plug.config.system_id) -system_secret=$(uci -q get ns-plug.config.secret) -if [ -z "$system_id" ] || [ -z "$system_secret" ]; then - # not subscription - exit 0 -fi -url="$(uci -q get ns-plug.config.alerts_url)/alerts/store" - -storage_status=$(storage-status) -status="OK" -if [ "$storage_status" = "error" ]; then - status="FAILURE" -fi - -/usr/bin/curl -m 180 --retry 10 -L -s \ - --header "Authorization: token $system_secret" \ - --header "Content-Type: application/json" \ - --header "Accept: application/json" \ - --data-binary "{\"lk\": \"$system_id\", \"alert_id\": \"storage:status\", \"status\": \"$status\"}" \ - "$url" > /dev/null diff --git a/packages/ns-threat_shield/Makefile b/packages/ns-threat_shield/Makefile index 43001ed90..2b66694bf 100644 --- a/packages/ns-threat_shield/Makefile +++ b/packages/ns-threat_shield/Makefile @@ -22,7 +22,8 @@ define Package/ns-threat_shield CATEGORY:=NethSecurity TITLE:=Threat shield block list URL:=https://github.com/NethServer/nethsecurity/ - DEPENDS:=+wget-ssl +adblock +jq + DEPENDS:=+wget-ssl +adblock +jq +ns-api + EXTRA_DEPENDS:=adblock (>=4.5.3), ns-api (>=3.6.2) PKGARCH:=all endef diff --git a/packages/ns-threat_shield/README.md b/packages/ns-threat_shield/README.md index cc8ace162..6943b0ebc 100644 --- a/packages/ns-threat_shield/README.md +++ b/packages/ns-threat_shield/README.md @@ -82,18 +82,13 @@ uci commit adblock /etc/init.d/adblock restart ``` -### Custom categories - -To add custom categories, create a file `/etc/adblock/custom.sources.gz` with the list of categories to block. -If such file is present, the `/usr/share/threat_shield/community-dns.sources.gz` will be ignored. - ### DNS redirect bypass Allow bypass of DNS redirect for a specific source IP: ``` -uci add_list adblock.global.adb_bypass=192.168.100.2 +uci add_list adblock.global.ns_tsdns_bypass=192.168.100.2 uci commit adblock -/etc/init.d/adblock restart +/etc/init.d/adblock reload ``` For more info see [adblock repository](https://github.com/openwrt/packages/tree/master/net/adblock). diff --git a/packages/ns-threat_shield/files/ts-dns b/packages/ns-threat_shield/files/ts-dns index f67484be0..01f6399f6 100755 --- a/packages/ns-threat_shield/files/ts-dns +++ b/packages/ns-threat_shield/files/ts-dns @@ -8,8 +8,8 @@ DEST_DIR=/etc/adblock NETHESIS_SOURCES=/usr/share/threat_shield/nethesis-dns.sources.gz COMMUNITY_SOURCES=/usr/share/threat_shield/community-dns.sources.gz -CUSTOM_SOURCES=/etc/adblock/custom.sources.gz -TMP_FILE=/tmp/combined.sources +CUSTOM_FEEDS="$DEST_DIR/adblock.custom.feeds" +TMP_FILE=/tmp/ts-dns.sources SYSTEM_ID=$(uci -q get ns-plug.config.system_id) SYSTEM_SECRET=$(uci -q get ns-plug.config.secret) @@ -17,42 +17,28 @@ TYPE=$(uci -q get ns-plug.config.type) TS_ENABLED=$(uci -q get adblock.global.ts_enabled) if [ "$TS_ENABLED" = 1 ]; then - # Setup new blacklist source - uci set adblock.global.adb_srcarc="$DEST_DIR"/combined.sources.gz - # Setup dnsmasq as backend uci set adblock.global.adb_dns='dnsmasq' uci set adblock.global.adb_dnsinstance='0' - # Setup wget with compression support - uci set adblock.global.adb_fetchutil='wget' - uci set adblock.global.adb_fetchparm="--compression=gzip --no-cache --no-cookies --max-redirect=0 --timeout=20 -O" - - if [ -f $CUSTOM_SOURCES ]; then - # Add custom sources, if present - gunzip -c $CUSTOM_SOURCES > $TMP_FILE - else - # Use community sources - gunzip -c $COMMUNITY_SOURCES > $TMP_FILE - fi + # Build custom feeds from community sources + gunzip -c "$COMMUNITY_SOURCES" > "$TMP_FILE" - # Setup Nethesis sources if the machine has a subscription - if [ ! -z "$SYSTEM_SECRET" ] && [ ! -z "$SYSTEM_ID" ]; then - # Replaces credentials and compress - gunzip -c $NETHESIS_SOURCES | sed -e "s/__USER__/$SYSTEM_ID/" -e "s/__PASSWORD__/$SYSTEM_SECRET/" -e "s/__TYPE__/$TYPE/" >> $TMP_FILE + # Merge Nethesis sources if the machine has a subscription + if [ -n "$SYSTEM_SECRET" ] && [ -n "$SYSTEM_ID" ]; then + gunzip -c "$NETHESIS_SOURCES" | sed -e "s/__USER__/$SYSTEM_ID/" -e "s/__PASSWORD__/$SYSTEM_SECRET/" -e "s/__TYPE__/$TYPE/" >> "$TMP_FILE" fi - # Merge the source list and compress it to the final file - jq -s 'reduce .[] as $item ({}; . * $item)' $TMP_FILE | gzip -c > "$DEST_DIR"/combined.sources.gz + # Merge all sources into a single JSON object and write to adblock.custom.feeds + jq -s 'reduce .[] as $item ({}; . * $item)' "$TMP_FILE" > "$CUSTOM_FEEDS" # Cleanup - rm -f $TMP_FILE + rm -f "$TMP_FILE" else - # Reset sources to origin file if threat shield is not enabled - uci -q delete adblock.global.adb_srcarc - uci -q delete adblock.global.adb_fetchparam - uci set adblock.global.adb_fetchutil='curl' + # Clear custom feeds when threat shield is disabled + : > "$CUSTOM_FEEDS" fi # Save changes uci commit adblock + diff --git a/packages/ns-ui/Makefile b/packages/ns-ui/Makefile index 171b73bde..db0c137a4 100644 --- a/packages/ns-ui/Makefile +++ b/packages/ns-ui/Makefile @@ -10,10 +10,12 @@ PKG_NAME:=ns-ui PKG_VERSION:=2.20.1 PKG_RELEASE:=1 -PKG_SOURCE:=nethsecurity-ui-$(PKG_VERSION).tar.gz -PKG_BUILD_DIR=$(BUILD_DIR)/nethsecurity-ui-$(PKG_VERSION) -PKG_SOURCE_URL:=https://codeload.github.com/nethserver/nethsecurity-ui/tar.gz/$(PKG_VERSION)? -PKG_HASH:=skip +PKG_SOURCE_PROTO:=git +PKG_SOURCE_URL:=https://github.com/NethServer/nethsecurity-ui.git +PKG_SOURCE_VERSION:=d95a3d9df55b18df2db9afbe01421901e655ac58 +PKG_SOURCE_SUBDIR:=nethsecurity-ui-$(PKG_SOURCE_VERSION) +PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) +PKG_MIRROR_HASH:=skip PKG_MAINTAINER:=Giacomo Sanchietti PKG_LICENSE:=GPL-3.0-only diff --git a/packages/python-jinja2/Makefile b/packages/python-jinja2/Makefile deleted file mode 100644 index f29df71ec..000000000 --- a/packages/python-jinja2/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -# This is free software, licensed under the GNU General Public License v2. -# See /LICENSE for more information. -# - -include $(TOPDIR)/rules.mk - -PKG_NAME:=python-jinja2 -PKG_VERSION:=3.1.2 -PKG_RELEASE:=2 - -PYPI_NAME:=Jinja2 -PYTHON3_PKG_WHEEL_NAME:=jinja2 -PKG_HASH:=31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 - -PKG_MAINTAINER:=Michal Vasilek -PKG_LICENSE:=BSD-3-Clause -PKG_LICENSE_FILES:=LICENSE.rst -PKG_CPE_ID:=cpe:/a:pocoo:jinja2 -HOST_BUILD_DEPENDS:= python-markupsafe/host - -include $(TOPDIR)/feeds/packages/lang/python/pypi.mk -include $(INCLUDE_DIR)/package.mk -include $(INCLUDE_DIR)/host-build.mk -include $(TOPDIR)/feeds/packages/lang/python/python3-package.mk -include $(TOPDIR)/feeds/packages/lang/python/python3-host-build.mk - -define Package/python3-jinja2 - SECTION:=lang - CATEGORY:=Languages - SUBMENU:=Python - TITLE:=Very fast and expressive template engine - URL:=https://palletsprojects.com/p/jinja/ - DEPENDS:= \ - +python3-light \ - +python3-asyncio \ - +python3-logging \ - +python3-urllib \ - +python3-markupsafe -endef - -define Package/python3-jinja2/description -Jinja2 is a full featured template engine for Python. It has full -unicode support, an optional integrated sandboxed execution -environment, widely used and BSD licensed. -endef - -$(eval $(call Py3Package,python3-jinja2)) -$(eval $(call BuildPackage,python3-jinja2)) -$(eval $(call BuildPackage,python3-jinja2-src)) -$(eval $(call HostBuild)) \ No newline at end of file diff --git a/packages/python-semver/Makefile b/packages/python-semver/Makefile index 9132e194e..4affdbf63 100644 --- a/packages/python-semver/Makefile +++ b/packages/python-semver/Makefile @@ -5,11 +5,11 @@ include $(TOPDIR)/rules.mk PKG_NAME:=python-semver -PKG_VERSION:=2.13.0 +PKG_VERSION:=3.0.4 PKG_RELEASE:=1 PYPI_NAME:=semver -PKG_HASH:=fa0fe2722ee1c3f57eac478820c3a5ae2f624af8264cbdf9000c980ff7f75e3f +PKG_HASH:=afc7d8c584a5ed0a11033af086e8af226a9c0b206f313e0301f8dd7b6b589602 PKG_MAINTAINER:=Tommaso Bailetti PKG_LICENSE:=BSD-3-Clause diff --git a/packages/telegraf/Makefile b/packages/telegraf/Makefile new file mode 100644 index 000000000..b2e7ffa91 --- /dev/null +++ b/packages/telegraf/Makefile @@ -0,0 +1,108 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +include $(TOPDIR)/rules.mk + +PKG_NAME:=telegraf +# renovate: datasource=github-tags depName=influxdata/telegraf +PKG_VERSION:=1.38.3 +PKG_RELEASE:=1 + +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz +PKG_SOURCE_URL:=https://codeload.github.com/influxdata/telegraf/tar.gz/v$(PKG_VERSION)? +PKG_SOURCE_SUBDIR:=telegraf-$(PKG_VERSION) +PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) + +PKG_HASH:=skip +PKG_MAINTAINER:=Tommaso Bailetti +PKG_LICENSE:=MIT + +PKG_BUILD_DEPENDS:=golang/host +PKG_BUILD_PARALLEL:=1 +PKG_BUILD_FLAGS:=no-mips16 + +GO_PKG:=github.com/influxdata/telegraf/cmd/$(PKG_NAME) +GO_BUILD_PKG:=github.com/influxdata/telegraf/cmd/$(PKG_NAME) +GO_PKG_LDFLAGS_X:=github.com/influxdata/telegraf/internal.Version=$(PKG_VERSION) +GO_PKG_TAGS:= \ + custom \ + inputs.bond \ + inputs.conntrack \ + inputs.cpu \ + inputs.disk \ + inputs.diskio \ + inputs.ethtool \ + inputs.dns_query \ + inputs.exec \ + inputs.file \ + inputs.mem \ + inputs.net \ + inputs.netstat \ + inputs.nstat \ + inputs.ping \ + inputs.processes \ + inputs.sensors \ + inputs.system \ + outputs.influxdb \ + outputs.prometheus_client \ + parsers.json_v2 + +include $(INCLUDE_DIR)/package.mk +include $(TOPDIR)/feeds/packages/lang/golang/golang-package.mk + +define Package/telegraf + SECTION:=base + CATEGORY:=NethServer + TITLE:=Telegraf + URL:=https://github.com/influxdata/telegraf + DEPENDS:= \ + $(GO_ARCH_DEPENDS) \ + +lm-sensors \ + +victoria-metrics \ + +python3-uci \ + +python3-jinja2 +endef + +define Package/telegraf/description + Telegraf is an agent for collecting, processing, aggregating, and writing metrics. +endef + +define Package/telegraf/conffiles +/etc/config/telegraf +endef + +define Package/telegraf/install + $(call GoPackage/Package/Install/Bin,$(1)) + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/telegraf.initd $(1)/etc/init.d/telegraf + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_CONF) ./files/telegraf.uci $(1)/etc/config/telegraf + $(INSTALL_DIR) $(1)/etc/telegraf + $(INSTALL_DATA) ./files/telegraf.conf $(1)/etc/telegraf.conf + $(INSTALL_DIR) $(1)/etc/telegraf.conf.d + $(INSTALL_DATA) ./files/telegraf.conf.d/os.conf $(1)/etc/telegraf.conf.d/os.conf + $(INSTALL_DATA) ./files/telegraf.conf.d/backup.conf $(1)/etc/telegraf.conf.d/backup.conf + $(INSTALL_DATA) ./files/telegraf.conf.d/storage.conf $(1)/etc/telegraf.conf.d/storage.conf + $(INSTALL_DATA) ./files/telegraf.conf.d/services.conf $(1)/etc/telegraf.conf.d/services.conf + $(INSTALL_DATA) ./files/telegraf.conf.d/mwan.conf $(1)/etc/telegraf.conf.d/mwan.conf + $(INSTALL_DIR) $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/uci-defaults/99-telegraf-migrate-netdata $(1)/etc/uci-defaults/99-telegraf-migrate-netdata + $(INSTALL_DIR) $(1)/usr/sbin + $(INSTALL_BIN) ./files/telegraf-config $(1)/usr/sbin/telegraf-config + $(INSTALL_DIR) $(1)/usr/libexec + $(INSTALL_BIN) ./files/telegraf-services $(1)/usr/libexec/telegraf-services + $(INSTALL_BIN) ./files/telegraf-backup-encryption $(1)/usr/libexec/telegraf-backup-encryption + $(INSTALL_BIN) ./files/telegraf-storage-status $(1)/usr/libexec/telegraf-storage-status + $(INSTALL_BIN) ./files/telegraf-mwan $(1)/usr/libexec/telegraf-mwan +endef + +define Package/telegraf/postinst +#!/bin/sh +[ -z "$${IPKG_INSTROOT}" ] && /etc/init.d/telegraf restart +exit 0 +endef + +$(eval $(call GoBinPackage,telegraf)) +$(eval $(call BuildPackage,telegraf)) diff --git a/packages/telegraf/README.md b/packages/telegraf/README.md new file mode 100644 index 000000000..b6cb55a73 --- /dev/null +++ b/packages/telegraf/README.md @@ -0,0 +1,145 @@ +# Telegraf + +## Overview + +Telegraf is the metrics collection agent that gathers host and service metrics and forwards them to Victoria Metrics for storage, alerting, and visualization. + +## Architecture + +``` +/usr/libexec/telegraf-services ← service status via ubus +/var/run/mwan3/iface_state/ ← WAN interface status via mwan3 state files +/proc filesystem ← CPU, memory, disk, network + │ + ▼ + Telegraf (inputs.exec, inputs.cpu, inputs.mem, …) + │ + ▼ +Victoria Metrics (http://127.0.0.1:8428) + │ + └─▶ vmalert (alert rules evaluation) +``` + +## Configuration Files + +| Path | Description | +|------|-------------| +| `/etc/telegraf.conf` | Main Telegraf agent config and InfluxDB output | +| `/etc/telegraf.conf.d/*.conf` | Additional Telegraf input configurations for plugins | + + +## Collected Metrics + +To see the list of metrics collected by Telegraf, use: +```bash +/usr/bin/telegraf --config /etc/telegraf.conf --config-directory /etc/telegraf.conf.d --test +``` + +### Service Health Monitoring (services.conf) + +**How it works**: Every 60 seconds, Telegraf executes `/usr/libexec/telegraf-services`, which queries procd via `ubus call service list`, filters the fixed monitored service whitelist, and converts the matching configured instances to metrics. + +**Metric format**: +``` +procd_service_running{service="nginx", instance="instance1"} = 1 (running) or 0 (down) +procd_service_pid{service="nginx", instance="instance1"} = process_id +procd_service_exit_code{service="nginx", instance="instance1"} = last_exit_code +``` + +Only these services are monitored: + +```text +banip +conntrackd +cron +dedalo +dedalo_users_auth +dnsmasq +dropbear +keepalived +mwan3 +netifyd +nginx +ns-api-server +ns-clm +ns-flashstart +ns-flows +ns-plug +ns-plug-alert-proxy +ns-stats +ns-ui +odhcpd +openvpn +qosify +rpcd +rsyslog +snort +swanctl +sysntpd +telegraf +victoria-metrics +vmalert +``` + +Services with no instances are skipped. + +Notable skipped services: +- `adblock`: excluded because ubus info is not reliable (always shows 1 instance even when disabled) + +##### Querying Service Status + +```bash +# All services and their running state +curl -s 'http://127.0.0.1:8428/api/v1/query?query=procd_service_running' + +# Run collection script manually to preview output +/usr/libexec/telegraf-services +``` + +### Multi-WAN Monitoring (mwan.conf) + +**How it works**: Every 60 seconds, Telegraf executes `/usr/libexec/telegraf-mwan`, which reads `/var/run/mwan3/iface_state/` to determine each WAN interface's online/offline state (maintained by mwan3 in real-time). + +**Metric format**: +``` +mwan_interface_online{interface="wan"} = 1 (online) or 0 (offline) +``` + +#### Querying WAN Status + +```bash +# All WAN interfaces and current state +curl -s 'http://127.0.0.1:8428/api/v1/query?query=mwan_interface_online' + +# Run collection script manually +/usr/libexec/telegraf-mwan +``` + +### Storage Status Monitoring (storage.conf) + +**How it works**: Every 60 seconds, Telegraf executes `/usr/libexec/telegraf-storage-status`, which runs `storage-status` and exports the current storage health as a metric. + +**Metric format**: +``` +storage_status_error = 1 (error) or 0 (ok / not configured) +``` + +## Advanced Configuration + +To add custom metrics or modify collection intervals, edit the `/etc/telegraf.conf.d/` files following [Telegraf documentation](https://docs.influxdata.com/telegraf/). Common customizations: + +- Modify collection intervals: change `interval` in main config +- Add new input plugins: append `[[inputs.plugin_name]]` sections + +After changes, restart Telegraf: +```bash +/etc/init.d/telegraf restart +``` + +## References + +- [Telegraf documentation](https://docs.influxdata.com/telegraf/) +- [Telegraf exec plugin](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/exec) +- [OpenWrt procd init scripts](https://openwrt.org/docs/guide-developer/procd-init-scripts) +- [OpenWrt ubus reference](https://openwrt.org/docs/techref/ubus) +- [Victoria Metrics integration](../victoria-metrics/README.md) diff --git a/packages/telegraf/files/telegraf-backup-encryption b/packages/telegraf/files/telegraf-backup-encryption new file mode 100644 index 000000000..a0026c6f3 --- /dev/null +++ b/packages/telegraf/files/telegraf-backup-encryption @@ -0,0 +1,17 @@ +#!/bin/sh +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Export backup encryption state for Telegraf. +# +# The metric is 1 when /etc/backup.pass exists and is non-empty, otherwise 0. + +if [ -s /etc/backup.pass ]; then + encrypted=1 +else + encrypted=0 +fi + +printf '[{"encrypted":%s}]\n' "$encrypted" diff --git a/packages/telegraf/files/telegraf-config b/packages/telegraf/files/telegraf-config new file mode 100644 index 000000000..7cc2b0d0c --- /dev/null +++ b/packages/telegraf/files/telegraf-config @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +import os +import subprocess +import glob +from jinja2 import Environment, BaseLoader +from euci import EUci + +PROMETHEUS_TEMPLATE = """# Prometheus client output +{% if enabled == '1' -%} +[[outputs.prometheus_client]] + listen = "{{ listen_addr }}" +{%- if basic_auth_username and basic_auth_password %} + basic_username = "{{ basic_auth_username }}" + basic_password = "{{ basic_auth_password }}" +{%- endif %} +{%- endif %} +""" + +PING_TEMPLATE = """# Ping input plugin - monitors ICMP ping to configured hosts +{% if pings -%} +[[inputs.ping]] + urls = [{{ pings|map('tojson')|join(', ') }}] + method = "native" + count = 1 + ping_interval = 1.0 + deadline = 10 + + [inputs.ping.tags] + influxdb_db = "ping-metrics" +{%- endif %} +""" + +DNS_TEMPLATE = """# DNS Query input plugin - monitors DNS resolution +{% if dns_domains -%} +[[inputs.dns_query]] + domains = [{{ dns_domains|map('tojson')|join(', ') }}] + servers = ["127.0.0.1"] + + [inputs.dns_query.tags] + influxdb_db = "ping-metrics" +{%- endif %} +""" + +SENSORS_TEMPLATE = """# Monitor sensors (requires lm-sensors package) +{% if sensors_available -%} +[[inputs.sensors]] + [inputs.sensors.tags] + influxdb_db = "os-metrics" +{%- endif %} +""" + + +def generate_prometheus_config(): + """Read UCI config and render Prometheus client output section.""" + e_uci = EUci() + + try: + enabled = e_uci.get('telegraf', 'output_prometheus', 'enabled', dtype=str, default='0') + listen_addr = e_uci.get('telegraf', 'output_prometheus', 'listen_addr', dtype=str, default=':9273') + basic_auth_username = e_uci.get('telegraf', 'output_prometheus', 'basic_auth_username', dtype=str, default='') + basic_auth_password = e_uci.get('telegraf', 'output_prometheus', 'basic_auth_password', dtype=str, default='') + except Exception as e: + print(f"Error reading UCI config: {e}") + return False + + # Render template + template = Environment(loader=BaseLoader()).from_string(PROMETHEUS_TEMPLATE) + rendered = template.render( + enabled=enabled, + listen_addr=listen_addr, + basic_auth_username=basic_auth_username, + basic_auth_password=basic_auth_password + ) + + # Write to drop-in directory + config_file = '/etc/telegraf.conf.d/prometheus.conf' + os.makedirs(os.path.dirname(config_file), exist_ok=True) + + try: + with open(config_file, 'w') as f: + f.write(rendered) + return True + except Exception as e: + print(f"Error writing config file: {e}") + return False + + +def generate_ping_config(): + """Read UCI config and render ping input section.""" + e_uci = EUci() + try: + pings = e_uci.get('telegraf', 'internet', 'pings', dtype=str, list=True, default=[]) + except Exception: + pings = [] + + template = Environment(loader=BaseLoader()).from_string(PING_TEMPLATE) + rendered = template.render(pings=pings) + + config_file = '/etc/telegraf.conf.d/ping.conf' + os.makedirs(os.path.dirname(config_file), exist_ok=True) + + try: + with open(config_file, 'w') as f: + f.write(rendered) + return True + except Exception as e: + print(f"Error writing config file: {e}") + return False + + +def generate_dns_config(): + """Read UCI config and render DNS query input section.""" + e_uci = EUci() + try: + dns_domains = e_uci.get('telegraf', 'internet', 'dns', dtype=str, list=True, default=[]) + except Exception: + dns_domains = [] + + template = Environment(loader=BaseLoader()).from_string(DNS_TEMPLATE) + rendered = template.render(dns_domains=dns_domains) + + config_file = '/etc/telegraf.conf.d/dns.conf' + os.makedirs(os.path.dirname(config_file), exist_ok=True) + + try: + with open(config_file, 'w') as f: + f.write(rendered) + return True + except Exception as e: + print(f"Error writing config file: {e}") + return False + + +def sensors_available(): + """Check if sensors command works by running it with -A -u flags.""" + try: + result = subprocess.run(['sensors', '-A', '-u'], capture_output=True, timeout=5) + return result.returncode == 0 + except (subprocess.TimeoutExpired, FileNotFoundError): + return False + + +def generate_sensors_config(): + """Render sensors input section based on system availability.""" + has_sensors = sensors_available() + + # Render template + template = Environment(loader=BaseLoader()).from_string(SENSORS_TEMPLATE) + rendered = template.render(sensors_available=has_sensors) + + # Write to drop-in directory + config_file = '/etc/telegraf.conf.d/sensors.conf' + os.makedirs(os.path.dirname(config_file), exist_ok=True) + + try: + with open(config_file, 'w') as f: + f.write(rendered) + return True + except Exception as e: + print(f"Error writing config file: {e}") + return False + + +if __name__ == '__main__': + generate_prometheus_config() + generate_sensors_config() + generate_ping_config() + generate_dns_config() + exit(0) diff --git a/packages/telegraf/files/telegraf-mwan b/packages/telegraf/files/telegraf-mwan new file mode 100644 index 000000000..ab6d12837 --- /dev/null +++ b/packages/telegraf/files/telegraf-mwan @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# +# Collect mwan3 WAN interface status from /var/run/mwan3/iface_state/. +# +# Each file in that directory is named after an mwan3 interface and contains +# a single word: "online" or "offline". The directory is managed by mwan3 +# and only exists when the daemon is running. +# +# Output metric: mwan_interface +# Tags: interface +# Fields: online (int 0/1) +# +# Prints a JSON array to stdout, consumed by telegraf inputs.exec with +# data_format = "json_v2" (parsers.json_v2 build tag). + +import json +import os + +IFACE_STATE_DIR = "/var/run/mwan3/iface_state" + + +def build_records(): + """Return one record per mwan3 interface found in the state directory.""" + if not os.path.isdir(IFACE_STATE_DIR): + return [] + + records = [] + for name in sorted(os.listdir(IFACE_STATE_DIR)): + path = os.path.join(IFACE_STATE_DIR, name) + if not os.path.isfile(path): + continue + try: + status = open(path).read().strip() + except OSError: + continue + records.append( + { + "interface": name, + "online": 1 if status == "online" else 0, + } + ) + return records + + +def main(): + records = build_records() + print(json.dumps(records)) + + +if __name__ == "__main__": + main() diff --git a/packages/telegraf/files/telegraf-services b/packages/telegraf/files/telegraf-services new file mode 100644 index 000000000..932049020 --- /dev/null +++ b/packages/telegraf/files/telegraf-services @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# +# Collect procd service status via ubus. +# +# Monitored services: fixed whitelist of NethSecurity services. +# Services with no instances are ignored because they are disabled or not +# configured. +# +# Usage: +# /usr/libexec/telegraf-services +# +# Output metric: procd_service +# Tags: service, instance +# Fields: running (int 0/1), pid (int), exit_code (int) +# +# Prints a JSON array to stdout, consumed by telegraf inputs.exec with +# data_format = "json_v2" (parsers.json_v2 build tag). +# +import json +import subprocess +import sys +MONITORED_SERVICES = { + "banip", + "conntrackd", + "cron", + "dedalo", + "dedalo_users_auth", + "dnsmasq", + "dropbear", + "keepalived", + "mwan3", + "netifyd", + "nginx", + "ns-api-server", + "ns-clm", + "ns-flashstart", + "ns-flows", + "ns-plug", + "ns-plug-alert-proxy", + "ns-stats", + "ns-ui", + "odhcpd", + "openvpn", + "qosify", + "rpcd", + "rsyslog", + "snort", + "swanctl", + "sysntpd", + "telegraf", + "victoria-metrics", + "vmalert", +} + +# Excluded service: adblock + +def get_service_list(): + result = subprocess.run( + ["ubus", "call", "service", "list"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode != 0: + print(f"Error calling ubus: {result.stderr}", file=sys.stderr) + sys.exit(1) + return json.loads(result.stdout) + + +def sanitize_tag(value): + # InfluxDB line protocol: tag values must not contain commas, spaces or equals + return value.replace(",", "_").replace(" ", "_").replace("=", "_") + + +def build_records(data): + """Return a list of dicts, one per configured monitored service instance.""" + records = [] + + for svc_name in sorted(MONITORED_SERVICES): + svc_body = data.get(svc_name) + instances = (svc_body or {}).get("instances") or {} + if not instances: + continue + + for inst_name, inst in instances.items(): + records.append( + { + "service": sanitize_tag(svc_name), + "instance": sanitize_tag(inst_name), + "running": 1 if inst.get("running", False) else 0, + "pid": inst.get("pid", 0), + "exit_code": inst.get("exit_code", 0), + } + ) + return records + + +def main(): + data = get_service_list() + records = build_records(data) + + # JSON array — consumed by telegraf inputs.exec with data_format=json_v2 + print(json.dumps(records)) + + +if __name__ == "__main__": + main() diff --git a/packages/telegraf/files/telegraf-storage-status b/packages/telegraf/files/telegraf-storage-status new file mode 100644 index 000000000..f7dd52113 --- /dev/null +++ b/packages/telegraf/files/telegraf-storage-status @@ -0,0 +1,21 @@ +#!/bin/sh +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Export the storage health state for Telegraf. +# +# The metric is 1 when storage-status reports "error", otherwise 0. + +if storage_status=$(/usr/sbin/storage-status 2>/dev/null); then + if [ "$storage_status" = "error" ]; then + error=1 + else + error=0 + fi +else + error=1 +fi + +printf '[{"error":%s}]\n' "$error" diff --git a/packages/telegraf/files/telegraf.conf b/packages/telegraf/files/telegraf.conf new file mode 100644 index 000000000..90f4b4061 --- /dev/null +++ b/packages/telegraf/files/telegraf.conf @@ -0,0 +1,24 @@ +# Telegraf Configuration +# Managed by NethSecurity — do not edit manually. + +[global_tags] + +[agent] + interval = "10s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "10s" + flush_jitter = "0s" + precision = "0s" + omit_hostname = true + skip_processors_after_aggregators = true + +# Victoria metrics output plugin configuration +[[outputs.influxdb]] + urls = ["http://127.0.0.1:8428"] + database = "nethsecurity" + database_tag = "influxdb_db" + exclude_database_tag = true + content_encoding = "gzip" diff --git a/packages/telegraf/files/telegraf.conf.d/backup.conf b/packages/telegraf/files/telegraf.conf.d/backup.conf new file mode 100644 index 000000000..65e4fdf1e --- /dev/null +++ b/packages/telegraf/files/telegraf.conf.d/backup.conf @@ -0,0 +1,16 @@ +# Backup encryption status monitoring +# Reports whether /etc/backup.pass is set so vmalert can alert on unencrypted backups. + +[[inputs.exec]] + name_override = "backup_encryption" + commands = ["/usr/libexec/telegraf-backup-encryption"] + interval = "60s" + timeout = "5s" + data_format = "json_v2" + + [[inputs.exec.json_v2]] + [[inputs.exec.json_v2.object]] + path = "@this" + + [inputs.exec.tags] + influxdb_db = "os-metrics" diff --git a/packages/telegraf/files/telegraf.conf.d/mwan.conf b/packages/telegraf/files/telegraf.conf.d/mwan.conf new file mode 100644 index 000000000..8658cfdcf --- /dev/null +++ b/packages/telegraf/files/telegraf.conf.d/mwan.conf @@ -0,0 +1,20 @@ +# mwan3 WAN interface status monitoring +# Reads /var/run/mwan3/iface_state/ — one file per interface, content is +# "online" or "offline". No-ops silently when mwan3 is not running. +# +# Uses parsers.json_v2 — available in the default NethSecurity Telegraf build. + +[[inputs.exec]] + name_override = "mwan_interface" + commands = ["/usr/libexec/telegraf-mwan"] + interval = "60s" + timeout = "10s" + data_format = "json_v2" + + [[inputs.exec.json_v2]] + [[inputs.exec.json_v2.object]] + path = "@this" + tags = ["interface"] + + [inputs.exec.tags] + influxdb_db = "os-metrics" diff --git a/packages/telegraf/files/telegraf.conf.d/os.conf b/packages/telegraf/files/telegraf.conf.d/os.conf new file mode 100644 index 000000000..be6368093 --- /dev/null +++ b/packages/telegraf/files/telegraf.conf.d/os.conf @@ -0,0 +1,80 @@ +# OS and system metrics collection +# Includes CPU, memory, disk, network, and kernel statistics +# All metrics from this section are tagged with influxdb_db=os-metrics + +[[inputs.cpu]] + percpu = true + totalcpu = true + collect_cpu_time = false + report_active = false + core_tags = false + [inputs.cpu.tags] + influxdb_db = "os-metrics" + +[[inputs.disk]] + ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] + [inputs.disk.tags] + influxdb_db = "os-metrics" + +[[inputs.mem]] + [inputs.mem.tags] + influxdb_db = "os-metrics" + +[[inputs.processes]] + [inputs.processes.tags] + influxdb_db = "os-metrics" + +[[inputs.system]] + [inputs.system.tags] + influxdb_db = "os-metrics" + +[[inputs.bond]] + [inputs.bond.tags] + influxdb_db = "os-metrics" + +[[inputs.ethtool]] + ## List of interfaces to include (default: all up interfaces) + # interface_include = ["eth0"] + ## List of interfaces to ignore (default: none) + interface_exclude = ["wg*", "ipsec*", "tun*", "br*"] + + [inputs.ethtool.tags] + influxdb_db = "os-metrics" + +[[inputs.net]] + ## Skip protocol stats, use inputs.nstat instead (fixes deprecation warning) + ignore_protocol_stats = true + [inputs.net.tags] + influxdb_db = "os-metrics" + +[[inputs.netstat]] + [inputs.netstat.tags] + influxdb_db = "os-metrics" + +[[inputs.nstat]] + proc_net_netstat = "/proc/net/netstat" + proc_net_snmp = "/proc/net/snmp" + proc_net_snmp6 = "/proc/net/snmp6" + dump_zeros = true + [inputs.nstat.tags] + influxdb_db = "os-metrics" + + +# Read metrics about disk I/O +[[inputs.diskio]] + ## By default, telegraf will gather stats for all devices. + ## Setting devices will restrict the stats to the specified devices. + # devices = ["sda", "sdb", "vd*"] + + ## Uncomment the following line if you need disk serial numbers. + # skip_serial_number = false + [inputs.diskio.tags] + influxdb_db = "os-metrics" + + +# Collect metrics from the nf_conntrack kernel module +[[inputs.conntrack]] + files = ["nf_conntrack_count", "nf_conntrack_max"] + dirs = ["/proc/sys/net/netfilter", "/proc/sys/net/netfilter"] + [inputs.conntrack.tags] + influxdb_db = "os-metrics" diff --git a/packages/telegraf/files/telegraf.conf.d/services.conf b/packages/telegraf/files/telegraf.conf.d/services.conf new file mode 100644 index 000000000..04c5f98c6 --- /dev/null +++ b/packages/telegraf/files/telegraf.conf.d/services.conf @@ -0,0 +1,20 @@ +# Procd service status monitoring +# Collects running state for a fixed whitelist of persistent services. +# Services with no instances are ignored by the collector. +# +# Uses parsers.json_v2 — available in the default NethSecurity Telegraf build. + +[[inputs.exec]] + name_override = "procd_service" + commands = ["/usr/libexec/telegraf-services"] + interval = "60s" + timeout = "10s" + data_format = "json_v2" + + [[inputs.exec.json_v2]] + [[inputs.exec.json_v2.object]] + path = "@this" + tags = ["service", "instance"] + + [inputs.exec.tags] + influxdb_db = "os-metrics" diff --git a/packages/telegraf/files/telegraf.conf.d/storage.conf b/packages/telegraf/files/telegraf.conf.d/storage.conf new file mode 100644 index 000000000..10e6efee2 --- /dev/null +++ b/packages/telegraf/files/telegraf.conf.d/storage.conf @@ -0,0 +1,16 @@ +# Storage status monitoring +# Reports whether the persistent data storage is mounted so vmalert can alert on storage:status. + +[[inputs.exec]] + name_override = "storage_status" + commands = ["/usr/libexec/telegraf-storage-status"] + interval = "60s" + timeout = "5s" + data_format = "json_v2" + + [[inputs.exec.json_v2]] + [[inputs.exec.json_v2.object]] + path = "@this" + + [inputs.exec.tags] + influxdb_db = "os-metrics" diff --git a/packages/telegraf/files/telegraf.initd b/packages/telegraf/files/telegraf.initd new file mode 100644 index 000000000..49197475e --- /dev/null +++ b/packages/telegraf/files/telegraf.initd @@ -0,0 +1,36 @@ +#!/bin/sh /etc/rc.common + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# shellcheck disable=SC3043 + +START=99 +USE_PROCD=1 + +PROG="/usr/bin/telegraf" + +start_service() { + /usr/sbin/telegraf-config + procd_open_instance + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param respawn 3600 5 0 + procd_set_param command $PROG + procd_append_param command --watch-config notify + procd_append_param command --config /etc/telegraf.conf + procd_append_param command --config-directory /etc/telegraf.conf.d + procd_close_instance +} + +reload_service() +{ + /usr/sbin/telegraf-config +} + +service_triggers() +{ + procd_add_reload_trigger telegraf +} diff --git a/packages/telegraf/files/telegraf.uci b/packages/telegraf/files/telegraf.uci new file mode 100644 index 000000000..b7073f50d --- /dev/null +++ b/packages/telegraf/files/telegraf.uci @@ -0,0 +1,11 @@ +config output_prometheus 'output_prometheus' + option enabled '0' + option listen_addr ':9273' + option basic_auth_username '' + option basic_auth_password '' + +config internet 'internet' + list pings '8.8.8.8' + list pings '1.1.1.1' + list dns 'google.com' + list dns 'cloudflare.com' diff --git a/packages/telegraf/files/uci-defaults/99-telegraf-migrate-netdata b/packages/telegraf/files/uci-defaults/99-telegraf-migrate-netdata new file mode 100644 index 000000000..2d5921b82 --- /dev/null +++ b/packages/telegraf/files/uci-defaults/99-telegraf-migrate-netdata @@ -0,0 +1,39 @@ +#!/bin/sh +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Migrate fping hosts from the legacy netdata configuration to the telegraf +# UCI config (telegraf.internet.pings). +# +# The old fping.conf format is a single line: +# hosts="1.1.1.1 8.8.8.8 google.com" + +FPING_CONF="/etc/netdata/fping.conf" + +[ -f "$FPING_CONF" ] || exit 0 + +# Extract the hosts string: strip 'hosts="..."' +hosts_line=$(grep '^hosts=' "$FPING_CONF" | head -n1) +[ -n "$hosts_line" ] || exit 0 + +# Strip key and surrounding quotes +hosts_val="${hosts_line#hosts=}" +hosts_val="${hosts_val#\"}" +hosts_val="${hosts_val%\"}" +[ -n "$hosts_val" ] || exit 0 + +# Add each host to telegraf.internet.pings if not already present +uci -q get telegraf.internet > /dev/null || uci set telegraf.internet='internet' + +for host in $hosts_val; do + # Skip if already in the list + uci -q get telegraf.internet.pings | grep -qw "$host" && continue + uci add_list telegraf.internet.pings="$host" +done + +uci commit telegraf + +# Remove bundled netdata configuration files since they are no longer used. +rm -rf /etc/netdata 2>/dev/null || true diff --git a/packages/victoria-logs/Makefile b/packages/victoria-logs/Makefile new file mode 100644 index 000000000..813fe68a3 --- /dev/null +++ b/packages/victoria-logs/Makefile @@ -0,0 +1,76 @@ +include $(TOPDIR)/rules.mk + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +PKG_NAME:=victoria-logs +# renovate: datasource=github-tags depName=VictoriaMetrics/VictoriaLogs +PKG_VERSION:=1.49.0 +PKG_RELEASE:=1 + +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz +PKG_SOURCE_URL:=https://codeload.github.com/VictoriaMetrics/VictoriaLogs/tar.gz/v$(PKG_VERSION)? +PKG_SOURCE_SUBDIR:=VictoriaLogs-$(PKG_VERSION) +PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) + +PKG_HASH:=skip +PKG_MAINTAINER:=Tommaso Bailetti +PKG_LICENSE:=Apache-2.0 + +PKG_BUILD_DEPENDS:=golang/host +PKG_BUILD_PARALLEL:=1 +PKG_BUILD_FLAGS:=no-mips16 + +GO_PKG:=github.com/VictoriaMetrics/VictoriaLogs +GO_PKG_BUILD_PKG:=github.com/VictoriaMetrics/VictoriaLogs/app/victoria-logs \ + github.com/VictoriaMetrics/VictoriaLogs/app/vlogscli +GO_PKG_LDFLAGS:= \ + -extldflags \ + -static +GO_PKG_LDFLAGS_X:=github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo.Version=$(PKG_NAME)-v$(PKG_VERSION) +GO_PKG_TAGS:= \ + netgo \ + osusergo \ + musl + +include $(INCLUDE_DIR)/package.mk +include $(TOPDIR)/feeds/packages/lang/golang/golang-package.mk + +define Package/victoria-logs + SECTION:=base + CATEGORY:=NethServer + TITLE:=Victoria Logs + URL:=https://github.com/VictoriaMetrics/VictoriaLogs + DEPENDS:=$(GO_ARCH_DEPENDS) +rsyslog +endef + +define Package/victoria-logs/description + VictoriaLogs — fast and easy-to-use database for logs. +endef + +define Package/victoria-logs/conffiles +/etc/config/victoria-logs +endef + +define Package/victoria-logs/install + $(call GoPackage/Package/Install/Bin,$(1)) + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/victoria-logs.initd $(1)/etc/init.d/victoria-logs + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_DATA) ./files/victoria-logs.conf $(1)/etc/config/victoria-logs + $(INSTALL_DIR) $(1)/etc/rsyslog.d + $(INSTALL_CONF) ./files/rsyslog-victoria-logs.conf $(1)/etc/rsyslog.d/victoria-logs.conf + $(INSTALL_DIR) $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/35_victoria-logs $(1)/etc/uci-defaults/35_victoria-logs +endef + +define Package/victoria-logs/postinst +#!/bin/sh +[ -z "$${IPKG_INSTROOT}" ] && /etc/init.d/victoria-logs restart +exit 0 +endef + +$(eval $(call GoPackage,victoria-logs)) +$(eval $(call BuildPackage,victoria-logs)) diff --git a/packages/victoria-logs/files/35_victoria-logs b/packages/victoria-logs/files/35_victoria-logs new file mode 100644 index 000000000..079e1a4eb --- /dev/null +++ b/packages/victoria-logs/files/35_victoria-logs @@ -0,0 +1,14 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +RSYSLOG_CONF="/etc/rsyslog.d/victoria-logs.conf" + +# Register the victoria-logs rsyslog drop-in in rsyslog UCI includes +if ! uci -q get rsyslog.syslog.includes | grep -qF "${RSYSLOG_CONF}"; then + uci add_list rsyslog.syslog.includes="${RSYSLOG_CONF}" + uci commit rsyslog +fi diff --git a/packages/victoria-logs/files/rsyslog-victoria-logs.conf b/packages/victoria-logs/files/rsyslog-victoria-logs.conf new file mode 100644 index 000000000..c2a66de3b --- /dev/null +++ b/packages/victoria-logs/files/rsyslog-victoria-logs.conf @@ -0,0 +1,18 @@ +# Rsyslog configuration for VictoriaLogs + +ruleset(name="victoria-logs") { + *.* action( + type="omfwd" + target="127.0.0.1" + port="5514" + protocol="tcp" + TCP_Framing="octet-counted" + Template="RSYSLOG_SyslogProtocol23Format" + + action.resumeRetryCount="-1" + queue.type="linkedList" + queue.size="10000" + ) +} + +*.* call victoria-logs diff --git a/packages/victoria-logs/files/victoria-logs.conf b/packages/victoria-logs/files/victoria-logs.conf new file mode 100644 index 000000000..84aba8dbe --- /dev/null +++ b/packages/victoria-logs/files/victoria-logs.conf @@ -0,0 +1,4 @@ +config victorialogs 'main' + option storage_path '/var/lib/victoria-logs' + option max_disk_usage '50MB' + option http_listen_addr '127.0.0.1:9428' diff --git a/packages/victoria-logs/files/victoria-logs.initd b/packages/victoria-logs/files/victoria-logs.initd new file mode 100644 index 000000000..820ae7877 --- /dev/null +++ b/packages/victoria-logs/files/victoria-logs.initd @@ -0,0 +1,43 @@ +#!/bin/sh /etc/rc.common + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# shellcheck disable=SC3043 + +START=99 +USE_PROCD=1 + +PROG="/usr/bin/victoria-logs" + +start_service() { + config_load victoria-logs + local storage_path max_disk_usage http_listen_addr + config_get storage_path main storage_path /var/lib/victoria-logs + config_get max_disk_usage main max_disk_usage 50MB + config_get http_listen_addr main http_listen_addr 127.0.0.1:9428 + + procd_open_instance + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param respawn 3600 5 0 + procd_set_param command $PROG + procd_append_param command -storageDataPath="$storage_path" + procd_append_param command -retention.maxDiskSpaceUsageBytes="$max_disk_usage" + procd_append_param command -httpListenAddr="$http_listen_addr" + procd_append_param command -syslog.listenAddr.tcp=127.0.0.1:5514 + procd_close_instance +} + +service_triggers() +{ + procd_add_reload_trigger victoria-logs +} + +reload_service() +{ + stop + start +} diff --git a/packages/victoria-metrics/Makefile b/packages/victoria-metrics/Makefile new file mode 100644 index 000000000..6510e5a0a --- /dev/null +++ b/packages/victoria-metrics/Makefile @@ -0,0 +1,78 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +include $(TOPDIR)/rules.mk + +PKG_NAME:=victoria-metrics +# renovate: datasource=github-tags depName=VictoriaMetrics/VictoriaMetrics +PKG_VERSION:=1.139.0 +PKG_RELEASE:=1 + +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz +PKG_SOURCE_URL:=https://codeload.github.com/VictoriaMetrics/VictoriaMetrics/tar.gz/v$(PKG_VERSION)? +PKG_SOURCE_SUBDIR:=VictoriaMetrics-$(PKG_VERSION) +PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) + +PKG_HASH:=skip +PKG_MAINTAINER:=Tommaso Bailetti +PKG_LICENSE:=Apache-2.0 + +PKG_BUILD_DEPENDS:=golang/host +PKG_BUILD_PARALLEL:=1 +PKG_BUILD_FLAGS:=no-mips16 + +GO_PKG:=github.com/VictoriaMetrics/VictoriaMetrics +GO_PKG_BUILD_PKG:=github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-metrics \ + github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert +GO_PKG_LDFLAGS:= \ + -extldflags \ + -static +GO_PKG_LDFLAGS_X:=github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo.Version=$(PKG_NAME)-v$(PKG_VERSION) +GO_PKG_TAGS:= \ + netgo \ + osusergo \ + musl + +include $(INCLUDE_DIR)/package.mk +include $(TOPDIR)/feeds/packages/lang/golang/golang-package.mk + +define Package/victoria-metrics + SECTION:=base + CATEGORY:=NethServer + TITLE:=Victoria Metrics + URL:=https://github.com/VictoriaMetrics/VictoriaMetrics + DEPENDS:=$(GO_ARCH_DEPENDS) +endef + +define Package/victoria-metrics/description + VictoriaMetrics time series database / single-node server. +endef + +define Package/victoria-metrics/conffiles +/etc/config/victoria-metrics +/etc/config/vmalert +endef + +define Package/victoria-metrics/install + $(call GoPackage/Package/Install/Bin,$(1)) + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/victoria-metrics.initd $(1)/etc/init.d/victoria-metrics + $(INSTALL_BIN) ./files/vmalert.initd $(1)/etc/init.d/vmalert + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_DATA) ./files/victoria-metrics.conf $(1)/etc/config/victoria-metrics + $(INSTALL_DATA) ./files/vmalert.conf $(1)/etc/config/vmalert + $(INSTALL_DIR) $(1)/etc/vmalert/rules + $(INSTALL_DATA) ./files/vmalert-rules/*.yaml $(1)/etc/vmalert/rules/ +endef + +define Package/victoria-metrics/postinst +#!/bin/sh +[ -z "$${IPKG_INSTROOT}" ] && /etc/init.d/victoria-metrics restart +[ -z "$${IPKG_INSTROOT}" ] && /etc/init.d/vmalert restart +exit 0 +endef + +$(eval $(call GoBinPackage,victoria-metrics)) +$(eval $(call BuildPackage,victoria-metrics)) \ No newline at end of file diff --git a/packages/victoria-metrics/README.md b/packages/victoria-metrics/README.md new file mode 100644 index 000000000..cd7c11728 --- /dev/null +++ b/packages/victoria-metrics/README.md @@ -0,0 +1,138 @@ +# Victoria Metrics + +## Overview + +This package provides **Victoria Metrics** and **vmalert** for time-series metrics storage and alerting in NethSecurity. Metrics are collected by Telegraf, stored in Victoria Metrics, and evaluated by vmalert according to alert rules. + +**Key Components:** +- **victoria-metrics**: Time-series database on port 8428 +- **vmalert**: Alert rule evaluator on port 8081 +- **Telegraf integration**: Host metrics, service health, WAN status, storage status +- **Mimir integration**: Optional centralized alerting (via ns-plug) + +## Quick Start + +### View Active Alerts + +```bash +# List all firing and pending alerts +curl http://127.0.0.1:8082/api/v1/alerts | jq + +# Get a specific alert status +curl 'http://127.0.0.1:8082/api/v1/rules?type=alert' | \ + jq '.data.groups[].rules[] | select(.name == "HighCpuUsage") | {name, state, lastEvaluation}' +``` + +### List Available Metrics + +```bash +# All metrics currently being stored +curl -s 'http://127.0.0.1:8428/api/v1/label/__name__/values' | jq -r '.data[]' | sort +``` + +## Configuration + +Configuration is located at `/etc/config/victoria-metrics`: + +``` +config victoriametrics 'main' + option http_listen_addr '127.0.0.1:8428' +``` + +**Required options:** +- `http_listen_addr`: Address and port for the HTTP server + +**Optional options:** +- `storage_path`: Where to store metrics data (default: `/var/lib/victoriametrics`, auto-detects `/mnt/data/victoriametrics` if available) +- `retention_period`: How long to keep metrics (`1d`, `7d`, `30d`, `1y`, etc.) (default: `7d`, auto-detects `1y` if not set) + +### Accessing the Web UI + +By default the server is accessible only on localhost for security. +The service also exposes a Web UI on port 8428 for browsing metrics and testing queries. + +To access the Web UI, you can change the `http_listen_addr` to `0.0.0.0:8428` to allow external access, but this is not recommended for production environments without proper security measures. +A safer approach is to use SSH port forwarding: +```bash +ssh -L 8428:127.0.0.1:8428 root@remote_host +``` + +Then open `http://127.0.0.1:8428` in your web browser to see all exposed endpoints. +The UI to query metrics is available at `http://127.0.0.1:8428/vmui + +## Alerting Rules + +All alert rules are defined as YAML files in `/etc/vmalert/rules/*.yaml`. Each file corresponds to a specific monitoring category. + +Some alerts implement a two-tier severity model with `warning` and `critical` levels and are designed to suppress lower-severity alerts when higher-severity ones are firing. + +Warning alerts use `unless` clauses to suppress them when their critical counterpart is already firing, reducing noise. For example, `HighCpuUsage` warning is silenced when `CriticalCpuUsage` is firing. + +See rule files for specific thresholds and suppression logic. + +An alert can be in one of three states: + +1. **Pending**: Condition is true but hasn't met the required `for` duration +2. **Firing**: Condition has been true for at least the `for` duration +3. **Resolved**: Condition is no longer true + +Example: An alert with `for: 5m` takes 5 minutes to transition from pending → firing. + +### Custom Alert Rules + +To add custom alerts, create a new YAML file in `/etc/vmalert/rules/`. +Example `my_alerts.yaml`: + +```yaml +groups: + - name: "my_alerts" + interval: "30s" + rules: + - alert: MyAlert + expr: 'metric_name > threshold' + for: "5m" + labels: + severity: "warning" + service: "my_service" + annotations: + summary_en: "Alert summary" + summary_it: "Riepilogo avviso" + description_en: "Value is {{ $value }}" +``` + +Then restart vmalert: +```bash +/etc/init.d/vmalert restart +``` + +## Mimir Integration (ns-plug) + +Mimir is a multi-tenant Prometheus-compatible long-term storage and alerting system used by nextgen [my](https://github.com/NethServer/my/) monitoring +platform. +When Mimir is configured via ns-plug, vmalert automatically forwards alerts. No manual vmalert configuration needed. + +**Enable Mimir forwarding:** +```bash +uci set ns-plug.config.my_url='https://mimir.example.com' +uci set ns-plug.config.my_system_key='your_api_key' +uci set ns-plug.config.my_system_secret='your_api_secret' +uci commit ns-plug +/etc/init.d/vmalert restart +``` + +**Disable (alert-proxy only mode):** +```bash +uci delete ns-plug.config.my_url +uci delete ns-plug.config.my_system_key +uci delete ns-plug.config.my_system_secret +uci commit ns-plug +/etc/init.d/vmalert restart +``` + +## References + +- [Victoria Metrics vmalert docs](https://docs.victoriametrics.com/vmalert/) +- [MetricsQL documentation](https://docs.victoriametrics.com/metricsql/) +- [Prometheus alerting rules](https://samber.github.io/awesome-prometheus-alerts/) +- [vmalert documentation](https://docs.victoriametrics.com/vmalert/) +- [Telegraf metrics collection](../telegraf/README.md) diff --git a/packages/victoria-metrics/files/victoria-metrics.conf b/packages/victoria-metrics/files/victoria-metrics.conf new file mode 100644 index 000000000..bf3541fe8 --- /dev/null +++ b/packages/victoria-metrics/files/victoria-metrics.conf @@ -0,0 +1,2 @@ +config victoriametrics 'main' + option http_listen_addr '127.0.0.1:8428' diff --git a/packages/victoria-metrics/files/victoria-metrics.initd b/packages/victoria-metrics/files/victoria-metrics.initd new file mode 100644 index 000000000..148804cab --- /dev/null +++ b/packages/victoria-metrics/files/victoria-metrics.initd @@ -0,0 +1,65 @@ +#!/bin/sh /etc/rc.common + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# shellcheck disable=SC3043 + +START=99 +USE_PROCD=1 + +PROG="/usr/bin/victoria-metrics" + +start_service() { + config_load victoria-metrics + local storage_path retention_period http_listen_addr + config_get storage_path main storage_path + config_get retention_period main retention_period + config_get http_listen_addr main http_listen_addr 127.0.0.1:8428 + + # Detect if external storage is mounted + local disk_mount + config_load fstab + config_get disk_mount ns_data target + + # Auto-detect storage_path if not customized + if [ -z "$storage_path" ]; then + if [ -n "$disk_mount" ]; then + storage_path="$disk_mount/victoria-metrics-data" + else + storage_path="/var/lib/victoria-metrics-data" + fi + fi + + # Set retention_period default based on storage availability + if [ -z "$retention_period" ]; then + if [ -n "$disk_mount" ]; then + retention_period="1y" + else + retention_period="7d" + fi + fi + + procd_open_instance + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param respawn 3600 5 0 + procd_set_param command $PROG + procd_append_param command -storageDataPath="$storage_path" + procd_append_param command -retentionPeriod="$retention_period" + procd_append_param command -httpListenAddr="$http_listen_addr" + procd_close_instance +} + +service_triggers() +{ + procd_add_reload_trigger victoria-metrics +} + +reload_service() +{ + stop + start +} diff --git a/packages/victoria-metrics/files/vmalert-rules/backup.yaml b/packages/victoria-metrics/files/vmalert-rules/backup.yaml new file mode 100644 index 000000000..30800186e --- /dev/null +++ b/packages/victoria-metrics/files/vmalert-rules/backup.yaml @@ -0,0 +1,20 @@ +# Victoria Metrics Alert Rules for backup encryption monitoring +# +# Monitors whether /etc/backup.pass is present and non-empty via the +# backup_encryption_encrypted metric collected by Telegraf. + +groups: + - name: "backup" + interval: "60s" + rules: + - alert: BackupEncryptionDisabled + expr: 'backup_encryption_encrypted == 0' + for: "2m" + labels: + severity: "warning" + service: "backup" + annotations: + summary_en: "Backup encryption is disabled" + summary_it: "La cifratura dei backup e disattivata" + description_en: "The backup passphrase file /etc/backup.pass is missing or empty. If the firewall has a subscription, the backup will not be sent to the remote storage server." + description_it: "Il file della passphrase dei backup /etc/backup.pass manca o e vuoto. Se il firewall ha una subscription, i backup non verranno inviati al server di archiviazione remoto." diff --git a/packages/victoria-metrics/files/vmalert-rules/host.yaml b/packages/victoria-metrics/files/vmalert-rules/host.yaml new file mode 100644 index 000000000..b01dae22f --- /dev/null +++ b/packages/victoria-metrics/files/vmalert-rules/host.yaml @@ -0,0 +1,96 @@ +# Victoria Metrics Alert Rules for Host and Hardware Monitoring +# +# Based on: https://samber.github.io/awesome-prometheus-alerts/rules/basic-resource-monitoring/host-and-hardware/ +# Adapted for Telegraf metrics names + +groups: + - name: "host_and_hardware" + interval: "30s" + rules: + # CPU Monitoring + - alert: HighCpuUsage + expr: 'round(100 - avg(cpu_usage_idle), 0.1) > 70 unless round(100 - avg(cpu_usage_idle), 0.1) > 85' + for: "5m" + labels: + severity: "info" + service: "host" + annotations: + summary_en: "High CPU usage detected" + summary_it: "Utilizzo elevato di CPU rilevato" + description_en: "CPU usage is {{ $value }}%" + description_it: "Utilizzo della CPU è {{ $value }}%" + + - alert: CriticalCpuUsage + expr: 'round(100 - avg(cpu_usage_idle), 0.1) > 85' + for: "2m" + labels: + severity: "warning" + service: "host" + annotations: + summary_en: "Critical CPU usage detected" + summary_it: "Utilizzo critico di CPU rilevato" + description_en: "CPU usage is {{ $value }}%" + description_it: "Utilizzo della CPU è {{ $value }}%" + + # Memory Monitoring + - alert: HighMemoryUsage + expr: 'round((mem_used / mem_total) * 100, 0.1) > 80 unless round((mem_used / mem_total) * 100, 0.1) > 90' + for: "5m" + labels: + severity: "info" + service: "host" + annotations: + summary_en: "High memory usage detected" + summary_it: "Utilizzo elevato di memoria rilevato" + description_en: "Memory usage is {{ $value }}%" + description_it: "Utilizzo della memoria è {{ $value }}%" + + - alert: CriticalMemoryUsage + expr: 'round((mem_used / mem_total) * 100, 0.1) > 90' + for: "2m" + labels: + severity: "warning" + service: "host" + annotations: + summary_en: "Critical memory usage detected" + summary_it: "Utilizzo critico di memoria rilevato" + description_en: "Memory usage is {{ $value }}%" + description_it: "Utilizzo della memoria è {{ $value }}%" + + # Disk Space Monitoring + - alert: DiskSpaceWarning + expr: 'round((disk_used / disk_total) * 100, 0.1) > 80 unless round((disk_used / disk_total) * 100, 0.1) > 90' + for: "5m" + labels: + severity: "warning" + service: "storage" + annotations: + summary_en: "Disk space low on {{ $labels.path }}" + summary_it: "Spazio disco in esaurimento su {{ $labels.path }}" + description_en: "Disk usage is {{ $value }}% on {{ $labels.path }}" + description_it: "Utilizzo del disco è {{ $value }}% su {{ $labels.path }}" + + - alert: DiskSpaceCritical + expr: 'round((disk_used / disk_total) * 100, 0.1) > 90' + for: "2m" + labels: + severity: "critical" + service: "storage" + annotations: + summary_en: "Disk space critical on {{ $labels.path }}" + summary_it: "Spazio disco critico su {{ $labels.path }}" + description_en: "Disk usage is {{ $value }}% on {{ $labels.path }}" + description_it: "Utilizzo del disco è {{ $value }}% su {{ $labels.path }}" + + # System Load Monitoring + - alert: HighSystemLoad + expr: 'system_load1 / system_n_cpus > 2' + for: "5m" + labels: + severity: "warning" + service: "host" + annotations: + summary_en: "High system load detected" + summary_it: "Carico di sistema elevato rilevato" + description_en: "System load is {{ $value }}" + description_it: "Carico di sistema è {{ $value }}" diff --git a/packages/victoria-metrics/files/vmalert-rules/mwan.yaml b/packages/victoria-metrics/files/vmalert-rules/mwan.yaml new file mode 100644 index 000000000..a4da2b96d --- /dev/null +++ b/packages/victoria-metrics/files/vmalert-rules/mwan.yaml @@ -0,0 +1,25 @@ +# Victoria Metrics Alert Rules for mwan3 WAN Monitoring +# +# Monitors WAN interface connectivity via the mwan_interface_online metric +# collected by /usr/libexec/telegraf-mwan. +# +# The metric is sourced from /var/run/mwan3/iface_state/ which mwan3 +# writes as "online" or "offline" based on its tracking probes. +# Only interfaces present in that directory are monitored — interfaces +# not managed by mwan3 are not included. + +groups: + - name: "mwan" + interval: "60s" + rules: + - alert: WanDown + expr: 'mwan_interface_online == 0' + for: "2m" + labels: + severity: "critical" + service: "network" + annotations: + summary_en: "WAN interface {{ $labels.interface }} is offline" + summary_it: "L'interfaccia WAN {{ $labels.interface }} non è raggiungibile" + description_en: "WAN interface {{ $labels.interface }} is down. Internet connectivity lost." + description_it: "L'interfaccia WAN {{ $labels.interface }} non è raggiungibile. Connettività Internet persa." diff --git a/packages/victoria-metrics/files/vmalert-rules/services.yaml b/packages/victoria-metrics/files/vmalert-rules/services.yaml new file mode 100644 index 000000000..f87b1c38c --- /dev/null +++ b/packages/victoria-metrics/files/vmalert-rules/services.yaml @@ -0,0 +1,22 @@ +# Victoria Metrics Alert Rules for Service Monitoring +# +# Monitors configured procd-managed services via the procd_service_* metrics +# collected by /usr/libexec/telegraf-services. +# +# Services with no instances are ignored by the collector. + +groups: + - name: "services" + interval: "60s" + rules: + - alert: ServiceDown + expr: 'procd_service_running == 0 and procd_service_exit_code != 0' + for: "2m" + labels: + severity: "critical" + alertgroup: "services" + annotations: + summary_en: "Service {{ $labels.service }} is down" + summary_it: "Il servizio {{ $labels.service }} non è attivo" + description_en: "Service {{ $labels.service }} (instance {{ $labels.instance }}) has been down for more than 2 minutes" + description_it: "Il servizio {{ $labels.service }} (istanza {{ $labels.instance }}) non è attivo da più di 2 minuti" diff --git a/packages/victoria-metrics/files/vmalert-rules/storage.yaml b/packages/victoria-metrics/files/vmalert-rules/storage.yaml new file mode 100644 index 000000000..fdcca20cf --- /dev/null +++ b/packages/victoria-metrics/files/vmalert-rules/storage.yaml @@ -0,0 +1,18 @@ +# Storage status monitoring +# +# The alert is driven by the Telegraf storage_status_error metric. + +groups: + - name: "storage" + interval: "30s" + rules: + - alert: StorageStatus + expr: 'storage_status_error == 1' + labels: + severity: "critical" + service: "storage" + annotations: + summary_en: "Storage is in error state" + summary_it: "Lo storage è in stato di errore" + description_en: "The configured data storage is not mounted or is otherwise in error." + description_it: "Lo storage dati configurato non è montato o è in errore." diff --git a/packages/victoria-metrics/files/vmalert.conf b/packages/victoria-metrics/files/vmalert.conf new file mode 100644 index 000000000..c40cc9c37 --- /dev/null +++ b/packages/victoria-metrics/files/vmalert.conf @@ -0,0 +1,3 @@ +config main 'main' + option datasource_url 'http://localhost:8428' + option http_listen_addr '127.0.0.1:8082' diff --git a/packages/victoria-metrics/files/vmalert.initd b/packages/victoria-metrics/files/vmalert.initd new file mode 100644 index 000000000..5ff4efd58 --- /dev/null +++ b/packages/victoria-metrics/files/vmalert.initd @@ -0,0 +1,69 @@ +#!/bin/sh /etc/rc.common + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +START=95 +STOP=5 +USE_PROCD=1 + +PROG="/usr/bin/vmalert" +RULE_DIR="/etc/vmalert/rules" + +start_service() { + config_load vmalert 2>/dev/null || true + + local datasource_url http_listen_addr + config_get datasource_url main datasource_url "http://localhost:8428" + config_get http_listen_addr main http_listen_addr "127.0.0.1:8081" + + # Check if Mimir integration is configured in ns-plug + local mimir_url mimir_key mimir_secret notifier_url + config_load ns-plug 2>/dev/null && { + config_get mimir_url config my_url "" + config_get mimir_key config my_system_key "" + config_get mimir_secret config my_system_secret "" + } + + # If all Mimir credentials are present, configure alert forwarding to Mimir + if [ -n "$mimir_url" ] && [ -n "$mimir_key" ] && [ -n "$mimir_secret" ]; then + notifier_url="${mimir_url%/}/collect/api/services/mimir/alertmanager" + else + notifier_url="" + fi + + procd_open_instance + procd_set_param command $PROG + procd_append_param command -rule="$RULE_DIR/*.yaml" + procd_append_param command -httpListenAddr="$http_listen_addr" + procd_append_param command -datasource.url="$datasource_url" + procd_append_param command -remoteRead.url="$datasource_url" + procd_append_param command -remoteWrite.url="$datasource_url" + procd_append_param command -evaluationInterval=30s + + # Always notify the local alert-proxy (handles unregistered machines gracefully) + procd_append_param command -notifier.url="http://127.0.0.1:9095" + + # Also forward to Mimir if credentials are configured + if [ -n "$notifier_url" ]; then + procd_append_param command -notifier.url="$notifier_url" + procd_append_param command -notifier.basicAuth.username="$mimir_key" + procd_append_param command -notifier.basicAuth.password="$mimir_secret" + fi + + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param respawn 3600 5 5 + procd_close_instance +} + +reload_service() { + stop + start +} + +service_triggers() { + procd_add_reload_trigger vmalert +} diff --git a/patches/feeds/packages/100-netdata-export-promethus.patch b/patches/feeds/packages/100-netdata-export-promethus.patch deleted file mode 100644 index 4db53cf7c..000000000 --- a/patches/feeds/packages/100-netdata-export-promethus.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/admin/netdata/Makefile b/admin/netdata/Makefile -index e471f27b2..def44a874 100644 ---- a/admin/netdata/Makefile -+++ b/admin/netdata/Makefile -@@ -62,7 +62,6 @@ CONFIGURE_ARGS += \ - --disable-plugin-freeipmi \ - --disable-plugin-cups \ - --disable-plugin-xenstat \ -- --disable-backend-prometheus-remote-write \ - --disable-unit-tests \ - --disable-ml \ - --disable-cloud diff --git a/patches/feeds/packages/100-strongswan-dpd-action-restart.patch b/patches/feeds/packages/100-strongswan-dpd-action-restart.patch new file mode 100644 index 000000000..8f16df2a3 --- /dev/null +++ b/patches/feeds/packages/100-strongswan-dpd-action-restart.patch @@ -0,0 +1,12 @@ +diff --git a/net/strongswan/files/swanctl.init b/net/strongswan/files/swanctl.init +--- a/net/strongswan/files/swanctl.init ++++ b/net/strongswan/files/swanctl.init +@@ -314,7 +314,7 @@ + hold) + dpdaction="trap" ;; + restart) +- dpdaction="start" ;; ++ dpdaction="restart" ;; + trap|start) + # already using new syntax + ;; diff --git a/patches/package/100-pppd-edit-bcopy.patch b/patches/package/100-pppd-edit-bcopy.patch deleted file mode 100644 index 2874da9fa..000000000 --- a/patches/package/100-pppd-edit-bcopy.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/network/services/ppp/patches/200-use-memmove-for-bcopy.patch b/network/services/ppp/patches/200-use-memmove-for-bcopy.patch -new file mode 100644 -index 0000000000..1632eb27b8 ---- /dev/null -+++ b/package/network/services/ppp/patches/200-use-memmove-for-bcopy.patch -@@ -0,0 +1,11 @@ -+--- a/pppd/pppd-private.h -++++ b/pppd/pppd-private.h -+@@ -523,7 +523,7 @@ int parse_dotted_ip(char *, u_int32_t *) -+ #define TIMEOUT(r, f, t) ppp_timeout((r), (f), (t), 0) -+ #define UNTIMEOUT(r, f) ppp_untimeout((r), (f)) -+ -+-#define BCOPY(s, d, l) memcpy(d, s, l) -++#define BCOPY(s, d, l) memmove(d, s, l) -+ #define BZERO(s, n) memset(s, 0, n) -+ #define BCMP(s1, s2, l) memcmp(s1, s2, l) -+ diff --git a/scripts/Readme.md b/scripts/Readme.md index f7b16b6cb..32fa85a37 100644 --- a/scripts/Readme.md +++ b/scripts/Readme.md @@ -99,7 +99,14 @@ This script retrieves open issues labeled "testing" from the NethServer/nethsecu ## netifyd-packages.sh -This scripts pulls the `.ipk` packages from the `netifyd-ipks` directory, unpacks them and puts the files inside the `netifyd` package. This script is useful when an update of `netifyd` requires changes in the integration meta package. +This script extracts Netify `.apk` packages from the `netifyd-apks` directory, unpacks them, and merges the files for analysis and integration. This script is useful when an update of `netifyd` requires changes in the integration meta package. + +### Prerequisites + +- **apk-tools**: The Alpine package extraction tool must be installed on your system. + - On Fedora/RHEL: `dnf install apk-tools` + - On Debian/Ubuntu: `apt-get install apk-tools` + - On Alpine Linux: Already included ### Usage @@ -107,4 +114,12 @@ This scripts pulls the `.ipk` packages from the `netifyd-ipks` directory, unpack ./netifyd-packages.sh ``` -All process will be handled by the script, there will be an output that lists the files that were being copied in the process, this output needs to be copied and pasted inside the `packages/netifyd/Makefile` file, inside the `define Package/netifyd/install` section. +The script will process all `.apk` files in the `netifyd-apks/{arch}/` directories and extract their contents into `netifyd-apks/tmp/{arch}/netifyd/`. The merged files can then be copied into the `packages/netifyd/` directory as needed. + +### How It Works + +1. Iterates over each architecture subdirectory in `netifyd-apks/` +2. For each `.apk` file found, extracts its contents to a temporary location +3. Merges all extracted files into a single `netifyd` output directory per architecture +4. Skips metadata files (`.PKGINFO`, install scripts, etc.) during extraction +5. Outputs the merged file structure in `netifyd-apks/tmp/{arch}/netifyd/` diff --git a/scripts/netifyd-packages.sh b/scripts/netifyd-packages.sh index 8425d380c..d3261af1c 100755 --- a/scripts/netifyd-packages.sh +++ b/scripts/netifyd-packages.sh @@ -1,28 +1,32 @@ #!/bin/bash -# This is a helper script to extract all Netify IPK packages -# into netifyd-ipks/tmp/{arch} directories for analysis and integration. +# This is a helper script to extract all Netify APK packages +# into netifyd-apks/tmp/{arch} directories for analysis and integration. # # Usage: ./netifyd-packages.sh # -# Input: netifyd-ipks/{arch}/*.ipk -# Output: netifyd-ipks/tmp/{arch}/netifyd/ — merged contents of all IPKs for that arch -# netifyd-ipks/tmp/{arch}/{pkg}/ — per-package extraction (intermediate) +# Input: netifyd-apks/{arch}/*.apk +# Output: netifyd-apks/tmp/{arch}/netifyd/ — merged contents of all APKs for that arch +# netifyd-apks/tmp/{arch}/{pkg}/ — per-package extraction (intermediate) set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -IPKS_DIR="${SCRIPT_DIR}/netifyd-ipks" -TMP_DIR="${IPKS_DIR}/tmp" +APKS_DIR="${SCRIPT_DIR}/netifyd-apks" +TMP_DIR="${APKS_DIR}/tmp" -if [ ! -d "${IPKS_DIR}" ]; then - echo "ERROR: IPKs directory not found: ${IPKS_DIR}" >&2 +if [ ! -d "${APKS_DIR}" ]; then + echo "ERROR: APKs directory not found: ${APKS_DIR}" >&2 exit 1 fi # Iterate over each arch subdirectory -for arch_dir in "${IPKS_DIR}"/*/; do +for arch_dir in "${APKS_DIR}"/*/; do [ -d "${arch_dir}" ] || continue + + # Skip the tmp directory + [ "$(basename "${arch_dir}")" = "tmp" ] && continue + arch="$(basename "${arch_dir}")" echo "==> Processing arch: ${arch}" @@ -31,13 +35,13 @@ for arch_dir in "${IPKS_DIR}"/*/; do rm -rf "${TMP_DIR:?}/${arch}" mkdir -p "${output_dir}" - # Extract each IPK into its own per-package subdirectory, then merge - for ipk_file in "${arch_dir}"*.ipk; do - [ -f "${ipk_file}" ] || continue + # Extract each APK into its own per-package subdirectory, then merge + for apk_file in "${arch_dir}"*.apk; do + [ -f "${apk_file}" ] || continue # Derive package name: strip version and arch suffix - # e.g. netify-plm_2026-01-01-v1.2.1-r8_x86_64.ipk -> netify-plm - filename="$(basename "${ipk_file}" .ipk)" + # e.g. netify-plm_2026-01-01-v1.2.1-r8_x86_64.apk -> netify-plm + filename="$(basename "${apk_file}" .apk)" pkg_name="${filename%%_*}" # Extract to a temporary directory first to avoid conflicts @@ -45,7 +49,7 @@ for arch_dir in "${IPKS_DIR}"/*/; do mkdir -p "${pkg_extract_dir}" echo " Extracting ${filename} -> ${pkg_name}/" - tar -xf "${ipk_file}" ./data.tar.gz -O | tar -xzf - -C "${pkg_extract_dir}" + apk extract --allow-untrusted --destination "${pkg_extract_dir}" "${apk_file}" # Merge extracted files into the single netifyd output directory cp -a "${pkg_extract_dir}/." "${output_dir}/" diff --git a/tools/cleanup/cleanup.py b/tools/cleanup/cleanup.py index 4c413e494..caa163bf4 100755 --- a/tools/cleanup/cleanup.py +++ b/tools/cleanup/cleanup.py @@ -2,13 +2,13 @@ # # Cleanup old development builds from DigitalOcean Spaces: -# - Keep all tagged releases -# - Keep at least 3 versions of each sub release +# - Keep the latest 5 versions of each channel # +# Version format: 8.7.2-dev.. or 8.7.2-branch.. import os import boto3 -from semver import VersionInfo +from semver import Version as VersionInfo region = "ams3" bucket_name = "nethsecurity" @@ -27,20 +27,35 @@ parsed_files = [] for file in files: file_name = file.lstrip(f'{prefix}/').rstrip('/') - version_parsed = VersionInfo.parse(file_name) - if version_parsed.build is None: + try: + version_parsed = VersionInfo.parse(file_name) + except ValueError: + print(f'Skipping {file_name} - not a valid semver version.') + continue + + # Check if it's a development build (has prerelease segment) + if version_parsed.prerelease is None: print(f'Skipping {file_name} as it is not a development build.') continue - build_split = version_parsed.build.split('.') + + # Extract timestamp from prerelease segment: "dev.." or "branch.." + prerelease_parts = version_parsed.prerelease.split('.') + if len(prerelease_parts) < 3: + print(f'Skipping {file_name} - prerelease segment does not contain timestamp.') + continue + + timestamp = prerelease_parts[1] # middle part is the timestamp parsed_files.append({ - 'timestamp': build_split[1], - 'file': file + 'timestamp': timestamp, + 'file': file, + 'version': file_name }) -# keep only the latest 5 dev builds +# Keep only the latest 5 dev builds, sorted by timestamp (descending) to_delete = sorted(parsed_files, key=lambda k: k['timestamp'], reverse=True)[min_versions:] for d in to_delete: - print(f"Deleting {d['file']} ...") + print(f"Deleting {d['version']} ...") objects_to_delete = s3_client.list_objects(Bucket=bucket_name, Prefix=d['file']) delete_keys = {'Objects': [{'Key': k['Key']} for k in objects_to_delete.get('Contents', [])]} s3_client.delete_objects(Bucket=bucket_name, Delete=delete_keys) +